ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-2: (2) No such fi

admin

** ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-2: (2) No such file or directory

' }2 Z4 M; `0 t9 X: M- Q4 Q2022-08-25 16:58:17.680 7f3029de9a80  0 set uid:gid to 167:167 (ceph:ceph)
% }' a3 D: U% }9 N2022-08-25 16:58:17.680 7f3029de9a80  0 ceph version 14.2.8-111.el7 (2e6029d57bc594eceba4751373da6505028c2650) nautilus (stable), process ceph-osd, pid 4848
2022-08-25 16:58:17.680 7f3029de9a80  0 pidfile_write: ignore empty --pid-file
' v+ O& t( X- g3 S, u9 g# f2022-08-25 16:58:17.721 7f3029de9a80 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (13) Permission denied
2022-08-25 16:58:17.721 7f3029de9a80 -1  ** ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-2: (2) No such file or directory
% n0 a; @7 S% }2022-08-25 16:58:19.246 7f14524dba80  0 set uid:gid to 167:167 (ceph:ceph)
8 \* f. h. ?+ ]  O+ Y2022-08-25 16:58:19.246 7f14524dba80  0 ceph version 14.2.8-111.el7 (2e6029d57bc594eceba4751373da6505028c2650) nautilus (stable), process ceph-osd, pid 4868
! @3 f* L6 w3 m* V2022-08-25 16:58:19.246 7f14524dba80  0 pidfile_write: ignore empty --pid-file
% h  k5 R2 _! D$ K' |' z- r2022-08-25 16:58:19.276 7f14524dba80 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (13) Permission denied
% |8 p" J- X6 T2022-08-25 16:58:19.276 7f14524dba80 -1  ** ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-2: (2) No such file or directory



解决办法：
( Q8 T* I/ c7 Z: t" ^' Z! s/ L5 ][root@ceph1 ~]# ceph-volume lvm activate --bluestore --all
--> Activating OSD ID 1 FSID 91e09831-defe-4a1c-9d54-a46ee2f00dbb
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1
Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-2c075439-91b9-49d2-8a83-2c1ac81db669/osd-block-91e09831-defe-4a1c-9d54-a46ee2f00dbb --path /var/lib/ceph/osd/ceph-1 --no-mon-config
( |! D* S8 H7 K% x: i7 Z0 A: G  x" [Running command: /usr/bin/ln -snf /dev/ceph-2c075439-91b9-49d2-8a83-2c1ac81db669/osd-block-91e09831-defe-4a1c-9d54-a46ee2f00dbb /var/lib/ceph/osd/ceph-1/block
. |3 z6 ]% X! E, Q2 nRunning command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block
0 X3 A8 Y8 g: FRunning command: /usr/bin/chown -R ceph:ceph /dev/dm-1
9 e& m# e8 [8 c( g  P& a  `Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1
% x+ H' H: \9 L- d5 \Running command: /usr/bin/ln -snf /dev/vdc1 /var/lib/ceph/osd/ceph-1/block.wal
1 u2 F+ V; x2 B; w) ?Running command: /usr/bin/chown -R ceph:ceph /dev/vdc1
% O, I: ~: N. N3 KRunning command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block.wal
Running command: /usr/bin/chown -R ceph:ceph /dev/vdc1
Running command: /usr/bin/systemctl enable ceph-volume@lvm-1-91e09831-defe-4a1c-9d54-a46ee2f00dbb
Running command: /usr/bin/systemctl enable --runtime ceph-osd@1
" ?& z' T) O$ q$ D7 q& i, RRunning command: /usr/bin/systemctl start ceph-osd@1
--> ceph-volume lvm activate successful for osd ID: 1
; D; J+ _1 ?' J) m--> Activating OSD ID 0 FSID c2f92a17-c127-4a3b-bb99-9006c7f8bbf5
& @* F4 |# l: M( \& _3 cRunning command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-8bfb62b9-db26-47b9-aa47-1f3ec09771d8/osd-block-c2f92a17-c127-4a3b-bb99-9006c7f8bbf5 --path /var/lib/ceph/osd/ceph-0 --no-mon-config
Running command: /usr/bin/ln -snf /dev/ceph-8bfb62b9-db26-47b9-aa47-1f3ec09771d8/osd-block-c2f92a17-c127-4a3b-bb99-9006c7f8bbf5 /var/lib/ceph/osd/ceph-0/block
Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block
Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
+ q; d+ E* R, \* z' uRunning command: /usr/bin/ln -snf /dev/vdb1 /var/lib/ceph/osd/ceph-0/block.wal
! b& m$ v5 O' QRunning command: /usr/bin/chown -R ceph:ceph /dev/vdb1
Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block.wal
3 K8 O# }! N2 I/ N- q( ^- ]4 T/ wRunning command: /usr/bin/chown -R ceph:ceph /dev/vdb1
Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-c2f92a17-c127-4a3b-bb99-9006c7f8bbf5
' g  h* Z4 G! ~* GRunning command: /usr/bin/systemctl enable --runtime ceph-osd@0
Running command: /usr/bin/systemctl start ceph-osd@0
--> ceph-volume lvm activate successful for osd ID: 0
+ ]' P+ \! e) q/ N- i# a[root@ceph1 ~]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
2 p% X+ H! t' Y" R4 e- H: `2 d-3       6.00000 root hdd                                   
/ g+ y1 A  w" V" v  r5 E# i& D-4       2.00000     host hdd-ceph1                        
4 w) f/ I4 p3 k1 }7 ?' P 0   hdd 1.00000         osd.0        down  1.00000 1.00000
1   hdd 1.00000         osd.1        down        0 1.00000
-5       2.00000     host hdd-ceph2                        
2   hdd 1.00000         osd.2        down        0 1.00000
3   hdd 1.00000         osd.3        down  1.00000 1.00000
-6       2.00000     host hdd-ceph3                        
; z( b7 N9 R& @% n3 ~2 Q 4   hdd 1.00000         osd.4        down  1.00000 1.00000
5   hdd 1.00000         osd.5          up  1.00000 1.00000
-1             0 root default                              
6 W8 i1 d) Y, `& O3 q) o[root@ceph1 ~]# ceph osd tree
7 `( O- \; m- S" @2 L% p% x8 x7 aID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-3       6.00000 root hdd                                   
5 A& E9 |% h( Y' S+ n$ g; B-4       2.00000     host hdd-ceph1                        
0 k; t& ]! c' k6 B4 z) D$ U 0   hdd 1.00000         osd.0        down  1.00000 1.00000
1   hdd 1.00000         osd.1        down        0 1.00000
; I  w- N/ Q* X& C-5       2.00000     host hdd-ceph2                        
2   hdd 1.00000         osd.2        down        0 1.00000
3   hdd 1.00000         osd.3        down  1.00000 1.00000
-6       2.00000     host hdd-ceph3                        
6 Z# S7 k3 ^* _0 y 4   hdd 1.00000         osd.4        down  1.00000 1.00000
5   hdd 1.00000         osd.5          up  1.00000 1.00000
-1             0 root default                              
" t1 _' k1 d+ p; f1 }: S[root@ceph1 ~]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-3       6.00000 root hdd                                   
-4       2.00000     host hdd-ceph1                        
0   hdd 1.00000         osd.0          up  1.00000 1.00000
( y3 m1 C0 d9 f% t. f3 J( r: Y 1   hdd 1.00000         osd.1          up  1.00000 1.00000
-5       2.00000     host hdd-ceph2                        
2 ~. p- K4 o2 _2 H. y4 U 2   hdd 1.00000         osd.2        down        0 1.00000
: S$ U4 p# u! M; t+ \& ^: W& f3 ~ 3   hdd 1.00000         osd.3        down  1.00000 1.00000
-6       2.00000     host hdd-ceph3                        
; G" m9 E# l6 d0 w, | 4   hdd 1.00000         osd.4        down  1.00000 1.00000
5   hdd 1.00000         osd.5          up  1.00000 1.00000
/ Z: T; {% s4 _-1             0 root default                              
! L1 s2 r$ t, J2 M# B3 Y

admin

问题
When redeploying an OSD, for example, after a hard drive replacement by using by using the "add-osd.yml" Ceph-Ansible playbook, the following error occurs in /var/log/messages on the OSD node:

, R3 f$ @9 M, S0 k5 p* NRaw
"ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-<nn>: (2) No such file or directory"
决议
. p$ d5 W+ t8 e) ?6 j5 {& wTo prevent this error occurring, list all the OSD mountpoints under /var/lib/ceph/ and identify the OSDs which are invalid.
7 {( y: ?' p, k. L; ^: HThen, remove the invalid mountpoints.
( D5 I' Z5 ?% ^9 v# GFor example, in the below output, OSD "ceph-03" is invalid and has been removed.
5 w3 ?' `' T) H% ]; ], S# n
. J6 ~" @4 b3 Q* Y  JRaw
# ls -lr /var/lib/ceph/osd/
total 0
+ z& ^" `# E) d$ `8 F, y8 g9 [drwxr-xr-x. 3 ceph ceph 231 Aug  6 10:32 ceph-01
; b3 y3 d  V9 `3 y5 g' P$ {- hdrwxr-xr-x. 3 ceph ceph 231 Aug  6 10:29 ceph-02
" H9 d5 i" z: e) Bdrwxr-xr-x. 2 root root   6 Feb 25  2017 ceph-03
drwxr-xr-x. 3 ceph ceph 231 Aug  6 10:26 ceph-04
drwxr-xr-x. 3 ceph ceph 231 Aug  6 10:23 ceph-05
; A1 d3 u* l3 V  g/ r
: r* n0 `: k  u) f# rmdir /var/lib/ceph/osd/ceph-03
1 B$ R2 d7 K5 [根源
8 O! {' U, k1 UThis error occurs because of orphaned directories under /var/lib/ceph/osd/ . These sub-directories should be empty and only serve as mountpoints. Because the sub-directories exist, Ceph will assume an OSD exists and try to mount them. Consequently, it will fail because they cannot be mounted.
: a; S( R3 i8 c; F
诊断步骤
; j6 t# ^! T7 j+ M* qThe following error occurs in /var/log/messages on the OSD node:

) s2 u8 a( L0 b! ORaw
ERROR: unable to open OSD superblock on /var/lib/ceph/osd/ceph-<nn>: (2) No such file or directory

admin

[root@ceph3 ~]# ceph-volume lvm activate --bluestore --all
. `6 k# E* g& x& m( U, f2 }  R2 I--> Activating OSD ID 5 FSID b71e3a1a-64ce-4cab-acc5-df852c8cf213
" W. D$ \1 s- R# ]3 ~( L9 SRunning command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5
Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-2d6d2a9e-f0ae-4d79-879e-22acef248b69/osd-block-b71e3a1a-64ce-4cab-acc5-df852c8cf213 --path /var/lib/ceph/osd/ceph-5 --no-mon-config
: a0 Z9 u- L! T5 J- x, bRunning command: /usr/bin/ln -snf /dev/ceph-2d6d2a9e-f0ae-4d79-879e-22acef248b69/osd-block-b71e3a1a-64ce-4cab-acc5-df852c8cf213 /var/lib/ceph/osd/ceph-5/block
Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block
1 \! C$ t" B" Z4 m& O  P* O, X- ]Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5
( o+ l) L, s3 Z) r' [# r: nRunning command: /usr/bin/ln -snf /dev/vdc1 /var/lib/ceph/osd/ceph-5/block.wal
5 `3 b2 ^8 I6 O3 k; aRunning command: /usr/bin/chown -R ceph:ceph /dev/vdc1
  _% A0 H1 ]3 ?Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block.wal
Running command: /usr/bin/chown -R ceph:ceph /dev/vdc1
- X$ Q3 |- {5 JRunning command: /usr/bin/systemctl enable ceph-volume@lvm-5-b71e3a1a-64ce-4cab-acc5-df852c8cf213
Running command: /usr/bin/systemctl enable --runtime ceph-osd@5
Running command: /usr/bin/systemctl start ceph-osd@5
) r1 s. E% |1 z; `--> ceph-volume lvm activate successful for osd ID: 5
# S: d7 Y" \! L9 L$ b; w--> Activating OSD ID 4 FSID 01cbcdc2-6d43-49b0-80ab-3c4af5ee48c2
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4
* V! W# d7 `6 u: k+ ]. eRunning command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-40e73c9a-a332-48d7-8f23-091e404b4829/osd-block-01cbcdc2-6d43-49b0-80ab-3c4af5ee48c2 --path /var/lib/ceph/osd/ceph-4 --no-mon-config
3 |/ E! A/ x' S" m+ M2 r" RRunning command: /usr/bin/ln -snf /dev/ceph-40e73c9a-a332-48d7-8f23-091e404b4829/osd-block-01cbcdc2-6d43-49b0-80ab-3c4af5ee48c2 /var/lib/ceph/osd/ceph-4/block
+ W9 m3 j. h  t% u( |Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block
; @- c+ D- [- Z* h- H& r7 gRunning command: /usr/bin/chown -R ceph:ceph /dev/dm-1
. J. o. `5 I" i( Q/ y6 eRunning command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4
' ]/ B6 i& c1 ]2 c  ^/ i( XRunning command: /usr/bin/ln -snf /dev/vdb1 /var/lib/ceph/osd/ceph-4/block.wal
* h# p# {/ j- _; I, lRunning command: /usr/bin/chown -R ceph:ceph /dev/vdb1
Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block.wal
Running command: /usr/bin/chown -R ceph:ceph /dev/vdb1
Running command: /usr/bin/systemctl enable ceph-volume@lvm-4-01cbcdc2-6d43-49b0-80ab-3c4af5ee48c2
Running command: /usr/bin/systemctl enable --runtime ceph-osd@4
+ U) V9 v: K9 A6 g9 lRunning command: /usr/bin/systemctl start ceph-osd@4
3 A- d5 r6 O8 X3 Y* n--> ceph-volume lvm activate successful for osd ID: 4
[root@ceph3 ~]# ceph osd tree
, V7 F. w' ^" O$ X- xID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-3       6.00000 root hdd                                   
3 `! ~* ^. d- f2 Q-4       2.00000     host hdd-ceph1                        
0   hdd 1.00000         osd.0          up  1.00000 1.00000
1   hdd 1.00000         osd.1          up  1.00000 1.00000
-5       2.00000     host hdd-ceph2                        
2   hdd 1.00000         osd.2          up  1.00000 1.00000
3   hdd 1.00000         osd.3          up  1.00000 1.00000
-6       2.00000     host hdd-ceph3                        
& L- Y9 b* i. \6 h. q2 }$ x# t! Q5 K 4   hdd 1.00000         osd.4        down  1.00000 1.00000
5   hdd 1.00000         osd.5        down  1.00000 1.00000
. x) c+ A% X8 r: }4 D-1             0 root default                              
[root@ceph3 ~]# ceph osd tree
  f- z) b* h; i* k; O6 HID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-3       6.00000 root hdd                                   
1 Z1 T& r3 j$ }! H4 D7 u% z9 [& g  p-4       2.00000     host hdd-ceph1                        
0   hdd 1.00000         osd.0          up  1.00000 1.00000
1   hdd 1.00000         osd.1          up  1.00000 1.00000
: j4 c" W* h6 J4 N3 L( y-5       2.00000     host hdd-ceph2                        
2 {0 F0 H+ Z6 C: p2 _/ f 2   hdd 1.00000         osd.2          up  1.00000 1.00000
0 [+ |4 a8 D: k6 F+ M 3   hdd 1.00000         osd.3          up  1.00000 1.00000
-6       2.00000     host hdd-ceph3                        
; P  c/ i, N# X 4   hdd 1.00000         osd.4        down  1.00000 1.00000
5   hdd 1.00000         osd.5        down  1.00000 1.00000
5 _/ S) m; q$ N: C" U2 b-1             0 root default                              
# O* |. N; @) E0 l3 \[root@ceph3 ~]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
) g/ _$ H5 ?& h$ v- \' I  [-3       6.00000 root hdd                                   
1 p( D6 S; R% R  T( h9 L-4       2.00000     host hdd-ceph1                        
: j2 M& L$ j# r. x' E 0   hdd 1.00000         osd.0          up  1.00000 1.00000
3 r  ?8 `2 S4 V) k2 @, R 1   hdd 1.00000         osd.1          up  1.00000 1.00000
-5       2.00000     host hdd-ceph2                        
2   hdd 1.00000         osd.2          up  1.00000 1.00000
' G/ N& x: H' o 3   hdd 1.00000         osd.3          up  1.00000 1.00000
  ?! Q' D# ^8 O9 N-6       2.00000     host hdd-ceph3                        
4   hdd 1.00000         osd.4          up  1.00000 1.00000
5   hdd 1.00000         osd.5          up  1.00000 1.00000
8 y2 @* V9 y' I* C- ^5 [-1             0 root default                              
[root@ceph3 ~]#