rbd: error: image still has watchers....Try again after closing/unmapping ...rbd

admin

[root@compute02 ~]# rbd -p vms info 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
rbd image '234abd4d-2375-44e3-b64a-0e84f247e2b4_disk':
( R, C. V8 a  |. {& L# m- ^( ] size 10 GiB in 1280 objects
order 23 (8 MiB objects)
snapshot_count: 0
, C, o( e7 J' Y3 |$ p id: 2a3c265c791c4
8 V: Q6 c% w  [5 D! q block_name_prefix: rbd_data.2a3c265c791c4
format: 2
features: layering
4 N% ~! T  R* b1 V3 X op_features:
flags:
create_timestamp: Wed Jul 21 15:36:47 2021
+ d, h; h% `  I. c2 [( L access_timestamp: Wed Jul 21 15:36:47 2021
modify_timestamp: Wed Jul 21 15:36:47 2021
. L: b9 j* o+ o: u, f) _2 z parent: images/64079697-b54d-4499-872b-a79d3daa48b8@snap
( z7 i. M6 M9 r% v overlap: 4 GiB
7 o0 i, @4 o8 |! [8 @3 J/ z[root@compute02 ~]# rbd -p vms rm 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
/ n* H% v9 S, Q: x) J2021-07-21 16:38:39.334 7f14abfff700 -1 librbd::image::PreRemoveRequest: 0x557dd1fd5c30 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
4 O1 ?4 a4 w1 m1 d9 Mrbd: error: image still has watchers
This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.
" i' f. g9 [! k- C4 ]
/ J* z7 F  `" i/ v/ r删除失败：
解决步骤如下：
! f) x$ l0 e' t( X# E[root@compute02 ~]# rbd status vms/2d05517a-8670-4cce-b39d-709e055381d6_disk
rbd: error opening image 2d05517a-8670-4cce-b39d-709e055381d6_disk: (2) No such file or directory
7 ]) Q/ B( W. H  l4 X, z7 W
[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
# f: d0 h1 R; GWatchers:
# X- J2 B  ~) f- F* w6 ^ watcher=192.168.0.77:0/3574198266 client.173021 cookie=93887687311360


. V" w1 s! A. w  r: M[root@compute02 ~]# ceph osd blacklist add 192.168.0.77:0/3574198266
blacklisting 192.168.0.77:0/3574198266 until 2021-07-21 17:40:57.398343 (3600 sec)


& x# j; @# ?  E; Z0 m+ d. D& P) I( N[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
Watchers: none


[root@compute02 ~]# rbd rm vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
1 s, A: T- I9 K6 b2 }Removing image: 100% complete...done.

: B9 Z9 y: r  Y! n7 ?删除成功
8 `) L4 ?7 u6 N8 p

admin

同样的问题再次出现，解决步骤如下：

[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
8 [( L0 m  {, ]1 }1 R6 g1 \2022-07-14 14:01:37.980 7f3cddffb700 -1 librbd::image::PreRemoveRequest: 0x55b914876b60 check_image_watchers: image has watchers - not removing
5 [0 V& H. F+ Y7 p/ Q- X0 jRemoving image: 0% complete...failed.
rbd: error: image still has watchers
, B" V( j% l' F- e$ S0 R& DThis means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.
2 w' r7 r( t) D! t

[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
* F9 b9 {6 z" x; X2022-07-14 14:01:51.812 7f1736f22700 -1 librbd::image::PreRemoveRequest: 0x560a2b06ea70 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
rbd: error: image still has watchers
& O( o5 y! m/ a" W# ~. J$ vThis means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.

) G) v0 J6 k/ H# C
[root@host09 ~]# rbd info volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
' I2 C8 x/ q9 C9 M+ Zrbd image 'volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak':
. J+ B: I. e" _- x: n8 Y3 Z$ H        size 50 GiB in 12800 objects
        order 22 (4 MiB objects)
6 j3 M  Z8 x9 R; \. s' Q9 t        snapshot_count: 0
8 g' Q( ]1 \1 i* d; d9 \        id: 1a910a2feef869
        block_name_prefix: rbd_data.1a910a2feef869
        format: 2
7 w/ l2 K  M1 s1 |: B' u( d        features: layering
5 K) M1 c( n$ B        op_features:
% b, l' s( O# f        flags:
3 w& Z) l! ]+ U  o" V. j) Y        create_timestamp: Fri May 14 11:39:01 2021
        access_timestamp: Tue Jul 12 11:19:29 2022
        modify_timestamp: Fri May 14 11:39:01 2021
% A( @$ M0 W" Z% w% L- q4 t        parent: images/0728be2a-d8d7-4e29-9e04-90c9ae91dca0@snap
        overlap: 2 GiB
8 H. v4 L( \! H6 Q7 |. ?
& @1 r+ i8 `  h( [8 Y) V* G' Q! M[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
) U5 h6 Z3 @1 G  AWatchers:
  p2 _4 {9 F0 V" r- S2 l4 w; _        watcher=172.16.129.12:0/3801911378 client.331035433 cookie=94155368121216
# _/ y' g1 q) B8 P/ O

[root@host09 ~]# ceph osd blacklist add 172.16.129.12:0/3801911378
blacklisting 172.16.129.12:0/3801911378 until 2022-07-14 15:07:44.315065 (3600 sec)

$ q- I6 N9 M+ q$ W% e
* v( n0 y9 s* H# U& e% J[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
Watchers: none

, o1 M( k8 i! [) w2 }8 M' C2 o5 N
[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
( A: T  x" }, G5 ~* h1 u0 IRemoving image: 100% complete...done.
* s* a7 @! c% d, d' X! `

& c0 N' g8 q5 k: _' P& \9 f[root@host09 ~]# ceph osd blacklist
& @. L& {; R7 n& l, W1 Bno valid command found; 3 closest matches:
- w& w. o3 R4 y; `5 ~1 |0 L3 T$ Qosd blacklist ls
osd blacklist clear
osd blacklist add|rm <EntityAddr> {<float[0.0-]>}


' q) m0 s4 {0 E' S" |' i[root@host09 ~]# ceph osd blacklist ls
/ a  W* m' Y' n/ v: X6 T& mlisted 1 entries
172.16.129.12:0/3801911378 2022-07-14 15:07:44.315065

* O) m9 v8 K2 o7 O
[root@host09 ~]# ceph osd blacklist rm 172.16.129.12:0/3801911378
un-blacklisting 172.16.129.12:0/3801911378
* z2 O. w; M' P. K

1 W4 ~/ P% c2 r" A$ }[root@host09 ~]# ceph osd blacklist ls
6 o8 {6 p1 C! C! ~8 k/ v- flisted 0 entries

7 x: r! x* j& I6 [