rbd: error: image still has watchers....Try again after closing/unmapping ...rbd

admin

[root@compute02 ~]# rbd -p vms info 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
rbd image '234abd4d-2375-44e3-b64a-0e84f247e2b4_disk':
9 }/ K1 n) w0 I9 T9 a size 10 GiB in 1280 objects
order 23 (8 MiB objects)
snapshot_count: 0
id: 2a3c265c791c4
3 C! d( d1 M' I% N3 Z block_name_prefix: rbd_data.2a3c265c791c4
format: 2
1 h+ Y6 |: }4 G features: layering
op_features:
flags:
. j1 j6 g+ Q3 c& f create_timestamp: Wed Jul 21 15:36:47 2021
/ w* ^" O9 K  L; k0 j& x+ I access_timestamp: Wed Jul 21 15:36:47 2021
modify_timestamp: Wed Jul 21 15:36:47 2021
- y& [* U6 s  k2 h; f parent: images/64079697-b54d-4499-872b-a79d3daa48b8@snap
overlap: 4 GiB
[root@compute02 ~]# rbd -p vms rm 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
7 X7 E5 ^* w- f5 h4 y+ w( B2 ?2021-07-21 16:38:39.334 7f14abfff700 -1 librbd::image::PreRemoveRequest: 0x557dd1fd5c30 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
rbd: error: image still has watchers
This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.

: `% s1 Y$ y9 ]/ X  m9 y% T! K删除失败：
! N# H; K# F* G. F3 g- ^0 [解决步骤如下：
[root@compute02 ~]# rbd status vms/2d05517a-8670-4cce-b39d-709e055381d6_disk
$ {( K: r' d* E& z7 Irbd: error opening image 2d05517a-8670-4cce-b39d-709e055381d6_disk: (2) No such file or directory

[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
. c) e! j) h3 v( C" x/ g1 i  wWatchers:
1 D5 a4 a4 e, ^2 m- Z. O watcher=192.168.0.77:0/3574198266 client.173021 cookie=93887687311360
* c, R5 v- S# @3 n) b) D! m
  l: L. ?" [; z" o( k
2 n. ~6 Y" v, s, ^6 |6 E5 }[root@compute02 ~]# ceph osd blacklist add 192.168.0.77:0/3574198266
' p2 V: W7 u1 G7 h% H+ fblacklisting 192.168.0.77:0/3574198266 until 2021-07-21 17:40:57.398343 (3600 sec)
$ K/ x3 ^1 V8 w- o

) P0 t" q" C; [/ B, d+ H- A: c[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
# U  Y; @: |3 @: U% C' yWatchers: none

% f  Z. o5 P1 R# J& _# H
[root@compute02 ~]# rbd rm vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
Removing image: 100% complete...done.

7 p8 ?  X$ v0 P- u; x/ i删除成功

admin

同样的问题再次出现，解决步骤如下：

+ ?4 K$ y$ \' [' L7 t$ T[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
, N/ C- p; q/ A- O! F$ I8 H2022-07-14 14:01:37.980 7f3cddffb700 -1 librbd::image::PreRemoveRequest: 0x55b914876b60 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
rbd: error: image still has watchers
) o. {+ j0 ]& b& K# u% a* jThis means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.
. j2 p% H( p% v3 B' f: t# D

; C& J/ H) B& U0 `" P0 f4 J) C[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
. J$ a5 [4 x, V0 i8 p2022-07-14 14:01:51.812 7f1736f22700 -1 librbd::image::PreRemoveRequest: 0x560a2b06ea70 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
! M; m/ o$ f  B( T9 o6 Hrbd: error: image still has watchers
! J0 c1 z+ Z* {4 ^This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.
4 z5 B* V7 s/ f2 d7 W

5 }" u' v7 A1 V, R- |[root@host09 ~]# rbd info volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
5 q( y- e$ S8 V$ m, [0 @rbd image 'volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak':
0 Q* f4 {+ x' \$ N        size 50 GiB in 12800 objects
' B! T$ |- h  o3 E# H        order 22 (4 MiB objects)
3 I" N. X) y% J4 x1 @        snapshot_count: 0
        id: 1a910a2feef869
        block_name_prefix: rbd_data.1a910a2feef869
        format: 2
        features: layering
        op_features:
- P- i2 F. }. d4 O) L/ j  U        flags:
& `, W: U, v4 Q% i" W        create_timestamp: Fri May 14 11:39:01 2021
# Y  p, G% ~' v1 C7 F5 o        access_timestamp: Tue Jul 12 11:19:29 2022
0 V: A) `9 m) q. j( [$ t3 i# x6 E. r1 E        modify_timestamp: Fri May 14 11:39:01 2021
        parent: images/0728be2a-d8d7-4e29-9e04-90c9ae91dca0@snap
        overlap: 2 GiB
  Q, c0 d: C+ j( c: t
  X5 x' G$ U1 q4 n8 t+ z[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
, E# X" l' v+ r# s/ b' \Watchers:
4 S6 V3 y5 _# Y; T) J        watcher=172.16.129.12:0/3801911378 client.331035433 cookie=94155368121216
3 R4 A& B; e" G. W! |9 Y
1 B  v$ M" `6 X! R! i/ B
[root@host09 ~]# ceph osd blacklist add 172.16.129.12:0/3801911378
blacklisting 172.16.129.12:0/3801911378 until 2022-07-14 15:07:44.315065 (3600 sec)
- W. J* ~/ o3 d4 c

[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
Watchers: none
  i/ D3 u% }3 V0 z2 U9 z/ [% A
8 M$ [" f! d' D. C7 P
5 n: n! n) ~; M; R[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
7 `$ c- v; [7 M- P% B" q4 [Removing image: 100% complete...done.
2 `% f  O7 y  j
0 c1 c1 w  s9 v! d
[root@host09 ~]# ceph osd blacklist
no valid command found; 3 closest matches:
; F; t# u4 d  f1 ~osd blacklist ls
osd blacklist clear
osd blacklist add|rm <EntityAddr> {<float[0.0-]>}
7 ^' W. A% [( ]& f) z2 q
  ]% Z' @- A0 k5 [) ~- t
[root@host09 ~]# ceph osd blacklist ls
" M* Z# L; G3 S! J' Flisted 1 entries
: ^  a( P: q; S3 \172.16.129.12:0/3801911378 2022-07-14 15:07:44.315065


[root@host09 ~]# ceph osd blacklist rm 172.16.129.12:0/3801911378
2 o( z  \" x! A9 H+ V2 h4 ~un-blacklisting 172.16.129.12:0/3801911378


7 J" ~+ O% H# |' \- @9 I[root@host09 ~]# ceph osd blacklist ls
listed 0 entries
' _: Z2 A' H, u; _0 h; L( S
' I9 E! v( e% I% K2 Y0 j& y0 i  L
5 w( V5 j7 `- D$ f6 g