rbd: error: image still has watchers....Try again after closing/unmapping ...rbd

admin

[root@compute02 ~]# rbd -p vms info 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
% S" d% O, }; h" b. Q& |, f9 V9 \rbd image '234abd4d-2375-44e3-b64a-0e84f247e2b4_disk':
2 D4 c, ]8 j8 f% U6 J size 10 GiB in 1280 objects
order 23 (8 MiB objects)
+ D" {  Q- F) b& W# g" p snapshot_count: 0
id: 2a3c265c791c4
; o4 D9 `6 D% J1 ?2 ^; g block_name_prefix: rbd_data.2a3c265c791c4
format: 2
; M- O( J; i8 n1 ~& O6 L features: layering
8 ^4 m( B9 a9 B$ ~# R( K, e# c op_features:
flags:
2 U. t1 g/ L' \1 T9 P, w' D- y create_timestamp: Wed Jul 21 15:36:47 2021
# X" o% X* K2 _, Q7 q4 { access_timestamp: Wed Jul 21 15:36:47 2021
modify_timestamp: Wed Jul 21 15:36:47 2021
parent: images/64079697-b54d-4499-872b-a79d3daa48b8@snap
overlap: 4 GiB
[root@compute02 ~]# rbd -p vms rm 234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
2021-07-21 16:38:39.334 7f14abfff700 -1 librbd::image::PreRemoveRequest: 0x557dd1fd5c30 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
rbd: error: image still has watchers
This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.

删除失败：
解决步骤如下：
[root@compute02 ~]# rbd status vms/2d05517a-8670-4cce-b39d-709e055381d6_disk
rbd: error opening image 2d05517a-8670-4cce-b39d-709e055381d6_disk: (2) No such file or directory
4 C8 c6 w$ \. l" {
7 y% Q' R  q5 i- _[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
  r; h+ |6 ~6 Z8 q( E3 m  B: M$ TWatchers:
watcher=192.168.0.77:0/3574198266 client.173021 cookie=93887687311360

9 X. r! n$ ^4 \, E
* ?& Q$ M7 Y' y[root@compute02 ~]# ceph osd blacklist add 192.168.0.77:0/3574198266
blacklisting 192.168.0.77:0/3574198266 until 2021-07-21 17:40:57.398343 (3600 sec)
+ T9 h8 B$ r) s' |' Q  @; t
( m  S, ?, V2 K: a1 k$ q4 o
2 r, D0 p1 R+ k. s9 B[root@compute02 ~]# rbd status vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
Watchers: none
9 e5 M3 B- i  S- M

3 D  |1 \9 z1 t! p  y[root@compute02 ~]# rbd rm vms/234abd4d-2375-44e3-b64a-0e84f247e2b4_disk
Removing image: 100% complete...done.

删除成功

admin

同样的问题再次出现，解决步骤如下：

% }- U- S5 X* ~0 v" x* E% v: y( x' r[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
* Z) Z. u! ?( `  u1 X2022-07-14 14:01:37.980 7f3cddffb700 -1 librbd::image::PreRemoveRequest: 0x55b914876b60 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
rbd: error: image still has watchers
9 \2 h  I( W8 l. X# P* gThis means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.


( f; _: z4 p$ V4 B' }1 }[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
% S6 I! N5 g' E+ }) ]0 K2022-07-14 14:01:51.812 7f1736f22700 -1 librbd::image::PreRemoveRequest: 0x560a2b06ea70 check_image_watchers: image has watchers - not removing
Removing image: 0% complete...failed.
  l+ e* t7 c/ b0 `rbd: error: image still has watchers
This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.
3 E5 w8 s& o; g5 r. t: m

4 n; T7 F$ ?/ _" D: V, _3 T[root@host09 ~]# rbd info volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
" S: s2 B& m, }* ]/ R. Jrbd image 'volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak':
        size 50 GiB in 12800 objects
        order 22 (4 MiB objects)
        snapshot_count: 0
6 I$ t0 B$ ~8 e1 u        id: 1a910a2feef869
        block_name_prefix: rbd_data.1a910a2feef869
! U+ s$ Z) L; F' g, z* y' L+ U        format: 2
9 _$ F3 n' E9 O  k) w        features: layering
" K8 f) \/ X) w        op_features:
        flags:
5 _2 i4 L  d1 y4 L8 s        create_timestamp: Fri May 14 11:39:01 2021
6 a  c! K; }+ L4 j$ y" k2 I$ H        access_timestamp: Tue Jul 12 11:19:29 2022
        modify_timestamp: Fri May 14 11:39:01 2021
; }% ~$ M8 }) f* E. V6 j        parent: images/0728be2a-d8d7-4e29-9e04-90c9ae91dca0@snap
        overlap: 2 GiB
0 A# [1 G. j7 p/ Y6 t9 Q
; F2 R# s! s6 M# o[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
Watchers:
, w8 w) r0 B% H. d8 s$ }        watcher=172.16.129.12:0/3801911378 client.331035433 cookie=94155368121216

4 n  d9 T; m* B+ {! ?
+ p. g1 c6 ]+ P2 u% j[root@host09 ~]# ceph osd blacklist add 172.16.129.12:0/3801911378
blacklisting 172.16.129.12:0/3801911378 until 2022-07-14 15:07:44.315065 (3600 sec)

' \; q: E9 Y0 a4 M
* c8 {; e6 O4 Y  q' l* K1 _[root@host09 ~]# rbd status volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
Watchers: none

+ y* s2 x( N1 J! D  `3 s
% `, z. c5 j3 D( W7 f[root@host09 ~]# rbd rm volumes-hdd/volume-98e50ac1-3b42-4393-b53f-6c6832fec4c4-bak
8 M5 ?* m& `1 D( y- L7 d& e# uRemoving image: 100% complete...done.
# v, s1 D  y+ x1 A) A6 _. r

5 G, l! I) V  Z0 @1 d* d0 Q[root@host09 ~]# ceph osd blacklist
- L9 y! R. ?' B4 q# {! Yno valid command found; 3 closest matches:
osd blacklist ls
# D: ]) G6 N3 k6 l6 O+ A0 ?. r0 uosd blacklist clear
$ {# |+ J: B5 m/ A, ?, @osd blacklist add|rm <EntityAddr> {<float[0.0-]>}
- @5 b5 E, ]8 z- N$ Y
) q- I+ D) b6 V  b: x- l% C
[root@host09 ~]# ceph osd blacklist ls
listed 1 entries
172.16.129.12:0/3801911378 2022-07-14 15:07:44.315065


[root@host09 ~]# ceph osd blacklist rm 172.16.129.12:0/3801911378
3 s2 g% F9 y, N9 H0 _% Q- D6 yun-blacklisting 172.16.129.12:0/3801911378


: f9 ^& E1 @+ \! V4 Y: J! Z+ n[root@host09 ~]# ceph osd blacklist ls
listed 0 entries