HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent 解决过程

admin

[root@controller1 ~]# ceph -s
+ c% o% D+ y) S2 H5 F  cluster:
    id:     2af51d38-db90-4a57-a43d-ea9f6ebd7482
    health: HEALTH_ERR
            1 scrub errors
            Possible data damage: 1 pg inconsistent
( e& Y0 [8 b/ M- e# \: Q
  services:
0 j3 B5 s8 B7 ~    mon: 5 daemons, quorum compute01,compute02,compute03,compute05,compute08 (age 13d)
* J$ f6 c2 j( l! ~6 k* h# j% T8 O    mgr: compute03(active, since 3M), standbys: compute02, compute01, compute08, compute05
4 K% n" t( i9 F6 ~0 p8 w" @    mds:  1 up:standby
. e$ y; \/ \6 y9 z$ S- ?; ?% z    osd: 32 osds: 32 up (since 4w), 32 in (since 3M)
, o) C1 F. y; b; e$ B: W
  data:
& z! i( B) v/ E: Y# t    pools:   7 pools, 3712 pgs
% s( b  i0 p( _6 G+ a" ^% b" v- i7 ]    objects: 1.71M objects, 6.5 TiB
: {" n# j: u2 q$ z6 X$ o- G    usage:   13 TiB used, 131 TiB / 144 TiB avail
    pgs:     3707 active+clean
             4    active+clean+scrubbing+deep
* [3 L. ]- f( J$ J# K             1    active+clean+inconsistent

  io:
    client:   290 KiB/s rd, 9.7 MiB/s wr, 400 op/s rd, 577 op/s wr
: K% _/ J( g. P2 F1 v
0 A. _9 \: L) m[root@controller1 ~]# ceph health detail
* W% V. X0 a& p0 I- q/ Y- CHEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
OSD_SCRUB_ERRORS 1 scrub errors
8 P7 W8 X+ r! U5 o3 j1 V' cPG_DAMAGED Possible data damage: 1 pg inconsistent
    pg 9.f1 is active+clean+inconsistent, acting [11,9]
( M4 W# X! V- D查找osd在哪里？
[root@controller1 ~]# ceph osd find  11
& P& r! R" p' @: m1 N: t3 e{
0 \$ o3 X9 Q: \/ E2 a    "osd": 11,
7 M3 \* V/ R7 F& [" ?3 o+ E1 O    "addrs": {
" S7 r. ~# p# @( ^: D- }* O        "addrvec": [
            {
                "type": "v2",
                "addr": "192.168.0.79:6800",
                "nonce": 2194
            },
            {
% e  f& ^  R4 o+ E6 S6 m                "type": "v1",
                "addr": "192.168.0.79:6801",
                "nonce": 2194
            }
        ]
- s8 n  M& H1 r9 V: L# F3 ^, p9 z  B( W    },
    "osd_fsid": "5bf2cf92-b7bc-45ee-bfc9-6a1cf553a1d3",
3 W& J$ i4 ?6 B! h+ a& p* g    "host": "compute03",
3 I; R' x: u$ E, J5 U    "crush_location": {
  ^6 L; H, }' R1 ?  d! G        "host": "hdd-0.6T-compute03",
+ y: e5 @; [& T5 D8 c, x        "root": "hdd-0.6T"
    }
}
9 }+ o' q; L& T0 _1 f, K[root@controller1 ~]# ceph osd find  9
5 D& n( s( W( @; M{
% L5 C5 U- h5 m3 N% j# F    "osd": 9,
8 v$ a4 a( C4 f9 P  i3 {    "addrs": {
  O, }7 A" t! L' w; G) L) ~" N7 l$ R        "addrvec": [
3 J0 y' b" V1 t, `. C7 |            {
. w5 ^: G. ?' z# @& N4 ^                "type": "v2",
                "addr": "192.168.0.75:6800",
+ g0 i0 |% q! U! i. H$ t" t                "nonce": 2046
            },
) S9 L3 m7 W4 k; O, f            {
8 d5 `# f" J- _! s                "type": "v1",
% A& F( y0 U& J2 A9 u                "addr": "192.168.0.75:6802",
0 x! a& J: O# d3 S                "nonce": 2046
            }
6 v- K. [6 t; r; Z6 ]        ]
    },
) i% W8 [8 G; ?& r% q# n' T) M% R    "osd_fsid": "40efbcad-2acc-4e43-82d0-da8e473054a1",
    "host": "compute01",
    "crush_location": {
, M2 o( {- O, K( l" [8 `* m% ^        "host": "hdd-0.6T-compute01",
; I; z9 u7 {9 k2 G6 h        "root": "hdd-0.6T"
    }
6 Y# }! r& g) S# ~}
; y7 @' X+ \7 X, e* ~
! |- d& O  [% i! H- ?& T
登录对应节点进行操作：
[root@controller1 ~]# ssh compute01
Last login: Fri Aug  5 12:35:04 2022 from 192.168.3.28
5 m/ _8 v. K$ P$ r[root@compute01 ~]# systemctl stop ceph-osd@9.service
+ f4 S4 @' Y! N% p, U$ s  N2 B, M$ C' r

[root@compute01 ~]# ceph-osd -i 9 --flush-journal
2022-08-08 09:23:54.750 7f714bfc2a80 -1 flushed journal /var/lib/ceph/osd/ceph-9/journal for object store /var/lib/ceph/osd/ceph-9
  Z" y/ ~" K! l: a; H[root@compute01 ~]# systemctl start ceph-osd@9.service
# C0 [% v7 Z) |
( R4 `- B/ H6 G! S7 B3 O% E3 p
" P& U: T7 O% ?! E" \6 s[root@compute03 ~]# ceph health detail
" w9 `8 ?- `5 h% W% hHEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
OSD_SCRUB_ERRORS 1 scrub errors
PG_DAMAGED Possible data damage: 1 pg inconsistent
    pg 9.f1 is active+clean+inconsistent, acting [11,9]
1 s& q" ]! d& W
[root@compute03 ~]# ceph health detail
HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
) A! t, q, }: P% D( `6 E$ b/ q3 Y) zOSD_SCRUB_ERRORS 1 scrub errors
5 \6 a2 t9 m8 F! t! Q; ?3 yPG_DAMAGED Possible data damage: 1 pg inconsistent
, M4 p( t) _3 u1 v+ m+ P" [- E    pg 9.f1 is active+clean+inconsistent, acting [11,9]
3 c7 H# Q; Q# q; Q1 T
; f5 b9 t  U5 {7 p) b- m使用repair方式修复：
4 i( B7 ~2 a3 U- }  v
[root@compute03 ~]# ceph pg repair 9.f1
. U" r% w  }  Q  W6 u8 I, e0 Cinstructing pg 9.f1 on osd.11 to repair

4 N& i+ W0 v3 u7 C[root@compute03 ~]# ceph health detail
HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
9 U' w) }/ P3 _, u1 vOSD_SCRUB_ERRORS 1 scrub errors
PG_DAMAGED Possible data damage: 1 pg inconsistent
* n. D! ]  Q5 c' E$ i) A; ?; n    pg 9.f1 is active+clean+inconsistent, acting [11,9]

+ H( r* G. g7 j! f) }; h$ P$ Z[root@compute03 ~]# ceph pg repair 9.f1
instructing pg 9.f1 on osd.11 to repair
6 G* }5 s" m) D
0 o3 W, I9 c* Q8 j[root@compute03 ~]# exit
logout
: q. {6 B& X: ]4 K5 g8 yConnection to compute03 closed.

$ a. b# j6 j& J& X- k, e: [& P: X' D[root@compute01 ~]# ceph pg repair 9.f1
instructing pg 9.f1 on osd.11 to repair

7 D) j7 }: d* x( ?7 H等待一小会同步后，即可：
[root@compute01 ~]# ceph -s
# U% }0 M3 k# A$ v# V3 s  cluster:
( R7 y; b6 f7 B" A: l: ]6 e    id:     2af51d38-db90-4a57-a43d-ea9f6ebd7482
    health: HEALTH_OK

* a1 I) o: e- }9 d! ]$ Y! E9 f  services:
8 C5 V* U3 J+ d6 M: H    mon: 5 daemons, quorum compute01,compute02,compute03,compute05,compute08 (age 13d)
2 ~3 t# b7 n  z9 p    mgr: compute03(active, since 3M), standbys: compute02, compute01, compute08, compute05
( K) D8 e6 y7 x  _3 c8 Q; n! j    mds:  1 up:standby
1 Q( g/ [* \* m- y6 M    osd: 32 osds: 32 up (since 112s), 32 in (since 3M)

  data:
    pools:   7 pools, 3712 pgs
: N. \" b* E5 ^9 F; i1 C1 p    objects: 1.71M objects, 6.5 TiB
    usage:   13 TiB used, 131 TiB / 144 TiB avail
* {- v8 P- E1 u7 s( D    pgs:     3707 active+clean
             4    active+clean+scrubbing+deep
             1    active+clean+scrubbing+deep+repair

  io:
    client:   442 KiB/s rd, 9.1 MiB/s wr, 573 op/s rd, 562 op/s wr
$ g; \* Q4 n" b
1 F) B% g0 U, p% O[root@compute01 ~]# ceph health detail
/ e7 w( b' X- h7 |HEALTH_OK
8 f' A2 ~/ H. X4 Q6 Z& u- P2 i; F