|
|
[root@controller1 ~]# ceph -s
! ?8 N' X! b& w, l! T cluster:
) t1 Q3 S& U) F8 ?% f, G* F) t, j id: 2af51d38-db90-4a57-a43d-ea9f6ebd7482
. K5 \+ Z; K1 a! p8 D8 J# ] health: HEALTH_ERR
' h. Y# a5 K/ _6 b8 _ 1 scrub errors$ [& R1 p) ^* O4 h( H4 A3 E
Possible data damage: 1 pg inconsistent
4 t$ s2 s: z& O$ `. n7 `, _9 \& N
2 ]% \+ @; U2 ? R) u f R( P services:
- L; ?& i* ^+ y; V: o mon: 5 daemons, quorum compute01,compute02,compute03,compute05,compute08 (age 13d)& e: J6 q! `4 v: Z5 J- Q6 A! z
mgr: compute03(active, since 3M), standbys: compute02, compute01, compute08, compute053 X; u. _4 D& m- o
mds: 1 up:standby0 J3 y+ s. Z- Z* |
osd: 32 osds: 32 up (since 4w), 32 in (since 3M); J. e& v& ?2 T( X
, {) J7 P& _6 F3 @+ P! b& l) \
data:. ]2 x: j5 @, M
pools: 7 pools, 3712 pgs7 k8 w" ]! D- B% B+ Q( [
objects: 1.71M objects, 6.5 TiB
9 d6 @' W! d( c' n2 }1 y; g$ P6 i& n usage: 13 TiB used, 131 TiB / 144 TiB avail4 R$ ~. N Z( d3 v- r; _- C$ r
pgs: 3707 active+clean9 z/ o" J( E2 H/ D0 ~
4 active+clean+scrubbing+deep5 c" a$ \) N$ T4 v. M p6 z; ^- j
1 active+clean+inconsistent) b: m! d8 h2 J2 c5 N) F) R* V
/ N; V P: m& q/ |: z: G) M G io:0 D6 b( X: L- Z J: R& U- L
client: 290 KiB/s rd, 9.7 MiB/s wr, 400 op/s rd, 577 op/s wr; I$ Z% M9 m1 }, o
/ A9 S0 u0 L# I$ v[root@controller1 ~]# ceph health detail . I% N* v# t! Q: I
HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
" Q/ i! |2 C! ^7 ]9 v) gOSD_SCRUB_ERRORS 1 scrub errors
. E( X. `7 X C& B5 APG_DAMAGED Possible data damage: 1 pg inconsistent; _; P9 b1 y: i+ Z
pg 9.f1 is active+clean+inconsistent, acting [11,9]! Q5 k+ \, P9 e( N
查找osd在哪里？
9 \9 [! [$ P4 |[root@controller1 ~]# ceph osd find 114 g: e V+ B9 K4 P% b+ O+ f
{
- i) L1 ]! P | "osd": 11,( E% r. @. V x7 z
"addrs": {
! {( q/ a- D& X "addrvec": [
7 H9 d; F9 I5 o" M: { {
& K: X. f* h4 _% h "type": "v2",
9 g! v4 L. f* N "addr": "192.168.0.79:6800",5 N- G4 G. u9 `
"nonce": 2194
$ q, s' \2 u9 Z4 { w4 g },7 t4 H; l; k* e% _9 S. Y+ K/ T
{$ i; v) U Y& k a7 S
"type": "v1",
( P! Q0 w+ ?- j+ d% P, d+ b "addr": "192.168.0.79:6801",
% O8 T) k- @; E% B# l "nonce": 2194: i6 y, S2 a0 J2 ?
}
) z5 U) x$ o# ^0 T, C ]
% `9 A) ~0 X) O$ _2 Z" V },
8 {7 \* |3 {! U/ G "osd_fsid": "5bf2cf92-b7bc-45ee-bfc9-6a1cf553a1d3",9 C: l; {' V6 o( T6 K
"host": "compute03",, F- Y" ~' X3 _/ ?0 u9 K& t
"crush_location": {& `2 H% A. [5 T9 ^* p) e0 v
"host": "hdd-0.6T-compute03",
3 M0 j Q9 n1 J& c3 R "root": "hdd-0.6T"2 O$ Q+ Y$ k8 ^0 j. H, t/ u
}
) N8 X5 K' S, [}
5 W1 {# w) ^$ ?5 n[root@controller1 ~]# ceph osd find 9
% O, Z8 v) p, D/ f{ ^) ]. U) l- ]+ k
"osd": 9,# P% j/ u9 \- V2 v! [ v/ R6 v8 O
"addrs": {: C$ D. _- Y2 J) U5 F- w
"addrvec": [/ }: A& _. J9 o, S& ^# p3 E
{
" M" a( ^/ V9 J, A, e# h "type": "v2",
) q' m7 K& p7 R' B% j5 p "addr": "192.168.0.75:6800",
& w" E5 T. I+ D/ p "nonce": 20460 g" d9 m5 Z/ d2 Z/ k# a; W
},
& H% _# A" J/ e s {# [7 k* E4 F' H* ^
"type": "v1",
: U7 K$ i) r# d, C" a, D "addr": "192.168.0.75:6802",5 X0 {0 {5 a k# y
"nonce": 2046) ?& b+ S; r0 V) d1 T9 Q
}
. U9 w9 j/ U; ]( L! Z9 } ]
0 A/ O8 o8 L) Z },( X6 g/ k( }5 X
"osd_fsid": "40efbcad-2acc-4e43-82d0-da8e473054a1",) O; _+ }2 P9 {3 }
"host": "compute01",8 A, F2 p5 R( \. x
"crush_location": {1 K) _: u% M8 G, I2 q" q2 X2 W
"host": "hdd-0.6T-compute01",
2 y) Z) ^+ }( K! j1 H& o "root": "hdd-0.6T") O! f$ l4 u! `# i! J
}7 n! o& c! H4 k
}
6 i6 _/ O+ _3 T8 P& V: R4 w# E
, C, @. m3 h* e+ s# Q4 C/ o* K& J8 p( M0 z" Q
登录对应节点进行操作：3 ~& H" E( b9 r1 z6 \ ]
[root@controller1 ~]# ssh compute01
3 v+ X. n5 X4 ]Last login: Fri Aug 5 12:35:04 2022 from 192.168.3.288 b, _7 @5 ?2 G% j) N. L
[root@compute01 ~]# systemctl stop ceph-osd@9.service ! P9 ^. @& N+ B# e# [; ]- z
3 U$ u8 x: e( r. V7 J! n, c
8 E" F& l$ S. h ^' c/ S- Q2 W+ @[root@compute01 ~]# ceph-osd -i 9 --flush-journal
" [3 c8 N* K& q2 x2022-08-08 09:23:54.750 7f714bfc2a80 -1 flushed journal /var/lib/ceph/osd/ceph-9/journal for object store /var/lib/ceph/osd/ceph-9
, U8 M8 D7 R" _[root@compute01 ~]# systemctl start ceph-osd@9.service $ O$ g' H1 n2 Q ?4 B2 C* C
" v0 [6 p' a+ u5 y- K! r" i
" C- o9 G8 \+ E
[root@compute03 ~]# ceph health detail 0 v( r5 p5 T, T. o
HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent; V- `, `0 j7 R) }# k: c- P9 I. {
OSD_SCRUB_ERRORS 1 scrub errors
, |" O! F) W9 ?4 J$ tPG_DAMAGED Possible data damage: 1 pg inconsistent9 }7 n o) v6 c
pg 9.f1 is active+clean+inconsistent, acting [11,9]
6 y$ @$ ?0 b( E3 ] U0 j% M! ^ Y
[root@compute03 ~]# ceph health detail
+ j. ]0 l6 N6 l+ o5 x+ x* {! s( dHEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent. U3 v' ~: N: K8 C; r7 ~# f9 l
OSD_SCRUB_ERRORS 1 scrub errors
: H" z5 Z( F$ N; P1 JPG_DAMAGED Possible data damage: 1 pg inconsistent
0 z- G) U- p* i" ^7 O pg 9.f1 is active+clean+inconsistent, acting [11,9]
, \. S0 O& W( Z& F: h* w* e8 h2 s, n9 c+ `% R. O
使用repair方式修复：! z. t. H( \9 d$ b
2 k: K6 f" R+ ?5 L
[root@compute03 ~]# ceph pg repair 9.f1
0 l( z- }9 K4 i+ R, Q1 J$ Zinstructing pg 9.f1 on osd.11 to repair
/ l2 @3 Y& w& L8 o5 ]& z" Q. u. u% S4 l$ Y$ Y/ s k9 J" J% f
[root@compute03 ~]# ceph health detail
, W3 U& L5 }* D) UHEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent* i: U# T* L1 r" w
OSD_SCRUB_ERRORS 1 scrub errors
8 L! X; F" G# W' n; R, T9 \- |0 NPG_DAMAGED Possible data damage: 1 pg inconsistent0 g' O. d4 I) [7 l% J
pg 9.f1 is active+clean+inconsistent, acting [11,9]
T& Y- `8 @& t8 F/ v9 E
. i' H' l! `7 \[root@compute03 ~]# ceph pg repair 9.f1
: n; _# K; R% k$ v7 Binstructing pg 9.f1 on osd.11 to repair& K5 Y. k7 m* M& e. E7 ]2 v
, X# R% W; R5 q% A, t[root@compute03 ~]# exit
; Q! K9 y+ e8 ^. qlogout
2 Y1 A* L5 e9 o# O, @+ ?6 H: iConnection to compute03 closed.6 H5 i& s. u" L
/ K* K1 c- B: ~( ^4 X[root@compute01 ~]# ceph pg repair 9.f1/ U6 K' H2 B. q) n+ P9 [
instructing pg 9.f1 on osd.11 to repair2 J% ]9 G8 K9 u. K
. }# }+ N* j# ^4 `- e5 e等待一小会同步后，即可：& m* ?% {, g9 {0 R9 G6 J
[root@compute01 ~]# ceph -s: m( x- i% S: O$ q/ C
cluster:; ^2 ]2 F% h+ K1 n- o w
id: 2af51d38-db90-4a57-a43d-ea9f6ebd7482
3 I i; v0 t" p- c& r3 D8 @ health: HEALTH_OK
1 D5 s% o) `. l; }# R1 o8 m) ] N" q) R7 }2 t8 [3 P
services:
2 Z f2 z K" Q! o9 b mon: 5 daemons, quorum compute01,compute02,compute03,compute05,compute08 (age 13d)8 U- C2 N: x0 r9 H3 \2 Q. P
mgr: compute03(active, since 3M), standbys: compute02, compute01, compute08, compute05
! L6 J# e$ b. K7 }. }6 T mds: 1 up:standby
C5 d: w; F$ E+ D* Z7 g osd: 32 osds: 32 up (since 112s), 32 in (since 3M)
# Q4 }+ Z; U- ?' w8 H
! ^5 S6 ]: \* `# n) n data:
, d- J8 y2 h8 T Y& e3 ~ pools: 7 pools, 3712 pgs0 M" A1 ~( L3 @. T2 _
objects: 1.71M objects, 6.5 TiB
$ r) e. K5 p/ j f8 g3 C: |) V9 i usage: 13 TiB used, 131 TiB / 144 TiB avail
) K6 t0 Z' h; M& t5 ~* ]6 I pgs: 3707 active+clean
% l) h) a0 v) e$ g. h2 N 4 active+clean+scrubbing+deep2 a1 J- \5 q. N" ?' H
1 active+clean+scrubbing+deep+repair
# o+ R* C& w' n4 T1 x, t' `- ]
" Y& p; M! u) s3 w$ o io:
2 _) @( q) m9 U& G9 P- u3 \1 z8 ^$ O client: 442 KiB/s rd, 9.1 MiB/s wr, 573 op/s rd, 562 op/s wr% t8 E. |4 N( M
/ z/ c+ K; g5 [[root@compute01 ~]# ceph health detail
% E, {7 S! j. n' bHEALTH_OK
6 S/ ?! T Y+ U3 h7 X1 d8 `! i: n
|
|
发表于 2022-8-8 09:19:38