|
|
楼主 |
发表于 2023-11-24 00:06:43
|
显示全部楼层
解决过程,发现无发正常同步,通过检查调整参数,达到修复的目的:
1 g; E4 T( m8 j- _6 n. m; ^& n[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 1 Z" N" v4 O B0 q, o4 n
"osd_max_backfills": "1",
/ R, k5 }3 b# r2 u) b8 w' s. X5 C+ N "osd_recovery_max_active": "1",
2 l, M* l% o+ X "osd_recovery_op_priority": "0",! I2 {7 @4 b5 L1 O, v$ ~
"osd_recovery_op_priority": "0", 默认是10
4 x( Z6 O! F4 E6 Z& c4 p8 U' `0 H0 y
( l8 k+ `0 x0 \- h/ B修改为默认值10 , r# k% N3 O. D: {/ s6 Y8 ?
' J" p, @2 ~1 ~' v[root compute1 ~]# ceph tell osd.* injectargs osd_recovery_op_priority=10
% X* W0 P3 g. Z4 J# Oosd.0: osd_recovery_op_priority = '10' (unchangeable)
- }9 a# a* h) D; T9 wosd.1: osd_recovery_op_priority = '10' (unchangeable)
" c8 U) Q9 w) ?4 m0 |osd.2: osd_recovery_op_priority = '10' (unchangeable) " K3 Y! M! c9 ~5 [7 ^
osd.3: osd_recovery_op_priority = '10' (unchangeable)
# {$ q Y8 G. d$ Qosd.4: osd_recovery_op_priority = '10' (unchangeable) 7 o7 t/ t o7 R2 P
osd.5: osd_recovery_op_priority = '10' (unchangeable) ! n2 w* y0 T6 Q4 W
osd.6: osd_recovery_op_priority = '10' (unchangeable)
% e) ~/ j9 ]( }* k( E, Zosd.7: osd_recovery_op_priority = '10' (unchangeable) % ?( t& V5 [/ R4 ]4 g0 M" f! T
osd.8: osd_recovery_op_priority = '10' (unchangeable) 2 N6 H1 _& q2 F/ }9 y8 v8 S
osd.9: osd_recovery_op_priority = '10' (unchangeable)
# } j( @$ _- p6 K8 Z4 v- Z/ qosd.10: osd_recovery_op_priority = '10' (unchangeable) 7 O' E2 f+ o! j& L0 m# p7 j" b
osd.11: osd_recovery_op_priority = '10' (unchangeable)
: l7 u0 R& E! c E$ posd.12: osd_recovery_op_priority = '10' (unchangeable)
, U. u2 A& L$ _osd.13: osd_recovery_op_priority = '10' (unchangeable) 7 s7 b- C1 Q0 t
osd.14: osd_recovery_op_priority = '10' (unchangeable)
; V! N) K* a* K" s9 U9 n; Qosd.15: osd_recovery_op_priority = '10' (unchangeable)
( z/ |1 ?/ J/ p7 k* A+ Eosd.16: osd_recovery_op_priority = '10' (unchangeable)
+ _# n. [6 t/ k( a3 gosd.17: osd_recovery_op_priority = '10' (unchangeable) 8 r8 N. A, F% C2 ^
osd.18: osd_recovery_op_priority = '10' (unchangeable) 2 w* y, D2 z/ {& T. t0 L4 F
osd.19: osd_recovery_op_priority = '10' (unchangeable)
9 {% S" _/ M" [5 e: r* Bosd.20: osd_recovery_op_priority = '10' (unchangeable) , h/ g! q- D$ n6 F
osd.21: osd_recovery_op_priority = '10' (unchangeable)
, s' W X' l) A- W+ a" P- d) Yosd.22: osd_recovery_op_priority = '10' (unchangeable) ) A5 T% q! f$ h6 K
osd.23: osd_recovery_op_priority = '10' (unchangeable) . A: f0 @3 |* \5 m) n1 E
osd.24: osd_recovery_op_priority = '10' (unchangeable) / X' q6 H. t L- B
osd.25: osd_recovery_op_priority = '10' (unchangeable) 2 z+ z S6 J+ s6 r
osd.26: osd_recovery_op_priority = '10' (unchangeable)
[/ ~ `) ]: t. O: tosd.27: osd_recovery_op_priority = '10' (unchangeable) 0 ^2 m; W( C" L% P" r" l( c
osd.28: osd_recovery_op_priority = '10' (unchangeable)
! Z$ V2 b( F; z4 E: S9 p% J& posd.29: osd_recovery_op_priority = '10' (unchangeable)
9 ~5 r9 a) _, Y3 {: y. ~: G8 rosd.30: osd_recovery_op_priority = '10' (unchangeable)
& C; s) c8 |+ yosd.31: osd_recovery_op_priority = '10' (unchangeable) 1 s! Q# r% o. H! `8 \
osd.32: osd_recovery_op_priority = '10' (unchangeable) 1 H' \* k. O. q# r |0 ~" h1 w
osd.33: osd_recovery_op_priority = '10' (unchangeable)
% ^+ M, F) W0 H% |. _osd.34: osd_recovery_op_priority = '10' (unchangeable) 4 B/ `1 Y$ {3 S; `
osd.35: osd_recovery_op_priority = '10' (unchangeable) 9 \! [7 I3 e" O2 H5 _0 @
osd.36: osd_recovery_op_priority = '10' (unchangeable)
) g, L5 i* f! c9 \- D3 O# o" ?, @osd.37: osd_recovery_op_priority = '10' (unchangeable)
3 _5 j6 r1 J$ o9 ^osd.38: osd_recovery_op_priority = '10' (unchangeable)
- q9 g" B) {0 |, y; Qosd.39: osd_recovery_op_priority = '10' (unchangeable) % D" |5 [, }4 U2 Q6 q5 H' b
osd.40: osd_recovery_op_priority = '10' (unchangeable)
/ k; e# g1 L0 I* i, G2 W# B! u/ wosd.41: osd_recovery_op_priority = '10' (unchangeable) * a4 X5 n" L# r/ y
osd.42: osd_recovery_op_priority = '10' (unchangeable) % ~8 P, z2 S# b& @
osd.43: osd_recovery_op_priority = '10' (unchangeable)
4 f( q: J4 N. ?- Y; s8 r# |osd.44: osd_recovery_op_priority = '10' (unchangeable) 3 c4 `1 y& w ?: |" L$ p" W
osd.45: osd_recovery_op_priority = '10' (unchangeable)
( p0 Y* s' n& S# A+ d2 N+ wosd.46: osd_recovery_op_priority = '10' (unchangeable) 7 I5 ^4 K7 ] t4 M* V
osd.47: osd_recovery_op_priority = '10' (unchangeable)
5 A; L# v( g! p# ^! E$ S" m# w2 Yosd.48: osd_recovery_op_priority = '10' (unchangeable)
% r4 p- W/ O# ^1 i* ?3 B$ K+ Gosd.49: osd_recovery_op_priority = '10' (unchangeable)
0 c D, ?% k& H8 L. sosd.50: osd_recovery_op_priority = '10' (unchangeable) # P2 O0 G4 q8 c) u g
osd.51: osd_recovery_op_priority = '10' (unchangeable)
, R5 t3 [0 `4 C& r7 o; Sosd.52: osd_recovery_op_priority = '10' (unchangeable)
# p A6 q7 H$ k1 losd.53: osd_recovery_op_priority = '10' (unchangeable) ( K; Q# F: Q/ C, E3 k! e
Error ENXIO: problem getting command descriptions from osd.543 b$ t4 V7 c" K& T* o) f; y7 f
osd.54: problem getting command descriptions from osd.54
) i- v; ]$ B* c! ~. i9 B( vosd.55: osd_recovery_op_priority = '10' (unchangeable)
% u9 _, H' i9 ^" @6 B( x$ H4 yosd.56: osd_recovery_op_priority = '10' (unchangeable)
4 M* X$ B: B" b, k) f& ~osd.57: osd_recovery_op_priority = '10' (unchangeable) 6 j3 H" K/ W5 [7 i G9 s
osd.58: osd_recovery_op_priority = '10' (unchangeable) 8 O4 L0 M$ r1 x% U/ T
osd.59: osd_recovery_op_priority = '10' (unchangeable)
# p- b; g4 d. U, Cosd.60: osd_recovery_op_priority = '10' (unchangeable)
, c0 l! R Y* iosd.61: osd_recovery_op_priority = '10' (unchangeable)
& _/ w& Q. b% b3 kosd.62: osd_recovery_op_priority = '10' (unchangeable)
) b# n6 S7 m8 _! posd.63: osd_recovery_op_priority = '10' (unchangeable) + M% A' A. _, ]( d+ C
osd.64: osd_recovery_op_priority = '10' (unchangeable) / ~: N( U' K4 z# p6 g3 Z: G0 k6 J
osd.65: osd_recovery_op_priority = '10' (unchangeable) 2 X( u) R% {& r) B
osd.66: osd_recovery_op_priority = '10' (unchangeable)
0 }. _) x) T. c7 }; [7 oosd.67: osd_recovery_op_priority = '10' (unchangeable) $ O+ _" F0 `- _" f
osd.68: osd_recovery_op_priority = '10' (unchangeable)
M) I2 Y! T/ M. }osd.69: osd_recovery_op_priority = '10' (unchangeable) ' d: i% C# l( }
osd.70: osd_recovery_op_priority = '10' (unchangeable) 3 v; [0 a. _- ?% |4 A: z9 `
osd.71: osd_recovery_op_priority = '10' (unchangeable) . }1 B' Q4 h" f
osd.72: osd_recovery_op_priority = '10' (unchangeable)
+ H5 p: f! g5 mosd.73: osd_recovery_op_priority = '10' (unchangeable) 5 C, r. V/ F8 A" k
osd.74: osd_recovery_op_priority = '10' (unchangeable)
" a1 q- n+ [. B. O$ W+ k9 e% dosd.75: osd_recovery_op_priority = '10' (unchangeable)
, R* v# i# q% U: }6 eosd.76: osd_recovery_op_priority = '10' (unchangeable)
9 ^/ ^( M$ L- O- r( Vosd.77: osd_recovery_op_priority = '10' (unchangeable)
9 \( y8 k' \2 `# o* P5 P7 vosd.78: osd_recovery_op_priority = '10' (unchangeable) 1 f' _( b$ d3 q! O! G+ g1 _8 ?
osd.79: osd_recovery_op_priority = '10' (unchangeable)
/ K" n: I- R5 w+ F7 [& gosd.80: osd_recovery_op_priority = '10' (unchangeable) 6 M: a3 ~9 X2 s& X( H0 v1 u( _' J
osd.81: osd_recovery_op_priority = '10' (unchangeable)
0 f& ~: O) }: Y4 M/ X3 B kosd.82: osd_recovery_op_priority = '10' (unchangeable) ; `5 R1 o5 i6 Q) [* n
osd.83: osd_recovery_op_priority = '10' (unchangeable)
8 t2 ~/ z, l8 fosd.84: osd_recovery_op_priority = '10' (unchangeable)
* a/ Q. X+ v7 Q) |. C, josd.85: osd_recovery_op_priority = '10' (unchangeable) 3 C' D: ^$ q2 C
osd.86: osd_recovery_op_priority = '10' (unchangeable) , j$ u. ^7 D% t" Y# C
osd.87: osd_recovery_op_priority = '10' (unchangeable) 2 j3 P, G* D+ M" e, v4 W4 s8 z
osd.88: osd_recovery_op_priority = '10' (unchangeable)
' o9 C1 s& I: O" S. N1 Eosd.89: osd_recovery_op_priority = '10' (unchangeable)
# t% F+ p, ~3 L& Losd.90: osd_recovery_op_priority = '10' (unchangeable) 5 b' M$ z3 ?# r0 G( N# ]
osd.91: osd_recovery_op_priority = '10' (unchangeable)
, ]6 q/ I2 z$ g. D/ bosd.92: osd_recovery_op_priority = '10' (unchangeable)
! W" s7 @" D$ p: Y4 d4 ]osd.93: osd_recovery_op_priority = '10' (unchangeable)
9 `# }) }- J$ Posd.94: osd_recovery_op_priority = '10' (unchangeable)
) s- J6 f) I+ _8 O3 J0 W8 `osd.95: osd_recovery_op_priority = '10' (unchangeable) ( m/ P9 X9 j& v" Z' F
osd.96: osd_recovery_op_priority = '10' (unchangeable) & @0 n$ c# j' _+ D
osd.97: osd_recovery_op_priority = '10' (unchangeable)
: C) e: k+ I H. [osd.98: osd_recovery_op_priority = '10' (unchangeable) & F1 D& M8 U$ ~$ ~% g8 E4 V O' l
osd.99: osd_recovery_op_priority = '10' (unchangeable)
$ ]- X6 `( o! Losd.100: osd_recovery_op_priority = '10' (unchangeable) r0 T5 n$ T! y$ v6 h1 B
osd.101: osd_recovery_op_priority = '10' (unchangeable) ) J/ |6 b B" X6 ]5 z
osd.102: osd_recovery_op_priority = '10' (unchangeable) 1 ?4 I, g% `2 U( g$ R
osd.103: osd_recovery_op_priority = '10' (unchangeable)
. Q! Q4 n: [7 @0 Yosd.104: osd_recovery_op_priority = '10' (unchangeable)
" ]2 v* ]1 a$ T+ T/ ]2 K) Y9 Kosd.105: osd_recovery_op_priority = '10' (unchangeable) ' L' \6 E) U3 O. T% p
osd.106: osd_recovery_op_priority = '10' (unchangeable)
4 M! I6 d* q- s% r/ B; Mosd.107: osd_recovery_op_priority = '10' (unchangeable)
$ y' _2 Q+ a# p, G6 y4 Aosd.108: osd_recovery_op_priority = '10' (unchangeable)
' w8 _0 s) o/ `; v4 p0 v! D: |osd.109: osd_recovery_op_priority = '10' (unchangeable) % }; _/ ]0 z2 R( X l6 p& F
osd.110: osd_recovery_op_priority = '10' (unchangeable) 9 B2 q* B2 V3 i+ h' n% T
osd.111: osd_recovery_op_priority = '10' (unchangeable)
$ n1 `9 F: b8 e& hosd.112: osd_recovery_op_priority = '10' (unchangeable)
& X8 O1 a5 ?/ ?& G0 |osd.113: osd_recovery_op_priority = '10' (unchangeable) 8 T( R8 z* d4 }8 b2 b3 _( ?$ `4 N
osd.114: osd_recovery_op_priority = '10' (unchangeable) ! {3 Y" G( E, N; H; a3 t7 M
osd.115: osd_recovery_op_priority = '10' (unchangeable)
' I( n. z; q, z" {1 c# losd.116: osd_recovery_op_priority = '10' (unchangeable)
! J7 B# h( |$ W3 V% k0 qosd.117: osd_recovery_op_priority = '10' (unchangeable)
( X0 W; c; s/ g& p {osd.118: osd_recovery_op_priority = '10' (unchangeable) / p" @" E H% V5 ^
osd.119: osd_recovery_op_priority = '10' (unchangeable) O# t* O" v3 H* e: j! K5 F) }& Q0 u
osd.120: osd_recovery_op_priority = '10' (unchangeable) , ]" \% i) P* p [
osd.121: osd_recovery_op_priority = '10' (unchangeable) 3 |. I# |/ e N' T9 n
osd.122: osd_recovery_op_priority = '10' (unchangeable) , t; \( K5 U5 q3 a5 H% v
osd.123: osd_recovery_op_priority = '10' (unchangeable) ' u# V5 \4 b9 O& B" M& j+ \
Error ENXIO: problem getting command descriptions from osd.124
# J0 G8 Q* Y5 K3 N% w5 x1 n" D5 qosd.124: problem getting command descriptions from osd.124& \( P# ~" o# G5 `
osd.125: osd_recovery_op_priority = '10' (unchangeable) % O# s! H$ ~4 u7 o8 [% w
osd.126: osd_recovery_op_priority = '10' (unchangeable) / X. u( e+ [) f# A+ H% @
Error ENXIO: problem getting command descriptions from osd.127
3 g, J! t s6 Z7 D$ Tosd.127: problem getting command descriptions from osd.127& z8 q3 C( u D! D
osd.128: osd_recovery_op_priority = '10' (unchangeable)
9 e! Z7 o5 }: L$ E. Vosd.129: osd_recovery_op_priority = '10' (unchangeable)
5 l. v6 I+ ?6 B3 y" J# _osd.130: osd_recovery_op_priority = '10' (unchangeable) . N5 b2 k+ K5 m% P% j# n
You have new mail in /var/spool/mail/root7 X2 }( h4 Y+ K8 }4 `- x5 ?
1 [$ o0 W" j* c( b7 e. H
所有的osd都修改:
5 @" O8 _, `# |) C4 f/ p, }. v+ M
* M8 o1 \1 K& Q- S5 D6 M) i, f9 g9 k执行过ceph pg repair pg-id3 |/ ?. T& B7 @" r5 G7 I
. b) G$ N1 l {" J
+ W& k; F8 d$ _# ^$ P2 N; C: \# R
状态出现修改过程:' j" u1 ~' `9 D& x9 {
[root compute1 ~]# ceph s9 i! A+ K5 v8 x( p. o, l1 S
cluster 70d27aec742e4a95b000-cf37ebba35d03 \9 |% w @! B3 @& }
health HEALTH_ERR
( Y& r/ M# i1 O* x/ O! G$ l 1 pgs inconsistent
' p( F$ K3 f6 |: [* w4 D# ~ 1 pgs repair: s: { C$ _( |+ t2 }1 W. Z+ J
1 scrub errors# Z7 x. k. a& r2 R0 ~
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
. ^9 G& L) g2 K# y: N election epoch 332, quorum 0,1,2 compute1,compute2,compute31 N8 K& _3 G- H1 c
osdmap e26725: 131 osds: 128 up, 128 in
# S. @1 ~' t" d+ b f9 x% P flags sortbitwise,require_jewel_osds
% k1 T1 t& Y$ e) u" E& u1 L pgmap v169370679: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
" ^, M* B' K4 F% ~/ K" W 309 TB used, 155 TB / 465 TB avail, p; K2 k: H; h, I5 q5 t
10229 active+clean
/ A) F9 x4 ]& O* X7 m 9 active+clean+scrubbing+deep ?4 L/ N$ M r
1 active+clean+scrubbing
4 \# l/ R, D4 N( M 1 active+clean+scrubbing+deep+inconsistent+repair
6 D0 R) S6 {+ l9 a8 D' ? client io 91086 kB/s rd, 67615 kB/s wr, 3119 op/s rd, 3569 op/s wr `1 s7 f, L) r8 J
[root compute1 ~]# ceph s% T5 X. E" y, O) e# g5 S- s, v
cluster 70d27aec742e4a95b000-cf37ebba35d0
3 F+ v3 f( z6 F/ x/ Y health HEALTH_ERR) N- y; o, ]" K) W
1 pgs inconsistent
( O; c }: F0 m3 L7 G' H 1 pgs repair8 x; O& A) q5 e8 T+ t7 J& \% m
1 scrub errors
5 v7 D5 U3 G5 {8 o7 I monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
8 D% S6 J; u5 M! I% m election epoch 332, quorum 0,1,2 compute1,compute2,compute3
/ n! C( y2 ]! v& r% M% G+ n osdmap e26725: 131 osds: 128 up, 128 in- L, R; J) j+ K+ X5 O
flags sortbitwise,require_jewel_osds
7 \) h% R: d7 D, V; O pgmap v169370681: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
' `, m% O. v% V4 T* w' b1 V& c 309 TB used, 155 TB / 465 TB avail
4 g) Q& H; t7 t2 O+ f2 a7 A# A 10230 active+clean7 `4 D* ^: d4 G0 D: g$ m
8 active+clean+scrubbing+deep
# Q& H( F( C' X* n/ ~ 1 active+clean+scrubbing
) X3 G6 r6 u9 A$ { Q 1 active+clean+scrubbing+deep+inconsistent+repair! i) ~6 n4 t' y$ _1 J. D
client io 104 MB/s rd, 67033 kB/s wr, 3479 op/s rd, 5013 op/s wr+ ~0 e+ d% |# T' ^% J# k2 k2 K% J
[root compute1 ~]# ceph s
2 u* ?% a" {% M cluster 70d27aec742e4a95b000-cf37ebba35d0
! w& ]5 N" Q$ x$ l; [! V health HEALTH_ERR
% `; q- u9 W7 L5 a5 D, `1 t( _ 1 pgs inconsistent
* v. m1 c/ K3 N3 m8 H0 M 1 pgs repair. h+ X( `6 `3 ^. u( @
1 scrub errors7 c8 Z4 ^/ B) ?" r4 }0 d
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}& G( {+ \) `7 y
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
8 V+ f" b& ?: w. [ osdmap e26725: 131 osds: 128 up, 128 in
0 u I6 S' R* v3 G4 } flags sortbitwise,require_jewel_osds
% i( k) ]; d$ [; X. x& t1 d( M pgmap v169370783: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
" [6 q& u9 A1 r4 p 309 TB used, 155 TB / 465 TB avail
8 l4 U- h, u! d 10232 active+clean1 j) ^+ Q" |. q7 s4 X' S
6 active+clean+scrubbing+deep: s# f2 r/ Z4 N' n0 n
1 active+clean+scrubbing! ]5 o* x) j& X6 Q
1 active+clean+scrubbing+deep+inconsistent+repair
* r# I# T, t; J! a8 i client io 147 MB/s rd, 57692 kB/s wr, 4584 op/s rd, 5944 op/s wr
7 \8 O1 V! O" N1 Z6 o6 N4 Y& q" KYou have mail in /var/spool/mail/root0 u8 L+ j* x, N+ {. \, B
[root compute1 ~]# ceph s
9 U$ y2 ?& a( `' ?5 S cluster 70d27aec742e4a95b000-cf37ebba35d0
?+ r. I- y4 Y( X health HEALTH_ERR
! k9 l% C5 ~( i& R/ s; C 1 pgs inconsistent/ U- H: G. \) O4 G
1 pgs repair' L3 d! R, b0 _) S
1 scrub errors
4 G. F$ |5 G/ v% V" S; B monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}# \3 _8 n# T! a8 i8 H/ g3 Q$ I
election epoch 332, quorum 0,1,2 compute1,compute2,compute3! j7 D. y- c% `1 h
osdmap e26725: 131 osds: 128 up, 128 in! e9 o$ i: T* L- V" _" K" @
flags sortbitwise,require_jewel_osds$ m" U. p/ Z3 N0 }7 i( _! P
pgmap v169370785: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
2 _6 r7 u# g' F3 n 309 TB used, 155 TB / 465 TB avail& `, _6 E- Z" k `
10232 active+clean
; {7 O+ s2 w& w! o) y1 L( {% o! ~* K! d 6 active+clean+scrubbing+deep* K) q' |, |* C1 ]2 n
1 active+clean+scrubbing0 |+ X" A) k$ p
1 active+clean+scrubbing+deep+inconsistent+repair
4 W5 v3 G1 }) C* {0 ` client io 104 MB/s rd, 49581 kB/s wr, 2830 op/s rd, 3282 op/s wr
. C' e P, J' S. E[root compute1 ~]# ceph s) W( q9 c. |6 ]7 l0 U- N; ~' V, W# K
cluster 70d27aec742e4a95b000-cf37ebba35d0+ V$ R8 s: \/ l! V' J+ u. T/ I Z
health HEALTH_ERR2 @0 q4 C* n+ L6 M' ^" j
1 pgs inconsistent
2 J& A) R6 M- a) P* N T 1 pgs repair: F) |0 ?! w; X0 T6 }5 ~: n& c
1 scrub errors
* {1 k- r! W/ f5 b |% `6 y monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}7 C# @. W7 y! a5 Z" O" _% ~
election epoch 332, quorum 0,1,2 compute1,compute2,compute3& _% }& U* f1 Z# P% k! _8 j7 K- `+ N
osdmap e26725: 131 osds: 128 up, 128 in& k5 C, \7 C; q) O. J: w
flags sortbitwise,require_jewel_osds
# b* M# B& T v9 D+ B pgmap v169370786: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
& [, v) W4 P, o$ g( ] 309 TB used, 155 TB / 465 TB avail
/ q1 W, k: ^' G; Q( T3 `: l 10232 active+clean
" B; y, a9 o0 L+ a5 d7 v! K7 y/ l 6 active+clean+scrubbing+deep+ X1 f/ N5 i9 R B
1 active+clean+scrubbing
" V2 X2 f/ G2 V* n7 j% n 1 active+clean+scrubbing+deep+inconsistent+repair
$ Z l w }# p8 j6 o client io 95762 kB/s rd, 44372 kB/s wr, 2258 op/s rd, 2660 op/s wr
; c w7 I* I" B% ~& I6 ^- E! F$ [0 o' ]% h
& c/ n' U$ k. Y0 q; R6 A[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" - g" Y& [/ ~: k) c/ f7 T' V5 w
"osd_max_backfills": "1",
7 J/ _; k* @3 @ "osd_recovery_max_active": "1",
. j5 l0 H0 r; s- A "osd_recovery_op_priority": "10",
E) p1 x* \% a[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" & }- N: o! t1 W1 j
"osd_max_backfills": "1"," p; Q/ a' \# s. y3 x8 G
"osd_recovery_max_active": "1",
6 b3 r6 L; ?. H6 l4 J# I "osd_recovery_op_priority": "10",
" w% x6 C1 p" G[root compute1 ~]# ceph s, C4 i H4 |( w6 Y+ M6 Q% ^; I
cluster 70d27aec742e4a95b000-cf37ebba35d0
4 ?- C: ]( Z! j/ c `% q8 A health HEALTH_ERR3 Q0 U4 J, S5 B9 Y
1 pgs inconsistent. m8 v3 S0 W: V1 ?$ q( y! ]
1 pgs repair, Z/ E! C% P0 j6 A, n- V Q
1 scrub errors
& j5 @* J# U+ D monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}5 N& U% [! u, l! r7 l# O; W" W x
election epoch 332, quorum 0,1,2 compute1,compute2,compute3% q6 P C' e, t
osdmap e26725: 131 osds: 128 up, 128 in
z0 ^' B/ Z8 G4 m% {$ |5 x flags sortbitwise,require_jewel_osds
) U/ o6 V E, p( e pgmap v169370799: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects# r8 l5 h' N5 |# A+ i+ d
309 TB used, 155 TB / 465 TB avail
4 o, U, \ Q0 t* Q 10232 active+clean% C; S* A# \! m$ @6 A, E$ p+ L
6 active+clean+scrubbing+deep
0 P7 M4 j/ {6 R8 H 1 active+clean+scrubbing
2 W* k; r7 c- Q- l. G" { 1 active+clean+scrubbing+deep+inconsistent+repair
$ I U# M# J0 N client io 134 MB/s rd, 117 MB/s wr, 3865 op/s rd, 4949 op/s wr$ y' [5 F' [% ]% O6 v; k9 J. E
+ t. O* A$ H' A
, q* u" z& @% g1 i[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 5 w6 Q+ M0 P* D( j
"osd_max_backfills": "1",- s3 U2 K5 f7 D% _; Q2 ~) g$ O
"osd_recovery_max_active": "1",
1 D% a: c: X' ~ "osd_recovery_op_priority": "10",$ a+ F' z5 M d: m# C
7 E; Z& I% o }
# a/ s7 y) D4 _, Q0 c; e# ~6 _1 t
. {3 N9 q% s" p5 i8 Y+ [! o[root compute1 ~]# ceph s
/ o' B% Y6 A/ @( c: ^& j; q cluster 70d27aec742e4a95b000-cf37ebba35d0. B( ~( S4 }4 P5 b! z
health HEALTH_OK
: G* y/ d Q, ^( T2 K( E9 p monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
% R+ h3 x: v: ] m election epoch 332, quorum 0,1,2 compute1,compute2,compute3
! i& \' T' u. [ osdmap e26725: 131 osds: 128 up, 128 in
' L& o# X8 j3 F2 ^& Y. y1 r i flags sortbitwise,require_jewel_osds1 j* x3 v* H8 [& l; e1 `, m) I
pgmap v169370916: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
' L8 F9 ~+ X6 z* F1 a 309 TB used, 155 TB / 465 TB avail2 e* @+ |; L% A4 [
10233 active+clean7 C: H4 h; L- K
6 active+clean+scrubbing+deep
K7 F& H5 y' B; J0 V 1 active+clean+scrubbing6 r" u& T# K6 E' ~; M7 E# ~/ y
client io 195 MB/s rd, 234 MB/s wr, 4404 op/s rd, 5993 op/s wr4 w5 R8 X/ l- f6 h6 T
You have mail in /var/spool/mail/root
$ P% u+ R; S2 \/ {% P' R( h+ y
( @" c* e4 w; A# l3 A[root compute1 ~]# ceph s
; q; w3 @' L6 A8 W' S cluster 70d27aec742e4a95b000-cf37ebba35d0+ y% a- }* a9 R( W2 R0 _3 A* \
health HEALTH_OK
4 a" X3 r/ A* Z monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
8 Z8 o9 N7 G3 F: }1 u. _: D% Y4 X( t* H) C election epoch 332, quorum 0,1,2 compute1,compute2,compute3
7 D) M8 `7 U! H osdmap e26725: 131 osds: 128 up, 128 in# J% G) q& Z+ V2 k7 s m
flags sortbitwise,require_jewel_osds
3 D! A; C1 m3 q6 S7 k: @2 A pgmap v169370921: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
: N4 W2 k6 I, c0 R6 | 309 TB used, 155 TB / 465 TB avail
6 W9 S" s8 H& u0 b, j9 t( q 10233 active+clean
0 Y# K, l' Y) D+ L& K- x 6 active+clean+scrubbing+deep
1 Z; P. Q! ?" L8 X) V9 @ 1 active+clean+scrubbing
; w& `/ z7 r' h. w$ o- u, W! Z client io 257 MB/s rd, 358 MB/s wr, 5025 op/s rd, 6587 op/s wr4 @+ L: y- m6 k
$ b( i0 i7 Y9 j8 [. J2 @* `
恢复正常:- z4 Q c1 h4 r
[root compute1 ~]# ceph s' d1 O0 z5 W* o' Y# v8 P
cluster 70d27aec742e4a95b000-cf37ebba35d02 j/ @) k+ r& E9 ]7 m1 s
health HEALTH_OK5 w7 D0 @3 _0 b! p) b$ i9 r) K
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}% ~9 U7 p2 ~6 s) \4 G4 q- w: C
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
+ E. q7 x4 [, F& ^# C& d& [ osdmap e26725: 131 osds: 128 up, 128 in I$ L/ ~, @" o# w3 x/ \9 F; y
flags sortbitwise,require_jewel_osds/ L# s' Z( n* ?9 E
pgmap v169370922: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
" |0 i1 c6 U! X) U4 q8 i 309 TB used, 155 TB / 465 TB avail+ {3 Q' x& `3 ~' t
10233 active+clean
" `5 h& @8 c* i, G+ b" I9 \! ? 6 active+clean+scrubbing+deep
8 `, w: M5 y) t" `! ^- ` 1 active+clean+scrubbing" q9 q: p3 b9 _
client io 254 MB/s rd, 390 MB/s wr, 5522 op/s rd, 6826 op/s wr
' l: B% U7 Y3 {8 ]; L: ^' z0 `0 N/ [, j+ I
4 j3 I$ e4 T9 r$ E |
|