- 积分
- 16843
在线时间 小时
最后登录1970-1-1
|

楼主 |
发表于 2023-11-24 00:06:43
|
显示全部楼层
解决过程,发现无发正常同步,通过检查调整参数,达到修复的目的:
1 \+ H1 @8 g* P! I/ P5 [ j3 l2 d[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" % h( R4 G+ R5 x2 w
"osd_max_backfills": "1",
, q* n! p* t8 i# G" c "osd_recovery_max_active": "1",
" A/ t' V4 w, Y" A( s _ q "osd_recovery_op_priority": "0",
g; j ]8 [4 o+ z1 O1 H"osd_recovery_op_priority": "0", 默认是10
+ X7 S0 w N" D0 c ]' R
0 e0 k8 n& D c! r1 V& m8 S修改为默认值10
* s( B$ Q$ t3 ^1 P. n% z" P
. i0 a8 l/ [. E- ^2 ?! V6 u4 L[root compute1 ~]# ceph tell osd.* injectargs osd_recovery_op_priority=10
0 C9 n _/ E) ^* @osd.0: osd_recovery_op_priority = '10' (unchangeable) . s* C' i4 n9 i! X8 u6 Q
osd.1: osd_recovery_op_priority = '10' (unchangeable)
+ M! V3 a- C& u& r8 Eosd.2: osd_recovery_op_priority = '10' (unchangeable)
! n6 T; o1 V4 Q; [ ~. q, ~3 _3 Rosd.3: osd_recovery_op_priority = '10' (unchangeable) $ L# K( t$ M0 e
osd.4: osd_recovery_op_priority = '10' (unchangeable) & z: I- ^% ~+ x) y
osd.5: osd_recovery_op_priority = '10' (unchangeable)
& P$ _1 t2 J+ cosd.6: osd_recovery_op_priority = '10' (unchangeable)
1 M- ?9 G6 u. j- Y+ Bosd.7: osd_recovery_op_priority = '10' (unchangeable) 1 `# e1 z9 t( N+ }3 t
osd.8: osd_recovery_op_priority = '10' (unchangeable) / p0 A$ I, l% [3 {
osd.9: osd_recovery_op_priority = '10' (unchangeable) 9 ?: r3 [1 {& A8 o
osd.10: osd_recovery_op_priority = '10' (unchangeable) , U, k a, q2 S, c+ b* C2 a a
osd.11: osd_recovery_op_priority = '10' (unchangeable)
! N5 _8 X9 ~7 J' W/ tosd.12: osd_recovery_op_priority = '10' (unchangeable) 5 b' v( Z' p1 S
osd.13: osd_recovery_op_priority = '10' (unchangeable)
& j" f- R- b3 ]" ?# `& c5 N* F% _9 Rosd.14: osd_recovery_op_priority = '10' (unchangeable) . V. |$ [+ C: ?: b7 K
osd.15: osd_recovery_op_priority = '10' (unchangeable) ) i1 U: o& o- j4 `* Q! T5 y7 ~2 d
osd.16: osd_recovery_op_priority = '10' (unchangeable) & e* s! A# e, d5 U
osd.17: osd_recovery_op_priority = '10' (unchangeable)
5 B' R: \+ G6 f9 J0 v. U( k; O/ j( ?7 ?osd.18: osd_recovery_op_priority = '10' (unchangeable)
) Q( p6 J# P4 j* R' w8 C3 G0 Sosd.19: osd_recovery_op_priority = '10' (unchangeable) 0 _/ g; D+ I' }: o
osd.20: osd_recovery_op_priority = '10' (unchangeable) ( P& f$ g# E) D
osd.21: osd_recovery_op_priority = '10' (unchangeable) 9 R' }7 b: G8 }8 [: E
osd.22: osd_recovery_op_priority = '10' (unchangeable)
1 }' p( g. W# W4 Cosd.23: osd_recovery_op_priority = '10' (unchangeable)
! |7 C9 u' G- p# [$ M Yosd.24: osd_recovery_op_priority = '10' (unchangeable) : G( w, {: I2 h6 ]7 t
osd.25: osd_recovery_op_priority = '10' (unchangeable)
9 b0 [+ F4 N" X) P3 ?# u! ?& Bosd.26: osd_recovery_op_priority = '10' (unchangeable) : U" W) h; M. Z
osd.27: osd_recovery_op_priority = '10' (unchangeable) " F0 j4 R$ s# g0 s
osd.28: osd_recovery_op_priority = '10' (unchangeable) 1 \# E( @. Y$ x- k
osd.29: osd_recovery_op_priority = '10' (unchangeable) 2 v0 E: Y' }) E; `2 k
osd.30: osd_recovery_op_priority = '10' (unchangeable)
/ U9 j, A) p3 W) |osd.31: osd_recovery_op_priority = '10' (unchangeable) 1 J5 r& g& v' y- k# a7 k5 `3 ~; F: i
osd.32: osd_recovery_op_priority = '10' (unchangeable)
( {# {( S4 } A5 U% v+ k Uosd.33: osd_recovery_op_priority = '10' (unchangeable)
9 v, T# G+ _) c9 [5 ?) M$ iosd.34: osd_recovery_op_priority = '10' (unchangeable) / i8 b" J _7 C* ?1 t
osd.35: osd_recovery_op_priority = '10' (unchangeable)
- b3 I6 e5 x! o0 h; Z, N2 y1 Sosd.36: osd_recovery_op_priority = '10' (unchangeable) 1 H! H. @" _* X q7 N2 c
osd.37: osd_recovery_op_priority = '10' (unchangeable)
. { ^' q) p# Vosd.38: osd_recovery_op_priority = '10' (unchangeable)
8 }' t. Y+ P* a- a. q( I/ o4 ^( cosd.39: osd_recovery_op_priority = '10' (unchangeable) ' C% x* P1 p: U4 x5 g( G
osd.40: osd_recovery_op_priority = '10' (unchangeable) 0 M" V+ C1 V( n! G3 F5 p% M/ X
osd.41: osd_recovery_op_priority = '10' (unchangeable)
% _8 v. N$ @8 s3 |0 w) fosd.42: osd_recovery_op_priority = '10' (unchangeable) ; Z; h* |, {5 c
osd.43: osd_recovery_op_priority = '10' (unchangeable) & x- x* m* R/ y1 j4 x0 T
osd.44: osd_recovery_op_priority = '10' (unchangeable)
% n9 d+ _6 J9 c6 cosd.45: osd_recovery_op_priority = '10' (unchangeable)
* L; v- U, B+ p: I. ~osd.46: osd_recovery_op_priority = '10' (unchangeable)
& c8 i" ?8 p2 C, R- bosd.47: osd_recovery_op_priority = '10' (unchangeable) 8 @$ Z- z. C% c$ D6 H$ Z( ?) e
osd.48: osd_recovery_op_priority = '10' (unchangeable) ; ~; x6 B" T2 C) K$ {2 n
osd.49: osd_recovery_op_priority = '10' (unchangeable) 8 ^$ \3 D! g. e9 t% @4 n# g) T
osd.50: osd_recovery_op_priority = '10' (unchangeable)
1 p& }3 I7 f# A" k+ ?1 `osd.51: osd_recovery_op_priority = '10' (unchangeable) & |% c* \% A+ Y8 ?5 s
osd.52: osd_recovery_op_priority = '10' (unchangeable)
6 n: e- W' p; R: x% a \6 p. ~osd.53: osd_recovery_op_priority = '10' (unchangeable) ; \% g- H0 T# H7 u8 W
Error ENXIO: problem getting command descriptions from osd.54) E$ V! }# }) V6 P- H
osd.54: problem getting command descriptions from osd.54; l& X3 s4 z8 K6 r
osd.55: osd_recovery_op_priority = '10' (unchangeable)
- \- i# S" g/ }osd.56: osd_recovery_op_priority = '10' (unchangeable) 7 e+ J U" J* u# S o3 H. A: E
osd.57: osd_recovery_op_priority = '10' (unchangeable)
7 S3 j7 _. {6 b( B; C$ I' vosd.58: osd_recovery_op_priority = '10' (unchangeable) 1 U+ `, n1 U% Q8 G4 N9 ~- {6 F
osd.59: osd_recovery_op_priority = '10' (unchangeable) # r+ B4 o5 W4 ?% X
osd.60: osd_recovery_op_priority = '10' (unchangeable)
. w i, f$ S' D( ?& M- Z" A3 f5 {osd.61: osd_recovery_op_priority = '10' (unchangeable)
1 }5 C; P! h, B# D5 G" o; }7 Dosd.62: osd_recovery_op_priority = '10' (unchangeable) 8 c* P% @+ L) v; I, Y
osd.63: osd_recovery_op_priority = '10' (unchangeable) . T3 N* V! D: i/ H# E3 o( f$ a
osd.64: osd_recovery_op_priority = '10' (unchangeable)
9 P1 ]1 a) a, M4 z$ g! j# Z4 O! | Bosd.65: osd_recovery_op_priority = '10' (unchangeable)
; E& Q5 g% G/ D1 Y {3 }osd.66: osd_recovery_op_priority = '10' (unchangeable) ! f& N( s8 T: o
osd.67: osd_recovery_op_priority = '10' (unchangeable) ' P' d; L1 h# R/ L3 C' ]
osd.68: osd_recovery_op_priority = '10' (unchangeable)
! m3 P+ q3 Y9 w/ ]) q+ S9 ^osd.69: osd_recovery_op_priority = '10' (unchangeable) 0 [/ v8 n0 z& n+ [/ \3 @
osd.70: osd_recovery_op_priority = '10' (unchangeable) 4 j+ L& `) n, X
osd.71: osd_recovery_op_priority = '10' (unchangeable)
9 K- s' m6 u5 C) ^osd.72: osd_recovery_op_priority = '10' (unchangeable) $ G0 ^9 b& C' c& N
osd.73: osd_recovery_op_priority = '10' (unchangeable)
! H) ^: p3 B6 e/ l' g" Zosd.74: osd_recovery_op_priority = '10' (unchangeable) # P- w, }' W: o0 i0 \$ {
osd.75: osd_recovery_op_priority = '10' (unchangeable)
1 q; E! A6 d" Q; V5 `8 @osd.76: osd_recovery_op_priority = '10' (unchangeable) * B3 Y' S' S% k6 E& _
osd.77: osd_recovery_op_priority = '10' (unchangeable)
' e: ^$ p' y% }osd.78: osd_recovery_op_priority = '10' (unchangeable) ! T: Y: z5 T" I7 p) }' m
osd.79: osd_recovery_op_priority = '10' (unchangeable)
0 C& {& M1 W+ D$ X# ^1 e2 Tosd.80: osd_recovery_op_priority = '10' (unchangeable)
/ L# {# _- S- p* n3 f( \osd.81: osd_recovery_op_priority = '10' (unchangeable) ; P) E- S0 m* r$ h
osd.82: osd_recovery_op_priority = '10' (unchangeable)
. x( N2 q3 A- `$ I. Rosd.83: osd_recovery_op_priority = '10' (unchangeable)
4 h) P4 | H; x( P/ i bosd.84: osd_recovery_op_priority = '10' (unchangeable) 2 A3 j8 m7 {5 s0 z, r( T. e. j
osd.85: osd_recovery_op_priority = '10' (unchangeable)
( g" y( C3 _8 m! G0 x+ N) aosd.86: osd_recovery_op_priority = '10' (unchangeable)
) j& V. e% A- l/ C+ ~- Dosd.87: osd_recovery_op_priority = '10' (unchangeable)
$ W3 d1 ]$ D! y8 Y+ E9 sosd.88: osd_recovery_op_priority = '10' (unchangeable)
: ? H) M7 L: @! J1 T" ^# hosd.89: osd_recovery_op_priority = '10' (unchangeable)
8 ]8 R1 a1 A& O- Uosd.90: osd_recovery_op_priority = '10' (unchangeable)
6 v. |- P' ~) gosd.91: osd_recovery_op_priority = '10' (unchangeable)
( A, E- ]0 o" _3 ~) U( gosd.92: osd_recovery_op_priority = '10' (unchangeable) ( [! u) k2 O( ]2 N" F
osd.93: osd_recovery_op_priority = '10' (unchangeable)
8 a8 r/ Z" F" y; `osd.94: osd_recovery_op_priority = '10' (unchangeable)
: d8 Q7 m4 P; u& Yosd.95: osd_recovery_op_priority = '10' (unchangeable) $ L, i7 t0 i. e# A5 J5 m7 q
osd.96: osd_recovery_op_priority = '10' (unchangeable) 7 J5 m. {0 E( y x8 K M% A
osd.97: osd_recovery_op_priority = '10' (unchangeable) ! c" _$ Y% j6 v. N) t: b
osd.98: osd_recovery_op_priority = '10' (unchangeable) , }1 T: [/ j; f: t
osd.99: osd_recovery_op_priority = '10' (unchangeable)
* Z- y* W0 K4 ]osd.100: osd_recovery_op_priority = '10' (unchangeable) . X7 O4 x0 g: `
osd.101: osd_recovery_op_priority = '10' (unchangeable) 9 ~" S) _0 j3 n' O0 L
osd.102: osd_recovery_op_priority = '10' (unchangeable) - N% w, D. O) G5 F( T
osd.103: osd_recovery_op_priority = '10' (unchangeable)
; H- r: [" X6 n# [4 s- mosd.104: osd_recovery_op_priority = '10' (unchangeable)
( p8 ^5 i$ C5 @5 p& G7 V2 Dosd.105: osd_recovery_op_priority = '10' (unchangeable)
; Q! w6 c: p8 V' Q6 }0 `) W7 Y# t/ J) tosd.106: osd_recovery_op_priority = '10' (unchangeable)
. j# A, l1 f( s! `: @osd.107: osd_recovery_op_priority = '10' (unchangeable)
# S+ R, p! K# u# {) ~7 ^( tosd.108: osd_recovery_op_priority = '10' (unchangeable)
9 u( h; ?0 P5 ~osd.109: osd_recovery_op_priority = '10' (unchangeable) ' y t8 W5 u6 ~2 ^
osd.110: osd_recovery_op_priority = '10' (unchangeable)
3 I0 }. ?! M+ v3 Kosd.111: osd_recovery_op_priority = '10' (unchangeable)
2 v4 G. s( Z4 Z8 Gosd.112: osd_recovery_op_priority = '10' (unchangeable) + ^: k# r ?8 L! `. M, @2 a: ?* f
osd.113: osd_recovery_op_priority = '10' (unchangeable)
! o% {% d# W- c' W+ J- [osd.114: osd_recovery_op_priority = '10' (unchangeable) 7 I2 m; l/ \8 [. r
osd.115: osd_recovery_op_priority = '10' (unchangeable)
" s X1 t) K0 c* g0 x k/ J+ wosd.116: osd_recovery_op_priority = '10' (unchangeable) ; F B9 C8 }, @' h5 z) G: l. W
osd.117: osd_recovery_op_priority = '10' (unchangeable)
5 i% a# p7 z3 f" p' o4 Zosd.118: osd_recovery_op_priority = '10' (unchangeable)
& H) M6 T' n3 o' Xosd.119: osd_recovery_op_priority = '10' (unchangeable) : e2 T W8 R1 h+ w6 K) _
osd.120: osd_recovery_op_priority = '10' (unchangeable) 3 q- C7 q, @5 f* T
osd.121: osd_recovery_op_priority = '10' (unchangeable) 6 d8 A* j$ F1 e4 [/ o3 a# ]1 J7 u
osd.122: osd_recovery_op_priority = '10' (unchangeable) 5 ~. [. y0 ?3 D3 O! g) f
osd.123: osd_recovery_op_priority = '10' (unchangeable)
1 e" n0 w5 }" o8 }' qError ENXIO: problem getting command descriptions from osd.124
3 I: D# \0 u5 z gosd.124: problem getting command descriptions from osd.1241 q+ O8 t9 O9 P( Q3 x3 a$ q
osd.125: osd_recovery_op_priority = '10' (unchangeable) " p6 l" ^; d. k0 C* y5 C
osd.126: osd_recovery_op_priority = '10' (unchangeable) ' i# r; u/ I4 `6 ?, ^- l& L" s
Error ENXIO: problem getting command descriptions from osd.127
: D0 p2 V* S0 u. k; Z5 d) f6 d; {osd.127: problem getting command descriptions from osd.127
0 |/ u! v/ P3 T5 b% B4 @' A) @osd.128: osd_recovery_op_priority = '10' (unchangeable) - W* s" h* L* \/ Y: z/ l" Z# R
osd.129: osd_recovery_op_priority = '10' (unchangeable) ) D6 t$ G6 j6 a$ \7 {# T
osd.130: osd_recovery_op_priority = '10' (unchangeable)
0 j" C; I5 e. [You have new mail in /var/spool/mail/root0 {6 {3 M" m& p7 w* C
" L! M! {+ p2 C' M" l3 ^所有的osd都修改:
2 y3 B) w+ |. v# Q
' f0 T; O1 q5 v+ R- M( u# y执行过ceph pg repair pg-id
8 ]- W' g' ?. {5 j
- h4 Z. `+ N m. A; J
s- O& t. }4 ~$ k4 f+ t状态出现修改过程:
8 z# @! g0 b- W) h2 B* B. o \[root compute1 ~]# ceph s
! K* l: Y, f% j9 Z9 J cluster 70d27aec742e4a95b000-cf37ebba35d0
* h. f2 j( U- y w health HEALTH_ERR
. n" b+ y! r! F( d7 F8 n/ I 1 pgs inconsistent
9 \" ?/ V: ?: \ 1 pgs repair, x, _" h$ U* p& u
1 scrub errors
2 }/ S5 k; U* t monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
9 e0 V7 E m( |! \$ c* w' G election epoch 332, quorum 0,1,2 compute1,compute2,compute3
, K, E V5 j- i8 @: H" v osdmap e26725: 131 osds: 128 up, 128 in
5 X8 \- Q, [- b flags sortbitwise,require_jewel_osds
1 J8 U C% X, s% u( I* |0 j# g5 q pgmap v169370679: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
0 q1 }% D. W, x; s6 v' S `; J 309 TB used, 155 TB / 465 TB avail
A+ S9 w9 @7 V2 a# j, _3 ~6 a+ ]$ C 10229 active+clean, \! a" V+ j2 j) k3 f* S% F+ P5 k
9 active+clean+scrubbing+deep
8 J( h7 }" Z3 n8 A 1 active+clean+scrubbing Z( C: _7 x8 y
1 active+clean+scrubbing+deep+inconsistent+repair& b3 i5 P. K' |' Y; i; b
client io 91086 kB/s rd, 67615 kB/s wr, 3119 op/s rd, 3569 op/s wr
( `! F% P8 M4 r! a" e& r5 V[root compute1 ~]# ceph s
6 A& R: K8 F) V j0 t; | cluster 70d27aec742e4a95b000-cf37ebba35d0* a% T& R; o3 q9 h6 E
health HEALTH_ERR$ ?- M+ V) I- a$ A7 s, H9 R v! k" {
1 pgs inconsistent
6 o5 {4 {( A* T3 o2 N. S 1 pgs repair X$ o- L8 x, c& Y
1 scrub errors' K: M: h- c- N8 s3 g( e
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}; H8 t! e- B" V2 R) j
election epoch 332, quorum 0,1,2 compute1,compute2,compute3! D4 J7 \ Z* {3 o# X
osdmap e26725: 131 osds: 128 up, 128 in
. p' D* v& k' v) }+ N8 A5 }8 F flags sortbitwise,require_jewel_osds
1 R! i; r$ }/ i7 C" y$ |5 b4 N pgmap v169370681: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects8 w; ^; V V2 `: L
309 TB used, 155 TB / 465 TB avail" a$ s) a) X0 z# a. O6 t# V/ o
10230 active+clean
1 Z, A3 ~$ b; S( M z+ t; N* {' @ 8 active+clean+scrubbing+deep- v: N" u/ @$ |4 S g) ]! o
1 active+clean+scrubbing3 Z' n& y# l0 @* P& t$ ~7 Q# j
1 active+clean+scrubbing+deep+inconsistent+repair3 L C, U& C4 E& w) ~* y0 U
client io 104 MB/s rd, 67033 kB/s wr, 3479 op/s rd, 5013 op/s wr
3 H) l! F& ]; N[root compute1 ~]# ceph s
3 w0 d2 P7 n1 t7 Q$ s" K cluster 70d27aec742e4a95b000-cf37ebba35d0
4 C! w9 J9 q0 r; {# \ health HEALTH_ERR
" S$ {6 {& a8 r- T+ o! Y 1 pgs inconsistent+ C7 j. w8 f: [& ?
1 pgs repair
, O/ y& r m' } 1 scrub errors7 `9 o/ }4 n) x# g$ C W( \% w" P3 t) `
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
$ I9 U, m, W, G" \ election epoch 332, quorum 0,1,2 compute1,compute2,compute3( {1 _2 Z ?2 s, D+ F
osdmap e26725: 131 osds: 128 up, 128 in( R2 W8 X3 o$ i6 V
flags sortbitwise,require_jewel_osds+ Q' O5 W% E1 f! r
pgmap v169370783: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
, H9 \; \* ~% ~* p! `3 F 309 TB used, 155 TB / 465 TB avail4 b0 C5 ^/ s$ `2 ]: I* X4 k
10232 active+clean. |5 A; g6 j# M* C
6 active+clean+scrubbing+deep' w9 j2 o( q9 X, z& ^
1 active+clean+scrubbing3 I5 N; I% g, i, w
1 active+clean+scrubbing+deep+inconsistent+repair
# r( s) w; a, j, ~, D% ~ client io 147 MB/s rd, 57692 kB/s wr, 4584 op/s rd, 5944 op/s wr+ \2 k5 d5 z. K" n& W
You have mail in /var/spool/mail/root& I2 D3 C) j q7 b7 y- g
[root compute1 ~]# ceph s
+ B9 @& c% i& S, X8 \ cluster 70d27aec742e4a95b000-cf37ebba35d0+ c2 W* |: C, i4 S# K1 m3 j
health HEALTH_ERR8 w% u' R1 {2 y6 |
1 pgs inconsistent
2 O: H- @. B7 h, ~6 {6 R2 C' g 1 pgs repair$ D6 M! f& ^! m
1 scrub errors$ E7 d9 Q+ [" ^; @' Y7 V
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}- A) V1 v/ m+ z- U2 a
election epoch 332, quorum 0,1,2 compute1,compute2,compute3' l. \- ^! W- ?7 W1 z% v+ l+ W1 X, x
osdmap e26725: 131 osds: 128 up, 128 in3 u8 _, D- u- v
flags sortbitwise,require_jewel_osds
; Y- ]) m* n/ w( w pgmap v169370785: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects, z3 j0 Q% s2 Z, ] w, X
309 TB used, 155 TB / 465 TB avail# g/ O: p$ L' k+ ?8 M
10232 active+clean
% |% h2 E' R( W' \; m- ^& I 6 active+clean+scrubbing+deep
$ f5 Q2 E5 p# j$ R9 b- c 1 active+clean+scrubbing( z. |% p: O0 k3 U* _
1 active+clean+scrubbing+deep+inconsistent+repair0 n* j+ M& b; {
client io 104 MB/s rd, 49581 kB/s wr, 2830 op/s rd, 3282 op/s wr$ ]7 V3 c. K+ `. G0 q# p
[root compute1 ~]# ceph s
8 W3 b" a( U) |" y9 | cluster 70d27aec742e4a95b000-cf37ebba35d0
6 a5 l5 m; J5 t health HEALTH_ERR2 V. X# x4 `! ?2 P
1 pgs inconsistent
1 U, j0 d0 I3 C% K 1 pgs repair5 { G. W4 P3 ^5 D4 q) \" y" I
1 scrub errors
4 T& o, \6 ^3 `5 l! f% L7 I monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}5 }# j/ V6 e- Q6 x+ N0 T8 t5 Z
election epoch 332, quorum 0,1,2 compute1,compute2,compute3! _$ X z9 ~" Z+ B! Q
osdmap e26725: 131 osds: 128 up, 128 in
4 w5 K6 E4 F7 ] flags sortbitwise,require_jewel_osds, v' ?9 |) }% Y
pgmap v169370786: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects6 m4 s5 M( b! `4 \! d, p
309 TB used, 155 TB / 465 TB avail* w4 u* v N! ]; V; H
10232 active+clean4 z( m, H2 n3 C: ~# w2 j( T
6 active+clean+scrubbing+deep
' w5 H/ |* J( P, s/ E( m 1 active+clean+scrubbing
( A; T2 H n2 ~! C6 T5 C6 b 1 active+clean+scrubbing+deep+inconsistent+repair U0 c+ E4 P9 F3 @# F0 s
client io 95762 kB/s rd, 44372 kB/s wr, 2258 op/s rd, 2660 op/s wr: ]9 _1 M5 a h4 q+ J* X
( f4 |; B3 P6 o6 l
8 {4 ?3 R( x' y- u[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
' @/ u4 z- N4 T7 Z% }; N3 v; Z "osd_max_backfills": "1",
- x8 x+ E" R. I$ k7 D! W' o( X "osd_recovery_max_active": "1",( B$ ]1 t& S$ b/ \7 @
"osd_recovery_op_priority": "10",
1 t* Z# `0 F6 {7 S" G[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
2 w7 A8 b! M" P+ W4 | "osd_max_backfills": "1",
" f9 V$ ~& ?2 G" ]5 ^ "osd_recovery_max_active": "1",
: }0 n, ~* R, K- B1 j "osd_recovery_op_priority": "10",: `* B) _% _ C2 E- g. W' T$ V) u% y
[root compute1 ~]# ceph s
% X( `2 ^6 `9 c; ~' Y% ~3 Y cluster 70d27aec742e4a95b000-cf37ebba35d0
; T3 `4 ]) D. a4 S: ^ health HEALTH_ERR
! y8 I7 p) Z* k0 P 1 pgs inconsistent
* M; ]& i/ k4 {$ o4 ? 1 pgs repair, o: h, _ z8 q( h5 ]
1 scrub errors0 c/ P5 W& Q2 a) u
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
% \- v3 Y& N2 V6 s election epoch 332, quorum 0,1,2 compute1,compute2,compute38 m6 r2 j0 |6 Y3 k' O
osdmap e26725: 131 osds: 128 up, 128 in# L* O) t# ` D3 T; u; c
flags sortbitwise,require_jewel_osds1 g! |7 e+ O5 k3 p
pgmap v169370799: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects9 N, e7 A4 o9 I8 C/ L
309 TB used, 155 TB / 465 TB avail
' N9 \- s# y) }; G! e, T 10232 active+clean
% `$ S# ?; z: f; T* X+ o9 {8 z 6 active+clean+scrubbing+deep
% A; k: |3 ^1 |2 r9 g! J0 e 1 active+clean+scrubbing# b+ o/ N5 ~& N* O1 u- [* g L8 x9 P
1 active+clean+scrubbing+deep+inconsistent+repair+ ^* C$ n* O; Y5 h' c% q
client io 134 MB/s rd, 117 MB/s wr, 3865 op/s rd, 4949 op/s wr
+ v; I5 g# w" ]+ j/ {% n8 @* `, K9 V8 l, ~
- D* \0 `0 n2 L$ [( g
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
; N; O/ w: F+ [7 r0 g! z5 g& {8 L "osd_max_backfills": "1",* q9 i3 O+ c& Z" k. U, i9 ~
"osd_recovery_max_active": "1",
3 ?2 j- m S8 q7 C& h: h& A "osd_recovery_op_priority": "10"," ~5 `2 o1 _% e/ ^2 u3 F B# S
1 U& P- a' @5 T* {1 A. U, P
* Y2 t9 h9 H* [! O
0 R: Y$ \4 f; ^! |[root compute1 ~]# ceph s
' l2 g% ]( w5 D. ~0 y- G. a cluster 70d27aec742e4a95b000-cf37ebba35d0
+ f M3 k+ D7 r6 { health HEALTH_OK1 t/ d% z: x: d8 s X
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}8 I. J/ a. g7 R0 L
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
0 ?2 q# |# Q/ N* u osdmap e26725: 131 osds: 128 up, 128 in
# E1 ~8 i5 i+ N0 f flags sortbitwise,require_jewel_osds
% z9 Y3 F' Z$ o1 g% Y5 Q ~# y' T; v pgmap v169370916: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects/ y! K. r/ Y \
309 TB used, 155 TB / 465 TB avail
+ P/ v) G# Z V, A- u A9 l* R 10233 active+clean
) o" l. l! d' |8 D/ O8 {& A 6 active+clean+scrubbing+deep
; L x2 L7 g& g 1 active+clean+scrubbing
) n9 G* h2 ~$ q client io 195 MB/s rd, 234 MB/s wr, 4404 op/s rd, 5993 op/s wr9 N& f4 F& R0 K; K: ^" h
You have mail in /var/spool/mail/root. L; r+ t% c: q; j$ F* ~
% ]3 v7 ?9 i5 r[root compute1 ~]# ceph s
& n( z' m) Y0 v$ f6 \ cluster 70d27aec742e4a95b000-cf37ebba35d07 F+ f( {1 H; w; L/ @3 K
health HEALTH_OK
/ T3 Y" c, ^& T$ M monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}5 V% R7 G9 \- V2 T
election epoch 332, quorum 0,1,2 compute1,compute2,compute31 V0 _5 w9 }/ J! e( X4 L
osdmap e26725: 131 osds: 128 up, 128 in
. V1 j3 e' P. V flags sortbitwise,require_jewel_osds
* _% t0 s6 O9 f0 s0 g- U pgmap v169370921: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
# H: Y0 q( p: e+ N. { 309 TB used, 155 TB / 465 TB avail
, s: _; ]1 C* \3 c2 n$ ?8 I- V8 } 10233 active+clean2 l7 F I; o( v7 E
6 active+clean+scrubbing+deep
+ \9 u i; p; q& i: R 1 active+clean+scrubbing; D3 E% r' A; m9 ^6 P8 e
client io 257 MB/s rd, 358 MB/s wr, 5025 op/s rd, 6587 op/s wr
. b' q. m! P; T+ N; t4 x
, J; O; W! [* e5 I3 t9 _恢复正常:
* X' H0 b: k' U$ C% @5 ][root compute1 ~]# ceph s8 k3 M7 B U4 k v
cluster 70d27aec742e4a95b000-cf37ebba35d0
! G& N$ N8 g8 w& a8 K% f health HEALTH_OK
_8 q; n" ~& W8 r4 ~8 _' } monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
* x, [+ R% c7 E' H% S' W election epoch 332, quorum 0,1,2 compute1,compute2,compute3
0 a) J- A W" g osdmap e26725: 131 osds: 128 up, 128 in
. n, ?$ o) l9 }4 @2 V flags sortbitwise,require_jewel_osds, L2 r n: A) i
pgmap v169370922: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
" z5 Z' b4 w1 _7 O/ ^, H1 W 309 TB used, 155 TB / 465 TB avail
+ o2 y) z; U- |7 T( D3 b 10233 active+clean( B0 n) s1 b, a, S1 q* q
6 active+clean+scrubbing+deep' o O n7 d C; I
1 active+clean+scrubbing
5 f; H3 e- ]6 L) ?2 q2 l/ ~ client io 254 MB/s rd, 390 MB/s wr, 5522 op/s rd, 6826 op/s wr. v6 q2 `( ?1 ^0 g: I
% a6 R4 F* r: ?8 Z5 Z' ^5 T
# L, H) h) V5 m; X |
|