- 积分
- 16843
在线时间 小时
最后登录1970-1-1
|

楼主 |
发表于 2023-11-24 00:06:43
|
显示全部楼层
解决过程,发现无发正常同步,通过检查调整参数,达到修复的目的:
. h; U+ k n1 n# J% w, k[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
2 B% T+ j$ `% D4 z1 V "osd_max_backfills": "1",
9 }8 r4 O( ]$ O8 Q' z4 ~ "osd_recovery_max_active": "1",; `, V9 \& ~$ x9 p9 p
"osd_recovery_op_priority": "0",- M9 R; ]" x* ], L
"osd_recovery_op_priority": "0", 默认是10
9 l2 l- G ^: C1 I8 W5 ]
6 ]% r' e8 Q+ c! t修改为默认值10
4 `" {/ Y, ~$ N4 r9 r: R
9 }8 {$ G2 ~% j2 F9 h1 @7 Z[root compute1 ~]# ceph tell osd.* injectargs osd_recovery_op_priority=10
0 d' p/ `* x6 E5 S3 q* F7 r! qosd.0: osd_recovery_op_priority = '10' (unchangeable) V; ^; m5 e6 u0 N! _+ N
osd.1: osd_recovery_op_priority = '10' (unchangeable)
/ e2 x7 x# i* G3 S) d' nosd.2: osd_recovery_op_priority = '10' (unchangeable) 9 j- r3 h' _3 h. e8 w) K
osd.3: osd_recovery_op_priority = '10' (unchangeable)
' J' u9 P4 ~ f; {osd.4: osd_recovery_op_priority = '10' (unchangeable)
" f a) a3 q& ^) |osd.5: osd_recovery_op_priority = '10' (unchangeable) 2 F2 [, C. A. u
osd.6: osd_recovery_op_priority = '10' (unchangeable)
s! s8 B, J, f" F! y: \3 o' p) O/ Yosd.7: osd_recovery_op_priority = '10' (unchangeable) " o! l5 Z) s1 b. h/ U% C; `
osd.8: osd_recovery_op_priority = '10' (unchangeable)
7 V2 o! ^. ]( f* v6 I: O/ Kosd.9: osd_recovery_op_priority = '10' (unchangeable)
, N0 c! y7 a7 c( g1 Qosd.10: osd_recovery_op_priority = '10' (unchangeable) 0 A: n; y: n, O% z. z) g
osd.11: osd_recovery_op_priority = '10' (unchangeable) & o N9 B! G- e& F, [8 l2 c
osd.12: osd_recovery_op_priority = '10' (unchangeable) ! ~; o8 G `: B3 E9 P, |+ e, C
osd.13: osd_recovery_op_priority = '10' (unchangeable)
6 U9 `8 X3 h! a! Zosd.14: osd_recovery_op_priority = '10' (unchangeable)
9 c0 z! {+ N; w+ Aosd.15: osd_recovery_op_priority = '10' (unchangeable)
4 o* j y# r$ wosd.16: osd_recovery_op_priority = '10' (unchangeable)
8 j; T" }6 @. [& H, rosd.17: osd_recovery_op_priority = '10' (unchangeable) ' h0 p/ T3 S+ g
osd.18: osd_recovery_op_priority = '10' (unchangeable) % Y) ?6 O9 I4 ]$ Y% J8 u& L
osd.19: osd_recovery_op_priority = '10' (unchangeable)
& P& R% _6 f* R0 ?9 K& ~/ G8 N! n& p" g# ?osd.20: osd_recovery_op_priority = '10' (unchangeable) . [# {; y! x3 r# ^# m# D0 j
osd.21: osd_recovery_op_priority = '10' (unchangeable) - u3 R% c) G4 H y5 f! F+ _
osd.22: osd_recovery_op_priority = '10' (unchangeable) 7 q6 w, b/ v3 C2 g/ u, n v: y
osd.23: osd_recovery_op_priority = '10' (unchangeable)
% Z$ l! i6 X8 z0 wosd.24: osd_recovery_op_priority = '10' (unchangeable) / |6 g) _8 g- [8 R( u
osd.25: osd_recovery_op_priority = '10' (unchangeable) ' t |8 l) ~. `0 e3 C- i
osd.26: osd_recovery_op_priority = '10' (unchangeable)
6 _6 h0 N4 `0 qosd.27: osd_recovery_op_priority = '10' (unchangeable)
& n& p; N% i4 j8 k- F3 P! V& b' _osd.28: osd_recovery_op_priority = '10' (unchangeable)
; A) L6 p* T6 | I7 x/ G- b) J6 yosd.29: osd_recovery_op_priority = '10' (unchangeable)
2 t g7 T* r/ m1 x7 E( w$ yosd.30: osd_recovery_op_priority = '10' (unchangeable)
- Y+ F7 G+ D5 G8 ~/ wosd.31: osd_recovery_op_priority = '10' (unchangeable)
* K( H3 t; ]8 k# [) S; s- Mosd.32: osd_recovery_op_priority = '10' (unchangeable)
) R) a& }+ v, e% I" x+ ^7 Josd.33: osd_recovery_op_priority = '10' (unchangeable) " F8 p, n! ^/ K8 A3 x
osd.34: osd_recovery_op_priority = '10' (unchangeable)
3 Y5 d& m% |- m" L2 A6 S0 mosd.35: osd_recovery_op_priority = '10' (unchangeable) # ~% S7 O9 Q" @/ H4 z% b5 }9 w
osd.36: osd_recovery_op_priority = '10' (unchangeable) 7 G$ R6 v: c0 @; ]" R) N- E3 O' q1 P
osd.37: osd_recovery_op_priority = '10' (unchangeable)
( n0 A6 y5 b# c; uosd.38: osd_recovery_op_priority = '10' (unchangeable)
3 h) x; N0 m) a8 L: hosd.39: osd_recovery_op_priority = '10' (unchangeable)
! L) n" S0 r, _4 b5 G7 D7 h; {2 qosd.40: osd_recovery_op_priority = '10' (unchangeable) 8 `% J9 x0 i8 g3 O! u
osd.41: osd_recovery_op_priority = '10' (unchangeable)
9 }) z' Y0 }" r' t& Cosd.42: osd_recovery_op_priority = '10' (unchangeable)
2 f: f& \! B5 W! {osd.43: osd_recovery_op_priority = '10' (unchangeable) 7 R# q8 `" L) h+ B4 L( }+ {9 B
osd.44: osd_recovery_op_priority = '10' (unchangeable)
* f$ Z6 S( ~4 {1 p+ p& V0 qosd.45: osd_recovery_op_priority = '10' (unchangeable) " ?3 s L0 F( J# w
osd.46: osd_recovery_op_priority = '10' (unchangeable) 2 c1 T$ t% D e& Q+ d% p( c2 Z$ N
osd.47: osd_recovery_op_priority = '10' (unchangeable) , d+ i% Z7 a$ R& Z0 {
osd.48: osd_recovery_op_priority = '10' (unchangeable)
! N n6 o4 C! |- Tosd.49: osd_recovery_op_priority = '10' (unchangeable) 1 H1 q6 \7 M2 h7 w$ l# _* [
osd.50: osd_recovery_op_priority = '10' (unchangeable)
$ L6 [$ E6 o6 [+ fosd.51: osd_recovery_op_priority = '10' (unchangeable)
5 _/ a, Y# x7 f; ~4 j6 Vosd.52: osd_recovery_op_priority = '10' (unchangeable) # b* |8 x) }5 r2 p( L+ X) v+ {) W
osd.53: osd_recovery_op_priority = '10' (unchangeable) 5 Q8 w2 c; y* Z8 E T& p, f. h
Error ENXIO: problem getting command descriptions from osd.54
s/ W r: o' c Cosd.54: problem getting command descriptions from osd.54
9 E: p& ~1 ?" K7 iosd.55: osd_recovery_op_priority = '10' (unchangeable) 8 l8 j8 ~/ Y5 m* `. h" I( ^
osd.56: osd_recovery_op_priority = '10' (unchangeable)
$ p! R0 A" Y& _# I% [osd.57: osd_recovery_op_priority = '10' (unchangeable) & Z8 }$ c6 n& _1 X, R C
osd.58: osd_recovery_op_priority = '10' (unchangeable) 4 g ]# M& k) y+ _
osd.59: osd_recovery_op_priority = '10' (unchangeable) 8 i* K1 K. [0 p
osd.60: osd_recovery_op_priority = '10' (unchangeable)
+ z8 J+ X0 _, [# ?# Tosd.61: osd_recovery_op_priority = '10' (unchangeable) ( P! V0 x. ~% m1 \+ h6 p0 a' S: M
osd.62: osd_recovery_op_priority = '10' (unchangeable)
# e- g- {- T( j" a! m3 h5 josd.63: osd_recovery_op_priority = '10' (unchangeable)
* b1 J! @* `/ W: Y3 G& M6 hosd.64: osd_recovery_op_priority = '10' (unchangeable)
! G6 I$ L& U- b* u2 D0 T6 Vosd.65: osd_recovery_op_priority = '10' (unchangeable)
" } u6 x1 X! `% ]6 L; D, R1 Losd.66: osd_recovery_op_priority = '10' (unchangeable) 5 @( j6 p% d7 Z' Q! k6 P
osd.67: osd_recovery_op_priority = '10' (unchangeable) : [5 _% d) x/ g+ n$ C* z' B% G
osd.68: osd_recovery_op_priority = '10' (unchangeable)
) P3 k* C( k# Z* F) B: y1 Xosd.69: osd_recovery_op_priority = '10' (unchangeable) 7 E* q: @: J* X1 R& C* F
osd.70: osd_recovery_op_priority = '10' (unchangeable)
- r' i K1 H( d d1 e+ H4 Xosd.71: osd_recovery_op_priority = '10' (unchangeable) / M" l9 S! u; ^9 @; B
osd.72: osd_recovery_op_priority = '10' (unchangeable)
+ |7 o0 r! A' B' V( aosd.73: osd_recovery_op_priority = '10' (unchangeable) 5 u5 B5 m2 @) \+ W0 H
osd.74: osd_recovery_op_priority = '10' (unchangeable)
: H* o& s( y P; |osd.75: osd_recovery_op_priority = '10' (unchangeable) 6 P* z( e3 C+ s$ W
osd.76: osd_recovery_op_priority = '10' (unchangeable) 0 b+ x* W& @: K" t; ^& t
osd.77: osd_recovery_op_priority = '10' (unchangeable)
! f' a5 Q. k* y- r8 ^osd.78: osd_recovery_op_priority = '10' (unchangeable) ) h9 g! C' J( u, |6 q" h
osd.79: osd_recovery_op_priority = '10' (unchangeable) ; N* q. ~7 T5 E
osd.80: osd_recovery_op_priority = '10' (unchangeable) * E; b) U& s1 E6 v9 o
osd.81: osd_recovery_op_priority = '10' (unchangeable) * b, V* `4 w5 Y! T6 v* X. q- Y
osd.82: osd_recovery_op_priority = '10' (unchangeable) 4 q9 Q- M% @9 t" F5 J3 R$ x. |% }& ]
osd.83: osd_recovery_op_priority = '10' (unchangeable)
6 p0 u9 Q) C, bosd.84: osd_recovery_op_priority = '10' (unchangeable)
# \, X$ f/ v2 f2 ~osd.85: osd_recovery_op_priority = '10' (unchangeable) : E9 F8 a- ?' p% m5 h4 z. [
osd.86: osd_recovery_op_priority = '10' (unchangeable) + `: {, ~9 m1 f
osd.87: osd_recovery_op_priority = '10' (unchangeable)
0 b4 s4 o+ L8 D$ V2 tosd.88: osd_recovery_op_priority = '10' (unchangeable) # E9 Q( p$ q3 c% ~2 Q9 M& O
osd.89: osd_recovery_op_priority = '10' (unchangeable)
K. X9 n1 `2 [ [- v- Yosd.90: osd_recovery_op_priority = '10' (unchangeable)
0 F! P/ T. S/ ^; {+ a3 f5 josd.91: osd_recovery_op_priority = '10' (unchangeable) 9 s) ^5 y# |. a4 Y) Q$ I
osd.92: osd_recovery_op_priority = '10' (unchangeable) ( N% p7 O' t2 N7 {$ J
osd.93: osd_recovery_op_priority = '10' (unchangeable) 6 r3 b% N2 f+ q- L$ h3 c! E
osd.94: osd_recovery_op_priority = '10' (unchangeable) 0 k# g" C# y* p, Q
osd.95: osd_recovery_op_priority = '10' (unchangeable)
3 q- l" O1 y0 ?! B* f; h* M) Qosd.96: osd_recovery_op_priority = '10' (unchangeable) 3 L. a* z- n( y n1 J
osd.97: osd_recovery_op_priority = '10' (unchangeable)
% k7 {, o2 i& Q* Xosd.98: osd_recovery_op_priority = '10' (unchangeable) ) I+ \7 }1 y: L2 r4 p
osd.99: osd_recovery_op_priority = '10' (unchangeable) 5 {# {0 f. U w: T' S, X
osd.100: osd_recovery_op_priority = '10' (unchangeable)
2 ]3 r( u+ n5 X5 P. }osd.101: osd_recovery_op_priority = '10' (unchangeable) ( D. w; z( ^9 S$ T
osd.102: osd_recovery_op_priority = '10' (unchangeable) 6 k% x2 \$ s0 K+ D
osd.103: osd_recovery_op_priority = '10' (unchangeable)
: E6 ~8 c, L! _# V; l3 e( B9 fosd.104: osd_recovery_op_priority = '10' (unchangeable)
5 E+ q% U7 L! F5 p, m* l- s$ w* Wosd.105: osd_recovery_op_priority = '10' (unchangeable) a/ X: Z( |* j9 P4 F0 z) L4 l
osd.106: osd_recovery_op_priority = '10' (unchangeable)
M2 g, {" K# Hosd.107: osd_recovery_op_priority = '10' (unchangeable) 4 S9 s& r J7 Z% A; {+ v6 U
osd.108: osd_recovery_op_priority = '10' (unchangeable) 0 ?) g1 A5 T7 m
osd.109: osd_recovery_op_priority = '10' (unchangeable) - `6 v6 Z5 ]# c
osd.110: osd_recovery_op_priority = '10' (unchangeable)
5 ~- ~' K: v' ?* n+ |osd.111: osd_recovery_op_priority = '10' (unchangeable) 0 p" t: }8 G [, D& ?: H
osd.112: osd_recovery_op_priority = '10' (unchangeable) % l* J% U5 `8 B& \& y4 ]
osd.113: osd_recovery_op_priority = '10' (unchangeable)
$ ~2 f/ ?1 I! B5 H" A5 T$ B- W3 I9 Y, w5 Posd.114: osd_recovery_op_priority = '10' (unchangeable)
# U) \, x8 v9 n( m. P. zosd.115: osd_recovery_op_priority = '10' (unchangeable)
" ]) v, q+ j" {, b. ~1 T6 f. losd.116: osd_recovery_op_priority = '10' (unchangeable) " F9 m1 |4 {# i, e3 V
osd.117: osd_recovery_op_priority = '10' (unchangeable) 5 I: Q/ L6 n( I' N( w- b
osd.118: osd_recovery_op_priority = '10' (unchangeable)
! \: d$ D, U, p* D2 a4 uosd.119: osd_recovery_op_priority = '10' (unchangeable) 5 f1 X, F, H) a) q) E
osd.120: osd_recovery_op_priority = '10' (unchangeable) - C- l# [5 c8 w8 t' f' L/ E J
osd.121: osd_recovery_op_priority = '10' (unchangeable)
. ]3 h) T9 O2 X) o* Eosd.122: osd_recovery_op_priority = '10' (unchangeable)
( T2 U7 P5 `" r1 bosd.123: osd_recovery_op_priority = '10' (unchangeable)
. o# n( ~% }8 a) X8 F0 @$ OError ENXIO: problem getting command descriptions from osd.1246 d% @( O4 v7 l3 |
osd.124: problem getting command descriptions from osd.124/ ~0 Y6 B4 s; Y8 ]1 }
osd.125: osd_recovery_op_priority = '10' (unchangeable)
4 q# g6 j( U1 l1 wosd.126: osd_recovery_op_priority = '10' (unchangeable) / |: ]* [0 ?8 k {8 w
Error ENXIO: problem getting command descriptions from osd.127
7 b8 [! ~3 @9 Z/ b nosd.127: problem getting command descriptions from osd.127
# L. Q, N r7 k( C/ K n: M" u: e [4 }osd.128: osd_recovery_op_priority = '10' (unchangeable)
% [: p/ F$ }. [8 }' Mosd.129: osd_recovery_op_priority = '10' (unchangeable) + A F6 e6 E- q- [& c) Z5 ? Y/ I
osd.130: osd_recovery_op_priority = '10' (unchangeable) , C0 {3 w. T( g
You have new mail in /var/spool/mail/root
( u/ [4 p, |. v7 ]- U3 I$ }3 D& B6 ~8 s/ z T: X& c
所有的osd都修改:
' k e9 V4 z9 y1 W# O x' |+ h4 }* h% k$ E# n9 m9 S
执行过ceph pg repair pg-id
0 O3 }. t& b: N$ L$ j
6 N0 K5 K0 _* E c( s' e' }6 u
9 Z% E2 I* x4 w7 n/ K' v( H状态出现修改过程:
; q6 Z1 W8 I0 y0 `[root compute1 ~]# ceph s+ n& E: [* B4 e3 J: M5 N
cluster 70d27aec742e4a95b000-cf37ebba35d0
) i' Y3 Z) p' L3 u; } health HEALTH_ERR% Y# _' w. v4 _- k4 d l5 g6 M
1 pgs inconsistent9 u7 ?- m4 G8 f3 N
1 pgs repair! Y+ N' ?1 P8 j9 D6 S* f% H% E
1 scrub errors$ q# L! B5 g) H
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}; @% ?/ I* H. E/ `# o2 E
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
4 i8 q* D( J x7 V8 a" S osdmap e26725: 131 osds: 128 up, 128 in& x7 `$ B. n7 h4 y
flags sortbitwise,require_jewel_osds
/ `, ]- _8 a/ U1 ] pgmap v169370679: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
" h. N5 }% P2 T3 w c b& x 309 TB used, 155 TB / 465 TB avail
* x4 \0 s9 C/ ]2 g7 g0 I) A$ { 10229 active+clean
2 p. M0 n) I/ \3 x. T 9 active+clean+scrubbing+deep& H! m' ~5 _, k
1 active+clean+scrubbing6 j* B) G1 a8 Y+ u
1 active+clean+scrubbing+deep+inconsistent+repair
& i9 _7 T9 n" o client io 91086 kB/s rd, 67615 kB/s wr, 3119 op/s rd, 3569 op/s wr
$ U) ^& j4 h( m$ M, k[root compute1 ~]# ceph s& l2 o- f' P5 r. L4 ^
cluster 70d27aec742e4a95b000-cf37ebba35d0' t5 l. s6 ]2 b
health HEALTH_ERR
' E# s' H9 p" W$ A) D0 Q; g. U 1 pgs inconsistent3 h2 m! B S J9 T) r
1 pgs repair
0 B' r$ y! }2 B$ I$ x8 i 1 scrub errors0 ?- q% a! ^$ _" e1 {+ S
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
' B. M! N0 l6 u+ d& c H election epoch 332, quorum 0,1,2 compute1,compute2,compute38 [) D7 y T1 ^
osdmap e26725: 131 osds: 128 up, 128 in% F: S/ T" e; h/ m7 Y
flags sortbitwise,require_jewel_osds
% R0 `) P0 \9 K8 }8 B2 j7 a pgmap v169370681: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
1 A4 b8 t; r2 p% N+ M 309 TB used, 155 TB / 465 TB avail
0 `( z2 k( f- D 10230 active+clean
0 ]- P2 W- p2 k. b$ P 8 active+clean+scrubbing+deep% \5 _& ^. b( _; [$ G" f& E! M
1 active+clean+scrubbing
6 C3 U) E& p& t4 D/ S 1 active+clean+scrubbing+deep+inconsistent+repair
9 E% x2 q4 b9 P1 Y8 G client io 104 MB/s rd, 67033 kB/s wr, 3479 op/s rd, 5013 op/s wr
2 s- ?7 S" K% K1 S; E[root compute1 ~]# ceph s# C0 }. C* e% B4 f4 Y
cluster 70d27aec742e4a95b000-cf37ebba35d00 o0 P* ~$ P* |7 _" [, N1 I
health HEALTH_ERR( [( `7 Y2 p3 t8 d/ \/ D
1 pgs inconsistent
K6 j k$ h* l/ m5 K 1 pgs repair6 }0 H& o7 ^* @+ o& S- O7 l, r
1 scrub errors9 X5 c& _' a7 o! i9 E' V
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0} M0 Q" I. [2 Z2 G ]9 W1 q
election epoch 332, quorum 0,1,2 compute1,compute2,compute39 _- ]1 [' m$ r" j0 O
osdmap e26725: 131 osds: 128 up, 128 in
: k) l& E) x: {% ?0 [3 |9 f flags sortbitwise,require_jewel_osds
2 ?0 G! j- v1 s a( q7 g pgmap v169370783: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
; ?) ~& S& s( i 309 TB used, 155 TB / 465 TB avail
. S1 A& C. J/ b- d 10232 active+clean
2 a' \8 n# ?/ S( ? [4 A+ ? 6 active+clean+scrubbing+deep8 L( x& `' ~7 G0 i! R1 |
1 active+clean+scrubbing
+ n7 k$ ^6 \9 ~* |4 } 1 active+clean+scrubbing+deep+inconsistent+repair
) \* `2 F# y# \9 O' \( c client io 147 MB/s rd, 57692 kB/s wr, 4584 op/s rd, 5944 op/s wr
7 R3 J7 t+ Q; _4 TYou have mail in /var/spool/mail/root& { W+ u- Z- h% ^) y3 Y& M
[root compute1 ~]# ceph s* t, s1 a2 m5 Z/ ]
cluster 70d27aec742e4a95b000-cf37ebba35d0/ F- a) }2 b, Q% y9 H/ l, c& |
health HEALTH_ERR
: O( s' p# \3 p9 f- y; {& X 1 pgs inconsistent/ A) ]/ r/ D& i. \6 e3 P2 M
1 pgs repair9 f! s/ \% ^% s6 y0 h
1 scrub errors
) Y* s4 y* N$ ~8 X monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}+ f/ C4 y* x! }" O
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
9 g5 Q8 o$ Q) V6 m osdmap e26725: 131 osds: 128 up, 128 in4 @7 M4 Q! @7 [; ?
flags sortbitwise,require_jewel_osds
6 Y2 v7 I; m }6 K6 x$ Z! w4 d1 a: U pgmap v169370785: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects3 _5 \1 q, o& U8 {+ c
309 TB used, 155 TB / 465 TB avail( Q6 ^# x2 M, p$ A, y! ]
10232 active+clean
( {: i4 ]2 x" V5 @$ k1 @& J 6 active+clean+scrubbing+deep! d" M- s4 L- t) Q7 o
1 active+clean+scrubbing& j0 L7 {, A5 n) A7 ] _& J
1 active+clean+scrubbing+deep+inconsistent+repair
1 P) b; y$ d, ~: J b$ g" k8 M client io 104 MB/s rd, 49581 kB/s wr, 2830 op/s rd, 3282 op/s wr; h( n; A% K8 E& a- w, C, \
[root compute1 ~]# ceph s
, O- U# Y' K' {& \# J cluster 70d27aec742e4a95b000-cf37ebba35d0( q7 Z, s5 G9 N+ f5 g1 ]' b. a
health HEALTH_ERR
% Z0 o9 ]3 c' t9 `' B$ Z 1 pgs inconsistent& Y. E4 K1 K) @9 l
1 pgs repair
: w* P/ V+ Y' e! J p/ u5 L 1 scrub errors5 q7 D2 |9 h, R u, K/ v( y
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
* P. `" r a( T2 v5 n: G election epoch 332, quorum 0,1,2 compute1,compute2,compute3
3 `# v* R$ j4 s8 z. k osdmap e26725: 131 osds: 128 up, 128 in" D9 C; w+ @: z! G& n" T
flags sortbitwise,require_jewel_osds
2 T8 h# e! l7 [. o& K" X pgmap v169370786: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects3 a$ L- K; A' y7 e! M
309 TB used, 155 TB / 465 TB avail. Y0 v! r& O' q
10232 active+clean
+ r3 D( c6 z, \2 w8 o! g+ ~" H 6 active+clean+scrubbing+deep- \) l4 t$ f" T1 i7 s
1 active+clean+scrubbing
9 e- i) j- V0 Z9 P4 s+ R 1 active+clean+scrubbing+deep+inconsistent+repair% n( g' r" N4 @' W4 F& Y
client io 95762 kB/s rd, 44372 kB/s wr, 2258 op/s rd, 2660 op/s wr$ w1 A, J4 T/ N" P; q
4 y% A; [* o( R- _6 ?& Y5 j
0 `' t/ \ T. n- b, k[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 8 s( t# L- k5 `8 {, R! P% W: F
"osd_max_backfills": "1",
- s3 \$ I! X) | "osd_recovery_max_active": "1",
. i( A' _ s9 W$ k5 e e- P "osd_recovery_op_priority": "10",
2 e* l. s! {3 H* K3 F[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
$ p4 j" U. W" S "osd_max_backfills": "1",; U$ m) X: l. J: E! I% R9 r9 `; T6 I
"osd_recovery_max_active": "1",& i' |9 @$ G% V3 D4 M0 ~6 |% \) G
"osd_recovery_op_priority": "10",/ }' n8 V# c* ~9 a r! Z& o- x
[root compute1 ~]# ceph s) W- S8 C3 R1 e, V0 J" m7 t
cluster 70d27aec742e4a95b000-cf37ebba35d0
" F9 D1 W( t4 P# l* P' ~% d health HEALTH_ERR
: a6 B2 H3 T. U! }% K 1 pgs inconsistent
' p3 d& h0 s* W) @# a, j; F7 c+ K 1 pgs repair
& d# n1 B$ T. M/ M2 j8 |. R5 T: O 1 scrub errors+ I0 I. X/ [! _ g0 i( p$ o9 L! E$ t
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}3 N* U# m; @. e; A" C
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
" `! F7 D. _8 _. F& Z osdmap e26725: 131 osds: 128 up, 128 in
: G: w/ E/ t. A' j. w1 n8 I% n flags sortbitwise,require_jewel_osds
7 y" J! A" L; j% W( r pgmap v169370799: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects( f$ Y# Y& n+ u" {- I6 ]! ^( X
309 TB used, 155 TB / 465 TB avail# P# w4 e) R* ]0 m3 X
10232 active+clean
$ l" Y7 B- p$ Q- M+ P0 ^, e2 {' M 6 active+clean+scrubbing+deep
- X2 x; f8 F L6 o: c* X 1 active+clean+scrubbing0 Y A" ?" z3 M3 D% C) ]
1 active+clean+scrubbing+deep+inconsistent+repair- ~8 E: P8 k! t- U. M3 n7 I# c
client io 134 MB/s rd, 117 MB/s wr, 3865 op/s rd, 4949 op/s wr
* j0 G& ]1 t. U: |4 d+ A+ p
* {8 O% g5 {- h- F a l5 S% q |" D3 y' P) T. h+ J
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" . W/ S4 B3 Y8 b/ F
"osd_max_backfills": "1",
3 ]3 }& Z& G3 G5 e( l "osd_recovery_max_active": "1",# r( P: \1 c# c" Y
"osd_recovery_op_priority": "10",
- E" L" q* `1 h, [! w, A0 p' _% O9 E; d, N! q5 P
! `; o- k4 p# d2 Y4 D8 ^3 r. u A& V' B& m# e) P: h
[root compute1 ~]# ceph s2 z6 ` O" Q/ A
cluster 70d27aec742e4a95b000-cf37ebba35d0
- [9 j2 h& _3 p: j health HEALTH_OK% c' e& |9 {2 m0 T
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
0 ~! V" ~+ O# x0 J$ T v: g- \0 I! N election epoch 332, quorum 0,1,2 compute1,compute2,compute3: a C+ m N8 l" d- {+ I/ G
osdmap e26725: 131 osds: 128 up, 128 in; P. d% C- n% ^ @) ~
flags sortbitwise,require_jewel_osds( Y- j: q, O6 ^6 E! R [
pgmap v169370916: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects, ]6 _/ [% w# R' p5 b( v" C9 ^: S0 Q
309 TB used, 155 TB / 465 TB avail1 N* I' F4 f6 m8 y
10233 active+clean2 k$ b1 [# M6 t5 i
6 active+clean+scrubbing+deep( ?5 p$ \. M2 e. ~2 z+ Q& i9 V
1 active+clean+scrubbing
& I# G7 t0 r4 k* D T client io 195 MB/s rd, 234 MB/s wr, 4404 op/s rd, 5993 op/s wr
( u8 I3 \- m/ q X S, P6 T% MYou have mail in /var/spool/mail/root1 U% n6 u* ~+ d: m
1 i X0 W/ j# \. u, e" X
[root compute1 ~]# ceph s2 I r/ `1 z: f' W$ a# T
cluster 70d27aec742e4a95b000-cf37ebba35d0
9 N6 {+ H( b. a; g health HEALTH_OK
1 ~( s1 L R, \2 K monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}9 A* O2 y! v- q7 I* s
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
8 P P$ S+ F% p3 D, r$ M2 C osdmap e26725: 131 osds: 128 up, 128 in/ N, y" j6 q& j; C/ [& Y
flags sortbitwise,require_jewel_osds
* v' E: E0 [8 Z1 b; Z% ]7 g, ? pgmap v169370921: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
' p- k. a- T% S$ K, M1 [ [8 S/ f 309 TB used, 155 TB / 465 TB avail$ l& C; @4 a! [. `" x& W. x! q6 B4 G
10233 active+clean
% [: Q- a0 l8 ], I9 }7 J. B 6 active+clean+scrubbing+deep, a4 J2 W1 y' i# ^. G
1 active+clean+scrubbing
% q% ?. G8 O9 B client io 257 MB/s rd, 358 MB/s wr, 5025 op/s rd, 6587 op/s wr& T$ }" H5 ^" k- t% D
* h6 `5 D7 v( |& E9 X- y# C恢复正常:1 N2 {3 y3 f) u; H8 M
[root compute1 ~]# ceph s
+ e! ~2 {/ C4 Z8 d0 e* J cluster 70d27aec742e4a95b000-cf37ebba35d08 y$ ~. d9 s. I: F
health HEALTH_OK
0 ]0 P [% e" j' e; w monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0} J0 `. H" s2 W2 A
election epoch 332, quorum 0,1,2 compute1,compute2,compute35 e$ ^+ V3 r4 U$ d
osdmap e26725: 131 osds: 128 up, 128 in
9 u4 U7 C4 m$ q) E% z ~. i1 w flags sortbitwise,require_jewel_osds
: k- v i1 `2 m, m6 C pgmap v169370922: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects, o' K; g) t9 s' a2 i
309 TB used, 155 TB / 465 TB avail
$ ]. N5 v) A/ |" `- | 10233 active+clean
1 h' o5 ~! F' a- J5 c 6 active+clean+scrubbing+deep. r+ G/ q# B3 \+ }
1 active+clean+scrubbing' Y; c8 h g9 O2 A8 M) Y; S# E
client io 254 MB/s rd, 390 MB/s wr, 5522 op/s rd, 6826 op/s wr5 I1 s, u) u" P* W' |
# v# |. L+ a2 O" V& R) D6 M0 t" f
6 A6 _2 t+ v% N, r |
|