|
|
楼主 |
发表于 2023-11-24 00:06:43
|
显示全部楼层
解决过程,发现无发正常同步,通过检查调整参数,达到修复的目的:
9 ?% ?# _; N' N: Z: j P[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 5 [2 W& L5 C2 J# p
"osd_max_backfills": "1",
5 c; q( R% Y' r: h "osd_recovery_max_active": "1"," a$ e) x* z9 M4 D6 J- g0 J
"osd_recovery_op_priority": "0",! @$ y5 m# \" |* A5 K) y0 Q
"osd_recovery_op_priority": "0", 默认是10
- G7 r, S: C2 l7 P! K0 N8 ?; v! z' w: J _9 O
修改为默认值10 7 g3 r# D: T Y! H( _5 D2 S* c
% ]6 X( t# ?. h2 m/ f9 O9 ]& \
[root compute1 ~]# ceph tell osd.* injectargs osd_recovery_op_priority=105 k$ c9 ^) c2 u: Q9 L
osd.0: osd_recovery_op_priority = '10' (unchangeable)
, e! ]/ b: t8 U5 yosd.1: osd_recovery_op_priority = '10' (unchangeable) ' }6 ]- r1 a3 u& F) B. V4 W# p
osd.2: osd_recovery_op_priority = '10' (unchangeable) 8 F4 d( h; ?! k
osd.3: osd_recovery_op_priority = '10' (unchangeable) : A7 t. R% S" V- S
osd.4: osd_recovery_op_priority = '10' (unchangeable)
" g5 ?, [! u9 z4 d) Hosd.5: osd_recovery_op_priority = '10' (unchangeable) ; ?+ @/ ?0 m( p2 u1 `: s+ H
osd.6: osd_recovery_op_priority = '10' (unchangeable) " i* G1 \2 j, v( V, a0 R- U
osd.7: osd_recovery_op_priority = '10' (unchangeable)
; r! L, L, |$ R7 eosd.8: osd_recovery_op_priority = '10' (unchangeable) : O2 r) ^9 V! `1 H# l6 x
osd.9: osd_recovery_op_priority = '10' (unchangeable) : Z, ?7 a- m5 g5 V' E! o* R+ \0 k, e
osd.10: osd_recovery_op_priority = '10' (unchangeable)
+ r* d- }6 _/ H- \0 Gosd.11: osd_recovery_op_priority = '10' (unchangeable) 8 A2 V& L# `* e! O; y
osd.12: osd_recovery_op_priority = '10' (unchangeable)
) o- B: Z& i" r% I* Y- P. qosd.13: osd_recovery_op_priority = '10' (unchangeable)
3 R+ @' i6 Y4 g; P4 B$ a/ |- a! X; Oosd.14: osd_recovery_op_priority = '10' (unchangeable) ; w1 K) I: ^+ L( e m5 i; D# Q
osd.15: osd_recovery_op_priority = '10' (unchangeable) $ j/ d; y- y2 C( h- N! W( b
osd.16: osd_recovery_op_priority = '10' (unchangeable) 0 F N9 d" c S) p
osd.17: osd_recovery_op_priority = '10' (unchangeable) # _. o4 X! w$ G; b/ v
osd.18: osd_recovery_op_priority = '10' (unchangeable)
4 U" B& U4 O* E- oosd.19: osd_recovery_op_priority = '10' (unchangeable) % k0 ]9 M# ]3 F8 c) k
osd.20: osd_recovery_op_priority = '10' (unchangeable) 8 N; R) d h7 A# q
osd.21: osd_recovery_op_priority = '10' (unchangeable) + y8 `0 x3 L, l. f, e, o7 U+ _/ \& W
osd.22: osd_recovery_op_priority = '10' (unchangeable) * _; P' M) W0 n0 V0 \3 ]. Y
osd.23: osd_recovery_op_priority = '10' (unchangeable)
E- ?$ E0 E) Q* posd.24: osd_recovery_op_priority = '10' (unchangeable)
3 G7 G$ m; y5 l1 G: D* Mosd.25: osd_recovery_op_priority = '10' (unchangeable)
% T8 u- ^! A' \$ M2 Nosd.26: osd_recovery_op_priority = '10' (unchangeable)
+ A Z. }1 T) k8 Q$ v$ ~6 t Tosd.27: osd_recovery_op_priority = '10' (unchangeable) C7 R! L5 u+ ^- A8 \+ `- b i+ k
osd.28: osd_recovery_op_priority = '10' (unchangeable)
4 c. d4 V; Z( ?3 k, ^osd.29: osd_recovery_op_priority = '10' (unchangeable) 1 O, @1 n/ ], [5 `! _9 J7 ^# a, a7 y
osd.30: osd_recovery_op_priority = '10' (unchangeable) " E: T: F* V1 E7 S: }8 I# V
osd.31: osd_recovery_op_priority = '10' (unchangeable) 7 G$ h( R( L" b: ?! z& P: o6 L
osd.32: osd_recovery_op_priority = '10' (unchangeable)
7 P9 _0 F0 ^! |% {0 @0 Fosd.33: osd_recovery_op_priority = '10' (unchangeable) * s) q3 M$ B* O& t. }1 E
osd.34: osd_recovery_op_priority = '10' (unchangeable)
D* B. b5 _/ D5 w$ H' \osd.35: osd_recovery_op_priority = '10' (unchangeable) 7 S2 {' s; E" K1 p7 W3 C
osd.36: osd_recovery_op_priority = '10' (unchangeable)
7 ], h8 ~2 F- U% s" Mosd.37: osd_recovery_op_priority = '10' (unchangeable)
4 c/ y4 O' d3 r1 X, _osd.38: osd_recovery_op_priority = '10' (unchangeable) 9 @$ n) O8 Q+ h9 B; `- }
osd.39: osd_recovery_op_priority = '10' (unchangeable) , D; _) R9 p: p
osd.40: osd_recovery_op_priority = '10' (unchangeable) 2 L2 o! {" [' w+ u
osd.41: osd_recovery_op_priority = '10' (unchangeable) ' d- C: B# h6 A" C! J" J
osd.42: osd_recovery_op_priority = '10' (unchangeable)
/ S f: \4 E4 C) Hosd.43: osd_recovery_op_priority = '10' (unchangeable) & ^% Z# p5 `1 t/ d; M+ `7 `# ~& `) B# G H
osd.44: osd_recovery_op_priority = '10' (unchangeable)
7 u1 W# I6 I- Rosd.45: osd_recovery_op_priority = '10' (unchangeable) / q% }2 u$ i+ C+ w7 a# g
osd.46: osd_recovery_op_priority = '10' (unchangeable)
2 l$ A; F5 B6 s( Mosd.47: osd_recovery_op_priority = '10' (unchangeable) 7 U w0 y$ s! }) f# Z
osd.48: osd_recovery_op_priority = '10' (unchangeable)
1 {2 c3 n- l1 yosd.49: osd_recovery_op_priority = '10' (unchangeable)
& u- k- U% C4 o! Rosd.50: osd_recovery_op_priority = '10' (unchangeable)
& b7 u) C. z. Q, f9 ]osd.51: osd_recovery_op_priority = '10' (unchangeable) ! `) U* z4 l" c' n9 R0 l2 ]$ o
osd.52: osd_recovery_op_priority = '10' (unchangeable) 8 W1 H4 c K4 h1 h7 ? v7 f5 ]
osd.53: osd_recovery_op_priority = '10' (unchangeable) ; X* P z7 C. T# u$ o6 v
Error ENXIO: problem getting command descriptions from osd.54- C7 q! h* t3 U: \ Q% j# K2 O5 ~5 I
osd.54: problem getting command descriptions from osd.542 w1 ~: A9 w: a5 j6 M6 X; c
osd.55: osd_recovery_op_priority = '10' (unchangeable)
7 X8 a4 R5 |5 }/ Losd.56: osd_recovery_op_priority = '10' (unchangeable)
- T' {2 e1 S( ^, m+ S- L3 wosd.57: osd_recovery_op_priority = '10' (unchangeable) + j# L! ~) g8 Z, s- G- c9 t
osd.58: osd_recovery_op_priority = '10' (unchangeable) - M \# a+ A3 Y/ E8 V. v
osd.59: osd_recovery_op_priority = '10' (unchangeable)
* b9 e6 G* o0 Q0 ]4 xosd.60: osd_recovery_op_priority = '10' (unchangeable) 4 K i* M* n0 L6 p! C( v, v
osd.61: osd_recovery_op_priority = '10' (unchangeable) # R3 u6 j6 C! f
osd.62: osd_recovery_op_priority = '10' (unchangeable) & m2 d# a& {/ F) |. I6 Q+ M" i3 E3 `2 r
osd.63: osd_recovery_op_priority = '10' (unchangeable)
5 d3 ]* o$ P5 o* M5 Posd.64: osd_recovery_op_priority = '10' (unchangeable) / b8 c' ?* r3 X* u3 N* h: L
osd.65: osd_recovery_op_priority = '10' (unchangeable) , V4 w5 W2 R8 w# X" `: o
osd.66: osd_recovery_op_priority = '10' (unchangeable) ( x7 |8 T0 F( o( {& @3 u
osd.67: osd_recovery_op_priority = '10' (unchangeable) 5 T4 @& {* Z- m) q
osd.68: osd_recovery_op_priority = '10' (unchangeable)
+ d4 r7 g' p& a6 {( Losd.69: osd_recovery_op_priority = '10' (unchangeable) 1 A3 Y. ]/ u- v# x8 |
osd.70: osd_recovery_op_priority = '10' (unchangeable) 9 z1 f+ t" ^/ q; t; _) r
osd.71: osd_recovery_op_priority = '10' (unchangeable) 3 m6 I g) j8 l& Z; K
osd.72: osd_recovery_op_priority = '10' (unchangeable) 8 C2 T. \2 G. B. d- s4 |7 t
osd.73: osd_recovery_op_priority = '10' (unchangeable) ! U4 y9 g8 o; `! ]) j4 @
osd.74: osd_recovery_op_priority = '10' (unchangeable)
. _, |1 q8 D0 j+ W( O. Oosd.75: osd_recovery_op_priority = '10' (unchangeable) + E# k. |1 X& w& \, W
osd.76: osd_recovery_op_priority = '10' (unchangeable)
5 W" q! W2 I9 i; ~/ d- `9 tosd.77: osd_recovery_op_priority = '10' (unchangeable)
& Z% O5 b$ A, ~; U; e: I: @osd.78: osd_recovery_op_priority = '10' (unchangeable) / _8 Z6 J8 L1 C# E( T8 d) p
osd.79: osd_recovery_op_priority = '10' (unchangeable) ! e+ Y7 L8 X, \: f& ^
osd.80: osd_recovery_op_priority = '10' (unchangeable) " J8 [5 O$ T Q+ K
osd.81: osd_recovery_op_priority = '10' (unchangeable)
. D* B8 m* t/ B6 y% \# U2 dosd.82: osd_recovery_op_priority = '10' (unchangeable)
& k8 l& z; }/ ^6 G0 J( ^6 t' k( j1 e" Tosd.83: osd_recovery_op_priority = '10' (unchangeable)
; s! Y# e( E: E; vosd.84: osd_recovery_op_priority = '10' (unchangeable) 9 D/ Q5 Y* N$ J5 ]# G
osd.85: osd_recovery_op_priority = '10' (unchangeable)
1 R/ Y* P; F# Zosd.86: osd_recovery_op_priority = '10' (unchangeable)
4 [% I' w: f- r. M4 ~osd.87: osd_recovery_op_priority = '10' (unchangeable)
5 H n/ A5 Z& E- Uosd.88: osd_recovery_op_priority = '10' (unchangeable)
2 q+ d% p) q$ ]& a& V& K% {osd.89: osd_recovery_op_priority = '10' (unchangeable)
9 r3 K2 h9 ~9 M( `: |. eosd.90: osd_recovery_op_priority = '10' (unchangeable) % N2 q4 {3 B! H/ p
osd.91: osd_recovery_op_priority = '10' (unchangeable)
2 @( \' \! X, K( B; }4 n" R" kosd.92: osd_recovery_op_priority = '10' (unchangeable) * ^2 E4 K, v2 Y' R- [7 E; C3 N
osd.93: osd_recovery_op_priority = '10' (unchangeable) : x8 ]5 K* \3 d
osd.94: osd_recovery_op_priority = '10' (unchangeable) - t4 V: i/ M* N4 p. X
osd.95: osd_recovery_op_priority = '10' (unchangeable)
$ Q: s& i2 ?6 u: g( nosd.96: osd_recovery_op_priority = '10' (unchangeable) # S& ]) V1 Z. s W+ O1 f
osd.97: osd_recovery_op_priority = '10' (unchangeable)
2 z/ }) G# o6 s7 i9 j6 eosd.98: osd_recovery_op_priority = '10' (unchangeable)
! N3 M% B @: X7 I, Rosd.99: osd_recovery_op_priority = '10' (unchangeable)
1 D: N3 C/ C" L9 V, V. Zosd.100: osd_recovery_op_priority = '10' (unchangeable)
, c( Z& ~3 t6 H# D$ h- Nosd.101: osd_recovery_op_priority = '10' (unchangeable) " U! @ b2 L0 I
osd.102: osd_recovery_op_priority = '10' (unchangeable) $ v1 _! v% l3 u$ C
osd.103: osd_recovery_op_priority = '10' (unchangeable)
: a4 k8 \% W" E2 a4 _osd.104: osd_recovery_op_priority = '10' (unchangeable)
1 c! ^( I/ v- y* Z0 c0 y6 i s% ?osd.105: osd_recovery_op_priority = '10' (unchangeable) . i1 q3 h+ O* s$ C
osd.106: osd_recovery_op_priority = '10' (unchangeable) & W5 J X; V0 Y# [5 E0 q7 n3 |
osd.107: osd_recovery_op_priority = '10' (unchangeable) 6 ]* u" |/ c1 l' F- a+ @
osd.108: osd_recovery_op_priority = '10' (unchangeable)
3 U+ k% j' A) A" A- Qosd.109: osd_recovery_op_priority = '10' (unchangeable)
! D, t: s0 J) dosd.110: osd_recovery_op_priority = '10' (unchangeable) 2 x/ _# E* @4 K( @- x
osd.111: osd_recovery_op_priority = '10' (unchangeable) $ w" h( U% U/ s* T' |0 M
osd.112: osd_recovery_op_priority = '10' (unchangeable) & N' s- G/ u H5 k# J+ b
osd.113: osd_recovery_op_priority = '10' (unchangeable) : k" \# y+ ?/ k; B* A
osd.114: osd_recovery_op_priority = '10' (unchangeable) ! o0 F7 l1 V! F4 w/ S
osd.115: osd_recovery_op_priority = '10' (unchangeable) , l z# P. e3 T3 Q2 y" D
osd.116: osd_recovery_op_priority = '10' (unchangeable) % V0 S% n, T; ^; l
osd.117: osd_recovery_op_priority = '10' (unchangeable) ' b \$ X. d0 F% V- k0 v3 v
osd.118: osd_recovery_op_priority = '10' (unchangeable)
: U& a: S0 c: x. F# [- Eosd.119: osd_recovery_op_priority = '10' (unchangeable) 8 L: R: G A( l. B
osd.120: osd_recovery_op_priority = '10' (unchangeable)
7 ]" n# B z( |6 Vosd.121: osd_recovery_op_priority = '10' (unchangeable) ' j/ }/ P$ T( v7 @$ B
osd.122: osd_recovery_op_priority = '10' (unchangeable) 1 |+ D. j/ J' h4 K
osd.123: osd_recovery_op_priority = '10' (unchangeable)
% M8 k3 U- \8 X- y3 LError ENXIO: problem getting command descriptions from osd.124+ P! S8 q- ?; \; u' N/ J# p
osd.124: problem getting command descriptions from osd.124
$ z8 t# }' R3 T8 U! P& `4 Qosd.125: osd_recovery_op_priority = '10' (unchangeable) 4 H2 A& t7 ? l
osd.126: osd_recovery_op_priority = '10' (unchangeable) 8 E2 i) K5 r# a: d3 e! g/ q
Error ENXIO: problem getting command descriptions from osd.1271 ~" P" c! `9 E* l Q
osd.127: problem getting command descriptions from osd.1272 k) C3 O+ c0 }2 K$ D
osd.128: osd_recovery_op_priority = '10' (unchangeable)
9 p! d% y; i6 S2 \% `, R% Oosd.129: osd_recovery_op_priority = '10' (unchangeable)
0 t4 ]3 \. |# Y. ]osd.130: osd_recovery_op_priority = '10' (unchangeable) 4 A$ ^! A- f5 F( ?/ [
You have new mail in /var/spool/mail/root
! p" x4 d# u6 _$ g+ t
. i9 {# v+ s! `6 c" p所有的osd都修改:& l' u. P; f. S3 ] _
7 E+ L0 E! b% R! I. U, j3 v' @4 s9 e
执行过ceph pg repair pg-id9 g3 p( \8 d' Q' @7 L( e( t" S
0 j( B0 n, L4 \( {: T! e
8 Z0 s# a2 H7 q/ g# L9 G状态出现修改过程: f/ r# v* \) g' @9 W
[root compute1 ~]# ceph s a/ r9 M0 ~/ r$ M
cluster 70d27aec742e4a95b000-cf37ebba35d0( A& i2 q% ~2 K1 I- k5 W0 r
health HEALTH_ERR; c6 {& j& T" z: J: Q
1 pgs inconsistent
5 A0 i# F4 i6 Q! Q1 e! ~ 1 pgs repair7 L. Z6 Q* S5 K2 N. h; D
1 scrub errors- i( f; f" c" D4 a6 X$ D* a
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
1 z3 I- w0 U" Q# k: S* E election epoch 332, quorum 0,1,2 compute1,compute2,compute3& @- z2 W2 O# Q/ C$ h
osdmap e26725: 131 osds: 128 up, 128 in" s; s. X7 w$ Z# C3 d3 y
flags sortbitwise,require_jewel_osds, v+ o5 e) c; ~: G$ o
pgmap v169370679: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
c3 ]! d: F2 C; a' P- p: T2 h 309 TB used, 155 TB / 465 TB avail8 a7 s- }0 V4 | m! P
10229 active+clean" |- |! H5 F2 f
9 active+clean+scrubbing+deep: g! K0 v9 P. D5 ?6 K5 r
1 active+clean+scrubbing
, Y( w8 @( Q& N p. }4 j$ u 1 active+clean+scrubbing+deep+inconsistent+repair2 h% @) i9 K7 A6 i4 x2 m
client io 91086 kB/s rd, 67615 kB/s wr, 3119 op/s rd, 3569 op/s wr8 y; B* i- h! j) y* K. h
[root compute1 ~]# ceph s: k5 r& ^" }2 B; f
cluster 70d27aec742e4a95b000-cf37ebba35d0- A6 i5 h; A) w6 y5 j
health HEALTH_ERR
5 v7 a+ ]! M$ b2 d" | 1 pgs inconsistent
: L1 V: w( W" N/ W+ ~ 1 pgs repair
( G1 j) U- n+ z/ ~8 ] 1 scrub errors
' e9 v% n1 v) y8 o monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
( ]! V0 ]% t- M8 y election epoch 332, quorum 0,1,2 compute1,compute2,compute33 l8 E: g" p0 m8 b' f1 l6 G
osdmap e26725: 131 osds: 128 up, 128 in
, d8 L9 B# V' c" O' d4 b flags sortbitwise,require_jewel_osds8 g+ t2 N0 a6 \5 w, e7 s1 B6 q
pgmap v169370681: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
# o9 j/ A+ u2 d% }/ c! H 309 TB used, 155 TB / 465 TB avail
8 W* i# ~; l7 F3 ] 10230 active+clean
% d- M6 h( F% L) g+ E 8 active+clean+scrubbing+deep
% N/ a0 ?3 ~8 r+ n9 M; D: t0 M 1 active+clean+scrubbing5 k! P' Q2 \0 @" r0 e
1 active+clean+scrubbing+deep+inconsistent+repair
( }0 V; G( P6 c! u client io 104 MB/s rd, 67033 kB/s wr, 3479 op/s rd, 5013 op/s wr+ H! G" R# Z* O- o! t8 g. ^% ?
[root compute1 ~]# ceph s c) d3 z# }/ j) I r) u
cluster 70d27aec742e4a95b000-cf37ebba35d0
$ ^! {! V3 ^2 T% ?" E& {4 n health HEALTH_ERR
8 i* t6 S6 J# e* ?, S% F' ^+ c 1 pgs inconsistent. V6 y8 y- T1 a9 V
1 pgs repair0 A0 w$ n: }8 C' d- ?- i/ }
1 scrub errors
! x6 v+ ]# B* r' f monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
2 p' p. g( I; f8 v election epoch 332, quorum 0,1,2 compute1,compute2,compute3
1 s& z4 i8 ]% O2 p( |- x# D osdmap e26725: 131 osds: 128 up, 128 in
; ~' t, P! W; [( s1 H; P flags sortbitwise,require_jewel_osds7 D/ G, z- t5 b- N. H
pgmap v169370783: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
# Y4 Q, K: q. }4 a8 v; ]+ U 309 TB used, 155 TB / 465 TB avail
0 Z' i1 W1 S3 ~5 } 10232 active+clean' K6 ?( s4 X+ V5 W
6 active+clean+scrubbing+deep& Y- X# L- S! e2 g
1 active+clean+scrubbing
2 \6 N8 H- w$ E' {1 e! [- [4 T 1 active+clean+scrubbing+deep+inconsistent+repair
7 ^0 x5 J9 K+ Y# T client io 147 MB/s rd, 57692 kB/s wr, 4584 op/s rd, 5944 op/s wr# A+ ^1 F6 a5 `& \' V
You have mail in /var/spool/mail/root
. o2 h' d3 o" d[root compute1 ~]# ceph s
+ p S! A* s' p* j- c cluster 70d27aec742e4a95b000-cf37ebba35d02 u8 a" [# K# f3 I- e8 m1 D
health HEALTH_ERR) v) q+ H9 A. I& s% L" A
1 pgs inconsistent$ F" R1 @3 ~' H8 F* l1 j
1 pgs repair
1 l( g* Y7 ^, o+ ?4 J 1 scrub errors5 H/ l3 [1 F& H2 Y& V! ]
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}! I, ?) L: t& K1 g+ N4 d
election epoch 332, quorum 0,1,2 compute1,compute2,compute32 b+ [' p0 o, H) ^6 A
osdmap e26725: 131 osds: 128 up, 128 in
7 y& ?% ~2 l6 q5 u' g- e flags sortbitwise,require_jewel_osds
* i, n/ V* d" N7 b8 _ pgmap v169370785: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects$ T) z# a8 E% d. \; w
309 TB used, 155 TB / 465 TB avail
& \( `1 \6 @& h A4 v8 w# I 10232 active+clean, t( r j8 S W* V k/ k; q, H
6 active+clean+scrubbing+deep9 h6 p5 o# k m; ^
1 active+clean+scrubbing
+ ^5 T( r9 @0 C, e9 K8 T( j 1 active+clean+scrubbing+deep+inconsistent+repair
' l, C! L( Q% r4 E" @ client io 104 MB/s rd, 49581 kB/s wr, 2830 op/s rd, 3282 op/s wr+ k5 L4 |. E4 Y
[root compute1 ~]# ceph s
+ U& j C9 {8 @ cluster 70d27aec742e4a95b000-cf37ebba35d0
" ]* I4 {; ^) n. Q9 w health HEALTH_ERR
3 z% ?% B% H: L0 ?8 W4 A- _% P; g 1 pgs inconsistent2 N( T. M" d; u4 ` J u
1 pgs repair
4 o8 H1 u: `% C- n; { 1 scrub errors
c. a0 A! B. D9 E. z% } monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}' q* s2 e7 w0 ^% Y
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
+ s, s. [6 X% L" f osdmap e26725: 131 osds: 128 up, 128 in% Z4 j1 g& Y5 [9 t6 p- X2 N
flags sortbitwise,require_jewel_osds* U0 X; ~- D' W; o1 N8 p
pgmap v169370786: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects) b t0 M5 j4 F0 w5 [) e7 e7 W
309 TB used, 155 TB / 465 TB avail, E7 j7 `7 _0 F* I, l/ _- d
10232 active+clean, Y! j, R4 K8 F/ d5 K: L ?; q
6 active+clean+scrubbing+deep
, ^* Y9 ]! E1 Z. Y% M( b, F3 c* _ 1 active+clean+scrubbing" B. [% d5 A, B
1 active+clean+scrubbing+deep+inconsistent+repair
1 L- h! `6 g/ ]' L client io 95762 kB/s rd, 44372 kB/s wr, 2258 op/s rd, 2660 op/s wr1 ]' M. y% Z: f8 n
. `- o, I3 w3 t0 e, h' T- z* m& y. `. J2 h; T
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 8 F+ x- i6 x1 S5 W
"osd_max_backfills": "1",
6 `, h% e9 Q9 c5 ^5 C6 y# G "osd_recovery_max_active": "1",9 p9 X$ a' i" x) a9 H
"osd_recovery_op_priority": "10",$ `+ `7 ?2 l) ^* C! t
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" , x! ]% d# x4 u& @
"osd_max_backfills": "1",
/ x: P: X% r% G/ O: _5 T0 p" @ "osd_recovery_max_active": "1",. r! V! ?! W4 ^$ n0 M9 g
"osd_recovery_op_priority": "10",
0 F, D+ |9 @- `; H[root compute1 ~]# ceph s
' _7 \' L2 C0 E5 T5 l0 d5 j4 ? cluster 70d27aec742e4a95b000-cf37ebba35d0) s+ ?& L* J9 W8 J
health HEALTH_ERR
7 A" B V. v9 K( w 1 pgs inconsistent
; u7 S9 S3 q" j. I2 u 1 pgs repair! P3 k4 I- c' q7 i
1 scrub errors
, k7 o, p' G/ F7 S( ]; H5 ~ monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}! n- d" U$ t6 v4 Q) g, G
election epoch 332, quorum 0,1,2 compute1,compute2,compute3' G) r. R5 `- ]$ Y u
osdmap e26725: 131 osds: 128 up, 128 in
P9 G) H/ A. p" s8 v1 o L$ O: v* b flags sortbitwise,require_jewel_osds, b5 P4 G4 c2 U, v1 Z
pgmap v169370799: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
6 \9 N7 W4 F- R' k; O4 y 309 TB used, 155 TB / 465 TB avail l5 U+ O% E; v; T1 Z
10232 active+clean
4 W2 ^% D& k. K; b3 P6 ~ 6 active+clean+scrubbing+deep& _, u' V1 B6 Y9 e$ I: c
1 active+clean+scrubbing
% I" [$ o: ~7 V* g' B+ k* m2 k 1 active+clean+scrubbing+deep+inconsistent+repair8 w2 g% M9 f6 O1 o
client io 134 MB/s rd, 117 MB/s wr, 3865 op/s rd, 4949 op/s wr
: C, c* \! m3 N# ?$ _& G4 b. x; Y$ C) w9 ^/ n$ r
, f" [, q/ Q8 u3 L) Z" }; Y
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" ! Q- n- |0 ^( q/ Y' m* l9 o+ t& n
"osd_max_backfills": "1",
# n3 }/ I5 S$ y! `5 Q+ K- p; V. ~% q "osd_recovery_max_active": "1",
! n i. r: c6 S" y "osd_recovery_op_priority": "10",( p6 g1 c. ~* v! p* y4 U
& R9 \" _$ o2 l( M
. E* \7 x! z% p: g- D% D2 }2 ]' i: K3 P5 |/ l( K
[root compute1 ~]# ceph s
' s8 O" x [) M7 O1 _ cluster 70d27aec742e4a95b000-cf37ebba35d0
' d0 c# O/ t; ]$ ^- f health HEALTH_OK! f/ Q$ r! g. T Q: E
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
6 ]/ q) H3 L$ }1 x& \" c election epoch 332, quorum 0,1,2 compute1,compute2,compute36 W4 _6 h& e7 {" n# p1 H
osdmap e26725: 131 osds: 128 up, 128 in6 a0 b+ r% D2 q: d+ a8 ^" e0 ~
flags sortbitwise,require_jewel_osds
; H9 g0 }& a* e! A1 Y% N6 q pgmap v169370916: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
* j9 C5 J: m8 k, Z( a 309 TB used, 155 TB / 465 TB avail3 ?& t9 r2 k* U5 L
10233 active+clean6 j- u# |* c- M/ B! `6 A
6 active+clean+scrubbing+deep4 u& r4 @3 ]) c$ K- D& v
1 active+clean+scrubbing$ _0 G( V9 S* w
client io 195 MB/s rd, 234 MB/s wr, 4404 op/s rd, 5993 op/s wr
1 V. c! J) H" x, V; w; VYou have mail in /var/spool/mail/root
! k& O, Q7 A( d: o# N# V
( G, m3 K _2 r) p# i[root compute1 ~]# ceph s3 R! Y3 a4 S% `( e- X
cluster 70d27aec742e4a95b000-cf37ebba35d0
" [4 [- z7 c% l2 ^# [ health HEALTH_OK
, M h$ M& m3 p, B6 J: y monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}! G7 E9 y" j# S4 W* S; x* ~+ `1 z
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
7 M# n6 R# c# S8 r4 G$ n( \4 | osdmap e26725: 131 osds: 128 up, 128 in
5 ~! }+ ?+ N. x, g& ~2 }, m0 @ flags sortbitwise,require_jewel_osds. | A4 }, Z! ^, h& r3 g& t
pgmap v169370921: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
6 @; b) O: H5 D# x 309 TB used, 155 TB / 465 TB avail& s) Q' L# C: t) `/ v, Q
10233 active+clean7 i8 {8 L; [7 x3 \6 I$ h5 z
6 active+clean+scrubbing+deep$ H8 R- q( q; v2 A# D$ ]
1 active+clean+scrubbing
# {: s+ M5 J: N! p; R N. ] client io 257 MB/s rd, 358 MB/s wr, 5025 op/s rd, 6587 op/s wr* E& u7 x+ X* D1 p1 N- l# {& O3 |* C
: ]* o. V9 n9 ?# |
恢复正常:# c& Q4 C, \' q- {" X9 e
[root compute1 ~]# ceph s. E; f! M- _2 `7 l* Y9 H( n6 |
cluster 70d27aec742e4a95b000-cf37ebba35d0
! P0 X! y3 j& S health HEALTH_OK
+ V! `% s% A* d! O% W+ ^ monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
8 B3 S4 y* K% S# h election epoch 332, quorum 0,1,2 compute1,compute2,compute3* G4 u! ^9 A J1 z# d" m
osdmap e26725: 131 osds: 128 up, 128 in/ m" Y& g- Q" |2 H
flags sortbitwise,require_jewel_osds; _8 `0 o8 l1 ^1 i/ j& }8 _5 D
pgmap v169370922: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
3 e+ T8 F0 n; s# A6 O, G 309 TB used, 155 TB / 465 TB avail1 |/ |: E% t1 u% f" C% _4 H
10233 active+clean( H& z& e6 S3 J5 g2 @
6 active+clean+scrubbing+deep
! U7 }3 r( F* t& K% e6 u) [5 d 1 active+clean+scrubbing
7 n% J; A9 @8 _! } client io 254 MB/s rd, 390 MB/s wr, 5522 op/s rd, 6826 op/s wr
, V. }- E1 g; q" M
$ E; l4 z0 D. o+ X d2 \! v4 {. z' ]/ r- W3 x: i' I
|
|