|
|
楼主 |
发表于 2023-11-24 00:06:43
|
显示全部楼层
解决过程,发现无发正常同步,通过检查调整参数,达到修复的目的:& T) d# k7 H$ y: I. K
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" 5 v% c2 e V6 p: k* L
"osd_max_backfills": "1",
2 o5 C9 M2 M+ s8 c. h, n3 c "osd_recovery_max_active": "1",
. W) n7 q0 w) f' N- K "osd_recovery_op_priority": "0",
/ O% x k: }; p"osd_recovery_op_priority": "0", 默认是10
1 t* O: _/ G: C) o
2 H: d+ N0 L6 c) C2 P# e+ F* S1 v修改为默认值10
9 J% v% f7 ?% T0 W9 S# g4 ^; ^ a: M9 {" Q
[root compute1 ~]# ceph tell osd.* injectargs osd_recovery_op_priority=10 v4 a+ p: e7 ]' P# i8 R- h
osd.0: osd_recovery_op_priority = '10' (unchangeable) s; m. {/ D: z8 H4 p" X1 }
osd.1: osd_recovery_op_priority = '10' (unchangeable)
% Z+ q' _( k, c' M' i! n! |osd.2: osd_recovery_op_priority = '10' (unchangeable)
/ ~+ l, I+ r! S! S- X! Rosd.3: osd_recovery_op_priority = '10' (unchangeable)
; Q4 L% |" }; Sosd.4: osd_recovery_op_priority = '10' (unchangeable)
, z2 ]- v; W: W {3 M% L( oosd.5: osd_recovery_op_priority = '10' (unchangeable) + o; ~; G8 Q& f2 k. ~
osd.6: osd_recovery_op_priority = '10' (unchangeable) * f* t+ v$ N u7 [1 e
osd.7: osd_recovery_op_priority = '10' (unchangeable)
+ j Z# ^" \" R% J5 E! F: Nosd.8: osd_recovery_op_priority = '10' (unchangeable)
( T% e( s3 F4 \) s; ?* Mosd.9: osd_recovery_op_priority = '10' (unchangeable) 5 a3 [6 ]7 y0 S8 R
osd.10: osd_recovery_op_priority = '10' (unchangeable) 2 J+ |* m+ t3 j" Y3 }8 z
osd.11: osd_recovery_op_priority = '10' (unchangeable) 3 Z- C/ |) R, p
osd.12: osd_recovery_op_priority = '10' (unchangeable) 3 `' y4 x6 T' [9 I& l
osd.13: osd_recovery_op_priority = '10' (unchangeable) $ h* U# x8 h- L
osd.14: osd_recovery_op_priority = '10' (unchangeable)
9 |9 |' ^' S; t1 [( posd.15: osd_recovery_op_priority = '10' (unchangeable)
( h/ \4 o' m& @1 l% D/ Eosd.16: osd_recovery_op_priority = '10' (unchangeable)
$ I& _ L- l. c& ]osd.17: osd_recovery_op_priority = '10' (unchangeable) 1 S) u/ @! T a4 C% m4 Y
osd.18: osd_recovery_op_priority = '10' (unchangeable) + p& C) r% ?+ x G) y
osd.19: osd_recovery_op_priority = '10' (unchangeable) ) q) z |! k$ `5 N# e; c
osd.20: osd_recovery_op_priority = '10' (unchangeable) # [0 C8 I; u, `, {
osd.21: osd_recovery_op_priority = '10' (unchangeable) . q4 q4 S& j+ v" }
osd.22: osd_recovery_op_priority = '10' (unchangeable)
) w5 f$ ^+ w5 s; {: Zosd.23: osd_recovery_op_priority = '10' (unchangeable)
9 q- B/ n& L* u+ H' z1 y4 d2 o- qosd.24: osd_recovery_op_priority = '10' (unchangeable) , Y% O* V. U# y2 l" S
osd.25: osd_recovery_op_priority = '10' (unchangeable) E: h1 e" W u
osd.26: osd_recovery_op_priority = '10' (unchangeable) + l/ F' _) o. U! m
osd.27: osd_recovery_op_priority = '10' (unchangeable) - r. T# f9 D8 D
osd.28: osd_recovery_op_priority = '10' (unchangeable)
; Z5 G& [) `* i5 aosd.29: osd_recovery_op_priority = '10' (unchangeable)
: o+ ?6 m2 Q) t2 \: Z7 Wosd.30: osd_recovery_op_priority = '10' (unchangeable)
' H) A- L9 h7 k8 j+ _- X; Xosd.31: osd_recovery_op_priority = '10' (unchangeable)
2 e0 q. G6 r0 J0 b6 Kosd.32: osd_recovery_op_priority = '10' (unchangeable) * q7 ]+ U1 u7 _: \; y6 L" S
osd.33: osd_recovery_op_priority = '10' (unchangeable)
% X7 w0 s( ^* n. Cosd.34: osd_recovery_op_priority = '10' (unchangeable)
9 m% T |2 `0 Rosd.35: osd_recovery_op_priority = '10' (unchangeable)
. I, t9 ~& J# l0 x8 |3 e2 m% Posd.36: osd_recovery_op_priority = '10' (unchangeable) $ x0 k) _$ g4 H) I0 j' T7 \3 f1 }) [$ S" ~
osd.37: osd_recovery_op_priority = '10' (unchangeable)
: ]2 c0 I, i8 xosd.38: osd_recovery_op_priority = '10' (unchangeable)
/ B e/ V. Q" X1 L. Z' Fosd.39: osd_recovery_op_priority = '10' (unchangeable)
: k& l i' P% q+ a# ^osd.40: osd_recovery_op_priority = '10' (unchangeable)
* t2 E& Z e9 f: h/ c+ Gosd.41: osd_recovery_op_priority = '10' (unchangeable)
3 F. Y! }/ O6 _/ Gosd.42: osd_recovery_op_priority = '10' (unchangeable)
2 N0 q% V& V0 Fosd.43: osd_recovery_op_priority = '10' (unchangeable)
" y. G0 c+ c$ ~2 j, Posd.44: osd_recovery_op_priority = '10' (unchangeable)
- F6 i7 R8 W$ f5 k, h: v4 Sosd.45: osd_recovery_op_priority = '10' (unchangeable)
; F* \$ @ O2 y2 |- Aosd.46: osd_recovery_op_priority = '10' (unchangeable) ( V. [ w4 ^5 l J* x( s; j
osd.47: osd_recovery_op_priority = '10' (unchangeable) 1 _. t( W1 n3 A+ j4 x
osd.48: osd_recovery_op_priority = '10' (unchangeable)
( ~# g% e. _6 |+ t8 rosd.49: osd_recovery_op_priority = '10' (unchangeable)
% R$ j9 A Q6 w5 @osd.50: osd_recovery_op_priority = '10' (unchangeable)
4 Y4 @/ _2 V" G/ R) Posd.51: osd_recovery_op_priority = '10' (unchangeable) / l+ @% j/ _6 L, S8 ~
osd.52: osd_recovery_op_priority = '10' (unchangeable) & X: I, B7 Z/ r
osd.53: osd_recovery_op_priority = '10' (unchangeable) & {7 j: s! u' R7 V5 B
Error ENXIO: problem getting command descriptions from osd.54# P9 W& ~; I8 u
osd.54: problem getting command descriptions from osd.54
6 v5 W" x; c! V$ |- sosd.55: osd_recovery_op_priority = '10' (unchangeable)
' o% a, M. F1 _; u+ g" K" a1 f$ Gosd.56: osd_recovery_op_priority = '10' (unchangeable)
9 z7 q' N0 b/ {7 q$ k" Aosd.57: osd_recovery_op_priority = '10' (unchangeable)
! s3 O! A9 w/ i4 N" |2 mosd.58: osd_recovery_op_priority = '10' (unchangeable) # K, C# `, j: p! L, u# L# t/ g
osd.59: osd_recovery_op_priority = '10' (unchangeable)
( @ o& r2 L7 |& z" eosd.60: osd_recovery_op_priority = '10' (unchangeable)
% L2 w% x0 o" D6 w/ I9 j: G& posd.61: osd_recovery_op_priority = '10' (unchangeable)
4 L: h/ i" H' [) o/ Wosd.62: osd_recovery_op_priority = '10' (unchangeable)
. J5 E. g' i& H5 U3 | r4 b( \osd.63: osd_recovery_op_priority = '10' (unchangeable) 5 L+ _9 j( w5 |( k& y
osd.64: osd_recovery_op_priority = '10' (unchangeable)
- f6 f4 H [- q7 _ ?osd.65: osd_recovery_op_priority = '10' (unchangeable) 7 S4 \2 F) g9 F
osd.66: osd_recovery_op_priority = '10' (unchangeable)
* m T8 |, d M- ?0 fosd.67: osd_recovery_op_priority = '10' (unchangeable)
# P4 T# U/ E6 josd.68: osd_recovery_op_priority = '10' (unchangeable)
4 E9 o$ t' J, ^- w. H$ _osd.69: osd_recovery_op_priority = '10' (unchangeable)
) `3 j9 u: b) F. N+ Q. x$ }osd.70: osd_recovery_op_priority = '10' (unchangeable)
1 p$ }2 o4 C; b' Dosd.71: osd_recovery_op_priority = '10' (unchangeable)
% c& @# ]5 I5 _2 v0 Qosd.72: osd_recovery_op_priority = '10' (unchangeable)
+ z+ R D2 ]7 i1 L Rosd.73: osd_recovery_op_priority = '10' (unchangeable)
. N4 L; r% B T( p; c) E/ S- j, u. Nosd.74: osd_recovery_op_priority = '10' (unchangeable)
7 B( J Y0 o0 D9 n% @0 l3 B+ A: M+ O% Cosd.75: osd_recovery_op_priority = '10' (unchangeable) : S d3 t: v& C, x$ I% v. T
osd.76: osd_recovery_op_priority = '10' (unchangeable)
- O4 Z3 T( h- [! r; ?osd.77: osd_recovery_op_priority = '10' (unchangeable) 7 L8 g- }8 G7 E( l, V
osd.78: osd_recovery_op_priority = '10' (unchangeable)
; `0 l3 a1 T1 E0 ^) losd.79: osd_recovery_op_priority = '10' (unchangeable) # m- a/ P0 ~% o' \4 T$ y2 e
osd.80: osd_recovery_op_priority = '10' (unchangeable)
# ]! D3 E h- T1 \osd.81: osd_recovery_op_priority = '10' (unchangeable)
/ M* P% p! I }+ }6 Kosd.82: osd_recovery_op_priority = '10' (unchangeable)
4 q8 g4 l% H& C8 n7 Eosd.83: osd_recovery_op_priority = '10' (unchangeable) ! t! \- T3 a; T' Z9 m
osd.84: osd_recovery_op_priority = '10' (unchangeable)
$ }9 n* v- j1 M7 N. h+ ?8 Z9 v5 U* gosd.85: osd_recovery_op_priority = '10' (unchangeable)
+ w; I- ?- ?- O& }# K+ eosd.86: osd_recovery_op_priority = '10' (unchangeable) # _; F4 I) S8 g8 O8 A. ~
osd.87: osd_recovery_op_priority = '10' (unchangeable)
( o' e% L' Q; Yosd.88: osd_recovery_op_priority = '10' (unchangeable) $ t+ [$ k8 h, ~1 k4 Z' Q( F
osd.89: osd_recovery_op_priority = '10' (unchangeable)
( ?& s3 x: j7 y& w2 A4 Losd.90: osd_recovery_op_priority = '10' (unchangeable)
& B* N+ Z; t$ `7 Losd.91: osd_recovery_op_priority = '10' (unchangeable) % T! G/ G+ O( w4 R* i
osd.92: osd_recovery_op_priority = '10' (unchangeable)
/ R5 \& Q! `1 mosd.93: osd_recovery_op_priority = '10' (unchangeable)
- I. |( z$ l# A% u8 r7 zosd.94: osd_recovery_op_priority = '10' (unchangeable) " s! Z( g& r0 T# z2 N2 B
osd.95: osd_recovery_op_priority = '10' (unchangeable)
* r' }+ s& j' v6 r' I: P) oosd.96: osd_recovery_op_priority = '10' (unchangeable) % l- j: t* r# e1 G" L
osd.97: osd_recovery_op_priority = '10' (unchangeable) & p2 Z" U6 g" m/ A6 m) [
osd.98: osd_recovery_op_priority = '10' (unchangeable)
- ]$ k# [' s. sosd.99: osd_recovery_op_priority = '10' (unchangeable) " z( A! a @1 r R( o
osd.100: osd_recovery_op_priority = '10' (unchangeable)
5 [' ]' k J: ]- {. K/ K$ Y, Rosd.101: osd_recovery_op_priority = '10' (unchangeable)
* x: f6 E' J) v9 O( S" ~" Q% Nosd.102: osd_recovery_op_priority = '10' (unchangeable)
# S( S: k3 B- S! Nosd.103: osd_recovery_op_priority = '10' (unchangeable) & X+ ` N: D/ O7 M. D
osd.104: osd_recovery_op_priority = '10' (unchangeable) W- M/ {$ c, c# b
osd.105: osd_recovery_op_priority = '10' (unchangeable) / F m* n- T2 s! S. D
osd.106: osd_recovery_op_priority = '10' (unchangeable) 6 [ J4 b. h) G$ ~6 a. Z
osd.107: osd_recovery_op_priority = '10' (unchangeable) % @# e5 f; d) d- f: X
osd.108: osd_recovery_op_priority = '10' (unchangeable) 3 S; ?. k1 B- @ ?
osd.109: osd_recovery_op_priority = '10' (unchangeable) , D) Q& D2 R5 v* I' t
osd.110: osd_recovery_op_priority = '10' (unchangeable) ! ?. l ^& V& L, _# v
osd.111: osd_recovery_op_priority = '10' (unchangeable)
# N+ n+ @; i2 n$ `osd.112: osd_recovery_op_priority = '10' (unchangeable)
; y8 y" L5 @$ k& r' d. posd.113: osd_recovery_op_priority = '10' (unchangeable) $ k& i0 I( _, {6 I
osd.114: osd_recovery_op_priority = '10' (unchangeable)
) z$ r, l" ^$ Aosd.115: osd_recovery_op_priority = '10' (unchangeable)
8 i% s1 b8 T7 F$ b4 | Nosd.116: osd_recovery_op_priority = '10' (unchangeable)
# Y9 r& c! |8 x! D7 i3 l i7 t3 r3 xosd.117: osd_recovery_op_priority = '10' (unchangeable) - o9 Y8 g4 y' a: {. C5 v
osd.118: osd_recovery_op_priority = '10' (unchangeable)
; R4 S% _, ?: x" y5 T6 m Zosd.119: osd_recovery_op_priority = '10' (unchangeable) # o; j8 W( k7 p, l
osd.120: osd_recovery_op_priority = '10' (unchangeable) 9 h: T& O% `5 h W
osd.121: osd_recovery_op_priority = '10' (unchangeable)
+ x+ @* v% a* N1 d6 h, `osd.122: osd_recovery_op_priority = '10' (unchangeable) 1 b1 O; F& `; D4 r4 N5 Q1 D7 n
osd.123: osd_recovery_op_priority = '10' (unchangeable)
( j3 [; R/ s9 V3 pError ENXIO: problem getting command descriptions from osd.124# b1 }5 |: z( U) X* y1 o
osd.124: problem getting command descriptions from osd.1243 F. u( J e5 N
osd.125: osd_recovery_op_priority = '10' (unchangeable) % N0 Y T+ ?) |
osd.126: osd_recovery_op_priority = '10' (unchangeable)
{. n$ M( m2 U) O5 a5 a: gError ENXIO: problem getting command descriptions from osd.127
* \5 \- D% c/ `1 Eosd.127: problem getting command descriptions from osd.127 d$ Z2 @' b- N4 ^% R4 M9 u0 i7 N" @
osd.128: osd_recovery_op_priority = '10' (unchangeable) $ I( {2 n# C8 p/ x- n$ ^, F' p
osd.129: osd_recovery_op_priority = '10' (unchangeable) 0 V4 A$ K/ n+ S' d. q
osd.130: osd_recovery_op_priority = '10' (unchangeable) " V& ]6 N X& h" o: G3 {
You have new mail in /var/spool/mail/root
9 N: b/ Z+ I$ O( @
7 i2 Y; q! D4 K' S所有的osd都修改:; f5 s* f( v5 S _2 h
+ S4 v& e' s: |0 \5 `执行过ceph pg repair pg-id+ P& f2 S- Y' ~, Q
4 \5 r6 C, ?5 j: ~7 k
8 F7 p$ k7 V. u状态出现修改过程:: R% r. } V& A
[root compute1 ~]# ceph s
0 j# N, r L* \0 T cluster 70d27aec742e4a95b000-cf37ebba35d0
( W6 B! f. [$ O& q3 T2 B7 j health HEALTH_ERR
* f& e8 _/ h3 c/ r2 Z. q 1 pgs inconsistent
" E" W$ o, Z4 D2 }7 u% p 1 pgs repair6 m! s: C: F/ Q, o1 D* t
1 scrub errors) T: R4 q! p# ? Q! P
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}% o* G' W- L- K
election epoch 332, quorum 0,1,2 compute1,compute2,compute39 r6 A4 ~( }1 A2 Z, ]
osdmap e26725: 131 osds: 128 up, 128 in
, Z: L# j5 g$ }5 N8 z y; V* E flags sortbitwise,require_jewel_osds3 g/ G; E0 I$ G' n- C
pgmap v169370679: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
4 h/ I! _' X8 y' S 309 TB used, 155 TB / 465 TB avail6 M3 A9 P1 _6 O0 M
10229 active+clean
- ]. e! I$ s( r \ 9 active+clean+scrubbing+deep) {8 ~2 a3 @4 {! ~5 n" E+ q' W5 m
1 active+clean+scrubbing7 G: M# u' a" o. m1 q! W
1 active+clean+scrubbing+deep+inconsistent+repair
$ R/ K2 L: R0 r& p6 C' {! k" v client io 91086 kB/s rd, 67615 kB/s wr, 3119 op/s rd, 3569 op/s wr k% t, f R+ o9 L' h. M
[root compute1 ~]# ceph s
0 |4 A# I- A9 A" I: |: u9 @ cluster 70d27aec742e4a95b000-cf37ebba35d0
* B' r/ j4 ~0 D' ^& {' { health HEALTH_ERR5 ~$ d& N% p3 @3 R8 ]0 N, V3 C
1 pgs inconsistent% _3 a S0 C/ w4 S$ `* b! ~
1 pgs repair- v! [3 G8 z3 @1 K! A% g! ]
1 scrub errors
* h. d* ?2 ?% x5 A monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
S4 n6 S9 ^, X* ~) g election epoch 332, quorum 0,1,2 compute1,compute2,compute3
9 [" v, {" e5 S d' E) }" x osdmap e26725: 131 osds: 128 up, 128 in6 L7 d/ j8 [, }
flags sortbitwise,require_jewel_osds9 ~) M0 D: T. e9 l+ m4 ?
pgmap v169370681: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects+ V0 E5 [8 ?- q; c7 ~
309 TB used, 155 TB / 465 TB avail' E6 e7 ]4 z: y- A: y% y
10230 active+clean5 O0 }. w& J, [, a% a
8 active+clean+scrubbing+deep
5 Z& Z7 U/ M& F& \+ | 1 active+clean+scrubbing
3 u* x* W5 A5 w! _! W0 h 1 active+clean+scrubbing+deep+inconsistent+repair
- Q: f6 w% G9 v6 f# n' T0 v client io 104 MB/s rd, 67033 kB/s wr, 3479 op/s rd, 5013 op/s wr
" R" y: |9 T* }; k[root compute1 ~]# ceph s% U+ m' C% h) ?% r
cluster 70d27aec742e4a95b000-cf37ebba35d0
5 w% h6 j& x) h; c) c& O$ [ health HEALTH_ERR
4 i s% F; O7 |' H6 L6 r# I- y 1 pgs inconsistent
$ n0 p" N; A9 D( P8 U8 _+ } 1 pgs repair0 f( @/ ~5 p( `5 g2 V6 \+ ]
1 scrub errors3 ^+ Y9 F. u7 @& m
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
! @! X$ _; G& R+ ?+ D; } election epoch 332, quorum 0,1,2 compute1,compute2,compute33 R2 y6 L6 n& t9 Q/ [( u
osdmap e26725: 131 osds: 128 up, 128 in
: V$ l& ~5 }0 @6 {7 P8 J* T6 H flags sortbitwise,require_jewel_osds
+ G- }- w# n% _# q' M; ~, x0 D0 i pgmap v169370783: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects1 w" l! Q8 n7 g8 c* L6 ^
309 TB used, 155 TB / 465 TB avail
, M: ~2 [, B; b% q, V @1 _/ }' h 10232 active+clean
+ i, F/ X- {! n 6 active+clean+scrubbing+deep
) E0 k; H0 l! Q7 @5 ]1 R3 K 1 active+clean+scrubbing8 l! s7 Z- e6 k l# A
1 active+clean+scrubbing+deep+inconsistent+repair9 P; X3 }# ^4 F2 z I* Y
client io 147 MB/s rd, 57692 kB/s wr, 4584 op/s rd, 5944 op/s wr
j9 e& T3 f& B/ u$ L+ |You have mail in /var/spool/mail/root
+ ?/ z3 \/ P% H( V) A[root compute1 ~]# ceph s0 |! ?3 ]$ M8 i! h: c' H h0 v
cluster 70d27aec742e4a95b000-cf37ebba35d0/ h8 |3 d6 Y4 t, P' M: N5 f9 @
health HEALTH_ERR
+ W) Y/ W( ~& u& m6 y2 s+ ^ 1 pgs inconsistent
' V& e' u# K; A1 @& d4 ? 1 pgs repair
/ ~' y [. p z, r$ X, u0 h, ] 1 scrub errors
" x. S0 F8 z! a monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
+ ^+ \* V/ t) [- P% G election epoch 332, quorum 0,1,2 compute1,compute2,compute3
0 b0 G0 h& [0 k# G- [ osdmap e26725: 131 osds: 128 up, 128 in( L" M, }3 {1 `3 K
flags sortbitwise,require_jewel_osds
$ l( u' X$ j* a+ z pgmap v169370785: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects! h" s; {/ A o7 E% z
309 TB used, 155 TB / 465 TB avail
7 [. A$ F8 C/ y' W. N 10232 active+clean
; F& N9 R( u2 | 6 active+clean+scrubbing+deep
3 |0 g- i+ Z+ r; h( X6 o: t) g 1 active+clean+scrubbing
/ l- a. ~% r1 r 1 active+clean+scrubbing+deep+inconsistent+repair
. B1 w3 `6 s0 C& U- G& i4 J8 H( A client io 104 MB/s rd, 49581 kB/s wr, 2830 op/s rd, 3282 op/s wr e4 b/ D1 k! v; m: C
[root compute1 ~]# ceph s! s! S! A9 Y8 x
cluster 70d27aec742e4a95b000-cf37ebba35d0- ?' d9 f1 Q; ~6 c, r0 c
health HEALTH_ERR
* b/ Q# b. L+ e, ^% _! b; j& g4 _ 1 pgs inconsistent
5 e4 ]& J0 {( B, x 1 pgs repair
! M; {" q& y) c7 l 1 scrub errors
$ \- j* }& r3 M. S0 G monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
" z8 U0 `: E' t2 L# f* l1 L9 H election epoch 332, quorum 0,1,2 compute1,compute2,compute3
0 P% }& B& X. M* Q$ ^1 I osdmap e26725: 131 osds: 128 up, 128 in
$ ^6 `# |3 A* k$ f q flags sortbitwise,require_jewel_osds
1 E+ c7 q0 u3 q, y# V pgmap v169370786: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
9 [. H& O0 O( r- h3 U 309 TB used, 155 TB / 465 TB avail1 Q6 m: O* _# V9 p0 L) j
10232 active+clean) y3 ?- M! a# K+ S' w
6 active+clean+scrubbing+deep: \( Z2 |, h6 ^, A+ c6 F' e" l: v% I
1 active+clean+scrubbing) C+ L. G# Q) f- H+ k$ @4 B+ H, V$ p
1 active+clean+scrubbing+deep+inconsistent+repair
+ j8 n, A' N( U/ T( _) r4 L6 X client io 95762 kB/s rd, 44372 kB/s wr, 2258 op/s rd, 2660 op/s wr' {/ M& ~0 @1 v7 w [# V6 r
- R( t6 @6 i+ s: ~: n0 z9 E; Q6 I! [6 Y+ {; w# m1 f$ r' v) S
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills"
) p* G6 ~3 V* s- F1 d* @ "osd_max_backfills": "1",) r0 s8 H3 X& } s+ d4 Z
"osd_recovery_max_active": "1",' a, K b: C: f; `( {
"osd_recovery_op_priority": "10",3 `* r$ I+ F3 E! L& M
[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" , }8 e% t) O2 |
"osd_max_backfills": "1",
( O. f' q/ x4 c( F+ d% Y: r "osd_recovery_max_active": "1",
; _2 B) w2 t w "osd_recovery_op_priority": "10",
3 x( @' [% f: V[root compute1 ~]# ceph s% Z7 t: }$ k% S3 b' s
cluster 70d27aec742e4a95b000-cf37ebba35d0
' h9 W; R5 z5 G) I. d6 x$ t health HEALTH_ERR, K* N& } {. W; [+ n* a m
1 pgs inconsistent
3 J+ [. v t$ y) F/ U 1 pgs repair
. t ^/ d# H6 u" w! e- c+ k 1 scrub errors
7 g( b; x& [* U6 K monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}3 d9 ]& W* ^- W& z: A# g
election epoch 332, quorum 0,1,2 compute1,compute2,compute3
$ U+ |7 |2 j0 h9 @7 o osdmap e26725: 131 osds: 128 up, 128 in* G: S2 n8 H( l$ D; A
flags sortbitwise,require_jewel_osds8 F# j, T* F$ D6 P1 z8 Q+ @" M9 i
pgmap v169370799: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
{# i) z4 p+ ^# [" i 309 TB used, 155 TB / 465 TB avail# K+ h% m0 {" c! d9 f* V: _% t# g
10232 active+clean
$ l3 H3 K# _9 {8 M 6 active+clean+scrubbing+deep
% w5 v; k5 g0 Q2 D; K( C# r 1 active+clean+scrubbing6 l' G- |# e6 ~8 m9 ~7 ]3 _1 I
1 active+clean+scrubbing+deep+inconsistent+repair6 ^% g* N t o# y7 p/ D5 M2 h: X
client io 134 MB/s rd, 117 MB/s wr, 3865 op/s rd, 4949 op/s wr
0 p5 {0 @! j$ `. E* S
) N/ e0 _; f. V% {8 [" \! X
$ w% n6 ]+ g0 n[root compute1 ~]# ceph daemon osd.0 config show |egrep "osd_recovery_max_active|osd_recovery_op_priority|osd_max_backfills" ' n' ?& H/ q( n% ?6 w
"osd_max_backfills": "1",
8 }$ E' o0 w" s t1 J9 ` "osd_recovery_max_active": "1",
+ V! A# D# j/ n" a8 I1 t- ^ "osd_recovery_op_priority": "10",
$ K4 E- p5 k" u7 V
5 _! W) N9 N! l C; \1 g5 H$ k( [8 _ g2 |3 z
# a( s* u( a8 M `
[root compute1 ~]# ceph s* A: e6 E0 P# n# f
cluster 70d27aec742e4a95b000-cf37ebba35d0
) W9 ?: D+ U4 r health HEALTH_OK
" j+ h7 m$ B4 R. O( B5 E7 I% e monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
* S' V4 P$ z1 a election epoch 332, quorum 0,1,2 compute1,compute2,compute36 |5 ^ l6 l, d/ K* @
osdmap e26725: 131 osds: 128 up, 128 in, p3 m9 G. h, k4 v1 I
flags sortbitwise,require_jewel_osds
- P) g7 |7 @; B6 m" z& B$ { pgmap v169370916: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
. q9 I2 |# \ c6 E8 }! q) h( } 309 TB used, 155 TB / 465 TB avail+ c6 t2 a/ E! j: v8 h$ r
10233 active+clean
1 }6 b5 g* y5 a6 D3 z# n 6 active+clean+scrubbing+deep
. v7 d9 k8 `! U; u8 d 1 active+clean+scrubbing
) J; F$ g' q# f9 H* z; |$ N; m client io 195 MB/s rd, 234 MB/s wr, 4404 op/s rd, 5993 op/s wr
6 I4 @! h' s- N6 pYou have mail in /var/spool/mail/root: e0 `9 d2 h. [
0 h* u" \( M* K( N; P[root compute1 ~]# ceph s$ }, m8 r" V; e3 T4 b/ J; |
cluster 70d27aec742e4a95b000-cf37ebba35d0 r: X) C( E9 s1 [2 l* `0 F
health HEALTH_OK* @1 _, ^9 s- \! ]1 q( G
monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
9 ^; J8 a7 }' w M election epoch 332, quorum 0,1,2 compute1,compute2,compute3
" A2 e" A! Q" n0 E osdmap e26725: 131 osds: 128 up, 128 in4 l/ g( `6 A1 {5 A
flags sortbitwise,require_jewel_osds6 y' c! U9 P x/ f
pgmap v169370921: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
1 m5 R# b2 ?" U+ \ 309 TB used, 155 TB / 465 TB avail
. w+ n, v; }2 m& A3 Y 10233 active+clean
1 U9 q2 y" z! k1 P( X 6 active+clean+scrubbing+deep
; J9 e( Y( O# ?- x$ L! N$ I 1 active+clean+scrubbing
* T# t9 A# ^6 Z0 c9 L8 A( } client io 257 MB/s rd, 358 MB/s wr, 5025 op/s rd, 6587 op/s wr5 ]; h' z' U! V( D, y; X2 w& S: M$ ^5 u
* q8 m, T% A) K& c恢复正常:
7 u. k* W, c9 W; ?[root compute1 ~]# ceph s0 k* a- M8 r% L% J
cluster 70d27aec742e4a95b000-cf37ebba35d0
6 s V% c2 V3 I2 m o9 p# f health HEALTH_OK
) h* [6 D9 E/ o. O* ]- `7 K monmap e3: 3 mons at {compute1=176.12.132.5:6789/0,compute2=176.12.132.6:6789/0,compute3=176.12.132.7:6789/0}
( a4 i! ^/ J b: e# j% q W election epoch 332, quorum 0,1,2 compute1,compute2,compute37 S6 ^& W5 d. _* Y X1 Z
osdmap e26725: 131 osds: 128 up, 128 in- C( d' `2 R5 I& N2 p
flags sortbitwise,require_jewel_osds/ r* L3 @2 c5 j5 g) o
pgmap v169370922: 10240 pgs, 1 pools, 103 TB data, 27242 kobjects
; l4 u/ a/ v' N 309 TB used, 155 TB / 465 TB avail0 R# w( o h& @, j. B7 _! W4 h! Y
10233 active+clean
/ s7 g8 A6 c Q7 t7 o 6 active+clean+scrubbing+deep
; H' [. H4 }+ S; m# C 1 active+clean+scrubbing& } ~$ G4 ^( @0 t' Z, E
client io 254 MB/s rd, 390 MB/s wr, 5522 op/s rd, 6826 op/s wr
$ r2 |4 f/ T. ?& a* Q6 q4 a2 }3 s0 R
+ |" h$ H l# y' O; N5 N) }6 w' ^3 l7 B$ K* Q! p! Q
|
|