|
|
cluster:
* W/ }% M* I C4 F% a[root@compute01 src]# ceph -s7 a3 y1 f0 N0 C! F. B0 m+ Q0 L) A
cluster:
r& h! s1 ]! U) y I1 ]/ h id: 31403b11-8a1e-432f-876e-5a2c852f9dcc7 x$ f- r/ I9 j1 |
health: HEALTH_WARN) ]0 p# d; K4 n
Reduced data availability: 640 pgs inactive
* O% l* }( B! D2 u) J, E ) N: v Z9 R- H o/ _7 u
services:1 }2 l( X/ r& `7 [ o6 V: G! R
mon: 3 daemons, quorum compute01,compute02,compute03 (age 42m), M; ?; N+ T1 f3 r$ q* G- [
mgr: compute01(active, since 42m), standbys: compute02, compute03# n# A3 s9 H+ u1 j" Q0 l1 a( `
osd: 3 osds: 3 up (since 26m), 3 in (since 26m)
5 u2 @1 U+ Y! G' m" }& |6 a
0 r$ Q* m( F$ N$ O3 J% O7 v, f data:
8 L8 w( v; ^$ s4 j pools: 6 pools, 640 pgs
) q$ |" Q0 A! ~ objects: 0 objects, 0 B E. [( x. V! z- |1 X' p4 g
usage: 3.1 GiB used, 3.3 TiB / 3.3 TiB avail8 L+ Z3 N; i9 a0 T- t! u/ d/ p: N) `
pgs: 100.000% pgs unknown
3 ]- ^. D/ b% e) _7 X% V 640 unknown
* {$ R4 Y3 Y$ R% D7 I: O0 b4 L0 a0 e! {, ?) r" O/ k$ N
遇到问题，一直处于这种状态：( f( }4 O' p& U5 F& D( ]2 w, S3 ~
! e T! J) t1 g; t7 @
导出文件：6 b" v: X. ^; D9 P4 i
1 ^; u M- }0 v Z7 R
[root@compute01 ~]# ceph osd crush tree& Z, l I$ z0 E# m
ID CLASS WEIGHT TYPE NAME ! n# o& v2 {7 k" x# x: ^% D
-1 0 root default
4 o. d# E$ l. t4 @
1 H. w0 F* l: J9 W# m1 K& U/ [& {6 _* k
发现什么都没有，缺少东西
5 i' V1 x' I* f/ i6 \7 o; O9 \8 ?
[root@compute01 ~]# ceph osd getcrushmap -o /tmp/mycrushmap, I9 d! `; J& ]
12
5 _8 ?8 }' u, d9 e9 J4 F导出的数据只有12行，少了很多。& b* r" H- V6 b. l
, a- W# }) G( n A
, F0 g0 [8 v5 M! Q' U转换成可以读的文件：
4 @9 ~6 z! k# j, }5 q. u
% Q( A2 \' w; k0 x4 W r[root@compute01 tmp]# crushtool -d /tmp/mycrushmap > /tmp/mycrushmap.txt
$ n( f4 @' R* A& N% y( g! |: n
. k- \+ w# P1 }) I) u' g, y[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2& @" f. p+ |3 x0 ^. ]+ i) G3 E+ e
item 'compute01' in bucket 'default' is not defined
, I9 J9 ?$ C/ g1 K9 f4 h% p[root@compute01 tmp]# vim mycrushmap.txt
3 a- {& b4 W f6 Q[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2
3 ]1 Y. U: R! d+ R转换的时候发现缺少东西；, F: J- o+ i# R! q9 \" u
再次编辑：
9 Y7 K+ \( E6 |' O4 X" c7 ?5 G. K[root@compute01 tmp]# vim mycrushmap.txt " v4 q6 B! t$ q$ O& r( T
& {# J: \% p) N3 G, P' o- v: B: F
* C0 ]5 U" V; l J& X Z' ^% _- h1 w
# begin crush map
( ]4 d _0 r0 b% l9 qtunable choose_local_tries 01 v! I, M6 Z4 H8 o2 A
tunable choose_local_fallback_tries 0% c& s2 p0 {0 I5 R) b
tunable choose_total_tries 50
) m2 D; o3 E* W" Otunable chooseleaf_descend_once 1 O, K1 W. h. I8 y7 d$ k
tunable chooseleaf_vary_r 1
: N/ P. \: z3 \+ F/ p6 ctunable chooseleaf_stable 15 W z7 M. L, r5 ]( ]7 U$ S5 a' Q
tunable straw_calc_version 1! I5 ~, H4 G4 ?2 ~+ i# X2 G
tunable allowed_bucket_algs 545 K2 l& U, b# Y' |. X9 d$ I8 F4 ~! z
# devices
% R$ ~+ F8 d1 E& R% Z2 xdevice 0 osd.0 class hdd( T. o. O( L. S, [
device 1 osd.1 class hdd
# P' Y) V3 y6 D& Q1 T' odevice 2 osd.2 class hdd0 y) c' e4 k5 b- e9 J
# types1 R5 K9 \) x k# x
type 0 osd5 T$ Q$ R. ` m5 [1 z
type 1 host
4 r4 G' o: w7 b2 }5 \$ o$ ^- G9 ptype 2 chassis$ J: f! G& Z$ o, `1 j
type 3 rack
7 r4 \5 K5 r; a$ Z ytype 4 row0 p$ t. _" H z, c
type 5 pdu% U/ z; i! e- t L5 H# L% H; y' G
type 6 pod" F5 W# L( j; w* D* Y
type 7 room
. O1 f' L6 O3 s7 [3 A! x* ~, Qtype 8 datacenter) m: ~' w! {" ], ~; ~. A7 f3 b
type 9 zone4 e C# ^8 M$ _2 U/ o1 n! {- u) m
type 10 region7 Q: b; G$ W4 J. N9 M/ ^
type 11 root0 y6 K" @( V" G- g1 P: V
! Z6 r( [" ^0 J A9 c. o( }# buckets. g3 f2 \1 V* D
root default {$ {6 D4 A4 x. V( ~: I3 G
id -1 # do not change unnecessarily
- B( p1 q& b9 U+ d id -2 class hdd # do not change unnecessarily
8 A; O: ~9 J3 E2 d, Q0 b7 u7 f9 H # weight 0.000
3 ]* m( x1 n, C# x1 u alg straw2
% o5 F& v3 a/ t; m5 _2 x hash 0 # rjenkins1
1 N$ `% D, H! |, \
1 u4 z# g! r7 t9 P5 A}
) \$ p! A4 w. n# rules) l! |. t) d$ D4 R4 \/ e2 V
rule replicated_rule {
" r8 Y( {! ]3 T0 h5 s3 @ id 0
# H& [ k1 w1 n$ } type replicated& n+ [0 g4 [1 d/ m3 a
min_size 11 z2 i+ r! f2 e( ? V6 ^/ @, ^# T
max_size 10$ O% L" f8 K% |& J' f$ {8 E
step take default
2 Z3 [" g3 w6 C4 T% a step chooseleaf firstn 0 type host4 ^# {% c3 i8 n# r" N8 o
step emit
9 Z: P; i7 B2 t) N. X; {; M}$ l; M1 e+ N* @: W
# end crush map
z. d( c+ F! |$ \& `8 [) K% L# b$ n' Z& f
. Y! t8 v7 T, R6 G$ [7 j
发现少了很多东西，添加上吧：
' e- i, ]( u2 y( u7 c$ a
0 @. S1 x- J/ n# begin crush map' l0 P$ q7 x8 j6 o! N7 [; q# N/ \5 h
tunable choose_local_tries 0
3 H9 Q7 ?: W, t9 H9 Jtunable choose_local_fallback_tries 0
. l5 p# O! I5 V7 Rtunable choose_total_tries 50
9 @6 n$ J- W3 z' n H2 l: Vtunable chooseleaf_descend_once 14 F' r: O1 w" x6 t! i; s% \
tunable chooseleaf_vary_r 1& K# `# q8 w, }
tunable chooseleaf_stable 1
$ y- N( D- V5 f' w; G; y' W1 p: Otunable straw_calc_version 1
1 ~) J* M, Y: n& s. S4 x& {tunable allowed_bucket_algs 54( i+ M" q7 h& I7 |
# devices; b' v5 |; T& K( Q$ f* r0 u
device 0 osd.0 class hdd; R" b8 w0 e4 }# ]1 x' l) ~* G8 { v
device 1 osd.1 class hdd
4 v) J# N& o/ W/ W/ J' @+ Rdevice 2 osd.2 class hdd- f% d' z- c6 j' x$ E& S; J% ?
# types
& V3 t* y2 C H3 ` Rtype 0 osd
2 w- E4 Q3 p. I5 c: F8 b N! |# Mtype 1 host
+ V7 c5 G; I( O" O1 Q' \type 2 chassis
' j0 Q7 u. J* c! W0 ?( Itype 3 rack
; J& y4 N$ D: w$ i& itype 4 row
, s- k: H8 d6 Ltype 5 pdu
7 M1 q. M% O- utype 6 pod
. l0 o' @/ D- y8 ?7 ltype 7 room
9 m$ r" y. X1 Ktype 8 datacenter( |" t' l: A9 T% o7 E; _
type 9 zone/ K9 p/ T, p) k# f1 Y
type 10 region# s+ j3 P% l( A& H6 |% n/ P2 s
type 11 root
% z$ d% t1 F1 Z0 z4 D2 rhost compute02 {
+ _' X2 F, g! R" h& e" ] id -3 # do not change unnecessarily9 D* E4 t S" g
id -4 class hdd # do not change unnecessarily
) |0 p( _5 B, Q9 x" e7 _ # weight 1.0000 R. Z( c( v; Q" b1 K* Y
alg straw2
: L [1 F. i. p$ F1 h hash 0 # rjenkins1
8 T! s/ {4 G$ ~7 g8 D) w+ u item osd.0 weight 1.0007 G0 X: }/ x4 A0 S2 |" ] d9 Q
}
( _* `- H. @9 nhost compute01 {/ m+ g1 t% M" k% v, D+ K
id -5 # do not change unnecessarily
( P( i0 D2 N4 l8 C& N id -6 class hdd # do not change unnecessarily
( {0 M6 k9 Q' x `0 o( } # weight 1.000+ P% |; O* Q& K
alg straw2
8 j* h# V1 M0 J hash 0 # rjenkins1
9 q6 S8 Y( `2 X2 v item osd.1 weight 1.000/ v, _8 z) ?0 o; t$ z7 c6 |0 n
}
. `; j8 Y+ ]/ Q. f8 Q; B/ @( I; rhost compute03 {
, P+ D$ d6 U) Z! z$ H id -7 # do not change unnecessarily
( p4 T T- ]5 F id -8 class hdd # do not change unnecessarily, q* D8 W. i4 O+ x. I1 u& `1 H4 P
# weight 1.000
. o( I% d% B9 [- O9 {: b: e alg straw28 [( t/ `& D _: r! o6 ~! f
hash 0 # rjenkins1* y, X4 l& _9 H% _- Y" [
item osd.2 weight 1.000
$ s+ C' E( r+ S: ^. _1 V S) A}
) A( q. ^2 _7 q, ]! @7 r$ l- \# buckets
4 ]1 n! P% n0 e* Droot default {
- J+ r2 `; h+ c id -1 # do not change unnecessarily% {9 h5 b# V$ W) Q
id -2 class hdd # do not change unnecessarily
2 N2 A2 l& O( e1 ~4 x N# N # weight 0.000
9 u. z/ W/ n: ?+ n* h' x5 _. D alg straw2
. C6 Y0 L$ |" r- c- O$ w. `2 Z hash 0 # rjenkins1
+ P9 ]* ]5 B* I" R item compute02 weight 1.000
( w; l& v0 G4 f, [0 ` item compute01 weight 1.000/ j9 Z+ Z$ a" t) O' O. q% O9 u. ~
item compute03 weight 1.000
& y% o/ ]+ a/ u3 C}
9 q0 Q: c( a( ]# rules
2 Z& n. G6 S* } W( u4 orule replicated_rule {& [8 h) P# T0 H
id 08 W# j( o! R9 x/ j$ |; m
type replicated+ L( ^+ Y6 q* D$ c4 {1 @
min_size 1
% u8 x5 T$ K2 z% x6 T max_size 109 f% f/ n0 N8 `# e3 D
step take default1 t8 d9 A! B0 |$ J8 c) g+ x
step chooseleaf firstn 0 type host
) e6 ?+ B6 p5 y! f9 y step emit
2 B6 i6 e+ ~) H. b0 {/ q} T) W" R9 a, D h/ [. W
# end crush map' x( R6 P( G k) p4 {
e# d) ]1 ~/ c2 b x; \# H1 |8 | c! d8 Z g" Q- d
添加好之后，检查下对应关系，因为ceph节点和osd添加顺序的问题，导致1节点和2节点颠倒了，要注意这个地方，其他忽略；: n, q9 z, [8 [$ P' D& a7 q8 H
转换成ceph认识的文件：: D7 P, L, L8 n+ Y
[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap28 t6 N* v' k4 k& s. h+ H& J9 j
! R( u0 }6 }0 {+ \+ K, O. O$ p# a[root@compute01 tmp]# ceph osd setcrushmap -i /tmp/mycrushmap2
* S& i2 r M7 N: T13
* Z0 ?) I6 m; [1 K[root@compute01 tmp]# ceph -s
* {5 \9 c! |' A+ M% H, j% U cluster:# [7 `6 `6 u0 `$ P* u% {0 J. Y( [2 v
id: 31403b11-8a1e-432f-876e-5a2c852f9dcc; K( \; b& i8 D0 u
health: HEALTH_WARN
2 O: ^1 ~9 u$ w9 I6 l Reduced data availability: 212 pgs inactive
; t8 e. S( K" V8 A$ \0 R7 ?' o$ ^
* [. M) t8 e3 q8 I3 z4 y s' ~8 u services:
2 y {& m [5 Q+ b1 f7 w; u mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)2 q% S, V Q' {! U9 }' `) G
mgr: compute01(active, since 56m), standbys: compute02, compute03
N! M `0 B: j! A! r7 _, q osd: 3 osds: 3 up (since 40m), 3 in (since 40m)
- u9 g: l6 q* \ , q ^( p& v/ K2 V0 [& ~ ?2 o
data:
' Z( v' D& S# h; e+ ] pools: 6 pools, 640 pgs5 Z0 x# d1 h$ b9 s5 O1 Y# \. ^4 k
objects: 0 objects, 0 B; v% p( k8 f: [
usage: 3.1 GiB used, 3.3 TiB / 3.3 TiB avail" k, |$ `1 {3 s9 }/ B* H9 {
pgs: 33.125% pgs unknown
' v; {5 \1 p5 Y* w' P2 s* J+ p 428 active+clean
6 z2 w" {& c( b; D) g# q 212 unknown$ F. ^4 D8 t* C1 h2 |3 ~8 M7 b
% N. N% D; k# I. E+ \[root@compute01 tmp]# ceph -s
( _# B. M7 s) z cluster:6 t, I, J, V! E% F% u* i
id: 31403b11-8a1e-432f-876e-5a2c852f9dcc$ `- q6 k$ o. i( C
health: HEALTH_OK- o5 W6 m' e+ \, b1 w' P; i
- w% E4 y, Y$ V
services:" s* g N; O) t8 v! T+ q
mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)3 n# j2 g8 r: \* B5 I! e( x
mgr: compute01(active, since 56m), standbys: compute02, compute03) ^7 F' y3 f X- _) L
osd: 3 osds: 3 up (since 40m), 3 in (since 40m), q/ |6 W( z# y; ~( Q c
' A5 H" T. @0 G2 C2 `
data:
% y% p& z0 e4 G: v) x4 B( W pools: 6 pools, 640 pgs
& Y& |$ @ q8 W. }7 N2 |8 q; m: }3 \ objects: 0 objects, 0 B
2 j& l' p9 R. p& T: J& R usage: 3.1 GiB used, 3.3 TiB / 3.3 TiB avail1 d/ s- S3 g( K+ O0 t2 \: E" N4 V( k9 v
pgs: 640 active+clean3 d/ F# r' e5 u$ a
J/ c6 U+ J/ O* E[root@compute01 tmp]# ceph -s i7 f! j' V; p: ~% y. Q
cluster:
4 z; Z) v& B& R8 g0 H id: 31403b11-8a1e-432f-876e-5a2c852f9dcc% i$ P: o: M) h- ~7 e' }6 _$ i
health: HEALTH_OK2 l6 p3 ?8 G! E
% r% U3 r: v( x' ~+ F services:
3 [ G; ^& K. R5 b mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
8 z8 l1 q5 B2 V3 w* { mgr: compute01(active, since 56m), standbys: compute02, compute03
2 H5 |. l0 m Y' j! E osd: 3 osds: 3 up (since 40m), 3 in (since 40m)
. M* a' G* @, b" y" F$ _; O
+ L) X" s: i2 @ [2 [5 Z) ~' L data:
1 m T1 ?, q) j8 ]% i* c1 n) c pools: 6 pools, 640 pgs
( o& L( {; h- q: s& L% J objects: 0 objects, 0 B1 `& r+ h) P9 m/ T
usage: 3.1 GiB used, 3.3 TiB / 3.3 TiB avail
' m( h; D9 H7 {( O- P% `' @ pgs: 640 active+clean
9 e2 k) T9 Z* ?
& m* I* C! |0 H- ~5 d恢复正常了，问题解决。
2 C+ m. |7 X* L+ Z4 t, f/ }6 j/ Y& v2 f* E0 {
总结下：遇到这种问题，重做依然问题存在，很头疼。只能检查到底什么原因导致的问题。1 a0 ]7 ]: A) p6 R+ h
|
|
发表于 2021-7-20 17:00:03