health: HEALTH_WARN Reduced data availability 100.000% pgs unknown

admin

cluster:
+ n: d. w1 V: q[root@compute01 src]# ceph -s
  cluster:
    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_WARN
" H$ ]2 n6 f) w" N* N2 G; y' l& c            Reduced data availability: 640 pgs inactive
) i8 Z$ {( t, j* o+ u
  services:
9 J- ]& b! c. t  e: j    mon: 3 daemons, quorum compute01,compute02,compute03 (age 42m)
    mgr: compute01(active, since 42m), standbys: compute02, compute03
; H$ k! h1 R9 w( B5 s3 Q& C! W    osd: 3 osds: 3 up (since 26m), 3 in (since 26m)

  data:
* W6 W, A$ f; N5 u: K! t    pools:   6 pools, 640 pgs
, M8 E2 ^( P* E/ D5 a    objects: 0 objects, 0 B
    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
    pgs:     100.000% pgs unknown
5 q3 o  @( l, S, W9 i0 c) J" g             640 unknown

1 x& n) z+ j9 ~6 {) N) @; E( ]遇到问题，一直处于这种状态：

导出文件：
& y' O4 x+ L1 I5 h
8 J/ k/ ^7 C3 E- t[root@compute01 ~]# ceph osd crush tree
  ^' I1 A. H0 V, XID CLASS WEIGHT TYPE NAME   
-1            0 root default

3 M- |# F) p; d8 C. o) ]4 Z1 S
发现什么都没有，缺少东西

[root@compute01 ~]# ceph osd getcrushmap -o /tmp/mycrushmap
12
7 t' c" ~; Y2 h导出的数据只有12行，少了很多。


转换成可以读的文件：

[root@compute01 tmp]# crushtool -d /tmp/mycrushmap > /tmp/mycrushmap.txt

[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2
item 'compute01' in bucket 'default' is not defined
" K$ c6 W8 ~( N) T  P9 J( W/ r[root@compute01 tmp]# vim mycrushmap.txt
[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2
7 V8 z! |  L7 Z7 I- R: D# J4 v转换的时候发现缺少东西；
! Q' |) S. [& q. h0 W再次编辑：
# l  c" |% @" n3 ^[root@compute01 tmp]# vim mycrushmap.txt


$ T+ v* [6 G& }& \( B1 }" b# r# begin crush map
7 z1 ~6 O/ r5 ptunable choose_local_tries 0
tunable choose_local_fallback_tries 0
9 {% P+ p" q! h5 ?' K- Ltunable choose_total_tries 50
8 H( f5 J, W+ D! Ptunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
1 C$ u1 S' t& }8 [2 s& e5 F7 `tunable chooseleaf_stable 1
0 v- x0 k- B7 z( a- J3 Jtunable straw_calc_version 1
tunable allowed_bucket_algs 54
) q/ g5 ]: i  i$ T3 P% ~4 A# devices
$ K8 K$ V6 C( J' H+ ldevice 0 osd.0 class hdd
2 ?9 n9 r, s2 k$ B  r' Pdevice 1 osd.1 class hdd
2 K# K' T: G' G; Sdevice 2 osd.2 class hdd
8 H3 S2 v% C; }! ~8 I# types
- i0 K6 X4 R7 ?' m. E: |5 \type 0 osd
5 U; w' [  D+ a, x. I& _type 1 host
type 2 chassis
type 3 rack
type 4 row
4 x. [# V% {4 ^  o+ Ftype 5 pdu
type 6 pod
type 7 room
type 8 datacenter
. `% W9 i, s: V  H. f% @type 9 zone
type 10 region
type 11 root
5 v: N% _# S) ^9 {' C% W' i% C# s7 A
# buckets
  e- C6 W. b) r3 Broot default {
* t: A8 S4 D6 o, s; s- `& ] id -1  # do not change unnecessarily
2 a/ ~+ I: x4 t/ V id -2 class hdd  # do not change unnecessarily
9 d" s- M; y0 R # weight 0.000
4 C! _* e7 q4 i2 o3 G7 N  s alg straw2
hash 0 # rjenkins1

}
, T7 h: k* Q) \, j# rules
rule replicated_rule {
& i# `: k1 N+ n9 X. x+ j id 0
type replicated
min_size 1
max_size 10
step take default
5 j0 {, \' }. T& I4 }# D step chooseleaf firstn 0 type host
step emit
}
. |3 V( x4 C7 `, m# end crush map

! z8 w1 L) C0 g# y, `: T. _' \
6 h3 W3 j$ E6 `9 E/ e7 ~8 m; u发现少了很多东西，添加上吧：
  n; Q' F0 j: |7 @) g9 j8 Y. R6 u
$ ?- ^; @" Y7 c( g. E! k# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
0 _- g- i4 U! [/ a' z- W! M" htunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
. O9 ~: i, u; q2 }* e% A# Dtunable allowed_bucket_algs 54
( }5 S' e4 o- g7 ?; M0 A# devices
device 0 osd.0 class hdd
" a4 ]% y: [$ |: B- ^device 1 osd.1 class hdd
% A+ o) D2 G8 }5 r+ R4 t7 ]  i3 Adevice 2 osd.2 class hdd
# l/ R3 E4 A$ z) G- F, @% J# types
type 0 osd
type 1 host
+ U" Z" L0 _% h- O% y( Qtype 2 chassis
9 ]6 A3 S2 ]* E9 a- z) |type 3 rack
$ \8 Y# n, \2 |% d$ P& A  a" @type 4 row
type 5 pdu
type 6 pod
% j/ Z; W. V  d. G+ Y' Mtype 7 room
* F# E$ H3 @! |1 q  X3 U0 `5 k# ctype 8 datacenter
# o2 u* [( r& p6 r, z/ Atype 9 zone
3 h  H; Q+ c. b2 v7 mtype 10 region
# v8 C' q, }: ^$ xtype 11 root
; E& @5 @5 o( |: ^host compute02 {
. Q. ?% q5 f6 O4 ]4 |( L$ R; ~: P+ d        id -3           # do not change unnecessarily
        id -4 class hdd         # do not change unnecessarily
        # weight 1.000
1 i$ |4 \3 d6 P  H7 l) B        alg straw2
        hash 0  # rjenkins1
3 r: h0 \- r) ]! |. b        item osd.0 weight 1.000
}
* r$ f" v' I/ N. O- O0 Chost compute01 {
        id -5           # do not change unnecessarily
        id -6 class hdd         # do not change unnecessarily
5 ~8 Z  u; q9 m: v1 k! j        # weight 1.000
% n" `6 h6 _/ W2 U+ L        alg straw2
: M& R8 }' Y& r4 b9 U# m1 N+ x        hash 0  # rjenkins1
        item osd.1 weight 1.000
( ~4 O) ?& @  C}
host compute03 {
        id -7           # do not change unnecessarily
- _6 @9 q' x& ?2 _7 c- S9 _1 O8 ~        id -8 class hdd         # do not change unnecessarily
        # weight 1.000
        alg straw2
+ \0 m3 s0 d  w1 m4 Q( X        hash 0  # rjenkins1
! X, i4 ?+ g% a        item osd.2 weight 1.000
}
# buckets
root default {
id -1  # do not change unnecessarily
id -2 class hdd  # do not change unnecessarily
3 [. P: S! E" {; s* t) r3 _ # weight 0.000
alg straw2
hash 0 # rjenkins1
       item compute02 weight 1.000
- `# H6 w3 K( l        item compute01 weight 1.000
        item compute03 weight 1.000
}
# rules
rule replicated_rule {
id 0
type replicated
min_size 1
max_size 10
) i# u1 q6 o0 u4 ^. l step take default
( ^6 o+ V- A  {6 [* q" f$ K. r4 y step chooseleaf firstn 0 type host
- j# h& v. A* g step emit
}
# end crush map


$ e2 \/ e, L# F4 \% x: ]4 {9 W添加好之后，检查下对应关系，因为ceph节点和osd添加顺序的问题，导致1节点和2节点颠倒了，要注意这个地方，其他忽略；
转换成ceph认识的文件：
[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2

[root@compute01 tmp]# ceph osd setcrushmap -i /tmp/mycrushmap2
13
' @! E2 W0 ]! h3 `- r[root@compute01 tmp]# ceph -s
  cluster:
    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_WARN
            Reduced data availability: 212 pgs inactive
$ e0 l$ Y  B; P( a
0 a" G3 x8 P( K% {# F! f  services:
5 z( x  F, v0 `5 i* D3 W    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
    mgr: compute01(active, since 56m), standbys: compute02, compute03
    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)

( L! l/ ]8 \) V  data:
, M7 o. ~0 c9 g+ {: e    pools:   6 pools, 640 pgs
    objects: 0 objects, 0 B
& g3 s4 U; }/ t( A: f0 I    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
  Z4 K  q8 W5 a$ k7 ]7 F& i    pgs:     33.125% pgs unknown
0 U* v& r. O" p9 q# l             428 active+clean
- H/ y2 N3 ^3 d6 H  O             212 unknown

/ g7 g% n! @9 e[root@compute01 tmp]# ceph -s
# X% H( o+ a+ Z" X  cluster:
0 S7 F3 s( g3 B/ F3 h    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_OK

  services:
" T' J, i% g+ T0 s+ l0 x) X    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
    mgr: compute01(active, since 56m), standbys: compute02, compute03
% B0 J( z, F9 q    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)

  data:
! g  a4 `. W4 T- n' N) Y    pools:   6 pools, 640 pgs
  [  y7 x; p+ `! R1 \    objects: 0 objects, 0 B
    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
* Y) X! t" i0 l6 K8 }8 }$ A" C    pgs:     640 active+clean
0 Q5 f3 t) ~5 o( G2 U9 Q* ^
, g6 e6 A% X5 m[root@compute01 tmp]# ceph -s
  cluster:
: T& _: f0 ~. X5 G; I    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_OK
: i& r+ ^' Y2 `8 S" x
  services:
, e" J# G* R) ^2 X    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
    mgr: compute01(active, since 56m), standbys: compute02, compute03
    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)
+ X. I6 K9 G1 @+ m/ q
  data:
- Q: N# s5 P+ `    pools:   6 pools, 640 pgs
; v, C0 k3 A- b4 m6 n3 H& ]    objects: 0 objects, 0 B
7 w4 A: e: x6 S' }. L0 f' x    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
' D, ]. X, r' i4 ]# U" w4 n    pgs:     640 active+clean

恢复正常了，问题解决。

总结下：遇到这种问题，重做依然问题存在，很头疼。只能检查到底什么原因导致的问题。

admin

[root@compute03 ~]# ceph osd tree
ID CLASS WEIGHT TYPE NAME    STATUS REWEIGHT PRI-AFF
1 W) \7 F0 z: u3 m/ W-1            0 root default                        
/ r. @* z& G' g  o/ e: o 0   hdd      0 osd.0            up  1.00000 1.00000
- o* r0 X9 u) A. ]6 W 1   hdd      0 osd.1            up  1.00000 1.00000
2   hdd      0 osd.2            up  1.00000 1.00000
) T9 S8 Z7 F1 Y9 j[root@compute03 ~]# ceph osd tree
  C' a9 ^, f5 w, Q" kID CLASS WEIGHT TYPE NAME    STATUS REWEIGHT PRI-AFF
  F+ r' u* f% M% l5 I/ }# z7 R-1            0 root default                        
0   hdd      0 osd.0            up  1.00000 1.00000
/ r+ `' B$ v! `1 Q9 G. S 1   hdd      0 osd.1            up  1.00000 1.00000
2   hdd      0 osd.2            up  1.00000 1.00000


刚开始并没有发现什么问题，但总觉得有点奇怪，哪里怪呢？就是有点别扭，反正没有找到。
" C4 ~" l$ r, Y. u( H
" E2 K. g8 z4 L后来正常了，才发现有些东西发生边了。少了一些描述：

) y$ \, U) l9 f+ h! L& _[root@compute01 tmp]# ceph osd tree
8 B+ N) i, O" j. cID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-1       3.00000 root default                              
" T' I* @5 l; A/ S-5       1.00000     host compute01                        
0 g. m& O$ N$ y' S" | 1   hdd 1.00000         osd.1          up  1.00000 1.00000
4 P+ v0 u) r$ D  C, ^5 v! N' u: W-3       1.00000     host compute02                        
/ g3 `+ l6 n* Z7 r; f 0   hdd 1.00000         osd.0          up  1.00000 1.00000
-7       1.00000     host compute03                        
7 n. x! ^# F$ R! {' b) _ 2   hdd 1.00000         osd.2          up  1.00000 1.00000

admin

完成修复过程。

admin

,解决步骤：

" N! Y4 @. k8 U9 l( `' _3 Fceph osd crush add-bucket ceph1 host
8 f# k) f) Z, M! Y/ c' Z2 ?多个主机节点，添加多个主机host名称，方便管理
# G3 f1 _' g5 k6 q$ ?默认的话就移动到default root下：
ceph osd crush move ceph1 root=default
将多个都移动到default下
0 k1 G  |; K/ w4 o6 W, G, L) B1 [
  a; _3 x' u# i; h" K, ~这里是默认下配置host
ceph osd crush set osd.0 1.00000 host=ceph1
ceph osd crush set osd.1 1.00000 host=ceph1

admin

ceph osd crush add-bucket ceph1 host
2 `# n3 f0 C+ C: j
默认的话就移动到default root下：
" `4 h0 H2 P$ Q+ q+ U% E1 {ceph osd crush move ceph1 root=default


& a  t; _3 U$ z, c6 I这里是默认下配置host
5 y0 n3 Y3 ^, w/ a2 Sceph osd crush set osd.0 1.00000 host=ceph1
0 O5 I' B5 }5 c% d9 q5 j) Eceph osd crush set osd.1 1.00000 host=ceph1
2 p5 `7 V/ N0 K2 `$ k即可完美解决上诉的问题，前面的姐姐方法虽然可以，但是比较繁杂，没有此项步骤简单快捷。
" Y) x/ q& M4 h& `* a