health: HEALTH_WARN Reduced data availability 100.000% pgs unknown

admin

cluster:
[root@compute01 src]# ceph -s
6 d& h. k) ?6 c  s: x- q% V  cluster:
& F" K- J* p4 _* N8 X7 l/ ]    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_WARN
            Reduced data availability: 640 pgs inactive

! e* j' ]4 R2 b8 G  services:
4 |  a' a2 E5 _. z5 I' q    mon: 3 daemons, quorum compute01,compute02,compute03 (age 42m)
( o, H' b; o/ g0 \    mgr: compute01(active, since 42m), standbys: compute02, compute03
; Y) L# U0 w/ [/ i9 o+ |    osd: 3 osds: 3 up (since 26m), 3 in (since 26m)
% g3 L" \( ~3 c& t- J; C+ Z
  data:
6 B1 Q1 z3 h5 Q# ^& d, J3 n. j    pools:   6 pools, 640 pgs
    objects: 0 objects, 0 B
  l. q# U% U1 Y  k/ z3 L/ x: _    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
2 @+ e) n( h/ @1 i# N- G2 e# D    pgs:     100.000% pgs unknown
             640 unknown
0 a7 c7 ?9 Z3 m" D- \; u3 `7 q
遇到问题，一直处于这种状态：

5 a7 R3 ~1 {6 H) a( O& C导出文件：

[root@compute01 ~]# ceph osd crush tree
- ^2 Q* e4 T0 yID CLASS WEIGHT TYPE NAME   
-1            0 root default
* v' x6 Q& `* Z0 d) d2 _+ y' z

1 @2 r" S6 a" d; M+ z) G% I& ~发现什么都没有，缺少东西
5 M& D( f  I4 _! k
3 l6 m8 u1 ]  E7 T  ~% T$ s5 g[root@compute01 ~]# ceph osd getcrushmap -o /tmp/mycrushmap
( E& n, {9 {5 U/ ^( _8 B12
- \& l5 r7 `' X3 r导出的数据只有12行，少了很多。

4 D5 ^( ~8 L, {
转换成可以读的文件：

[root@compute01 tmp]# crushtool -d /tmp/mycrushmap > /tmp/mycrushmap.txt

/ M1 ]/ l9 P2 Y) l- O* I# f: W2 g[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2
item 'compute01' in bucket 'default' is not defined
[root@compute01 tmp]# vim mycrushmap.txt
[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2
3 Y8 [) ?: b! R  `7 t* U转换的时候发现缺少东西；
- y. @, Q: ?. g* Q* a0 v% {- n再次编辑：
7 r! R! a% _) p$ H' h: f* C: S[root@compute01 tmp]# vim mycrushmap.txt


# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
$ r$ }* `6 F) o% Q$ B6 [* `- ~tunable chooseleaf_vary_r 1
! Q6 H8 ?; l$ i* A2 Q9 Gtunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
( _; L7 [$ @/ k, g" P5 Zdevice 0 osd.0 class hdd
, q* s8 h$ c& A$ m3 |/ l2 Qdevice 1 osd.1 class hdd
6 d9 Y+ F" ~9 g) A; V( Hdevice 2 osd.2 class hdd
3 n$ y& k8 G( g1 Z' a, S" l9 a# types
  O3 f/ q3 P4 z/ J# l" D: Ctype 0 osd
type 1 host
type 2 chassis
type 3 rack
* O! b2 F) A- j5 @type 4 row
type 5 pdu
7 f. b6 _+ Q1 P* A( i! dtype 6 pod
type 7 room
( c) f" r& d( c. V9 i, S/ V; i' }  X# ptype 8 datacenter
type 9 zone
9 p( |- q1 Q/ ~# C+ jtype 10 region
type 11 root

. `$ g0 p+ h# g7 [& j# buckets
. K4 R4 C+ \' [6 k1 Croot default {
id -1  # do not change unnecessarily
& R2 B: C1 M4 V id -2 class hdd  # do not change unnecessarily
# weight 0.000
alg straw2
  Z) @1 Q4 \' d hash 0 # rjenkins1
6 ?! y, R6 D" `* G8 |( ?' ]9 }
, a' Y# u# F0 C0 e}
# rules
rule replicated_rule {
6 b; }. P/ f8 S) d8 E0 j id 0
2 S; \4 O$ _- `8 ~; X9 d: s type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
% C9 t: @  z1 r6 }( W step emit
: ^6 `2 B! Q% u( Z. }2 J}
# end crush map
+ g0 ]9 a- @/ j. Y, U, J% d

发现少了很多东西，添加上吧：

# begin crush map
tunable choose_local_tries 0
' b; `/ N# m* R( x% K. btunable choose_local_fallback_tries 0
1 C+ _0 P6 z# X# D9 stunable choose_total_tries 50
9 W! a: o1 U) U7 G7 t5 M4 q. e# qtunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
( _- F# C/ B- @2 d% n& Qtunable allowed_bucket_algs 54
% ]- Q7 r6 e* @* G, ?" `# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
6 B7 X1 W+ C2 S" A) t$ Xdevice 2 osd.2 class hdd
& P3 M# x6 W+ O' G6 H$ F# types
  ]# {; Q2 A$ G3 h6 etype 0 osd
: y1 [8 n- g; J5 T! N# ttype 1 host
type 2 chassis
- u; x+ y) B  etype 3 rack
type 4 row
% \2 r- l0 {! ]type 5 pdu
/ _1 m, @, V3 w  i# r* d3 Stype 6 pod
7 a2 v2 R% @: {2 j/ p0 x" k# {7 Mtype 7 room
8 O5 ]+ W+ \' W# Y8 k& l+ Btype 8 datacenter
1 {& C) E( W* V3 E3 Z% h& Ftype 9 zone
3 X2 q2 _5 l# V' T" N( Wtype 10 region
type 11 root
host compute02 {
        id -3           # do not change unnecessarily
' p( l* _- I8 Z4 j" H        id -4 class hdd         # do not change unnecessarily
9 T( Y3 G. r% t* f( O        # weight 1.000
) c: I! U. a7 h' g& f. {/ x        alg straw2
        hash 0  # rjenkins1
/ T* a. _+ q& b0 i- X' H$ [        item osd.0 weight 1.000
}
3 P8 x* B. ^/ }  zhost compute01 {
2 x& ?+ i6 s& u$ T7 f        id -5           # do not change unnecessarily
# ~. ?1 S. b7 c' v9 _4 A        id -6 class hdd         # do not change unnecessarily
        # weight 1.000
        alg straw2
        hash 0  # rjenkins1
9 W# K' `. ~8 m; I        item osd.1 weight 1.000
}
/ i6 h% ^1 A' w+ |6 }host compute03 {
        id -7           # do not change unnecessarily
2 S) d& b+ p& d; ~9 m" K- c. m, O        id -8 class hdd         # do not change unnecessarily
% U( ]5 V9 u/ Q# o        # weight 1.000
  l* z- e5 N- V1 J! U        alg straw2
. v0 e0 F9 @8 R8 H        hash 0  # rjenkins1
2 ]+ A5 N6 }5 E# Q7 G        item osd.2 weight 1.000
3 z" o1 G+ T* I/ e1 m9 w! [, Z7 W}
# buckets
root default {
" c) A0 J8 m& F  |) o8 Z id -1  # do not change unnecessarily
8 J, q" m! l6 o5 i3 T0 Z! } id -2 class hdd  # do not change unnecessarily
1 q% _" ]" V! Q1 D$ H # weight 0.000
alg straw2
; W/ ]% e4 \6 m" Z& G4 b hash 0 # rjenkins1
  V8 f- E* l1 U( |6 E) ]2 Y        item compute02 weight 1.000
        item compute01 weight 1.000
        item compute03 weight 1.000
}
/ j( y* k. c0 S' m/ u, I& L# rules
rule replicated_rule {
id 0
type replicated
4 n# ^1 k( q+ F0 ` min_size 1
4 ]& N$ ~3 r/ {2 N/ ~) B" q max_size 10
$ W( l; }" ^8 r4 W step take default
, ]* Z% I; Y4 n4 z- r. c9 m  @: `( w step chooseleaf firstn 0 type host
! Z" O3 R9 w, \) p( K8 q8 z) M step emit
. q2 I  R2 q, P$ L& r}
1 V6 o7 \9 N* F& H3 W5 g# end crush map
- ^/ f, @" K: L
: |* E0 `; b5 v% ?, L8 ]( t$ t% ?
& H6 t$ y- M% ?- f添加好之后，检查下对应关系，因为ceph节点和osd添加顺序的问题，导致1节点和2节点颠倒了，要注意这个地方，其他忽略；
转换成ceph认识的文件：
9 q' u" v- v8 a+ r& o0 E[root@compute01 tmp]# crushtool -c mycrushmap.txt -o mycrushmap2

[root@compute01 tmp]# ceph osd setcrushmap -i /tmp/mycrushmap2
13
+ L+ P' u  k+ l& W6 Q* [1 x# P$ f' }3 S[root@compute01 tmp]# ceph -s
  cluster:
    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_WARN
            Reduced data availability: 212 pgs inactive
8 \% ]% f5 r. _6 S% t5 V' T
  services:
. m6 n1 f* d9 O3 u" A+ e* |% d    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
  {  Y1 ]: g1 Y5 d# j    mgr: compute01(active, since 56m), standbys: compute02, compute03
+ U8 i0 L8 `/ _- J5 A    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)
) R: ~! o6 Y( N& N7 o6 j0 v
  data:
    pools:   6 pools, 640 pgs
6 |4 Z/ u# L) o9 |  I& I    objects: 0 objects, 0 B
    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
& _: C0 S$ y. x" m    pgs:     33.125% pgs unknown
             428 active+clean
             212 unknown

( I: b7 |+ W% t7 G3 T$ n: u[root@compute01 tmp]# ceph -s
0 y# x- c/ s; ]/ h( F  cluster:
    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
    health: HEALTH_OK

5 Z( W- D$ h6 Y: d  services:
    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
    mgr: compute01(active, since 56m), standbys: compute02, compute03
, b* V- P" r3 U4 f1 b1 y    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)
; i$ W$ L9 ?. J9 R1 `. r
) s, q# R, n! @# O# p& K  data:
, x1 O& K- i! K! X. q    pools:   6 pools, 640 pgs
2 b- z" ]( G5 p# Y' @  ^    objects: 0 objects, 0 B
    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
/ ]) C; i* F; f8 P& i4 q3 ]$ M    pgs:     640 active+clean
& ]1 _' h4 E2 R6 T# [5 E
7 _! n$ q0 b% \- T. j  \5 @[root@compute01 tmp]# ceph -s
  cluster:
    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
. z/ V3 v8 K7 Y+ {/ t, B    health: HEALTH_OK
5 M! L- o7 ]- q  w0 `/ y
) k# V1 [! J$ `0 ~  services:
/ N3 Y4 ]' [% g5 P) C    mon: 3 daemons, quorum compute01,compute02,compute03 (age 56m)
    mgr: compute01(active, since 56m), standbys: compute02, compute03
    osd: 3 osds: 3 up (since 40m), 3 in (since 40m)

' g; [$ ~% e& w5 N( `5 ]  data:
    pools:   6 pools, 640 pgs
- X1 P2 B; ?$ e8 B- B    objects: 0 objects, 0 B
    usage:   3.1 GiB used, 3.3 TiB / 3.3 TiB avail
  v0 v4 ~. p% I    pgs:     640 active+clean
! c0 M3 w8 U0 ]: {# o
9 c! C3 B/ _7 i% s/ @恢复正常了，问题解决。
3 H* f0 {, b8 j! m
总结下：遇到这种问题，重做依然问题存在，很头疼。只能检查到底什么原因导致的问题。

admin

[root@compute03 ~]# ceph osd tree
5 ~: P/ n1 ~, f* u- v6 ?ID CLASS WEIGHT TYPE NAME    STATUS REWEIGHT PRI-AFF
-1            0 root default                        
0   hdd      0 osd.0            up  1.00000 1.00000
1 `- }4 `% M/ a7 W% K 1   hdd      0 osd.1            up  1.00000 1.00000
: z6 j! t5 S- O 2   hdd      0 osd.2            up  1.00000 1.00000
4 T3 I( \* r/ ~) V[root@compute03 ~]# ceph osd tree
ID CLASS WEIGHT TYPE NAME    STATUS REWEIGHT PRI-AFF
-1            0 root default                        
0   hdd      0 osd.0            up  1.00000 1.00000
1   hdd      0 osd.1            up  1.00000 1.00000
" M! R7 F& |, y# S$ A2 S. N# d 2   hdd      0 osd.2            up  1.00000 1.00000


刚开始并没有发现什么问题，但总觉得有点奇怪，哪里怪呢？就是有点别扭，反正没有找到。
3 m1 J& w  s% S
后来正常了，才发现有些东西发生边了。少了一些描述：
& l1 p0 C, n' \7 j, g
% {" |' V4 f9 P. E4 _[root@compute01 tmp]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
-1       3.00000 root default                              
* B& U. J: s8 o: D2 j2 X-5       1.00000     host compute01                        
- n# V6 D" t# g! N5 w 1   hdd 1.00000         osd.1          up  1.00000 1.00000
-3       1.00000     host compute02                        
$ n' \9 p& f* N9 [+ l3 A 0   hdd 1.00000         osd.0          up  1.00000 1.00000
/ Z8 _3 w, `! ?% ?3 [-7       1.00000     host compute03                        
- X2 S: b. N8 k7 Y5 x 2   hdd 1.00000         osd.2          up  1.00000 1.00000

admin

完成修复过程。

admin

,解决步骤：

  Q. K* R4 m! w1 x; vceph osd crush add-bucket ceph1 host
多个主机节点，添加多个主机host名称，方便管理
默认的话就移动到default root下：
ceph osd crush move ceph1 root=default
: H  l0 ^9 I, y- f将多个都移动到default下

这里是默认下配置host
ceph osd crush set osd.0 1.00000 host=ceph1
ceph osd crush set osd.1 1.00000 host=ceph1

admin

ceph osd crush add-bucket ceph1 host

默认的话就移动到default root下：
5 t, q* t! O1 |( uceph osd crush move ceph1 root=default


3 z) K+ o7 k( I# k' h4 h6 @这里是默认下配置host
, x: f# l  L$ G3 ^9 u& _ceph osd crush set osd.0 1.00000 host=ceph1
9 q- v: L9 ?; j9 m# fceph osd crush set osd.1 1.00000 host=ceph1
( h6 ^8 ?. R5 N- l即可完美解决上诉的问题，前面的姐姐方法虽然可以，但是比较繁杂，没有此项步骤简单快捷。
6 J* T' Y+ n0 ?- j0 S& Y