Degraded data redundancy: 1 pg undersized ceph status状态异常

admin

[root@controller ~]# ceph -s
4 T2 w) Z2 U- e* d( U; _9 l6 G' P  cluster:
" A' z- u, m/ X) s/ j: a" B; K. A  t    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
    health: HEALTH_WARN
            Degraded data redundancy: 1 pg undersized
/ Z- w! }' ?8 o3 B& W8 |
  services:
    mon: 1 daemons, quorum controller (age 87m)
3 l" \+ V) v. z* t    mgr: controller.horbtx(active, since 87m)
# D( i: @2 ^7 q7 k    osd: 6 osds: 6 up (since 6m), 6 in (since 6m); 1 remapped pgs
) f# D" F+ U+ a$ a! N) j( E
; r2 ?. K5 \% E+ |% H3 F' {+ I  data:
    pools:   1 pools, 1 pgs
    objects: 0 objects, 0 B
/ L$ z, O. E7 @2 u7 }+ l. M    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
    pgs:     1 active+undersized+remapped

解决过程：

' t# ~' y% W! O6 Y9 l" s5 P5 |2 u[root@controller ~]# vim /etc/ceph/ceph.conf

  osd_class_update_on_start = false

# u: |5 G4 r/ ]- ?$ \$ l
[root@controller ~]# ceph health detail
4 q3 V8 d$ B! l" O" WHEALTH_WARN Degraded data redundancy: 1 pg undersized
+ O) ?* S3 W/ \& [1 A" x+ E[WRN] PG_DEGRADED: Degraded data redundancy: 1 pg undersized
; L( _- |5 y: T    pg 1.0 is stuck undersized for 86m, current state active+undersized+remapped, last acting [1,0]
& h; ]' }3 C! ]' v* L0 k
修改配置后，需要重启osd服务:

' a3 r# o) ?8 Cceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.0.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.3.service
# }) ?! S4 L6 _1 Xceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.1.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.4.service
9 i; `" s* w, E8 c' O6 V. T& F. Jceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.2.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.5.service
[root@controller ~]# systemctl restart ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.0.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.3.service
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.1.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.4.service
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.2.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.5.service
[root@controller ~]# systemctl restart ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.*
[root@controller ~]# ceph -s
! O5 Y: I2 k9 v* d% `  cluster:
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
    health: HEALTH_WARN
8 A# c) Q/ O! d5 F# s& d5 F1 t  P            4 osds down
. [0 [7 I$ {2 R. W8 {! O2 f            Degraded data redundancy: 1 pg undersized

  T6 G9 I/ ~) V, k7 a2 R. l  services:
) w1 f+ D6 @3 O' t; D    mon: 1 daemons, quorum controller (age 89m)
4 @/ ^6 p) H. @% r3 X% b4 Z    mgr: controller.horbtx(active, since 88m)
    osd: 6 osds: 2 up (since 0.641904s), 6 in (since 8m)
* b8 {0 p8 O7 ^$ o# b8 ^# ], m2 \
0 e! @4 m# F% D% C' h" e  data:
: e3 F; u. [# E$ j! j  d    pools:   1 pools, 1 pgs
    objects: 0 objects, 0 B
    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
' K( \+ g: E" Q1 L    pgs:     1 stale+active+undersized+remapped
* |& g: l. ~8 w1 a: {9 x
) A* e- v# g! j# b% q[root@controller ~]# ceph -s
, @2 ?: L3 j% m' r  cluster:
& A2 [# e. p% r& f/ K    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
! L2 F- g  \4 I0 ^) Z! x    health: HEALTH_OK
+ v, ~# }) v( U' o# p' H! b
3 _4 }. m% F7 z, z5 n. p  services:
    mon: 1 daemons, quorum controller (age 89m)
- w; ?6 \/ _7 h) K9 D& g    mgr: controller.horbtx(active, since 89m)
% ?0 p( M% G% P( Y( y& W    osd: 6 osds: 6 up (since 6s), 6 in (since 8m); 1 remapped pgs

  data:
    pools:   1 pools, 1 pgs
' f5 T) @3 m' \! j    objects: 0 objects, 0 B
* t! c* @+ T$ X% d! P- x# y    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
    pgs:     1 active+undersized+remapped
' R: i2 A+ l6 d. m4 |" N6 }  |, s
0 f" p, s' P5 T  V- W[root@controller ~]# ceph -s
  cluster:
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
# O0 P, ]0 p" C# w    health: HEALTH_OK
, B1 c$ E; O* b6 j) G* w  B6 _, U
! [" T# S4 a& g' u  services:
    mon: 1 daemons, quorum controller (age 89m)
    mgr: controller.horbtx(active, since 89m)
    osd: 6 osds: 6 up (since 8s), 6 in (since 9m); 1 remapped pgs
; o. K  g5 X1 \/ S  |
  data:
    pools:   1 pools, 1 pgs
9 n' h' V! b, ?+ r$ q9 j# ~  j    objects: 0 objects, 0 B
! l( d( O' l1 L. ?" ]/ w1 r- w    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
    pgs:     1 active+undersized+remapped
4 s* T/ I# @5 f
[root@controller ~]# ceph -s
: n) @1 l+ x2 P  cluster:
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
' C, U: I' A4 q; z; Y9 |/ B" z    health: HEALTH_OK
7 s' o; I* z( J7 s5 A3 e) g' s& E6 V
/ C7 R% C3 B! M  services:
1 `3 P0 @; G9 o3 r    mon: 1 daemons, quorum controller (age 89m)
    mgr: controller.horbtx(active, since 89m)
" X" A8 l' U2 h  m- a1 ?    osd: 6 osds: 6 up (since 9s), 6 in (since 9m); 1 remapped pgs

  data:
% N! j3 x' z( A3 P" l( {9 }    pools:   1 pools, 1 pgs
; C  W( E# V  q6 R3 v    objects: 0 objects, 0 B
4 K, H3 v+ y8 T3 @4 T    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
    pgs:     1 active+undersized+remapped

0 i. q, X! i/ |9 l8 O7 H7 O/ r[root@controller ~]# ceph -s
  cluster:
3 t( f6 v5 O, b* B9 E; K    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
! g, u  g: `9 L. W( J    health: HEALTH_OK

+ P9 @$ l9 F2 ]  A# ]3 E  services:
    mon: 1 daemons, quorum controller (age 89m)
4 Y; q) N- Z8 C% R9 i( |' o    mgr: controller.horbtx(active, since 89m)
    osd: 6 osds: 6 up (since 10s), 6 in (since 9m); 1 remapped pgs
1 b  r+ Z/ i$ L/ ?# [6 M3 G% e
  data:
    pools:   1 pools, 1 pgs
7 ~4 P5 y4 |  C    objects: 0 objects, 0 B
    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
3 P! N* ~+ t& U    pgs:     1 active+undersized+remapped
0 ~+ `8 n9 n4 x, _; p& |
[root@controller ~]# ceph health detail
HEALTH_OK
5 Z& T- w- S3 p9 m" Q( J[root@controller ~]# ceph health detail
1 _+ ^$ N3 t/ R, `- o" d' wHEALTH_OK
[root@controller ~]#

admin

3.1.1 说明
降级：由上文可以得知，每个PG有三个副本，分别保存在不同的OSD中，在非故障情况下，这个PG是active+clean 状态，那么，如果PG 的 副本osd.4 挂掉了，这个 PG 是降级状态。
3.1.2 故障模拟
a. 停止osd.1
) \  h  I, p7 @ $ systemctl stop ceph-osd@1
b. 查看PG状态
$ bin/ceph pg stat 20 pgs: 20 active+undersized+degraded; 14512 kB data, 302 GB used, 6388 GB / 6691 GB avail; 12/36 objects degraded (33.333%)
3 n. R2 k% A8 h+ m) r, M. Nc. 查看集群监控状态
5 P. A# ?; k( F $ bin/ceph health detail
HEALTH_WARN 1 osds down; Degraded data redundancy: 12/36 objects degraded (33.333%), 20 pgs unclean, 20 pgs degraded; application not enabled on 1 pool(s)
/ M7 j- I' T+ yOSD_DOWN 1 osds down     
. [& e: w& F5 }0 {7 U+ H   osd.1 (root=default,host=ceph-xx-cc00) is down
/ b* r' a6 \$ U2 N  \$ @2 _PG_DEGRADED Degraded data redundancy: 12/36 objects degraded (33.333%), 20 pgs unclean, 20 pgs degraded     
   pg 1.0 is active+undersized+degraded, acting [0,2]     
   pg 1.1 is active+undersized+degraded, acting [2,0]
/ U* }5 j. v( v4 h6 Fd. 客户端IO操作
: a" M8 r) F, u/ @0 E #写入对象
$ bin/rados -p test_pool put myobject ceph.conf  

#读取对象到文件
$ bin/rados -p test_pool get myobject.old  
8 A% A6 y+ w! I. o9 w1 o
#查看文件
$ ll ceph.conf*
9 m; r/ Z/ w% U; i3 G-rw-r--r-- 1 root root 6211 Jun 25 14:01 ceph.conf
-rw-r--r-- 1 root root 6211 Jul 3 19:57 ceph.conf.old
9 ~: h  }; O) j+ }( K2 l4 E9 E故障总结：
为了模拟故障，(size = 3, min_size = 2) 我们手动停止了 osd.1，然后查看PG状态，可见，它此刻的状态是active+undersized+degraded,当一个 PG 所在的 OSD 挂掉之后，这个 PG 就会进入undersized+degraded 状态，而后面的[0,2]的意义就是还有两个副本存活在 osd.0 和 osd.2 上, 并且这个时候客户端可以正常读写IO。
3.1.3 总结
0 `* x# R- R5 _, m# @# ?降级就是在发生了一些故障比如OSD挂掉之后，Ceph 将这个 OSD 上的所有 PG 标记为 Degraded。
8 r0 ^4 e3 \, T降级的集群可以正常读写数据，降级的 PG 只是相当于小毛病而已，并不是严重的问题。
4 h4 b# n1 T/ b' ^  J  l8 wUndersized的意思就是当前存活的PG 副本数为 2，小于副本数3，将其做此标记，表明存货副本数不足，也不是严重的问题。
2 ^; O: q2 _- }! l3.2 Peered
3.2.1 说明
! S) _6 m6 e- H9 o% JPeering已经完成，但是PG当前Acting Set规模小于存储池规定的最小副本数(min_size)。
, E+ \" j$ Y% }  C3.2.2 故障模拟
a. 停掉两个副本osd.1,osd.0
9 X8 X$ }4 D8 g5 } $ systemctl stop ceph-osd@1
$ systemctl stop ceph-osd@0
5 c- g* Y8 q  M  _* Y: D1 \/ k( ]6 e& c

3.2.1 说明
1 Y5 c3 l1 X7 U: LPeering已经完成，但是PG当前Acting Set规模小于存储池规定的最小副本数(min_size)。
3.2.2 故障模拟

a. 停掉两个副本osd.1,osd.0

$ systemctl stop ceph-osd@1
4 p; C- X% R3 j$ B+ ]% Z $ systemctl stop ceph-osd@0
8 ~5 ]. {% ~! x3 l, ]! k
2 Q/ z7 C' ^- W. o4 _7 B/ r/ Ab. 查看集群健康状态
7 Z. |% A& d1 U+ U# k# V
% i4 D# ^. I& p1 G9 C $ bin/ceph health detail
HEALTH_WARN 1 osds down; Reduced data availability: 4 pgs inactive; Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded; application not enabled on 1 pool(s)
OSD_DOWN 1 osds down     
' X: o1 ]3 \3 z' _    osd.0 (root=default,host=ceph-xx-cc00) is down
PG_AVAILABILITY Reduced data availability: 4 pgs inactive     
$ n  H  B  ]8 v& ]2 K) k    pg 1.6 is stuck inactive for 516.741081, current state undersized+degraded+peered, last acting [2]     
: H6 I' A$ k6 G7 J    pg 1.10 is stuck inactive for 516.737888, current state undersized+degraded+peered, last acting [2]     
: M3 ?2 s  \( i$ {) a% K: ?% {/ y    pg 1.11 is stuck inactive for 516.737408, current state undersized+degraded+peered, last acting [2]     
2 m2 j/ D) m# C; p6 r+ \    pg 1.12 is stuck inactive for 516.736955, current state undersized+degraded+peered, last acting [2]
4 J: V; ]+ R. b, n7 J! OPG_DEGRADED Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded     
& w& @$ g0 c& N    pg 1.0 is undersized+degraded+peered, acting [2]     
8 V$ a3 X; C- r    pg 1.1 is undersized+degraded+peered, acting [2]
c. 客户端IO操作(夯住)

4 |$ i# D+ d8 e  o# U% ~ #读取对象到文件，夯住IO
$ bin/rados -p test_pool get myobject  ceph.conf.bak
故障总结：

现在pg 只剩下osd.2上存活，并且 pg 还多了一个状态：peered，英文的意思是仔细看，这里我们可以理解成协商、搜索。
这时候读取文件，会发现指令会卡在那个地方一直不动，为什么就不能读取内容了，因为我们设置的 min_size=2 ，如果存活数少于2，比如这里的 1 ，那么就不会响应外部的IO请求。
$ K8 c/ j- N' a" q0 id. 调整min_size=1可以解决IO夯住问题
: N6 F( T0 h1 V6 M5 A
3 {( v+ M2 F2 k  Q #设置min_size = 1
$ bin/ceph osd pool set test_pool min_size 1
set pool 1 min_size to 1
3 i! o& x  ?4 Q# x; {e. 查看集群监控状态
# W$ f  \, G5 i0 g
; [* p; H7 G0 o+ t# `0 b2 u* j $ bin/ceph health detail
HEALTH_WARN 1 osds down; Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded, 20 pgs undersized; application not enabled on 1 pool(s)
1 ]( V9 t  l- N  v2 q0 \8 E' Q1 zOSD_DOWN 1 osds down     
   osd.0 (root=default,host=ceph-xx-cc00) is down。
PG_DEGRADED Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded, 20 pgs undersized     
pg 1.0 is stuck undersized for 65.958983, current state active+undersized+degraded, last acting [2]     
5 t4 N! \% h5 m1 f6 C- |8 l- ]pg 1.1 is stuck undersized for 65.960092, current state active+undersized+degraded, last acting [2]     
2 F/ Y0 j% O4 o) ~  S8 f8 [3 lpg 1.2 is stuck undersized for 65.960974, current state active+undersized+degraded, last acting [2]
5 j, S4 T3 O0 G8 m6 m) }f. 客户端IO操作
" S) ]" ?* S: C' s' L
#读取对象到文件中
+ z/ c2 [, U% C" P8 k7 D3 q) p+ {9 G$ ll -lh ceph.conf*
7 J: H& w0 s# A/ h9 ^1 {3 y, Z! G-rw-r--r-- 1 root root 6.1K Jun 25 14:01 ceph.conf
-rw-r--r-- 1 root root 6.1K Jul 3 20:11 ceph.conf.bak
7 P0 Q8 ]/ F3 K  ?; a-rw-r--r-- 1 root root 6.1K Jul 3 20:11 ceph.conf.bak.1
' T4 P1 T: D1 C; J故障总结：
6 p1 T+ [" z) ?# ^) f
1 [8 `- w6 w, J9 y) y可以看到，PG状态Peered没有了，并且客户端文件IO可以正常读写了。
当min_size=1时，只要集群里面有一份副本活着，那就可以响应外部的IO请求。