浏览此站
联系我们相册切换到窄版 Language

 找回密码
 注册
易陆发现论坛»论坛 › 云计算开源区 › ceph分布式存储 › Degraded data redundancy: 1 pg undersized ceph stat ...
返回列表 发新帖
查看: 4246|回复: 1

Degraded data redundancy: 1 pg undersized ceph status状态异常

[复制链接]
admin

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
发表于 2021-6-9 15:00:16 | 显示全部楼层 |阅读模式
[root@controller ~]# ceph -s
" b, K4 \" q: L3 h, x  cluster:# E3 h' @# P, K" O9 b
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de9 G' F* w7 u; _. t! Z/ y3 \
    health: HEALTH_WARN
$ m$ s* y+ V0 E9 ~# v! }, D            Degraded data redundancy: 1 pg undersized
/ X, u% l4 g. E
4 c/ v$ @% x/ m- B  R  services:
3 n6 d+ r7 o. h  O+ S% n2 i    mon: 1 daemons, quorum controller (age 87m)6 V" k- J$ P* C# J% \* B
    mgr: controller.horbtx(active, since 87m)
4 Z/ _# ^! N8 _1 R: F, ~8 u    osd: 6 osds: 6 up (since 6m), 6 in (since 6m); 1 remapped pgs
% K; U) H: v/ P
1 q4 `/ y+ p: j; ^1 q5 b  data:
/ \! n) {* N; o6 T) |0 A5 F    pools:   1 pools, 1 pgs
/ p( M& ~3 i0 \+ @5 g( {    objects: 0 objects, 0 B
1 ]- Z* E+ b4 U" r. j/ b9 r& w    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
1 C- j1 j4 N" t( ]2 b- D9 Q    pgs:     1 active+undersized+remapped
; \9 V" \' z/ |$ x' A. W
8 {& g) N  S5 k+ B4 J解决过程：# v$ G" o* I( ?
$ o' T7 {% _* o# a) K3 i! ]$ N9 J
[root@controller ~]# vim /etc/ceph/ceph.conf
: X. c- Y1 g2 E2 }6 `
2 @1 R% m. B  M; i/ G1 z  osd_class_update_on_start = false7 g. `6 z* o2 y7 S/ X, W2 c: k
/ A$ N' p* j! n' t3 B- ?

4 x7 @! U0 h. g* L- i[root@controller ~]# ceph health detail
0 h. K- g8 Y4 T; @  D2 b5 @+ CHEALTH_WARN Degraded data redundancy: 1 pg undersized
' b# H" z5 ?5 k+ M7 ?[WRN] PG_DEGRADED: Degraded data redundancy: 1 pg undersized% A4 Y9 k8 `! h: t
    pg 1.0 is stuck undersized for 86m, current state active+undersized+remapped, last acting [1,0]
) o& e" [, N% M, n" X
- p$ y- M0 t4 |7 H8 W修改配置后，需要重启osd服务:  E; g' V; j* s5 Y1 U. Y

1 w. r6 \$ z/ L+ @& Mceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.0.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.3.service% e  V$ z! N: g$ a4 d: E+ o
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.1.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.4.service
& j5 h8 u& C7 t3 F- Kceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.2.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.5.service+ ~; }1 I8 v% V+ H: N5 F
[root@controller ~]# systemctl restart ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.* F7 q6 J4 }! c2 _& _5 t' d; S
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.0.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.3.service* c; V0 _1 c/ a
ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.1.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.4.service
+ N" B9 X; B- b) [ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.2.service  ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.5.service. q  g! f% d$ N; ^# w
[root@controller ~]# systemctl restart ceph-a4bb5236-c8ca-11eb-a67b-000c29ad02de@osd.*
6 c5 I! C' p1 b8 m& C& d; @[root@controller ~]# ceph -s
5 ~. M- z; o' ^% |4 b8 X  cluster:/ |5 r" d' k- g1 v6 L3 U
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
% G1 W: @0 `. D    health: HEALTH_WARN- [# D# g$ ~# x  S  v
            4 osds down
3 k+ l% Y$ @9 E1 R3 I0 o            Degraded data redundancy: 1 pg undersized
# i8 e* H% ~; b' z 7 z. B) V, V" ^7 y- Q% W* l
  services:" A# z" U% C! Q5 v8 [" b) t; f) I
    mon: 1 daemons, quorum controller (age 89m)4 M( o- Y$ y/ r4 W* R2 s1 V" R0 j
    mgr: controller.horbtx(active, since 88m)! u! S' ]2 E1 M: o
    osd: 6 osds: 2 up (since 0.641904s), 6 in (since 8m)$ c, ?( h0 k) L) E3 J
! E& D3 ?) C' f
  data:- ~2 X, M6 m' z# y' w; m; ~- V
    pools:   1 pools, 1 pgs; g+ q- @8 u) D* a9 I
    objects: 0 objects, 0 B6 R4 ~& X' {2 D6 _# U5 C; Y
    usage:   6.0 GiB used, 114 GiB / 120 GiB avail$ m* f7 v2 n# t- I
    pgs:     1 stale+active+undersized+remapped- U4 k& R9 L9 f, i. E) l0 U

6 G, c. N3 R3 W! g* E( I[root@controller ~]# ceph -s
' u+ h5 a7 w' A  cluster:  G) N% D+ t6 n/ k7 I0 s0 _
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de" G) b& u9 i. T/ b
    health: HEALTH_OK: t' W- v, O2 `6 G& d' x2 D
8 r% J; E; r% a
  services:
7 a9 l1 Q0 {. j# H; B+ [4 E4 E    mon: 1 daemons, quorum controller (age 89m)6 I9 y+ m' L/ g5 G6 S1 y3 ?7 S
    mgr: controller.horbtx(active, since 89m): ~9 X2 f  b, V) W# G& y
    osd: 6 osds: 6 up (since 6s), 6 in (since 8m); 1 remapped pgs$ |4 Z1 N% y+ L! J
9 d  Q3 W) Z0 U7 n% d2 j
  data:
* p8 M, L6 F5 E1 L& v9 W& ?( D) C4 T; s    pools:   1 pools, 1 pgs
3 w" i6 F+ u6 c' H5 E" L9 `! m) x) v" \    objects: 0 objects, 0 B8 O7 I1 e) S" b: r/ r/ R- ^
    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
& R- u0 r( C* I1 U1 R    pgs:     1 active+undersized+remapped
4 O- I; g! h% z5 z- [* w 4 m# n5 K( q- o# q# n* d
[root@controller ~]# ceph -s
* t0 Q/ j: c$ C, A% {  cluster:1 X3 L5 h( c+ Q+ Z% ?% W
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
2 ^: t& f2 `# ?* z$ i    health: HEALTH_OK
: l6 i' V' ]. V: H- U1 A1 a- ?& l
7 e! _9 m/ ^+ K4 ^4 ^4 [  services:% l' }; Z9 T+ E' }- L
    mon: 1 daemons, quorum controller (age 89m)
( ?+ W! b" f- z  A. k2 y    mgr: controller.horbtx(active, since 89m)$ g2 Q$ j/ ~+ j& d
    osd: 6 osds: 6 up (since 8s), 6 in (since 9m); 1 remapped pgs
# o% k0 w, w1 U$ t8 N
" d: G* g, Z' X: m& _  l+ K% d  data:3 d; B* t! q/ d+ U/ l* W
    pools:   1 pools, 1 pgs
2 c* E$ D: m+ z3 y1 W. A    objects: 0 objects, 0 B
' ^4 V& o2 m. o+ ]    usage:   6.0 GiB used, 114 GiB / 120 GiB avail
0 g* M" l, N% Q% Y* F( u8 v) `    pgs:     1 active+undersized+remapped
" q% n0 j( E' _9 B9 t, E$ }
0 l8 o' @2 |- d( v1 s: \[root@controller ~]# ceph -s
( S. h6 J7 u) G  cluster:
$ ]7 u9 B: j$ e4 m( W    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de/ v) }$ r4 [2 G0 ~+ k) L
    health: HEALTH_OK
, m) M- B  L7 `4 L% c+ }+ G7 V+ T ! g% t2 _: D* Z8 ~3 c% p% D
  services:
# P# a! w1 O) H% t, p% k( u0 \    mon: 1 daemons, quorum controller (age 89m)8 `$ ^: e7 u7 C# V& G) d
    mgr: controller.horbtx(active, since 89m)
+ H  j' Q: D$ q5 U& C& C, E4 z2 T    osd: 6 osds: 6 up (since 9s), 6 in (since 9m); 1 remapped pgs
8 e. ~+ q0 |. A+ H& O
5 h3 h9 Q" y5 k- G1 P  data:
+ _9 }9 W- ?$ W    pools:   1 pools, 1 pgs- Q- W9 d( a  q2 X( {9 B- |
    objects: 0 objects, 0 B+ P( {' q* A# Z' h6 b3 T
    usage:   6.0 GiB used, 114 GiB / 120 GiB avail4 ~5 H; A4 J6 |9 V, {2 @
    pgs:     1 active+undersized+remapped# P# C1 r- m6 y; M
* j# l% w/ Q! r% e3 K
[root@controller ~]# ceph -s
' `$ z/ h* j' C9 C( ?  cluster:9 z& L3 L% y; z8 F
    id:     a4bb5236-c8ca-11eb-a67b-000c29ad02de
* F, z4 n# ?: H  K  B6 E    health: HEALTH_OK
3 T. B; C9 \7 Q3 i 7 J% }4 `- t& Z; j  i  ?
  services:
4 E  T/ a% \* d# L& F- N5 _    mon: 1 daemons, quorum controller (age 89m)$ J/ Y2 J! {' S0 s2 M- Z
    mgr: controller.horbtx(active, since 89m)
, b$ H3 Q( e( h    osd: 6 osds: 6 up (since 10s), 6 in (since 9m); 1 remapped pgs3 M: \, ]4 K" h; z5 N$ D
' @/ N! t& q  \/ C4 w
  data:# M/ n2 Y) S. |7 M& T
    pools:   1 pools, 1 pgs; s! n$ k. G4 X( D5 e7 B2 c
    objects: 0 objects, 0 B
6 G7 |# q4 w4 j6 R    usage:   6.0 GiB used, 114 GiB / 120 GiB avail# K- p5 \% J8 H2 B
    pgs:     1 active+undersized+remapped
+ P$ d3 v! u, {. J" A, l- g
6 Q# W, {7 s8 y3 a' P[root@controller ~]# ceph health detail . n1 l# P& i: O: V0 N7 o3 m3 L
HEALTH_OK" q4 Y5 G) X; W/ {2 P" M
[root@controller ~]# ceph health detail
4 o: o* _3 D. ?0 P# yHEALTH_OK" v# g1 H9 A  q7 g9 y' s/ Q" ?. P
[root@controller ~]#
4 X+ p5 @* X0 v
回复

举报

admin

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2021-6-9 15:00:17 | 显示全部楼层
3.1.1 说明& t2 D% d7 D8 c, N
降级：由上文可以得知，每个PG有三个副本，分别保存在不同的OSD中，在非故障情况下，这个PG是active+clean 状态，那么，如果PG 的 副本osd.4 挂掉了，这个 PG 是降级状态。: E3 k$ C2 R; J$ A
3.1.2 故障模拟
( V+ P# S0 u2 l5 W0 Qa. 停止osd.1
4 `8 n: o* f  n $ systemctl stop ceph-osd@1 ! ?; S! @3 h, ^- a% T
b. 查看PG状态
: M9 w; g( j* m' I $ bin/ceph pg stat 20 pgs: 20 active+undersized+degraded; 14512 kB data, 302 GB used, 6388 GB / 6691 GB avail; 12/36 objects degraded (33.333%)
, _; i4 Q$ f6 [9 R( D# N7 qc. 查看集群监控状态( M0 c7 ^6 ]& O% ^& q/ @5 m
$ bin/ceph health detail
) J4 H) v9 x( w0 o+ H1 `HEALTH_WARN 1 osds down; Degraded data redundancy: 12/36 objects degraded (33.333%), 20 pgs unclean, 20 pgs degraded; application not enabled on 1 pool(s)
; S8 \& t& o7 }: ^+ O) I+ R* q2 DOSD_DOWN 1 osds down     
/ U7 X( m; s- u   osd.1 (root=default,host=ceph-xx-cc00) is down * x% `- @: a" B( x# t+ Q
PG_DEGRADED Degraded data redundancy: 12/36 objects degraded (33.333%), 20 pgs unclean, 20 pgs degraded     $ X6 ^3 r, i8 l. a+ r1 o2 v- i& T
   pg 1.0 is active+undersized+degraded, acting [0,2]     4 b. B! X: z) b- @. y8 r% U
   pg 1.1 is active+undersized+degraded, acting [2,0]
) m. X3 Q, C3 T7 w% ^/ c4 Yd. 客户端IO操作
& z( M1 G) ]! ?6 w& I) Z, W5 \+ L* i" ] #写入对象
8 T' o8 P* w# [- S! j2 l% r$ bin/rados -p test_pool put myobject ceph.conf  6 U/ p  x; a/ y$ W

) L& g& b; ^4 u/ x#读取对象到文件 " ]$ T7 I+ {% u; a. |! X
$ bin/rados -p test_pool get myobject.old  
7 T! k+ G. H. B5 D; D0 y2 ^
" r" C0 Y  u6 F# s#查看文件 ; L2 }, B0 @. K9 H5 E. f
$ ll ceph.conf*
9 h9 N7 [- K+ x6 R2 |! l-rw-r--r-- 1 root root 6211 Jun 25 14:01 ceph.conf
$ u3 Y5 w) t; {- K4 r0 R8 m-rw-r--r-- 1 root root 6211 Jul 3 19:57 ceph.conf.old
0 D, u. H+ r/ }4 A7 b故障总结：
) h2 q# j8 R/ E为了模拟故障，(size = 3, min_size = 2) 我们手动停止了 osd.1，然后查看PG状态，可见，它此刻的状态是active+undersized+degraded,当一个 PG 所在的 OSD 挂掉之后，这个 PG 就会进入undersized+degraded 状态，而后面的[0,2]的意义就是还有两个副本存活在 osd.0 和 osd.2 上, 并且这个时候客户端可以正常读写IO。
" \. `7 O  x( V3.1.3 总结( r: D/ f8 Z/ X* x1 W& |' P
降级就是在发生了一些故障比如OSD挂掉之后，Ceph 将这个 OSD 上的所有 PG 标记为 Degraded。( G, U' k5 V+ d8 p% T
降级的集群可以正常读写数据，降级的 PG 只是相当于小毛病而已，并不是严重的问题。! I, D( z6 W) y2 t) g, b
Undersized的意思就是当前存活的PG 副本数为 2，小于副本数3，将其做此标记，表明存货副本数不足，也不是严重的问题。
6 o* ?2 x8 ]1 @$ n1 O# g, l3.2 Peered4 d% N* B; y. Q  @& R
3.2.1 说明0 E" |. W1 ?& q. E
Peering已经完成，但是PG当前Acting Set规模小于存储池规定的最小副本数(min_size)。$ t# w8 h" l+ @  k
3.2.2 故障模拟2 a# O% z! z9 F
a. 停掉两个副本osd.1,osd.0
0 b$ L) m( {. ~ $ systemctl stop ceph-osd@1
0 t! u2 n# R$ p  x4 _  Q" e $ systemctl stop ceph-osd@0
- \" p. i5 N2 \: t/ _  M+ R! w3 @
# Q, U& H% t) h% U' E1 A% K$ u1 B: v+ I
3.2.1 说明. f  `- q2 F  @% t
Peering已经完成，但是PG当前Acting Set规模小于存储池规定的最小副本数(min_size)。# D/ o; `$ m+ v: `) n
3.2.2 故障模拟7 b6 ?% M! w% d
) `) o' K+ c( h5 t, u4 z
a. 停掉两个副本osd.1,osd.0
8 F! S/ r) N( n" T3 G) J& y* R
9 B* d" o* ^& k1 C% B$ E $ systemctl stop ceph-osd@1 9 c6 ~# ?3 g6 O2 u2 G7 B
$ systemctl stop ceph-osd@0
3 g- K/ M& O6 o/ R9 t
$ B+ X" Y3 C9 r: `b. 查看集群健康状态
: K) y# i0 t! s5 j: M9 U  O; C8 i0 e* p
: l0 i2 J8 i( z0 L2 a+ P $ bin/ceph health detail
0 D1 ?0 r0 q- E2 ]HEALTH_WARN 1 osds down; Reduced data availability: 4 pgs inactive; Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded; application not enabled on 1 pool(s) 3 t* P( w6 X' \! a  q# P
OSD_DOWN 1 osds down     + R; i: a& }2 b8 t, j
    osd.0 (root=default,host=ceph-xx-cc00) is down
0 C7 N" @& I7 ?: u5 {; B; wPG_AVAILABILITY Reduced data availability: 4 pgs inactive     / ?  m/ g3 g; D0 T( I; y+ M3 `
    pg 1.6 is stuck inactive for 516.741081, current state undersized+degraded+peered, last acting [2]     . d7 h2 ~- Y0 ]+ x
    pg 1.10 is stuck inactive for 516.737888, current state undersized+degraded+peered, last acting [2]     & g1 @! M1 F( }" V* f( m
    pg 1.11 is stuck inactive for 516.737408, current state undersized+degraded+peered, last acting [2]     
2 G& D8 V' j! y! v% z5 h    pg 1.12 is stuck inactive for 516.736955, current state undersized+degraded+peered, last acting [2]
/ B! S' Z" m( |% [+ W( ~. SPG_DEGRADED Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded     % a% o$ m: B: W' ]6 M- q6 O# L
    pg 1.0 is undersized+degraded+peered, acting [2]     
2 e2 Z' }$ t) x. f* B+ ], s    pg 1.1 is undersized+degraded+peered, acting [2] 5 v7 |( ]2 G7 J& L/ u7 E) j5 u; }
c. 客户端IO操作(夯住): e& Q* L& F; [% D% S5 d) u

7 F( m3 x/ {  t7 R- {3 { #读取对象到文件，夯住IO " N% G; z8 M% |
$ bin/rados -p test_pool get myobject  ceph.conf.bak - `5 q. ^$ J0 r! l$ D8 I/ Z
故障总结：$ L2 o& |: G3 i( y

9 V  _1 a6 B  S4 J现在pg 只剩下osd.2上存活，并且 pg 还多了一个状态：peered，英文的意思是仔细看，这里我们可以理解成协商、搜索。- q+ s, t, z6 ]1 _5 C
这时候读取文件，会发现指令会卡在那个地方一直不动，为什么就不能读取内容了，因为我们设置的 min_size=2 ，如果存活数少于2，比如这里的 1 ，那么就不会响应外部的IO请求。
1 Z% J8 n3 L$ q. b% V' Y9 rd. 调整min_size=1可以解决IO夯住问题
# o# }0 a* o+ w, V
" w6 G/ {- x% v #设置min_size = 1 2 a1 c4 |9 h% J, O9 ~+ G3 ^
$ bin/ceph osd pool set test_pool min_size 1
4 c6 j4 y( T& Q; nset pool 1 min_size to 1
2 O; z/ c/ k5 Y, ~+ ?1 n) o2 r! Y4 D9 me. 查看集群监控状态9 K7 K: |: H0 g# Q1 u' d$ F& f

! _7 D' l3 h1 Q- m- ` $ bin/ceph health detail * ^! E2 S9 l$ Y! y0 i9 H
HEALTH_WARN 1 osds down; Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded, 20 pgs undersized; application not enabled on 1 pool(s) # v8 \5 [" J+ V. T' O4 i& N
OSD_DOWN 1 osds down     
4 P3 W# l* [( @/ A   osd.0 (root=default,host=ceph-xx-cc00) is down。. b" O2 S& Z! ?- o: U% q
PG_DEGRADED Degraded data redundancy: 26/39 objects degraded (66.667%), 20 pgs unclean, 20 pgs degraded, 20 pgs undersized       \2 `/ f' @2 @2 V; |
pg 1.0 is stuck undersized for 65.958983, current state active+undersized+degraded, last acting [2]     
9 t4 r% b% v- o* m! Z, v' Npg 1.1 is stuck undersized for 65.960092, current state active+undersized+degraded, last acting [2]     
. C, P2 K! L5 {pg 1.2 is stuck undersized for 65.960974, current state active+undersized+degraded, last acting [2]
, f! f6 e; o* g1 T- X- |f. 客户端IO操作+ k" E# H2 V- D8 m0 g9 x  e
# k* V* ]( ]  v$ j# b9 h
#读取对象到文件中
. v) q9 C/ ?1 s1 |7 ~& n$ ll -lh ceph.conf*
" w9 a" f7 c* \' b1 U+ O; l6 R5 s-rw-r--r-- 1 root root 6.1K Jun 25 14:01 ceph.conf
0 a/ l( K7 L. f# t0 ]-rw-r--r-- 1 root root 6.1K Jul 3 20:11 ceph.conf.bak 0 g$ w/ L; j0 N. }; [
-rw-r--r-- 1 root root 6.1K Jul 3 20:11 ceph.conf.bak.1
& y! I% ^. X5 U5 |( E# w/ R故障总结：8 ^- R! _, O, k. b7 f  @- M
) o+ r* a- u  ]
可以看到，PG状态Peered没有了，并且客户端文件IO可以正常读写了。
+ E. ?! K! d; X3 ~当min_size=1时，只要集群里面有一份副本活着，那就可以响应外部的IO请求。
8 Z' b- S+ ?" x% i, T& }0 V; s# X5 r; ~% l
回复

举报

返回列表 发新帖
高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 | 注册

本版积分规则

返回首页|Archiver|手机版|小黑屋|易陆发现技术论坛 ( 蜀ICP备2026014127号-1 )

GMT+8, 2026-6-12 02:06 , Processed in 0.024268 second(s), 22 queries .

Powered by Discuz! X5.0

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表