|
|
模拟删除mds 元数据恢复过程，
4 `9 R+ v; L" `, M测试环境ceph版本号： ceph L版本：+ d7 b; @: I3 G. ~5 ?5 q; a1 r
[root@ceph-3 ~]# ceph -v
# b1 V6 v" T ]ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)
( @7 Q) E, e k ~5 y7 U; V) B$ M! g1 ^( H9 Y/ @
7 x3 s& A) g+ G$ n* x检查osd的状态：" O$ L) R( X% ]6 V
[root@ceph-3 ~]# ceph osd tree
) F0 K& _" J# Y7 [! K9 `0 t) N9 A; zID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
5 p9 \6 X0 K: ?7 W) |% L-9 0 host root # w! n- q O( o8 Z1 ~* w7 [
-1 6.00000 root default
$ @6 e( b! c" D9 K2 [1 Z6 M% o- |% e, c-2 2.00000 host ceph-1
' [1 r& v( d" k" n3 _/ D 0 hdd 1.00000 osd.0 up 1.00000 1.00000
8 C% b/ S6 u' T! L4 E7 a 1 hdd 1.00000 osd.1 up 1.00000 1.00000
' T+ w$ @. o# T2 ~-3 2.00000 host ceph-2
1 K9 z( l; S; D+ p/ W 2 hdd 1.00000 osd.2 up 1.00000 1.00000 : V4 }0 h# W K; `8 a: U' |/ ^
3 hdd 1.00000 osd.3 up 1.00000 1.00000
8 b* ^) r; W$ E% }# y4 |3 e3 K* Y# A5 x-4 2.00000 host ceph-3
( @! x9 Q3 z5 }! M# I+ F) R 4 hdd 1.00000 osd.4 up 1.00000 1.00000
3 q0 O) B+ }1 k2 c5 f 5 hdd 1.00000 osd.5 up 1.00000 1.00000
q3 B/ }" k% k1 Y" z/ a# g
( m: M( s0 S4 @2 E: aceph 容量使用情况:. M; M" G2 N. O' [
[root@ceph-3 ~]# ceph df
: A$ s! E' H9 g* }GLOBAL:
9 ?! J+ v7 p) S6 @ SIZE AVAIL RAW USED %RAW USED 2 G. s' ^! M) k1 Z' ^1 p
585GiB 535GiB 50.2GiB 8.58 - }4 K+ s# z3 o+ y8 [
POOLS:9 U3 W \6 I" @ X: B; {! V
NAME ID USED %USED MAX AVAIL OBJECTS
' X; c$ y: E6 u8 f7 M6 [: p) j& j metadata 6 8.03MiB 0 167GiB 23
6 F A, x5 t+ J/ C fsdata 7 16.5GiB 8.96 167GiB 5016 ' f( p+ A9 p- I1 T% F7 l
recovery 8 2.19KiB 0 251GiB 21
. |, c% i8 d% { Y, Z( M* f6 o1 ^) f3 R
# \+ F n A* \: b, W; j6 e' r4 t
[root@ceph-3 ~]# ceph df
9 d. v7 I/ C( d) q0 ZGLOBAL:
! }: n6 ~: ]) |9 f h, v+ s SIZE AVAIL RAW USED %RAW USED
3 f! | y/ t) `& z) V" D; A 585GiB 535GiB 50.2GiB 8.58 0 t$ }8 F8 b2 W+ h) N$ e% p& B: \
POOLS:
9 D+ _/ G3 N8 { NAME ID USED %USED MAX AVAIL OBJECTS 6 c1 |# u* o1 q
metadata 6 8.03MiB 0 167GiB 23 , F$ \- e: F" @- R7 g2 K
fsdata 7 16.5GiB 8.96 167GiB 5016 9 G6 w* P$ Y/ o1 Q, X- M! Y& q( m
recovery 8 2.19KiB 0 251GiB 21
$ ~% e3 W% x, ^8 n/ ^6 M9 m% C6 |4 ?" A! C9 p7 k9 ^" ?0 I' [
% V( a5 K( {0 l2 b: S2 n5 V0 ]8 x( V& L/ o a& |2 j
查看metadata的数据记录：! m; U7 }5 g. V9 C0 j
[root@ceph-3 ~]# rados -p metadata ls8 k. X$ ]# d X- _3 p2 G
601.00000000 C# i7 |5 d, T8 ]1 V
602.00000000- v" Q( W' Y% A8 z
600.00000000' ~2 h8 q7 g0 A" S6 G: G" d% S
603.00000000: {/ f7 h0 x7 u6 S- l) Y. m j
1.00000000.inode4 O# ^+ l- \# M; U% A
200.00000000
2 P+ p5 z# r# d$ N2 c+ o200.00000001
' s, w r/ L' Y606.00000000
3 A: r1 u* ~; j5 x h9 L607.00000000
& C' Q0 y/ k" x; |8 ]' U608.00000000
# H; l: }% c6 }4 C* F# y6 S% g6 Y604.00000000; v+ N" b1 w, U/ Q7 p
500.00000000
% M! X5 ~/ G% s, \0 b4 v0 tmds_snaptable
* q, v7 B- j5 w$ G* v605.00000000' g- d4 W$ t4 [) z6 _: I
mds0_inotable
/ c2 X+ p; A3 U; f: \7 n0 C7 Z- y" Q100.00000000
/ }4 N/ L5 U( p' }3 u" E; Emds0_sessionmap
* K5 x, l6 d& {$ }% g200.000000037 t# e( T" R* F! U4 w
200.000000028 {; h+ I$ @" U* {
609.00000000+ v* ?4 O2 o$ k3 ^) B% B# n* b
400.00000000
1 J k" r: T- `: y0 l; m- U) i \100.00000000.inode- k* i3 H& }# V I k; w
1.000000009 o0 `/ z" ]! O& T1 T
' A& w5 R4 E+ N: q: ^
" ?, ~7 }6 ~' @- L4 |; r" G
模拟删除metedata pool池的源数据：7 b2 q x' E( [0 u
[root@ceph-3 ~]# rados -p metadata ls |xargs -i rados -p metadata rm {}
) x7 Z1 n1 D- n) T# F( k: L W, Z- q[root@ceph-3 ~]#( V5 `1 p, U) H
检查还在不在：4 P7 Y3 n! A- L+ I9 N& B
[root@ceph-3 ~]# rados -p metadata ls
- G9 ?" T. ?9 X- I; ^& U[root@ceph-3 ~]#+ R5 l/ b1 D" O4 F7 u
$ t9 B) R; G% n+ g( x
! N' A5 _& p+ P3 A: s1 w$ O检查状态：
2 A( i: ]9 W p* R Every 2.0s: ceph -s Fri Aug 5 09:37:17 20228 F4 k" x* u% c$ e
cluster:5 z. h5 F- x9 ]. L
id: 57bf4711-2218-46af-99d6-9c68ae230ce10 S! d! B7 ]5 P. i. A
health: HEALTH_OK' Y3 O( Y3 l2 b1 U# ^9 _
5 A+ f6 `6 h! Y
services:! g' d2 @. f) T2 |) O& @, ?
mon: 3 daemons, quorum ceph-2,ceph-1,ceph-3$ w6 M" H. T" L2 e0 ]( k; k
mgr: ceph-2(active), standbys: ceph-1, ceph-3' G$ _- j2 e, L( V
mds: filefs-1/1/1 up {0=ceph-2=up:active}, 2 up:standby0 u6 B3 k4 t' p( m" @+ I; M
osd: 6 osds: 6 up, 6 in
2 p; T$ Q/ n0 {" D
9 y! U- H$ a0 A/ R% O/ R data:
) `" D) ~- A: W4 n pools: 3 pools, 1280 pgs% M! {: P, C6 F5 I
objects: 5.04k objects, 16.5GiB3 }: h# \0 B( Q' _( E
usage: 50.1GiB used, 535GiB / 585GiB avail: D: e m+ F: X
pgs: 1280 active+clean
$ G4 b* q5 J' x7 ]4 J8 n9 V7 E4 R9 a5 f/ Z3 ^( o! a) i
- r% B, ^ t9 @4 O% d/ x( q# H看到metadata池里面没有对象了已经，重启下mds看效果，因为mds里面会缓存元数据信息，所以要重启下mds：( R- U, w: a. Y, n" r5 T' N
3 z2 Z; Y3 C Q4 K: }
[root@ceph-3 ~]# ceph df
) X- i& i L0 A. o( C4 EGLOBAL:
7 M' E4 J: z' I7 N( \5 u SIZE AVAIL RAW USED %RAW USED
) I+ i2 o9 _0 g% B" ^. m; x 585GiB 535GiB 50.1GiB 8.58 - t- x- Y1 D. j3 U& y
POOLS:% N/ u5 s* y5 B
NAME ID USED %USED MAX AVAIL OBJECTS
& E7 j! B' I+ g- R+ _8 v; { metadata 6 0B 0 167GiB 0
! I* j0 U ^ x0 i fsdata 7 16.5GiB 8.96 167GiB 5016 : u- z( v5 k, _7 R
recovery 8 2.19KiB 0 251GiB 21
x, T0 m/ a* ^: g/ t检查ceph fs文件系统：9 M# [7 }8 _6 ]0 r8 ~ Z
[root@ceph-3 ~]# ceph fs status filefs - 0 clients2 T- }9 b+ a+ y2 Y# Q8 C0 ]3 ?
======4 n* P5 j: b/ m
+------+--------+--------+---------------+-------+-------+$ F+ Y( f! q4 O# b, h
| Rank | State | MDS | Activity | dns | inos |, a A; T6 j3 q: f: Q
+------+--------+--------+---------------+-------+-------+- l6 O- l( V8 H5 a/ N% f
| 0 | active | ceph-2 | Reqs: 0 /s | 1001 | 1003 |
" x I' L: r2 ]( j3 ~! v* f+------+--------+--------+---------------+-------+-------+
6 r& f$ G/ v+ V R t+----------+----------+-------+-------+! R; K9 h6 Z6 y$ e2 I( Z: j- V
| Pool | type | used | avail |6 F8 o( [ B: w5 B f
+----------+----------+-------+-------+
. v* Z; {, O, U4 |2 J| metadata | metadata | 0 | 167G |" [$ e" B" C9 ]7 D8 F- H
| fsdata | data | 16.4G | 167G |
7 c& o# ]3 n: u& O3 H2 F, A* L+----------+----------+-------+-------+
1 n8 U; r) s A4 ]3 F
; T& E6 O. q6 P7 I2 L2 M/ H+-------------+
, ~- l6 j5 i$ }, T1 F| Standby MDS |9 Z4 u. ^6 P5 F/ G
+-------------+0 |: y2 ]! j) V' F$ ~7 }/ Z; x
| ceph-1 |
/ q$ [& J) n+ e4 Q7 C5 B| ceph-3 |
7 u2 y$ c! I. A2 r+-------------+
& e. X0 T) Y8 C; _MDS version: ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)- z6 O# J6 x& h6 O
[root@ceph-3 ~]#
$ ]7 W5 S, w: X% R6 \; [( F# p0 _1 L1 [- q' q1 b) U& h
重启mds服务：) c( B' H+ @! h$ Y" U3 ~
[root@ceph-2 ~]# systemctl restart ceph-mds@ceph-2.service , t9 }+ r, X" H& o i4 y; O
* w8 P( p2 G# V' x e4 l9 x9 C& R
查看ceph 的状态：3 s1 ?8 a( p; S; \5 u# q
- o3 ?7 t$ Y, A' x. e
[root@ceph-1 ~]# ceph -s: p9 N7 V! ]6 H! ^: D7 {
cluster:/ e% z' G) [7 ?% ~! R0 F5 I. ?
id: 57bf4711-2218-46af-99d6-9c68ae230ce1
0 l4 @+ p+ o2 K( | health: HEALTH_ERR) d T" G5 O/ B
1 filesystem is degraded y- }. |0 n6 u3 S! n% {
1 mds daemon damaged, t f7 a2 C$ p" n
- Z- y1 g* u. B2 i% E; K2 B2 l services:3 E' j, Y1 ^- v/ K W" l
mon: 3 daemons, quorum ceph-2,ceph-1,ceph-3" P2 x% j8 w. k; w) T8 T
mgr: ceph-2(active), standbys: ceph-1, ceph-36 W9 Z7 i5 }$ o1 [" D9 `# Z
mds: filefs-0/1/1 up , 3 up:standby, 1 damaged
* V: H1 F; x- E* u osd: 6 osds: 6 up, 6 in
3 ]* E# c1 {6 Z+ j% g8 H2 q% K: |3 x& ^8 y6 ~0 ~: F
data:
8 Q5 N& _' C0 r$ n( F pools: 3 pools, 1280 pgs# `2 r- n: C. W. C8 ?) ~" V
objects: 5.04k objects, 16.5GiB3 E/ O! S# j% q) t5 w6 t
usage: 50.1GiB used, 535GiB / 585GiB avail0 x8 l4 m& x# {) z4 f
pgs: 1280 active+clean2 C% [2 ]/ [& u
( b0 q5 K: u$ ]) n9 a5 L8 k. _
[root@ceph-1 ~]# systemctl restart ceph-mds@ceph-1.service 5 s1 i) J% Z. q/ P4 Z
[root@ceph-1 ~]#
' P% L9 t \: W( ~$ Y2 R4 H( @1 ?/ s5 R- f( i4 \; ?
[root@ceph-3 ~]# systemctl restart ceph-mds@ceph-3.service
. _3 f8 r6 P, J# ~% V
- r$ g, U* N+ V3 Y! ~( n ?[root@ceph-3 ~]# ceph fs status ' J: P0 O5 K# T
filefs - 0 clients, k) ^/ z5 T$ T; ^& w
======- s; l1 a0 {# k5 \) G2 M
+------+--------+-----+----------+-----+------+% H2 n: p& z+ I/ ]
| Rank | State | MDS | Activity | dns | inos |: v6 k7 V+ D+ g
+------+--------+-----+----------+-----+------+
+ `# p6 `6 u( ]# O# U| 0 | failed | | | | |
, p' M& ^% O2 v/ V4 ^2 ?+------+--------+-----+----------+-----+------+, @" D2 @1 H8 Y+ E2 S* |7 A/ Q
+----------+----------+-------+-------+
; J8 E' t8 b: j( Q1 I| Pool | type | used | avail |& Z6 P6 v; B: P3 p5 Q
+----------+----------+-------+-------+
5 M: I9 ~2 }+ M6 [- d| metadata | metadata | 112 | 167G |
( Z o- [: R0 s2 y| fsdata | data | 16.4G | 167G |' Y6 }) _) l' `
+----------+----------+-------+-------+
& b: T5 p, W, U9 }1 E
2 d2 F( A' K% ]$ g( Y+-------------+
. C" M* n: r+ v| Standby MDS |
2 ?; t, {4 {! T3 a8 f+-------------+
! |* b: o+ s4 Z| ceph-2 |9 `# b- M3 e/ b' C
| ceph-1 |% t, |: a0 X3 u5 C) O
| ceph-3 |
8 q3 ^5 f4 D8 ]5 v9 e' ]0 G6 h0 W# f+-------------+
% t) O' u/ _- v4 T/ L5 d; i8 iMDS version: ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)
r4 `6 t0 U# e) p" G2 y8 G[root@ceph-3 ~]#/ y: q& r- X5 ~. \" N
看到集群现在不正常了，访问kc里面的数据卡住，说明数据已经无法正常读取了。9 ?/ V4 [6 J# k$ }! L9 s% R
( W3 {; W1 b; F7 h
: a+ D& @0 q" m0 w& O[root@ceph-3 ~]# mount -t ceph 192.168.120.31:6789:/ /mnt/mycephfs/ -o name=admin,secret=AQBH+tRiATMVCRAAdTQnt4IFUWD45zGEZQa7A==
3 I/ Z: L2 ~6 Dsecret is not valid base64: Invalid argument.
: W8 I7 Q6 H) w. e3 r8 [1 Y: qadding ceph secret key to kernel failed: Invalid argument.4 t9 q( U2 y; T+ i. T, d
failed to parse ceph_options; W% r8 w3 ^ e) e0 Q. d, i
: t9 k, q* i2 `
[root@ceph-3 ~]# ceph df detail 4 X6 h# _' M8 D8 M1 N& m1 r
GLOBAL:
% {2 `3 f: v* A' @* ~ SIZE AVAIL RAW USED %RAW USED OBJECTS % \0 ?* u5 y8 P1 ~# a% D
585GiB 535GiB 50.2GiB 8.58 5.04k
: [. M3 A7 ?. T/ M+ pPOOLS:+ P7 f& k/ _ n! Q& ]6 Q }- e
NAME ID QUOTA OBJECTS QUOTA BYTES USED %USED MAX AVAIL OBJECTS DIRTY READ WRITE RAW USED + O# m9 g, ~$ ]& E/ e. w; Z
metadata 6 N/A N/A 112B 0 167GiB 2 2 176B 1.25KiB 336B % J: J6 ~8 ^7 J' {5 |% Q8 i
fsdata 7 N/A N/A 16.5GiB 8.96 167GiB 5016 5.02k 3.19KiB 4.91KiB 49.4GiB
2 A4 g# e3 N2 @3 z recovery 8 N/A N/A 2.19KiB 0 251GiB 21 21 32B 44B 4.39KiB
# ]* ?4 H8 q7 s. i, l+ x: j: K[root@ceph-3 ~]#
0 i) z8 W$ W/ J4 @: f
6 f/ g+ V7 r$ @' {) U k; [/ p) L+ X开始恢复使用一开发者编写的py脚本(文末给出了源码)恢复，把脚本放到集群任意一台节点上执行： ( O+ a% V) u6 g5 ]
[root@ceph-3 ~]# python recovery_cephfs.py -p fsdata
, N/ }2 E# I8 L/ u9 W) N* n- |4 B, |3 b
在日志里看到的问题：
0 l; L# q7 c; o2022-08-05 09:59:03 : INFO exec_cmd():: cmd: for obj in $(rados -p fsdata ls);do rados -p fsdata get ${obj} /root/recoveryobjs/${obj};done8 a* ^/ P% `1 r- P2 f% V5 E! m, v- x
2 o/ g! @6 b4 f7 ^
* \9 u* G( M, q8 C8 c* }" a! s1 ~
[root@ceph-3 ~]# tail -f recovery.log ) n- @& O/ K& d4 ?8 I- P
7
& o7 L( j( [; `; Y; ?5 _1 v ],9 A7 ~4 K- r8 M7 R* P3 ]$ Z
"data_pools": [
/ l8 A5 o, i. y% Q "fsdata"
; E7 u W0 J" r; c6 I! |$ T ]
/ M3 Y! D! S( Z2 J+ l }2 G5 \, _% M! x+ ^0 x, {) a B
]3 E4 q% O( Q8 M" l- I
, code: 03 R- H9 f, ^; c# Z7 w v. C7 g" [) m
2022-08-05 09:59:03 : INFO === main():: recovery start& Q& e9 H. p4 F( Y( p* e- B; a
2022-08-05 09:59:03 : INFO exec_cmd():: cmd: for obj in $(rados -p fsdata ls);do rados -p fsdata get ${obj} /root/recoveryobjs/${obj};done
6 ]; Q, Z" k( |$ i3 k6 u4 ?& ?4 s. j
- n J" J9 J( X9 c3 g9 G. g2 V# b2022-08-05 10:35:36 : INFO exec_cmd():: cmd exec out: , code: 06 c7 I" S" N* Q: H5 X
2022-08-05 10:35:36 : INFO exec_cmd():: cmd: ls /root/recoveryobjs: y( v4 d/ B i) T: X Q/ o
3 R! `! v4 Z( h' ^
- p% Q4 G- ~# u; e9 C, t4 y) z
9 \4 Y( A8 U- `4 H
- |/ R9 t# ]9 g; h, w: [6 G |
|
发表于 2022-8-5 09:29:06