|
|
楼主 |
发表于 2022-8-4 13:39:30
|
显示全部楼层
1、前言7 O& |4 x+ R K9 x X3 V
这里实验在使用cephfs时，如果cephfs的元数据损坏或丢失了，那该如何恢复出用户数据。下面就为大家演示下如何恢复。1 H7 d0 S* ^$ c
; t' P8 S( z3 _- Q8 w8 j& E7 r
2、准备测试环境
# D B' n6 I7 O2.1、准备测试集群
' B, ^# S' Y, t3 |8 z我是基于L版本做的实验，J版也是可以的。测试环境如下：& B7 |' T7 y! w5 G" i- b9 u
+ l7 n/ }' i1 G2 U8 _[root@ceph05 ~]# ceph -v
6 F# z: u2 J9 g5 Vceph version 12.2.11 (26dc3775efc7bb286a1d6d66faee0ba30ea23eee) luminous (stable)& Z3 l, H8 _0 Y' {* B' ~
j2 Z: d5 E6 r; \) L) P! @
[root@ceph05 ~]# ceph osd tree
' B4 T5 i) [4 fID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
' r0 b6 b+ e( V- b& v-1 0.07397 root default
& |6 b1 \; A) I" d-3 0.03699 host ceph05
, a. c* Y8 W) } 0 hdd 0.01799 osd.0 up 1.00000 1.00000 2 R5 t) y- H8 C* C; s
1 hdd 0.01900 osd.1 up 1.00000 1.00000 5 {# I2 y$ j5 F
-5 0.03699 host ceph06
2 i0 W1 \2 Y+ E 2 hdd 0.01900 osd.2 up 1.00000 1.00000
; R* _& G ?% z5 K 3 hdd 0.01900 osd.3 up 1.00000 1.000007 w9 A1 L$ t: r: g8 u7 }% D. |+ N
+ g/ Z; U" x8 p! X$ X% W e
[root@ceph05 deployceph]# ceph -s
+ F, L( {! X% A' o& d& J cluster:
0 y# c! d% X# I2 [( u4 P id: 176feab8-ca22-47bf-b809-202deac53c6f% p6 S# q3 }# e# k
health: HEALTH_WARN) K# y' r& ~6 ~7 T$ K% p
crush map has straw_calc_version=0' U4 n; s# T/ k ~" ?' Y% s
9 C2 Z9 i$ f0 |
services:
9 q2 W! c4 c/ |3 Z: D% `3 d2 ]" E mon: 1 daemons, quorum ceph050 Q5 V9 S- N4 y: d
mgr: ceph05(active)
2 y( o3 U' t( s" A mds: cephfs-1/1/1 up {0=ceph05=up:active}- w& l" T+ n# X; k" h. t
osd: 4 osds: 4 up, 4 in
$ I+ _: d$ D f0 F& _, H) z( l2 d; a/ S4 j ^- A
data:
{+ P$ ]! A+ U# |9 ]7 r' a+ N pools: 10 pools, 304 pgs
& c' d! H/ w" y objects: 918 objects, 2.60GiB
- G; F& E, d$ h, v usage: 6.70GiB used, 71.3GiB / 78.0GiB avail* }# z3 M, n5 B
pgs: 304 active+clean/ a( A7 [! l) `1 s) K' U
2.2、准备测试数据1 T$ [# s5 B* O, D
挂载kc
- H4 o) c+ [! Y; F) P M0 D& C! S: ?, X8 j$ a0 K8 |
[root@ceph05 deployceph]# mount -t ceph 192.168.10.30:/ /cephfs
8 {( u- g4 C5 x7 u; r( i+ n[root@ceph05 deployceph]# df -h|grep ceph# z% S2 y7 A7 f! K3 @, s6 c) w
···
/ r# {' d! t0 C) P# e192.168.10.30:/ 78G 6.8G 72G 9% /cephfs0 T6 ^7 d7 y& g9 f0 J
···+ a: y. A$ f6 `' n8 \
写入数据（这里我写入了几个比较有代表性的文件类型：txt、jpg、png、pdf、word、excel）- ^5 O( Q9 R* ~& A6 C
5 a% b+ k$ V# r1 X[root@ceph05 deployceph]# ll /cephfs/9 y3 l. G4 S, f
total 5912
: F% j: Y4 Z' r& O* K& B. v-rw-r--r-- 1 root root 31232 Mar 15 12:18 111.doc
) B0 f" k8 \7 d }4 C9 k-rw-r--r-- 1 root root 20593 Mar 15 12:18 22.xlsx
) [( ^: r7 |4 b-rw-r--r-- 1 root root 12494 Mar 15 12:17 5be23a3eec2c0.png
! U1 }9 R, b8 T. ]% H- Y; H0 |-rw-r--r-- 1 root root 3189 Mar 15 12:17 cmap.txt& i) p0 {0 u! A
-rw-r--r-- 1 root root 5985243 Mar 15 12:17 hello0.pdf- z" c2 n1 g/ n* Y& p0 Q
3、模拟故障, ]0 ~" e; b3 [( J0 j) u
这里直接模拟元数据丢失的情况，删除metadata池里面所有的元数据对象：0 Y4 u/ U- f6 A7 D8 n& d5 e
+ H/ k: O! Y. t& L! o s G[root@ceph05 deployceph]# rados -p metadata ls|xargs -i rados -p metadata rm {}
+ _; @8 H: `8 i5 q( @% S[root@ceph05 deployceph]# ceph df- [! |: i, V. X) Z
GLOBAL:6 E: s$ E* ]+ c, r* `( p% ?+ {
SIZE AVAIL RAW USED %RAW USED , }5 `& j# ^5 h0 Q; v8 N3 G! a
78.0GiB 71.3GiB 6.71GiB 8.60 / Z$ X0 U0 {* Q
POOLS:
$ J; C. e I9 e6 G% V7 U NAME ID USED %USED MAX AVAIL OBJECTS / u5 ~3 r6 {$ F2 L
.rgw.root 1 1.09KiB 0 63.3GiB 4
) x9 h% j* K4 u [% x default.rgw.control 2 0B 0 63.3GiB 8 % P: `( ^: J9 }, m- R. @& ]# t" Y
default.rgw.meta 3 720B 0 63.3GiB 5
7 i/ V. K$ C( y( @+ H default.rgw.log 4 0B 0 63.3GiB 207
$ U) \) H8 r% Y9 k9 `9 ? default.rgw.buckets.index 5 0B 0 63.3GiB 1 / {# F, I1 G" B" {1 W
default.rgw.buckets.data 6 1.02KiB 0 63.3GiB 2 ! b/ f7 \, y) J: [( b- F) v- S
pool01 7 2.60GiB 3.94 63.3GiB 666
+ Q4 g3 i1 j$ z1 h# O rbd 8 36B 0 63.3GiB 4
+ A! j/ Q7 Q. H1 v( q# c! h2 H# ] metadata 17 0B 0 63.3GiB 0 " d& F" `1 B& K4 ] ~
data 18 5.77MiB 0 63.3GiB 6
' d2 _5 v3 ?- \7 l看到metadata池里面没有对象了已经，重启下mds看效果，因为mds里面会缓存元数据信息，所以要重启下mds：
5 F0 q) a0 m/ i/ v! _* G% |" J) ^; O6 B7 Q+ `% z* T O
[root@ceph05 deployceph]# systemctl restart ceph-mds@ceph05
# z% A' x( K9 s Q/ K9 k0 \4 m3 \[root@ceph05 deployceph]# ' {1 v) S. L: u$ s& d& m( v
[root@ceph05 deployceph]# ( a2 M J8 @8 B0 B; }! n. M
[root@ceph05 deployceph]# ceph -s F. N+ r5 z8 V
cluster:
( b5 B S& u3 W/ J4 o, o8 b7 M id: 176feab8-ca22-47bf-b809-202deac53c6f
Q9 |. N3 B+ }7 {; F4 ~; ? health: HEALTH_WARN
" P# J2 E7 d1 g0 O6 f 1 filesystem is degraded
Y6 Q6 }/ f: R: e9 T! F3 ~: X 1 filesystem has a failed mds daemon
' D8 `7 h$ [0 n. O crush map has straw_calc_version=0
- ~% F0 t* _3 f& L8 v
1 Q7 T) X6 v# g! K7 b services:* F1 ]- L/ P( n& R* s
mon: 1 daemons, quorum ceph05
7 C% |" ?2 c* M4 r mgr: ceph05(active)
- B2 y* A6 f: \& R: u mds: cephfs-0/1/1 up , 1 failed h5 \0 j+ O4 |. K/ q( D# k
osd: 4 osds: 4 up, 4 in/ ]0 G5 h& x# G& {% x8 B# I3 p! c
+ ^( p8 M6 u( k+ A9 c1 M* X t* \& [
data:5 D4 `' Z$ |, c5 I/ k& o
pools: 10 pools, 304 pgs
& r4 |# g" f. y' ~ objects: 905 objects, 2.60GiB# F6 \) ?4 I. Q3 n! ~
usage: 6.71GiB used, 71.3GiB / 78.0GiB avail
2 \6 i2 s/ u0 k* A' P2 W7 F pgs: 304 active+clean$ w; }- T1 _& b3 _3 h( ~
看到集群现在不正常了，访问kc里面的数据卡住，说明数据已经无法正常读取了。# q" Q9 h! ^" Z% [/ H+ S: C3 {
) B) _! B& q, U6 g
4、开始恢复* a# ]/ R# b/ p
使用我编写的py脚本(文末给出了源码)恢复，把脚本放到集群任意一台节点上执行：
0 k6 _1 H, n7 |
; v0 ^! X ?& v Z( b% {[root@ceph05 rcy]# python recovery_cephfs.py -p data a# q3 C; h/ L8 a9 i h2 W4 V/ o
-p指定cephfs的数据池，运行完之后会在当前目录下产生两个文件夹和一个运行脚本的日志文件。& |* n y. t1 N
5 ]" ]0 C# I' y; k$ ]+ C[root@ceph05 rcy]# ll3 l# G, h. s5 ]/ m% g5 [/ N
total 16
9 [2 @* \+ b8 V, d-rw-r--r-- 1 root root 3826 Mar 15 13:57 recovery_cephfs.py
; n6 n$ H5 a% k) z; ?drwxr-xr-x 2 root root 120 Mar 15 13:57 recoveryfiles
3 G; T+ P7 i- ^- P-rw-r--r-- 1 root root 4804 Mar 15 13:57 recovery.log" u* @, G" m( y) C# C9 Z
drwxr-xr-x 2 root root 4096 Mar 15 13:57 recoveryobjs+ J# _( I1 c: f& R2 ?9 p# [6 u
查看恢复出来的文件在recoveryfiles文件夹下：6 r2 E% x" H3 q1 H
/ y1 L; b9 G' I3 M" |) {[root@ceph05 rcy]# ll recoveryfiles/( ^ i- y8 O# Y5 h B9 ~1 Q! j
total 123645 Z; r$ z. X' ?' B
-rw-r--r-- 1 root root 5985243 Mar 15 13:57 10000000000-pdf. X b& T2 W$ G8 d4 ?" a" @
-rw-r--r-- 1 root root 3189 Mar 15 13:57 10000000001-text& u5 C# y- S5 s6 o+ n% c8 ^
-rw-r--r-- 1 root root 12494 Mar 15 13:57 10000000002-png3 j4 Q w% R7 U# T: w
-rw-r--r-- 1 root root 31232 Mar 15 13:57 10000000003-text
% p! D8 D7 t# W3 o, T+ }. s% H-rw-r--r-- 1 root root 20593 Mar 15 13:57 10000000004-excel
9 u- O5 l g% B$ s文件名格式为”文件在cephfs里面的inode-该文件可能的类型“。恢复出来的文件名后面会给出该文件的类型。这样就可以使用合适的软件打开该文件来验证文件是否完整。
# h2 o2 j5 t$ W3 T, o
! k% Y v" r2 ~2 Y& j5、总结
1 R, h0 A# T2 a' p8 @3 m! y在cephfs文件系统的元数据完全损坏的情况下，只要数据池对象不丢失，就可以恢复出完整的数据。恢复的思路如下：% \, m$ _/ Q, i% `, M4 y3 `6 j
) A& h; J e1 Z+ v- u
获取数据池对象
/ |: _- I/ @9 \' f5 Z根据inode找到该文件的所有对象' U$ e" Y& t" N2 q/ h
拼接对象
+ g# Q+ t' G# Y3 y8 J4 d. y9 k' N使用脚本注意事项：
! g& R/ Z# {5 g! M* n0 ~9 x
" x8 p. t# n8 n现在的脚本在只加入了txt、jpg、png、pdf、word、excel这些文件类型的识别，需要其他的就需要自己加入到脚本里面了( p8 c. ?' @+ r4 e
只适合副本池 y+ b3 Z3 M) ~0 A8 G
如果数据量特别大，不适合使用脚本，不过可以参考脚本的思路去一个一个文件恢复
) |" _& \: n3 r1 ]% b( N6、脚本
! v# S! d) t$ a! r% q# coding: utf-85 z1 e6 \; d7 Z! t* `
import os& K! D0 |$ s2 s( F. A5 R9 B3 ?% E) L
import shutil# V+ F6 Q8 s( h' q
import json2 e3 w7 ^3 h) `0 B, N/ x! Z
import sys
! g3 U: {7 p& _: s9 y4 [- d4 ?import subprocess
; r3 w& e8 E! q3 i2 C p9 f! P- cimport copy! u/ R- s0 @% ^; A) s
import logging, j' H1 G4 `8 u2 A. i4 |
import argparse6 i3 E9 y$ d7 e) M+ w+ D, e
& m+ \+ k3 [, X; A2 S4 |6 [__auth__ = 'ypdai'
8 {- \, `+ U/ n% p$ M2 u6 E* @- `3 `+ X' f8 W; u& E# D0 C
SLEEP_INTERVAL = 1. G4 T5 V1 h$ c8 d- {
logging.basicConfig(filename='./recovery.log', format='%(asctime)s : %(levelname)s %(message)s',
) }* `3 n6 h' x8 t; h. m level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')8 X, ?* A q$ D! z" l, W
% G: E% H4 ?8 {6 ~# D6 T1 ~
BASE_DIR = os.path.dirname(os.path.abspath(__file__))+ i0 t9 W) O6 w3 p
RECOVERY_OBJ_DIR = os.path.join(BASE_DIR, 'recoveryobjs')0 z; z- _9 g' _- W8 J) N9 N
RECOVERY_FILE_DIR = os.path.join(BASE_DIR, 'recoveryfiles')
2 r; ~ d6 x+ u% ~% m
+ i# |" g' m$ ?2 A. O/ X0 Y4 C9 {
/ v8 G2 y5 a# j$ B2 {% [def exec_cmd(cmd):
: U. u" w& a8 r """
/ G/ z& R% O/ e+ L- g 执行shell命令，并返回标准输出和执行状态码
- m `* t7 O8 ]4 ~* R i :param cmd:
6 p. ]6 Y* V6 D5 V ]1 M2 w :return:
C3 W {: ^/ E6 c- w5 k """. F5 i) F2 ^% o) l( [
logging.info('exec_cmd():: cmd: {}'.format(cmd))/ a2 E( o6 I9 Y: r
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)4 R4 W/ f' m0 }$ }& s5 b3 F" I1 a
p.wait()
; b2 r. K: R9 u; ]5 ~ out = p.stdout.read()
. E/ b+ N! a, x* F# q: g4 I: F- m code = p.returncode
5 y+ T" _0 R/ m3 I/ i logging.info('exec_cmd():: cmd exec out: {}, code: {}'.format(out, code))
* H1 }4 g5 P, \" [: ~$ e' c0 c% s return out, code
& r0 B# ^. u% `8 v L% ?
& m: C3 E3 V$ o: _7 o+ d* P1 j' p5 e5 X1 J
def prepare(pool_name):
# J# l9 I; B) f F """/ @7 Z3 X& `* T+ s' ^: U( w) c$ w! {
准备恢复条件- n: A9 ?3 J! p) i4 Y
1、检查所给pool name是不是cephfs的数据池9 Y/ U7 m: @' U* t
2、创建recoveryobjs和recoveryfiles文件加 y8 D# y5 z$ R6 P1 ?
:param pool_name:
; u! H$ r' P) I z/ U* ^' b :return:
$ n1 f$ w% r/ G3 i3 A """
+ A5 t: ~3 z5 X1 B8 Z cmd = 'ceph fs ls -f json-pretty'
& m; I0 X: {4 s: {" x: ?7 P" E% z out, code = exec_cmd(cmd)
4 \( |) M* @/ ^% j& I7 _: O2 i$ F out = json.loads(out)
- P7 t+ M& {0 Q8 {* J( v6 [ for cnt in out:4 p# I1 Y$ l0 w8 h+ Q5 Y: h9 U4 m
if pool_name not in cnt.get('data_pools'):
3 j6 u" t' Z. w: p: Q6 z- R9 o, J return False4 x; _; v" M- K: C$ g/ j1 \. W
0 \) }' X2 C' T if os.path.isdir(RECOVERY_OBJ_DIR):
. B0 l5 Q+ L7 J5 f4 w shutil.rmtree(RECOVERY_OBJ_DIR)1 Y: j, E4 H' z- A) v4 ~& `2 r
os.mkdir(RECOVERY_OBJ_DIR)8 ^9 w( [; d+ f. ?) \
2 j6 A$ ?) n6 u& H1 q if os.path.isdir(RECOVERY_FILE_DIR):
# i7 c; \2 l* L6 V shutil.rmtree(RECOVERY_FILE_DIR)
G* M% _0 q+ v8 Q# j( c os.mkdir(RECOVERY_FILE_DIR) W0 b9 w- V, F
- a' a: j; o9 T% C3 `( O& r% o* R
return True
* R- Q0 T9 u4 \& _3 f9 F4 w8 Q( K+ n% F% t. A
% z/ w- B: l% Y- F
def get_file_type(file_path):% j& j9 { V2 l( C, t$ Z- m4 e9 @
cmd = 'file %s' % file_path% b) n9 C4 f9 B5 q- `2 h
out, code = exec_cmd(cmd): K' p9 B8 N# Y+ e: H- C
out = out.split(':')[-1].lower()" S, x5 w* r8 c' G- A: U+ d
/ j( o% R0 y7 a, I file_type = 'text'3 @, B: @% q+ Z2 V) j
if 'word' in out:
- j0 k, t4 Y4 @% a1 R file_type = 'word'8 t: v4 _$ v% P! j& r6 `* U; a
elif 'excel' in out:
* v. r$ `8 w/ n: s# Y file_type = 'excel'2 I& o3 R" l/ Z( V! h0 b
elif 'pdf' in out:- P$ b. ^( F1 C6 \0 y# q
file_type = 'pdf'
- U5 l1 K& K4 g) W* J elif 'text' in out:* o. u+ d! L. Q( U
file_type = 'text'" Q' W8 g; v# _/ p
elif 'jpeg' in out:
, s' A; k. ~* b4 s' d' _ file_type = 'jpg'
6 k% F$ f) v! @( }' D elif 'png' in out:
% M0 c5 f8 P- l) Q* T6 H3 X' i file_type = 'png'8 T) ]3 w( ?0 L9 f, T7 B
( ]/ }8 X' }- Q
return file_type
9 B( e7 `& d( [, E( F
6 B+ h' n& l/ }, c0 k$ d0 O+ H. |, W
def do_recovery(pool_name):
i6 f* A4 e1 W0 S8 t """
# n& S/ j3 W0 p" e 具体执行恢复，大概恢复逻辑如下：
& i. @% F' _8 \% x, y3 p 1、从数据池里面获取所有的数据对象
1 t( A, p7 P' ?$ a# K2 |- `, a 2、找到每个文件的head对象，然后把数据这个文件的其他对象内容写入head对象里面
" V" U( Z- k7 W* O/ Y( {: ~ 3、根据head对象的文件类型，推测该文件的实际类型, P) H+ e8 k' P& o1 E
:param pool_name:
8 d3 p# x8 y6 G7 p :return:
" Z, p! y' f; j% h """6 K1 Q/ e4 y6 k+ T( k m
cmd = 'for obj in $(rados -p %s ls);do rados -p %s get ${obj} %s/${obj};done' % (
! E. w: ]8 S( ]2 }$ J* l- k( Q pool_name, pool_name, RECOVERY_OBJ_DIR)& c3 M! f. A5 w& i$ j# s8 ~% c
out, code = exec_cmd(cmd)
: U7 {2 J# j& i7 k7 p# k
" h' S' {$ [% \. Z' ] if code != 0:
1 j' I b: w0 w2 A T& B# O logging.error('do_recovery():: get obj from rados failed.'); V* Y% R' t9 O% d; x5 r# W3 P
return
( a' E+ B1 U1 M% J4 Z' P# u
' a, p4 Z1 Q9 w7 A4 o( C V! |. V cmd = 'ls %s' % RECOVERY_OBJ_DIR2 p$ I' b' C- f: L6 S, d& G& M
out, code = exec_cmd(cmd)8 A/ |3 c% d9 e
if code != 0:: [5 `) j) h2 ~$ Q' j
logging.error('do_recovery():: list obj failed.')9 K* K* q# w4 u0 j: R" a" f, Q5 P
return4 W0 V- j: f* `+ }% L& |8 Z9 c5 F
! J) O; g1 c$ P" o8 }" o
done_lst = []$ U! c1 r5 W2 f, p
objects = out.split(), m5 Q( q' o% W. j( j. }
for obj in objects:# |( n% G, ?8 b5 t& Z
inode, number = obj.split('.')
7 A# j- y' S& ^5 W" v" _ if inode in done_lst:" U, l- x9 r6 V% d/ \, B
continue
7 M& S" V& O% z
9 N$ ?2 P5 x- u$ h; K7 m cmd = '''ls -l %s | awk '{print $NF}' | grep ^%s |sort''' % (RECOVERY_OBJ_DIR, inode)+ ~, r5 t* g, v9 Q- R: k
out, code = exec_cmd(cmd); D% ~1 [3 S- L1 U. B" S
files = out.split('\n')5 r# s; @3 ^7 x5 M
5 |3 w9 B3 m( O$ @5 z5 z/ _
head_file = files[0]8 R8 h* p/ t% h o; S# f& [8 D0 |
file_type = get_file_type('%s/%s' % (RECOVERY_OBJ_DIR, head_file))
0 e. @' x6 w* B, q8 E- E5 S cmd = 'cp %s/%s %s/%s-%s' % (RECOVERY_OBJ_DIR, head_file, RECOVERY_FILE_DIR, inode, file_type)( m! [6 A& {$ J N& E
out, code = exec_cmd(cmd)! ^9 v) z r T+ g. |
for f in files[1:]:
8 C6 F' I H7 ?3 ? if not f:/ b2 }, A2 I7 t8 H5 v5 V1 v
continue9 U' [ q3 l( z0 U$ O
cmd = 'cat %s/%s >> %s/%s-%s' % (RECOVERY_OBJ_DIR, f, RECOVERY_FILE_DIR, inode, file_type)
( H2 R/ K+ O. f# t out, code = exec_cmd(cmd)" S& t H. x/ @ S: N
8 x6 `+ P" m" N }
done_lst.append(inode)
! B% |# z1 {- `/ [ I
+ N- X2 j, K, Z: Y" u/ x7 a, k. r* y# F
if __name__ == '__main__':. d+ Y& a- J1 ~1 V" N9 t6 W
parser = argparse.ArgumentParser()
+ t6 S# X) k) n6 Y/ E8 e3 L2 { parser.add_argument('-p', '--pool', required=True, type=str, dest='pool',, ~$ F2 L4 h& G" ]8 C5 E
help='select given cephfs data pool by name')
/ g* Q( x! P2 L$ a args = parser.parse_args()
- I% y [4 d' o* l0 u
$ `% x6 l g) w' v( ~3 e' c8 H if not prepare(args.pool):, ~) N& @" b, S. n2 A
logging.error('main():: invalid pool name.')
. p- x3 k" ~% Q; Z sys.exit(1)
7 H: Y$ [: S+ o: P% a. M$ U) P) \3 J( |9 {' `4 X% O% {$ Y' V! L. h
logging.info('=== main():: recovery start')
& u$ W: h0 |( W( p do_recovery(args.pool)7 D7 L: C& q5 j S3 U0 [; S5 h
logging.info('=== main():: recovery done') |
|
发表于 2022-8-3 11:02:40