|
|
1、启动一个ceph 进程
( Y2 W& I) x5 e, n" I* H启动mon进程
! i6 |9 o/ H: [4 x" B$ y) yservice ceph start mon.node1
7 m( y) Y0 `+ r! z1 I0 [7 V7 K启动msd进程5 s- k) }4 H5 v) F8 c2 B; W) s
service ceph start mds.node13 C, G U$ S/ U" N9 q" k
启动osd进程
$ z: t. \# e4 K& a service ceph start osd.0* E0 D. \. j' J7 T
2、查看机器的监控状态& @9 B5 y( ~3 \. c0 b1 S
[root@client ~]# ceph health
) a$ m$ H* _0 d( E4 qHEALTH_OK
5 S; R- q: K: M6 J( c3、查看ceph的实时运行状态7 p1 N8 K+ X) w5 r. ~+ W# P
[root@client ~]# ceph -w
% \: b# U$ N# [# _5 K" v cluster be1756f2-54f7-4d8f-8790-820c82721f17
; P# M& M& F1 b' D2 k" g0 ]0 Q8 Q health HEALTH_OK2 a% t6 `9 R& ^& N7 Q) v
monmap e2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3
n* S: f# H+ b" ~+ O3 } mdsmap e95: 1/1/1 up {0=node2=up:active}, 1 up:standby
/ D) N4 H! G( Y* b/ @ osdmap e88: 3 osds: 3 up, 3 in
# H& Y3 `7 w9 k/ F pgmap v1164: 448 pgs, 4 pools, 10003 MB data, 2520 objects c3 R9 Q8 [& u
23617 MB used, 37792 MB / 61410 MB avail& p; S: Z/ T; n( p
448 active+clean7 g" X$ a; Z0 T0 H9 }, _7 p
2014-06-30 00:48:28.756948 mon.0 [INF] pgmap v1163: 448 pgs: 448 active+clean; 10003 MB data, 23617 MB used, 37792 MB / 61410 MB avail9 Z" o. a0 \9 z0 m# a0 |
b0 S+ T1 }7 m9 F# q* N4、检查信息状态信息
- U% D3 l6 Y6 }[root@client ~]# ceph -s
" w$ o. L! y! H cluster be1756f2-54f7-4d8f-8790-820c82721f17
6 g2 @- X2 e! H% |$ L health HEALTH_OK; _6 W+ r$ g* r/ z
monmap e2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3, n$ E6 y8 e- M1 j4 ^3 w
mdsmap e95: 1/1/1 up {0=node2=up:active}, 1 up:standby
) [6 k1 {7 _6 r; \, ^- I. Q$ k4 t osdmap e88: 3 osds: 3 up, 3 in m/ [6 i0 \. H$ [8 @
pgmap v1164: 448 pgs, 4 pools, 10003 MB data, 2520 objects+ V" o5 Z/ Z8 ]8 _
23617 MB used, 37792 MB / 61410 MB avail
- s+ X+ R6 l: q4 y& b+ n 448 active+clean
8 g# A$ M. z' J[root@client ~]#
3 y& m$ e; B$ H
7 G4 m2 |) W# P5 H/ m2 s- B5、查看ceph存储空间
7 m$ Z8 `: b" f. m1 t[root@client ~]# ceph df
$ o/ p$ {% s- k R) b+ FGLOBAL:2 C+ e6 a8 ^7 | }4 W
SIZE AVAIL RAW USED %RAW USED 1 m. J1 Z6 t) N3 ^& L
61410M 37792M 23617M 38.46
: W. ^/ U* Q3 W7 aPOOLS:
1 x5 W$ c2 N5 r* u! l& }' b2 b h NAME ID USED %USED OBJECTS
$ q6 c0 y8 y A, s5 U. B, E data 0 10000M 16.28 2500 2 B+ T" g% L. t+ K& B
metadata 1 3354k 0 20 / P0 W$ u" e9 Q# `& J+ B% C0 Q8 W
rbd 2 0 0 0 - G% {2 Z1 q1 S0 O9 x
jiayuan 3 0 0 0 % U6 T% h7 |; m* t8 Q! o ?* X: p; l
[root@client ~]# : \5 F% b3 V1 H/ b/ l6 d
6、删除一个节点的所有的ceph数据包
* ~* \- n! t% \; q# d: M2 p, R3 y[root@node1 ~]# ceph-deploy purge node1
! c. A& X, E" p7 [. r[root@node1 ~]# ceph-deploy purgedata node1
5 g+ C) B/ u# ~* i5 Y: h; ]7、为ceph创建一个admin用户并为admin用户创建一个密钥，把密钥保存到/etc/ceph目录下：2 t9 O( V, Y, e$ x+ k9 p
ceph auth get-or-create client.admin mds 'allow' osd 'allow *' mon 'allow *' > /etc/ceph/ceph.client.admin.keyring
( U" r5 Q9 T8 P0 Z/ Y或# R- f& o( _& H, t) J/ a; T. O
ceph auth get-or-create client.admin mds 'allow' osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.admin.keyring. S5 L- d' W2 T) r0 `" @; W$ ?, |
8、为osd.0创建一个用户并创建一个key
/ [& p' S) Z0 c W/ E. O e$ Vceph auth get-or-create osd.0 mon 'allow rwx' osd 'allow *' -o /var/lib/ceph/osd/ceph-0/keyring
0 r) m& y) e5 D7 S1 `$ }, l. ^9、为mds.node1创建一个用户并创建一个key+ w, X5 w+ j( g! l
ceph auth get-or-create mds.node1 mon 'allow rwx' osd 'allow *' mds 'allow *' -o /var/lib/ceph/mds/ceph-node1/keyring; S F: E9 Z% e! [0 o
10、查看ceph集群中的认证用户及相关的key, g# N+ x c' ^+ A1 L; ?( d ~* l S
ceph auth list L, O/ S( Y& S; t; v, F
11、删除集群中的一个认证用户! F+ A) Q6 }, X& {6 R, z+ y
ceph auth del osd.0
/ N7 @" ~( ]6 d; Z* p+ i) N' `12、查看集群的详细配置, c% p; Y- l4 b
[root@node1 ~]# ceph daemon mon.node1 config show | more
2 D4 A% Y6 `2 S. R) k1 R13、查看集群健康状态细节. z3 z2 M) b% `. \2 k
[root@admin ~]# ceph health detail5 M Z1 B, \9 J- f9 R' _9 P
HEALTH_WARN 12 pgs down; 12 pgs peering; 12 pgs stuck inactive; 12 pgs stuck unclean
; U, N. f/ W0 |5 D+ c: T- _pg 3.3b is stuck inactive since forever, current state down+peering, last acting [1,2]8 w& Y3 K1 K6 ]% [
pg 3.36 is stuck inactive since forever, current state down+peering, last acting [1,2], ~" a! w1 A+ t! k b
pg 3.79 is stuck inactive since forever, current state down+peering, last acting [1,0]
5 l W7 B# L7 dpg 3.5 is stuck inactive since forever, current state down+peering, last acting [1,2]2 K! B6 ~* C! K: c
pg 3.30 is stuck inactive since forever, current state down+peering, last acting [1,2]) e, S7 R* Y/ |7 D8 T) u& }. O5 @; F
pg 3.1a is stuck inactive since forever, current state down+peering, last acting [1,0]
6 D/ v" G2 Z! g' S3 w$ H* {" vpg 3.2d is stuck inactive since forever, current state down+peering, last acting [1,0]' _* w8 y3 J: V0 @( `
pg 3.16 is stuck inactive since forever, current state down+peering, last acting [1,2]4 ?- ?* f. W t, ^' N8 v6 ^1 g
: T) L! x1 x" u" w8 K2 L
14、查看ceph log日志所在的目录
/ p6 v% S! N$ l/ r6 p4 b4 ^[root@node1 ~]# ceph-conf --name mon.node1 --show-config-value log_file
2 o7 r6 q8 b* Z! h( b/var/log/ceph/ceph-mon.node1.log+ K& ?+ u! J8 J5 V% y7 Y' p
- Y+ e! e* r9 w! c; S" R5 A
% n9 D( F0 w2 W$ n! _+ H8 k+ T
二、mon8 }# k+ S0 @: Y5 u X; C0 S
1、查看mon的状态信息# b0 C! u! [, J% D* |& |* E) f0 S
[root@client ~]# ceph mon stat$ W* m5 X5 Z8 R$ U
e2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3, f: i1 \- M, k* M8 s
2、查看mon的选举状态
0 |* u) B1 t7 O, L5 V b* `[root@client ~]# ceph quorum_status
2 b. ]0 Z- W3 Y# j1 W5 ?{"election_epoch":294,"quorum":[0,1,2],"quorum_names":["node1","node2","node3"],"quorum_leader_name":"node1","monmap":{"epoch":2,"fsid":"be1756f2-54f7-4d8f-8790-820c82721f17","modified":"2014-06-26 18:43:51.671106","created":"0.000000","mons":[{"rank":0,"name":"node1","addr":"10.240.240.211:6789\/0"},{"rank":1,"name":"node2","addr":"10.240.240.212:6789\/0"},{"rank":2,"name":"node3","addr":"10.240.240.213:6789\/0"}]}}
* A7 Q3 i6 L& ? h" t3、查看mon的映射信息
4 A0 x7 g9 ~; V+ M[root@client ~]# ceph mon dump
! q, y4 V }# S8 o! t/ t, Xdumped monmap epoch 2
: d) G9 z; x. Aepoch 2- L: e1 y7 s4 ]5 a
fsid be1756f2-54f7-4d8f-8790-820c82721f17
h0 k5 C% Q7 w4 r/ nlast_changed 2014-06-26 18:43:51.671106
7 Q& }0 B% g8 L- ?5 @/ ]created 0.000000
/ B! ]& E: r) }0: 10.240.240.211:6789/0 mon.node1
% G) o& T, E8 K& r, t5 V1: 10.240.240.212:6789/0 mon.node2( I5 _0 i. n4 Q- e( X6 N
2: 10.240.240.213:6789/0 mon.node3, V2 ~' Q& V0 H% A: n; M
4、删除一个mon节点
3 T1 m( k& G, k( T& G7 R$ Z: V[root@node1 ~]# ceph mon remove node1
6 \6 r/ }4 e6 l" `8 m N7 T+ Tremoved mon.node1 at 10.39.101.1:6789/0, there are now 3 monitors
: [: Q7 X% x. K) v9 V$ {4 Q8 j2014-07-07 18:11:04.974188 7f4d16bfd700 0 monclient: hunting for new mon
. [. p& n% h4 n$ b" Z$ I! [ j. L5、获得一个正在运行的mon map，并保存在1.txt文件中
/ k7 L1 |& ~6 I; f[root@node3 ~]# ceph mon getmap -o 1.txt) ~8 i/ X9 h6 Q6 l% i
got monmap epoch 6/ w& M- {4 {4 f$ r% S7 Y
6、查看上面获得的map
" e* u2 v5 T. b[root@node3 ~]# monmaptool --print 1.txt 1 N0 E( N% t$ g" @+ O- l5 I# {) Y
monmaptool: monmap file 1.txt
3 d; ]' G, D+ R2 Gepoch 6) m$ s5 c6 j8 Q" t& C% ?+ H
fsid 92552333-a0a8-41b8-8b45-c93a8730525e
8 \+ [" w6 ]4 u/ Clast_changed 2014-07-07 18:22:51.927205# {( _7 f }8 Y! K5 ~
created 0.000000
6 |) ]9 X! G! R( W, u2 B5 f! q0: 10.39.101.1:6789/0 mon.node1/ z! C$ I5 Y1 r7 Z" Y
1: 10.39.101.2:6789/0 mon.node2
1 S% K" \: {1 |/ W2: 10.39.101.3:6789/0 mon.node3
; K+ i" \ |' A j1 x$ D$ U[root@node3 ~]#
% u# v. _5 C8 }2 h7、把上面的mon map注入新加入的节点, y4 u! B2 w0 }/ W" ^
ceph-mon -i node4 --inject-monmap 1.txt
" I' t/ g0 K7 ^, E, l8、查看mon的amin socket5 ?% T/ ]5 W/ Z+ A" Q! j
root@node1 ~]# ceph-conf --name mon.node1 --show-config-value admin_socket
1 s% Y) V; |6 s8 i) A# L/var/run/ceph/ceph-mon.node1.asok ; H0 w5 F$ h/ i* ?/ ^
9、查看mon的详细状态
! q6 @9 G% n; m+ h( i) Q q0 w+ y[root@node1 ~]# ceph daemon mon.node1 mon_status
; y k% v" Y6 L& `. v6 s3 ~. k- U{ "name": "node1"," Z% g0 S- }, B+ }1 e, D, P* O
"rank": 0,
+ S, w1 v0 y! P3 Q0 q$ E/ r9 Y6 L/ S "state": "leader",
( m3 T% F0 c; ^# K "election_epoch": 96,& L; X+ B6 R7 P4 y: S! Z
"quorum": [% Y4 p' q6 h$ n: E# F- K, |
0,* [" F$ b: d" E- S
1,$ \0 d, t# S" M; }; P* h4 v2 e
2],$ C0 G& D2 v" Q0 y' f/ D( F$ x
"outside_quorum": [],
. r0 |+ g- [- G# U5 I: R. x$ s1 }# n "extra_probe_peers": [
. e" C# e; U5 V( l' _. h3 E5 ~ T8 x "10.39.101.4:6789\/0"],
; x' \% z1 l& [4 w1 R+ S2 D* z& f8 k "sync_provider": [],
- Q3 Q0 ]1 {; J7 ~9 f! U "monmap": { "epoch": 6,+ ~" c5 Q, F( f% d: U' u: i
"fsid": "92552333-a0a8-41b8-8b45-c93a8730525e",
9 M4 u! l& ]+ Q& q4 k. j. q "modified": "2014-07-07 18:22:51.927205",1 a- h/ E6 T. @
"created": "0.000000",
" p* F: l; s! ?0 ~. E "mons": [- f( N8 X1 g) W) j' q' k$ n
{ "rank": 0,
* `, x+ Q* d* v8 P# j+ f "name": "node1",
( @1 O; U4 l! }% N: I; { "addr": "10.39.101.1:6789\/0"},
, T3 Z. @5 G8 U1 I& [; \ { "rank": 1,
) \4 G. f1 G# X% ]% d9 g; O "name": "node2",
8 w2 [4 ^0 ?1 J3 ?% T2 F3 h "addr": "10.39.101.2:6789\/0"},
5 l7 A5 I2 N) `2 L- P& E { "rank": 2,: ^ H4 g6 A: N6 n( ?: \4 J( e
"name": "node3",
$ O. h4 Y8 t5 e. O! \! e( T "addr": "10.39.101.3:6789\/0"}]}6 Z2 [% Z. ?6 t: a+ h
10、删除一个mon节点- _! Z" S+ Q" W, n/ h/ F
[root@os-node1 ~]# ceph mon remove os-node1" p* P2 j5 z- z3 p
removed mon.os-node1 at 10.40.10.64:6789/0, there are now 3 monitors& G# N9 ~* b5 N' d0 ?0 C% s
- G z ]2 [3 t% _! L# n三、msd+ q. v/ ?, K Q# \9 a( _( G9 T
1、查看msd状态* B: e* B- ]. d# \
[root@client ~]# ceph mds stat
j- T* n3 U: U( L0 oe95: 1/1/1 up {0=node2=up:active}, 1 up:standby- F# q I4 m7 r3 w3 ]
2、查看msd的映射信息- d% v9 d( p9 K: v) k
[root@client ~]# ceph mds dump
: ?3 V/ ~" Q2 I& x; d5 Odumped mdsmap epoch 95
L; c5 H9 }2 a) ?: O% A ?epoch 95
& D5 k0 H- k+ H6 N5 h- fflags 09 l- m. v; @1 K9 H8 P
created 2014-06-26 18:41:57.686801) X+ {/ _5 @% ^) A
modified 2014-06-30 00:24:11.749967
# l( Z! b2 R; q: u* s+ Mtableserver 0& K, J. U% I7 d/ g! @7 N
root 0
: x* O3 ^2 P. d# s( ?session_timeout 60
- L4 ~- Y8 u; D6 c4 Ysession_autoclose 300
$ H) O5 X0 k: _$ \8 l; g" q3 `max_file_size 1099511627776: \! Z$ C& J2 c1 m: R
last_failure 84* g8 _( @# T( U) E
last_failure_osd_epoch 81
# x$ J3 q n1 w$ a) pcompat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap}' D0 r5 E7 w0 l% Z& K
max_mds 1: Q6 c( U; r# c
in 0
8 }) v) k( t! W/ Y& sup {0=5015}! `1 K4 I9 K: Q$ y) S, s7 `
failed
7 h9 I+ h/ L& o/ @7 hstopped; _% [+ N- J) Q
data_pools 0
8 V) |/ S! A }) ?; Vmetadata_pool 1
3 ~. P; H* ]4 B2 C cinline_data disabled% [2 k+ Z! l/ s- F9 a
5015: 10.240.240.212:6808/3032 'node2' mds.0.12 up:active seq 30
8 ^+ ~' P& A$ i# Z" T; U" ?: h7 T5012: 10.240.240.211:6807/3459 'node1' mds.-1.0 up:standby seq 38
! {$ c, ~3 _9 N% v( e3、删除一个mds节点
# D' x% L7 ?8 ~! w[root@node1 ~]# ceph mds rm 0 mds.node1
( L8 y0 A/ e$ ^* ` C- _& s mds gid 0 dne1 N' ^0 k8 ~( x! W% O* @) K
( X- P* x/ B* F四、osd) z n# P& @2 B4 u; u. G
1、查看ceph osd运行状态, `" n3 |/ Z1 T$ o0 b
[root@client ~]# ceph osd stat
9 Y" t1 E! ~' u) w1 `; [& `" d osdmap e88: 3 osds: 3 up, 3 in
, b& b" ^0 ?7 y! i$ ^4 F: H" Z' {0 p/ }: D2 j* \, c# g4 h
2、查看osd映射信息1 U# w/ ?* j) ^' F; F
[root@client ~]# ceph osd dump! q8 e8 @: `+ {: ?. _
epoch 88
5 B- _$ z9 _$ j0 v) g. @fsid be1756f2-54f7-4d8f-8790-820c82721f17( |2 |# p0 w* F1 Z% @
created 2014-06-26 18:41:57.687442
- }1 z# l+ T! T- n$ o. Vmodified 2014-06-30 00:46:27.1797935 T) L0 A& @: ]% q/ T
flags
* `$ Z' e6 ]8 p* w: {3 Tpool 0 'data' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 06 P+ L# N/ k3 d& B. Q! n3 g0 h
pool 1 'metadata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0' b u3 P, A* H9 h; x
pool 2 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 01 _+ c' q# B; q+ i
pool 3 'jiayuan' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 256 pgp_num 256 last_change 73 owner 0 flags hashpspool stripe_width 0, ^+ y+ I9 w/ \3 O
max_osd 3
e3 l2 M. Q% P1 Wosd.0 up in weight 1 up_from 65 up_thru 75 down_at 64 last_clean_interval [53,55) 10.240.240.211:6800/3089 10.240.240.211:6801/3089 10.240.240.211:6802/3089 10.240.240.211:6803/3089 exists,up 8a24ad16-a483-4bac-a56a-6ed44ab74ff0- l' ^. u# H7 C$ C2 w2 L" X: Z9 I
osd.1 up in weight 1 up_from 59 up_thru 74 down_at 58 last_clean_interval [31,55) 10.240.240.212:6800/2696 10.240.240.212:6801/2696 10.240.240.212:6802/2696 10.240.240.212:6803/2696 exists,up 8619c083-0273-4203-ba57-4b1dabb893394 M3 z8 `% i5 R/ P2 L% r. ^
osd.2 up in weight 1 up_from 62 up_thru 74 down_at 61 last_clean_interval [39,55) 10.240.240.213:6800/2662 10.240.240.213:6801/2662 10.240.240.213:6802/2662 10.240.240.213:6803/2662 exists,up f8107c04-35d7-4fb8-8c82-09eb885f0e58 S/ n/ j2 _, q! s0 S3 W& J1 U: q
[root@client ~]# $ W4 H3 ^( `9 T( z" e* b: S
- }* M; T! N6 B' w9 |2 P8 x3、查看osd的目录树8 u c5 X: u" r5 M
[root@client ~]# ceph osd tree
6 r$ H7 K2 a8 ]' _# id weight type name up/down reweight% T* g9 w( |) q) U1 g8 @
-1 3 root default8 v9 e1 g, l2 E9 |
-2 1 host node1+ |* Z8 _5 x6 |/ d" q; f
0 1 osd.0 up 1- M* Q. a5 R3 U
-3 1 host node2
7 N$ u; w. M+ b3 ?" V! y5 `1 1 osd.1 up 17 W* j6 {$ Z5 S+ G7 }
-4 1 host node3- T. n- T% ~# n2 j
2 1 osd.2 up 1
" ^9 Y5 H$ i4 U) p* q4 k2 M. C& {' v
: s R0 x; R) J7 C$ G4、down掉一个osd硬盘* N( h7 M0 ]% o) F+ J% }2 M
[root@node1 ~]# ceph osd down 0 #down掉osd.0节点
$ X. o) i$ S" Y3 {* i3 T5、在集群中删除一个osd硬盘/ T- X- R9 q" W9 }
[root@node4 ~]# ceph osd rm 0" R) Y3 o: M( X# f0 i
removed osd.03 @8 Z3 C% @* c$ f0 ?
6、在集群中删除一个osd 硬盘 crush map8 |: E1 H/ P8 g5 b: l; c5 H6 N
[root@node1 ~]# ceph osd crush rm osd.0
+ q5 ~; l, [- h# o0 \* K7、在集群中删除一个osd的host节点
O7 \6 b1 P$ k0 I% H/ v+ {[root@node1 ~]# ceph osd crush rm node1! x( t( b' S6 i# R
removed item id -2 name 'node1' from crush map
( `. q) F" i* x/ ^0 o( F9 ^+ A4 A3 |( P9 |8 V" M
查看最大osd的个数 9 }( R/ ?; `/ h1 I* p+ C% ~9 L
[root@node1 ~]# ceph osd getmaxosd4 b5 H2 B# O0 x2 q" D- L
max_osd = 4 in epoch 514 #默认最大是4个osd节点
" U U# Y K: N0 C8、设置最大的osd的个数（当扩大osd节点的时候必须扩大这个值）9 F% q Q( T7 |
[root@node1 ~]# ceph osd setmaxosd 10
5 r( q3 L( h- r) u# p3 O9、设置osd crush的权重为1.0
1 L* k" A& t: d' W p: Vceph osd crush set {id} {weight} [{loc1} [{loc2} ...]]
+ J, {) s1 j& \7 U8 ?: x# I例如：0 L: u" ^+ P# x z1 p5 o
[root@admin ~]# ceph osd crush set 3 3.0 host=node4
/ U9 m4 e, L" Y+ n. b0 Iset item id 3 name 'osd.3' weight 3 at location {host=node4} to crush map
- ]- C6 k+ Y) u. q3 f% W+ b$ J9 U1 E[root@admin ~]# ceph osd tree) x; _. B# ~+ N1 R# J
# id weight type name up/down reweight
5 B q d/ `, g+ Z8 o3 K-1 6 root default
6 X( z# Q5 _: |' C9 t6 E-2 1 host node1! J1 V: b2 _0 Z! F7 b
0 1 osd.0 up 1
9 d# A2 E# i* i c. ~9 o. f-3 1 host node2
3 U/ w9 l* n+ D5 p t5 z7 d1 1 osd.1 up 1
8 b; m& G+ p. n4 ~* p. D-4 1 host node3
. U' R& L( R2 `$ c2 1 osd.2 up 1
/ O" S O [0 b4 v) I4 b-5 3 host node4
8 {; u% l9 q, K2 D( |2 ?# t2 C, @3 3 osd.3 up 0.51 J. j# J2 H' j1 o
# w( i* V" X5 B& g或者用下面的方式7 \3 l# {6 \1 f# w6 e
[root@admin ~]# ceph osd crush reweight osd.3 1.0! F" `- |& V$ W) b
reweighted item id 3 name 'osd.3' to 1 in crush map+ I$ ^& h: [" w4 U! B0 [
[root@admin ~]# ceph osd tree
- [8 y1 u% d% \$ F6 F) f# id weight type name up/down reweight
( ~4 k4 B: l+ V-1 4 root default* o8 B/ C( v/ u9 c8 O$ t
-2 1 host node15 D) B- K' _" ~, T) _2 I
0 1 osd.0 up 12 p7 {2 x. b# s, {" N+ }
-3 1 host node2
! H$ _7 P1 G! G' ~# `. d" z! ^( e1 1 osd.1 up 18 W" S* X n2 z. `
-4 1 host node3 I3 G7 C9 k3 d, z4 O% E
2 1 osd.2 up 1* r) A8 S S3 Z3 ?
-5 1 host node4$ C, ^9 o" a- J
3 1 osd.3 up 0.5, K* m; |2 F* ?
10、设置osd的权重2 `( N6 q+ n( G5 [) |' d. R; _
[root@admin ~]# ceph osd reweight 3 0.5; z6 n4 o; q6 Y' B: e \' C, ?
reweighted osd.3 to 0.5 (8327682). l% Y' D' i3 D$ s
[root@admin ~]# ceph osd tree" d" @5 P9 k" {. M. i' h+ y
# id weight type name up/down reweight% w0 N" \. h" r4 k7 _. s ?" R
-1 4 root default d2 X' i$ @- v. E% A D
-2 1 host node1
3 ~. W0 W- [! `7 ]6 u$ d0 1 osd.0 up 1
9 Y, w( E' E$ l* s-3 1 host node2
3 v0 ^8 d% E5 A1 1 osd.1 up 1
9 j7 m9 ^% k0 G-4 1 host node3
( _6 R, C6 r- U3 `) Y2 1 osd.2 up 1. N. w0 f9 b; ~% b
-5 1 host node4" ], X' O$ s& ~6 _
3 1 osd.3 up 0.5! V: s9 B- X# j" K, r
11、把一个osd节点逐出集群( m0 G# c0 R [) [8 ^ U
[root@admin ~]# ceph osd out osd.3
3 q1 l% X* f. C: `( W* Vmarked out osd.3. ) V1 |& ~8 ?( @( r! a, B. y
[root@admin ~]# ceph osd tree# V2 n5 N5 g+ o' S
# id weight type name up/down reweight
/ R& j- ^2 O+ O# b8 L7 r-1 4 root default" F4 \2 O$ s( Y1 W' k
-2 1 host node1+ P# V; ^; ?" d5 X& s) v# r1 |
0 1 osd.0 up 1; E1 |3 V0 E4 y, R$ k* ?" _) q3 z* B
-3 1 host node2
3 u3 r; ?5 } ^/ v4 t1 1 osd.1 up 1* n+ t$ r8 b/ `3 l3 }
-4 1 host node31 D, {3 K& @+ w* {
2 1 osd.2 up 1+ J% H" e% F6 A
-5 1 host node4
- Z% w/ H1 O5 u5 W N8 M3 1 osd.3 up 0 # osd.3的reweight变为0了就不再分配数据，但是设备还是存活的
9 F! _- G) S+ R12、把逐出的osd加入集群
# k; v7 h4 D$ X% ]) V: z% H[root@admin ~]# ceph osd in osd.3/ n+ o; |1 d$ M3 N7 i
marked in osd.3. * r9 N) C$ O& k# e H/ k
[root@admin ~]# ceph osd tree! Z0 i: V; o1 r
# id weight type name up/down reweight. r6 s6 J4 }( d0 ]& [$ i
-1 4 root default/ s+ M; x( h6 f1 E3 W7 `- b+ y' j9 v
-2 1 host node1
& K9 L6 k1 X( I7 M& Q0 1 osd.0 up 1. N2 E! O8 g, W! K) P6 K! w! j
-3 1 host node2
3 @# B+ W0 q* k V! M1 a2 |* C. h1 1 osd.1 up 1
X) U0 U3 t( U6 H' C" T-4 1 host node3, i- A7 W) j0 W- m, Z
2 1 osd.2 up 1
, q! G6 {" w$ ?0 Q" |# j( K4 {# p-5 1 host node4) P) Y6 h" N2 Z
3 1 osd.3 up 1. U$ U# e( [/ p" i5 y, D
13、暂停osd （暂停后整个集群不再接收数据）# D: n: A/ d. j# [- c6 m; C; \) z
[root@admin ~]# ceph osd pause* ?* d( ^8 j9 k B6 C* X0 S
set pauserd,pausewr 2 n" X! ^) `- J" d8 J
14、再次开启osd （开启后再次接收数据）
; j' f! t# B( O7 f9 N# V% o[root@admin ~]# ceph osd unpause
/ v. G7 _- X7 B: m- Cunset pauserd,pausewr: f+ C" A4 d9 n; G7 T
0 O) J- y( X1 i15、查看一个集群osd.2参数的配置, G/ l6 F z# H& k
ceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config show | less6 F9 J4 @( u" q7 E$ g" G6 H; L# W
' ?+ I$ e3 P! N) g" T
- {: A: i7 t9 v0 m3 m/ u' N) r0 }* D0 C/ L# K9 J( ]
五、PG组
# b; {6 a! y3 P5 s9 V4 J1 S5 e' O8 I1、1、查看pg组的映射信息! L' h! ~5 \, W: u b
[root@client ~]# ceph pg dump
* P. x1 p& M- n6 T! d/ Tdumped all in format plain
_& e6 i7 Z, `, K5 j' \version 1164
1 @/ V r/ n2 L8 ^+ ]$ @, ?+ Kstamp 2014-06-30 00:48:29.754714) U$ A: U* g2 a9 ?+ C0 Q, P
last_osdmap_epoch 88# {; I3 c) e/ p6 E. `, G: H5 s
last_pg_scan 73
- @7 b9 n5 |- J* V0 q! L; zfull_ratio 0.951 h7 y% X( i) e. g& m C9 @
nearfull_ratio 0.85
/ N' X, i* n" X( i' Wpg_stat objects mip degr unf bytes log disklog state state_stamp v reported up up_primary acting acting_primary last_scrub scrub_stamp last_deep_scrudeep_scrub_stamp* b4 s5 v* p# Q2 @ D: p/ S) V/ `
0.3f 39 0 0 0 163577856 128 128 active+clean 2014-06-30 00:30:59.193479 52'128 88:242 [0,2] 0 [0,2] 0 44'25 2014-06-29 22:25:25.282347 0'0 2014-06-26 19:52:08.521434! {% ~8 H1 X: K& j3 Y* C1 v4 Z
3.3c 0 0 0 0 0 0 0 active+clean 2014-06-30 00:15:38.675465 0'0 88:21 [2,1] 2 [2,1] 2 0'0 2014-06-30 00:15:04.295637 0'0 2014-06-30 00:15:04.295637/ |6 x* q& f, x' _
2.3c 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.583702 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:29:13.701625 0'0 2014-06-26 19:52:08.845944
8 b* {% {) y1 g% J4 e1.3f 2 0 0 0 452 2 2 active+clean 2014-06-30 00:10:48.596050 16'2 88:66 [2,1] 2 [2,1] 2 16'2 2014-06-29 22:28:03.570074 0'0 2014-06-26 19:52:08.6552924 ?7 N: d- a: h7 t
0.3e 31 0 0 0 130023424 130 130 active+clean 2014-06-30 00:26:22.803186 52'130 88:304 [2,0] 2 [2,0] 2 44'59 2014-06-29 22:26:41.317403 0'0 2014-06-26 19:52:08.518978. ~2 z# i: ]4 s7 L+ P/ x! b
3.3d 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:57.548803 0'0 88:20 [0,2] 0 [0,2] 0 0'0 2014-06-30 00:15:19.101314 0'0 2014-06-30 00:15:19.1013145 k. |4 z# K2 a5 C# Z- Q; f
2.3f 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.750476 0'0 88:106 [0,2] 0 [0,2] 0 0'0 2014-06-29 22:27:44.604084 0'0 2014-06-26 19:52:08.864240
# G6 X$ H( `$ ^+ ~1.3c 1 0 0 0 0 1 1 active+clean 2014-06-30 00:10:48.939358 16'1 88:66 [1,2] 1 [1,2] 1 16'1 2014-06-29 22:27:35.991845 0'0 2014-06-26 19:52:08.6464706 E: a( J/ M0 G" L
0.3d 34 0 0 0 142606336 149 149 active+clean 2014-06-30 00:23:57.348657 52'149 88:300 [0,2] 0 [0,2] 0 44'57 2014-06-29 22:25:24.279912 0'0 2014-06-26 19:52:08.5145264 h s# k3 l5 k5 u4 I& ?
3.3e 0 0 0 0 0 0 0 active+clean 2014-06-30 00:15:39.554742 0'0 88:21 [2,1] 2 [2,1] 2 0'0 2014-06-30 00:15:04.296812 0'0 2014-06-30 00:15:04.296812) e+ r/ g( |: L; |3 a8 W% G( Z3 M2 j
2.3e 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.592171 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:29:14.702209 0'0 2014-06-26 19:52:08.855382
7 w" h& N$ v- z( ]. t3 Y0 w1.3d 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.938971 0'0 88:58 [1,2] 1 [1,2] 1 0'0 2014-06-29 22:27:36.971820 0'0 2014-06-26 19:52:08.650070- T5 y( I# Y5 E }/ K/ S2 ^0 N- v
0.3c 41 0 0 0 171966464 157 157 active+clean 2014-06-30 00:24:55.751252 52'157 88:385 [1,0] 1 [1,0] 1 44'41 2014-06-29 22:26:34.829858 0'0 2014-06-26 19:52:08.513798
/ `; d6 }2 ~" E3.3f 0 0 0 0 0 0 0 active+clean 2014-06-30 00:17:08.416756 0'0 88:20 [0,1] 0 [0,1] 0 0'0 2014-06-30 00:15:19.406120 0'0 2014-06-30 00:15:19.406120. {6 E3 Q/ R U2 N
2.39 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.784789 0'0 88:71 [2,0] 2 [2,0] 2 0'0 2014-06-29 22:29:10.673549 0'0 2014-06-26 19:52:08.8346441 g; j1 w! n# U2 |: r
1.3a 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.738782 0'0 88:106 [0,2] 0 [0,2] 0 0'0 2014-06-29 22:26:29.457318 0'0 2014-06-26 19:52:08.642018
1 j/ t9 `8 p) Q, ~* w$ \0.3b 37 0 0 0 155189248 137 137 active+clean 2014-06-30 00:28:45.021993 52'137 88:278 [0,2] 0 [0,2] 0 44'40 2014-06-29 22:25:22.275783 0'0 2014-06-26 19:52:08.510502+ }) e3 r2 Z. t4 J
3.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:13.222339 0'0 88:21 [1,0] 1 [1,0] 1 0'0 2014-06-30 00:15:05.446639 0'0 2014-06-30 00:15:05.446639 S$ L, E3 Y4 a7 x9 T
2.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.783103 0'0 88:71 [2,0] 2 [2,0] 2 0'0 2014-06-29 22:29:06.688363 0'0 2014-06-26 19:52:08.827342$ e1 R4 x* G, F6 X
1.3b 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.857283 0'0 88:78 [1,0] 1 [1,0] 1 0'0 2014-06-29 22:27:30.017050 0'0 2014-06-26 19:52:08.644820
1 {) F6 s9 o8 ?( A4 f1 B* r0.3a 40 0 0 0 167772160 149 149 active+clean 2014-06-30 00:28:47.002342 52'149 88:288 [0,2] 0 [0,2] 0 44'46 2014-06-29 22:25:21.273679 0'0 2014-06-26 19:52:08.5086546 b# _# m0 O( v% z- f
3.39 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:13.255056 0'0 88:21 [1,0] 1 [1,0] 1 0'0 2014-06-30 00:15:05.447461 0'0 2014-06-30 00:15:05.447461
; m) x# t6 [2 _( \; Y5 Y# m2.3b 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.935872 0'0 88:57 [1,2] 1 [1,2] 1 0'0 2014-06-29 22:28:35.095977 0'0 2014-06-26 19:52:08.844571
; M0 `: X1 b7 R8 n- R; d1.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.597540 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:28:01.519137 0'0 2014-06-26 19:52:08.633781' H& a% e# m5 \; U8 G) w7 u: [
0.39 48 0 0 0 201326592 164 164 active+clean 2014-06-30 00:25:30.757843 52'164 88:432 [1,0] 1 [1,0] 1 44'32 2014-06-29 22:26:33.823947 0'0 2014-06-26 19:52:08.504628" t8 j6 t9 C/ q T2 t
下面部分省略
( J( Y4 J' k5 F; l& |
3 Q- N B! e1 n% ~3 b; t2、查看一个PG的map" r. D9 m L! w( [! k, q
[root@client ~]# ceph pg map 0.3f
' [# C; m4 X% N: U2 `' O& O8 R# b! B. aosdmap e88 pg 0.3f (0.3f) -> up [0,2] acting [0,2] #其中的[0,2]代表存储在osd.0、osd.2节点，osd.0代表主副本的存储位置' f2 H) h& Y7 _ U( F# O& W& {
3、查看PG状态
2 e0 o' m3 t/ A( @. A* h[root@client ~]# ceph pg stat
x- ]/ q6 M; y _v1164: 448 pgs: 448 active+clean; 10003 MB data, 23617 MB used, 37792 MB / 61410 MB avail
H0 D* P. o1 @/ n- i6 a% X' r& a4、查询一个pg的详细信息# [% z8 [" N' ]2 n4 P
[root@client ~]# ceph pg 0.26 query
% O7 v4 }6 D0 e: t5、查看pg中stuck的状态
/ n K1 G. Q* ~3 m% @- A5 ^[root@client ~]# ceph pg dump_stuck unclean
" _0 H# G; K& x6 y% Ook
6 L. w y2 d8 t3 @* K+ u( g! H[root@client ~]# ceph pg dump_stuck inactive
+ s" f |0 R& Y1 z8 O: b5 C& Aok/ |/ T g- D- H( S' z9 Q
[root@client ~]# ceph pg dump_stuck stale+ w# g" [. C: ?( B
ok
* ^- H* I5 y1 D( `6、显示一个集群中的所有的pg统计7 Q/ g; p0 t* i/ A5 v* I& B
ceph pg dump --format plain6 n& W, x( N4 y& p/ @! t' c
7、恢复一个丢失的pg) t' U3 M5 g, a8 U# J
ceph pg {pg-id} mark_unfound_lost revert$ ~% R# k7 B. X0 }0 I4 W+ d- Z
8、显示非正常状态的pg
7 G0 F x* {* w- e1 Vceph pg dump_stuck inactive|unclean|stale% Q/ H9 P5 y6 j$ F+ G& w8 T
六、pool D" I/ N D+ a- z
1、查看ceph集群中的pool数量
2 ^% s% p+ D% u v9 ^8 \& D[root@admin ~]# ceph osd lspools0 G+ x5 A5 [+ v* i f" u
0 data,1 metadata,2 rbd,# {0 F2 b3 ~, H& \, Z3 ~! I% G: u
2、在ceph集群中创建一个pool8 T% y* k2 X7 k* H
ceph osd pool create jiayuan 100 #这里的100指的是PG组
% ?$ ~1 W$ j* t1 C" b& n5 D3、为一个ceph pool配置配额
9 p% z3 _6 [0 f1 f9 { X1 ~( |ceph osd pool set-quota data max_objects 10000
( `- N' s* [; |4 M( B' e4、在集群中删除一个pool- q4 U4 J9 W* h: o4 |( A
ceph osd pool delete jiayuan jiayuan --yes-i-really-really-mean-it #集群名字需要重复两次3 R5 i; `) [2 J
5、显示集群中pool的详细信息2 {) U5 v& [$ S! S/ ?" Q$ O& S
[root@admin ~]# rados df
' U7 x5 h# ?2 ~% Dpool name category KB objects clones degraded unfound rd rd KB wr wr KB: z8 z0 S2 S0 Y: `; @& d% @
data - 475764704 116155 0 0 0 0 0 116379 4757647043 k) O1 S3 }$ k0 H8 E
metadata - 5606 21 0 0 0 0 0 314 58338 `, j/ C! ^2 ]( R: S
rbd - 0 0 0 0 0 0 0 0 0# c1 H; L. {5 Y; _( b- H0 B/ I9 K
total used 955852448 116176! X! s8 w& H k6 f. }
total avail 639497596
- U' W6 w# C% d% B5 w, U. Q total space 15953500441 e, N9 o3 h- v
[root@admin ~]#
# P8 @8 y; P) M3 D7 |6、给一个pool创建一个快照" e3 `- x% B9 x; R; p) v
[root@admin ~]# ceph osd pool mksnap data date-snap
5 @7 l3 e5 S& v$ b. ]8 d4 pcreated pool data snap date-snap
/ d. m9 X8 N' o, S, q7、删除pool的快照
; }& \; D& ~8 r( p5 S; l3 a[root@admin ~]# ceph osd pool rmsnap data date-snap7 n% [9 u/ y0 ]- `% L+ }9 Z6 S
removed pool data snap date-snap) d8 E) v: g) c5 A# K$ _! m& S
8、查看data池的pg数量
7 W9 O' [1 y; k' X3 E; {[root@admin ~]# ceph osd pool get data pg_num# j! P1 h; a) S& e5 Z6 y
pg_num: 64
8 \" ]2 ]) }6 `* t9、设置data池的最大存储空间为100T（默认是1T)6 G+ C- J" z0 e
[root@admin ~]# ceph osd pool set data target_max_bytes 100000000000000; k# E& \% J' x8 ]5 I
set pool 0 target_max_bytes to 100000000000000* ^( `) w, _* n
10、设置data池的副本数是3
7 X# j5 d3 s# L p1 I[root@admin ~]# ceph osd pool set data size 3
( X+ W; l$ u u6 ]0 r; ^# t, {& Oset pool 0 size to 3, b$ _9 n, }& O" \2 w' m I
11、设置data池能接受写操作的最小副本为2# a9 a6 _* C. V9 u7 y9 b
[root@admin ~]# ceph osd pool set data min_size 2
4 G1 A, f0 V/ C0 Z/ Y; _4 @: Vset pool 0 min_size to 2( M: X& x- k, n* c; Q+ r8 D
12、查看集群中所有pool的副本尺寸0 f% T+ M4 G# q" ] W0 [4 s* v) H& A
[root@admin mycephfs]# ceph osd dump | grep 'replicated size'
& u" Y: v9 i; W% }, Wpool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 26 owner 0 flags hashpspool crash_replay_interval 45 target_bytes 100000000000000 stripe_width 0
2 U; O, t! G/ v! D! bpool 1 'metadata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0/ f: }0 _/ k' p+ ?5 R/ o' Q
pool 2 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0 P$ |/ J; `6 ?; M- L6 r4 X
13、设置一个pool的pg数量6 c+ t+ y5 a( U0 E4 Y' `- j1 ^0 z
[root@admin ~]# ceph osd pool set data pg_num 100
' q+ E: l7 { i8 o1 h, Pset pool 0 pg_num to 100) L9 }6 O0 ^ k2 q8 s7 u
14、设置一个pool的pgp数量
4 a1 k; H$ E+ V' K8 J% B, z" T' v* E+ Q[root@admin ~]# ceph osd pool set data pgp_num 100/ b9 W t7 |. u j1 N* W0 f3 ~/ G2 v2 }
set pool 0 pgp_num to 100% L/ t8 [3 t" A `8 F9 Q; o6 \6 A6 m: |
: l+ P1 s R: [' S& T0 g" ^' m3 C七、rados和rbd指令
6 Z8 m L; I5 R& |( Q0 G; T" M1、rados命令使用方法
$ z7 U+ H8 z4 `, V7 @" F- \! T& V（1）、查看ceph集群中有多少个pool （只是查看pool)) s2 B( g& G* Z! ~% S. \, [: S
[root@node-44 ~]# rados lspools! W. Q( \8 R' ?4 E3 }1 R+ P
data
, c8 ?* y) s* e7 Z2 v7 K- ~metadata) i/ g- \* G* J" k+ O; G
rbd
- B* c1 F! I' c+ I1 limages
% f D; D% n! u) gvolumes O$ j- `) J7 U8 r1 d; O8 V P
.rgw.root
0 G2 ^8 O) U3 l+ M9 dcompute$ ~5 S2 y6 D* I; m# y4 I( {
.rgw.control
" S, j2 k9 w. Y.rgw
# O3 a/ ^+ \& x C% A.rgw.gc6 o6 Y8 R4 |2 i3 ]
.users.uid
# n$ ~+ p( a5 t（2）、查看ceph集群中有多少个pool,并且每个pool容量及利用情况
9 E: j' Z+ }; {$ M' r0 P- F, b8 d1 m+ q[root@node-44 ~]# rados df
* F, i% Y! C3 Z' j2 upool name category KB objects clones degraded unfound rd rd KB wr wr KB
7 ~4 C7 G+ W u8 d+ C2 V& d7 k.rgw - 0 0 0 0 0 0 0 0 0
5 L- w6 r5 ^$ H, B.rgw.control - 0 8 0 0 0 0 0 0 0& P3 |/ M W& s, e
.rgw.gc - 0 32 0 0 0 57172 57172 38136 09 r5 Y: R- c, q5 U) {
.rgw.root - 1 4 0 0 0 75 46 10 10
/ P, } c) ?! v.users.uid - 1 1 0 0 0 0 0 2 1
3 u- T; ^/ U2 ^/ Ycompute - 67430708 16506 0 0 0 398128 75927848 1174683 222082706% q/ C8 B4 @3 Q7 _( V
data - 0 0 0 0 0 0 0 0 08 {) `$ Y: Z$ m! r' K: S2 G
images - 250069744 30683 0 0 0 50881 195328724 65025 388375482
; T3 u: u" |' ?metadata - 0 0 0 0 0 0 0 0 04 [# I$ {4 r9 M5 V2 Q/ b
rbd - 0 0 0 0 0 0 0 0 04 f7 D% B1 _2 T- D% w
volumes - 79123929 19707 0 0 0 2575693 63437000 1592456 1638121726 P/ w7 e) b, c3 r( e' B. L
total used 799318844 66941
3 ?) A! W8 Z5 l3 w4 | total avail 11306053720
# O1 d- G% d' b2 }+ I total space 12105372564( t, d+ y% H8 T# Z" z& A
[root@node-44 ~]#
3 F7 U9 r0 s Y# N+ F# \4 M（3）、创建一个pool; U/ s7 l/ }4 f, e$ Q* j$ w
[root@node-44 ~]#rados mkpool test- g+ `/ _! f6 A( Z7 T& v+ X1 A) {
(4)、查看ceph pool中的ceph object （这里的object是以块形式存储的）
. I; e4 L6 Y: d. A- o[root@node-44 ~]# rados ls -p volumes | more+ Q! l! L7 T2 Y/ ]' M" L9 `
rbd_data.348f21ba7021.0000000000000866
- v k8 j, U U* j; krbd_data.32562ae8944a.0000000000000c79
+ N7 C9 j8 b, E$ r# P& M) arbd_data.589c2ae8944a.00000000000031ba
4 D/ l" D/ o/ z2 xrbd_data.58c9151ff76b.00000000000029af5 u5 S$ f4 R2 ~
rbd_data.58c9151ff76b.0000000000002c19+ e8 C5 P9 D5 @8 h7 I
rbd_data.58c9151ff76b.0000000000000a5a
1 H8 ^0 n# F, }, Nrbd_data.58c9151ff76b.0000000000001c69
* T; E7 W; r2 z( h0 C! c# prbd_data.58c9151ff76b.000000000000281d B/ T3 ?. z8 L$ g2 ~
rbd_data.58c9151ff76b.0000000000002de1+ e: \! f; B# U
rbd_data.58c9151ff76b.0000000000002dae
: r7 s4 l% X: P（5）、创建一个对象object 3 ?) ~# X, i% @" D Y
[root@admin-node ~]# rados create test-object -p test- Q2 x r6 j3 s, H; P
2 l4 O( d* ?1 D& E/ K! D
[root@admin-node ~]# rados -p test ls3 [ q2 @. [/ [1 [% k- L; S0 }
test-object. b6 f" v+ t- i: J# P. o
（6）、删除一个对象
0 I0 r" h8 |7 w" A. F[root@admin-node ~]# rados rm test-object-1 -p test* ]1 A* X3 ]9 ?7 Y
6 t6 M( X5 J2 I
( A5 y# o7 ^0 a8 [% U2、rbd命令的用法 4 d$ S& |6 c8 `, y, `$ |. \
& E9 \- p0 {+ H6 ]0 n0 {4 |
（1）、查看ceph中一个pool里的所有镜像8 [4 Z8 t: K& Z0 f
[root@node-44 ~]# rbd ls images
7 |5 I. z& u& P v4 H5 \/ T# c2014-05-24 17:17:37.043659 7f14caa6e700 0 -- :/1025604 >> 10.49.101.9:6789/0 pipe(0x6c5400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x6c5660).fault
4 i3 {( T3 A# n/ Y2182d9ac-52f4-4f5d-99a1-ab3ceacbf0b9
! h1 S9 I' I) W' }. ]& ]3 d34e1a475-5b11-410c-b4c4-69b5f780f03c
- X4 e2 Q- {! q) ~5 s476a9f3b-4608-4ffd-90ea-8750e804f46e
5 R& z6 W: C$ G! {* r60eae8bf-dd23-40c5-ba02-266d5b942767$ R! n0 j, W7 @4 M. W
72e16e93-1fa5-4e11-8497-15bd904eeffe# [ ]5 l5 [8 m
74cb427c-cee9-47d0-b467-af217a67e60a0 F$ e; g, }6 H$ C
8f181a53-520b-4e22-af7c-de59e8ccca78
, S" l2 B7 U. M9 U9867a580-22fe-4ed0-a1a8-120b8e8d18f4
8 I6 O7 @" ^) i' h5 F* Yac6f4dae-4b81-476d-9e83-ad92ff25fb133 C4 B0 j& Q) V ^$ {1 }# L% x" K
d20206d7-ff31-4dce-b59a-a622b0ea3af67 s7 m2 G9 @, f$ u, K* w! @0 _
+ J( U4 q3 D* q[root@node-44 ~]# rbd ls volumes
% B; m+ l6 }& O4 g+ l2014-05-24 17:22:18.649929 7f9e98733700 0 -- :/1010725 >> 10.49.101.9:6789/0 pipe(0x96a400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x96a660).fault0 @% n7 q: @# X; n- H t
volume-0788fc6c-0dd4-4339-bad4-e9d78bd5365c
. p z& J7 G1 g4 U: Rvolume-0898c5b4-4072-4cae-affc-ec59c2375c51# {1 s8 {( c. b3 n" e$ ~
volume-2a1fb287-5666-4095-8f0b-6481695824e22 w# c) s) ]) z7 x' C
volume-35c6aad4-8ea4-4b8d-95c7-7c3a8e8758c5
+ G! d# v# f; i! X/ N' b6 `8 ?volume-814494cc-5ae6-4094-9d06-d844fdf485c4
6 v4 R$ a3 ` V7 X2 H* B/ pvolume-8a6fb0db-35a9-4b3b-9ace-fb647c2918ea
3 ?. J) j% n, Y) `7 s* L3 F7 qvolume-8c108991-9b03-4308-b979-51378bba2ed1
. A* |- c- Y! x- O* J' m |volume-8cf3d206-2cce-4579-91c5-77bcb4a8a3f8
; G5 g- w1 M& xvolume-91fc075c-8bd1-41dc-b5ef-844f23df177d; A* N/ f; L: x
volume-b1263d8b-0a12-4b51-84e5-74434c0e73aa+ K& Q( }& O! m/ i5 O8 Z
volume-b84fad5d-16ee-4343-8630-88f265409feb* x# J6 ]0 P9 t8 s- _
volume-c03a2eb1-06a3-4d79-98e5-7c62210751c3
9 {% z2 }& a. Y$ ^/ Cvolume-c17bf6c0-80ba-47d9-862d-1b9e9a48231e( \/ h2 Q1 g) J1 o0 Z" T+ D% Y
volume-c32bca55-7ec0-47ce-a87e-a883da4b4ccd' I! S2 H8 e: Y/ E
volume-df8961ef-11d6-4dae-96ee-f2df8eb4a08c
* [1 S y M5 }volume-f1c38695-81f8-44fd-9af0-458cddf103a3
# S3 s+ ~: \( U2 L
6 g1 H8 ? @) S8 C3 a（2）、查看ceph pool中一个镜像的信息
$ `8 P- s& ^( m: |0 w[root@node-44 ~]# rbd info -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a/ F2 W& W; ]; M/ K9 ?9 e2 H6 }6 m
rbd image '74cb427c-cee9-47d0-b467-af217a67e60a':: Q. y2 W+ H+ M
size 1048 MB in 131 objects
" M4 N4 k- \4 u order 23 (8192 KB objects)
) Y, _5 Y3 \0 L M0 ^4 N block_name_prefix: rbd_data.95c7783fc0d0( l0 D3 v# ~4 z) N1 z! v
format: 2
o, s$ z k, a2 X2 T2 c! L. F. f features: layering
* A. ~4 [3 }5 l% ]（3）、在test池中创建一个命名为zhanguo的10000M的镜像7 @- U+ I4 V7 S7 c
[root@node-44 ~]# rbd create -p test --size 10000 zhanguo
% [3 M; b! K$ R' L[root@node-44 ~]# rbd -p test info zhanguo #查看新建的镜像的信息
- R" t, u# E( Y, u- h5 arbd image 'zhanguo':
$ T. ~4 s" S5 c! i8 I size 10000 MB in 2500 objects
8 J) a& Y) ~0 X* F) }. C" v, a order 22 (4096 KB objects)1 o/ D( y# X0 o$ t+ S: p
block_name_prefix: rb.0.127d2.2ae8944a( p9 F* V2 J5 m0 U
format: 1' u1 B! c# v3 t; A4 f" X% `5 T5 ?0 V
[root@node-44 ~]# c4 _5 I# }* q. q# p1 V9 M
（4）、删除一个镜像) n9 N2 {4 N; `$ E P! E- ~7 a! F0 t
[root@node-44 ~]# rbd rm -p test lizhanguo7 Z3 u% ~1 W" m( x2 k5 Q* S
Removing image: 100% complete...done., l6 P' k3 k- C) \" i
（5）、调整一个镜像的尺寸+ X" q! L# E0 P: ?1 {3 }6 q& p
[root@node-44 ~]# rbd resize -p test --size 20000 zhanguo
) v+ U8 ~5 {1 j- A t0 jResizing image: 100% complete...done.
$ O+ l4 {; V) f( m1 x[root@node-44 ~]# rbd -p test info zhanguo #调整后的镜像大小
7 o, c) g2 i! J( B& X; e# grbd image 'zhanguo':( _2 v6 j% l8 [
size 20000 MB in 5000 objects" X7 k, o l- Q
order 22 (4096 KB objects)* v1 `& j O5 t+ f# I/ q O
block_name_prefix: rb.0.127d2.2ae8944a4 `. \4 @7 A% F: M2 N
format: 1
( D- d M) F1 M b7 o% V[root@node-44 ~]# # C! E5 k+ f1 a. H: A" G6 ~1 J1 p
（6）、给一个镜像创建一个快照
: Q9 M5 H9 G: Q% b f[root@node-44 ~]# rbd snap create test/zhanguo@zhanguo123 #池/镜像@快照; F& Q0 W8 \# \0 S/ l3 Z3 M; @! B
[root@node-44 ~]# rbd snap ls -p test zhanguo
. p# J6 o; ], T% J! vSNAPID NAME SIZE * T2 j7 a1 ?- ?4 p$ R
2 zhanguo123 20000 MB
/ A/ j4 x# y: A% x[root@node-44 ~]# # S; e+ k* ?, e0 a
[root@node-44 ~]# rbd info test/zhanguo@zhanguo123& S$ ~1 \+ n) `% D
rbd image 'zhanguo':3 c; R2 L9 I* O3 a5 f. O6 ~% r
size 20000 MB in 5000 objects! e8 X* l7 i4 H, P/ F. @: D& X
order 22 (4096 KB objects)+ Z8 A+ i/ q# d4 Q/ G- ? Z f
block_name_prefix: rb.0.127d2.2ae8944a
3 M0 R& p; \( ~ format: 1
: q; g: h, n7 @ protected: False+ s- }) n7 g: p: D/ n9 R' h& z
[root@node-44 ~]#
& p; d* o: J0 b5 \6 ]) J（7）、查看一个镜像文件的快照0 b5 [+ J% w1 w" b. L
[root@os-node101 ~]# rbd snap ls -p volumes volume-7687988d-16ef-4814-8a2c-3fbd85e928e4" m9 g( m: {$ G2 U0 c# O
SNAPID NAME SIZE
. |+ m. Q; y/ U( `: f 5 snapshot-ee7862aa-825e-4004-9587-879d60430a12 102400 MB 8 c% |$ l% R- M% `. C6 D* Z7 B
（8）、删除一个镜像文件的一个快照快照
K' t) ^3 O& h4 B7 L* N' _ 快照所在的池/ 快照所在的镜像文件 @ 快照9 e( r7 |0 e8 {6 r/ k
[root@os-node101 ~]# rbd snap rm volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a12
" h. C4 g9 F# q: @ C2 u; N! E: a2 ^! lrbd: snapshot 'snapshot-60586eba-b0be-4885-81ab-010757e50efb' is protected from removal.
0 u' A: ^$ Q/ O% a; D2014-08-18 19:23:42.099301 7fd0245ef760 -1 librbd: removing snapshot from header failed: (16) Device or resource busy
" X1 n3 P, ~' l2 [ D上面不能删除显示的报错信息是此快照备写保护了，下面命令是删除写保护后再进行删除。4 r4 e: `' {: _, m( I- Q
[root@os-node101 ~]# rbd snap unprotect volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a12( K+ V" C6 e1 \1 {- S' v( _
[root@os-node101 ~]# rbd snap rm volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a124 l6 m. n7 G6 o' r
（9）删除一个镜像文件的所有快照
" a: a1 _& i* y' L[root@os-node101 ~]# rbd snap purge -p volumes volume-7687988d-16ef-4814-8a2c-3fbd85e928e4
- M/ w1 B6 V# rRemoving all snapshots: 100% complete...done.
: @# n# P! [5 i9 A# ?! S8 \3 W, x. f# W0 @, S9 K
% \+ \$ J v7 E
（10）、把ceph pool中的一个镜像导出& X% o4 Q( D* v! |1 ]3 i( M
导出镜像: _& n% _% t! { y: W O( o
[root@node-44 ~]# rbd export -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a /root/aaa.img! J& o, F X$ g" `
2014-05-24 17:16:15.197695 7ffb47a9a700 0 -- :/1020493 >> 10.49.101.9:6789/0 pipe(0x1368400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x1368660).fault
* {2 q' \' J) y# i* {5 t" R7 ? JExporting image: 100% complete...done.
& x' g1 s% x Q, _' k2 q) a" I1 \& E! D: \: Q. Z; }
导出云硬盘
2 \6 y9 V0 [% J' P( Q U[root@node-44 ~]# rbd export -p volumes --image volume-470fee37-b950-4eef-a595-d7def334a5d6 /var/lib/glance/ceph-pool/volumes/Message-JiaoBenJi-10.40.212.24
6 H9 `! P9 ?: O) O2014-05-24 17:28:18.940402 7f14ad39f700 0 -- :/1032237 >> 10.49.101.9:6789/0 pipe(0x260a400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x260a660).fault
- N7 ~% t3 n* R6 q, ~Exporting image: 100% complete...done.
! d6 K# J q/ }; H1 Y* a, @" G i. |- N* u& H" @
（11）、把一个镜像导入ceph中 （但是直接导入是不能用的，因为没有经过openstack,openstack是看不到的）8 [% e: r0 g% |
[root@node-44 ~]# rbd import /root/aaa.img -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a
& s3 P. _: [* j* B0 R" G5 uImporting image: 100% complete...done.6 V. [/ m5 h, Y' p' S+ P
1 w) W4 w$ I" h
|
|
发表于 2018-10-6 16:15:42