|
|
1、启动一个ceph 进程" h5 B; V& n( l2 [
启动mon进程
" W! k$ e4 Q% ?+ z* sservice ceph start mon.node1/ m+ |6 O2 P6 N8 q7 z" x
启动msd进程
8 ?( [' N' v( J+ {service ceph start mds.node1
# M- u3 X! t2 v5 s启动osd进程 ^, \" b+ o5 n' `/ m9 z4 g9 J; b# n
service ceph start osd.0; l% r- x1 \. D. ?- U# j5 I; H
2、查看机器的监控状态
* `3 c! h: ?" L( F[root@client ~]# ceph health4 f2 Q3 w1 r' R4 p! {$ Z
HEALTH_OK f' Z( k5 {$ ]/ o" L, }
3、查看ceph的实时运行状态0 b: N+ h& C# v, r8 ^
[root@client ~]# ceph -w* }: i' j; d: E3 R2 |3 L
cluster be1756f2-54f7-4d8f-8790-820c82721f17! b+ Q9 ^9 T2 k$ L" _* r' k
health HEALTH_OK
; Q' A5 m: o& ^$ w9 _4 q3 Y monmap e2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3, l, r! M4 q; u5 ^
mdsmap e95: 1/1/1 up {0=node2=up:active}, 1 up:standby m7 t2 R8 ~. c3 G5 \
osdmap e88: 3 osds: 3 up, 3 in
: U; R- o3 [% y T: x% @& v pgmap v1164: 448 pgs, 4 pools, 10003 MB data, 2520 objects% L/ G8 B/ T% t* A* u7 F
23617 MB used, 37792 MB / 61410 MB avail
1 H3 Y, g" T; k4 X) ]9 l2 y, N" r 448 active+clean
- f2 L$ Z( } k d4 n2014-06-30 00:48:28.756948 mon.0 [INF] pgmap v1163: 448 pgs: 448 active+clean; 10003 MB data, 23617 MB used, 37792 MB / 61410 MB avail+ I' Y& y5 B1 }5 I8 Q) ^6 t
) U* a( m% K6 v# B# G
4、检查信息状态信息, d1 _& g8 q5 a3 \
[root@client ~]# ceph -s& b; {$ [# X9 v
cluster be1756f2-54f7-4d8f-8790-820c82721f17
. I4 {: d% m5 s6 Q: \ i health HEALTH_OK
2 O1 Z' `- t1 M5 J8 m6 @: R monmap e2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3
) |# h2 B* X* ?. o( p) x! \! D0 } mdsmap e95: 1/1/1 up {0=node2=up:active}, 1 up:standby
- h2 r6 n; |, H# W3 L( c2 z+ d4 t' x7 b osdmap e88: 3 osds: 3 up, 3 in
2 `6 n7 U% J4 f, ^, ]: _# T7 j* a pgmap v1164: 448 pgs, 4 pools, 10003 MB data, 2520 objects
6 u8 F! y$ F# q+ Y u 23617 MB used, 37792 MB / 61410 MB avail
- K' z% v# p1 K- m. Q8 e 448 active+clean2 g* E$ W) q8 a" {' Y2 V, e
[root@client ~]# # G c! C0 |8 Y' R
+ P. D7 f) I' ?5、查看ceph存储空间) G4 N0 p: v5 l- Q! o7 u# i, h+ L
[root@client ~]# ceph df
, T6 n, y S0 Y9 H' wGLOBAL:
7 \4 a a1 s$ I7 [ SIZE AVAIL RAW USED %RAW USED 8 j. B0 ]$ K* X6 [
61410M 37792M 23617M 38.46
5 l5 u& o1 v6 X" cPOOLS:! g: d9 V6 B1 P8 q: t% I7 Z: H6 k
NAME ID USED %USED OBJECTS
+ n. d+ D" O6 `: e7 Q data 0 10000M 16.28 2500
' J" G, d8 O9 L. P metadata 1 3354k 0 20
# t: g: G/ g7 r6 b" D0 f; C9 p rbd 2 0 0 0 2 O: x0 w' D: e' `) Z0 F" l4 D
jiayuan 3 0 0 0 6 P l9 f& T# V, ]: T
[root@client ~]#
( P3 c$ d2 `! E' y; x& T6、删除一个节点的所有的ceph数据包
7 z+ q3 L' ~2 y) r5 j[root@node1 ~]# ceph-deploy purge node1
* z. Y/ e# p% d m5 a1 x[root@node1 ~]# ceph-deploy purgedata node1: b! v, S# u' ^% w7 x
7、为ceph创建一个admin用户并为admin用户创建一个密钥，把密钥保存到/etc/ceph目录下：& S# D7 O5 S( F0 d% I/ R* B! H* C' o
ceph auth get-or-create client.admin mds 'allow' osd 'allow *' mon 'allow *' > /etc/ceph/ceph.client.admin.keyring# \- f/ F; N& ^/ [3 ^
或2 c9 i" l: m. P) ?" r
ceph auth get-or-create client.admin mds 'allow' osd 'allow *' mon 'allow *' -o /etc/ceph/ceph.client.admin.keyring
0 h g, R$ g- }8、为osd.0创建一个用户并创建一个key$ _) d) h* \6 u
ceph auth get-or-create osd.0 mon 'allow rwx' osd 'allow *' -o /var/lib/ceph/osd/ceph-0/keyring! Z$ {& ?; r* i' h2 g
9、为mds.node1创建一个用户并创建一个key7 A1 A. I! r0 @% r
ceph auth get-or-create mds.node1 mon 'allow rwx' osd 'allow *' mds 'allow *' -o /var/lib/ceph/mds/ceph-node1/keyring
) Y8 d7 w) E1 N/ @( }10、查看ceph集群中的认证用户及相关的key
" u8 v, U- L7 S* j, Q5 qceph auth list
/ w7 H/ D) d0 Z q( A l11、删除集群中的一个认证用户0 F7 M# M4 R7 h) o7 ], L) k/ ?9 C
ceph auth del osd.0% l3 R4 m; G2 d: K
12、查看集群的详细配置
* B* H6 l7 `* P; f+ } N& u0 X* x* T[root@node1 ~]# ceph daemon mon.node1 config show | more
: k/ Q; d" ^8 \' R& a6 A7 ?, Y13、查看集群健康状态细节( m, R8 u& v n, X% D
[root@admin ~]# ceph health detail6 w3 t. w6 S6 a5 {' U; ^9 H
HEALTH_WARN 12 pgs down; 12 pgs peering; 12 pgs stuck inactive; 12 pgs stuck unclean
7 B6 R$ H$ h/ l4 Apg 3.3b is stuck inactive since forever, current state down+peering, last acting [1,2]! G5 r) {" p+ ^: l4 |1 g# s, q
pg 3.36 is stuck inactive since forever, current state down+peering, last acting [1,2]
! W) a- X1 _/ e& k) K* J1 S. Epg 3.79 is stuck inactive since forever, current state down+peering, last acting [1,0]
- P2 I' n& F8 ]5 O opg 3.5 is stuck inactive since forever, current state down+peering, last acting [1,2] u. x2 V( Q& w( T" L" D5 I
pg 3.30 is stuck inactive since forever, current state down+peering, last acting [1,2]
( o% O! V7 G) ~pg 3.1a is stuck inactive since forever, current state down+peering, last acting [1,0]
# F+ S# t6 _. {& ?' |; O; X R9 d% qpg 3.2d is stuck inactive since forever, current state down+peering, last acting [1,0]
! Y8 E. H' D& _+ Jpg 3.16 is stuck inactive since forever, current state down+peering, last acting [1,2]
/ Z+ d$ L) C/ q" ?
* z; p5 g2 h8 R% h# t% x* D& ^14、查看ceph log日志所在的目录: ^2 q& c: O7 k9 `( Z; R/ J
[root@node1 ~]# ceph-conf --name mon.node1 --show-config-value log_file7 h# @$ F* J* X) A
/var/log/ceph/ceph-mon.node1.log
1 B$ E5 g1 T1 T7 r5 j7 J% |
9 i+ d0 c+ s7 u% M4 g4 y8 E8 M! Q' k6 r
二、mon
/ x8 U* n# E$ b$ H' m, F) }# _1、查看mon的状态信息
: o" m1 W' D8 ^0 \# k0 P[root@client ~]# ceph mon stat
7 U0 C- z: |( ?, {. X% J* ue2: 3 mons at {node1=10.240.240.211:6789/0,node2=10.240.240.212:6789/0,node3=10.240.240.213:6789/0}, election epoch 294, quorum 0,1,2 node1,node2,node3# F. \. r. j1 u0 B; y( r* J8 ?: D
2、查看mon的选举状态
( V: a4 \& S! k[root@client ~]# ceph quorum_status5 G8 r. d5 X: O( i% H
{"election_epoch":294,"quorum":[0,1,2],"quorum_names":["node1","node2","node3"],"quorum_leader_name":"node1","monmap":{"epoch":2,"fsid":"be1756f2-54f7-4d8f-8790-820c82721f17","modified":"2014-06-26 18:43:51.671106","created":"0.000000","mons":[{"rank":0,"name":"node1","addr":"10.240.240.211:6789\/0"},{"rank":1,"name":"node2","addr":"10.240.240.212:6789\/0"},{"rank":2,"name":"node3","addr":"10.240.240.213:6789\/0"}]}}; p9 D8 \/ U! i' |
3、查看mon的映射信息# Q$ ]& [0 I3 ~( Z+ D8 h; d0 L% w% J' G0 C F
[root@client ~]# ceph mon dump. M# E0 x- ]9 p& `
dumped monmap epoch 2
+ }0 T1 h! p' kepoch 2- v. g3 Z( P8 B- l% v
fsid be1756f2-54f7-4d8f-8790-820c82721f171 i$ ~; O% j5 N4 ^7 h
last_changed 2014-06-26 18:43:51.671106! t( C& i! V4 c" r2 }* z# ?
created 0.000000
. x$ H8 X @' R3 T4 Q* ]$ E' B; @0: 10.240.240.211:6789/0 mon.node1
1 |7 x) ?; _$ H% _( S1: 10.240.240.212:6789/0 mon.node2
( f# z4 p( [( {! Z2: 10.240.240.213:6789/0 mon.node36 t" u7 A& {9 N$ }
4、删除一个mon节点9 s# M [6 Q) @
[root@node1 ~]# ceph mon remove node1' _% E9 }3 j/ N, `% N% }- \
removed mon.node1 at 10.39.101.1:6789/0, there are now 3 monitors
: k3 f1 F1 s( W: {! r) c+ I* v2014-07-07 18:11:04.974188 7f4d16bfd700 0 monclient: hunting for new mon
3 o: L, N1 e5 f% @5、获得一个正在运行的mon map，并保存在1.txt文件中
1 m$ v* X' k4 r[root@node3 ~]# ceph mon getmap -o 1.txt
# @6 z, b4 T( {$ b* S) zgot monmap epoch 6
% V# f# V d* t9 }8 b7 K6、查看上面获得的map* f0 \# ^# h* Z9 G$ \
[root@node3 ~]# monmaptool --print 1.txt ( f( `$ ?- C ~' w' }
monmaptool: monmap file 1.txt$ P6 O' V1 w, ]8 _- u. w1 A" \
epoch 6* U5 y$ Z! k) S( D
fsid 92552333-a0a8-41b8-8b45-c93a8730525e" O- Z/ d) i; t# Z0 `
last_changed 2014-07-07 18:22:51.927205
; a% k# e' Q g1 Acreated 0.000000
; R* C+ Y) ?6 }+ d3 S3 W) ?0: 10.39.101.1:6789/0 mon.node1% J1 Y8 S) G, W. |0 n/ F$ c3 w$ I
1: 10.39.101.2:6789/0 mon.node2
+ k0 l- B0 l& g( T1 U& M2: 10.39.101.3:6789/0 mon.node3
: U* H8 P" N; K4 L3 E6 w/ m( w[root@node3 ~]#
" o" t0 I% X( X% Q1 [ \6 r7、把上面的mon map注入新加入的节点
! _9 _7 V1 C4 @+ G) X. m0 Kceph-mon -i node4 --inject-monmap 1.txt' |, Y) `5 s& P3 N
8、查看mon的amin socket
% I- |2 o4 v5 ~) G" K* O7 d" vroot@node1 ~]# ceph-conf --name mon.node1 --show-config-value admin_socket
0 k* | x0 N' S4 @/var/run/ceph/ceph-mon.node1.asok
2 Q5 e/ l" r: {4 @& w H! [9、查看mon的详细状态& e) K6 s' W6 R6 P0 D9 V
[root@node1 ~]# ceph daemon mon.node1 mon_status
# v% o! {+ l. @- f4 c6 @{ "name": "node1",: K- C' t/ `- x& t" T
"rank": 0,3 r* Y& Z2 Z! Y" N* ~! ^
"state": "leader",7 E/ [# K! ?; j8 i+ G
"election_epoch": 96,
8 c# I. n$ {( h @8 Z+ }9 V* a "quorum": [! [# n+ f* d( O* ~( X0 J' x- v
0,2 K( o& w% V0 K4 [9 n4 Y
1,3 [3 L% Q7 R# _( x
2],
# e6 {& I; @) e "outside_quorum": [],
8 Q, p) B% X$ ~ "extra_probe_peers": [
4 \" I& P& Z9 g* n* m "10.39.101.4:6789\/0"],- d" A: \9 w2 e4 S8 o
"sync_provider": [],; [+ J$ {: H3 ?5 N. I
"monmap": { "epoch": 6,
1 z X5 z! ^/ c% j "fsid": "92552333-a0a8-41b8-8b45-c93a8730525e", m! f6 i/ l; D7 G7 ^6 H6 W7 ]0 |
"modified": "2014-07-07 18:22:51.927205",6 A% y8 S! a; l. P, V( Z- v/ }& w
"created": "0.000000",* m' r$ s+ I D; U
"mons": [6 e. [7 Z/ z3 j( H. N! V. O
{ "rank": 0,3 f* {$ Z& S2 M1 | D* z
"name": "node1",
9 H, n' T9 k$ i# f "addr": "10.39.101.1:6789\/0"},
, H4 k3 a. u6 S! v" D* m { "rank": 1,7 x- Q, X2 C& |% K# @( ]* H
"name": "node2",
( U! A( n8 q6 L "addr": "10.39.101.2:6789\/0"},
) l" O# D$ t- ?- [- [ { "rank": 2,
6 s1 I4 l% }& W5 \9 ^1 E "name": "node3",) b1 F+ B) C" P& h1 b; ?
"addr": "10.39.101.3:6789\/0"}]}
7 i1 \/ ^$ R1 ?% w& y0 z. w/ m- x" C10、删除一个mon节点9 ~9 Y3 p8 K- p; P0 A
[root@os-node1 ~]# ceph mon remove os-node1
- b( d( S4 R! h: v# {& L* j5 dremoved mon.os-node1 at 10.40.10.64:6789/0, there are now 3 monitors" P! O9 {2 D8 M6 M; |
2 I* ]$ ^; n* x) ^( V三、msd
; r( l B, ^7 m) v+ o1、查看msd状态
}' b% `3 M9 w& G* x0 |. P[root@client ~]# ceph mds stat- Q$ }4 i+ H" X8 A7 u' m
e95: 1/1/1 up {0=node2=up:active}, 1 up:standby0 u* G- E2 ]1 z a# p l5 l
2、查看msd的映射信息/ F$ b. i8 e) Y) E2 ]. v% x
[root@client ~]# ceph mds dump) d2 A( T/ s' h5 G+ p2 d/ P
dumped mdsmap epoch 957 F- E! S M: N# `
epoch 95
" {: E4 j% `( Q$ k: c& iflags 0; z' _; j. q: J3 U0 B. A
created 2014-06-26 18:41:57.6868018 Y: W5 \/ ?* V" P1 q7 `: i' s" @9 d
modified 2014-06-30 00:24:11.749967/ }5 r, W3 b6 _
tableserver 0/ e1 M( y# Z( c" @4 n7 s ]2 ^& g
root 0, J' C8 K+ A, [9 A5 w& D! ~ v
session_timeout 60, N3 h4 G' b' P
session_autoclose 3004 p. n7 x+ d$ z0 j* Y- i$ {
max_file_size 10995116277764 m5 w4 C. [$ T0 ?' d
last_failure 84
/ B! l q& Z j! w3 W: d6 ilast_failure_osd_epoch 81
& k+ J: t+ }% lcompat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap}
# N2 ?6 ^- M) omax_mds 1
/ }8 P7 O: W( c9 h4 f4 ein 0! E" c X! r* L7 ?1 m
up {0=5015}
7 E3 ^( u5 Z* M6 Kfailed: G* j" O" w2 W+ @. L
stopped
6 N8 E2 J8 Z4 N( Odata_pools 0* t' I& h3 N: @+ e2 r
metadata_pool 1
7 i1 F6 k2 g# X, z. q8 J7 {6 minline_data disabled8 W% ^: |8 Z; x" @
5015: 10.240.240.212:6808/3032 'node2' mds.0.12 up:active seq 30! ^; q; C; N' N% e/ y# o$ o |
5012: 10.240.240.211:6807/3459 'node1' mds.-1.0 up:standby seq 38
: v' x- I$ V2 f* N% ]3、删除一个mds节点& H2 Q$ u" t( y; }- t: D* s: o
[root@node1 ~]# ceph mds rm 0 mds.node1! f0 u) a) M+ d2 R/ y
mds gid 0 dne) ?1 W# Q- j/ ^
3 ~5 J1 d& G5 a s. n2 k0 H9 h! F四、osd
* F, t5 Q( k1 S' u1、查看ceph osd运行状态
- E$ x$ J8 E+ \[root@client ~]# ceph osd stat
# f3 h7 p" p, v2 K) v4 [ osdmap e88: 3 osds: 3 up, 3 in* _+ i! l0 s" M! `: E" C
5 \: G) z7 P0 [1 o+ ]' i
2、查看osd映射信息6 J; w& F: V6 U7 U8 X& c6 j
[root@client ~]# ceph osd dump. X) C) W3 n7 L9 _' n e) T
epoch 88
$ c1 L6 H& H8 s+ D& g$ z* ]fsid be1756f2-54f7-4d8f-8790-820c82721f176 w3 p2 R5 K* j6 W9 i8 {
created 2014-06-26 18:41:57.687442) E& E1 [! U5 Y5 c C4 v
modified 2014-06-30 00:46:27.179793
4 p7 |& \. v$ Qflags
5 B0 D3 N2 b. D' Hpool 0 'data' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool crash_replay_interval 45 stripe_width 0
0 T% v7 X4 i2 M! M/ J$ qpool 1 'metadata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0/ |8 ^ |9 R8 _3 W% r
pool 2 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0
. M) @* q* }' p$ e$ @: I; q+ Zpool 3 'jiayuan' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 256 pgp_num 256 last_change 73 owner 0 flags hashpspool stripe_width 0
2 @; U n- j& U# kmax_osd 33 h5 k' L. l3 l: l6 A0 C, s6 {
osd.0 up in weight 1 up_from 65 up_thru 75 down_at 64 last_clean_interval [53,55) 10.240.240.211:6800/3089 10.240.240.211:6801/3089 10.240.240.211:6802/3089 10.240.240.211:6803/3089 exists,up 8a24ad16-a483-4bac-a56a-6ed44ab74ff0, T5 L3 q& J8 o+ b ~! l" U Q
osd.1 up in weight 1 up_from 59 up_thru 74 down_at 58 last_clean_interval [31,55) 10.240.240.212:6800/2696 10.240.240.212:6801/2696 10.240.240.212:6802/2696 10.240.240.212:6803/2696 exists,up 8619c083-0273-4203-ba57-4b1dabb893392 b, T8 P/ {! l: ?/ r
osd.2 up in weight 1 up_from 62 up_thru 74 down_at 61 last_clean_interval [39,55) 10.240.240.213:6800/2662 10.240.240.213:6801/2662 10.240.240.213:6802/2662 10.240.240.213:6803/2662 exists,up f8107c04-35d7-4fb8-8c82-09eb885f0e58
$ x' g8 Q3 R$ K6 X+ n[root@client ~]#
" ?1 q- u7 I8 O( T) @- l' D3 h! y. @4 H
3、查看osd的目录树
8 H* v! ~6 n8 \* M[root@client ~]# ceph osd tree
3 k% R6 @+ T% A$ W/ b, s. L; v# id weight type name up/down reweight
9 r9 J5 T; e+ o-1 3 root default
X( s( ]7 S0 P! O. W: |-2 1 host node1+ i- o+ q' J) z3 ^3 a7 F$ J
0 1 osd.0 up 15 _% T' y0 V: h# D
-3 1 host node2! {* c z" Y6 f- v2 E3 E+ r$ M
1 1 osd.1 up 10 _( x3 @' q# i7 \, i; S# n% G" u
-4 1 host node34 C5 L$ m1 W- @, n, a* c+ c3 @' r) A
2 1 osd.2 up 1
- I2 V$ y! p3 ]0 B" f
5 G0 ?9 W+ P7 s& H4、down掉一个osd硬盘, K8 c- R0 @2 B* @% {- O; `+ R6 {
[root@node1 ~]# ceph osd down 0 #down掉osd.0节点
% I6 c+ N0 o' N& }# h. G z5、在集群中删除一个osd硬盘
! p9 _/ C) c! t: n3 L8 L% V[root@node4 ~]# ceph osd rm 0
% l# @: ?; d4 P: L! Hremoved osd.09 T8 o% K, [& g. p& b
6、在集群中删除一个osd 硬盘 crush map
; `0 B. y6 S) R! g[root@node1 ~]# ceph osd crush rm osd.0
% h1 K$ W$ c, M; h/ o* H7、在集群中删除一个osd的host节点/ O. b* I6 H. Z! Y3 h8 w
[root@node1 ~]# ceph osd crush rm node1/ h7 j1 Y2 u4 Y+ b P! p+ T" a
removed item id -2 name 'node1' from crush map
9 _( ]9 n2 I( s: p$ X5 F* N P/ G# S& q& v: u
查看最大osd的个数 2 l: W: h& v8 Q, B0 [
[root@node1 ~]# ceph osd getmaxosd
; F$ M) W; a: t: ~" umax_osd = 4 in epoch 514 #默认最大是4个osd节点' o0 P" b+ v! `3 J0 ?7 {+ o
8、设置最大的osd的个数（当扩大osd节点的时候必须扩大这个值）1 z- ^* V. s7 h
[root@node1 ~]# ceph osd setmaxosd 10
5 q0 B& k) n* R" n9、设置osd crush的权重为1.0
9 w& k% r/ B6 ]! ]/ L+ I; o6 Tceph osd crush set {id} {weight} [{loc1} [{loc2} ...]]- i! l+ D" \2 L. Z( C
例如：2 {8 W8 k/ n5 G
[root@admin ~]# ceph osd crush set 3 3.0 host=node4! ? j, p$ w7 |, {
set item id 3 name 'osd.3' weight 3 at location {host=node4} to crush map
9 s% r0 G) S Q[root@admin ~]# ceph osd tree8 C4 ]& T6 l9 N( S3 P# B3 k* S! A: I
# id weight type name up/down reweight4 A9 U! f1 D* z1 T3 M% \; ~
-1 6 root default% Q' z2 Z/ |6 c
-2 1 host node18 H5 j$ s: x2 Q5 I
0 1 osd.0 up 1
; [3 P4 i2 q, }% g& |-3 1 host node2* U0 R" q' W" k9 k% ^5 i
1 1 osd.1 up 1+ Q) F; y6 d4 I7 n9 L( U) m
-4 1 host node3% e6 G- \* _$ ], w) Q2 b
2 1 osd.2 up 1
0 G0 k: _8 `' Y |, _3 ^; b6 K-5 3 host node4
: p4 U* e" Z' n* W V9 i, X3 3 osd.3 up 0.56 o2 u/ C. r5 K% p+ E& [+ }
! l' o+ l0 I0 |& I- H8 L: y或者用下面的方式
/ f4 h% v8 f7 z9 E0 |0 j8 d[root@admin ~]# ceph osd crush reweight osd.3 1.0
4 k( t5 `9 W/ x. F E& I" preweighted item id 3 name 'osd.3' to 1 in crush map
6 x* r8 `* ]- M1 Q+ Z3 O, o[root@admin ~]# ceph osd tree/ |7 B4 t, X% Q9 t3 E6 F
# id weight type name up/down reweight' w3 n9 ]) n$ O/ d& }/ Q6 b+ L
-1 4 root default1 C& l' o0 Z( y& |) N- C
-2 1 host node1
2 a+ G/ |$ Z' J' t2 l) z4 t0 1 osd.0 up 1
5 t6 n" Y& L" Z- D. ?-3 1 host node28 A6 d) c+ A( }" I( x! i/ ^
1 1 osd.1 up 1
' F% ?( T9 ?$ V-4 1 host node3 @( F8 l+ p! m2 [8 _
2 1 osd.2 up 15 w! u2 _7 G a+ o0 f
-5 1 host node4! v# k$ E5 S, }6 C3 I* {" I
3 1 osd.3 up 0.5
+ U; V( O. b) K* Z: \) A c10、设置osd的权重
h: [4 o9 s3 L) f C' w: y* d[root@admin ~]# ceph osd reweight 3 0.50 w, e$ s# X& z' j/ ^1 N; z' U
reweighted osd.3 to 0.5 (8327682)
6 K2 b4 _8 v8 a$ Y7 G% D[root@admin ~]# ceph osd tree
/ g! G3 I9 E( R0 E1 p' _& J6 m# id weight type name up/down reweight
# j& ~* e! ^2 J& k' C-1 4 root default- |5 \! G! H9 K* f" T
-2 1 host node17 X6 d/ j# ? o
0 1 osd.0 up 1
1 r+ Y! Y* c% d. B-3 1 host node21 x) X8 l) y' t2 H3 E% O
1 1 osd.1 up 1' V) r7 J- q0 |" G" V5 _) S9 T
-4 1 host node35 k% N# |: q! P3 ~
2 1 osd.2 up 10 ~* H* N$ {% u: m& s
-5 1 host node4$ w5 p1 R( {2 t$ P0 ~' b: b
3 1 osd.3 up 0.5& f: }5 t( G7 C/ W+ t
11、把一个osd节点逐出集群
9 j- A8 \4 x8 {; Y[root@admin ~]# ceph osd out osd.3/ f4 w7 n( Z+ e; Y1 q5 q
marked out osd.3. % u. y- ]1 K- f$ o- P# H
[root@admin ~]# ceph osd tree2 p2 B7 ?- K; W6 j7 h
# id weight type name up/down reweight
4 [* C% F, K! X, q-1 4 root default2 R6 e4 q& t; p: ]& D# Z$ g& h6 W
-2 1 host node1
' c: n# a2 A7 }' n4 [4 X0 1 osd.0 up 1) e5 Q3 ~3 U0 w# p, G$ V
-3 1 host node20 o9 ]# B4 m9 r
1 1 osd.1 up 1; \4 }0 k4 R! p; ^% Y! X, g
-4 1 host node3" [* s( s q# l8 ]) e: T# h2 m1 p
2 1 osd.2 up 1
3 J% w8 r3 k$ W-5 1 host node4
8 k4 a6 i/ r5 m3 1 osd.3 up 0 # osd.3的reweight变为0了就不再分配数据，但是设备还是存活的
3 {. q7 T* b; L4 d6 u12、把逐出的osd加入集群6 A5 Y8 C! W3 [. C0 @, p
[root@admin ~]# ceph osd in osd.3
2 u/ V4 s F1 Q" a9 s, G7 \7 u }marked in osd.3. % w: T4 p5 M# O$ K
[root@admin ~]# ceph osd tree
/ M2 m2 W0 s4 ^( @; R! ]# id weight type name up/down reweight! U7 G% I9 C9 O% j
-1 4 root default: T: A6 z5 Y0 x% [$ p
-2 1 host node1! f# ]! c/ O! J: d. e3 y
0 1 osd.0 up 1
# ^8 w! \; D2 k! G) ]$ ^-3 1 host node2, n7 M/ l- J6 v
1 1 osd.1 up 1$ o' C2 q+ Y/ C
-4 1 host node3+ h, u& z0 s2 T! k, ^( ~6 Q
2 1 osd.2 up 1
# z- W: V' M( R- l* d' l( s-5 1 host node4
* H3 W! |9 i8 N: d; K8 d# h3 1 osd.3 up 1
6 b, V7 q% u8 H. Z+ D" {( e2 `13、暂停osd （暂停后整个集群不再接收数据）5 |* v4 q* S/ g# I0 ?2 \5 y
[root@admin ~]# ceph osd pause! I1 ?; z" F- H) R
set pauserd,pausewr 8 s! k+ i( p- o9 J3 a
14、再次开启osd （开启后再次接收数据）
) ?! X* B+ f( V+ {8 t0 N9 H+ \5 A[root@admin ~]# ceph osd unpause
6 p1 c' N7 }1 v% j+ A7 yunset pauserd,pausewr
1 j! l5 c- n- c& c. G
- K7 _4 z2 V A$ t. C9 Y/ b: Z- a15、查看一个集群osd.2参数的配置
9 Q6 o( O0 b# g9 u( h$ ^5 k- Kceph --admin-daemon /var/run/ceph/ceph-osd.2.asok config show | less
0 d' }( y- T F; o, S. J+ S: m: X1 F8 ?: n2 r6 |, J* k
6 s3 {' ]) j- B8 u, Q) B/ D* `$ w/ U4 s
五、PG组
; ?! a8 I$ L$ c1、1、查看pg组的映射信息
: _7 r1 _5 w4 c8 B$ s1 H- @[root@client ~]# ceph pg dump4 m- y0 C4 b3 h% T, m1 o
dumped all in format plain
5 j4 _$ p9 X% w0 C" t1 G2 Gversion 1164' R" [. C4 r9 u9 f2 b# V7 X
stamp 2014-06-30 00:48:29.754714
" U1 w' w) r! Y7 C: T( J) l+ |" G. v# Y$ Dlast_osdmap_epoch 88: K( Z* Q( |3 ]3 z5 i7 b1 J$ w: M4 I
last_pg_scan 73
4 l7 w% T' k. T- D: h) {full_ratio 0.95* ^, S) G4 w* P# |. r. w2 I: \
nearfull_ratio 0.85
+ b$ L+ } Q4 i. Mpg_stat objects mip degr unf bytes log disklog state state_stamp v reported up up_primary acting acting_primary last_scrub scrub_stamp last_deep_scrudeep_scrub_stamp
( e C: p- A% G, l- ^# Y0.3f 39 0 0 0 163577856 128 128 active+clean 2014-06-30 00:30:59.193479 52'128 88:242 [0,2] 0 [0,2] 0 44'25 2014-06-29 22:25:25.282347 0'0 2014-06-26 19:52:08.521434
3 d6 D" o. w2 D9 x- D* E, m3.3c 0 0 0 0 0 0 0 active+clean 2014-06-30 00:15:38.675465 0'0 88:21 [2,1] 2 [2,1] 2 0'0 2014-06-30 00:15:04.295637 0'0 2014-06-30 00:15:04.295637
/ M% h. v* @$ ` s6 c2.3c 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.583702 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:29:13.701625 0'0 2014-06-26 19:52:08.845944( c' v% y q7 l* c
1.3f 2 0 0 0 452 2 2 active+clean 2014-06-30 00:10:48.596050 16'2 88:66 [2,1] 2 [2,1] 2 16'2 2014-06-29 22:28:03.570074 0'0 2014-06-26 19:52:08.655292
/ N2 }8 W3 k: S! i+ _( D0.3e 31 0 0 0 130023424 130 130 active+clean 2014-06-30 00:26:22.803186 52'130 88:304 [2,0] 2 [2,0] 2 44'59 2014-06-29 22:26:41.317403 0'0 2014-06-26 19:52:08.518978" t- d, _3 K8 q& j2 P+ G) \& p0 U
3.3d 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:57.548803 0'0 88:20 [0,2] 0 [0,2] 0 0'0 2014-06-30 00:15:19.101314 0'0 2014-06-30 00:15:19.101314 q+ ?" `4 B) y' I& X7 _
2.3f 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.750476 0'0 88:106 [0,2] 0 [0,2] 0 0'0 2014-06-29 22:27:44.604084 0'0 2014-06-26 19:52:08.864240
4 f) i# T4 B4 [, j( a; ~ M" g1.3c 1 0 0 0 0 1 1 active+clean 2014-06-30 00:10:48.939358 16'1 88:66 [1,2] 1 [1,2] 1 16'1 2014-06-29 22:27:35.991845 0'0 2014-06-26 19:52:08.6464704 R, p4 `' C: Y7 D$ T9 ^
0.3d 34 0 0 0 142606336 149 149 active+clean 2014-06-30 00:23:57.348657 52'149 88:300 [0,2] 0 [0,2] 0 44'57 2014-06-29 22:25:24.279912 0'0 2014-06-26 19:52:08.514526& ?9 }, m& ^9 m# R7 \1 y% B$ }
3.3e 0 0 0 0 0 0 0 active+clean 2014-06-30 00:15:39.554742 0'0 88:21 [2,1] 2 [2,1] 2 0'0 2014-06-30 00:15:04.296812 0'0 2014-06-30 00:15:04.2968129 i7 J. c& X& Z
2.3e 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.592171 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:29:14.702209 0'0 2014-06-26 19:52:08.855382
3 V! h6 Q. k) h2 W2 P/ |1.3d 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.938971 0'0 88:58 [1,2] 1 [1,2] 1 0'0 2014-06-29 22:27:36.971820 0'0 2014-06-26 19:52:08.650070' Q: ~! i) v: d9 |3 J3 T' \# `1 ~
0.3c 41 0 0 0 171966464 157 157 active+clean 2014-06-30 00:24:55.751252 52'157 88:385 [1,0] 1 [1,0] 1 44'41 2014-06-29 22:26:34.829858 0'0 2014-06-26 19:52:08.513798
2 j0 I5 q* w: b! y. B0 G4 k3.3f 0 0 0 0 0 0 0 active+clean 2014-06-30 00:17:08.416756 0'0 88:20 [0,1] 0 [0,1] 0 0'0 2014-06-30 00:15:19.406120 0'0 2014-06-30 00:15:19.406120
' E1 j, T3 W6 ^! q9 K5 [* f+ q. b2 O2.39 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.784789 0'0 88:71 [2,0] 2 [2,0] 2 0'0 2014-06-29 22:29:10.673549 0'0 2014-06-26 19:52:08.834644! m' ]7 o2 \" n% W; u
1.3a 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.738782 0'0 88:106 [0,2] 0 [0,2] 0 0'0 2014-06-29 22:26:29.457318 0'0 2014-06-26 19:52:08.642018# e3 W9 K2 V$ ]$ c( _ Y
0.3b 37 0 0 0 155189248 137 137 active+clean 2014-06-30 00:28:45.021993 52'137 88:278 [0,2] 0 [0,2] 0 44'40 2014-06-29 22:25:22.275783 0'0 2014-06-26 19:52:08.510502& m" U1 i- f+ a& V* P
3.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:13.222339 0'0 88:21 [1,0] 1 [1,0] 1 0'0 2014-06-30 00:15:05.446639 0'0 2014-06-30 00:15:05.446639 F* x4 A2 W* v7 _8 Y J9 d2 u
2.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.783103 0'0 88:71 [2,0] 2 [2,0] 2 0'0 2014-06-29 22:29:06.688363 0'0 2014-06-26 19:52:08.8273427 O* R) W: C) L7 E; A k, K, V
1.3b 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:58.857283 0'0 88:78 [1,0] 1 [1,0] 1 0'0 2014-06-29 22:27:30.017050 0'0 2014-06-26 19:52:08.6448205 H% n4 w0 r3 A: ?3 Y
0.3a 40 0 0 0 167772160 149 149 active+clean 2014-06-30 00:28:47.002342 52'149 88:288 [0,2] 0 [0,2] 0 44'46 2014-06-29 22:25:21.273679 0'0 2014-06-26 19:52:08.508654
( J9 v* [0 i. d3.39 0 0 0 0 0 0 0 active+clean 2014-06-30 00:16:13.255056 0'0 88:21 [1,0] 1 [1,0] 1 0'0 2014-06-30 00:15:05.447461 0'0 2014-06-30 00:15:05.447461
/ O* q; [+ U/ e1 I, b2.3b 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.935872 0'0 88:57 [1,2] 1 [1,2] 1 0'0 2014-06-29 22:28:35.095977 0'0 2014-06-26 19:52:08.844571
* F: p- {' _9 E0 a1.38 0 0 0 0 0 0 0 active+clean 2014-06-30 00:10:48.597540 0'0 88:46 [2,1] 2 [2,1] 2 0'0 2014-06-29 22:28:01.519137 0'0 2014-06-26 19:52:08.633781
, K+ g. }& Z. h# q ?) W3 ]0.39 48 0 0 0 201326592 164 164 active+clean 2014-06-30 00:25:30.757843 52'164 88:432 [1,0] 1 [1,0] 1 44'32 2014-06-29 22:26:33.823947 0'0 2014-06-26 19:52:08.504628' y& o, J8 s( I9 o8 S: y! w
下面部分省略
& O3 D' C0 `) i7 D6 q
" F- C) b+ j7 L5 }- j2、查看一个PG的map
2 v4 I$ c# T+ ~4 h6 C[root@client ~]# ceph pg map 0.3f1 T0 M# L' Y- |5 P) O# V, p. G2 K
osdmap e88 pg 0.3f (0.3f) -> up [0,2] acting [0,2] #其中的[0,2]代表存储在osd.0、osd.2节点，osd.0代表主副本的存储位置
" M* f8 e- p. ~, e3、查看PG状态5 ]3 g' x& S% z) {- [" D
[root@client ~]# ceph pg stat# e& S1 M: g4 n7 o6 y
v1164: 448 pgs: 448 active+clean; 10003 MB data, 23617 MB used, 37792 MB / 61410 MB avail
* }" c1 h+ g4 J8 a4、查询一个pg的详细信息
1 t8 B" ~: X9 K% f$ \$ R[root@client ~]# ceph pg 0.26 query$ w% J3 q8 s: M2 _: y" \
5、查看pg中stuck的状态
+ @* p+ [& ]" u4 ^0 a: l[root@client ~]# ceph pg dump_stuck unclean
+ V- ~2 x/ D# D) }ok+ q+ k# Y3 |+ T; ]2 d5 ~; x
[root@client ~]# ceph pg dump_stuck inactive3 B3 C i7 v( |+ t% I
ok' {( S8 Z+ x3 E2 V! H m% z9 \
[root@client ~]# ceph pg dump_stuck stale" u/ b$ N w# P- N
ok& u/ r1 @. ~2 q
6、显示一个集群中的所有的pg统计
7 ]. ~3 h$ r; k0 E( j4 wceph pg dump --format plain- d6 C; F6 G4 W+ Z
7、恢复一个丢失的pg, Y# G8 L/ X' k/ A
ceph pg {pg-id} mark_unfound_lost revert
7 {* P% A) |' t: ^4 Y8、显示非正常状态的pg
4 |! b7 ^" v1 d# \ceph pg dump_stuck inactive|unclean|stale
& J: d- h# I. r2 _8 A六、pool! V3 t' }1 ], F* S5 X$ ^9 y
1、查看ceph集群中的pool数量0 [% z0 A3 ~+ o" F
[root@admin ~]# ceph osd lspools% n2 _- }: P+ J# V5 Y" c3 v
0 data,1 metadata,2 rbd,+ g2 M# F( ]3 K& V2 A5 j/ J+ p
2、在ceph集群中创建一个pool
4 I: n- W( j* `8 \& t! _+ N$ A2 D+ |! Dceph osd pool create jiayuan 100 #这里的100指的是PG组
0 y3 I0 E7 U0 ?3、为一个ceph pool配置配额
- g+ U3 b& H% w- i5 Y* G: `9 \ceph osd pool set-quota data max_objects 10000/ f# n% R5 b! M2 F f" O
4、在集群中删除一个pool+ L& c3 v5 c0 ^. H8 }* Y8 b u' {
ceph osd pool delete jiayuan jiayuan --yes-i-really-really-mean-it #集群名字需要重复两次
7 n7 a) f4 W2 O, l& }5 B5、显示集群中pool的详细信息
: r' L! Q# O# }. E6 N6 J: U9 [$ }[root@admin ~]# rados df4 M- s; x# A0 N# l4 m. [) k5 ?) a
pool name category KB objects clones degraded unfound rd rd KB wr wr KB |0 ^1 a/ t0 i! _( O" V4 O- ]
data - 475764704 116155 0 0 0 0 0 116379 475764704$ G1 e- h0 z* c
metadata - 5606 21 0 0 0 0 0 314 5833
% k4 J! B( V- Z/ F) O rrbd - 0 0 0 0 0 0 0 0 09 T- ^5 Z1 C3 K" Y% p" ?
total used 955852448 116176
- c$ Q2 W: C) \4 k- B total avail 639497596
: K0 I8 Q9 x" I4 e0 Q. {' `! Q total space 1595350044
0 V! y! i& d. a6 h+ u+ h[root@admin ~]# 6 {4 \" z( J' k3 i- Z. y; h
6、给一个pool创建一个快照/ J) j6 J" o1 r1 c! _
[root@admin ~]# ceph osd pool mksnap data date-snap
0 s' z9 N. k* k$ ^2 O/ k8 Vcreated pool data snap date-snap
8 z5 \6 N+ j) ], h9 S* E' t0 m7、删除pool的快照( g+ [' J8 E% w7 s% H
[root@admin ~]# ceph osd pool rmsnap data date-snap" k0 L* H# i! ?9 M# H3 e3 H4 F$ x- v
removed pool data snap date-snap
' n @, E( X' X& h/ Z( h8、查看data池的pg数量
9 C7 G% A0 ^1 q[root@admin ~]# ceph osd pool get data pg_num* d, w" H4 b h+ @
pg_num: 64
& I `: {5 p1 N9、设置data池的最大存储空间为100T（默认是1T)1 r2 Q6 V; f" {: |) E& l. x1 W
[root@admin ~]# ceph osd pool set data target_max_bytes 100000000000000
8 ^0 R5 S6 y* t/ h; {set pool 0 target_max_bytes to 100000000000000# T$ s6 e0 l& `1 L. {; |! z9 L
10、设置data池的副本数是3
( C k- r9 c, k; r, c0 e3 M[root@admin ~]# ceph osd pool set data size 3
8 A0 j9 ?4 V* W8 Q) _7 z; H3 Yset pool 0 size to 3
7 v p+ ?, M0 m5 V11、设置data池能接受写操作的最小副本为29 S3 ^. ?8 H! }) o3 O: t
[root@admin ~]# ceph osd pool set data min_size 2
2 K) l. x0 N9 V! O$ h# pset pool 0 min_size to 2- T9 }% t' V, Z5 o" z0 L' M
12、查看集群中所有pool的副本尺寸
5 f$ _5 u& J- g5 N" k) w8 C# B[root@admin mycephfs]# ceph osd dump | grep 'replicated size': S8 L' e) {& d) F- w
pool 0 'data' replicated size 3 min_size 2 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 26 owner 0 flags hashpspool crash_replay_interval 45 target_bytes 100000000000000 stripe_width 0
- H3 z8 _0 L" N7 a" E) Ppool 1 'metadata' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0
6 c7 d' n) o/ [pool 2 'rbd' replicated size 2 min_size 1 crush_ruleset 0 object_hash rjenkins pg_num 64 pgp_num 64 last_change 1 owner 0 flags hashpspool stripe_width 0
) [1 U. X; ^. L4 T13、设置一个pool的pg数量4 K$ s) `# F5 l7 X" O: |6 ?
[root@admin ~]# ceph osd pool set data pg_num 100
& E J' m" g( O( f0 {4 Eset pool 0 pg_num to 100
+ O# X, w7 t y2 K" C14、设置一个pool的pgp数量
. i6 U1 I$ t3 Z4 x[root@admin ~]# ceph osd pool set data pgp_num 100
) e1 W. w) q& I0 ]" b" Cset pool 0 pgp_num to 100- }( W5 D* U4 X( J4 W
; O$ [ q9 @& E% W七、rados和rbd指令' K) R9 S% Z7 y0 {
1、rados命令使用方法/ U! l3 f" t$ q, c
（1）、查看ceph集群中有多少个pool （只是查看pool)
# @% {4 p9 ~# V7 E' r2 _[root@node-44 ~]# rados lspools
2 Q, c( a( x g. ?% [/ h: Edata
. ^3 Y6 ?: W7 C9 D: ?metadata. j% h5 H6 x1 \ F" p% V% _
rbd4 j2 F! Q& O5 v4 p% ^8 `) `7 o
images) E0 Z' ?4 @- A$ A7 {
volumes9 H2 s( d9 a/ a" S4 l
.rgw.root
0 o6 t8 a [" H$ x9 @compute
# l3 {* X1 G4 `4 u0 [1 Z M/ y.rgw.control
. E2 S4 [, u$ K# F1 p% I.rgw" C; @/ Z8 w5 o5 d- T* _+ Q
.rgw.gc
8 _5 t" Q9 Y- d N.users.uid, D6 W7 M8 } J$ N; M b
（2）、查看ceph集群中有多少个pool,并且每个pool容量及利用情况
& t2 v2 J! j1 S# X$ o e[root@node-44 ~]# rados df
' ^9 q+ }) q$ z% k/ Ipool name category KB objects clones degraded unfound rd rd KB wr wr KB) G' O/ ?0 Z( I
.rgw - 0 0 0 0 0 0 0 0 05 T) X+ m. {( ?& _
.rgw.control - 0 8 0 0 0 0 0 0 0
9 _% F9 o) K a! q! W! A: q! l.rgw.gc - 0 32 0 0 0 57172 57172 38136 0- k, S4 Q2 p6 G, t |- Q
.rgw.root - 1 4 0 0 0 75 46 10 10
( f, U* V( F9 e: C7 i4 p! S9 H) m.users.uid - 1 1 0 0 0 0 0 2 1" Y/ k5 k8 r) E( u
compute - 67430708 16506 0 0 0 398128 75927848 1174683 222082706
8 C( ~+ R; |" N+ v" E' jdata - 0 0 0 0 0 0 0 0 00 Q% e b. R0 G& b& a5 g0 O
images - 250069744 30683 0 0 0 50881 195328724 65025 388375482% [# c1 S3 t# |! e/ r. s
metadata - 0 0 0 0 0 0 0 0 0) \6 w. n! G" H4 x, ]1 {2 d
rbd - 0 0 0 0 0 0 0 0 0
) U& J7 z- p6 }( O( U8 k% ^$ Svolumes - 79123929 19707 0 0 0 2575693 63437000 1592456 163812172
* }7 {$ U2 E5 C1 k9 ?( N total used 799318844 66941
5 ?0 _: a/ G8 ~# v( v total avail 11306053720
Q4 v, u% u6 C+ |3 J+ g9 i total space 12105372564
+ x( D. z2 J1 s4 y[root@node-44 ~]#
2 E+ A2 u5 n3 ?( x4 r（3）、创建一个pool
. t1 R7 L% x8 \8 C, s1 l[root@node-44 ~]#rados mkpool test x$ `) _6 C* I
(4)、查看ceph pool中的ceph object （这里的object是以块形式存储的）" \( S2 J x( \9 Q
[root@node-44 ~]# rados ls -p volumes | more
. S% r6 s8 E3 u- t8 jrbd_data.348f21ba7021.0000000000000866
4 t4 L1 s; V t/ frbd_data.32562ae8944a.0000000000000c79) ^1 R( }+ K, {9 M4 C& ^$ l" U
rbd_data.589c2ae8944a.00000000000031ba* r& P% X0 W8 X8 {0 |6 @% \1 W7 v0 S
rbd_data.58c9151ff76b.00000000000029af
0 N5 A- g, y+ Z% grbd_data.58c9151ff76b.0000000000002c194 v, M2 }* L. U1 b$ \4 b' }
rbd_data.58c9151ff76b.0000000000000a5a7 M& I9 O3 B4 c( O
rbd_data.58c9151ff76b.0000000000001c69
7 K9 k \/ _. h j* t' Rrbd_data.58c9151ff76b.000000000000281d* u& c, \0 s- a: u2 s4 Z& A
rbd_data.58c9151ff76b.0000000000002de1
8 ~+ B/ d, {4 Prbd_data.58c9151ff76b.0000000000002dae
; E0 S' ~4 u, e/ t- F（5）、创建一个对象object
3 w! f6 k$ h) s[root@admin-node ~]# rados create test-object -p test
7 W; s* {1 f# V5 \$ @9 P1 t/ `4 ~2 M1 H- b: g
[root@admin-node ~]# rados -p test ls
) @5 C0 y1 F. {' T! `( rtest-object
4 E9 L( T7 D5 ~, f c（6）、删除一个对象8 s# a1 E. @, l
[root@admin-node ~]# rados rm test-object-1 -p test6 a$ m6 l- v6 R X
8 h9 t9 p, z! j* h2 x+ }0 r: l: o
* S' {+ h7 o* I: X9 G) [2、rbd命令的用法 + @2 ]/ V. d6 h5 {: L
F4 |' f( g9 r3 K
（1）、查看ceph中一个pool里的所有镜像) a% D M- e8 ~
[root@node-44 ~]# rbd ls images
; ` z7 N6 R/ ?, t2014-05-24 17:17:37.043659 7f14caa6e700 0 -- :/1025604 >> 10.49.101.9:6789/0 pipe(0x6c5400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x6c5660).fault% m4 p, Y) e3 m- R8 s
2182d9ac-52f4-4f5d-99a1-ab3ceacbf0b9- @+ |6 {4 q" v& @( `
34e1a475-5b11-410c-b4c4-69b5f780f03c
( y: A% k1 E6 I' L! \/ t" V# G476a9f3b-4608-4ffd-90ea-8750e804f46e- }& `& [( t G H: \5 @
60eae8bf-dd23-40c5-ba02-266d5b942767
5 M8 g1 e, s, r% g5 F72e16e93-1fa5-4e11-8497-15bd904eeffe
2 K" E+ o% x+ f& S/ f4 u7 e# x74cb427c-cee9-47d0-b467-af217a67e60a
, H; R+ {; C5 \" H9 K8f181a53-520b-4e22-af7c-de59e8ccca78
5 P7 ]! s: c$ ?' Z: a% d0 ]$ i* t9867a580-22fe-4ed0-a1a8-120b8e8d18f4; U2 W+ R8 }# X G b4 X2 d) X
ac6f4dae-4b81-476d-9e83-ad92ff25fb13
^( {; g# @7 A: h# E% \" Ed20206d7-ff31-4dce-b59a-a622b0ea3af6
]9 T L" d8 y# P) R- p
6 X4 i6 _, f0 c# w$ I6 v! F2 e[root@node-44 ~]# rbd ls volumes' J2 F* g6 k1 R" r9 g
2014-05-24 17:22:18.649929 7f9e98733700 0 -- :/1010725 >> 10.49.101.9:6789/0 pipe(0x96a400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x96a660).fault
v: X! g, \, k( \4 lvolume-0788fc6c-0dd4-4339-bad4-e9d78bd5365c# L2 Q% R' ~# I$ d8 U9 w9 a2 x2 ^
volume-0898c5b4-4072-4cae-affc-ec59c2375c51
2 ~9 J: W4 k1 Y+ {, Mvolume-2a1fb287-5666-4095-8f0b-6481695824e2' j8 T4 j, Q& b5 |
volume-35c6aad4-8ea4-4b8d-95c7-7c3a8e8758c5. t2 Y: E! K$ _1 v
volume-814494cc-5ae6-4094-9d06-d844fdf485c4
. j6 X' d8 V2 Ivolume-8a6fb0db-35a9-4b3b-9ace-fb647c2918ea( e5 ? V0 p# {% a. k
volume-8c108991-9b03-4308-b979-51378bba2ed1
# _ z8 _* y: w4 B* h. j8 A" ?" P' rvolume-8cf3d206-2cce-4579-91c5-77bcb4a8a3f8
) ]) W) |: L" Y# F/ O" I$ I- w# Fvolume-91fc075c-8bd1-41dc-b5ef-844f23df177d
% a8 z. c; d6 g& P7 F* Y+ |7 _volume-b1263d8b-0a12-4b51-84e5-74434c0e73aa, u/ ^! x+ o. ?( l% a
volume-b84fad5d-16ee-4343-8630-88f265409feb: D. T& z2 F! `% t i0 y O
volume-c03a2eb1-06a3-4d79-98e5-7c62210751c3
: q; L' P& U( K) ^' ?9 K* Ivolume-c17bf6c0-80ba-47d9-862d-1b9e9a48231e
! G0 O2 Z. K0 T& uvolume-c32bca55-7ec0-47ce-a87e-a883da4b4ccd6 a1 I* o' n" p% ^
volume-df8961ef-11d6-4dae-96ee-f2df8eb4a08c! C# [2 {$ v2 l2 b* p
volume-f1c38695-81f8-44fd-9af0-458cddf103a3
9 [+ A6 t) T, `" U4 n9 O
% J* r- `8 y+ x4 b! O（2）、查看ceph pool中一个镜像的信息8 q* D% Q9 k7 U$ e8 t0 |
[root@node-44 ~]# rbd info -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a
- \ S# `5 ^ Y! Krbd image '74cb427c-cee9-47d0-b467-af217a67e60a':1 G$ _; G8 I9 U7 j$ ?
size 1048 MB in 131 objects
! s: v3 D; Y. f order 23 (8192 KB objects)
f' D. I- _9 t/ A8 |: `/ X. V0 p block_name_prefix: rbd_data.95c7783fc0d0
1 _8 c$ i& S0 G% ]2 } format: 2! u" k8 h0 V( l1 e- r1 v" t- s
features: layering2 w( l3 p' b1 G
（3）、在test池中创建一个命名为zhanguo的10000M的镜像) E& r D# w+ v1 c9 Y9 \- ~( b5 Q
[root@node-44 ~]# rbd create -p test --size 10000 zhanguo
8 z& d h; `7 j8 m[root@node-44 ~]# rbd -p test info zhanguo #查看新建的镜像的信息5 X. d+ f3 }1 M6 }0 B3 U
rbd image 'zhanguo':
" \" X5 Z6 F, Q- v4 Z5 T# p size 10000 MB in 2500 objects
# y% o" _4 g" U* x* w( L order 22 (4096 KB objects)5 i/ F, n3 R2 J6 [! C" S4 C
block_name_prefix: rb.0.127d2.2ae8944a' p* a# u. b* u4 S o
format: 1
2 R) B) E( c9 [1 }- q" l7 H( J[root@node-44 ~]# ( F" M0 x, X8 O; H8 w# Y' E
（4）、删除一个镜像
9 Q9 L& O U }, Z; }/ ?[root@node-44 ~]# rbd rm -p test lizhanguo
/ D6 p" ]2 p, h$ ~" bRemoving image: 100% complete...done.
8 Y' j. e$ T! O: {（5）、调整一个镜像的尺寸3 [2 d6 K# j' x. g; x
[root@node-44 ~]# rbd resize -p test --size 20000 zhanguo7 P f/ M( ]- S. B- }2 m, E
Resizing image: 100% complete...done., \# X' `0 l- x' x5 L# L
[root@node-44 ~]# rbd -p test info zhanguo #调整后的镜像大小
5 D; i1 `5 @/ b3 I$ {3 Vrbd image 'zhanguo':
# m3 u* l- y T; O2 B, z! E size 20000 MB in 5000 objects, w0 K4 e. t2 N: _
order 22 (4096 KB objects)
6 w' I$ Y; y4 o+ W block_name_prefix: rb.0.127d2.2ae8944a
0 h6 A# ^+ w Y" Q" }+ i% x4 z2 ~) q format: 1' b2 w7 G0 X) `" N* ]
[root@node-44 ~]# * j. t: U$ O4 H) @2 B5 G
（6）、给一个镜像创建一个快照5 O& w! u% U3 y9 V1 U
[root@node-44 ~]# rbd snap create test/zhanguo@zhanguo123 #池/镜像@快照
3 P) s/ a3 Z" v6 t, y[root@node-44 ~]# rbd snap ls -p test zhanguo- k$ M1 z0 `; W
SNAPID NAME SIZE " |, j5 K9 ]2 t! I, S3 ?& t& x
2 zhanguo123 20000 MB
& u" ?2 R$ W# }! ?& n& ][root@node-44 ~]#
- H1 O$ S, f: q# l[root@node-44 ~]# rbd info test/zhanguo@zhanguo123. @7 h" e7 y( G% i
rbd image 'zhanguo':
1 x% w+ A9 i- ^6 B5 x size 20000 MB in 5000 objects
( J) h2 X0 \7 `! p- D order 22 (4096 KB objects)% u$ ^1 k! n8 n7 b1 C
block_name_prefix: rb.0.127d2.2ae8944a
4 Z* i+ J/ `5 }7 W+ c% ~# c format: 1
- I6 _% S; r' U; I4 r protected: False
7 E! }/ @ j) `1 w0 l[root@node-44 ~]#
! Y& m* \; |$ D% I2 X* t（7）、查看一个镜像文件的快照
$ a8 ?" k5 ]1 \$ _- R# G[root@os-node101 ~]# rbd snap ls -p volumes volume-7687988d-16ef-4814-8a2c-3fbd85e928e4
3 k: T e& L' ~. a7 q& \9 i) ^/ q) d bSNAPID NAME SIZE
. a* K) U4 ?6 ?8 a" x 5 snapshot-ee7862aa-825e-4004-9587-879d60430a12 102400 MB & G. Y% X# w. d9 X4 {4 O% m& j% n4 j
（8）、删除一个镜像文件的一个快照快照
6 f1 q% }1 M0 x) N6 q 快照所在的池/ 快照所在的镜像文件 @ 快照# a5 Y i; b1 K1 g$ }* d
[root@os-node101 ~]# rbd snap rm volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a128 ~, u+ `$ F5 L6 N& k
rbd: snapshot 'snapshot-60586eba-b0be-4885-81ab-010757e50efb' is protected from removal.. E! \: F; a3 Z" S
2014-08-18 19:23:42.099301 7fd0245ef760 -1 librbd: removing snapshot from header failed: (16) Device or resource busy
; z# q$ x1 b+ g" c上面不能删除显示的报错信息是此快照备写保护了，下面命令是删除写保护后再进行删除。
5 r# t2 w) L) h[root@os-node101 ~]# rbd snap unprotect volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a12
/ @' p8 a4 d! {, L2 P0 Q4 N[root@os-node101 ~]# rbd snap rm volumes/volume-7687988d-16ef-4814-8a2c-3fbd85e928e4@snapshot-ee7862aa-825e-4004-9587-879d60430a12
# \4 \ k5 F1 M+ u& }, e（9）删除一个镜像文件的所有快照2 ]) U+ t2 U e3 t/ ~) P
[root@os-node101 ~]# rbd snap purge -p volumes volume-7687988d-16ef-4814-8a2c-3fbd85e928e42 F" f- }' A- T
Removing all snapshots: 100% complete...done.
0 W6 M' D! O- T. J. G+ |* Z! m6 u" {# _, R0 _5 D
9 T8 h$ n$ I. Q Y, E
（10）、把ceph pool中的一个镜像导出8 m8 L6 @* E, u6 z5 A
导出镜像& A: `0 e3 k5 v5 Q1 e$ ]
[root@node-44 ~]# rbd export -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a /root/aaa.img& ~8 u* g: j% u5 U" Z* |; @
2014-05-24 17:16:15.197695 7ffb47a9a700 0 -- :/1020493 >> 10.49.101.9:6789/0 pipe(0x1368400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x1368660).fault% n6 @3 b2 S/ V) D) O. S
Exporting image: 100% complete...done.
3 A/ ~+ |( O# T- N4 ^' h
. p I- x; O$ j4 P8 b4 M导出云硬盘4 M- N5 w/ U" T: o) C
[root@node-44 ~]# rbd export -p volumes --image volume-470fee37-b950-4eef-a595-d7def334a5d6 /var/lib/glance/ceph-pool/volumes/Message-JiaoBenJi-10.40.212.24" h {" B7 b1 N
2014-05-24 17:28:18.940402 7f14ad39f700 0 -- :/1032237 >> 10.49.101.9:6789/0 pipe(0x260a400 sd=3 :0 s=1 pgs=0 cs=0 l=1 c=0x260a660).fault4 t) r0 y8 E5 {, `% W
Exporting image: 100% complete...done.
6 A$ f7 c) H, ~6 [) N% ]2 v+ M) c$ A
（11）、把一个镜像导入ceph中 （但是直接导入是不能用的，因为没有经过openstack,openstack是看不到的）: f t% R" G* O/ d
[root@node-44 ~]# rbd import /root/aaa.img -p images --image 74cb427c-cee9-47d0-b467-af217a67e60a 6 z- W9 K! y* R. F( J: i: u1 ~
Importing image: 100% complete...done.
* A& l1 n$ L" G+ S1 @7 `# F3 U* y/ u% R! B0 X/ T6 Z; s& L
|
|
发表于 2018-10-6 16:15:42