|
|
#openstack pike与ceph集成
9 C: `& I1 h3 ?9 E###########################+ G/ `& A9 b/ a# G* P: \4 J2 a
#openstack节点
9 \: u$ o( R+ D5 V/ s# _( Q#openstack节点配置ceph源
$ X8 j7 \! ~+ r- Y, y! g0 G#使用阿里源 #rm -f /etc/yum.repos.d/*.repo
9 R6 U) I1 l* |$ A3 X" B0 w0 Rwget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
1 [0 X6 _9 Z4 j# L2 o4 Fwget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
3 o* G+ f* r& a* \: G: i' b% Ised -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址3 d3 I- D. A# w5 C' |+ `8 @% p. H/ }
wget -O /etc/yum.repos.d/ceph-luminous-aliyun.repo http://elven.vip/ks/yum/ceph-luminous-aliyun.repo
' z. l8 j# j1 e0 F cyum clean all && yum makecache #生成缓存$ B6 ?7 Y8 }( b( v, s0 e, t
#ceph客户端安装
& ? s @7 `$ i+ M- oyum -y install ceph-common) P% @$ o0 b$ i$ A5 W' n. S
c2 i4 Q2 ^, }###########################/ ? M q. k+ l0 H
#ceph管理节点
2 H9 Z$ ]3 e. `3 w" y#创建POOL
s. W% M) y7 k1 v oceph osd pool create volumes 128 8 w7 `% q1 g( B6 s4 ^8 |
ceph osd pool create images 128 ' q" V* j: ?8 P$ z" R
ceph osd pool create vms 128 I. @3 V4 ~- Q# Z
#ssh免密验证
0 T5 M' s$ ? D9 C E8 D5 ~curl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh( c) m: O, L. ` F
#认证用户及密码#4 F+ t" Y9 I" R! O$ r
echo ", K% o( V) Q" {! n+ [2 U
USER=root* Z$ ~. G9 b. v) H0 w! x' y: t! \- c
PASS=123321' H5 d+ ?' q$ P9 `6 w( } b
">my.sh.conf) u9 f" @4 C, e) `' D
#hosts设置
" R5 H, e" A- L2 x) c( A: t/ @echo "3 ^3 T: v! Q. O% E: z3 F C$ t, |
#openstack. U5 Z2 h2 M- Q, X
192.168.58.17 controller
( ^4 N) ~4 [+ y0 a+ G E4 F192.168.58.16 compute01
4 R' p6 J0 X" y" O( u- b192.168.58.14 storage1" t0 h& d6 i# R6 u) v
">>/etc/hosts
- H: Z* \8 B% D. x5 v; t8 ?) Q: ~# Y#ssh批量认证#
/ N/ s5 A" N0 K) A& D; k$ k0 X- Z0 dsh ./sshkey.me.sh controller compute01 storage1
$ |6 d% O( n! P' K* N- q2 m b#推送ceph配置到client* H* x1 K! v, T8 ^3 d. a
cd /etc/ceph/
1 l8 `' L. \; n( O/ fceph-deploy config push controller compute01 storage1# F% I5 y& h8 V
###########################4 J, Y1 J& I7 z5 Q c% s, ?
#创建ceph用户和密钥
; ~7 F6 F) @5 Cceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' 1 J) U: W4 ~! X0 K7 U
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
+ t& G3 e: ^# I p* P- B& ]ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
/ }/ C9 @1 O$ |6 p+ {#查询用户，写入文件
7 s+ ?* i! E; W4 J& [9 ^/ s1 ]0 @" Iceph auth get-key client.cinder >/etc/ceph/ceph.client.cinder.keyring
( c6 f& @5 k" Oceph auth get-or-create client.glance >/etc/ceph/ceph.client.glance.keyring
, s; `3 z' b3 u9 q+ z# scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/7 w* Y8 z7 s' ~ w, q5 r' F3 t3 V
# scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/. b7 E$ L6 r7 c5 X# \# c" Z* M- L
########################### H" i9 n3 G, U0 |& O1 G
#拷贝秘钥到对应节点,修改权限
8 l4 h* |/ `& j' K) W; q9 \#(nova,cinder都使用client.cinder)
8 D" g& m( b3 O0 j s" I" k! S9 E#glance9 \1 |' E& @$ W; s
Node=controller
& L( o" o! z9 `! A4 H- f) Lscp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/% b7 w' w; y8 S) Z* ^
ssh $Node sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
' G& A* ]9 a' X8 F L#nova compute
% O4 l* L5 Z k; @8 zNode=compute01! G2 Q" T6 a) I
scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/3 ]/ d5 u b( f5 H5 r1 E, ^( V' N
ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.cinder.keyring
- S+ ^3 K3 [: Sscp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/
" B- }' j0 p" a* E hssh $Node sudo chown nova:nova /etc/ceph/ceph.client.glance.keyring. [! Y( N' c9 Q- Y* A
#cinder storage
; {8 O* {! n* v: o2 J6 jNode=storage1
7 p$ k. x' P! J8 d+ y% {8 J( jscp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/% O4 J8 m/ z$ J( f; T S" k
ssh $Node sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
; {/ J+ t4 U) K' ~- H8 ?###########################6 |" Q6 R# j5 G) P& t/ f) y" V
#openstack glance配置
i- m8 p' `# ols -l /etc/ceph/
& B. D3 L2 R& E#更改glance默认存储为ceph
% P0 e$ u8 U/ Y" T: Scp -f /etc/glance/glance-api.conf{,bak2}
# D" N- [& n! k) Psed -i 's/^stores/#&/' /etc/glance/glance-api.conf , n# V6 w- p* N! B% n7 y
sed -i 's/^default_store/#&/' /etc/glance/glance-api.conf 9 O" |' d5 X8 X. I
echo '#[glance_store]- W7 M3 Z" D9 |4 T o
stores = rbd,file
8 r+ X" F1 f; Tdefault_store = rbd
/ Y7 I- E/ Z. k; Z+ }rbd_store_pool = images3 K8 x8 v0 ~; S/ E( \
rbd_store_user = glance" x9 t6 G6 N9 l2 K3 X
rbd_store_ceph_conf = /etc/ceph/ceph.conf
+ R; Y+ t% ?1 Q2 Wrbd_store_chunk_size = 8
- u0 U5 @/ T, \2 u'>>/etc/glance/glance-api.conf
0 h2 _1 a8 }, w5 ~ o. p Q9 N#重启服务4 L9 l, V* j a& b$ R( t
systemctl restart openstack-glance-api openstack-glance-registry
& l( h1 K3 r. W9 Z+ {7 y, A9 }8 d###########################
! ^. [) H9 O8 j0 k: @( \- H& C2 J#nova计算节点+ ]( N' i2 e) f. d
ls -l /etc/ceph/
% y& |3 z* W+ f#ceph5 V* f. K7 t! D. D
echo '
+ o- F. A7 O/ g* C6 X8 i[client]
% R4 |% D; G+ Z0 J% _rbd cache = true 0 y" a5 F( v8 K
rbd cache writethrough until flush = true
% I6 E" d& Y. [; Zadmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok : g1 y/ U! M& ^1 T% I
log file = /var/log/qemu/qemu-guest-$pid.log 9 }! ]1 ~6 V( Z1 h* A
rbd concurrent management ops = 20 Z; N6 f0 i5 u
[client.cinder]
: N/ S+ t4 i7 ~8 pkeyring = /etc/ceph/ceph.client.cinder.keyring0 P0 y D7 _- H8 ~
'>>/etc/ceph/ceph.conf
5 W3 |+ \2 g( Xmkdir -p /var/run/ceph/guests/ /var/log/qemu/ . ]: w8 o4 N- y. ?
chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/ * S, P5 O4 l0 ~+ n
#密钥加进libvirt
/ R8 | j$ `' K u3 j#MyUID=$(uuidgen) && echo $MyUID #生成UID后面会用到#
8 I( p O+ t; QMyUID=5d8bc172-d375-4631-8be0-cbe11bf88a550 L' [' d4 t2 Z
Key=$(awk '/key/ { print $3 }' /etc/ceph/ceph.client.cinder.keyring)8 s! V( n6 s% j; N8 A
echo '
' I4 C( r7 j0 `! P<secret ephemeral="no" private="no"> , @. p q m# l( Z$ Q3 U
<uuid>'$MyUID'</uuid>
+ x+ K: |7 ^+ b/ m$ ]<usage type="ceph"> * Q" F2 H2 \- Z! q" u6 t7 Q& @
<name>client.cinder secret</name>
2 G5 A8 \8 q! u1 m% R9 |</usage> * C1 i. w' C: B! P% g6 w* G( H" b
</secret>
* l% ?! R/ J5 c- U1 n! ]) C'>ceph.xml$ ^& m; ]" [; O0 ~4 n% ~6 N0 I
virsh secret-define --file ceph.xml
2 W9 f/ f. U" ]( nvirsh secret-set-value --secret $MyUID --base64 $Key
. d) G* p" \- q1 m3 Y#nova配置# c! V- h+ s( } h
#注释原[libvirt]部分
' z% R+ Y. I( L$ @0 }sed -i 's/\[libvirt\]/#&/' /etc/nova/nova.conf3 ]1 Y7 X3 F: t" n8 S% V( R ]
sed -i 's/^virt_type/#&/' /etc/nova/nova.conf
; j- n, W7 D. N#使用ceph存储 C% U m# z0 R$ t5 W: O
echo '( Y% z' C J& N$ l* z, P
[libvirt]
; b* Z" O$ o: t% O: v2 p; ovirt_type = qemu
; v" A }+ p' e- {images_type = rbd# n4 y# b$ K5 f+ S) @6 G
images_rbd_pool = vms
3 F% k2 Q$ [5 s3 D- eimages_rbd_ceph_conf = /etc/ceph/ceph.conf/ g! ~: ?$ r, G z
rbd_user = cinder
$ h# y6 I- r- r* E$ nrbd_secret_uuid = '$MyUID'
" V- C8 a0 K$ j( v% v7 Hdisk_cachemodes="network=writeback"
9 `. Z+ m# r2 }6 i5 `: g' ylive_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
) _0 f% y$ t5 X/ r) Q/ @. @5 K#禁用文件注入#
" C" i2 M9 j6 f. L* nlibvirt_inject_password = false
* C# O: H. W# _* e) B6 @ e0 Ylibvirt_inject_key = false7 m( D1 P" x7 s- C4 ?6 @5 d7 ~
libvirt_inject_partition = -2
0 j7 h# U- w7 y'>>/etc/nova/nova.conf
( {9 l1 e: L# y% M: D2 y/ v#重启服务
5 O# t; M8 k) |( psystemctl restart libvirtd.service openstack-nova-compute.service5 X8 p# b; A4 z' j$ I& s8 p2 J* j/ @' x
###########################
; w6 [3 [' k$ ^( l4 z3 Z: i+ {& D#Cinder storage 添加Ceph存储
$ L. v2 x _6 V4 y* m#enabled_backends添加ceph
, R; }% W C4 w) Y' I7 V3 X' Dsed -i 's/^enabled_backends.*/&,ceph/' /etc/cinder/cinder.conf
, U2 s( ~6 }* @3 X) C1 L6 Recho ') @7 |9 G7 \$ ^# l; f$ L
[ceph]
+ c# j" C r5 p1 N( b A9 mvolume_driver = cinder.volume.drivers.rbd.RBDDriver
$ q4 w; a7 P* g9 f" h8 urbd_pool = volumes
+ r0 d' \& K5 ~5 nrbd_ceph_conf = /etc/ceph/ceph.conf
, ^# m2 |8 a4 {! ^6 J- a: j* }1 V+ qrbd_flatten_volume_from_snapshot = false
4 n+ ?% F# g; l5 K$ @. h! ~rbd_max_clone_depth = 56 u5 C9 |& R9 u* I$ W5 j9 h* `" N
rbd_store_chunk_size = 4
( j# h. u1 A1 `; I' L% g0 Trados_connect_timeout = -11 W* F$ Q+ h: W1 D3 C
glance_api_version = 21 F B) J, z9 @& N& k
rbd_user = cinder
3 ~- U: K% j' _rbd_secret_uuid = 5d8bc172-d375-4631-8be0-cbe11bf88a55 3 V2 P+ W; ]7 `7 u; E/ t: n
'>>/etc/cinder/cinder.conf9 S( \1 f' m6 l+ J% E% N7 C; k5 s" N
#重启服务 " c! E8 O* ^2 K2 J6 X4 M. [5 E
systemctl restart openstack-cinder-volume.service# `: l- S3 V3 M$ b- F: j0 g
###########################
2 ^% a2 U" ^2 A2 N4 q( o4 b9 c% A3 g7 H- L# E( ~+ h
1 o+ h% S6 Y8 `( N2 c4 \, m9 W5 \* k5 B
* S7 B+ ^) L! D9 k: w9 f6 q3 n. B
###########################
! S( i/ U# n: W' Y1 Y$ B#检测# r9 `5 x! p+ w
#在openstack管理节点
# E) O3 U/ ]( n+ |; s* Bsource admin-openstack.sh
2 o; W2 v: Q, v$ k& a/ \#查看cinder是否有@ceph存储
8 {9 g$ p/ R2 ~8 J0 |cinder service-list. S; _7 ~1 J" J: ~" p
#使用raw磁盘格式,创建镜像+ I3 y# n0 M2 K. f
source ./admin-openstack.sh9 o% o7 u Y' S8 ]% h7 r
openstack image create "cirros2" \
7 {# ?% A2 r6 j# X3 f--file cirros-0.3.5-x86_64-disk.img \
2 [; ^1 G9 D$ ]--disk-format raw --container-format bare \
9 T3 e; S; r( `9 ~--public# j% f3 a1 F- q4 n7 y; k9 ]
#检查是否上传成功
5 w5 F* z3 G, N' ^7 Q! Yopenstack image list! B$ P' w, p+ O- T# c7 {
#创建VM (cpu16是可用域)
|" o: `+ c4 C3 H1 ANET=de98a7e6-6aaf-4569-b0bf-971cfb4ffbc8
+ `3 h1 [4 H1 ?0 unova boot --flavor m1.nano --image cirros2 \9 W. w, f3 j( p9 x
--nic net-id=$NET \* U0 C' d. X" D2 A4 g
--security-group default --key-name mykey \
1 q1 C) r5 ]! n% l& p--availability-zone cpu16 \0 I" t0 E0 D! p
kvm04
( T! D6 m i3 M/ r! i2 @% Y) U0 v$ _#检查, {! j( { Y: y2 |6 c; L0 G+ P6 ]
openstack server list
2 |4 B9 }! S V. z y#虚拟控制台访问实例
# T; J: ~+ N* W2 S* Eopenstack console url show kvm042 V' e, d" b# }- ?4 C
#创建云盘volume4 S7 E4 L% Y# L3 d' E
openstack volume create --size 1 disk01
2 q8 J+ Y0 q9 ~( X X5 e#openstack volume list
, ]; y, `9 q6 ~& Y" c: D4 g#给虚机kvm04添加云盘
H0 r+ H- [1 g5 i' m) Xopenstack server add volume kvm04 disk014 T) `. }9 v7 J4 ]9 ` t
###########################
, Z0 ~& K: T/ Y8 I#在ceph管理节点查看5 X4 j4 p7 p& d) u6 F* s
ceph df9 W. ?2 W9 {8 M
#查看pool7 Y* `& F0 \! k. |. j/ K/ C/ `/ E
rbd -p vms ls
^% N% w" P! ]7 m0 ]rbd -p volumes ls4 C, m4 p7 \0 }% J* X- M4 H$ r
rbd -p images ls
. ]! r& q: {% K; |+ ~###########################
% ]: J/ C; a8 x########################### |
|
发表于 2018-1-10 00:02:43