|
|
#openstack pike与ceph集成
5 k o0 d% o' a7 D, K5 H: n! M* Y###########################* s# k$ o$ p: I' w! t0 d9 X
#openstack节点* T+ {, F( t- E4 }! [; e2 z
#openstack节点配置ceph源2 [. A- k! h& D) t L7 D
#使用阿里源 #rm -f /etc/yum.repos.d/*.repo
3 h% g! U; K7 Z) d; `) ?wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo, q; h: w9 c1 P# T9 u2 I: C0 Z! K7 Y
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
1 {4 r) G. h" E& Osed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址
* `4 t- f! I1 z/ s) \6 {2 Ywget -O /etc/yum.repos.d/ceph-luminous-aliyun.repo http://elven.vip/ks/yum/ceph-luminous-aliyun.repo' C) Y$ [+ S$ c/ {5 k: o
yum clean all && yum makecache #生成缓存
% l( N4 \- u3 \; |' t2 G#ceph客户端安装8 R4 g e9 O4 I$ v
yum -y install ceph-common! k( y9 E& T" V, U* T
i$ M- b( E( v# ]' R###########################/ E* U" {5 c! e V) N
#ceph管理节点
/ o6 M: t+ z% f4 a: Y5 | E- [#创建POOL! S0 f" ^* H- `) {# J
ceph osd pool create volumes 128 6 W: c4 S( l6 _% h' T7 t$ v
ceph osd pool create images 128 / m$ m, |- L" V i* D1 @8 o
ceph osd pool create vms 128 $ v& M! m, H! `" k
#ssh免密验证
. z3 A5 ~1 |6 C2 J5 }curl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh6 U$ c2 a$ b$ Z4 w4 D; T8 M }
#认证用户及密码#; C' @" ^9 W k0 p9 w! ?# K
echo "7 p5 U. h8 W0 p) F
USER=root
& M1 N8 K {: ?/ j0 fPASS=123321% c; A4 |7 \+ B. o: Q
">my.sh.conf
# E8 |7 d. [, p. V) I0 l# d ^#hosts设置
) U6 U* I7 g! s+ Secho "
- u% P9 Z, [* V) t/ F6 {#openstack
7 \8 P8 Y7 w4 c4 m; m192.168.58.17 controller
% E p, K v# Q& |192.168.58.16 compute01* I" y9 o8 d) B0 y5 @8 {- b
192.168.58.14 storage1
* r$ \' T. R$ u/ @; S">>/etc/hosts4 `* ?8 d7 `) V% u J
#ssh批量认证#: c: k6 s+ `9 d+ O+ l* i- S+ S4 i
sh ./sshkey.me.sh controller compute01 storage1" O, `! N% X4 I7 ]& S y/ M. r
#推送ceph配置到client
0 e1 w$ y8 j/ @- c2 B3 gcd /etc/ceph/! {- O/ l" J ]8 d
ceph-deploy config push controller compute01 storage1
8 m$ R7 s- _4 \, ?* G; {4 E! t###########################6 |8 r0 C$ P: I
#创建ceph用户和密钥
+ M2 ?( }! U# Z9 h* t4 nceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' 2 \4 H' k0 l7 e% w6 _( c7 I
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images' 7 \. k. M8 H& i2 G$ k/ R
ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
& O. u+ t# Z2 c0 G#查询用户，写入文件
' }) C/ a" M' q' ~ceph auth get-key client.cinder >/etc/ceph/ceph.client.cinder.keyring' \* i+ `% A3 S
ceph auth get-or-create client.glance >/etc/ceph/ceph.client.glance.keyring$ c, ~% w: z/ w- j3 Y% j7 _% W
# scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
$ G& s, m6 M" Y# e4 }8 w2 ^& {# scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/2 ~0 ], j- `$ o
###########################
1 u6 W- ?- h8 s+ F#拷贝秘钥到对应节点,修改权限
4 a. ]+ x0 T; f! W: g2 d( c5 `#(nova,cinder都使用client.cinder)
( U; K3 ]2 `0 Y$ }$ j! P# k5 k; F7 d#glance4 B$ M' H8 @7 i! ~
Node=controller
3 u8 M5 n. S) Y9 Q! g; h2 Jscp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/7 n9 j9 O6 w9 Z6 Z& `
ssh $Node sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
' C9 }. V2 y6 D4 C/ q l( w#nova compute
/ F0 S' x3 `) N% ]+ w( w' l2 cNode=compute01
6 T6 a6 \$ y( @. Z. b! tscp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/* B4 F& Q2 m. @& E6 J4 r
ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.cinder.keyring- P( Y# f# p2 I5 Y( @0 }. r
scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/+ {* M* K7 P8 @& w
ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.glance.keyring$ ^ ]# t1 m- M/ A1 Q; t7 F! H8 p0 e
#cinder storage
3 `5 d3 S( q2 P% p9 V HNode=storage1
3 Y9 t S' ^) T( }8 p4 k& u5 m, wscp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
6 a: V& p0 m0 }2 Y" K$ P: b) ossh $Node sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
2 a5 ]/ l: e' O2 }% m1 V3 Q y###########################
9 t; I( P8 w4 |& z4 Z: @#openstack glance配置+ K! x3 M3 Q6 J5 a3 s }
ls -l /etc/ceph/( |" P( ]! F/ ]4 J; s1 V
#更改glance默认存储为ceph
( {( P9 z) J, scp -f /etc/glance/glance-api.conf{,bak2}
' B0 r, F+ v/ t: _. |sed -i 's/^stores/#&/' /etc/glance/glance-api.conf
, ^' q: {0 z* S) R7 X, u4 P; Lsed -i 's/^default_store/#&/' /etc/glance/glance-api.conf 4 Q! y9 _! L, _) e0 _: F
echo '#[glance_store]" q/ ^! b* y+ R' o( Z2 M
stores = rbd,file4 u! e! B' @0 C% W. Q+ p
default_store = rbd* B! `% D0 P$ `5 M* k: D
rbd_store_pool = images& {* [1 C' z0 b( z
rbd_store_user = glance* o$ E+ t- A L7 _# l; H$ ^% |
rbd_store_ceph_conf = /etc/ceph/ceph.conf
/ ^6 i9 w# J( s5 c8 nrbd_store_chunk_size = 8* U0 W! ?, ^/ B6 a! ]3 l2 _
'>>/etc/glance/glance-api.conf( Y5 m; C+ x1 Y2 p
#重启服务
4 G, i6 h- @ R0 @. h1 lsystemctl restart openstack-glance-api openstack-glance-registry/ a5 v( X2 S6 H) G n7 g1 C
###########################6 l9 f0 ~- W* Q5 D+ z! Y; \
#nova计算节点
1 z/ }5 F" p5 B Q8 Els -l /etc/ceph/
% s- t9 c% R5 C) |1 Z4 }0 A# o#ceph8 C2 G/ g! h0 s2 P9 \) z
echo '
8 C' t/ L' e5 O% m. y. |& c8 {[client] 1 C: o4 J9 \/ Q2 `" u
rbd cache = true , Y+ X3 }( p( h/ s' x
rbd cache writethrough until flush = true ! g, e% K8 G1 C
admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
9 O( y7 N( @" J/ e2 c1 B# flog file = /var/log/qemu/qemu-guest-$pid.log 8 l- I. }5 K' ?; O6 F1 X. |
rbd concurrent management ops = 20 3 k3 @$ P# k& S" ^, p# _- y5 J
[client.cinder]8 @6 ]- F+ W! z& y3 D( q9 W
keyring = /etc/ceph/ceph.client.cinder.keyring& F/ J% c+ v# {
'>>/etc/ceph/ceph.conf8 M" O, O6 c) d2 d& v0 Q
mkdir -p /var/run/ceph/guests/ /var/log/qemu/ " G7 a) m( Y# i& E/ x
chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/ 5 |2 m* b, @7 y. R4 o6 r6 p5 R. E
#密钥加进libvirt
9 E& @; L9 i) l# y! \- i; B+ H#MyUID=$(uuidgen) && echo $MyUID #生成UID后面会用到#. f# R+ q/ T2 a0 D4 l7 f8 T
MyUID=5d8bc172-d375-4631-8be0-cbe11bf88a55
! ]* |/ i5 e4 Q9 Y! ZKey=$(awk '/key/ { print $3 }' /etc/ceph/ceph.client.cinder.keyring)* r: V4 B5 Q( K- U0 ~+ r
echo '
6 u$ H; {; c4 }! \/ g9 W5 Q6 T<secret ephemeral="no" private="no"> 6 c- u. E& t* V8 a. o4 c3 Y
<uuid>'$MyUID'</uuid> ' o& d; J. s+ Y+ n6 N4 M
<usage type="ceph"> 5 z6 i8 _( B; D- m' D" n3 K
<name>client.cinder secret</name> " J% L% o/ x5 h* _8 D
</usage>
" J ^; x; W: Z5 Z3 b</secret>
2 {0 i0 {$ k( b+ o6 W4 a+ t3 _'>ceph.xml. v- D1 P% I' a$ m0 m
virsh secret-define --file ceph.xml 2 h- H' ~% {. M9 t; k& j. R
virsh secret-set-value --secret $MyUID --base64 $Key/ H( [) p8 q& p# G3 |! L7 ~! q
#nova配置: b+ L7 F4 V3 k
#注释原[libvirt]部分
/ l2 I; j }) E6 Y( t& t: }) wsed -i 's/\[libvirt\]/#&/' /etc/nova/nova.conf
6 Q) G5 Y* w2 U9 X$ wsed -i 's/^virt_type/#&/' /etc/nova/nova.conf
H4 G, k b/ ]/ B1 E6 M f#使用ceph存储* c' l& N2 y8 Z' Z
echo '
) W# \+ d. h R- p' z* i[libvirt]/ k/ t, S1 g# p O" X# P" {
virt_type = qemu# i3 d% [$ ?! ^6 u. Q
images_type = rbd
" e/ z; C2 b' h2 E4 k# {images_rbd_pool = vms
6 e% |/ ] [, o- ?/ himages_rbd_ceph_conf = /etc/ceph/ceph.conf
$ T. V4 i$ n+ {) U2 s' b% q4 nrbd_user = cinder
# I4 F; r% b2 K* _0 l+ Frbd_secret_uuid = '$MyUID'; V2 Q3 }4 A( v' Y. E
disk_cachemodes="network=writeback"
9 {9 o5 V+ N3 p. U. o7 `! r Slive_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"/ F- l4 ~ i; A7 k$ d2 E: V: l
#禁用文件注入#' A4 i8 c, g) _7 i. B: B) F" q, L1 r
libvirt_inject_password = false
% v. r( W* M9 d% Slibvirt_inject_key = false
: e. h2 j, T0 r$ a) \libvirt_inject_partition = -2! S* }5 `7 P0 c0 A6 n
'>>/etc/nova/nova.conf
/ o" R) t% `% ^( E, S5 N#重启服务
' z" ^5 ]9 s W# s7 a" F" [! z Q! osystemctl restart libvirtd.service openstack-nova-compute.service; p6 @3 |5 P) S- n9 L V0 _
###########################
/ E# ~. A1 q1 V' F/ V% O#Cinder storage 添加Ceph存储
7 ?9 x: `1 b6 u: }+ v$ K) ]- l#enabled_backends添加ceph
; L9 h. ]6 Z7 K& `" ased -i 's/^enabled_backends.*/&,ceph/' /etc/cinder/cinder.conf( X0 j0 i) F- o, ~9 Q8 c
echo '
4 `) H- t i) H+ F" d[ceph]/ w$ Y4 H) \0 G9 X6 {+ h& @
volume_driver = cinder.volume.drivers.rbd.RBDDriver
3 q7 @7 w6 R) L# j% o" ]rbd_pool = volumes
! j! c' |4 Y$ ~rbd_ceph_conf = /etc/ceph/ceph.conf
/ V& \1 S! f1 C& orbd_flatten_volume_from_snapshot = false1 { F" o8 B5 D5 {: Q+ c
rbd_max_clone_depth = 59 l; r/ X; S" h; e
rbd_store_chunk_size = 4
+ p, g2 N8 _% |0 y$ trados_connect_timeout = -1& p) b1 F* b# o' j7 w* H$ j
glance_api_version = 2
% a( X$ B0 {, k2 wrbd_user = cinder
( E' b' v! X) ]; u9 Wrbd_secret_uuid = 5d8bc172-d375-4631-8be0-cbe11bf88a55
0 T" I/ G( g8 z1 i8 B3 }8 L'>>/etc/cinder/cinder.conf( V: u# Y# a2 Y5 p1 `# H
#重启服务
. ]8 n a5 a/ O% X& b# qsystemctl restart openstack-cinder-volume.service0 `& I7 A* u2 V: F2 Q8 l' v1 o
########################### {4 }- w/ M- V7 p o0 g
) q8 g( x" F |' ^; {& J" o1 _5 K
5 d& U6 Q5 d6 C8 L0 y1 }4 P9 k; c1 {# r: Y' _
$ A Z/ Z) Y$ T) l P+ g$ h+ g###########################4 O R$ e& S) n/ t5 c. L
#检测+ a9 z0 _$ y. @8 [. u
#在openstack管理节点
$ l" ?/ w. K) x+ }( P7 F9 w) Bsource admin-openstack.sh: B$ D0 H7 N1 ?- I
#查看cinder是否有@ceph存储
( C7 @/ [6 T8 N2 s7 ^- P1 e" l4 {* pcinder service-list/ a( {1 m9 D+ A( Z# W
#使用raw磁盘格式,创建镜像
6 S& C1 Z5 q: isource ./admin-openstack.sh
, w7 K! {& y1 H" lopenstack image create "cirros2" \3 M2 d/ X7 o. c; t
--file cirros-0.3.5-x86_64-disk.img \7 H+ j$ i; ^3 |4 G! Y. k; `% t7 G! {
--disk-format raw --container-format bare \
# b( @# b# m; e+ Z" O--public
0 q; `' T6 r/ M, K7 K8 K7 G5 M#检查是否上传成功
6 j; V) B! o0 \( f5 _0 ?openstack image list
! Y, y: ^1 b; A#创建VM (cpu16是可用域)
7 a, a8 [/ r, x0 k4 T5 dNET=de98a7e6-6aaf-4569-b0bf-971cfb4ffbc8
1 k ]! w* E! _! ?nova boot --flavor m1.nano --image cirros2 \
; G0 ?$ e! S8 g8 V. [--nic net-id=$NET \9 A( ?0 `& ]! p# i% Z
--security-group default --key-name mykey \ P% j& w1 S' E' g. A& @
--availability-zone cpu16 \- w5 X; D3 g. q4 q! k! F, x
kvm04
- H8 Z( C% }. O/ B$ z#检查5 O& I" r; t% U% S/ W$ e% H
openstack server list. I/ q: m' i( L$ X) v
#虚拟控制台访问实例/ T k( ?4 K0 T: i) h
openstack console url show kvm04
! D3 j: y- h' p# F#创建云盘volume
9 Z( L1 s* g6 o0 X/ Hopenstack volume create --size 1 disk01' s7 T# e$ E8 Q% R A
#openstack volume list$ O7 w: |: I8 a- a: ]6 t, V9 }
#给虚机kvm04添加云盘9 A& ]- w3 M) K) W4 ?
openstack server add volume kvm04 disk01
4 P1 j) K- d& m# Y###########################7 D5 H! E9 c: D( _9 M
#在ceph管理节点查看/ N3 Y6 x; Y+ j5 a/ F
ceph df# p: q, q$ }2 N" \- H3 b
#查看pool* L: } z# d( f4 a" R
rbd -p vms ls& E( [" W: S/ c8 o
rbd -p volumes ls
2 D/ Y0 d# I% i4 z" A- Xrbd -p images ls
- L( y$ ]9 j, i K############################ k/ v& K5 U1 u9 O
########################### |
|
发表于 2018-1-10 00:02:43