|
|
#openstack pike与ceph集成
9 T( \8 ]- Z/ {$ n, R###########################
5 O7 B' p3 d! F4 X4 a#openstack节点
6 ^. K: J4 A) e: l- `$ S#openstack节点配置ceph源+ j( v8 e7 @2 Z; a9 f8 z1 Z
#使用阿里源 #rm -f /etc/yum.repos.d/*.repo# K1 b8 k# E, u/ f% d; s" A
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
# b$ \1 i7 C; b* v1 G! b& fwget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo8 y, Y( \ V: r9 ^1 l9 }4 t U# Z
sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址" C* s0 v, |1 V
wget -O /etc/yum.repos.d/ceph-luminous-aliyun.repo http://elven.vip/ks/yum/ceph-luminous-aliyun.repo
0 C% y6 D0 e! b: R' qyum clean all && yum makecache #生成缓存
; t/ r- R2 ~8 q) N8 m( ^#ceph客户端安装
! ^& i4 b, y$ ~: \4 B# v/ {yum -y install ceph-common4 x0 [& U- W b
" M z2 h3 l+ l: n3 C @+ t###########################
+ P/ D" K4 |2 _# X! z#ceph管理节点
! H, O) f' A( G#创建POOL4 w1 q5 t2 I0 U
ceph osd pool create volumes 128
. J, c; W7 r$ D7 _: N3 qceph osd pool create images 128
, W& M5 o; K# R9 U: Wceph osd pool create vms 128 - J. S' V9 {+ t. C: [1 F9 ^- Y
#ssh免密验证
# k$ i _/ t! L) q/ D4 Scurl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh' z( a. L: ^" x3 G- C
#认证用户及密码#' [6 @" {- p2 |6 H0 {/ J
echo "
; B, l& i4 x+ o& ^5 `USER=root2 c$ b$ D" q; Z7 h3 x- P
PASS=123321! j/ O' |- n2 ?! g0 ~: }
">my.sh.conf9 M/ a8 W ]1 {+ x: h. ?
#hosts设置
& ^9 h, B8 M$ Hecho "
- Z6 r3 x1 b5 p, T- A#openstack
. U3 r& R! y) m/ j; d* U/ f192.168.58.17 controller
: N8 S; b6 _5 Z: x192.168.58.16 compute01
$ B h1 S$ ]3 ]3 `/ n+ x192.168.58.14 storage1
5 d) [. v0 u( N">>/etc/hosts
1 Y3 H' |' E5 z#ssh批量认证#; k$ `' }. q; m7 p/ ~$ P( e
sh ./sshkey.me.sh controller compute01 storage1
" j* J. I L: P/ ~6 ~( [#推送ceph配置到client
0 W) J1 O9 n( u2 Icd /etc/ceph/. P* E% @* [, X0 v
ceph-deploy config push controller compute01 storage18 g2 h! u: V% t, @- c: m
###########################% k& t3 F) Y: _, H
#创建ceph用户和密钥
. d% V$ \# y" A3 r: Z* }ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
$ z9 L5 B; a( v- ~ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
! t, a0 _) }. I) k! s( n; sceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups' - i; s4 D4 @/ E) s5 U! Q% @
#查询用户，写入文件
" l4 [) Q( D& @4 q: d! Kceph auth get-key client.cinder >/etc/ceph/ceph.client.cinder.keyring
9 G, r$ v4 Y$ i2 ` A1 O+ bceph auth get-or-create client.glance >/etc/ceph/ceph.client.glance.keyring, T) y9 J' R8 n) S0 L! E$ ~2 U
# scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
9 J9 ]$ V- B! Z2 D/ _( G8 q# scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/
4 A- t, z3 N' B) O& a###########################
1 f6 m5 j3 T7 a' E9 B% a#拷贝秘钥到对应节点,修改权限 H1 x. L8 j6 F8 C/ L% |- m8 `8 A/ M
#(nova,cinder都使用client.cinder)
, W& x" ^- q% A2 x I% h#glance
; T$ k8 v& r% L7 A6 `" YNode=controller) b9 F/ }2 u+ L
scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/8 X2 q: Z8 N& @
ssh $Node sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring- D1 X* B4 k- x0 G: l# H
#nova compute- Z8 H* B2 m$ o& i. O' f% X+ z
Node=compute01( U& d6 u( C& Q. G' W7 |7 m
scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/# r; [9 i2 D/ }/ ~, `1 y
ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.cinder.keyring
) \# x; H- [) |scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/1 T% L- l$ O1 P# ]# t6 I
ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.glance.keyring
9 y& \- z0 K! F) x( B* v9 {#cinder storage
) \, B# Q6 W7 jNode=storage1/ h. _; y$ n( m
scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
" o, r( ~: ^8 |1 _! L+ O) ussh $Node sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
6 C3 W- p/ Y! A+ P###########################& p3 B7 `' x" {2 w) c
#openstack glance配置- g# \) i) [$ z% I
ls -l /etc/ceph/) L( t( n. K) z3 B, q2 \
#更改glance默认存储为ceph, H1 e& a: V% S3 \5 F
cp -f /etc/glance/glance-api.conf{,bak2}2 e& {+ k& C; C
sed -i 's/^stores/#&/' /etc/glance/glance-api.conf
; Q: k7 p7 }9 F0 m! ?" @0 \sed -i 's/^default_store/#&/' /etc/glance/glance-api.conf
, n! b2 K% v7 L9 E; iecho '#[glance_store]
4 q. ]& s7 K( ]stores = rbd,file
7 J$ Z4 p, i. N; ddefault_store = rbd
& v" x4 r" W6 u7 b, Nrbd_store_pool = images5 n" L, F, O2 P1 Z9 b
rbd_store_user = glance* x- G2 |; X0 o |2 V) \
rbd_store_ceph_conf = /etc/ceph/ceph.conf
& s b1 ]' {" j+ U, H2 q4 Orbd_store_chunk_size = 85 S4 G3 ^( r5 \
'>>/etc/glance/glance-api.conf4 g3 F- a$ g' H) R8 {
#重启服务
" d [( G/ d: b$ P# @2 @7 Gsystemctl restart openstack-glance-api openstack-glance-registry
# ]0 x! K: w$ h$ l, f- M###########################3 [) p8 [$ e M' h- a! B8 {
#nova计算节点, E$ K0 I8 F1 X. l. ]; E/ w
ls -l /etc/ceph/1 B7 T5 _1 c) q* n9 R0 D' k
#ceph- p5 Z f. P; y9 b# T, N/ D" {
echo '( s5 w. r4 Y0 R8 R
[client]
* W6 F2 d. B: O$ S# Wrbd cache = true
- H8 F: o% C, v; B: Orbd cache writethrough until flush = true
) V6 `; o7 d( ]! H% c! t: Q; j! Sadmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
* M3 C$ d0 y s9 ?' X3 k6 S8 Ulog file = /var/log/qemu/qemu-guest-$pid.log
5 U; ^, J# Y/ g9 W$ D7 Trbd concurrent management ops = 20
5 \3 I6 I. p1 C- y% `[client.cinder]
- ]% K G! u/ o" P0 Okeyring = /etc/ceph/ceph.client.cinder.keyring
q7 S2 e& i S. B" E/ l; S'>>/etc/ceph/ceph.conf
6 a2 P# Y1 Y9 z* X$ rmkdir -p /var/run/ceph/guests/ /var/log/qemu/
! B+ m& g5 P7 Ychown qemu:libvirt /var/run/ceph/guests /var/log/qemu/
6 X3 T! A( J9 s; p1 @#密钥加进libvirt
4 }1 p- z1 `! W0 M, C. ~#MyUID=$(uuidgen) && echo $MyUID #生成UID后面会用到#
0 ]8 {5 c5 P$ e8 H$ rMyUID=5d8bc172-d375-4631-8be0-cbe11bf88a55
( X: q0 K& O4 {8 [; Q {Key=$(awk '/key/ { print $3 }' /etc/ceph/ceph.client.cinder.keyring)
+ `. f( m! J7 v1 Q7 \8 uecho '$ g% ^! G: B1 \7 D7 \7 z5 Y% a
<secret ephemeral="no" private="no"> 7 ~) f1 S0 \- i5 h6 x3 s
<uuid>'$MyUID'</uuid> ! E0 b$ A/ E2 D; `
<usage type="ceph"> ! Q! t. l4 M4 ~# a2 |% h' R! V
<name>client.cinder secret</name> * M. p) s& f$ u& l' A$ b- f
</usage> 5 [/ h. m+ ^5 P; u) U" {
</secret> ' ~! b4 a8 w- d8 L. Q7 G
'>ceph.xml7 Y9 d. ^( d# T$ p/ a8 T5 J
virsh secret-define --file ceph.xml
4 G- d. W) O. U/ g1 ?: g* n. h$ hvirsh secret-set-value --secret $MyUID --base64 $Key. m7 j! l& q! z+ ]. T6 x
#nova配置1 w1 Q' ]+ h' J. M/ W
#注释原[libvirt]部分, u+ }5 b8 I( k! v# s# x
sed -i 's/\[libvirt\]/#&/' /etc/nova/nova.conf6 R8 `2 v7 T% q' O
sed -i 's/^virt_type/#&/' /etc/nova/nova.conf: {+ z v- B! L! }& m( W
#使用ceph存储
2 U4 }& i5 K2 ^; hecho '
7 Z" b6 j' Q( V2 n' b/ O& U[libvirt]" g+ |9 n0 y3 G. ]
virt_type = qemu, b2 i- @! A7 G' Q$ b1 W
images_type = rbd4 X3 n! @3 m. L5 f0 L) Z- ?6 Z8 f& ?$ {
images_rbd_pool = vms
0 b& `: t$ M9 A( }images_rbd_ceph_conf = /etc/ceph/ceph.conf0 {; l! v2 A/ f) x! Y) B, T3 |( `
rbd_user = cinder
) _$ j- o. z! i- I' H2 t4 ~rbd_secret_uuid = '$MyUID'
. Y$ k6 I* s$ U4 C" I Z4 R6 k: bdisk_cachemodes="network=writeback"
( Q$ [1 F. V6 V& xlive_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
5 Y# S, h8 ]# Z K V#禁用文件注入# y' N% }" ] Y1 j- d
libvirt_inject_password = false
+ l. B6 n' X7 j5 b6 ]7 jlibvirt_inject_key = false- Q0 l1 ]% ~1 z
libvirt_inject_partition = -2 z% a! e; r- W% ^8 @
'>>/etc/nova/nova.conf5 T. Y T; x+ Q+ @2 R6 G' B, v
#重启服务
% j# F* V. G- p9 |systemctl restart libvirtd.service openstack-nova-compute.service: H5 b1 L: p8 _: O1 ~1 b- _
###########################
, v; a5 N/ G% A3 D#Cinder storage 添加Ceph存储
7 ?, j4 b- D* Z: q+ O" T) F% |#enabled_backends添加ceph$ X j2 ?! H5 j$ p' g5 A: s
sed -i 's/^enabled_backends.*/&,ceph/' /etc/cinder/cinder.conf
- l' t9 r+ m9 u% M3 _( `# Uecho '# H' |0 M/ a. m. A0 k( S
[ceph]& ]' I7 a- w4 ^! A$ v( m
volume_driver = cinder.volume.drivers.rbd.RBDDriver
( @8 ]+ O' ] H; j; C9 Urbd_pool = volumes
0 S, f) w( ? F) Jrbd_ceph_conf = /etc/ceph/ceph.conf
3 m T0 W5 s& k& irbd_flatten_volume_from_snapshot = false
3 s/ M/ O. R" \+ O) V {8 W- M% `rbd_max_clone_depth = 5
9 ~9 N( {: ]( q* G6 t, B$ J' j: ?rbd_store_chunk_size = 4# H# l# E% ~, M. u3 [
rados_connect_timeout = -18 j) b- b- I$ F/ F8 }
glance_api_version = 2( N* Z# m. s* ]
rbd_user = cinder 3 S, F& [8 q$ z7 e
rbd_secret_uuid = 5d8bc172-d375-4631-8be0-cbe11bf88a55
8 E5 A! ~- {2 ~, J. M- u5 \ Y) E( e- V'>>/etc/cinder/cinder.conf
, b6 u7 s( N, V I0 k#重启服务 0 t+ c7 f2 _0 K( }, z/ A2 K' k9 N* v' }
systemctl restart openstack-cinder-volume.service: O; B* W: ?8 h( O8 ?) I5 O7 @$ {1 s
###########################* R( F" H' x8 K8 v0 g+ M
. E6 e3 s2 a% |$ `! H' r+ q6 {% ^; W* o' n. F( p
# j' L. h2 a6 S8 e3 U' H+ R
5 _( ?! A1 V' c7 T* P, B3 n7 F0 m; p
###########################
2 e; e; V+ v0 T; R( S2 b( J' d0 D2 a#检测
: @- S! J2 c4 c, C! t3 W#在openstack管理节点2 r( O# z: r# D! g
source admin-openstack.sh
% |% i- K+ {) h: V& I#查看cinder是否有@ceph存储0 |+ L3 {5 b+ h( \8 p g- \
cinder service-list1 z5 d* G0 n: D( t0 T# \! `4 G
#使用raw磁盘格式,创建镜像
1 ?) y& f6 c' m `source ./admin-openstack.sh
! o- }7 k6 E5 E: G: A: O3 {openstack image create "cirros2" \1 e ]4 Y# v; @! t$ |) A
--file cirros-0.3.5-x86_64-disk.img \3 l/ S- u- t+ q; n1 v# l6 S3 K
--disk-format raw --container-format bare \
4 z5 `" `* Y6 X& F& _7 H6 p% J--public
! m4 \+ r/ ?1 I; d#检查是否上传成功
8 j# E( ^, Q6 |" popenstack image list9 t& y* M& G0 s, h5 n& s
#创建VM (cpu16是可用域)
0 n. G0 L2 s% UNET=de98a7e6-6aaf-4569-b0bf-971cfb4ffbc81 A" u% i) Y/ L* }! ]6 i6 k2 S
nova boot --flavor m1.nano --image cirros2 \
5 }( z' ]1 ?9 N/ }( Z--nic net-id=$NET \: I/ i5 a3 K) g3 c
--security-group default --key-name mykey \
; k: ~8 p% ?( _8 A. |--availability-zone cpu16 \+ P0 r& k* @9 f5 [% U3 U
kvm04/ ~/ `/ `( M% r( ^7 w' s) o" d
#检查
) A$ J" V& T; ]4 }- W# fopenstack server list9 p/ j5 N' W2 v
#虚拟控制台访问实例
" H& ?1 {; i6 P# k8 `* y! ^openstack console url show kvm04
6 F+ L- W2 M. ^# Z' W, h" A#创建云盘volume* g0 w# a$ U, B5 y6 l( b
openstack volume create --size 1 disk01
' `. _; k5 @* X0 _. p! E7 Y8 y#openstack volume list9 o0 w0 @2 N$ P* V) l: T! o
#给虚机kvm04添加云盘. ^: Q$ N( Q: ^
openstack server add volume kvm04 disk018 l. j$ @. U4 E5 Q: E- Y) v2 w* ?; Z
###########################* o( {( d# h; H1 V7 }
#在ceph管理节点查看
- N* z) k. e) |9 D, o. r Yceph df8 c( q1 E }" @
#查看pool
+ \ I" c, j) @, B! ~$ G- H" ]rbd -p vms ls
1 o- y1 I$ f4 e& t i/ _8 V. {rbd -p volumes ls0 X$ o: O2 j$ y& X. I% O
rbd -p images ls8 q2 l9 z3 v% n/ F- A( W9 j; K
###########################7 {. Q2 i4 V }, a7 C: f# A' N
########################### |
|
发表于 2018-1-10 00:02:43