|
|
#openstack pike与ceph集成
( i: w1 v' _ g" e) X###########################" a4 E) `& ^8 [1 @ ~3 } p h
#openstack节点% z, Q3 b" ~3 |* r
#openstack节点配置ceph源0 r% [) [" O2 }% A- }
#使用阿里源 #rm -f /etc/yum.repos.d/*.repo
" ^" I$ L1 c6 gwget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo; i( D, y& q0 w0 {0 O4 }
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo% P* y4 A, L* l8 K
sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址
: R% ?- o0 J% q9 g' gwget -O /etc/yum.repos.d/ceph-luminous-aliyun.repo http://elven.vip/ks/yum/ceph-luminous-aliyun.repo2 P9 A- K+ E* V. i" g3 q
yum clean all && yum makecache #生成缓存2 o8 _ D4 Q9 g6 {
#ceph客户端安装8 W5 G* k. p- E
yum -y install ceph-common
) x# C: i3 N8 |* ~& W
O/ r* j) }. n4 S6 _###########################
+ }- u& K) g. J" Y% Y' V1 Y$ Y# v#ceph管理节点
- x# Z6 [& f1 [! S: p% p#创建POOL. }3 Z% L, C9 j+ l- W9 e0 F
ceph osd pool create volumes 128 8 {3 e! a% z9 q/ O2 M
ceph osd pool create images 128
2 S4 h5 u: q2 w7 O0 T9 Z" Hceph osd pool create vms 128 D7 V" U$ Q3 t& S3 Q' Z d. [6 @: ?
#ssh免密验证
1 f* ?# Q% E9 O3 B; {curl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh
7 ~/ J9 g+ ^0 g4 F9 x g#认证用户及密码#
9 r. S7 Y ?" C1 h+ g* b1 Kecho "
7 o$ @' f. c' z& i) W4 o3 }USER=root/ K% X1 P, W1 K6 b% P' F* z
PASS=1233214 X) T7 h* P5 K8 |0 ], {- Q$ y
">my.sh.conf; E$ ?* E4 j/ Z1 v. |) I+ K
#hosts设置3 f" x, i; }$ a3 F
echo "! l1 h) `- I* ?, `
#openstack
# X1 w5 ]& K' A( f. B( O( p192.168.58.17 controller6 ?* y, C0 \" y) J" q
192.168.58.16 compute01' Z3 |) I$ r, O
192.168.58.14 storage1
+ w. u! _% c7 @; C- p">>/etc/hosts
% Q% p K1 I1 ?! Z/ E1 e: L! ?8 y. O#ssh批量认证#
5 a% W% K" L3 T/ M0 o* E) hsh ./sshkey.me.sh controller compute01 storage14 R% N8 D7 T1 ^: C: d9 U' X( U+ y
#推送ceph配置到client3 [$ g, u/ r, r, F
cd /etc/ceph/
! A3 O* ]* r1 e/ F" {* _ceph-deploy config push controller compute01 storage1# ~- N0 X) m2 }7 r' ~0 x0 j
###########################
. \& K. v4 H3 j$ h( h7 l#创建ceph用户和密钥
% F3 k* p! \0 F- s; U' L( sceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images' 1 P* b {1 k; Q6 B0 s$ L
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
, R! N- m' ]* o X& J1 P3 g3 bceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups' . Z- B3 L# M3 A) w7 T5 J8 R
#查询用户，写入文件, `4 v t, i: N7 F$ ^' I
ceph auth get-key client.cinder >/etc/ceph/ceph.client.cinder.keyring% z% o% o* Q$ w; R7 \
ceph auth get-or-create client.glance >/etc/ceph/ceph.client.glance.keyring* W9 h* G/ ?$ O0 A& c& w1 \
# scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
& |8 m% J6 B* ~% f/ {# scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/2 W5 N* v- L1 m" f1 W( r
###########################
4 y1 E9 f; D- C+ v. l#拷贝秘钥到对应节点,修改权限
5 l9 G) `0 B4 J4 K#(nova,cinder都使用client.cinder)
& s, @% q) v8 ]; ~, s#glance
; ~, |2 X% e* v, s* I# m- mNode=controller' X$ ~- B5 q, h) t5 ]
scp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/
' m2 I: T: r% S7 A7 F& jssh $Node sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
" z$ ^' z- N a/ j! ?#nova compute# G( {$ x% N+ x3 Q% e/ Z
Node=compute01
4 K2 J U' H# i$ H6 S! d1 S. Sscp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
w" W! d/ Z5 Z: V7 F: M- A: ^ssh $Node sudo chown nova:nova /etc/ceph/ceph.client.cinder.keyring
$ ?5 j* I3 S0 W5 [3 Cscp /etc/ceph/ceph.client.glance.keyring $Node:/etc/ceph/
* E$ ^7 }) y9 M9 C0 Issh $Node sudo chown nova:nova /etc/ceph/ceph.client.glance.keyring
1 P6 r, Q; n* w* k" n/ {: N* {#cinder storage
- l$ P, b, l/ t& `1 O) eNode=storage1
" H- z& q' t+ s) a. u0 } X, j2 c7 k: \scp /etc/ceph/ceph.client.cinder.keyring $Node:/etc/ceph/
2 Q9 D7 l7 C9 issh $Node sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring, r2 b( R8 h& F6 I3 _; W
###########################
" {: K& D' i Y( l7 v0 W0 [#openstack glance配置0 W. P( d1 O; ]$ R- R1 F% P6 e
ls -l /etc/ceph/: r! m7 a4 C3 d3 t/ _% L" S: n
#更改glance默认存储为ceph
& h$ i3 |) M* j0 scp -f /etc/glance/glance-api.conf{,bak2}
/ o5 Y' B- \: m; V6 P: q" fsed -i 's/^stores/#&/' /etc/glance/glance-api.conf 8 K* ~! N; l' N, y2 H
sed -i 's/^default_store/#&/' /etc/glance/glance-api.conf ' `% p% R3 V& _
echo '#[glance_store]
4 s3 R5 j& r7 s0 c- Nstores = rbd,file
: ~/ \# m9 D3 X, {1 j4 Ndefault_store = rbd/ K3 @- Q2 g7 g6 w0 S( z
rbd_store_pool = images
7 k4 n0 h3 O& }9 A0 c% i. Crbd_store_user = glance! x0 I2 ~2 e8 D7 s# e! [0 n: B
rbd_store_ceph_conf = /etc/ceph/ceph.conf
& q% W$ J' J4 n; y1 G+ K; Prbd_store_chunk_size = 8
" @. z0 e+ ]/ M& u; w u- {" n'>>/etc/glance/glance-api.conf5 m; {% U2 a9 }6 N$ r: [- f
#重启服务& g, z. u' O( P3 H% {3 S0 C3 k& l# I0 ~
systemctl restart openstack-glance-api openstack-glance-registry, w7 J# A9 s/ P& Z! I$ o3 Y
###########################
$ L; |& @8 s- o- ]& z4 _2 L#nova计算节点9 f' [, M$ X, w$ v: K
ls -l /etc/ceph/9 f5 @# `" X' ]" u) C
#ceph
$ E; H [3 [7 O f$ fecho '+ i8 G# W+ Y, G c( X/ D/ l- n+ ?, F
[client] : {# z' `9 R2 }: B5 A1 L+ x
rbd cache = true 4 A/ N8 h: |9 `* h9 B+ _
rbd cache writethrough until flush = true
$ E, P2 {& F3 B/ Gadmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok + R- d- d( t+ W4 J3 J- s+ H3 N8 Z/ H) `
log file = /var/log/qemu/qemu-guest-$pid.log
3 x1 R. C3 g. }rbd concurrent management ops = 20
5 A' F5 s$ I S' R8 f# f) E! n[client.cinder]
4 X& ?/ X9 E$ l# E5 d6 A' w2 tkeyring = /etc/ceph/ceph.client.cinder.keyring
6 o$ M* y# E$ O; ` b; z+ @'>>/etc/ceph/ceph.conf. a' @2 p" ?- T8 _% I% ~0 j* ^
mkdir -p /var/run/ceph/guests/ /var/log/qemu/ : X" T5 E* b" \& w$ o
chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/ 7 w2 A9 G2 |$ G3 f" E8 L1 _$ Q
#密钥加进libvirt
" O* }! w# x' G3 s; V#MyUID=$(uuidgen) && echo $MyUID #生成UID后面会用到# G% \1 w* _1 N! U3 }
MyUID=5d8bc172-d375-4631-8be0-cbe11bf88a554 z4 c7 s1 e' Y# p/ T
Key=$(awk '/key/ { print $3 }' /etc/ceph/ceph.client.cinder.keyring)
% T, ~5 z6 z0 ]7 _! oecho '% i% p, u) V8 T" G8 \: k: T
<secret ephemeral="no" private="no"> 7 _. I+ w+ }! |1 g, Z
<uuid>'$MyUID'</uuid>
+ a K! H1 F* r( s4 Y4 x<usage type="ceph">
% T( }0 R- V7 i$ C; `4 F<name>client.cinder secret</name> 3 B( I) i% a; p" A" O, a: Q+ y0 F
</usage> ! g* U, l5 k$ x4 Y5 x
</secret>
3 R' _) E. z# e( K8 c! s7 U'>ceph.xml" o0 k0 x }! p* e/ h) D. z
virsh secret-define --file ceph.xml % W' W. A% l" g' a
virsh secret-set-value --secret $MyUID --base64 $Key5 k2 e x+ j3 `0 ^. n6 |
#nova配置$ X9 \, x7 j- ^
#注释原[libvirt]部分
9 l# |4 f* H/ k# L8 Jsed -i 's/\[libvirt\]/#&/' /etc/nova/nova.conf
6 n, \: o* [' j; }6 R0 psed -i 's/^virt_type/#&/' /etc/nova/nova.conf) ~# |6 z c8 D* M
#使用ceph存储
3 s7 _/ z) e& |" y' l6 Techo ': c; Z3 a) x& I, `5 y* }- U
[libvirt]
- i8 u' G7 I. g* {! \virt_type = qemu
# B# f" m g- U3 Vimages_type = rbd
0 L# l ?" a1 P& y8 ~& jimages_rbd_pool = vms
: i5 X; p' z. I0 w4 v, U, zimages_rbd_ceph_conf = /etc/ceph/ceph.conf
2 E) @6 `0 _" w* v3 `, Prbd_user = cinder
- X' T5 U& r2 O7 r7 drbd_secret_uuid = '$MyUID'
8 q+ {2 n% c0 {disk_cachemodes="network=writeback"3 P, V" y: B+ O
live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"8 k$ b$ \0 {3 O" Z4 l; Y* T3 H
#禁用文件注入#
! r. g1 b# K% w: c! {! ^9 T; `libvirt_inject_password = false
3 J: l% m: Z& s4 g. Glibvirt_inject_key = false
% m# N4 k) {& o- p: ?0 R/ tlibvirt_inject_partition = -2' P5 i z! J3 u
'>>/etc/nova/nova.conf
A# U5 J& D& r4 Z' {9 R#重启服务4 k2 }! x/ J1 o
systemctl restart libvirtd.service openstack-nova-compute.service
2 v9 s. x- d' @1 R! X' @###########################, v: j1 h7 H2 F5 f
#Cinder storage 添加Ceph存储
( ~6 Z3 P5 J" Z/ B4 L3 K#enabled_backends添加ceph
% L9 c9 {* G+ \1 zsed -i 's/^enabled_backends.*/&,ceph/' /etc/cinder/cinder.conf
6 ?' _, m+ e& Y& a) u+ Gecho '
: l+ g9 F5 [& w: k, e! y3 A* X[ceph]
& o* V" b. _+ H3 N( evolume_driver = cinder.volume.drivers.rbd.RBDDriver
: D8 b7 l, M5 nrbd_pool = volumes
F' _. O" [, [7 t/ d! d1 lrbd_ceph_conf = /etc/ceph/ceph.conf
2 a V# {" H. p+ Arbd_flatten_volume_from_snapshot = false
% B+ Z* N. J6 F- k7 Qrbd_max_clone_depth = 58 P1 Z; g6 ?% l2 U/ ]
rbd_store_chunk_size = 4! m- e+ A+ `. J
rados_connect_timeout = -10 L) i; Z i. |
glance_api_version = 2! z) ]; K4 E% A p
rbd_user = cinder ; M' R% S# Y" ~4 g; b
rbd_secret_uuid = 5d8bc172-d375-4631-8be0-cbe11bf88a55 7 _; P% s& ^; e+ s7 Q
'>>/etc/cinder/cinder.conf
! l1 } A/ L+ M#重启服务
+ h2 g; {* |) Q% K! ysystemctl restart openstack-cinder-volume.service
4 @# u$ o: z( V5 I###########################) n2 U0 u/ M6 w/ `
+ u7 {$ K0 N9 W4 N/ z( r0 e8 {! w* [7 q' Q
- a/ Z2 D0 r$ e* n& V/ `* h# I7 p% u8 ~
7 _/ G# s( ]: }$ R3 e c( R- O###########################: G+ p/ |$ W9 I! n
#检测
* g! S2 v- s+ |: E/ v#在openstack管理节点( M# P5 f5 t7 a. x" w/ E j- X
source admin-openstack.sh
7 D+ F$ t4 @; R. w V; ?* n#查看cinder是否有@ceph存储9 S9 d9 u; d$ {# M9 c1 p
cinder service-list- v* o0 g4 s @( [# F
#使用raw磁盘格式,创建镜像" f6 Z& p5 Q4 @. y3 q
source ./admin-openstack.sh
- _# a. F. `; M, T% {- hopenstack image create "cirros2" \( n1 L. B h. ]% L
--file cirros-0.3.5-x86_64-disk.img \
$ a* W- ~$ t8 W; g* I--disk-format raw --container-format bare \
! Q) m" p! G0 l7 l- `: `--public
. D& o1 I% L& h, G#检查是否上传成功
$ T/ G2 D: t4 p! ~7 Iopenstack image list
$ f' h6 C4 b9 w5 }; ^1 g#创建VM (cpu16是可用域), l* |8 V$ Y0 g( K. ?0 S7 c
NET=de98a7e6-6aaf-4569-b0bf-971cfb4ffbc82 H, u! Z5 T% @& W" ^* `
nova boot --flavor m1.nano --image cirros2 \8 j+ q8 p, S r+ R" y
--nic net-id=$NET \
, ^( r/ S7 U8 {; m--security-group default --key-name mykey \) u m! r0 [' N5 q, c
--availability-zone cpu16 \
0 E- _& v7 j' e* E2 I" P4 Nkvm04! _. V7 Z) ^! m5 ]
#检查/ P# L. @8 a- S% V1 v6 h5 ~
openstack server list
* S' E6 j, ?0 ^" U* d+ _! c8 i#虚拟控制台访问实例- a' w L) ?! v. b
openstack console url show kvm04$ G D( K v) @/ D! x5 k
#创建云盘volume
% J( S; c5 J0 v" E V( copenstack volume create --size 1 disk016 A& n* k: U) }5 j& R) L7 h
#openstack volume list( K1 ? P+ w1 e
#给虚机kvm04添加云盘6 B9 c' D+ d# m( M* C( i
openstack server add volume kvm04 disk01
% c) k( L' \8 t0 h6 f' k# `/ B. h2 |########################### v1 k( t; R5 L( B+ G' \) Y
#在ceph管理节点查看
) d" h4 b" w8 B [1 Q- Z6 e( nceph df
3 B% u& \& V& O#查看pool& \ Q( X: } X Z. {; d" |
rbd -p vms ls
4 Y1 p. B4 G& j% k0 T5 Krbd -p volumes ls
. q( w6 v. z m+ z1 qrbd -p images ls- u3 Q: o& u( b7 P
###########################. y$ o/ z9 X; a; _2 ]+ \ Q/ V1 n
########################### |
|
发表于 2018-1-10 00:02:43