ceph集群与openstack对接

admin

创建cinder并设置权限

[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'

设置密钥

[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring
  \$ R+ X# m# Z: C
#传送密钥到computer

[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key

5 ~3 e$ s! h2 X' y0 u/ ~2 u[root@controller ~]# scp client.cinder.key computer:/root/

#修改权限

[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
6 n# D8 A9 Z/ [" T

设置密钥

#computer生成uuid

[root@computer ~]#uuidgen
  l1 f' Z3 t( H# n
1fad1f90-63fb-4c15-bfc3-366c6559c1fe #创建密钥文件
3 C/ V! y8 h5 E' w# Q
[root@computer ~]# vi secret.xml
3 q( X5 x& u+ y) C! j  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret

* D& c$ N9 I9 J3 u& H1 b
/ W( o" q3 e7 |" s#定义密钥
virsh secret-define --file secret.xml
! {. |8 N' ~& `' M- j
3 o8 l) k! e2 _6 R
#设置密钥

& E6 ?+ b' a( ]) R
virsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
) h& i& W% N: ?0 b! _4 u  V6 C+ `

; J* x0 D0 z0 v. C' g) u% N设置对接cinder模块

修改配置文件

[root@controller ~]# vi /etc/cinder/cinder.conf

[default]

1 A" C+ B2 v% }) C/ frpc_backend = rabbit
% Z. V' w5 {; j  B
auth_strategy = keystone
. [' G) ?( `! Y' I. J5 x
my_ip = 192.168.8.65

+ M* ~+ ?/ X4 l' q8 a4 c4 |! Venabled_backends = ceph-1
$ I0 V) Z; N1 j$ r* |+ F
[ceph-1]
9 {0 r" I* c3 I( m7 Z
/ \+ B" e; {, M' W4 w% qdefault_volume_type= ceph-1

( _0 i9 f1 U6 m7 v8 ^glance_api_version = 2  
/ A; [8 W; P  a, T6 l: }( {" X
7 X7 U5 o7 j* cvolume_driver = cinder.volume.drivers.rbd.RBDDriver

$ y: W) I5 Y2 s6 avolume_backend_name = ceph-1

rbd_pool = volumes
. m. p8 R! `! t+ Z
rbd_ceph_conf = /etc/ceph/ceph.conf
& |2 u7 _! M( _, V
rbd_flatten_volume_from_snapshot = false
. y0 v0 i5 r1 K- C$ v, f3 b8 f9 O
# d6 S+ B2 Y% I' j+ L. z# [: xrbd_max_clone_depth = 5
% \1 q/ C+ r1 ?8 j, N) f( J" E) }
( t9 T0 V- A# grbd_store_chunk_size = 4

: J, T( l1 F: a& R/ e* c0 mrados_connect_timeout = -1
0 ^1 O/ E+ ]. R( g' B
: ^8 s8 Z. O! l) H  [5 v! S7 w  \rbd_user = cinder

5 U8 }4 p2 B7 i* N#对应computer创建的uuid

rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe

同步数据库

#若已经有数据库，对数据库进行删除并重新创建和同步

[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder

重启服务

[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service

设置ceph的类型和存储类型

[root@controller ~]# source admin-openrc
0 v( |. T2 J" q+ i5 R0 \: k
[root@controller ~]# cinder type-create  ceph-1

# N- {. d5 _0 g! d& a9 Z[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph-1
6 X' c( f4 X) C+ N' x' `( a6 }
对接nova-compute模块

computer结点修改配置文件

[root@computer ~]# vi /etc.nova/nova.conf
: s+ j, A# Q4 t. Y) Y/ s7 R
[libvirt]

+ V, E2 |! `9 z: g! Yvirt_type = qemu
8 d" D4 ~2 O6 _6 T7 }, {
; `1 p& H5 A! |& zinject_password = true
. K3 B) N# a# Z: @/ H! a$ B
! [+ H+ `# ~- D$ E6 I) i. sinject_partition = -1
$ Y# ]5 D1 i1 F) F# h
2 d, e7 _$ p7 M; |. m1 I# iimages_type = rbd
% g# m! w. q. A! n2 |! D
2 W+ N- {. J0 H& M9 @, Vimages_rbd_pool = vms

images_rbd_ceph_conf = /etc/ceph/ceph.conf

3 V0 q1 }0 Z+ {1 P: vrbd_user = cinder
/ r& t3 i1 x6 K$ F
5 U, }, |- w5 h# Z/ v! ]rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
' P( S3 r$ J4 q- t4 |
disk_cachemodes = "network=writeback"

. D) ~2 m& U; o% vlive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
6 }& ]. n0 Q  n  ]  l2 _7 v% @3 R
3 T2 Q" n. M  P3 ?3 \# p7 Ahw_disk_discard = unmap  
4 y" t4 g4 t9 _2 T
[root@computer ~]# vi /etc/ceph/ceph.conf
$ y1 z% F7 d  v. U
* x5 \6 ^" Q; J[client]

rbd cache=true
4 ~# X* \! Q" ^0 g* Y" }4 K; L4 ?  K
0 B1 C) W0 I8 [9 Z: {0 l2 \4 h8 t7 qrbd cache writethrough until flush=true

admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
' p9 m7 g; f! V. N3 i+ u" M% `4 l
. z4 x7 P+ q- ~5 L& h: E- olog file = /var/log/qemu/qemu-guest-$pid.log

rbd concurrent management ops = 20
$ n) L' b' c: u7 k, r

创建日志目录

[root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/

[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
, W. z1 `% U+ |" C; I: H& u6 m

controller下发密钥

[root@controller ~]# cd /etc/ceph

) s0 j2 ]6 H- L[root@controller ~]# scp ceph.client.cinder.keyring root@compute01:/etc/ceph
8 \( O% e" s1 M  R

重启服务

[root@computer ~]# systemctl stop libvirtd openstack-nova-compute
& b3 [1 Q7 A) y  a! a3 M
$ ]% p4 @) w$ r6 v% Z1 a- h# _[root@computer ~]# systemctl start libvirtd openstack-nova-compute

admin

创建cinder并设置权限
* L3 ?0 U) E5 [5 q4 e3 B" I
4 o% Y2 e4 s( v* Z[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'
设置密钥
6 s6 I' A. u  \& E7 c5 l
[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring
  O$ Q  m6 A# a& @
#传送密钥到computer
/ w7 b* _, m  [1 C! L! z& t[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key
[root@controller ~]# scp client.cinder.key computer:/root/

#修改权限
2 X) c6 F4 }$ h( `; H$ h4 |[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
设置密钥

' ?' M+ ^7 s; c# e#computer生成uuid
3 L: V$ E$ @- A+ f/ g9 ][root@computer ~]#uuidgen
" o, H9 \- y' u2 o9 a: k1fad1f90-63fb-4c15-bfc3-366c6559c1fe

3 T$ r! b; J0 k" d#创建密钥文件
5 a3 U* F5 L( \' s! E6 I* Z; i[root@computer ~]# vi secret.xml
  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret
: ]" F; _* D( X! c" _2 L6 N; @#定义密钥
virsh secret-define --file secret.xml
/ M# _+ r. O4 s1 A5 U0 P1 j! m
#设置密钥
# O1 P" x5 H! E# G  Q; O# p0 cvirsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
设置对接cinder模块
% C4 G& C- \2 ?8 H9 \1 T- X修改配置文件
& B1 A6 s: z  Y6 O
[root@controller ~]# vi /etc/cinder/cinder.conf
[default]
! Z  E9 a* [8 `6 C: Wrpc_backend = rabbit
8 ~: R. P7 g: N; Zauth_strategy = keystone
2 h. C5 H2 N( r, L; {, U4 S( }my_ip = 192.168.29.148
enabled_backends = ceph
6 R# \* n5 ~5 Y4 {3 ~
[ceph]
: k: ?8 f7 l1 e7 R6 v2 C/ @default_volume_type= ceph
glance_api_version = 2  
volume_driver = cinder.volume.drivers.rbd.RBDDriver
" y+ E& {% q# @+ Vvolume_backend_name = ceph
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
, V' ]/ |. e) x, B4 W/ T+ }/ g' drbd_store_chunk_size = 4
rados_connect_timeout = -1
rbd_user = cinder
$ m. c. r# D6 u$ y  ^9 V  O0 i#对应computer创建的uuid
rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
同步数据库
1 V1 J# T% ^6 U7 X! ^5 ?8 d
#若已经有数据库，对数据库进行删除并重新创建和同步
, `$ N: W2 ~, B! z[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder
重启服务
! G9 }; b1 Z! n: p( C8 }
[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service
8 B1 N  N, v; ]6 m& B设置ceph的类型和存储类型

, @1 C: ^! y* t! O5 S[root@controller ~]# source admin-openrc
( Z* ^3 ~0 I( Z+ P; v; S[root@controller ~]# cinder type-create  ceph
/ b' G. L1 A- o( x( h[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph
对接nova-compute模块
# L2 d8 _  F7 f4 X* O: g1 h# k0 dcomputer结点修改配置文件

[root@computer ~]# vi /etc.nova/nova.conf
- `! Y" |/ S9 l: ~' y" P  P[libvirt]
% C$ Z6 |% i9 o( x2 o; v3 V- Yvirt_type = qemu
inject_password = true
inject_partition = -1
images_type = rbd
- y; T$ Z# U' G* `0 T: {images_rbd_pool = vms
5 n0 b+ O: n) ]( Bimages_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
, i4 I# x+ B& ]8 {rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
disk_cachemodes = "network=writeback"
5 Z1 S* b0 _4 e6 Z+ S7 Z; Ylive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
1 `, T. y  L- ]6 H% f8 J; `7 B; Vhw_disk_discard = unmap
[root@computer ~]# vi /etc/ceph/ceph.conf
[client]
rbd cache=true
rbd cache writethrough until flush=true
$ v; \2 `2 D8 b, a+ u% F( Q1 K( f# f# sadmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
! J1 i/ v0 @. A. d1 i' D, S9 llog file = /var/log/qemu/qemu-guest-$pid.log
rbd concurrent management ops = 20
创建日志目录

) N5 f4 s% _: H4 `4 ][root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/
7 V. P0 N) L. e[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
controller下发密钥
, c0 K! K- l2 {7 C8 Z) Y% r
[root@controller ~]# cd /etc/ceph
# z5 Z) {- |5 O0 p* g( A[root@controller ~]# scp ceph.client.cinder.keyring root@computer:/etc/ceph
重启服务
( u  z& U  X- t5 N& p' T3 G: K* G
[root@computer ~]# systemctl stop libvirtd openstack-nova-compute
[root@computer ~]# systemctl start libvirtd openstack-nova-compute