ceph集群与openstack对接

admin

创建cinder并设置权限

[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'

设置密钥

[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring
" }0 |' I) h; |/ s( P6 s$ f8 R
' a0 W6 M0 B( M" a2 \+ _! w#传送密钥到computer

[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key
1 ?  D, C7 P' s) ^3 U; ?
! h4 p" |+ k  f* A6 p& v$ K* B( Y[root@controller ~]# scp client.cinder.key computer:/root/

#修改权限

1 Z' ?# O6 ~3 i[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
0 ^- b2 @. U; |' p; I6 j) a0 f

设置密钥

#computer生成uuid
* ~" `$ F; i7 Z. E
[root@computer ~]#uuidgen
- J7 g; y2 O" y# H) p: k
1fad1f90-63fb-4c15-bfc3-366c6559c1fe #创建密钥文件

- N% K4 M' ^4 J7 ^: A[root@computer ~]# vi secret.xml
, G" Z2 p+ f0 \$ X- u  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret
& \( B0 E( L$ N" W7 I

#定义密钥
0 a0 Z7 Q9 E6 }virsh secret-define --file secret.xml

6 a, [: R9 s/ ^0 J5 ?! C/ G$ c8 \
, ?. |8 _5 S. a5 ^8 }#设置密钥
- n8 P# I9 c" H- E7 n0 M- o

virsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
1 N/ x& u. a. D4 A- b- O
) U, R2 g) p6 n
  j6 z8 V! U/ e7 K# c设置对接cinder模块

修改配置文件

[root@controller ~]# vi /etc/cinder/cinder.conf

[default]

7 b7 {' K7 F7 \- }7 W: P6 nrpc_backend = rabbit

auth_strategy = keystone
9 F( ^, r8 z: m
my_ip = 192.168.8.65

, }7 e! W- e8 I; s" [2 henabled_backends = ceph-1
& v1 Z- v- a% R8 v- s7 v4 e* z/ |
! `2 z8 q# p- y% d+ A[ceph-1]
3 @3 ?" {+ n- ]! _
: D, [3 {/ |- V  \- G4 T9 j5 h6 S$ Ldefault_volume_type= ceph-1

glance_api_version = 2  

4 b4 v8 F7 f. f: x! s7 {+ o2 Qvolume_driver = cinder.volume.drivers.rbd.RBDDriver

6 }3 F$ u4 X; |7 B, r' n/ _; x4 X. dvolume_backend_name = ceph-1

, z& w4 f7 w& s, K! R, ]rbd_pool = volumes

rbd_ceph_conf = /etc/ceph/ceph.conf
8 U( ^/ n+ A, t  W
rbd_flatten_volume_from_snapshot = false
4 J  Y: y4 b3 t# g
; w4 a' y& f& s. A+ Erbd_max_clone_depth = 5

rbd_store_chunk_size = 4
% l; H9 d- F4 @6 F3 s- G
rados_connect_timeout = -1

rbd_user = cinder
  F6 t1 G! \: X' Z6 C* A
& E. H& W1 d& a( [9 t#对应computer创建的uuid

rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
" l* q" A6 j) H. B

同步数据库

#若已经有数据库，对数据库进行删除并重新创建和同步

[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder

重启服务

[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service

设置ceph的类型和存储类型

[root@controller ~]# source admin-openrc
6 [8 l( ?4 T( H+ m( T6 i3 e
[root@controller ~]# cinder type-create  ceph-1

[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph-1
9 @4 X8 G: F' S& e) A; [' ^: {
. C: Q& `. p( V; C对接nova-compute模块

computer结点修改配置文件

[root@computer ~]# vi /etc.nova/nova.conf
- v1 P# o% N1 d0 Y. h
, e6 X! z7 }/ ?% v) ]8 n0 E[libvirt]
/ u1 q5 e8 b- R$ Y: W" o8 P/ c
( p) e: K% y# f. P8 i) a) N. uvirt_type = qemu
; P) l" b, V; a3 a
inject_password = true

inject_partition = -1

images_type = rbd

" ^/ K  M' t+ ~5 y( Oimages_rbd_pool = vms

# A& k" v, x* \" E' K+ B$ J$ Fimages_rbd_ceph_conf = /etc/ceph/ceph.conf
. d  \* P; T, R# v9 v3 v- B4 p# X0 C
rbd_user = cinder
& D( k3 W- x' [4 B8 f$ d1 M
rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe

" T6 @! Q9 U: [$ b$ Gdisk_cachemodes = "network=writeback"
- [- v4 ]6 f. B' @
' p9 P; D( R- M: q) wlive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"

/ i' c' o) F% A4 p0 }hw_disk_discard = unmap  
+ q: B* a6 B0 I) A0 h* l: h% A. B
[root@computer ~]# vi /etc/ceph/ceph.conf

6 w' W) t% m* g[client]
& b, {: I, H4 f  `) o8 \
1 ], ^3 r/ M+ d5 Lrbd cache=true

rbd cache writethrough until flush=true

admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok

+ Z! N6 ~' G" z; ^5 N6 X! O7 hlog file = /var/log/qemu/qemu-guest-$pid.log

/ i  C3 W/ g' r, f6 }rbd concurrent management ops = 20
; G7 U* y% G8 i2 l9 l. m8 c2 e, G

创建日志目录

[root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/

4 K4 h7 `0 U4 O3 R[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
4 D9 q, g7 _& |# }  ^; P5 }

controller下发密钥

[root@controller ~]# cd /etc/ceph
: [4 A3 b3 z# i# H5 ?! f0 {+ d
* o# G$ D. r9 B. ]* V5 c2 d[root@controller ~]# scp ceph.client.cinder.keyring root@compute01:/etc/ceph

重启服务

[root@computer ~]# systemctl stop libvirtd openstack-nova-compute

( |- u2 U* }4 `- Y[root@computer ~]# systemctl start libvirtd openstack-nova-compute

admin

创建cinder并设置权限
2 z) E6 w6 \  F2 {( S1 A9 ]0 k
- S4 H( K, p  u2 R4 X! N7 b[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'
设置密钥
* s2 ?5 G7 z, b3 u: O
[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring

#传送密钥到computer
( G, m8 v' T" j* p! [! }- R7 a[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key
[root@controller ~]# scp client.cinder.key computer:/root/
! q+ l0 Y6 K( N2 ^% Q
#修改权限
[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
& |2 \; o8 a4 i) o' X8 s设置密钥
+ b4 r: `8 P7 l. D, J9 g2 z
#computer生成uuid
* l$ l9 _( i# B[root@computer ~]#uuidgen
/ Z9 I! ~0 ?3 v0 C8 c6 ^1fad1f90-63fb-4c15-bfc3-366c6559c1fe
6 M6 X4 F1 f( D8 ~" V( j
! Q% L* {- }) }2 o1 D0 f# C) w' Q#创建密钥文件
[root@computer ~]# vi secret.xml
  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret
; V2 v" `" t5 [1 H: G# Z9 P#定义密钥
virsh secret-define --file secret.xml
+ c0 q5 t$ U. {# J5 b; H! ~0 G
#设置密钥
/ t3 |* |+ v# F2 ]. w/ tvirsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
+ ]: c8 Y7 a! h: S  R设置对接cinder模块
修改配置文件
  U1 q2 \1 y; ~. S0 X" s
[root@controller ~]# vi /etc/cinder/cinder.conf
- B8 Z0 L* y5 Z9 O4 s6 t! ]/ I[default]
9 C& W% b2 K  urpc_backend = rabbit
0 F/ v; h" ]' m3 k3 i; bauth_strategy = keystone
  u, M+ f5 `" j9 o5 I2 Wmy_ip = 192.168.29.148
enabled_backends = ceph
: a) z5 x! e$ K# x& }
[ceph]
default_volume_type= ceph
glance_api_version = 2  
- x# ^- I; q' r) ?8 [; V6 \( Mvolume_driver = cinder.volume.drivers.rbd.RBDDriver
! `' I& W8 ~9 `8 G; `  ~) Gvolume_backend_name = ceph
2 O+ D  d" H2 p+ N$ U' Urbd_pool = volumes
- k" m5 p; ?) s, f- irbd_ceph_conf = /etc/ceph/ceph.conf
2 ~" X6 I( e1 vrbd_flatten_volume_from_snapshot = false
! T: @( p3 j2 H' R& Drbd_max_clone_depth = 5
rbd_store_chunk_size = 4
rados_connect_timeout = -1
rbd_user = cinder
; l9 D: |* X9 x, n, y#对应computer创建的uuid
; `$ W7 g: k: U$ Z* \rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
+ M" J. g! N6 ]7 C$ e/ L; M5 @同步数据库
; [1 B. T/ W) T2 `  F, N" k
4 A& y/ Q1 R$ J3 F, H+ [) @& o#若已经有数据库，对数据库进行删除并重新创建和同步
$ I) V" \; x( ~+ }6 p3 t[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder
$ p5 Y: s- x1 W8 ]. T3 r6 e重启服务
3 i4 }: f# W- R, P. Y
[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service
; S; l1 j1 p  g. S$ ]8 X9 M设置ceph的类型和存储类型
7 Z; B9 V' b% S% s" g" o) ?
6 K+ e% I+ e6 i7 Q1 k: H[root@controller ~]# source admin-openrc
' L: w9 G! y& l4 J9 X( F[root@controller ~]# cinder type-create  ceph
[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph
对接nova-compute模块
7 u% J: y+ a6 _- F2 b! Hcomputer结点修改配置文件
( C; q! d$ Y$ D3 Z
3 Z( Z- B" ^. K; E[root@computer ~]# vi /etc.nova/nova.conf
[libvirt]
$ q% L% @# w3 Z0 P' v$ yvirt_type = qemu
inject_password = true
. Y5 C/ g9 ~$ N& ^inject_partition = -1
images_type = rbd
# d. e" b2 e* @' simages_rbd_pool = vms
4 S2 ?% v4 p$ Bimages_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
$ W3 i% d6 p. c* Orbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
9 k1 v7 U1 E, S$ ]: [+ ~5 ^disk_cachemodes = "network=writeback"
' ^3 T7 ^* R+ G+ z9 Flive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
7 F# K' \3 m, \/ h& X% d2 ?% whw_disk_discard = unmap
: ^5 [' F! a6 @* w( T0 q* T[root@computer ~]# vi /etc/ceph/ceph.conf
[client]
rbd cache=true
2 q) U! L6 }, crbd cache writethrough until flush=true
admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
log file = /var/log/qemu/qemu-guest-$pid.log
rbd concurrent management ops = 20
创建日志目录

[root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/
% G# v% \6 `/ W& J5 }[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
( D5 Z3 E1 [- f' s1 p( Ncontroller下发密钥
* R- D4 g# {* e) l% X/ a+ k
[root@controller ~]# cd /etc/ceph
[root@controller ~]# scp ceph.client.cinder.keyring root@computer:/etc/ceph
% I! y2 Y4 d, P0 p" }7 T. e5 _重启服务

[root@computer ~]# systemctl stop libvirtd openstack-nova-compute
[root@computer ~]# systemctl start libvirtd openstack-nova-compute