ceph集群与openstack对接

admin

创建cinder并设置权限

[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'

设置密钥

[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring

5 l, n  X* K& b; F5 A1 ?#传送密钥到computer

% f+ x4 R5 X3 `8 {4 X2 o1 |' C[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key

[root@controller ~]# scp client.cinder.key computer:/root/

#修改权限

8 X" c- e+ z* ]0 [- O[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
, |5 S' V9 z, O8 I# R; M0 H

设置密钥

#computer生成uuid
$ n- Y2 J1 a, {7 |( D5 E- |
[root@computer ~]#uuidgen

1fad1f90-63fb-4c15-bfc3-366c6559c1fe #创建密钥文件

$ }2 M# h2 Q; S& k/ s; n[root@computer ~]# vi secret.xml
  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret
9 {' J8 }! d: a7 o! z& s# q

#定义密钥
$ u: Z% p* h) H& W) Ivirsh secret-define --file secret.xml
, g) ?1 ?' k3 c# j) D7 k* a# X

( L2 u9 T) f/ G( K6 N; t. R: D" `#设置密钥
4 h" R4 |' D: R
; t% [& D; Z5 A0 n/ y
  g  J6 {* H! R7 L. _; @- @virsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
/ ~- D1 h& ]- p
1 v+ U' p& |' e& K% z4 ^) T
设置对接cinder模块
3 V0 ]5 p! d; h

修改配置文件

[root@controller ~]# vi /etc/cinder/cinder.conf

: u& Y9 |9 @7 Q' j( I  d& e; C[default]
& I2 b& ]& H/ A  X) w
$ \9 m, H: p* k3 r" k( b; f5 |rpc_backend = rabbit
- z1 `% c* s# }  Q  E& T
  K8 z5 I+ `! p5 hauth_strategy = keystone
7 ^  B" g' c& _
0 ]4 }/ r" K  Umy_ip = 192.168.8.65

+ B- O# n. j2 N  U! w$ r8 y. Kenabled_backends = ceph-1

[ceph-1]

default_volume_type= ceph-1

glance_api_version = 2  
* _, V. w! S  t0 t
volume_driver = cinder.volume.drivers.rbd.RBDDriver

volume_backend_name = ceph-1

rbd_pool = volumes

rbd_ceph_conf = /etc/ceph/ceph.conf
1 o9 f  d( q& p: o
rbd_flatten_volume_from_snapshot = false
( y  T( g2 |) _. I9 ^
rbd_max_clone_depth = 5
6 N) o! q/ w7 {+ e
rbd_store_chunk_size = 4

' D- W7 ~% j2 p+ Y' J, T; g0 lrados_connect_timeout = -1

rbd_user = cinder

1 n# Q( u9 `# J( A* _#对应computer创建的uuid
% h) ]" [; t7 {% Z2 W( [
* |5 f2 f6 Z- ?9 u" k, j3 D# Yrbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
, h4 j5 n$ M/ e/ n( O) u  ^

同步数据库

#若已经有数据库，对数据库进行删除并重新创建和同步
+ c! G3 v: K/ l8 J- x( n! E/ I
[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder
1 n+ P4 I3 R+ \& @# f

重启服务

[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service

设置ceph的类型和存储类型

[root@controller ~]# source admin-openrc

[root@controller ~]# cinder type-create  ceph-1
' ^- n/ G; D3 K; {2 z% |
3 O* F, ]  s5 @8 K: X3 `) U. G! J[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph-1

% ?* |$ O4 A6 L对接nova-compute模块

computer结点修改配置文件

[root@computer ~]# vi /etc.nova/nova.conf
4 M6 w' N9 m3 j' M/ h
[libvirt]

virt_type = qemu

inject_password = true
7 m" b. F1 R8 S3 [% W2 G! G
inject_partition = -1

& j2 j) U7 c' g8 Y* i7 f7 ~images_type = rbd

images_rbd_pool = vms
8 K9 |9 t! j2 b: Q8 W, \3 h! n% l. c
( n) F0 j  r' C4 C2 |) `+ C8 d' K# h) Gimages_rbd_ceph_conf = /etc/ceph/ceph.conf
- h7 X  a" @9 Y2 i+ |
rbd_user = cinder

' y. x7 D( I( P" \) }# O4 Xrbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
  p& G* {- ^9 {: B/ @; E( O# i+ o
; V) n2 i4 W/ y% O2 j4 Cdisk_cachemodes = "network=writeback"
+ C, a' p5 _1 ^' @2 x
4 U3 D4 l; c1 ^2 Plive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
% {2 V3 [' {3 n" c8 |" ]
hw_disk_discard = unmap  
" y( A- V7 m$ P* x! u' K  u
[root@computer ~]# vi /etc/ceph/ceph.conf

! f$ i7 t( B- p. n9 Y[client]
' F! U" M( p8 u# u
rbd cache=true
3 ]. |8 }7 p2 v& E5 n9 |4 C
rbd cache writethrough until flush=true
% S: j3 G" U( K' n! \, @: a  u  N' I
: Z$ o2 `3 G3 q6 ^4 Tadmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
, D  o0 w( O: i! Q
log file = /var/log/qemu/qemu-guest-$pid.log

rbd concurrent management ops = 20

创建日志目录

[root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/
, t. [" p; W& X1 p  [
8 k+ O5 a: [1 O, s5 K[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
; L7 P4 X5 Q, o- H( a

controller下发密钥

[root@controller ~]# cd /etc/ceph

[root@controller ~]# scp ceph.client.cinder.keyring root@compute01:/etc/ceph
6 o' x2 j* ?5 C# g, u

重启服务

[root@computer ~]# systemctl stop libvirtd openstack-nova-compute
- p2 O' m: ^# l* ]: T. }9 z
( j' c% \) M+ N% Y9 Q[root@computer ~]# systemctl start libvirtd openstack-nova-compute

( ?# Q4 ]2 q3 t; Q
' Q/ l. ~# M6 f$ r2 G+ t

admin

创建cinder并设置权限

[root@controller ceph]# ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms'
2 H; e- a" f' _+ @0 W设置密钥
" h2 N  K0 M8 @# a  C- W
[root@controller ceph]# ceph auth get-or-create client.cinder | tee /etc/ceph/ceph.client.cinder.keyring
1 f, f% L2 w) ~2 N/ Y. k& M: p- N
/ a! m# K% Q( D#传送密钥到computer
[root@controller ~]# ceph auth get-key client.cinder > client.cinder.key
[root@controller ~]# scp client.cinder.key computer:/root/
: `( i( M% i7 ]7 r" K
#修改权限
; k" m  m" L/ h" W& a[root@controller ceph]# chown  cinder.cinder /etc/ceph/ceph.client.cinder.keyring
设置密钥
* \9 U# E% e; U, F# H  T
, L+ B  W: v. s: {1 M#computer生成uuid
[root@computer ~]#uuidgen
2 e8 h% k0 q8 K2 y1fad1f90-63fb-4c15-bfc3-366c6559c1fe
5 U- @& m7 w* A2 g: Q9 i
#创建密钥文件
[root@computer ~]# vi secret.xml
' V- D' \1 z. {) a  1fad1f90-63fb-4c15-bfc3-366c6559c1fe   client.cinder secret
+ X1 x8 M- u* ^" ?4 l. F#定义密钥
: t, R" I2 E( x- Z8 A' Z- k5 F% jvirsh secret-define --file secret.xml

; T) I( e- J" x2 F3 \  _' I#设置密钥
5 D3 Y. \6 J3 a/ a% mvirsh secret-set-value --secret 1fad1f90-63fb-4c15-bfc3-366c6559c1fe  --base64 $(cat client.cinder.key) && rm -rf client.cinder.key secret.xml
1 d3 \: d+ c  _  j9 ]# h设置对接cinder模块
修改配置文件

[root@controller ~]# vi /etc/cinder/cinder.conf
[default]
; E/ I' `% D6 C$ K  u8 c( g! Prpc_backend = rabbit
" l1 {3 @4 Q- _2 }* m" m/ wauth_strategy = keystone
; t8 Q  R: r1 U) e3 Amy_ip = 192.168.29.148
enabled_backends = ceph
9 i; f+ L3 u5 W, b+ S) g1 S; l5 E
& W- M; I& G8 V! A[ceph]
9 s8 C' e* a4 [6 Q' Jdefault_volume_type= ceph
glance_api_version = 2  
" N3 ~, O; X3 o  ?volume_driver = cinder.volume.drivers.rbd.RBDDriver
. i1 e3 y  y9 _% _5 uvolume_backend_name = ceph
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
/ w' t5 ~# D( U1 ?, y  ]rbd_max_clone_depth = 5
+ I; W8 o8 ^2 e$ Q) y4 B. A# _( q/ l# Jrbd_store_chunk_size = 4
+ ^8 e* }' U) x" p6 p( wrados_connect_timeout = -1
8 ^' E: h. R1 c# b+ N! @% a3 [rbd_user = cinder
#对应computer创建的uuid
, j0 I* D0 z1 ]: Frbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
同步数据库
4 d' }2 @: J) B" D0 {- w9 c  m
$ ?  _) g6 }+ L4 H" v; K#若已经有数据库，对数据库进行删除并重新创建和同步
( e0 F' p9 p* S! x% Z[root@controller ~]# su -s /bin/sh -c "cinder-manage db sync" cinder
, w4 ^; i1 W! ^1 t重启服务

/ l; v- Z; Z3 g4 U1 z1 W9 g( x( c; \  V$ N[root@controller ~]# systemctl restart openstack-cinder-api.service openstack-cinder-scheduler.service openstack-cinder-volume.service
1 N% [- g5 [% w! Y+ q9 C5 q设置ceph的类型和存储类型
0 }+ m  E7 N% z
[root@controller ~]# source admin-openrc
( V! A/ k. b$ Y# ?[root@controller ~]# cinder type-create  ceph
! Z: p: N! ^! C+ z7 _[root@controller ~]# cinder type-key ceph set volume_backend_name=ceph
+ s* @) }1 [4 c; n" l对接nova-compute模块
% A, N4 u& B' G) l, Ecomputer结点修改配置文件

9 X- e) j0 ]) ^0 T[root@computer ~]# vi /etc.nova/nova.conf
% V1 z( v: I* \2 [[libvirt]
virt_type = qemu
inject_password = true
inject_partition = -1
images_type = rbd
images_rbd_pool = vms
images_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
rbd_secret_uuid = 1fad1f90-63fb-4c15-bfc3-366c6559c1fe
disk_cachemodes = "network=writeback"
# @8 z. |1 Q0 l2 m' U9 p; F2 slive_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
: U# Q) L2 W0 C2 s5 J3 f; V2 l2 {( Shw_disk_discard = unmap
[root@computer ~]# vi /etc/ceph/ceph.conf
[client]
rbd cache=true
# O8 ?' T/ i. _( O) ^rbd cache writethrough until flush=true
admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
log file = /var/log/qemu/qemu-guest-$pid.log
$ f  \9 _1 _! i* E0 m7 B, v* yrbd concurrent management ops = 20
6 ?- k* m7 P" m' n) w: Q创建日志目录

[root@computer ~]# mkdir -p /var/run/ceph/guests/ /var/log/qemu/
5 S! z+ O, ^2 T[root@computer ~]# chown 777 -R /var/run/ceph/guests/ /var/log/qemu/
8 z8 m4 m; N3 [% m9 ^- K  S* [0 ucontroller下发密钥
$ P, @0 A5 Y) o: |9 D! R/ o# z
9 x$ P/ {' o" @, p2 C6 ^[root@controller ~]# cd /etc/ceph
7 k, @6 L) K1 s; E' ]' P) U) m% i' q[root@controller ~]# scp ceph.client.cinder.keyring root@computer:/etc/ceph
. y, r/ X* ^  ?0 m1 f. e( k重启服务
% B" S' }9 Y8 Y- {8 K* p& `
[root@computer ~]# systemctl stop libvirtd openstack-nova-compute
( S1 L; v: \. W8 ]. ~' J2 o[root@computer ~]# systemctl start libvirtd openstack-nova-compute