最新Ceph 版与openstack Pike对接

admin

一、在所有服务器上操作

#使用阿里源

yum install wget -y

$ E7 r/ I8 `) t1 V" G2 j

rm -f /etc/yum.repos.d/*

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址

+ o4 @& Q6 z' h7 k* ~3 j+ ^

#创建ceph源

echo '#阿里ceph源

" V1 D2 k! g# C

[ceph]

9 g. R; a- q3 g  @9 w' |, J

name=ceph

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/

+ F3 `0 w: ?  e( I9 R( r

gpgcheck=0

5 ^+ A, H* e: D& _

[ceph-noarch]

# Y' |/ e% W: m

name=cephnoarch

; p* o5 i2 H1 z, L$ l

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/

gpgcheck=0

! f8 p( d) }: ]* u. R/ b

[ceph-source]

name=ceph-source

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/

7 G" R( q+ b$ G9 j; L& e

gpgcheck=0

. P% G7 A( }( M! I

#'>/etc/yum.repos.d/ceph.repo

' x9 I' w6 Q1 @2 M) L. {

yum clean all && yum makecache #生成缓存

4 R& V+ i! T, s) j

#关闭selinux、防火墙

. I! U$ r6 j7 U9 H, A

systemctl stop firewalld.service

) h& }: Z7 N7 l' R( l

systemctl disable firewalld.service

1 p2 c  q" X+ d/ z0 E6 L' u

firewall-cmd --state

sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config

9 X$ U- ]: Z# K) F, U. C% q

sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config

2 B) F, q' |. Q/ E9 S

grep --color=auto '^SELINUX' /etc/selinux/config

9 G6 b; G* M2 n4 d6 w! @0 W# j$ K0 B

setenforce 0

二、只在node1上操作

三台服务器分别为node1 node2 node3   其中我们在node1上面安装ceph官方的自动化部署工具，也就是说我们只需要操作node1即可完成部署

# E6 j+ I; }- H. ]6 N

#添加服务器主机名

echo '

% c( j* A8 I$ m4 w: d" Z: ~  a4 Z, P9 @% L

10.0.0.11    node1

10.0.0.22    node2

0 O8 o5 m9 `% f+ \0 j& b

10.0.0.33    node3

3 A2 j7 c# }8 }' P/ ?: t! M

10.0.0.44    node4

6 c6 v1 M0 V/ C9 h* O2 j

10.0.0.55    node5

10.0.0.66    node6

( ?& E( R  H$ k" _* L9 [  b

'>>/etc/hosts

: G0 u. l+ Y. |1 X4 ]

#创建秘钥

' i' ~2 G, e; {( [

ssh-keygen

ssh-copy node2

ssh-copy node3

8 n1 Q  @# l: m- v7 W6 p

#将node1的hosts文件发送到其他服务器上面

scp /etc/hosts node2:/etc

6 |) X' \9 O6 W- y. ]

scp /etc/hosts node3:/etc

: [3 S6 f3 n* }
( `) ?( T1 J- q, S" W1 y

三、开始在node1上安装ceph

; G4 q5 p0 l* t6 O) _, O& C

#安装ceph-deploy配置工具

yum install -y ceph-deploy

yum install python-setuptools

#创建配置目录

5 Q5 B8 K4 P4 @

mkdir /etc/ceph

cd /etc/ceph/

* y+ R0 B" l/ w7 x4 x

#初始化Mon配置

ceph-deploy new node1

$ T/ Z/ j9 @0 m; Y  B1 z: G; `- M2 J) M

# #修改冗余份数为2，日志大小2G

3 z$ Y& L7 r7 z

# #配置网络,单网卡忽略

echo '

public network = 10.0.0.0/24

cluster network = 10.0.0.0/24

3 q) P0 }! d0 c3 j2 v

mon_clock_drift_allowed = 2   

osd_journal_size = 4086

. g: P: d2 H3 [, B$ \

osd_pool_default_pg_num = 128

osd_pool_default_pgp_num = 128

, y$ _! ?) I( b

osd pool default size = 2

' r( l2 f3 G/ {  R* Z

osd pool default min size = 1

+ G; ~! ^6 s! x) k8 z

rbd_default_features = 1

client_quota = true

' T4 w4 Z" P+ W+ w8 @; ]0 ]' @

'>>./ceph.conf

9 ?6 S7 ]& @) }! T: ?

#安装Ceph

8 {) f, c, f0 `6 l) f0 o. ?

ceph-deploy install node1 node2 node3

#yum install -y ceph ceph-radosgw #实际上是安装这2个rpm

#初始化monitor和key

cd /etc/ceph/

' }4 D, T, c$ H; L

ceph-deploy --overwrite-conf mon create-initial

, r/ R0 O; v/ E5 _, S
. `7 E- [: w' I, n9 w1 e$ Q

#创建osd

8 D% z7 G" O9 W

ceph-deploy osd create --data /dev/sdb node1

( a( x8 q- C4 J; n* @

ceph-deploy osd create --data /dev/sdc node1

; S  c9 `2 L. x* F1 `

ceph-deploy osd create --data /dev/sdd node1

ceph-deploy osd create --data /dev/sdb node2

ceph-deploy osd create --data /dev/sdc node2

( U! c7 m+ x9 W1 D3 f; q

ceph-deploy osd create --data /dev/sdd node2

ceph-deploy osd create --data /dev/sdb node3

* [9 o6 R' N5 y( k+ ]

ceph-deploy osd create --data /dev/sdc node3

9 |  D. H, q& B

ceph-deploy osd create --data /dev/sdd node3

1 D7 I/ x8 S1 B# M

#拷贝配置及密钥

7 p% h4 b. s' _  S3 @+ E: X

ceph-deploy admin node1 node2 node3

) U( Z# ~1 C' L) U1 x' ]1 z  `3 R

chmod 644 /etc/ceph/ceph.client.admin.keyring

( k' _/ N* M( @, f3 U2 O, l! d0 _) `

#创建mon

ceph-deploy --overwrite-conf mon create node1

ceph-deploy --overwrite-conf admin node1

#添加mgr

#ceph 12开始，monitor必须添加mgr

( m. m( D9 \* j# L8 P

ceph-deploy mgr create node1:mon_mgr

5 Z: {+ H- Y% I! t6 P! w

#启用dashboard (在node1节点)

$ z6 {  s5 H4 I) d) o

ceph mgr module enable dashboard

web访问http://10.0.0.11:7000   即可看到ceph的运行情况

#查看相关命令

; ?' Y1 q. s6 H' I  c

ceph health

ceph -s

+ K) ]# _4 e8 g& Z9 [

ceph osd tree

+ U% V* `' W& z6 ?: I

ceph df

! }& U: b! _) u/ C& ^0 ]3 F

ceph mon stat

ceph osd stat

$ e# r- t0 d  h8 \8 q9 s

ceph pg stat

2 e3 f" G/ B2 f7 i8 V, P
. q" q; s9 |, }; a) A

ceph osd lspools

ceph auth list

5 R1 ^; [2 M6 R! o$ w6 B& x7 K/ }+ d

Ceph L版与openstack Pike对接

* W) G5 ?/ z& Y

环境opestack Pike的controller、computer、cinder三个节点已经安装完毕，并且能够正常的使用

6 `" H. w) Q% Q4 Z

node4为controller

node5为computer和cinder

# b% [/ C( g, @: |9 P! @

一、环境准备，所有openstack主机都要操作

#所有openstack节点都添加主机名

#添加服务器主机名

echo '

& k, [* k6 K  E

10.0.0.11    node1

10.0.0.22    node2

3 y" t$ }' W/ W- o# h. Z* L

10.0.0.33    node3

3 k! ^* P  y% E

10.0.0.44    node4

10.0.0.55    node5

10.0.0.66    node6

'>>/etc/hosts

#创建ceph源

( Y& o/ _5 p. N2 l7 I$ W

echo '#阿里ceph源

[ceph]

name=ceph

! J/ v& f8 }; S& p9 C

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/

1 V$ n+ k0 h& l4 v

gpgcheck=0

[ceph-noarch]

( z+ D/ M% W; K0 d$ b) W2 b

name=cephnoarch

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/

gpgcheck=0

[ceph-source]

& @5 F/ h% v0 K

name=ceph-source

; {. g3 k. E& U; M2 Z

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/

# L& N. {8 g1 I1 u9 b

gpgcheck=0

* e( u5 c* S1 P( b% r

#'>/etc/yum.repos.d/ceph.repo

6 N) ]7 {. q0 m6 ?5 U, a+ Y3 @! c

yum clean all && yum makecache #生成缓存

#关闭selinux、防火墙

5 Q) s6 u3 {0 W0 A) h& \

systemctl stop firewalld.service

systemctl disable firewalld.service

firewall-cmd --state

" t3 |" l  Z" ~. S5 v5 O/ {3 I

sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config

sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config

3 j7 f4 z, N% h- {$ ]

grep --color=auto '^SELINUX' /etc/selinux/config

setenforce 0

3 {* [& K3 X1 j4 Q6 x. y3 ]; G

#将glance组件的节点安装ceph客户端

$ g. e4 \- u+ t" q( z; j- p  x

yum install python-rbd

#将nova和cinder组件的节点安装ceph客户端

" L- `: [! }6 q7 o. H

yum install ceph-common

#创建POOL

ceph osd pool create volumes 128

ceph osd pool create images 128

ceph osd pool create vms 128

& ^! ]2 m1 r3 a# E$ i

#初始化POOL

rbd pool init volumes

* I& ]6 L5 r$ L9 h: t, B3 F0 H/ P

rbd pool init images

5 l6 ~. J' ?  Z3 h+ H: \

rbd pool init vms

% ^5 M4 ^( v$ l

#将ceph的配置文件导入到各个openstack节点上

ssh node4 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf

, O. s9 R0 I& Q4 X8 b1 r/ V3 S

ssh node5 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf

$ ?% K' c7 I) L4 `  A1 y$ ^

#创建ceph用户和密钥

8 ~0 s2 D( p/ s2 m

ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images'

ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images'

ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups'

#查询用户，写入文件

ceph auth get-or-create client.glance | ssh node4  tee /etc/ceph/ceph.client.glance.keyring

ssh node4 chown glance:glance /etc/ceph/ceph.client.glance.keyring

- v2 \/ A! ~$ r% P8 C

ceph auth get-or-create client.cinder | ssh node5  tee /etc/ceph/ceph.client.cinder.keyring

ssh node5  chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring

, o# K" I4 K8 J1 O1 Z8 [8 e1 Q

ceph auth get-or-create client.cinder-backup | ssh node5  tee /etc/ceph/ceph.client.cinder-backup.keyring

ssh node5  chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring

& {5 Y7 W( Y" |% E

ceph auth get-key client.cinder | ssh node5 tee client.cinder.key

, D6 P9 T6 ~1 y* b0 x
3 M. A* e- Q; ~* ^1 b/ I

ceph与glance对接

#更改glance默认存储为ceph

3 G2 ?# j3 U! i. k# b0 {

#修改/etc/glance/glance-api.conf文件，添加一下内容

[DEFAULT]

default_store = rbd

/ f% I. y( l& r# o! Y

show_image_direct_url = True

[glance_store]

; @" o" n; ^6 p% q& H3 z8 {

#stores = file,http

+ w: J9 Z% M. K5 c/ @) Y

#default_store = file

* i* `6 S9 G2 @% s, H$ r( V3 v

#filesystem_store_datadir = /var/lib/glance/images/

stores = rbd

default_store = rbd

: d# }4 }8 _% r

rbd_store_pool = images

rbd_store_user = glance

3 j+ O+ z' h; e; ~! s! K5 {

rbd_store_ceph_conf = /etc/ceph/ceph.conf

rbd_store_chunk_size = 8

1 e- l8 B( M1 f

#重启服务

& \1 S! h( Q5 ?# T

systemctl restart openstack-glance-api openstack-glance-registry

* t' I" E* T5 M# t* Q; ~4 T# V1 o" Y

注意：在glance节点你需要查看/etc/ceph目录下是否有glance秘钥

ceph与nova对接，在computer节点上操作

' \' U4 r$ }+ t9 \

#密钥加进libvirt

uuidgen

cat > secret.xml <<EOF

<secret ephemeral='no' private='no'>

  <uuid>ff2e1190-30f5-4849-9c1c-886b1e1ee181</uuid>

  <usage type='ceph'>

- P5 p$ E& E! r- W& Q& \

    <name>client.cinder secret</name>

  </usage>

</secret>

EOF

virsh secret-define --file secret.xml

4 \7 H6 @. n9 O9 d" o# H

virsh secret-set-value --secret ff2e1190-30f5-4849-9c1c-886b1e1ee181 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml

virsh secret-list

' R" z% j8 [& G  O/ `' E

UUID                                  用量

' @- I! Y1 v5 I3 t

--------------------------------------------------------------------------------

* d1 O  I9 [' u; M8 }

ff2e1190-30f5-4849-9c1c-886b1e1ee181  ceph client.cinder secret

' t3 L8 z9 N" Y3 u8 p

在computer节点的ceph配置文件里添加一下内容

; w% d( ^" V9 @* I9 v; u

ls -l /etc/ceph/

#ceph

echo '

5 X1 `( G5 L8 l* R' w

[client]

rbd cache = true

rbd cache writethrough until flush = true

admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok

+ h  Z; t+ [, Z! i# V! j0 p

log file = /var/log/qemu/qemu-guest-$pid.log

$ F2 K' x4 E# O: V5 Q$ \

rbd concurrent management ops = 20

* ?0 ?& L% G( `6 P% i2 v0 {

[client.cinder]

4 e$ W( j0 n' o2 B5 v) V2 o& J3 a

keyring = /etc/ceph/ceph.client.cinder.keyring

'>>/etc/ceph/ceph.conf

: _1 c* M6 j- `' k, D  }

mkdir -p /var/run/ceph/guests/ /var/log/qemu/

% e$ `+ X4 C1 g9 W/ a, Q4 C

chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/

#使用ceph存储，在/etc/nova/nova.conf添加以下内容

[libvirt]

1 e- ^0 @. s" T$ P! W$ B/ \: r

virt_type = qemu

images_type = rbd

6 b, F9 y( w. z$ B1 |* t9 O

images_rbd_pool = vms

& L0 F- _- |9 ~9 w

images_rbd_ceph_conf = /etc/ceph/ceph.conf

rbd_user = cinder

rbd_secret_uuid = 'ff2e1190-30f5-4849-9c1c-886b1e1ee181'

" I2 ?6 b! E- t

disk_cachemodes="network=writeback"

3 G" J3 V6 u4 f

live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"

$ P! U- }( m( U. e& |/ s

libvirt_inject_password = false

8 L. M9 }" w5 U" I( N

libvirt_inject_key = false

/ y) u& q8 `) O$ O" ^( V/ j

libvirt_inject_partition = -2

3 A" h9 x8 a# ^. }  m

#重启服务

/ X4 q1 {+ m! V; G" _  N

systemctl restart libvirtd.service openstack-nova-compute.service

ceph与cinder对接

- t1 i6 y" b$ b! ~& m8 x

#修改/etc/cinder/cinder.conf配置文件，注意：将之前的lvm的配置都删掉

[DEFAULT]

( Y2 x2 f) E6 k' Z+ z! B% R

#enabled_backends = lvm

enabled_backends = ceph

2 W0 i/ c; t- R+ d
7 ?  o  n' g4 S. e

[ceph]

volume_driver = cinder.volume.drivers.rbd.RBDDriver

rbd_pool = volumes

& F* l" t1 ^; u

rbd_ceph_conf = /etc/ceph/ceph.conf

/ U5 ^. }+ _" |, |* N" l1 u

rbd_flatten_volume_from_snapshot = false

* c; l3 N; L0 w3 t; u8 E

rbd_max_clone_depth = 5

) F6 g3 _& x2 ^, ], k$ w

rbd_store_chunk_size = 4

rados_connect_timeout = -1

  R" ^5 L) D) A8 W2 C/ i4 @

glance_api_version = 2

rbd_user = cinder

rbd_secret_uuid = ff2e1190-30f5-4849-9c1c-886b1e1ee181

( v. L, c' u( m4 }9 e+ u- z  @" F
9 D6 t' @) e  [% c' r; R
8 W* \7 G9 w+ v! g' ~

#重启服务   

systemctl restart openstack-cinder-volume.service

& Z% Y9 a$ s8 G5 I

admin

1、创建一个POOL
Ceph的块设备默认使用“rdb” pool，但建议为Cinder和Glance创建专用的pool。
ceph osd pool create volumes 128
ceph osd pool create images 128
ceph osd pool create backups 128
, T. q* L- E% b) |- U+ }ceph osd pool create vms 128
( r" H* x: S7 z5 K
2、配置OPENSTACK CEPH CLIENTS
环境的准备，需要事先在ceph管理节点到openstack各服务节点间建立起免密钥登录的关系，且需有使用sudo的权限。
" f) \8 ?$ [4 Y, s4 o! X. l4 t
安装ceph客户端软件包：
% i1 z% R" m/ V2 m在glance-api节点：sudo yum install python-rbd
在nova-compute, cinder-backup and on the cinder-volume节点：sudo yum install  ceph （both the Python bindings and the client command line tools）
2 ?# P6 {2 x' f4 i% h. h
- Y/ e) B4 m! j8 {1 }在OpenStack中运行glance-api, cinder-volume, nova-compute ，cinder-backup服务的主机节点，都属于Ceph的客户端。需要配置ceph.conf.
使用以下命令把ceph.conf复制到每个ceph客户端节点：
ssh {your-openstack-server} sudo tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf

3、设置Ceph Client Authentication
如果启用了cephx认证，那么就需要为Nova/Cinder创建一个新用户：
2 E" R4 \' U* Q2 g$ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
- p) l8 D! P( K6 O3 J. P/ [: Q2 p$ }$ ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
* s2 Q/ L: J; y! W" j$ ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
+ a/ [6 O3 u3 ]4 F0 j" k, K# B! W+ h6 o
' E& y5 Y: G( W将密钥（client.cinder, client.glance, and client.cinder-backup）分发至对应的主机节点，并调整属主权限：
6 e" g: V1 c! E( P. N8 U2 V. wceph auth get-or-create client.glance | ssh {your-glance-api-server} sudo tee /etc/ceph/ceph.client.glance.keyring
ssh {your-glance-api-server} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
7 [: `9 F5 K& s+ @' S% r! t2 gceph auth get-or-create client.cinder | ssh {your-volume-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
ssh {your-cinder-volume-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder-backup | ssh {your-cinder-backup-server} sudo tee /etc/ceph/ceph.client.cinder-backup.keyring
ssh {your-cinder-backup-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring

+ _, f7 ]2 {7 ?& q: x7 ^# F. Z9 i8 E运行nova-compute的节点需要使用cinder密钥：
6 w$ t( |  `) e, ~$ b/ M  H7 r9 qceph auth get-or-create client.cinder | ssh {your-nova-compute-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
, G& C/ s; f4 Y7 E  z
! y/ H1 L: b9 v7 C7 Glibvirt同样也要使用client.cinder密钥：
Libvirt进程在从Cinder挂载一个块设备时，需要使用该密钥访问Ceph存储集群。
先要在运行nova-compute的节点上创建一个密钥的临时拷贝：
$ L7 F0 r( ~9 G) u( gceph auth get-key client.cinder | ssh {your-compute-node} tee client.cinder.key

2 b, l+ d0 E; |; z: c然后登录到compute节点上，将密钥增加到libvirt配置文件中并删除上面的临时文件：
$ uuidgen
22003ebb-0f32-400e-9584-fa90b6efd874
4 r% c" M2 b1 r. b9 [! M
8 ~6 V. e( M, D, gcat > secret.xml <<EOF
: \8 i* Q; r: W<secret ephemeral='no' private='no'>
9 f/ F9 O. E: S% P! L<uuid>22003ebb-0f32-400e-9584-fa90b6efd874</uuid>
: }2 D8 ]' f4 l6 W, g<usage type='ceph'>
<name>client.cinder secret</name>
! v5 C7 i7 o' @9 a- S8 Q! T# u</usage>
7 e: @/ D) T- g. c( S</secret>
EOF
# virsh secret-define --file secret.xml
# W' _0 g' Q! j#Secret 22003ebb-0f32-400e-9584-fa90b6efd874 created
2 c5 y( Z+ s) y) ^; _' A! A# virsh secret-set-value --secret 22003ebb-0f32-400e-9584-fa90b6efd874 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml
Secret value set
3 h. {# A* k% u* A- k
3 K0 m& s: d9 P: {; Z, I! X为了便于管理，建议在有多个计算节点时，在上在的操作中使用相同的UUID。
+ r4 z7 B. |2 z- u$ s# v1 n
4 `% j0 z7 Y. d  G
: V3 p7 S+ k6 F4 |4、设置Glance集成Ceph
JUNO，编辑/etc/glance/glance-api.conf：
/ D- `) j4 f) j7 Y# K! P# S  C[DEFAULT]
...
default_store = rbd
2 o3 q' q7 A+ @; S, K...
' @2 L, e" L6 I0 H/ p[glance_store]
6 T" V6 @% F2 V2 O' dstores = rbd
rbd_store_pool = images
3 V" E$ }. G1 g( c/ Xrbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_chunk_size = 8

* G  E) ^8 \8 r  z0 [# b启用copy-on-write功能：
在[DEFAULT]中增加以下参数，
4 w/ L4 U2 O; L3 l, ^show_image_direct_url = True

4 W5 g# J4 v: f$ p" {怎样关闭Glance的镜像缓存：
* U# I/ Q' ]' A3 ]4 G  Z; \修改以下参数如以下所示，
[paste_deploy]flavor = keystone

其它建议设置的Glance参数：
# N) a( I1 Y7 k# {$ W- hw_scsi_model=virtio-scsi: add the virtio-scsi controller and get better performance and support for discard operation
4 F* t" x! l; `, t7 H' r- hw_disk_bus=scsi: connect every cinder block devices to that controller
9 H0 y/ J' {4 Q( D0 n/ q- L- hw_qemu_guest_agent=yes: enable the QEMU guest agent
- os_require_quiesce=yes: send fs-freeze/thaw calls through the QEMU guest agent

, B7 ]8 d2 a1 `  x. b+ K6 A4 VOpenStack官网配置参考：
) W1 V+ K7 B# O- X; n+ Ohttp://docs.openstack.org/libert ... -image-service.html

5、设置Cinder集成Ceph
$ A! B$ w3 {; u. u编辑/etc/cinder/cinder.conf：
[DEFAULT]
. e& O1 K9 Q% p  d8 A  V8 p...
enabled_backends = ceph
) j( b8 j& G( B. B...
[ceph]
1 T6 L# y& e2 m' o  p0 lvolume_driver = cinder.volume.drivers.rbd.RBDDriver
3 r3 o+ w. G0 J+ h1 Qrbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
% v% b7 ^3 s0 [0 Erbd_store_chunk_size = 4
rados_connect_timeout = -1
& Q* I7 @7 o4 ~9 Uglance_api_version = 2
" C2 a2 d& i! X% ^7 g' e! C% h
在启用了cephx认证时，还需要配置认证信息：
[ceph]
4 \. I1 g" J0 K+ h6 L0 r...
3 C& z* J' \5 irbd_user = cinder
rbd_secret_uuid = 22003ebb-0f32-400e-9584-fa90b6efd874

+ p% [4 D  S* M注：如果你需要配置多个cinder back ends，一定要在 [DEFAULT] 部分设置glance_api_version = 2 。
% T  j. E9 c: z# u* h) T2 s
下表来自OpenStack官网Liberty的配置文档：
http://docs.openstack.org/libert ... ent/ceph-rados.html
/ L. ~1 ^8 l/ S& e5 I' t& \+ {Description of Ceph storage configuration options
Configuration option = Default value        Description
[DEFAULT]
rados_connect_timeout = -1        (IntOpt) Timeout value (in seconds) used when connecting to ceph cluster. If value < 0, no timeout is set and default librados value is used.
rados_connection_interval = 5        (IntOpt) Interval value (in seconds) between connection retries to ceph cluster.
rados_connection_retries = 3        (IntOpt) Number of retries if connection to ceph cluster failed.
rbd_ceph_conf =        (StrOpt) Path to the ceph configuration file
; o$ l) y8 }( drbd_cluster_name = ceph        (StrOpt) The name of ceph cluster
rbd_flatten_volume_from_snapshot = False        (BoolOpt) Flatten volumes created from snapshots to remove dependency from volume to snapshot
rbd_max_clone_depth = 5        (IntOpt) Maximum number of nested volume clones that are taken before a flatten occurs. Set to 0 to disable cloning.
rbd_pool = rbd        (StrOpt) The RADOS pool where rbd volumes are stored
$ s" N% r; L7 j" d+ s7 \2 B3 nrbd_secret_uuid = None        (StrOpt) The libvirt uuid of the secret for the rbd_user volumes
rbd_store_chunk_size = 4        (IntOpt) Volumes will be chunked into objects of this size (in megabytes).
rbd_user = None        (StrOpt) The RADOS client name for accessing rbd volumes - only set when using cephx authentication
volume_tmp_dir = None        (StrOpt) Directory where temporary image files are stored when the volume driver does not write them directly to the volume. Warning: this option is now deprecated, please use image_conversion_dir instead.
1 ^5 M/ A- v/ d# `! E

# `+ |) R; u" D: h6 M
6、设置Cinder Backup集成Ceph
$ \& J8 j8 m2 u5 Y+ d5 YOpenStack Cinder Backup需要一个专门的进程。在你的Cinder Backup节点上，编辑/etc/cinder/cinder.conf：
) U+ G$ P3 c# r. j( ]7 c' abackup_driver = cinder.backup.drivers.ceph
8 C! q. G6 a( z1 ]! dbackup_ceph_conf = /etc/ceph/ceph.conf
backup_ceph_user = cinder-backup
7 x0 W- L* ]/ X3 t* l7 e& V: hbackup_ceph_chunk_size = 134217728
backup_ceph_pool = backups
8 ?5 x" p; p  }: @; b/ w1 Dbackup_ceph_stripe_unit = 0
6 t) ]& b0 Q3 d% m* vbackup_ceph_stripe_count = 0
restore_discard_excess_bytes = true
" j" C* F/ v; Y. @6 D
/ I0 ~8 }4 O4 h以下是来自OpenStack官网对Cinder Backup集成Ceph的配置说明：
To enable the Ceph backup driver, include the following option in the cinder.conf file:
9 @; K& N2 u: R8 @( F+ @
: C- N& @% n, _. I6 t7 Zbackup_driver = cinder.backup.drivers.ceph
* n. }, D9 S* }0 _+ {; L0 FThe following configuration options are available for the Ceph backup driver.
/ ]/ q: \0 z) P& z. V
Table 2.52. Description of Ceph backup driver configuration options
Configuration option = Default value        Description
4 n7 d, S/ c! P  A/ I) U! _: J[DEFAULT]
% q+ D+ Q" l/ w& Xbackup_ceph_chunk_size = 134217728        (IntOpt) The chunk size, in bytes, that a backup is broken into before transfer to the Ceph object store.
backup_ceph_conf = /etc/ceph/ceph.conf        (StrOpt) Ceph configuration file to use.
backup_ceph_pool = backups        (StrOpt) The Ceph pool where volume backups are stored.
backup_ceph_stripe_count = 0        (IntOpt) RBD stripe count to use when creating a backup image.
5 [1 l5 n6 r5 M3 ]# q, M/ `# ~backup_ceph_stripe_unit = 0        (IntOpt) RBD stripe unit to use when creating a backup image.
3 V; i. F" c9 M5 O' ?backup_ceph_user = cinder        (StrOpt) The Ceph user to connect with. Default here is to use the same user as for Cinder volumes. If not using cephx this should be set to None.
3 u  i/ u7 o7 n9 V1 ^8 ^restore_discard_excess_bytes = True        (BoolOpt) If True, always discard excess bytes when restoring volumes i.e. pad with zeroes.
This example shows the default options for the Ceph backup driver.

! \* w. q" l2 Tbackup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user = cinder
backup_ceph_chunk_size = 134217728
backup_ceph_pool = backups
backup_ceph_stripe_unit = 0
backup_ceph_stripe_count = 0

8 i  P; e5 }# L, {7、设置NOVA集成Ceph
; N! r2 W8 N1 Y% Y( z为了直接基于Ceph存储启动虚机，还需要为Nova配置一个临时的存储后端。同时，建议使用RBD缓存，启用admin socket。
admin socket可以通过以下方法访问：
     ceph daemon /var/run/ceph/ceph-client.cinder.19195.32310016.asok help
% O$ a/ A$ s" F
在你的每个compute节点上，编辑Ceph配置文件：
: Q. b! S. r' S[client]
rbd cache = true
3 C. N- T4 b+ m5 Erbd cache writethrough until flush = true
admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
( V- a1 n0 F$ i7 t( Wlog file = /var/log/qemu/qemu-guest-$pid.log
  o3 ]3 g" o$ s. d) Zrbd concurrent management ops = 20
1 H, q( e# C2 `9 w3 P8 v
调整权限：
' F( G: e! ?' Gmkdir -p /var/run/ceph/guests/ /var/log/qemu/
chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/
  Z6 w7 w. e$ A; s) U% a
3 @& c) y$ b1 d7 e! U+ z注：以上的qemu用户和libvirt组是基于RedHat相关系统的。
7 k- a9 c) E: ^& o' R' A
9 t2 T' i* E- K& B: u以配置好后，如果虚机已经在运行，则可以重启使上面的配置生效。

JUNO
在每个compute节点上编辑/etc/nova/nova.conf文件：
: Y9 f8 _* ~6 n8 U' G/ l9 n7 c9 i/ a8 z[libvirt]
6 l  t( J6 ~$ @2 Q# Nimages_type = rbd
images_rbd_pool = vms
7 c. F0 h# C+ A2 @3 `" H! V" {  Zimages_rbd_ceph_conf = /etc/ceph/ceph.conf
7 s: ?+ C4 f" f. ^rbd_user = cinder
! n& ?$ b; M. v  Arbd_secret_uuid = 22003ebb-0f32-400e-9584-fa90b6efd874
disk_cachemodes="network=writeback"
) D% g& e8 i5 [5 J1 N$ K1 W
建议关闭nova的密钥注入功能，而是使用基于metadata服务、cloud-init实现类似功能：
( O6 ]  \* e, K在每个计算节点上，编辑/etc/nova/nova.conf：
inject_password = false
inject_key = false
inject_partition = -2

启动热迁移支持：
在[libvirt]部分增加：
live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
" B% q2 \7 h! E: y1 Y8 k7 J
# W6 z$ U7 X3 o  r( I8、重启OpenStack服务
sudo service openstack-glance-api restart
( }$ d! |0 a, ?' v6 g% @1 W/ msudo service openstack-nova-compute restart
sudo service openstack-cinder-volume restart
sudo service openstack-cinder-backup restart
. R2 [; v2 I2 X/ {