最新Ceph 版与openstack Pike对接

admin

一、在所有服务器上操作

2 @$ a' d  P0 y, Z6 N

#使用阿里源

2 M; w  x- I+ p  D( U

yum install wget -y

" X' c8 x, y% x0 z; @+ H1 P3 M* B# H

rm -f /etc/yum.repos.d/*

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

* Y6 ^# v5 O9 r: N7 _. V. D" m

sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址

#创建ceph源

" t4 i. F: ?! B0 J5 ]; o

echo '#阿里ceph源

[ceph]

" H6 V$ \* m& S

name=ceph

; q" b1 J; }6 m' j

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/

gpgcheck=0

[ceph-noarch]

. x) l3 x8 {3 w9 ~5 W& o

name=cephnoarch

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/

gpgcheck=0

[ceph-source]

, m) \! ?2 u/ }0 w1 C" a9 c* v

name=ceph-source

! G8 Z0 W5 G" D0 _

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/

& L/ A9 k. E0 y& [) M+ r

gpgcheck=0

#'>/etc/yum.repos.d/ceph.repo

+ r4 r8 S% h) c4 S

yum clean all && yum makecache #生成缓存

#关闭selinux、防火墙

systemctl stop firewalld.service

systemctl disable firewalld.service

firewall-cmd --state

3 [- b. o+ K. D" M3 `8 ?

sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config

$ ]+ t2 j2 |% P3 N! `0 Z% Y: F

sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config

) Y) t2 |0 L% _5 e7 Y5 l$ p

grep --color=auto '^SELINUX' /etc/selinux/config

setenforce 0

4 ?6 _8 A$ V# [
' E: w; }) Z2 H

二、只在node1上操作

三台服务器分别为node1 node2 node3   其中我们在node1上面安装ceph官方的自动化部署工具，也就是说我们只需要操作node1即可完成部署

) ^/ E' o+ F- p: I  A" _' F4 A2 K

#添加服务器主机名

echo '

10.0.0.11    node1

# a9 e5 L9 |: m( x6 T1 H; j1 l

10.0.0.22    node2

! t+ Q0 C. ~/ N; `! F7 D. N

10.0.0.33    node3

10.0.0.44    node4

10.0.0.55    node5

2 K4 z6 P  M3 v$ u# r: t( M

10.0.0.66    node6

1 ^, l2 l- F- j" k; l

'>>/etc/hosts

#创建秘钥

ssh-keygen

, _6 P+ L6 h# w+ N2 _7 t: u4 N

ssh-copy node2

ssh-copy node3

! G3 D, C2 \% l/ _! s
% a+ s& k& [/ Y7 u6 s9 @& G" D" \

#将node1的hosts文件发送到其他服务器上面

/ ^  G% R' m! R6 C; y2 k

scp /etc/hosts node2:/etc

- a; b9 [1 w5 c/ ~/ }! C

scp /etc/hosts node3:/etc

% \$ ^1 f; o: W$ }% l: Y

三、开始在node1上安装ceph

) [. F$ v2 W8 q, h; l+ p" \- B& N

#安装ceph-deploy配置工具

7 C* S' `4 F9 `5 G& y5 p# s/ ^

yum install -y ceph-deploy

yum install python-setuptools

$ Y6 Q5 k5 s( |; a& D
- z( e  i# O3 W4 b; |# I

#创建配置目录

mkdir /etc/ceph

2 N* G  F7 Y, Z7 U

cd /etc/ceph/

' r2 H& X. D- u0 M) M+ `& x9 M
) z  z0 `" b8 ]

#初始化Mon配置

0 q" w9 ~6 X: |2 e

ceph-deploy new node1

  |6 }1 R9 @* A5 a! Y

# #修改冗余份数为2，日志大小2G

* b2 T6 R4 }, E! X( C. |

# #配置网络,单网卡忽略

$ b$ U$ a( ^  C5 q

echo '

- Z' D  \2 @, U

public network = 10.0.0.0/24

cluster network = 10.0.0.0/24

+ W6 l" X- z/ p7 q* A: Q9 ^1 J9 V

mon_clock_drift_allowed = 2   

osd_journal_size = 4086

osd_pool_default_pg_num = 128

6 n( ^4 X1 q! B

osd_pool_default_pgp_num = 128

- {, A1 s( \9 U

osd pool default size = 2

3 Y0 W' G- R& M; `

osd pool default min size = 1

# V: f  P+ S3 U9 b

rbd_default_features = 1

, k! @0 g' p3 S) y: g( w/ P

client_quota = true

: P6 O7 C5 ?* R+ W1 e+ A: a

'>>./ceph.conf

! @" s/ H9 @+ X0 \, ]" r$ t

#安装Ceph

ceph-deploy install node1 node2 node3

* }4 j! J( u$ q$ Q

#yum install -y ceph ceph-radosgw #实际上是安装这2个rpm

4 h4 w' e6 X' Q" X8 B

#初始化monitor和key

cd /etc/ceph/

0 X; l. t1 r. z2 k

ceph-deploy --overwrite-conf mon create-initial

#创建osd

ceph-deploy osd create --data /dev/sdb node1

8 i" E7 M3 a! g

ceph-deploy osd create --data /dev/sdc node1

. J+ z- y" R0 Q/ |* n8 j+ |

ceph-deploy osd create --data /dev/sdd node1

ceph-deploy osd create --data /dev/sdb node2

! K, r* d3 w, U8 }

ceph-deploy osd create --data /dev/sdc node2

ceph-deploy osd create --data /dev/sdd node2

ceph-deploy osd create --data /dev/sdb node3

ceph-deploy osd create --data /dev/sdc node3

ceph-deploy osd create --data /dev/sdd node3

#拷贝配置及密钥

ceph-deploy admin node1 node2 node3

3 s' z% [1 K# ^

chmod 644 /etc/ceph/ceph.client.admin.keyring

# ?; x5 W+ [& K) Z+ n* u7 @2 w1 r5 v

#创建mon

( C1 s" g5 u; ^/ h2 V& Y

ceph-deploy --overwrite-conf mon create node1

# Z" o6 a$ {. m+ T+ Q$ e. k

ceph-deploy --overwrite-conf admin node1

7 [: d; _# w' ?( w. Z" o

#添加mgr

% r6 `$ Y- f% Z  H

#ceph 12开始，monitor必须添加mgr

1 a+ h( r* ^) |/ g# z4 h+ M) ^

ceph-deploy mgr create node1:mon_mgr

#启用dashboard (在node1节点)

ceph mgr module enable dashboard

7 d" v; Q0 k+ M0 r; I

web访问http://10.0.0.11:7000   即可看到ceph的运行情况

. `3 [3 w& S5 H9 I* Y9 t) A
0 e1 d/ j8 m& \! Q5 d9 |0 M7 M" B8 U

#查看相关命令

ceph health

ceph -s

8 ]$ o6 i4 B7 R  h

ceph osd tree

" H7 q$ m9 Y5 c  n6 ~/ n6 w
& p1 u- F- C8 [; y, b$ u( v: u

ceph df

& c- ~4 k/ W/ R8 G6 U1 G

ceph mon stat

$ o9 w4 a2 L, L. z8 W

ceph osd stat

ceph pg stat

  h' W# ]0 J7 p* p
* x( `# q1 A2 `; S6 f" E+ ~( l. w

ceph osd lspools

& W1 D$ H( g; [

ceph auth list

1 a3 k( ?6 Z- c2 Z! B: p( V

Ceph L版与openstack Pike对接

; I8 E( S* Z9 P0 F- x+ u

环境opestack Pike的controller、computer、cinder三个节点已经安装完毕，并且能够正常的使用

( t3 f$ K0 `, }; D+ A' J

node4为controller

node5为computer和cinder

一、环境准备，所有openstack主机都要操作

#所有openstack节点都添加主机名

' Z7 w' ~7 i" D( G4 `- s5 y. ~3 t5 F

#添加服务器主机名

echo '

10.0.0.11    node1

10.0.0.22    node2

10.0.0.33    node3

* k' N5 @) J5 k& R

10.0.0.44    node4

10.0.0.55    node5

- n4 ^* J5 n* I" G# |& w$ G

10.0.0.66    node6

'>>/etc/hosts

6 }  m7 n1 l$ G+ J) T: ~+ s

#创建ceph源

9 }7 t! j7 H) Z9 Z7 @5 M) f

echo '#阿里ceph源

[ceph]

name=ceph

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/

gpgcheck=0

[ceph-noarch]

name=cephnoarch

% \5 {  J: E# u+ {) z6 N- _& ?  t# O

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/

- T& I+ D" v: O: u) @! X; g3 t

gpgcheck=0

. D' C, ]+ }2 [6 z2 t& V! f

[ceph-source]

name=ceph-source

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/

) ~; k  e1 A" _

gpgcheck=0

#'>/etc/yum.repos.d/ceph.repo

* q9 f) C+ q  }' s

yum clean all && yum makecache #生成缓存

2 L$ m- t+ M7 L

#关闭selinux、防火墙

) ~/ K( f  t+ \3 m

systemctl stop firewalld.service

systemctl disable firewalld.service

firewall-cmd --state

sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config

sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config

% W8 Y/ B5 ^5 Q

grep --color=auto '^SELINUX' /etc/selinux/config

/ s& X3 Q  }# ^* y3 m

setenforce 0

' j2 U. b; K' _7 m8 `( J% W1 C
9 x6 d$ L- V% H1 s; q2 b5 m

#将glance组件的节点安装ceph客户端

yum install python-rbd

; ^: M, U& v6 }

#将nova和cinder组件的节点安装ceph客户端

0 ]6 ~2 d7 k* G

yum install ceph-common

1 d: b1 I$ _' P6 f) _9 F4 h
9 Y- ^, l$ j- f8 d

#创建POOL

ceph osd pool create volumes 128

ceph osd pool create images 128

; Z6 L; p+ h5 {2 S" x: E2 f

ceph osd pool create vms 128

# f8 \! ?  \0 W( @7 l/ t

#初始化POOL

& F% g' r0 l- w, ^, `2 G1 ]

rbd pool init volumes

/ X( K' l" x! L7 l

rbd pool init images

rbd pool init vms

' ]: X6 z* L9 o. I0 n9 M3 x! U

#将ceph的配置文件导入到各个openstack节点上

$ _2 O1 c  U  t+ K

ssh node4 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf

ssh node5 tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf

0 D% t7 n1 _6 a% K2 e
; r5 W( v6 J3 U$ Q2 s% g

#创建ceph用户和密钥

ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images'

ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images'

& e: R) D0 H. F$ _/ L; J4 R

ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups'

( w9 W+ D. T0 E! T2 E" A; t

#查询用户，写入文件

ceph auth get-or-create client.glance | ssh node4  tee /etc/ceph/ceph.client.glance.keyring

ssh node4 chown glance:glance /etc/ceph/ceph.client.glance.keyring

) D3 K$ r& }5 T" W2 @

ceph auth get-or-create client.cinder | ssh node5  tee /etc/ceph/ceph.client.cinder.keyring

ssh node5  chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring

ceph auth get-or-create client.cinder-backup | ssh node5  tee /etc/ceph/ceph.client.cinder-backup.keyring

7 Z# z7 n/ m6 m4 m1 B

ssh node5  chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring

2 t8 U+ t) t/ E: O

ceph auth get-key client.cinder | ssh node5 tee client.cinder.key

$ i2 j2 ]) i" H8 K  M, Z4 _. l

ceph与glance对接

; w! h* c5 ]) P* q8 m

#更改glance默认存储为ceph

#修改/etc/glance/glance-api.conf文件，添加一下内容

6 }% x" W5 b* n( q

[DEFAULT]

! g2 \( e# X% [/ w3 m8 P( C* P

default_store = rbd

9 O' l2 K* _1 \3 W& q3 ]- C6 k

show_image_direct_url = True

% ^' t9 _# Z( r; _' U
2 R' K2 k# M' }" b3 k3 N

[glance_store]

#stores = file,http

#default_store = file

! B9 J( C! D* `8 B3 t

#filesystem_store_datadir = /var/lib/glance/images/

stores = rbd

default_store = rbd

rbd_store_pool = images

rbd_store_user = glance

$ q) ?) i  V" F( N" M) F" B8 [

rbd_store_ceph_conf = /etc/ceph/ceph.conf

  Z1 T4 Q8 Y9 T8 a4 Z3 \5 h: ^) h

rbd_store_chunk_size = 8

3 J0 ]/ T  S" Q0 T1 D6 d

#重启服务

systemctl restart openstack-glance-api openstack-glance-registry

2 |" n; E' j' I* L! [, _5 w

注意：在glance节点你需要查看/etc/ceph目录下是否有glance秘钥

/ `3 M3 t# J; T- C4 t( E; b; `
) Z5 W, I$ O% X0 |& \' v& r

ceph与nova对接，在computer节点上操作

#密钥加进libvirt

uuidgen

  ^2 u( N& I. i1 t/ X" \& n

cat > secret.xml <<EOF

<secret ephemeral='no' private='no'>

% F: ^( Y+ Y0 W$ r

  <uuid>ff2e1190-30f5-4849-9c1c-886b1e1ee181</uuid>

  <usage type='ceph'>

: m) K$ P4 x4 s7 r% {

    <name>client.cinder secret</name>

  </usage>

</secret>

EOF

; Y3 }+ j  r. V

virsh secret-define --file secret.xml

& A/ K4 `: d1 w0 b5 A

virsh secret-set-value --secret ff2e1190-30f5-4849-9c1c-886b1e1ee181 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml

virsh secret-list

UUID                                  用量

/ V4 z6 E7 V$ j6 \* P& Q. n

--------------------------------------------------------------------------------

ff2e1190-30f5-4849-9c1c-886b1e1ee181  ceph client.cinder secret

, ?3 g5 o; ^& ~8 s1 p' L9 c

在computer节点的ceph配置文件里添加一下内容

ls -l /etc/ceph/

7 L% ^$ C& g( r+ W$ S

#ceph

echo '

[client]

0 W' G3 }, g0 o8 r  R

rbd cache = true

rbd cache writethrough until flush = true

8 N- H) J- {4 \+ A

admin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok

( W9 X* v9 X2 ]6 x2 S

log file = /var/log/qemu/qemu-guest-$pid.log

rbd concurrent management ops = 20

[client.cinder]

keyring = /etc/ceph/ceph.client.cinder.keyring

6 u) u& {3 Y1 [6 _# L  y

'>>/etc/ceph/ceph.conf

+ Z. E9 N, [% I2 ^
! L8 {% |# B' j" o! N2 t* H

mkdir -p /var/run/ceph/guests/ /var/log/qemu/

; S; e5 D/ L' B9 b/ q3 Q0 N

chown qemu:libvirt /var/run/ceph/guests /var/log/qemu/

4 s) z9 t0 s8 i0 w8 v  U

#使用ceph存储，在/etc/nova/nova.conf添加以下内容

[libvirt]

8 \2 R0 Y/ k1 U1 y

virt_type = qemu

2 B! ~  f! m' ?' T

images_type = rbd

2 G, ~, ]/ G! K! r. \- B! M3 i+ h

images_rbd_pool = vms

images_rbd_ceph_conf = /etc/ceph/ceph.conf

) n0 H* O( B' O" u

rbd_user = cinder

! b4 m. Y- b! w( I1 l

rbd_secret_uuid = 'ff2e1190-30f5-4849-9c1c-886b1e1ee181'

disk_cachemodes="network=writeback"

live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"

libvirt_inject_password = false

libvirt_inject_key = false

$ R: L5 D+ ^' V1 u

libvirt_inject_partition = -2

/ x9 v0 ~1 K7 o1 g3 F
$ T6 V0 x5 |: e

#重启服务

systemctl restart libvirtd.service openstack-nova-compute.service

  [( h. F# k  c& S. p  T9 M

ceph与cinder对接

; y( X% X9 D! @1 }! P* J1 f& d
7 o/ \# c, p8 n4 F

#修改/etc/cinder/cinder.conf配置文件，注意：将之前的lvm的配置都删掉

) x% A3 a- Q' @! D. G

[DEFAULT]

; m8 V! ?; f' R& Y. h) L

#enabled_backends = lvm

5 H1 B. ~6 G: G# A; d: H

enabled_backends = ceph

  y4 {) }: b+ d0 P

[ceph]

volume_driver = cinder.volume.drivers.rbd.RBDDriver

rbd_pool = volumes

+ c) [6 g1 x3 p4 {4 P! ~

rbd_ceph_conf = /etc/ceph/ceph.conf

rbd_flatten_volume_from_snapshot = false

6 w/ I8 `$ u2 h$ j

rbd_max_clone_depth = 5

8 l0 i5 J$ J  J

rbd_store_chunk_size = 4

! j1 r$ `: ~2 ~  z) P4 D3 j. d

rados_connect_timeout = -1

) i- {  g0 ^# R& ]& r

glance_api_version = 2

rbd_user = cinder

" W9 z4 C4 }/ G/ ]( U. C( U6 ?& a

rbd_secret_uuid = ff2e1190-30f5-4849-9c1c-886b1e1ee181

: a! Z; C, Z3 K" w, s) u

#重启服务   

systemctl restart openstack-cinder-volume.service

8 X! K9 e1 C( Y3 J# S$ P2 j$ `

admin

1、创建一个POOL
* [# A6 }, Q. g! B% {Ceph的块设备默认使用“rdb” pool，但建议为Cinder和Glance创建专用的pool。
ceph osd pool create volumes 128
ceph osd pool create images 128
3 c: h+ Q, B5 k2 m: P+ W) C7 l: i: X2 D! Iceph osd pool create backups 128
2 D( m% q. ~0 Fceph osd pool create vms 128

2、配置OPENSTACK CEPH CLIENTS
环境的准备，需要事先在ceph管理节点到openstack各服务节点间建立起免密钥登录的关系，且需有使用sudo的权限。
( r9 T: K. h3 f
$ f& I& H" ]6 ]9 R- T) `2 ^4 Y安装ceph客户端软件包：
5 m) c, g& {5 a# k% [" D4 K/ p在glance-api节点：sudo yum install python-rbd
在nova-compute, cinder-backup and on the cinder-volume节点：sudo yum install  ceph （both the Python bindings and the client command line tools）

& ^8 f# t) }6 q  F+ n在OpenStack中运行glance-api, cinder-volume, nova-compute ，cinder-backup服务的主机节点，都属于Ceph的客户端。需要配置ceph.conf.
# l8 W% W# K% v. o  |使用以下命令把ceph.conf复制到每个ceph客户端节点：
$ B* S8 ^% P4 w* v& _/ P, Zssh {your-openstack-server} sudo tee /etc/ceph/ceph.conf </etc/ceph/ceph.conf
; T  m- `) I+ q' {: p/ s9 ~
3、设置Ceph Client Authentication
4 w% u1 O/ U& o/ ?如果启用了cephx认证，那么就需要为Nova/Cinder创建一个新用户：
5 d! \  o& S+ `( L$ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
$ ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'
8 N7 q. ~6 w2 N& Z1 J4 k& Y$ ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'

将密钥（client.cinder, client.glance, and client.cinder-backup）分发至对应的主机节点，并调整属主权限：
9 a' `# D9 [; z4 }% mceph auth get-or-create client.glance | ssh {your-glance-api-server} sudo tee /etc/ceph/ceph.client.glance.keyring
& B7 z( p, n, fssh {your-glance-api-server} sudo chown glance:glance /etc/ceph/ceph.client.glance.keyring
* i5 Z" ^* F  oceph auth get-or-create client.cinder | ssh {your-volume-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
5 u( Y6 h/ a% P" [; l5 b2 Jssh {your-cinder-volume-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder.keyring
ceph auth get-or-create client.cinder-backup | ssh {your-cinder-backup-server} sudo tee /etc/ceph/ceph.client.cinder-backup.keyring
0 ]6 E& e/ L3 Q' Essh {your-cinder-backup-server} sudo chown cinder:cinder /etc/ceph/ceph.client.cinder-backup.keyring
3 Q* p9 h# k* f  v0 `% x
1 n2 X! R) M! T# X) Y* U+ c' |! ?# u运行nova-compute的节点需要使用cinder密钥：
ceph auth get-or-create client.cinder | ssh {your-nova-compute-server} sudo tee /etc/ceph/ceph.client.cinder.keyring
1 I; o: _4 k$ H, A2 w4 i
libvirt同样也要使用client.cinder密钥：
: V! L$ h) P5 M" L. D5 VLibvirt进程在从Cinder挂载一个块设备时，需要使用该密钥访问Ceph存储集群。
先要在运行nova-compute的节点上创建一个密钥的临时拷贝：
2 N' m* C. f' g" S/ ~3 ?ceph auth get-key client.cinder | ssh {your-compute-node} tee client.cinder.key

8 I4 J9 ~0 m% n) v  @1 i% w* A# n: b然后登录到compute节点上，将密钥增加到libvirt配置文件中并删除上面的临时文件：
+ r1 P: Y; `- [( n6 \$ uuidgen
22003ebb-0f32-400e-9584-fa90b6efd874
- p. Y5 K% X$ X
cat > secret.xml <<EOF
<secret ephemeral='no' private='no'>
<uuid>22003ebb-0f32-400e-9584-fa90b6efd874</uuid>
<usage type='ceph'>
<name>client.cinder secret</name>
& Q: f3 ~% ?5 Y% v</usage>
6 l  ^& d. Y2 C( U/ [/ f</secret>
+ E4 h/ R3 }! W) ^3 {, T8 dEOF
# virsh secret-define --file secret.xml
#Secret 22003ebb-0f32-400e-9584-fa90b6efd874 created
# virsh secret-set-value --secret 22003ebb-0f32-400e-9584-fa90b6efd874 --base64 $(cat client.cinder.key) && rm client.cinder.key secret.xml
Secret value set

0 `, M3 _2 {' u5 a! S- O: C为了便于管理，建议在有多个计算节点时，在上在的操作中使用相同的UUID。


5 G- ]5 S& L9 P4、设置Glance集成Ceph
JUNO，编辑/etc/glance/glance-api.conf：
; Z) W0 ?- u6 M& c. \[DEFAULT]
...
" M3 Z# }) b( c0 qdefault_store = rbd
...
# e7 P. o2 N+ ], o- R[glance_store]
6 |0 c7 y/ \, e6 W% vstores = rbd
6 M5 g5 g/ p- x3 \rbd_store_pool = images
rbd_store_user = glance
! W4 \1 }4 \" N+ M" n. g( M7 Prbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_chunk_size = 8

启用copy-on-write功能：
# u: y/ Z" ^& J( X* z; G1 ?: R在[DEFAULT]中增加以下参数，
show_image_direct_url = True

怎样关闭Glance的镜像缓存：
" j* R5 x3 `+ D2 [修改以下参数如以下所示，
6 ~0 O8 `% i% \+ _[paste_deploy]flavor = keystone
% C0 l& M9 B2 Q+ G
其它建议设置的Glance参数：
- hw_scsi_model=virtio-scsi: add the virtio-scsi controller and get better performance and support for discard operation
- hw_disk_bus=scsi: connect every cinder block devices to that controller
/ u/ f, [  W+ \4 D/ V* Y, n- K2 G- hw_qemu_guest_agent=yes: enable the QEMU guest agent
- os_require_quiesce=yes: send fs-freeze/thaw calls through the QEMU guest agent
1 T4 k7 |: h! ~  L, U" I
6 @: i* a4 w/ J/ O. w# GOpenStack官网配置参考：
http://docs.openstack.org/libert ... -image-service.html
; q. s+ ?/ ?# a, W5 h( U. W& N+ O
: t# S6 ^+ n3 E/ e0 ]: ~% E2 k. Z; e5、设置Cinder集成Ceph
' B4 g2 F1 a+ Q, ^编辑/etc/cinder/cinder.conf：
6 j7 k) ^  [! m! Z/ Y" @: _[DEFAULT]
...
' `; b$ Z( C% [. x# Eenabled_backends = ceph
...
[ceph]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
& x; `/ [" D) p1 m- p8 X. Zrbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
4 Y/ }# e% \* L& L( M) Orbd_store_chunk_size = 4
6 J7 Y2 i3 ~1 m3 `rados_connect_timeout = -1
$ u6 \; p. Y1 {* o' ~glance_api_version = 2
& E) C% O8 J7 ^: t3 U% x$ z
在启用了cephx认证时，还需要配置认证信息：
7 F  a; k8 P, Z[ceph]
...
- N. x3 X! c) Y0 Xrbd_user = cinder
rbd_secret_uuid = 22003ebb-0f32-400e-9584-fa90b6efd874

注：如果你需要配置多个cinder back ends，一定要在 [DEFAULT] 部分设置glance_api_version = 2 。

下表来自OpenStack官网Liberty的配置文档：
2 V' h* c5 {" m( E; u; }7 {% O. N  hhttp://docs.openstack.org/libert ... ent/ceph-rados.html
Description of Ceph storage configuration options
Configuration option = Default value        Description
[DEFAULT]
rados_connect_timeout = -1        (IntOpt) Timeout value (in seconds) used when connecting to ceph cluster. If value < 0, no timeout is set and default librados value is used.
rados_connection_interval = 5        (IntOpt) Interval value (in seconds) between connection retries to ceph cluster.
+ d2 C  Y, C" V5 T8 `) b5 s( arados_connection_retries = 3        (IntOpt) Number of retries if connection to ceph cluster failed.
rbd_ceph_conf =        (StrOpt) Path to the ceph configuration file
! `4 H- ^9 v% x) crbd_cluster_name = ceph        (StrOpt) The name of ceph cluster
* `: X* v) A, d( h& Vrbd_flatten_volume_from_snapshot = False        (BoolOpt) Flatten volumes created from snapshots to remove dependency from volume to snapshot
rbd_max_clone_depth = 5        (IntOpt) Maximum number of nested volume clones that are taken before a flatten occurs. Set to 0 to disable cloning.
rbd_pool = rbd        (StrOpt) The RADOS pool where rbd volumes are stored
  `6 R3 W4 Y0 H% v4 Z* g7 T4 Yrbd_secret_uuid = None        (StrOpt) The libvirt uuid of the secret for the rbd_user volumes
& q$ y5 j9 e) b& \/ }rbd_store_chunk_size = 4        (IntOpt) Volumes will be chunked into objects of this size (in megabytes).
rbd_user = None        (StrOpt) The RADOS client name for accessing rbd volumes - only set when using cephx authentication
volume_tmp_dir = None        (StrOpt) Directory where temporary image files are stored when the volume driver does not write them directly to the volume. Warning: this option is now deprecated, please use image_conversion_dir instead.



7 C# k& P5 f" F: W8 v: _+ q) T6、设置Cinder Backup集成Ceph
OpenStack Cinder Backup需要一个专门的进程。在你的Cinder Backup节点上，编辑/etc/cinder/cinder.conf：
backup_driver = cinder.backup.drivers.ceph
backup_ceph_conf = /etc/ceph/ceph.conf
( [8 O8 b2 _0 c2 |' nbackup_ceph_user = cinder-backup
backup_ceph_chunk_size = 134217728
backup_ceph_pool = backups
backup_ceph_stripe_unit = 0
9 Q& E) |0 m, j3 W; Ubackup_ceph_stripe_count = 0
* J/ \/ s6 p+ K7 X  ^restore_discard_excess_bytes = true

以下是来自OpenStack官网对Cinder Backup集成Ceph的配置说明：
To enable the Ceph backup driver, include the following option in the cinder.conf file:

backup_driver = cinder.backup.drivers.ceph
+ Y! l2 a; X# B) X1 }1 i7 V* i$ wThe following configuration options are available for the Ceph backup driver.
/ i' U* p) @5 }
Table 2.52. Description of Ceph backup driver configuration options
% Y/ I* U$ p& `: \3 h" k' D. }Configuration option = Default value        Description
[DEFAULT]
backup_ceph_chunk_size = 134217728        (IntOpt) The chunk size, in bytes, that a backup is broken into before transfer to the Ceph object store.
9 D; \9 v6 W- @5 B- vbackup_ceph_conf = /etc/ceph/ceph.conf        (StrOpt) Ceph configuration file to use.
backup_ceph_pool = backups        (StrOpt) The Ceph pool where volume backups are stored.
backup_ceph_stripe_count = 0        (IntOpt) RBD stripe count to use when creating a backup image.
1 K: [" V" r; |backup_ceph_stripe_unit = 0        (IntOpt) RBD stripe unit to use when creating a backup image.
) s, y" @/ S7 ~/ |2 E3 r: T3 u$ nbackup_ceph_user = cinder        (StrOpt) The Ceph user to connect with. Default here is to use the same user as for Cinder volumes. If not using cephx this should be set to None.
4 z( v( l9 O! crestore_discard_excess_bytes = True        (BoolOpt) If True, always discard excess bytes when restoring volumes i.e. pad with zeroes.
  N3 s3 y8 a0 @# P# l! @4 N5 Z2 o3 bThis example shows the default options for the Ceph backup driver.
1 w  `" L) u; ^* F7 N/ L6 e$ P
! [- O! q7 O2 b6 Dbackup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user = cinder
; [- S$ ~7 T$ {; x" A7 ?; Bbackup_ceph_chunk_size = 134217728
/ x7 t. P: E6 P$ ^; [6 Ibackup_ceph_pool = backups
backup_ceph_stripe_unit = 0
+ T4 b' b/ [: c2 B+ D0 N8 ibackup_ceph_stripe_count = 0

7、设置NOVA集成Ceph
为了直接基于Ceph存储启动虚机，还需要为Nova配置一个临时的存储后端。同时，建议使用RBD缓存，启用admin socket。
admin socket可以通过以下方法访问：
( y% E! w$ ^9 M* Z3 g3 {. g     ceph daemon /var/run/ceph/ceph-client.cinder.19195.32310016.asok help
. r8 ~# m3 ^" f1 J# b6 y, w4 H
在你的每个compute节点上，编辑Ceph配置文件：
( X, C; |+ j6 T+ y. e' F0 n[client]
rbd cache = true
rbd cache writethrough until flush = true
! j9 C7 n$ U$ T7 S! q0 badmin socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
( A8 y. w+ {8 W1 C2 Y  a2 Blog file = /var/log/qemu/qemu-guest-$pid.log
* J. v0 {$ Y& X9 t/ F2 ^rbd concurrent management ops = 20
0 v- S- v3 e! P- S2 g+ Y- o  S
7 W' D3 ~+ ]3 X' Q调整权限：
# N( C% ]  k1 ]; _6 j0 s  Cmkdir -p /var/run/ceph/guests/ /var/log/qemu/
1 g1 d6 l- Z8 dchown qemu:libvirt /var/run/ceph/guests /var/log/qemu/

注：以上的qemu用户和libvirt组是基于RedHat相关系统的。
; z! |1 f) N0 i4 @! T" M4 i" g7 W
2 u4 H+ r8 o) V以配置好后，如果虚机已经在运行，则可以重启使上面的配置生效。

JUNO
: @3 f% W! M$ d4 b4 v* v在每个compute节点上编辑/etc/nova/nova.conf文件：
! r! Y; K& v( d3 J[libvirt]
( e8 ~# ]! o; G/ K- o2 A1 |images_type = rbd
  C0 ^1 i) l4 |( O! _( A$ yimages_rbd_pool = vms
7 ~( ]3 _5 B: S# a; limages_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
# P3 V  W, e, a5 jrbd_secret_uuid = 22003ebb-0f32-400e-9584-fa90b6efd874
disk_cachemodes="network=writeback"

4 O8 q+ u2 c/ o% p. E& z5 ^' N$ q建议关闭nova的密钥注入功能，而是使用基于metadata服务、cloud-init实现类似功能：
1 C  Z" k6 \! p& l" A在每个计算节点上，编辑/etc/nova/nova.conf：
inject_password = false
% |  H) ~$ X- Q! Oinject_key = false
inject_partition = -2

, a6 w2 D6 _5 {1 C. }+ n) N启动热迁移支持：
! N9 M, x' Q# u# y4 g( w在[libvirt]部分增加：
; {5 S0 ~/ n# o: R$ E/ I0 hlive_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"

0 X# T2 o: b0 T% ~8 v8、重启OpenStack服务
6 R% L& j; \/ d: M0 Jsudo service openstack-glance-api restart
sudo service openstack-nova-compute restart
sudo service openstack-cinder-volume restart
! Q2 p/ F" O; s, P( _0 i- j, s% ^sudo service openstack-cinder-backup restart