扩容ceph添加到原来的openstack节点

admin

配置步骤
9 b- V0 l6 b) i4 E1.拷贝配置文件
#拷贝配置文件、cinder账户key到openstack的cinder节点
/etc/ceph/ceph2.conf
/etc/ceph/ceph.client.cinder2.keyring
# s( `( k. Y6 p! w! U6 I2 F#此处使用cinder账户，仅拷贝cinder2账户的key即可

2.创建存储池
. @, T5 Y) D  B8 N$ e# Z( z#OSD添加完成后，创建存储池，指定存储池pg/pgp数，配置其对应功能模式
ceph osd pool create volumes 512 512
- t2 X3 F9 T. S4 xceph osd pool create backups 128 128
ceph osd pool create vms 512 512
* W& Z. I% d5 l, w, S6 C+ Zceph osd pool create images 128 128

ceph osd pool application enable volumes rbd
ceph osd pool application enable backups rbd
ceph osd pool application enable vms rbd
0 ]# T% b- G5 _) S% Q- c0 Hceph osd pool application enable images rbd

3.创建集群访问账户
. J, Q1 \0 S1 w9 c. O+ q( d6 g+ _' `+ K% Zceph auth get-or-create client.cinder2 mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
* f2 H* I$ P) r6 D, Fceph auth get-or-create client.cinder2-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'

4.查看进程信息
$ }! K8 l, c3 P! Y#查看当前openstack的cinder组件服务进程
& \) q* \+ Y" Q* E6 Asource /root/keystonerc.admin
5 B' }' N& _$ z0 ]# \( M; zcinder service-list

5.修改配置文件
#修改cinder配置文件
[DEFAULT]
enabled_backends = ceph1,ceph2

[ceph1]
% Z6 W7 e( p+ b1 |; w8 Q$ Zvolume_driver = cinder.volume.drivers.rbd.RBDDriver
0 x' ?9 `( _/ j/ w9 Dvolume_backend_name = ceph1
rbd_pool = volumes1
  U; ^( Y" B) j' ]rbd_ceph_conf = /etc/ceph1/ceph1.conf
3 R) j, X! j; Q/ q! e4 `4 _2 _3 crbd_flatten_volume_from_snapshot = false
" k! s) z- Z- H( r, lrbd_max_clone_depth = 5
& n1 l& ~4 |5 B5 f. K' R4 zrados_connect_timeout = -1
glance_api_version = 2
rbd_user = cinder1
# `0 m2 K2 l2 H5 x: A& D) mrbd_secret_uuid = **

[ceph2]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
. D2 J* i4 O$ {: E! N5 Z& x& _volume_backend_name = ceph2
: K. F4 {5 J/ M6 r$ K1 v3 Rrbd_pool = volumes2
5 F/ T5 d, p4 T5 S( |6 R  }) O) }rbd_ceph_conf = /etc/ceph/ceph2/ceph2.conf
rbd_flatten_volume_from_snapshot = false
" I6 K+ i7 z+ F' L4 Y" s' Nrbd_max_clone_depth = 5
rados_connect_timeout = -1
" \7 l- S5 p5 |5 L4 tglance_api_version = 2
rbd_user = cinder2
rbd_secret_uuid = **

6.重启服务
#重启cinder-volume服务
service openstack-cinder-volume restart Redirecting to /bin/systemctl restart openstack-cinder-volume.service
: G7 [8 d' R2 w5 O2 o$ }2 zservice openstack-cinder-scheduler restart Redirecting to /bin/systemctl restart openstack-cinder-scheduler.service

7.查看进程
cinder service-list

8.创建卷测试
#卷类型绑定
cinder type-create ceph1
cinder type-key ceph1 set volume_backend_name=ceph1
1 {( y" \1 a" c) M2 ^8 qcinder type-create ceph2
0 H3 f  w$ M7 wcinder type-key ceph2 set volume_backend_name=ceph2

9.查看绑定结果
cinder create --volume-type ceph1 --display_name {volume-name}{volume-size}
cinder create --volume-type ceph2 --display_name {volume-name}{volume-size}

配置libvirt
! a$ N3 \' K# x# C) `0 r9 t! n1.将第二套ceph的密钥添加到nova-compute节点的libvirt
0 W3 f0 w6 q5 L$ F#为了使VM可以访问到第二套cephRBD云盘，需要在nova-compute节点上将第二套ceph的cinder用户的密钥添加到libvirt
ceph -c /etc/ceph2/ceph2/ceph2.conf -k /etc/ceph2/ceph.client.cinder2.keyring auth get-key client.cinder2 |tee client.cinder2.key

#绑定之前cinder.conf中第二个ceph集群的uuid
cat > secret2.xml <<eof

' ?+ \5 k: A( w6 u( T5 l***

# h. t& C+ t% z, @: C* t( aclient.cinder2 secret

; F1 C* f, D, E
- {5 }) Q" x5 Z" s$ Y# a% w6 D#以上整段拷贝执行即可，替换uuid值</eof
( I: g8 \- |4 b8 F8 S* _( L% I. P

sudo virsh secret-define --file secret2.xml

sudo virsh secret-set-value --secret ***** --base64 $(cat client.cinder2.key) rm client.cinder2.key secret2.xml
) S, x$ M" u, b7 Y. Q, y9 h! A3 v$ G* Z#删除提示信息，输入Y即可

2.验证配置是否生效
/ [, A1 C# k9 N" N6 a#通过之前创建的两个类型的云盘挂载到openstack的VM验证配置
nova volume-attach {instance-id}{volume1-id}
; t8 @% j% A6 q# H+ wnova volume-attach {instance-id}{volume2-id}

admin

1、SJ-6-Cloud121  192.168.170.121 操作

创建ceph 磁盘类型

cinder type-create  sata02

cinder type-key sata02  set volume_backend_name=CEPH02_SATA

cinder type-create  sata03

cinder type-key sata03  set volume_backend_name=CEPH03_SATA

配置cinder文件

mkdir /etc/ceph/ceph02/

mkdir /etc/ceph/ceph03/

scp 192.168.170.36:/etc/ceph/ceph.conf /etc/ceph/ceph02/

scp 192.168.170.36:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph02/

scp 192.168.170.41:/etc/ceph/ceph.conf /etc/ceph/ceph03/

scp 192.168.170.41:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph03/

enabled_backends=sata02,sata03

[sata02]

volume_driver=cinder.volume.drivers.rbd.RBDDriver

volume_backend_name=CEPH02_SATA

rbd_pool=volumes

rbd_ceph_conf=/etc/ceph/ceph02/ceph.conf

rbd_user=cinder

rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae

rbd_max_clone_depth=10

rbd_flatten_volume_from_snapshot = false

rbd_store_chunk_size=4

: u+ H" _% Q0 Q% E- o+ y2 ?! R

[sata03]

volume_driver=cinder.volume.drivers.rbd.RBDDriver

volume_backend_name=CEPH03_SATA

rbd_pool=volumes

rbd_ceph_conf=/etc/ceph/ceph03/ceph.conf

rbd_user=cinder

rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae

rbd_max_clone_depth=10

rbd_flatten_volume_from_snapshot = false

rbd_store_chunk_size=4

192.168.170.121 获取cinder glance key

获取默认ceph  cinder glance 认证key

scp /ect/ceph/ceph.client.cinder.keyring 192.168.170.36:/etc/ceph/

scp /ect/ceph/ceph.client.glance.keyring 192.168.170.36:/etc/ceph/

! p  L+ C1 d, A& Y8 V

ssh 192.168.170.36

导入ceph02 auth key

删除

ceph auth  import -i ceph.client.cinder.keyring

ceph auth  import -i ceph.client.glance.keyring

更新权限ceph

ceph auth caps client.glance  mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=images'

192.168.170.121 获取cinder glance key

获取默认ceph  cinder glance 认证key

scp /ect/ceph/ceph.client.cinder.keyring 192.168.170.41:/etc/ceph/

scp /ect/ceph/ceph.client.glance.keyring 192.168.170.41:/etc/ceph/

ssh 192.168.170.41

导入ceph03 auth key

删除

ceph auth  import -i ceph.client.cinder.keyring

ceph auth  import -i ceph.client.glance.keyring

更新权限ceph

ceph auth caps client.glance  mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=images'

重启cinder服务服务

openstack-service restart openstack-cinder-api

openstack-service restart openstack-cinder-scheduler

openstack-service restart openstack-cinder-volume

配置完此处之后，glance从镜像启动创建磁盘时，rabbitmq消息，多个后端cinder-volume都有可能接收到；此时应该指定默认的类型来接收；

[url=][/url]
1 故障2 3 消息4     Build of instance 9865bdf5-12e5-4147-b1cf-d0e33a5ae88d aborted: Failure prepping block device.5 编码6     5007 详情8     File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2235, in _do_build_and_run_instance filter_properties) File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2349, in _build_and_run_instance 'create.error', fault=e) File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 85, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2319, in _build_and_run_instance block_device_mapping) as resources: File "/usr/lib64/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2458, in _build_resources reason=msg) [url=][/url]
9 i0 j9 K2 J5 U+ I: i, n* M3 T
% ]  l9 b  N; \
- k3 d7 m- p# E( [* a! y9 \

cinder配置多ceph后端：并指定默认pool
创建type： cinder type-create  XXX
cinder type-list
- i6 O4 G; @* ]' I1 u) W) u+-------------------------------------------------------+--------+
|                  ID                 　　   　　　　       |  Name  |
+-------------------------------------------------------+--------+
0 e1 H+ P  ?8 A. J| 1908f00c-a865-472c-8f90-c44d29361bd3   | sata03 |
, g% |1 W/ U) Y3 M5 X4 ?| 3b6e978f-839b-4164-b5e5-8e6e2d63fdde  |  sata    |
, l7 A9 ^4 R- k| ead23626-0bb9-4515-9906-0b5e6f41320a | sata02 |
2 i7 d5 y- ]8 g7 U6 s+-------------------------------------------------------+--------+

7 f* Z& |; G1 g  B! q6 L, T指定extra-specs：cinder type-key XXX set volume_backend_name=CEPH_XXX
, a$ ?: V$ c4 _& mcinder extra-specs-list
+------------------------------------------------------------+--------+------------------------------------------------------------+
|                  ID            　　　                                 |  Name  |               extra_specs                                        |
+------------------------------------------------------------+--------+------------------------------------------------------------+
+ N5 k! w7 F) Y1 b1 ^) o$ k9 w| 1908f00c-a865-472c-8f90-c44d29361bd3          | sata03 | {u'volume_backend_name': u'CEPH03_SATA'} |
| 3b6e978f-839b-4164-b5e5-8e6e2d63fdde         |  sata    |  {u'volume_backend_name': u'RBD_SATA'}      |
* \8 E  @' ^3 D+ X* _& s. `# c| ead23626-0bb9-4515-9906-0b5e6f41320a        | sata02 | {u'volume_backend_name': u'CEPH02_SATA'} |
+------------------------------------------------------------+--------+------------------------------------------------------------+

修改cinder.conf:
在DEFAULT段下指定default_volume_type = XXX

' N6 O0 d" X* N[DEFAULT]
...
default_volume_type=XXX  #glance创建的磁盘将位于此处
+ a& n0 h& o, [5 B2 x' penabled_backends=sata02, XXX
* H" g8 E* j; ]1 x
' N+ y6 z  _) k# q[XXX]
( k5 K) U) k4 [volume_driver=cinder.volume.drivers.rbd.RBDDriver
volume_backend_name=CEPH_XXX
! a% T) \* x: Z9 B- Yrbd_pool=volumes
# [) g( `7 X' z& R5 W- P5 V6 i; K: _rbd_ceph_conf=/etc/ceph/ceph02/ceph.conf
rbd_user=cinder
rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae
rbd_max_clone_depth=10
rbd_flatten_volume_from_snapshot = false
; h, O  X/ @( z% }+ k4 u4 X8 Lrbd_store_chunk_size=4