扩容ceph添加到原来的openstack节点

admin

配置步骤
$ ]- D( U$ M! d: ?% O& D1.拷贝配置文件
9 b6 Z7 R3 O' N1 V4 A#拷贝配置文件、cinder账户key到openstack的cinder节点
/etc/ceph/ceph2.conf
! _1 W/ L7 n* l/ m7 v1 e% T. L4 q+ D/etc/ceph/ceph.client.cinder2.keyring
- p9 R* z& n; ~. l, N: O% }8 V#此处使用cinder账户，仅拷贝cinder2账户的key即可

2.创建存储池
9 B3 a1 k2 J/ K#OSD添加完成后，创建存储池，指定存储池pg/pgp数，配置其对应功能模式
; ~& A# s, \  j# D; Q% Lceph osd pool create volumes 512 512
ceph osd pool create backups 128 128
5 z+ W) @9 N# K% bceph osd pool create vms 512 512
! ]( V, H! D  ]$ |5 Lceph osd pool create images 128 128

ceph osd pool application enable volumes rbd
ceph osd pool application enable backups rbd
  s. O6 n# ~5 \* c' Y- t- bceph osd pool application enable vms rbd
0 Z6 A5 t% p0 |4 I/ E5 }ceph osd pool application enable images rbd

3.创建集群访问账户
ceph auth get-or-create client.cinder2 mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=vms, allow rx pool=images'
  F, {/ l$ ]4 O% Y" f$ \1 uceph auth get-or-create client.cinder2-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=backups'
ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=images'

4.查看进程信息
#查看当前openstack的cinder组件服务进程
( l; q  Y8 {6 @( Y  a6 S, ?1 Psource /root/keystonerc.admin
5 o9 O- B& X8 ?2 V* ]3 W  Zcinder service-list

5.修改配置文件
( {5 `" b7 h# `, c2 ~#修改cinder配置文件
[DEFAULT]
enabled_backends = ceph1,ceph2

[ceph1]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
volume_backend_name = ceph1
rbd_pool = volumes1
rbd_ceph_conf = /etc/ceph1/ceph1.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
/ F. g& L$ K4 p* \: d8 Y2 z, @1 Xrados_connect_timeout = -1
3 [; s- @, o0 H. k( r  l9 hglance_api_version = 2
rbd_user = cinder1
  ^: X$ O, \) w2 [0 [rbd_secret_uuid = **

[ceph2]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
% d/ _8 v& Q  m# [. svolume_backend_name = ceph2
- H" T9 l$ g0 arbd_pool = volumes2
# D6 O5 [6 ]6 rrbd_ceph_conf = /etc/ceph/ceph2/ceph2.conf
# I; A8 L& Z! o! g3 H% b  qrbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
rados_connect_timeout = -1
$ i3 d1 @1 e6 w+ i, h0 oglance_api_version = 2
rbd_user = cinder2
rbd_secret_uuid = **

6.重启服务
' U+ [; |8 ?, h$ d#重启cinder-volume服务
service openstack-cinder-volume restart Redirecting to /bin/systemctl restart openstack-cinder-volume.service
service openstack-cinder-scheduler restart Redirecting to /bin/systemctl restart openstack-cinder-scheduler.service

7.查看进程
' K  T3 n; d* Wcinder service-list

8.创建卷测试
#卷类型绑定
cinder type-create ceph1
; Z' d" N+ X5 k0 e( E; Zcinder type-key ceph1 set volume_backend_name=ceph1
, f$ y+ k0 \- [- s8 \cinder type-create ceph2
cinder type-key ceph2 set volume_backend_name=ceph2

9.查看绑定结果
cinder create --volume-type ceph1 --display_name {volume-name}{volume-size}
; c( j. o" _- O, s; X* ncinder create --volume-type ceph2 --display_name {volume-name}{volume-size}

配置libvirt
. |4 A9 J/ |6 S6 O7 D0 @6 z2 ?1.将第二套ceph的密钥添加到nova-compute节点的libvirt
0 o' U: X% ~  z  d  J" P#为了使VM可以访问到第二套cephRBD云盘，需要在nova-compute节点上将第二套ceph的cinder用户的密钥添加到libvirt
, f( @  B& l/ E1 k) T) c/ E- _4 Zceph -c /etc/ceph2/ceph2/ceph2.conf -k /etc/ceph2/ceph.client.cinder2.keyring auth get-key client.cinder2 |tee client.cinder2.key

#绑定之前cinder.conf中第二个ceph集群的uuid
cat > secret2.xml <<eof
+ a) q" ^- m$ |# ]
***

client.cinder2 secret
8 z) K7 ~) X+ r8 c+ J

#以上整段拷贝执行即可，替换uuid值</eof
: d0 f, A; }2 d8 \4 z

sudo virsh secret-define --file secret2.xml

sudo virsh secret-set-value --secret ***** --base64 $(cat client.cinder2.key) rm client.cinder2.key secret2.xml
#删除提示信息，输入Y即可

2.验证配置是否生效
1 V7 j  o7 u" \: s' j#通过之前创建的两个类型的云盘挂载到openstack的VM验证配置
3 W; t" o+ E) D: Tnova volume-attach {instance-id}{volume1-id}
; ~, ^2 g+ K7 Y+ C* A9 T$ }7 U, vnova volume-attach {instance-id}{volume2-id}

admin

1、SJ-6-Cloud121  192.168.170.121 操作

创建ceph 磁盘类型

cinder type-create  sata02

cinder type-key sata02  set volume_backend_name=CEPH02_SATA

cinder type-create  sata03

cinder type-key sata03  set volume_backend_name=CEPH03_SATA

配置cinder文件

mkdir /etc/ceph/ceph02/

mkdir /etc/ceph/ceph03/

scp 192.168.170.36:/etc/ceph/ceph.conf /etc/ceph/ceph02/

scp 192.168.170.36:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph02/

scp 192.168.170.41:/etc/ceph/ceph.conf /etc/ceph/ceph03/

scp 192.168.170.41:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph03/

enabled_backends=sata02,sata03

[sata02]

volume_driver=cinder.volume.drivers.rbd.RBDDriver

volume_backend_name=CEPH02_SATA

rbd_pool=volumes

rbd_ceph_conf=/etc/ceph/ceph02/ceph.conf

rbd_user=cinder

rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae

rbd_max_clone_depth=10

rbd_flatten_volume_from_snapshot = false

rbd_store_chunk_size=4

( i& B$ a: a% z* A( ?/ X* }

[sata03]

volume_driver=cinder.volume.drivers.rbd.RBDDriver

volume_backend_name=CEPH03_SATA

rbd_pool=volumes

rbd_ceph_conf=/etc/ceph/ceph03/ceph.conf

rbd_user=cinder

rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae

rbd_max_clone_depth=10

rbd_flatten_volume_from_snapshot = false

rbd_store_chunk_size=4

7 g6 P! S7 W$ k; F2 c

192.168.170.121 获取cinder glance key

获取默认ceph  cinder glance 认证key

scp /ect/ceph/ceph.client.cinder.keyring 192.168.170.36:/etc/ceph/

scp /ect/ceph/ceph.client.glance.keyring 192.168.170.36:/etc/ceph/

ssh 192.168.170.36

导入ceph02 auth key

删除

ceph auth  import -i ceph.client.cinder.keyring

ceph auth  import -i ceph.client.glance.keyring

更新权限ceph

ceph auth caps client.glance  mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=images'

" Z7 G( n1 L$ n1 S5 Q

192.168.170.121 获取cinder glance key

获取默认ceph  cinder glance 认证key

scp /ect/ceph/ceph.client.cinder.keyring 192.168.170.41:/etc/ceph/

scp /ect/ceph/ceph.client.glance.keyring 192.168.170.41:/etc/ceph/

ssh 192.168.170.41

导入ceph03 auth key

删除

ceph auth  import -i ceph.client.cinder.keyring

ceph auth  import -i ceph.client.glance.keyring

更新权限ceph

ceph auth caps client.glance  mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rwx pool=images'

重启cinder服务服务

openstack-service restart openstack-cinder-api

openstack-service restart openstack-cinder-scheduler

openstack-service restart openstack-cinder-volume

配置完此处之后，glance从镜像启动创建磁盘时，rabbitmq消息，多个后端cinder-volume都有可能接收到；此时应该指定默认的类型来接收；

" p+ S8 b* E, V3 G: q3 r8 b6 G8 S[url=][/url]
1 故障2 3 消息4     Build of instance 9865bdf5-12e5-4147-b1cf-d0e33a5ae88d aborted: Failure prepping block device.5 编码6     5007 详情8     File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2235, in _do_build_and_run_instance filter_properties) File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2349, in _build_and_run_instance 'create.error', fault=e) File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 85, in __exit__ six.reraise(self.type_, self.value, self.tb) File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2319, in _build_and_run_instance block_device_mapping) as resources: File "/usr/lib64/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2458, in _build_resources reason=msg) [url=][/url]
( p/ F5 k* C2 C' m. ?' ^0 H; ^% t

cinder配置多ceph后端：并指定默认pool
+ u" p5 b! }3 D2 k创建type： cinder type-create  XXX
cinder type-list
; g3 K& ^2 A$ L5 y1 N. i  W" k+-------------------------------------------------------+--------+
|                  ID                 　　   　　　　       |  Name  |
+-------------------------------------------------------+--------+
| 1908f00c-a865-472c-8f90-c44d29361bd3   | sata03 |
| 3b6e978f-839b-4164-b5e5-8e6e2d63fdde  |  sata    |
| ead23626-0bb9-4515-9906-0b5e6f41320a | sata02 |
+-------------------------------------------------------+--------+
; y) ?! L9 r9 Y: v5 A& o
指定extra-specs：cinder type-key XXX set volume_backend_name=CEPH_XXX
/ E9 X6 s6 U& e9 e4 [cinder extra-specs-list
; M0 _# R4 ?2 G! J* V+------------------------------------------------------------+--------+------------------------------------------------------------+
4 Z6 g8 U0 {# u" K0 f|                  ID            　　　                                 |  Name  |               extra_specs                                        |
" L+ U! ?4 T" Y: Y' b, Q* [. s  J+------------------------------------------------------------+--------+------------------------------------------------------------+
9 a2 c* O6 F) ~' C; c& w; L| 1908f00c-a865-472c-8f90-c44d29361bd3          | sata03 | {u'volume_backend_name': u'CEPH03_SATA'} |
| 3b6e978f-839b-4164-b5e5-8e6e2d63fdde         |  sata    |  {u'volume_backend_name': u'RBD_SATA'}      |
& P- W. y' }  U5 ?# g6 S| ead23626-0bb9-4515-9906-0b5e6f41320a        | sata02 | {u'volume_backend_name': u'CEPH02_SATA'} |
+------------------------------------------------------------+--------+------------------------------------------------------------+

) e; a/ S: g8 V' Z修改cinder.conf:
在DEFAULT段下指定default_volume_type = XXX

4 ^5 f; g7 D/ [3 L7 m[DEFAULT]
6 d7 a" L8 n0 [/ [...
default_volume_type=XXX  #glance创建的磁盘将位于此处
; ?# F! }& E# q) g! G; S# renabled_backends=sata02, XXX
) q- X6 n4 E4 L+ [! v
+ g& c/ U1 }0 M2 a9 q+ C[XXX]
volume_driver=cinder.volume.drivers.rbd.RBDDriver
5 u5 Z0 f/ `$ [- |* P* Svolume_backend_name=CEPH_XXX
rbd_pool=volumes
/ p6 p. U! I  _$ D( x/ o+ R( Mrbd_ceph_conf=/etc/ceph/ceph02/ceph.conf
  R6 B( l, Q/ g# Yrbd_user=cinder
rbd_secret_uuid=62b1ed02-5de2-4a53-be1e-0477d1b385ae
rbd_max_clone_depth=10
% x" |' `- M5 T  Z8 U6 I! Arbd_flatten_volume_from_snapshot = false
# E) B1 u: g9 c) W, X5 y. N' h2 j+ \6 urbd_store_chunk_size=4