|
原理比较简单，利用ceph的rbd导出nfs，然后配置keepalived，在不同主机映射的rbd块间进行vip漂移。 1. 创建并映射rbd块设备试验机器为ceph中的osd1和osd2主机，ip分别为192.168.32.3、192.168.32.4，vip为192.168.32.5。 先创建一个rbd块设备，然后在两台机器上导出此相同的块：
H y: C! r+ s2 U( G8 ]9 B: J0 M) v, f, g, i! Q9 \: B
[root@osd1 keepalived]# rbd ls test- P8 C4 t' _' ~/ p% z f
k; K) y+ O& ~3 m9 Q6 m
2 j4 K |9 ^: ]: j+ F' `( }8 W/ z7 q; S1 v/ ~7 ~, c) B8 I
test.img* ]& [( h& x2 v( Z Q2 e) _
* I2 P9 N6 ^( S* p- 7 p B# I% ^- C) C
8 n( N* Z: _1 r3 x[root@osd1 keepalived]# rbd showmapped
3 ~6 A' i V/ q: v1 L/ ~! `9 I8 D/ g( i k7 V
# _. G v8 m# L7 C" a* w- L7 D6 T2 I
/ |2 y1 y! z% t2 j$ k! W7 t0 eid pool image snap device2 K+ L4 \1 f) Q& a& n
# K5 X7 p; j) g9 G, Z' {- ) y ~, q) [( Z( m% {
4 U& ]: _/ W) G# ?$ j. G0 test test.img - /dev/rbd0$ u# w: T; N/ \3 N8 @
8 E( r4 V5 @9 K% W T9 m% i
- ' L& t6 F( y+ ~) \, z3 ^
& o/ W7 W; c6 C7 K! K! `6 r; F {
[root@osd2 keepalived]# rbd showmapped1 w. B3 L, U* _& U: C9 u
3 }( a1 i2 ^5 B: L$ m9 M - 5 N' u1 S1 p' M/ A" Z" I& c
; p& D# G+ D5 w( `0 m+ Mid pool image snap device
" T0 T5 W( k+ A# r; O3 x9 f3 x
, f0 b1 `; A9 @ I& Y - - A" I$ P/ s+ ~9 h( E J
: o+ n6 Y d" a9 r1 g+ \
0 test test.img - /dev/rbd0
o% @# h" h9 U8 [' j2 P6 q& _, _5 R; x4 M6 L$ o5 ?$ Y; o
7 o ^* H- g2 I0 a, {
然后将/dev/rbd0格式化后进行挂载，如/mnt目录。 2. 配置keepalived
9 x. q8 S1 E5 ]7 m9 P( m3 U从http://www.keepalived.org/download.html下载最新的keepalived 1.2.15版本，安装比较简单，直接按照INSTALL的说明进行默认安装即可。为了方便管理keepalived服务，需进行以下操作： cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/mkdir /etc/keepalivedcp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/cp /usr/local/sbin/keepalived /usr/sbin/chkconfig --add keepalivedchkconfig keepalived onchkconfig --list keepalived配置/etc/keepalived.conf。osd1的配置如下： - ; X1 X8 E! [; ]" W
6 h, N& W4 J; I( \* B) V[root@osd1 keepalived]# cat keepalived.conf5 N7 _' f- Z* t7 R' S' \
7 I6 Z U( L# M' }( e8 p8 K( r
- / o1 v. b2 K; ~! m) I7 Y
/ N! i' L/ K* y8 p3 yglobal_defs4 o @& P; }4 I0 C9 v
, f8 j" p/ l+ F' j+ u z8 ] - ) b$ ~- {% w5 N+ ?
; h y) u2 P. |$ a8 t( l( g0 X. Z{
d! A/ x1 E6 E" r1 @. H! u
0 j9 k& v9 T9 r - 8 ?: m; H: g6 o" u
1 D$ y& Z d3 Y notification_email
0 Z) T0 y9 `4 V Q
5 q L: M& k7 D1 e - * p+ s, ~2 A. a: B/ ~: k- q
1 d4 H6 S3 Q# j$ U8 w9 f" t {/ k1 c2 k, i$ u2 Z4 L; U' p
5 W( ?! ?' o; U4 d - / X! ^5 K& u6 v( n7 e/ k9 l
# P% ~2 Y4 z$ s. Y5 N# T) V }4 K% I' I0 p- S
5 m: H! H$ }* y$ E! V- ]" L
4 L; T0 W/ S( ?) v% i( D, }+ T* S, {0 r" k9 R! u3 u' r/ f# ^
router_id osd1
5 a5 b0 V# r3 ]4 {& [' K7 W
; a) a$ O& P: q1 }, I# v
0 P* k7 o$ `& ?/ k! `6 U
3 y" z8 Q+ v/ e; J x& {0 _}
) G/ ^; C) q [% a% p* J8 Q$ X: F' ]% J
* |" x! q$ n8 m: C
( z; m- S& z7 L; U5 F, b
$ ~" _9 S' X/ W; Z! P' `$ ^3 ^! A8 q& m) s
- $ c5 v3 ?" y6 E1 H# A$ ]
$ P% e/ V/ t9 T0 Z6 I2 Svrrp_instance VI_1 {
4 ~9 F i9 z/ H, ]5 a8 H+ n( F7 B: o% B E+ s
- 3 g E3 V+ M! k- s. @! R8 x0 Q5 B! L
; s5 H0 Q5 L# O k state MASTER1 b; f2 }# n/ w; J2 G& q) J0 A; P K
6 A3 t" L0 K' z& v! b, ~
5 ]/ |( J% q- f! ~# y! }6 C
1 O: R# _) R( C+ R interface em1
" m7 p" G% {3 p7 ~ J( ?& T2 n" v) _* W/ n, z1 t
- $ b# ~# {& }( t I8 V# ]& W6 t
6 u) R1 l4 N9 Z. [8 I) ]. V
virtual_router_id 100
2 F4 J6 I+ }( u; g; ?# v1 [- F9 X" k+ p3 ]/ n
: B" e! e" \4 s5 p y7 n' g! t0 i: `; m! K+ a
priority 1009 X; e! K% M. H1 \
, Y2 k: L6 [" [ R, i
- " g: R* T" _: ]
, A5 w9 J9 {+ Y4 V @ advert_int 18 w. _- r% e8 D( Y
5 C+ G( i& c- J" W! E - M: |' y& B/ R# D+ \ a
- Q% ?- o; L. i# c( G
authentication {2 t( H" r8 Z7 d) M. n6 E( ^
8 ?1 ^, P1 a: J# ]4 x, E y - + w7 M9 i, f* U0 P
$ w: O" f- e* M" m+ C- Q4 ^& m
auth_type PASS
7 B7 [- j5 ]5 y* X
4 j- P* c4 O' C( q) w1 d/ ` - 4 F& z$ \4 k/ E' I$ K8 u" n
2 n+ B1 x2 X) K3 i, b auth_pass 11115 J! _8 W ]) K; m( l- K
3 Q4 M6 @5 E9 R+ A
* K9 U R! k. b+ x, H# U1 x: T
' Z2 c5 n+ K9 { Q; Z2 q7 X }. R1 Y4 u$ D/ V" p+ r& f0 Z
% Z, W$ h# g" f! u3 c8 }
5 ~. L& V0 R# I3 Z+ |" Q- Q5 p! `& ?* x: i$ d% p( ^7 S& G5 f
virtual_ipaddress {* u ^) D1 C& [( y5 A/ m
: G. j8 B. F- G! ]9 t% B; `- : W B5 X8 z2 Q' `
8 [2 E0 z& G1 H
192.168.32.5/24
; t* s* H1 c- V8 }. D3 r; Y+ H0 A8 t3 i; E& s7 W
- 2 u. V% `7 f) R, p6 Z
! z8 h3 Y& t% |/ U }
4 l9 q! @ d5 L$ T8 z* `+ J0 \/ a- \% M: q6 x5 G
5 T$ @4 T; V# x# {1 T2 U3 }, p# ]$ C0 p9 y2 c
}
5 n4 L+ n% n# A/ P
! i9 n! s% w8 a# r$ l, A9 k" _2 _0 R/ n: Q
osd2的配置如下： - 0 q0 @, |( g! l/ d- O
4 w. m* l" N( [. m6 e5 c8 m
[root@osd2 keepalived]# cat keepalived.conf
: P$ H2 s) v' i8 _* Y! d% n4 U
! b: |$ ^0 y7 a1 q - + u6 x0 L$ c! g; K2 c
$ P- ^: ]4 v3 d2 s- i1 S$ A' B6 `& ^
global_defs. b, W/ l6 j3 ^# w$ ?$ A- g$ X
# x% ?) i; o% `! n8 f. n Y
- & @/ Q! J7 f) U
5 o8 e, ^8 k' x" W; d. W/ r2 B
{
+ v. g) G8 E4 u2 x+ j( `; M
& D! c: m* ]8 a2 m8 T
0 R" |1 N1 m5 H4 F
: l$ g; I* q# d' Q' Y notification_email
1 `5 _6 [2 H* _% _& G7 |
9 R/ p, D: h0 W% U/ f- 5 C' H" l( s+ w4 c6 V6 q( U. ~
$ W+ j/ M6 b& l' ~' T. [2 i {
9 f: a6 j6 W2 j* e" O4 Z0 ?9 r, f p U/ ]* k8 @3 M) n5 |
. A4 u6 o7 ~" ^+ V7 k7 k# g/ j; ~1 B0 `( J! d
# admin@example.com
- L; s( k! v* s* r! b% Q! N4 o' `- A' x9 r) p" u$ L2 x y
6 p( P: a( J. {, P, c3 k; ^
5 @1 P3 j" G; b+ }& I: g3 L t3 J }
# m6 C0 @/ k7 O2 a. w4 V/ H9 q* x, U, E6 k: B
- 4 B, {* u9 u% \$ T. U
" t' i' m+ F4 M, I8 }: F0 h" n #notification_email_from admin@example.com1 \" {) g& B. d+ f- p3 q+ M" b1 q9 ]/ x
0 M" X+ ~! b" J- v; z. J; k/ i) K
; G$ A( |& |( \4 a; B% w; P g k) O; l0 B) D3 p
#smtp_server 127.0.0.1. O$ {( A \2 [- R
# P" G7 C* y/ i, U& n, ]- " |6 T$ w1 T9 p# Z" }/ M
7 B: m' p2 ^% d, V" D& C, Q
#stmp_connect_timeout 30
. V3 \' }7 F% g6 k: x' f9 U: ]- H, f7 U* I
- 2 e+ J9 H6 J* k! [! R2 V
9 r5 M/ ^6 J4 m3 C4 w. | router_id osd2
1 k7 {; w6 z" p2 c( T
. w1 h W4 X1 A; [8 T. X8 _ - 2 Q( S9 P4 b* w; w- G
7 \2 L+ \5 w0 @2 N
}# U( b7 ^" A; y6 {
2 g" D Y5 n" I0 A
4 d! b U) q9 U- C
, r# U3 q ]0 N' b( `* e
* u6 j( ~ b6 Q3 k; n7 D
$ m& h& X+ m3 w3 a' _4 D
1 g* N0 A6 ^$ c; {
n9 k5 I& X2 s2 A5 p9 v+ J9 Y. @) lvrrp_instance VI_1 {! j3 ]. B0 H8 J' h' k6 z# H2 D7 D
5 i1 E/ `$ U; E" U i; D
2 h$ q- J- ]$ C) S* Y# |1 i1 x% q+ t9 \3 \. o! ]
state BACKUP
' s/ ^5 |4 y" q9 X7 b( w
) Z' c3 @! q& Y/ `! X8 T$ w
6 y# n- V( b9 K6 j3 s
& Z0 u' K# N$ Y+ }" ~! x( A# Z7 P# g3 ` interface em1
6 L& {; I9 ?" o& J) C3 r/ Z
) Z1 T a6 G* `& x
* s( j/ o7 u. \+ O) W
# A) _8 N% W1 a, Z! _; C2 h9 C virtual_router_id 100+ u# R' J( M, b: X6 ^
; O2 d: M) Q9 G' Z: _$ J
6 ?6 [# k+ Q6 V
' ?% T; O* Q1 z7 |5 |6 j* f& F priority 99
2 X! m6 P* x5 y% g3 {
, p& H$ h2 k" G- N% R. z- 2 ~% x |+ ]; r# j' B
- Z% w) f/ l: u9 j: ~9 c
advert_int 1
& [6 J; K& p: a7 S0 N
/ W4 k7 q# W, f9 \
{) D7 n% {% {8 l) R* k" x" t) J
authentication {
3 l0 L- R5 ]" ]0 M$ L4 v: g$ e" H! k8 u$ t$ S; E
1 n) k# U, @1 R. s
; S0 K& f. a9 ^0 q3 I" R a* F( \ auth_type PASS
6 }5 v1 w% X* c8 R: L2 ?+ a# D) U$ P! b
+ p% ^. y8 p$ ^1 z6 T# M
6 J6 s3 P+ }2 o+ {7 f0 s( j auth_pass 1111
, G9 @: U& X2 B- A+ {/ K
) u" I/ E, P# }- \7 g7 p- " d% r2 P7 j! [5 U: x8 A: C6 d1 q7 _! {
/ x) |2 k# v5 o1 d, \: E" ?8 [/ Y$ x4 @ }
$ b- L* X" C% |, s" b
x# b+ u5 _( _* k/ D
7 }1 w# Q4 \4 J: |6 I+ x6 w, B+ g- u# X" g3 @
notify_master "/etc/keepalived/ChangeToMaster.sh"
8 w& [4 ^0 C7 V; f- k7 T/ O! t3 E3 P& Y+ q) j+ @+ C) q" \# s: p- g
- 6 C/ A6 }7 d( A' w$ B; }
9 K. Z% `4 L8 Y) e" E
notify_backup "/etc/keepalived/ChangeToBackup.sh"
1 {+ u* c1 u* J/ m& E6 Z
9 H K. y) m( t N6 _ P - 9 ?( ~) z5 w$ t8 w/ a g
/ t9 W" q/ Q- ]9 } virtual_ipaddress {
" G' O, |+ A9 C4 ?# J Y" K$ @
* w6 C+ Z+ U! A& y0 ~3 s - ! i, T: K+ R' U+ c6 Y
* |* u4 @. U! p2 M- Q 192.168.32.5/24# R; Z3 O6 I' S
4 a2 X Z4 a) B
: ^9 r# x" g3 d7 w1 P
9 G8 b& e3 c _7 w( g- i) { }- h2 t" S3 C9 l: ^( r
3 L6 a' A( J; ]1 t3 t2 P# C
. p. Q* z" ?# _
4 N/ Y0 U6 W$ B8 v2 t9 e}
' ^( V3 Y9 x6 C; z3 ~0 m
/ T2 [7 r- W. @* W$ K% F! z
( d7 _8 i4 d6 u, `) t1 [
在osd2写了两个控制脚本，用于osd2的keepalived状态改变时执行。ChangeToMaster.sh： [root@osd2 keepalived]# cat ChangeToMaster.sh#!/bin/bashservice nfs startssh lm "umount -f /mnt"ssh lm "mount -t nfs 192.168.32.5:/mnt /mnt"ChangeToBackup.sh: [root@osd2 keepalived]# cat ChangeToBackup.sh#!/bin/bashssh lm "umount -f /mnt"ssh osd1 "service nfs stop"ssh osd1 "umount /mnt"ssh osd1 "rbd unmap /dev/rbd0"ssh osd1 "rbd map test/test.img"ssh osd1 "mount /dev/rbd0 /mnt"ssh osd1 "service nfs start"ssh lm "mount -t nfs 192.168.32.5:/mnt /mnt"service nfs stopumount /mntrbd unmap /dev/rbd0rbd map test/test.imgmount /dev/rbd03. 配置nfs) c: T) [* g# r1 ^. U
在ceph的一个节点利用rbd map一个块设备，然后格式化并挂载在一个目录，如/mnt。在此节点上安装nfs的rpm包： yum -y install nfs-utils设置挂载目录： cat /etc/exports/mnt 192.168.101.157(rw,async,no_subtree_check,no_root_squash)/mnt 192.168.108.4(rw,async,no_subtree_check,no_root_squash)启动并导出： service nfs startchkconfig nfs onexportfs -r客户端查看一下： showmount -e mon0Export list for mon0:/mnt 192.168.108.4,192.168.101.157然后挂载： mount -t nfs mon0:/mnt /mnt需要注意的是，NFS默认是用UDP协议，如果网络不稳定，换成TCP协议即可： mount -t nfs mon0:/mnt /mnt -o proto=tcp -o nolock 4. 测试* x+ r6 r, d. g( w M7 O
关闭osd1的网卡em1查看结果： [root@osd1 keepalived]# ifdown em1[root@osd1 keepalived]# ip a1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever2: em1: <BROADCAST,MULTICAST> mtu 1500 qdisc mq state DOWN qlen 1000 link/ether c8:1f:66:de:5e:65 brd ff:ff:ff:ff:ff:ff查看osd2的网卡： [root@osd2 keepalived]# ip a1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever2: em1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000 link/ether c8:1f:66:f7:61:5d brd ff:ff:ff:ff:ff:ff inet 192.168.32.4/24 brd 192.168.32.255 scope global em1 valid_lft forever preferred_lft forever inet 192.168.32.5/24 scope global secondary em1 valid_lft forever preferred_lft forever inet6 fe80::ca1f:66ff:fef7:615d/64 scope link valid_lft forever preferred_lft forevervip已经漂移到osd2，查看客户端挂载情况： [root@lm /]# df -hTFilesystem Type Size Used Avail Use% Mounted on/dev/sda1 ext4 454G 79G 353G 19% /tmpfs tmpfs 1.7G 4.6M 1.7G 1% /dev/shm192.168.32.5:/mnt nfs 100G 21G 80G 21% /mnt打开osd1的网卡em1： [root@osd1 keepalived]# ifup em1Determining if ip address 192.168.32.3 is already in use for device em1...[root@osd1 keepalived]# ip a1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever2: em1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000 link/ether c8:1f:66:de:5e:65 brd ff:ff:ff:ff:ff:ff inet 192.168.32.3/24 brd 192.168.32.255 scope global em1 valid_lft forever preferred_lft forever inet 192.168.32.5/24 scope global secondary em1 valid_lft forever preferred_lft forever inet6 fe80::ca1f:66ff:fede:5e65/64 scope link valid_lft forever preferred_lft foreverosd2的em1： [root@osd2 keepalived]# ip a1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever2: em1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000 link/ether c8:1f:66:f7:61:5d brd ff:ff:ff:ff:ff:ff inet 192.168.32.4/24 brd 192.168.32.255 scope global em1 valid_lft forever preferred_lft forever inet6 fe80::ca1f:66ff:fef7:615d/64 scope link valid_lft forever preferred_lft forever现在的客户端： [root@lm /]# df -hTFilesystem Type Size Used Avail Use% Mounted on/dev/sda1 ext4 454G 79G 353G 19% /tmpfs tmpfs 1.7G 4.6M 1.7G 1% /dev/shm192.168.32.5:/mnt nfs 100G 21G 80G 21% /mnt[root@lm /]# ls /mnt31.txt a.txt b.txt c.txt etc linux-3.17.4 linux-3.17.4.tar m2.txt test.img test.img2也可以为ceph rbd的iscsi配置vip，后面会进行记录。 |
发表于 2018-11-21 21:56:12