|
部署ISCSI服务2.1 安装软件 ceph相关的ISCSI软件包可以从redhat通过的源代码进行编译或者下载centos已经编译好的。 - : w" t- N8 \- N
# c9 F8 w' P ^( N8 L+ H7 S! i
[root@ceph01 ~]# yum install ceph-iscsi-cli tcmu-runner ceph-iscsi-tools
8 y2 c$ y+ M: g* r- K; _: O k' R _1 G8 E
- 1 M5 b- N4 P5 F7 M
) j; D) J' L) _3 D& V: L4 d[root@ceph01 ceph]# ceph osd pool create rbd 150 150( L9 ]6 ~# ^+ i+ a9 _, d
5 ~( q, a1 u7 g/ [2 B
; B; C4 n( I0 r% U$ m
9 c7 k" |3 N! L5 L6 x[root@ceph01 ceph]# ceph osd pool application enable rbd rbd --yes-i-really-mean-it# N: q, d, ~ v U* q
# y+ ~( z: c! {
+ }; r' o: a: q1 P' O
2.2 创建配置文件创建iscsi-gateway.cfg，此文件主要设置iscsi服务的网关。 - 5 u1 H; e' n: M5 R8 w6 P
- z5 |, u. W- E" l/ F8 P9 D[root@ceph01 ~]# vi /etc/ceph/iscsi-gateway.cfg 5 \; \7 Q9 J [- i$ B5 R, e
8 K* R4 n. r a0 y0 p7 w
3 Z7 ~" Z( B7 W$ S# f" x6 h2 J
[config]6 p) x+ t6 c& Y+ U
/ m; N4 W7 C0 j' f0 K+ Q& l Y; E
- 2 _5 W* o: a1 j2 c* `
j5 D9 [8 Y8 Y4 j* A/ w8 K3 ]
# Name of the Ceph storage cluster. A suitable Ceph configuration file allowing0 V7 x% M5 Z8 L' q* W
' E A5 m( ~ R. v5 u$ [
0 q* T6 U2 O5 ]# T) u. r; t" d H- D# q0 w" L
# access to the Ceph storage cluster from the gateway node is required, if not( G* c2 |) o6 ~5 [ E
4 c. |# ]0 d7 \* V& E
- ) R$ \% J- w7 c+ G$ R4 }
; d: j6 p1 n( o3 ?
# colocated on an OSD node.
6 F$ I/ P: P* V& i X' h6 N. ]- c0 w' D2 j+ Y h* y
9 E% t6 r4 [4 L! K6 {; P3 N* i+ X; L( T
cluster_name = ceph
" D$ P' C f1 f7 F- z: `8 U3 T% @+ m5 y- ]
B: t( G" h" \: P3 m7 G& G( t% a" Y) K9 y8 e- [; i- D' x
+ R, [( J( ~3 L: i3 R! t! ]+ X1 g$ p
$ ?/ {; a$ S$ R/ q" h F
0 w& P5 a: ]4 }7 h- N
4 }7 c0 Z/ a7 G) F+ V# Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph' ~% U1 p% I2 s1 _# ]
7 H6 y* g8 d4 s, X
% y1 X* l h* u1 ?: N h2 S2 G7 N" ?8 q& P- Q
# drectory and reference the filename here! Z7 x- _, o4 G
' N/ r% O2 z+ B, L7 j& u
* k: I. m/ ~: @0 N8 g! u r# V: m7 `: ]5 S# ?
gateway_keyring = ceph.client.admin.keyring- |: Q* p, q3 X4 O5 x
. k! ?, v0 Z% q+ Y( A! p
. \2 H' ], z) {; ?7 m; L o8 j
: l w9 J ~" ?2 F1 M7 @. `3 T7 w3 ~) T) ?' y
) o( c$ {3 Z6 }
- , f" J$ U: X$ l4 ^
* C. a( R+ D& L$ ?4 N! O
# API settings.6 V, c& A/ A+ h& G" E6 j$ t" r
) J8 h* ^. S# U5 v% p - 2 B% V8 }# ^4 N7 F: X
. W4 \3 o! C U# g: H0 q2 n4 m# The API supports a number of options that allow you to tailor it to your
5 U/ x& v Y- k# c
# X2 o7 o4 e8 v; l - : k" l( n9 ?6 `. |3 L1 u
) {7 S; \* {" T$ I i: P# local environment. If you want to run the API under https, you will need to3 a4 y, F- r' q6 t3 A x# K
. W x9 i. R2 M
- 0 a0 M3 A5 z" ^ q
) q, f2 b% j( Z, H" N4 G8 F# create cert/key files that are compatible for each iSCSI gateway node, that is$ m! s. J" W* X/ S! l6 C
2 k; r" }" t) S" o: f, V9 E
h5 z3 H9 b- C, L) _6 d* p+ g' z) Z& i8 A/ W0 l
# not locked to a specific node. SSL cert and key files *must* be called
: p& g( ~" y) M& w) r3 i& q, H5 s! I
7 `) U( g) Q! L" V6 S6 L% k
# N. Y1 Z; y& }3 k) c# 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory) g, g8 O1 J# R D* e
* m: P' l0 }0 d8 K/ D; y. |
- 5 `: h. _) i, L3 u+ e+ `
: n. B2 Y1 r9 g8 i7 I( ~$ e! V/ O, {2 r- a# on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'' K7 e# T/ w) X
0 @2 p5 Y" ~( y4 Q) m; k
# ^5 h, ]1 B8 ?8 J/ }6 ~) S k3 j5 E
& W4 ~' J: G# r, a4 c. Q' d# to switch to https mode.' ]- J( F' ?% \% a( U8 J
- s$ Z$ }* A' | x/ X$ o4 u- , |$ B# J" x) h6 ?! S, t
- o2 p5 L T# }$ h+ U5 h
0 k' j( W% J' J; N3 f! M3 q
2 a8 I6 \9 w+ x7 d) r% M0 U! T
- / w# }4 R/ t6 r/ D2 ^2 R3 o
5 L8 |" ?. \. R% q# To support the API, the bear minimum settings are:
0 l( j9 J% i3 Z8 t4 \( p4 ~5 \
; U0 q. V1 o+ p3 h0 \/ V
1 I7 o/ \* V6 P1 I' {$ _/ S& \' I+ H+ e0 D
api_secure = false' J$ M7 h" I2 I' I; r3 M
+ h G! q, I! \: N. n
U" B# V2 O* G0 |5 ?9 [3 P. B$ b
0 e- m' P8 l' x5 O1 e" g4 l4 w" g( k& s* K6 j
( A0 [* G, o6 T4 o- @6 i n* N. p
( L! `+ y, ~, k5 P' T$ i& }) q# C/ \# d
# Additional API configuration options are as follows, defaults shown.
; c! M( g/ f4 I; p2 a5 N0 Y8 Z7 w: b
- ! I- v, ]! t8 v0 G% d4 p
6 |5 z: j9 r4 Y, f" X: K& b0 ` z# api_user = admin8 L2 r$ ]. T* _; `
, D+ n% t& W7 ~" M4 T: ^0 g - 8 D# B( I/ t9 `
) s- [% R* _) V2 t" p5 ?8 U
# api_password = admin0 p$ N3 _( I8 s( A$ o% g
$ Y( r' g2 X1 e+ Y9 _ - ! n1 z7 O' f, @, h+ ~ {- a
5 n/ f* ^. E! ]& U# api_port = 5001
9 o0 m5 i3 H9 T5 {5 _. B6 n
+ r7 G2 ~3 M( N - 6 q" z% w+ I+ A( B, ~% b9 L
9 ~( u( N8 U; o2 g& b) c- z
trusted_ip_list = 192.168.120.81,192.168.120.82,192.168.120.83& \; L8 R* B: ~* Z) t8 K- m/ D8 Q
9 ] N" N& Y h6 r4 @6 t; Q9 H8 Z/ \/ w( W( c5 D
2.3 同步文件到其他节点
5 D9 g! Z, _3 t8 O" R: H) z5 ]# `9 \- P
[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph02:/etc/ceph: |/ Z3 L1 N; N6 I. \4 [- d, L
1 P' x( a1 b4 F7 }/ B
) P- U9 H+ S {" B
; a; {1 n" |/ F5 ?[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph03:/etc/ceph
* L+ F! T+ l5 v+ V- c/ {8 L: Z+ p) V" {! l; S. G5 F
& \4 }2 A. }0 P. W3 {2 C
2.4 启动API服务
0 s4 K. |. R. h: G0 t' t7 c
4 _: Y3 @+ ]; \# r# G" a[root@ceph01 ~]# systemctl daemon-reload
" n7 K6 ?$ b# p; _& u$ K, {* A/ r! o1 c' Y5 b7 n
C. F0 j$ V' a k9 m! ^ q/ H
, {( X" E4 t, W$ J! }" `[root@ceph01 ~]# systemctl enable rbd-target-api' b" V- a( S- N8 s) V, q, y; e/ W
7 I- q4 W; i- O. K# I' w; P2 c- 9 A! m/ E9 R$ m- `
& i0 @: t+ q& B8 K# }[root@ceph01 ~]# systemctl start rbd-target-api
0 z) \/ \* D# |% M% i+ U6 Z2 z! n& h5 ]- B0 B2 F, R
- ) ?8 ]- B% q" ?1 a$ P- G9 O+ T
, j) a, ~, S+ W- l( H[root@ceph01 ~]# systemctl status rbd-target-api
2 q- \8 x/ I' q; A* ]: o$ o: S$ J( J: s" v) j
- & E8 L/ A8 V- [2 j9 D2 V
/ K' C& v* _2 T" u3 [4 A● rbd-target-api.service - Ceph iscsi target configuration API
# y+ Z( S& ]9 V, |: D. N0 ~5 W. J9 o0 p9 q9 W
% w6 D% R& ~! B' q( L
; J/ k& ]9 S5 L" {" M# J Loaded: loaded (/usr/lib/systemd/system/rbd-target-api.service; enabled; vendor preset: disabled)
& |) P+ e* `! D4 C( Z" }
/ M n% Z$ U. C( F1 p ?
! c0 ] a, W) @8 X6 g4 ?
7 h! ^; T P! s d- y3 m, f8 Y6 } Active: active (running) since Thu 2018-05-31 11:35:04 CST; 4s ago) @* V3 R9 }: B9 w- e. K& q
. K, t6 i' e: g- n* D
$ g/ P$ T/ p' j% y' T& ?8 G d$ D) o& K6 W. q) K v& ?
Main PID: 25372 (rbd-target-api)% x3 L N' S! ^0 T) T- ]) D! P! { d
3 S/ u2 o4 h" T9 f) h. N- & L9 w6 [7 I( ^2 c
, D3 T. d' L5 H i" M8 P CGroup: /system.slice/rbd-target-api.service k) L7 o) z/ J" l7 }0 g
% O( x1 O: w0 T" z$ j+ t! _ - ) X# l5 S7 K. l) c1 i6 {
. i, T" h+ f/ X' y3 @ {5 g
└─25372 /usr/bin/python /usr/bin/rbd-target-api( D$ d) G2 I5 f& }7 Y) I
3 G( ~. |% ]# _- R3 T8 B0 Y
' D+ U$ C$ b3 i
+ _2 }2 F' T' o' `9 J" R) C7 ^$ x9 C$ v
! z, ?+ M1 U; z6 O! |6 m. v- 4 ^+ o, K5 K' [3 ^( B/ p7 F
- e+ U) |! H \( x1 K
May 31 11:35:04 ceph01 systemd[1]: Started Ceph iscsi target configuration API.
. b' ^# ]5 h6 |/ S3 z- s+ p3 X3 C; l/ m
6 d8 q9 R" ?* ?% U
3 ^& Z9 w) A& }9 y4 f/ uMay 31 11:35:04 ceph01 systemd[1]: Starting Ceph iscsi target configuration API...
7 q" \: a1 H% e- {
9 d; m E& M8 x+ @6 |- 2 z4 {. g& l( E
% I. Z+ {& W# B8 rMay 31 11:35:05 ceph01 rbd-target-api[25372]: Started the configuration object watcher: M% W: h! X! ?, M9 r
$ h; {# Y% ] x) z
: g2 ~$ X. w: [7 B+ O6 D4 Y7 f1 r
6 f4 c0 k+ a) \8 q* l/ lMay 31 11:35:05 ceph01 rbd-target-api[25372]: Checking for config object changes every 1s
1 {: Y; x& O w! _1 W a/ G6 r
- \9 g6 v: b4 m0 s7 y- - v( D- T& j: e" q
0 s& V" R2 D: W$ Y6 i' Z
May 31 11:35:05 ceph01 rbd-target-api[25372]: * Running on http://0.0.0.0:5000/
, H9 N8 a5 X! m3 P. d2 _/ k! C* m
9 D, N- F. H* a% F, z! O
三、配置ISCSI服务3.1 创建target- 6 q2 ` D8 A; S1 @# ?' K; h) j4 n0 t
- ^/ L, V: ^" E$ p H[root@ceph01 ~]# gwcli
9 Q, Z9 u3 k& c2 Y3 p# g* v/ Y* W3 B8 D. M9 V, e3 W
9 e: E$ B5 D' t, z6 W& w
! O6 d$ E, k" g5 S8 N; O5 R+ d- K/> cd iscsi-target
! c+ k3 I) J# _- h/ e4 t
# f! q; m, g/ j- : A# Y3 K7 z/ ]/ S
9 V: T% U p2 Z/iscsi-target> create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw
1 x- d2 t$ t; G0 F* ?" e2 X
& B) [7 w6 t1 X4 F+ V/ L
! Q; u+ F# r% H" ^0 ^4 ?
3.2 创建ISCSI网关
0 \) M/ X* f( s) k, E* R5 m8 M' Y1 m/ ^" @! b! \
/iscsi-target...-igw/gateways> create ceph01 192.168.120.81 skipchecks=true
* d8 S$ S [8 f8 C1 V
/ X! v: D' n( j# U
% x# r% z, M" t* d/ u5 k+ \) F7 k& `& r- e4 a/ o3 A
OS version/package checks have been bypassed3 O5 m3 }6 l1 k6 D0 X" h
% Y$ J& D* ^- v0 I+ O- 3 [0 X, f8 B) H2 V$ C1 n7 Y* b+ b
: `+ s/ R& |2 q; @Adding gateway, sync'ing 0 disk(s) and 0 client(s)& O: t: ^! H- P3 i1 V% c; y; {. i
2 Q$ x4 D) y* g7 w
& X- W' z' `* s3 s/ D7 G0 A8 I9 w9 B# S& X) \* s# t
ok
9 ^ a- Y9 n0 Q& z, p
, T. T Z7 v6 p7 z. _2 Q6 K9 d u: l- . J( l; T. H3 g1 o" F9 |7 n, k; A& i
- \) \& E$ b6 {/iscsi-target...-igw/gateways> create ceph02 192.168.120.82 skipchecks=true6 E' i; K1 z1 b$ N$ c: R, l
2 d( }) F0 I- w1 ^# Y - # ]- v8 [7 c% ]
! w: @. o) P4 I& M/ I3 S, M
OS version/package checks have been bypassed$ u" v4 V! h' r3 v7 t8 G+ J! q
6 u4 D1 s/ m" n. j9 f - 3 X/ Q/ m2 k6 w, o R+ U( `6 w
V* H# G; X- H$ |- o
Adding gateway, sync'ing 0 disk(s) and 0 client(s)
9 [( D, E! ]# c& i t+ |
( l2 N4 U1 Y5 a! I8 |
# x; u: q+ h$ L2 H1 k& n+ l5 I$ _% s1 j- t. {1 t! Y, Z! Z% E
ok
. I# T* b' D6 ^; a0 r% [9 a* Y* g. m6 R
/ a4 q0 m9 u) y8 Z3 l7 }$ b4 I T8 z+ U7 I
/iscsi-target...-igw/gateways> create ceph03 192.168.120.83 skipchecks=true; \. `0 {; U5 ~1 ~5 f9 T2 T2 w/ E
) X' n" v, B" m% n
: N3 g1 V u/ ^5 y2 \" W" q$ o
OS version/package checks have been bypassed* [7 C b C* O' i- q! ?% Y4 K; u8 l
& F" T, V: G2 v
- $ `* o. s8 X; |$ n0 N' R1 h/ u& i
+ j6 @+ Z5 g" Q1 z( ZAdding gateway, sync'ing 0 disk(s) and 0 client(s): T6 M4 j' @2 H, `0 A) i
6 w, P% U# A# M" j3 G
! k. C; M, Q& Y2 L4 \7 G9 V' E# e( r$ n+ S! o& C
ok
6 D& U$ ^! }, ^; C1 E: _) A) d# \* B2 m) [
# c$ X+ I; r' [, h! v* ^+ K2 Q, W- & Z3 t, |9 m5 l6 Q" W$ z
$ `7 H5 E8 V: N; D/iscsi-target...-igw/gateways> ls, {! z3 H, X: }
( J5 i1 z- I- Q; F: D5 K) r( F
1 f% j: B; N' L) i
1 D- B9 m+ T. }, [9 V# l7 y, _o- gateways .................................................................................................. [Up: 3/3, Portals: 3]
: H' L. Y0 s$ ]' t, r G, P! M( u
- `! Q" a( ]. _6 R8 C/ K
/ P7 h# \. j5 f' ]' y2 M" e$ @& ?) v3 |. M, {) _* M: L
o- ceph01 .................................................................................................. [192.168.120.81 (UP)]
: a% a5 D* ]7 |% b$ n$ a/ E# v# v; ~) \% W; O* C( Y. C1 ]
- $ `; A! l" I7 O" H3 X# f, Z' r
e2 u8 m8 K8 u
o- ceph02 .................................................................................................. [192.168.120.82 (UP)]+ l7 d* O9 N: k0 g( A1 H
1 R3 r9 Z3 ]3 M5 X# I/ d
& Z! f, F2 R! ?7 y+ w8 I( G
' M, p; @1 W3 T; O o- ceph03 .................................................................................................. [192.168.120.83 (UP)]4 V& B$ I: T, A% L4 z
! }; x [8 i3 ^3 ?" b0 q5 T3 s# y
7 ^, B' }/ J, }- C- _
如果操作系统非Centos或redhat，则需要加skipchecks=true参数。 3.3 创建RBD image
. _& y" T! I) I0 v( i7 Q. k' _' |' L/ @) G
/iscsi-target...-igw/gateways> cd /disks6 w% L% ]/ ]5 Q# d, ?. `# x
+ N7 u" o" K/ B9 o- ' e% G! p% ~! n( R9 v! f
) _# u" [3 M# S3 p
/disks> create Oracle vol01 100G1 o- u) X4 f* A# e% j1 T, t9 q
U% ^7 Q+ P, c2 h2 W* d0 g3 o" f
h& N l9 U, [. h' D- k8 d# j0 w5 F0 n0 V9 y( X$ p: V& b o
ok
: L4 X/ c$ W- k& d2 k) i& ]" d
( |' w7 @( S, m2 L: s- ; V, z$ I: x& ]5 Z& Y: s5 v' M2 n* Z
" k2 D* X9 @* b- l/ c2 W+ b
/disks> create Oracle vol02 300G
4 T. l% A! A# @1 W- v" o# j0 n7 [ h. _
& E. o) C) C& @7 p( P$ o
& D' h' R' C! W( r2 ~, Mok
5 V( g! V3 ^- }' i5 t8 p
) a6 J# ^# U9 i
7 b8 I: B4 V- _0 v! O$ L
3.4 创建客户端名称Linux平台可以查看/etc/iscsi/initiatorname.iscsi文件获取InitiatorName。如果修改了默认的名称，必须重启iscsid服务，否则在登录iscsi服务端的时候会报错。 - ' M6 p: x. L7 L2 L) ^4 l% Q9 m
. t$ T' [* e3 v/ V- r! i6 L) B
/disks> cd /iscsi-target/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts
+ r' l. K% u/ S5 T7 F/ @
" i6 y+ D! d. X" Z6 F1 E - ( e" A% d( r$ x. N- i; Q2 B3 M
- ~ T& O8 \2 N- |
/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
5 V# N) w* t. [9 [. b
4 P. q' y& [5 b
4 ~: ]- p9 c$ E7 J Y1 @: l6 v- r& R! A# d$ x9 |
ok1 S" N& t4 P4 k, F, t- ^
9 _* _* H+ B3 C3 h' S
6 b) ~) {" f# ?5 R
7 x o& z/ I& w/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:ccd061606e1:odb04' y- X; B) f6 D0 ^+ u
: Z1 P5 x, a$ v" `0 ^$ _9 t R k
2 e; X: Z1 Z! C: G% ?9 [, g- Y; Y' Y( S, q: N% ]9 O
ok9 ^. l, l; F& M
, G. ^/ ^& K; s9 ~; @$ \. l! Y5 O! {9 N1 P1 ~! t+ d, ^+ A4 g
3.5 设置客户端认证
# x9 p& _' T# d: V8 u7 r9 L: K- @; z4 O( Z) c1 n, z" z1 Z8 L2 I& U9 C
/iscsi-target...csi-igw/hosts> cd iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
) u- Q- H) @$ R) \$ l7 j: e N' I, H1 ~% I; T) x) W9 D3 G1 W0 p! v
- ( X% s; X) J0 Z; F
) ?1 _; f1 b- S' B7 \- _* P
/iscsi-target...odb03> auth chap=admin/redhat+ h6 K) J) S$ }) i
! [* N0 L4 p5 ~# ?' B9 }
2 [2 U* E2 U2 ]' u
1 }2 @2 S+ w6 {; i& x/ ~, j1 o/iscsi-target...odb04> auth chap=admin/redhat
0 Q; M% s8 c5 `0 N6 a6 u8 ?* D. L3 h+ r( ~/ z; A
/ R! p/ N+ n {3 o
3.6 客户端映射磁盘
- N1 v; w0 y$ _4 i/ M9 [5 D' |9 s# `+ U3 X% ^3 q+ l! I
/iscsi-target...odb03> disk add vol01
# z9 [3 u/ x$ q& ^2 c3 p7 @- f* C+ I# p0 |
, a* ]" `, D- U( X
% U F0 J& U( N* M8 aok
7 r* M3 O+ Z! E$ H) n. V7 _0 ~1 O! V- W8 j+ V
- 9 H# n- d0 ]0 {( C( A
( G9 N) z4 R) y( p( Z. Z& h/iscsi-target...odb03> disk add vol02
8 ]8 M' O8 V! E" G) y5 b9 c$ U% o" @! p+ F
- 5 C* B% B4 V4 P! L" q- D" l
8 ?# h% n6 u/ M# Y- u4 k' j8 R( g2 ~ok2 @! P# K J/ q4 e5 P
. l5 S4 k6 c7 D6 H- a) x4 F
- 9 f& W4 j x! n- U$ _/ D
' I8 Y( K5 P* `
/iscsi-target...odb04> disk add vol01
1 m; i% \' D# F" Z b: |& f: p
( v" @3 i1 l# E - 3 U1 t* h# d/ v! y
( B) Q1 T) X0 ^* w1 ~: v
ok
: a9 V" M5 m4 O+ _0 Y4 u7 S& [* n0 e" d8 _+ y8 k$ |+ T2 _0 {6 _
) ~& k' o# n0 G- ]
* a H v$ i2 r! n& y/iscsi-target...odb04> disk add vol020 V8 e/ O# t. O( H
4 l7 U* [* x& s7 i& S
2 I% i6 u# K: ]
. k- G2 E7 @. z; F5 {( f1 i2 ]ok) N) ^ o* ~$ U: y3 @) l; L1 e9 d
7 R+ F# g1 F; \: o
% m4 h6 E1 r, o: ~# w
最后的结果如下图所示：- T4 c( l: `* a: S7 e8 ^) W& ^
 |
发表于 2021-7-9 17:51:57