|
部署ISCSI服务2.1 安装软件 ceph相关的ISCSI软件包可以从redhat通过的源代码进行编译或者下载centos已经编译好的。 - * }/ \" J/ L4 Y- \6 F; z) |
0 @2 v3 q0 y h7 b1 g
[root@ceph01 ~]# yum install ceph-iscsi-cli tcmu-runner ceph-iscsi-tools ; Z' ^( v! l- i: p
% D# h1 }( `9 \+ B
+ v4 K$ a1 ?% ]; R# A! C0 {' v% r% ~! ?8 o$ R* Z6 @
[root@ceph01 ceph]# ceph osd pool create rbd 150 150
3 P* j$ j. b. a0 O+ P" G' l, R4 R- X
- 4 S8 w6 G! p9 Y& `3 S
: s& J6 W# O7 u! F3 P% }( m[root@ceph01 ceph]# ceph osd pool application enable rbd rbd --yes-i-really-mean-it
P( `, R$ I W! t3 ^+ X2 n: Y( y; k6 K' u4 M
/ W$ m$ E+ Y' i5 O3 M: g
2.2 创建配置文件创建iscsi-gateway.cfg，此文件主要设置iscsi服务的网关。
* S5 ] k1 ^' i6 s$ K& ?+ |1 t( S" B5 t! V7 I5 L. L. S8 w
[root@ceph01 ~]# vi /etc/ceph/iscsi-gateway.cfg 6 _5 @ R1 M+ W. w$ H
$ F% ^6 x* }7 \8 {
3 j9 j* ]; k0 d" N* r' W R0 D: S* Z: M' W
[config]. g1 v# `5 F: l0 P B
- b A% ]" N# a h- @- q
- : c8 M+ J7 p. ~$ K1 i
5 M4 O7 E' P `( F# Name of the Ceph storage cluster. A suitable Ceph configuration file allowing
4 m+ A3 t+ Z0 d/ z: y/ A
5 n: f5 T$ A9 h% s - , ~ x! }( x% [) V3 s; i/ e
! J2 X& D- ~3 y# r4 m- a5 F
# access to the Ceph storage cluster from the gateway node is required, if not( o9 [) ^; u: v% ?) c
6 u4 [, Z% t, ]" b" j
- * Y" d3 w! T% i' ], P
) I0 @/ A- p4 [' u# colocated on an OSD node.
$ c& c. e, B1 a, ~! |2 P! A- w) V% p* w/ X$ W
- & o$ E+ Z7 s# N5 ^& a( j
( s4 l( C: y c( `# }: P) Ycluster_name = ceph1 g/ k) h* f3 u& N8 ]7 a: h7 q5 H; b
* ]9 E& S: M' H( z1 u6 R- T - ' a' ^# w7 r) ^( G" s
# c. d! |% M% c( O
7 S7 z' c8 T2 p( J& V: H2 L& h% `9 a/ K, g& Y9 T
+ f; f0 N) f; J6 t/ C1 R
3 M) K1 t; U( Q# Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph
) g+ U- w; q; A/ q" [- w5 k0 W5 H# D1 }% ?% U8 Y8 K2 R/ R
- % a) I4 B% x" a
3 D9 L6 {: d, Q9 i
# drectory and reference the filename here; d7 z* \2 O; F
) q! u9 |+ y6 b' ~9 C% ? x - % D$ O+ W5 ?1 _4 b
' d, Q. F1 d, r4 H/ @
gateway_keyring = ceph.client.admin.keyring
+ V; C* n2 _" G5 X$ ?! I$ N- q( V$ Q; ]" @( a$ c
- , v5 x/ |* P' i8 Y4 ?* P1 m
3 q/ q4 m# X8 L N4 z2 P" B$ e* |, s$ {# L& `' S- l: ?! ~) _
% L/ x& R: o5 b% n( x2 V4 n* V# w
- $ B( U+ U6 b, u& A n" h2 L( p
- i3 u! W; I2 s
# API settings.
9 a0 r7 p- V4 m. |
6 J$ X8 t8 z5 d6 J' F
) w1 F6 z9 {0 {( _( B
( n, B$ B( I9 D# The API supports a number of options that allow you to tailor it to your
8 j* p0 [. \) S- e$ h5 S, [6 c4 O: |0 F
- ) W3 Z: x8 Z( t# ~+ `0 J7 m* ] \
" e$ z6 V8 J; h: Q( y, ~$ ~# i! M# local environment. If you want to run the API under https, you will need to
& ?9 X! ^, {" ?1 g9 q3 I; X6 [( {7 t8 `0 l0 ]9 H. ^, F7 p
8 E7 R# [& M1 U& t' b
% k# S% y/ q9 ^- e- T# create cert/key files that are compatible for each iSCSI gateway node, that is
: D0 ]5 d# f D2 L2 }: q- c) D/ b" P/ c
- ; H1 P* ^$ u3 |0 e2 r8 C
8 [1 i2 B1 k3 e! P
# not locked to a specific node. SSL cert and key files *must* be called
) v" F, T \; s$ q4 O) q7 H# G+ ?: D+ m' M+ \: S9 N/ [; v$ g, Y
6 N9 j2 O4 M6 m" K4 Z- Q
5 M' ?6 p8 K! t8 w3 y5 ~% \$ g9 S# 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory# }6 \& M5 Z1 E# ?
1 {! t0 v/ A8 E: {+ I
- 2 f8 v2 A2 y6 t) ?# p$ ~
9 U/ N/ `( F/ K. B/ ^$ {$ m" D
# on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'
' W3 ?6 D- P; ?, E$ _
+ }. s: }2 s" h/ i8 p: q, r S0 j1 A - % l6 d- L" C' q; w" C/ f- x
% `" b3 v% E D3 C* o! @6 s# to switch to https mode.
8 g) g. q: c0 y) `8 A& {4 R& i7 ?0 u( X4 M/ \0 ]
2 @6 c: [; y9 x+ z. j+ w8 |) U; a9 O" |' @
2 l" [0 G* Q% v4 \. W; K3 @2 E5 p$ y5 F' S3 m% m0 `& R
. O) A/ H! | V; |+ h) h; o! ~' x/ v8 q5 q& Q" _
# To support the API, the bear minimum settings are:9 G+ s+ f; S% N3 v1 {; r
2 d" |- f' n9 o7 S3 L$ W0 k1 ^
3 S. N' z" b4 g/ M5 `: B. P7 l3 \* ]: ~: `% n& h
api_secure = false
) P" u- Z. x- S5 A! N- [# A) E9 Z, p: e, e1 S
! T# P# X- e6 B$ q r: s8 C; X& b* {6 R# \( y0 C. S
3 |" q) u6 o: q% W" m# D" k
- Q1 x5 Z$ U! i9 ~' ?& K8 z% f
- a0 R7 J4 r5 O+ M" u( {" o, w5 e; s9 K0 l" I* s5 i4 g9 S0 h& i0 s
# Additional API configuration options are as follows, defaults shown.
/ l* [* k: g7 d/ K3 `% l
+ l" ~5 A% H- O: U" E
+ h0 m z! A, h- b$ P+ M N# [/ Z9 z- ^: c6 l% n
# api_user = admin `/ U3 k6 L1 ~- C* j- m
% N" E |' m# }4 Z
7 z5 N2 }1 Q7 S* j" p( F0 T0 B9 i
# api_password = admin* N8 h! h+ H& G' Y7 t4 B- b
* G. O3 P! d V4 s; ^' h- % l' a+ r0 m0 Y/ }- c
* u* o- O# g3 ~2 _6 A, X# api_port = 50014 S/ Y( B+ |5 U' T3 t" _
3 P( o8 y, d. G# {+ N! s1 `3 t
- : ^; ~* G) k: k0 R' P
: v- X- C' B7 i/ G# W, |4 ~
trusted_ip_list = 192.168.120.81,192.168.120.82,192.168.120.83( m3 S' F, {! V# g2 g+ e
2 C4 Q& e- [% [5 k! j. \, L. n y5 [+ t: b. K) i$ R/ x A; c" \. F0 R2 |/ I
2.3 同步文件到其他节点- ' d/ `! I. z3 u- ?
& J1 O* }& T `& Q: e* K
[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph02:/etc/ceph
4 W. ~) f: s1 s8 \2 S$ ]0 b/ |0 ~6 J1 i4 Z( I. I( c
- ' `( g1 g# ^) a6 S' z7 C5 l/ q
$ K$ W% p7 K- P S
[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph03:/etc/ceph# a2 M9 o! L1 y: m3 a7 X& Y
5 f( ^ s' r5 J, W# V0 b, Q
/ S' v$ H3 E% W0 o6 o
2.4 启动API服务
8 F3 x( l: B" d/ b6 }. M$ E5 G
& z1 q! H, h ~) _7 z( Z[root@ceph01 ~]# systemctl daemon-reload+ X( {2 O5 a) g6 L" w. C# ^9 {- J
% Q( F4 ]2 U8 n2 A5 s& R; T1 z. X- & {1 E. y( N L1 e/ J* T, ]# V
) i, P* q0 h- M0 z
[root@ceph01 ~]# systemctl enable rbd-target-api
+ N$ `! y; E. Z. f
3 p' \' ^0 K* Y( F8 H& ~/ F \
2 Z/ Z) J" L9 e8 T& |7 p; m6 U0 G$ B9 b" w# r6 }: I
[root@ceph01 ~]# systemctl start rbd-target-api; d, W8 V, v+ u1 f
* _3 s$ S6 ]6 S1 I6 ^3 N1 A$ _
$ H( z) V+ }" u! B
; O' @+ D# K, L Z9 s# J[root@ceph01 ~]# systemctl status rbd-target-api
) B. p1 t' _' o s4 F3 N7 ?$ v0 V6 B/ L' Q" z+ W$ q% m1 i
- + Q- W) D& @* J
/ ]# C/ s! D" n. Z: p
● rbd-target-api.service - Ceph iscsi target configuration API' u& [4 a, t# ^, v. V
% Q6 |: R2 }9 x2 v1 T# M1 Z$ G, d
1 m' h, M$ t1 I a; g; H- r d& P B8 L$ Z, o
Loaded: loaded (/usr/lib/systemd/system/rbd-target-api.service; enabled; vendor preset: disabled)
6 c% S0 x+ C* t) e6 d3 n8 z
5 W! U: B( \! ^. r1 d
, ^& C; U+ X9 m$ S5 l0 O7 [0 R. D: x3 s% d
Active: active (running) since Thu 2018-05-31 11:35:04 CST; 4s ago7 n" L; P/ e2 S* e; C
1 t$ v/ W4 Y7 }7 X- h9 H% T
2 w5 k7 u# l* E/ W' R0 x
1 h- u& z, I$ \. ^# S Main PID: 25372 (rbd-target-api)2 }# Y( Z! r% w& E! H$ b/ w' _
3 N( R3 V: l* j/ E+ V
& A/ Z/ G+ `$ N/ E- b. U8 z) {8 X
3 c$ q& B4 Z: u9 b' \8 D+ @ CGroup: /system.slice/rbd-target-api.service
9 R1 W6 Z& ]$ P2 M; E [8 n- P2 O
, S& s, n( ^* {0 L4 T' J( S- : M' F) h7 y: X7 H2 Y! N
6 g" [0 E0 \4 s) G* p3 i
└─25372 /usr/bin/python /usr/bin/rbd-target-api
! L5 d2 S0 H* B9 i; r# w2 d" s0 P" _7 |. o# v2 S& @) @1 ^
Y% ~. r% W* L$ p' \& {
9 g% S: s" K" k) F8 i7 n2 ?1 i' w7 t2 Q
1 k7 I5 W' k% L, k, v( E' v& S8 U5 i3 L: k s0 O
; J) {! p: B, m0 h5 Z$ m3 _" c0 O' Q; V3 v& [4 Z
May 31 11:35:04 ceph01 systemd[1]: Started Ceph iscsi target configuration API.
* ~4 m8 X$ ]6 L. I1 O) x% j7 |& k( v
6 q, H9 U$ q) g4 ^4 }2 d# x- 3 J) ?$ r& j; @# K+ B; \1 q" T
" ^: C1 l( Q; R) O
May 31 11:35:04 ceph01 systemd[1]: Starting Ceph iscsi target configuration API...
! G w( C! Y8 t9 a- V: i( a+ f
' w9 O. ^4 a5 \# s; G - 9 \( E4 r( g6 r" O' f2 @
7 D. @. [$ g" l9 j' O6 R' W1 D) h
May 31 11:35:05 ceph01 rbd-target-api[25372]: Started the configuration object watcher
9 N0 _, J2 M3 [6 E* @5 [% S* K+ W& b' g0 H' n, Z0 F% n
/ c, o1 i. ^0 ]8 v" q' m5 d' D) ?5 T! P: f0 ^. v
May 31 11:35:05 ceph01 rbd-target-api[25372]: Checking for config object changes every 1s
U5 n# m H( ~( e
5 A% Z# U% }$ g9 V9 b; V$ U0 g( c; `% w- ' V$ \ O; u' h% F- r3 r
8 x) w: \9 F# S) R9 {, H$ k$ RMay 31 11:35:05 ceph01 rbd-target-api[25372]: * Running on http://0.0.0.0:5000/+ Q4 l% U8 w4 q5 j5 @) ]
1 P# S# W( ]& w( r, W4 i3 i6 g/ L* P; p! l+ J& R$ j) y
三、配置ISCSI服务3.1 创建target- p; k! a( e& e- o! z# Q
) i0 _+ ~1 Q$ M$ w- d
[root@ceph01 ~]# gwcli5 Y7 [: b( O4 A% d0 ?$ b* y6 Q
1 {( g1 S7 i* l: C: O1 C - + C) p( a1 B, R& j4 W
1 [: Z" v! J. b
/> cd iscsi-target
. z! g# f7 T+ s4 D; Y) U
- y, b& Z* d, z# M" p
& ~& c! @. D* _, c2 f J
# x6 w ^( T7 {- g+ l/iscsi-target> create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw
( Z. d$ Y1 o+ M% D T; V
; M) Q( d/ W( g6 L7 C/ Z# p8 N9 i+ q1 N' j3 x' y. R: ~6 F
3.2 创建ISCSI网关
% ^) K& p! h% P2 ]0 l+ T8 _" {; x/ k2 f
/iscsi-target...-igw/gateways> create ceph01 192.168.120.81 skipchecks=true
+ `+ ]8 M9 n; t( Q0 ?* ~( U$ p# w" Z7 B9 s+ ~! i6 F# J4 C) H
- * q+ q0 b+ M, P3 b1 @
* C4 ^8 W, {4 SOS version/package checks have been bypassed( C4 d7 r: F, b! F
3 b# B0 J, \ r' v" b( t4 u - * N+ g1 z0 l( |" N7 h0 f y
; r5 N; Z% i8 Z& j
Adding gateway, sync'ing 0 disk(s) and 0 client(s)
& t6 Y, i9 l: L* i& u
( W7 O: u1 H* v2 o( V+ @4 r - * O5 F o% T3 U
8 n/ Q. k9 O& Q* C8 z
ok
8 n( Z" g9 @$ o3 e, L6 T$ ~9 o7 I2 D9 m
9 G; l. Q# U% N) F' {' | N; S* O! X: c% }$ B
/iscsi-target...-igw/gateways> create ceph02 192.168.120.82 skipchecks=true
[" z) [+ L$ {+ [8 i) L* m7 F, ~, q& V8 q
- ) ?: M3 k2 _ M. P4 _& N) A
2 I' l' A+ ?9 L) U7 ]- L
OS version/package checks have been bypassed' O, M2 ?! x! _/ j9 l% Z
: ~3 d2 w& A# S- e7 o
- * t7 L6 p. `- L6 |! q9 @
2 h2 |7 a. W9 i# K
Adding gateway, sync'ing 0 disk(s) and 0 client(s)$ O, d" j- D& J( T1 l% M* G( c
. i7 x( `7 l7 P }& E- |8 O# x7 M - 8 W: c1 |6 A; Y. E$ r
3 f7 y: o1 l) @8 Z* z0 ?1 g
ok: o+ T. z# p& ^* e
0 |% p$ L: e7 V- Y
7 l1 `' c$ @7 D ^! f) c% ?6 ?: [) {1 {1 E& C4 @/ Q, e2 @
/iscsi-target...-igw/gateways> create ceph03 192.168.120.83 skipchecks=true! l* o/ w$ y0 V+ q. P
: ^: ?6 q$ `3 L" n$ X! b' M
) k) m& P: Z7 u4 d1 G
4 q' q; U& O5 A7 sOS version/package checks have been bypassed
, ^2 h I) \9 y1 ^5 F# ~# u- A* j6 K" a8 |7 c- K* f9 |: Z
, U& i+ F# Y3 v5 T. Q s Z8 b) }: U2 {# W8 b5 O; \
Adding gateway, sync'ing 0 disk(s) and 0 client(s) c2 b/ a+ S2 `1 V- U
, j2 Q7 I% Z6 f8 P8 f
8 \7 ?, ~ D2 \2 `3 k+ c4 ~4 Q
+ n4 H- I" [- }/ u/ k! P/ xok1 Y6 Q: L- B( A! m8 R! m
# b8 q* v3 }0 ]7 N) v- s0 B- 8 v7 p2 W6 i; M7 s
3 Z5 ?. ?6 Q( u1 S Y; j" B0 @/iscsi-target...-igw/gateways> ls
; I- p' y" ?/ Y+ S1 o) w! e& I9 }) ]( ]9 `# |( ~; q1 X
- 8 B( @9 b+ L8 {" R0 U- ]# G* ?. F
' R7 n0 l" `6 X$ b7 Ho- gateways .................................................................................................. [Up: 3/3, Portals: 3] g4 \# T' ?+ y; F9 O
$ U% X# Q: j4 n - / y7 v9 H2 ^* b0 D B* b
- v3 K$ j7 v9 W: K" V. S
o- ceph01 .................................................................................................. [192.168.120.81 (UP)]0 e3 k; a6 r9 J( }1 w9 t
* w. w% H+ N- E; v
% X P; k4 q! e! L; x- H$ N( |# F1 Z, Q, A! Y2 K- I
o- ceph02 .................................................................................................. [192.168.120.82 (UP)]
3 ~: g+ O) Z9 }
) a6 _! H' U$ Z* s- 6 j, m- A4 ?8 u6 ^- w) r
6 J) _0 _4 w; `3 z/ X: \
o- ceph03 .................................................................................................. [192.168.120.83 (UP)]5 H/ a! e7 w1 G7 q+ F& |
& i' i4 z3 u( {- d: j) J( Z0 V% D' d
如果操作系统非Centos或redhat，则需要加skipchecks=true参数。 3.3 创建RBD image- : O T8 H; m3 M% ~ H1 ?
& v$ y4 ]( X3 X3 R' C1 z/iscsi-target...-igw/gateways> cd /disks( R ~" p0 N+ |
5 U! l" q. m/ r. `, [) A" j
0 F) n) t7 o+ ^% C0 b( |9 U
: G3 ?0 y4 b/ B! N1 v; E' b+ g/disks> create Oracle vol01 100G2 u7 s9 ^* z. I0 s; A! r
9 A) S- u( m$ x3 m% o# n6 y4 A
, J4 D9 T( I$ I1 [+ p5 {0 V K2 M6 ?* [7 l* P! _0 B
ok$ c* @2 Q+ e$ w+ b) G% x
! j! y% Z. J/ {) j6 n( F8 l
' Z( W; o5 O# ]7 ~3 d3 s: |2 X4 ]; l5 t, t- @3 p
/disks> create Oracle vol02 300G
( x. g$ U$ A4 C. ~- n! ?+ ~& X0 P$ D [7 p( ]6 p. Q5 Y
* _7 i$ G* q0 h# [2 H3 E9 N% D# Z; x( M: V
ok! L' R" M/ c/ d: M( E( j
6 }5 Q: |" M3 @/ R
3 s) M3 Q: B# s- }
3.4 创建客户端名称Linux平台可以查看/etc/iscsi/initiatorname.iscsi文件获取InitiatorName。如果修改了默认的名称，必须重启iscsid服务，否则在登录iscsi服务端的时候会报错。 - ' u P( q. _2 }0 \
* k$ U Z7 s( P' g, ?% V" y$ [+ n/disks> cd /iscsi-target/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts
! Q* G3 g5 d/ s# d4 L
5 B- c+ J! o* F/ I
2 O0 j& K# s/ r9 @; B- V. f7 _! f" f' ]6 P8 `2 \! W
/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
/ j- g U! ?6 P" M4 v; B5 ?$ q. o2 @4 ?
# T0 U, C( D5 [- ! N1 [- n3 g+ ^% E- f: j" u0 S9 |
: W$ U9 `" P0 _, k: F! jok6 r+ C# F; I; e7 _
* A$ P' b7 a9 o1 [% u$ B5 Z4 V
% p. h; u5 k7 ^% j. }5 E/ D, S6 h' i e6 Y' z1 \
/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:ccd061606e1:odb04
; S t8 C8 m6 z- x# Q' i; Q% ]7 C4 }3 D* }
- 1 c- Y# u& R5 N; ~6 f
* ~' S# V0 o( c5 P2 ^8 Eok
8 ^) u) c9 ]3 E, g) H: x
- P4 @4 n; M {
8 H2 E/ [+ j" _: V. }' K
3.5 设置客户端认证
/ q1 [0 B% y, }% ^0 d3 A
8 e1 R* c) v/ q/iscsi-target...csi-igw/hosts> cd iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
; u9 R, S3 D( |3 j# M/ l( @6 A8 W
, K) Q+ i5 N* ]* T, y, T- , C7 `# h) f/ h
( S, k3 V! J, H: K4 R3 i; ^/iscsi-target...odb03> auth chap=admin/redhat6 Q; T* O1 c5 S: X/ D
$ e8 h+ H* i% f% X3 f: R, v" D4 V - + v- t/ V1 L8 C# r' P1 E B' e
! u- E1 E( P$ q4 q: R$ A8 G' f @1 r/iscsi-target...odb04> auth chap=admin/redhat* e- _- t( h8 B8 x" w6 ]" U
- V/ @* j$ j x0 G6 j/ f
! @5 G3 e7 H5 l' s
3.6 客户端映射磁盘- & O- j! Q5 p0 ]- [, S
8 K# @: I8 U& N0 ^7 z/iscsi-target...odb03> disk add vol01) ]9 T$ C0 z- k* S' Z! a" n r' ^
9 T: P0 S- b% X% U; `' Z3 c2 J% y- D
- 7 T: Y* t# l: R7 v9 Y
" X# @5 J& Q5 o6 ^: eok* l1 F H0 }9 w* b! n
* J, H. N* k! f& H
- \1 |& k0 P* [9 Z. D
5 ?: p& F" T7 E7 m6 N8 a
/iscsi-target...odb03> disk add vol02
7 F7 k j o2 Z8 @4 j& X/ T6 c" ?8 Z
- % @/ V7 W i: j1 [4 s% R$ A5 Y
* _1 Y- Z( V) I! n. aok: _. `- ?4 W z9 [# y$ s i
! L* q* ~+ H) k
- w* D% B$ H3 V8 y- O. [
, c5 J/ o# X% J! m6 ?
/iscsi-target...odb04> disk add vol01$ a9 \! v) u. M( Z0 g' q, q' \ X
; V2 T# J X4 F W. C
- " r R8 |$ v2 D( ~) F
8 U" M$ P4 Q; a' N/ q" ]# M! }ok3 Y/ R6 y/ Y2 N
+ L6 ?1 U) [- E: S( d8 s
' R- O. @+ Q t# D- R$ w# f, {3 q2 ]- }
/iscsi-target...odb04> disk add vol02$ M8 i# o8 G7 I- B
' g0 X" G2 R* [) P& j- E# }- * J1 O" s! Q0 \& h: w
* e# K( |$ r1 |
ok) h: R0 T. T; X9 }$ L
) D# I" I: U* g+ o8 X! K: i! ]4 e; M" \- l# s6 y
最后的结果如下图所示：' P) \# W, C, s# t( d9 g
 |
发表于 2021-7-9 17:51:57