|
部署ISCSI服务2.1 安装软件 ceph相关的ISCSI软件包可以从redhat通过的源代码进行编译或者下载centos已经编译好的。 - & Y+ O9 O) G* Y: u
) c' p# m% J1 U0 M7 W
[root@ceph01 ~]# yum install ceph-iscsi-cli tcmu-runner ceph-iscsi-tools
( v3 ^/ ]) a! b a v; K% b6 P3 H& \' N) W1 T- \0 I* D# n
3 I/ a0 M. N. `/ b' c: r
" l! F% ^9 j8 w6 ]7 b" S' @8 [1 l[root@ceph01 ceph]# ceph osd pool create rbd 150 150
/ E) z) u+ u4 \7 j3 P
* k0 ^: l0 F# z$ z- ( C! c& |- v: A5 Y9 H
& a/ e! D) l( {9 ?$ u5 P[root@ceph01 ceph]# ceph osd pool application enable rbd rbd --yes-i-really-mean-it6 E: P6 q1 `- {( ?. P& K
& _4 v1 P# o( ^
) V0 L1 I8 N4 X( D
2.2 创建配置文件创建iscsi-gateway.cfg，此文件主要设置iscsi服务的网关。
7 o! D, W4 R M. Q; @. c" H* {# @ ~
[root@ceph01 ~]# vi /etc/ceph/iscsi-gateway.cfg + N4 o6 N9 }! T1 p% J, h
2 u" L+ Q% S! n$ k
7 Z" G }7 Q+ @3 \1 C ~3 \ k4 T, J; p% r$ g: j
[config]
8 T! q$ ]: b' ~. f( G' K% K
# H S' o; s( Q9 N- s- 6 j! M. s @/ _
/ v4 e2 R2 [; S# Name of the Ceph storage cluster. A suitable Ceph configuration file allowing4 q" ]4 b' ?: D2 z
/ N1 G7 }$ h$ d1 j) p
7 w5 ^5 @) i; R0 e2 _) o
" o* a3 w j+ \3 B1 W0 v s; L* E# access to the Ceph storage cluster from the gateway node is required, if not8 p% `' h' x- }$ S7 e; P- O
1 ]! J. ^6 G% s4 Z/ ^
[, R/ b% a9 L0 Q
, ?5 [* N8 k j- r ~# colocated on an OSD node.
' p/ D1 x( Q, `, b2 S0 s3 _, `% i5 O- L4 _3 h- W0 g! u8 q
* B+ R: `, Q8 E
2 p/ c& L; w( E2 T" {. V1 jcluster_name = ceph K& U" T5 G9 d$ T4 ^
( e4 W* X) Z# ^8 \
- ( D2 j) R; i0 k, Q9 H1 l& F
6 w! u* I: J7 v3 Y! S. @
8 |% E+ j: O& i" j& q z# `* _
h M" K2 w+ ^- ]; F; f
- 8 A* F* L/ n# \) N* Z& z" K F! t8 f
. i o" f5 i) i# \, o. n# Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph" M0 Y5 J5 i3 y7 n- g. H9 k! k$ b
. L k- h) {; E" G( A( E - - \7 C F- T. _9 J4 V3 `" z n g/ ]
, m& e7 ~& \6 t: Q# drectory and reference the filename here8 d% {, Z) R' { s7 \+ t, |
" j3 _: ^6 O6 Q- V
5 t" ?- i% ^ s& ?7 `
0 [" \7 d4 `1 v C, g; }gateway_keyring = ceph.client.admin.keyring0 u& l7 p. F! h* n# z
# H9 B2 o u2 i* S
8 C+ k9 N) A# i# [5 k' D N; G- e
2 e6 ?5 Z. F% O+ Z" f& `( Q! A! K [: a* k* W" V" C" _
' g% L2 M4 [% V2 t: c& T3 |8 P
- ; h$ O9 { T! u$ ~# o8 Z6 o+ O% l
& a# ]& ~3 W R- \$ ^# API settings.
- t% |! c4 T U3 P5 ]* f
% L `6 d- Z f/ N9 O9 R
' r) k9 ~- q, `' E0 t2 S: ?' ]" |4 s: \7 ^: g: B+ |" y
# The API supports a number of options that allow you to tailor it to your4 d& y) ~1 X4 |3 f8 Y# [* n
+ w0 I. c5 X8 A& h
" D' o3 ~2 Q% E
6 p1 O# q' ?8 {4 G3 c- v1 ~' O# local environment. If you want to run the API under https, you will need to
0 o# l" o3 O6 m9 V+ }9 T2 |/ [, j9 \( B1 u8 V0 _
- # B0 [7 c1 M/ o: q* v' `+ k
4 t0 J% c }# H3 B7 A
# create cert/key files that are compatible for each iSCSI gateway node, that is- \; }& Q' M2 t$ l- `( ~
5 e1 L& u0 H! A) [ - & h% R- _ c( L
8 {* s. M8 S5 ~, ^7 @; S8 q. H# not locked to a specific node. SSL cert and key files *must* be called
0 i+ o5 w( V- o) y0 e* ?' G
0 b2 M0 `4 d) t0 @, D. G
$ Z% {1 b! z Y' a$ _( y
3 `8 b" \7 |( R5 U1 |" J& {# 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory
6 T! X6 W; f! ?. S+ M8 u
- A8 F( D# t& u- _" |3 Y! V1 c. G- 9 |9 |+ o' L7 [+ n) |
- E) g: v7 ]- s! l R, N3 v
# on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'1 S; N5 d+ C e2 Q; ]- j. q( o
# c% X) g7 \$ U" X0 W; K
- 3 Z9 C* \2 Y" c4 H
) z! j U. U$ `. Z$ @4 f
# to switch to https mode.6 K7 |. \+ \6 ~7 i3 f
4 R2 v4 x7 f4 E* Y5 d5 U3 m; G
# ^( c8 b: u2 w5 }& i7 x& z. F
% V s$ ~/ a1 [/ X! V, C" ^
+ U4 s, O7 w3 q- z9 i8 L2 X4 L& N) q) d
- , h4 R% z% z% o. s4 f
+ K& Y0 a& U' X r: s7 a# To support the API, the bear minimum settings are:) r7 B5 w2 }" M0 ~
$ ~9 G/ ~8 X+ `' Q/ R- |) D
/ K* s1 F) R+ r( k r* f
: T4 m0 Z. v7 [# r% Lapi_secure = false' I/ x: k9 e X0 i; H" m7 Y
' S8 K& K5 n! C$ d- . I+ B4 Y" x$ {4 o3 }5 a! G+ S
9 q, h* ~4 H, Z$ F! S- f
3 Y" e# X/ q8 ]& v
4 j" j! @, P( ?- c V4 Z8 p
" I3 P; m4 ~9 h* [; l* {& k; n- Q
: Q2 j1 M6 U1 a: N# Additional API configuration options are as follows, defaults shown.
! D/ M1 A+ ^/ d3 [3 H" k5 B4 z B4 _/ B k! i
- b; c$ M: p8 u8 x/ ~" z! ~% N8 z$ @9 Q; ]$ y2 k9 K
# api_user = admin
, b; W3 m$ G9 h4 B8 `
1 K0 d, {- o; y. k
7 I+ c3 t5 ~% w+ T" Q1 g$ D+ `8 [% m/ M" A
# api_password = admin
8 H# N5 j T+ H/ W3 d
* X3 q6 [; P& c+ A/ e
8 g9 p1 ~: _ M" l: z! P
8 g, w6 H* o7 d# api_port = 5001
6 C0 w1 |. r8 Q" q" @6 Z0 N
. Z2 h) u& ] I9 H# [, \# a- 9 \- e: s Q" w3 x$ d; z* L
' e& t$ A0 M3 J: W6 m8 {
trusted_ip_list = 192.168.120.81,192.168.120.82,192.168.120.83; i, _% @. I7 i# h/ A* g) I
9 K; P% j, j; w2 @& t2 O
/ s: [0 [: x, o4 y) t
2.3 同步文件到其他节点
6 D/ @" O9 Q' Z7 t4 I
- j2 h7 K$ ?5 k7 l[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph02:/etc/ceph
$ ^1 K9 F4 A7 q1 [" C
; F0 \4 ~: `1 n, }4 g; [- e- 4 K5 ]5 R, d! t W2 {
3 r h) B1 D, i2 t9 K
[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph03:/etc/ceph I8 u! o) D# N# H
: B7 J$ Z7 c$ v; `1 X" K
4 ^/ U- j- |9 L0 s3 T. K
2.4 启动API服务- ' k* J) n: K9 E6 D; \ t
$ x8 g) e) \3 f( S, C7 v
[root@ceph01 ~]# systemctl daemon-reload* F! Z' b, m/ O) W
" Q# H3 C* \3 o- x
- 4 n" M/ D% Q8 b* H( `; x% @
/ [0 N5 j0 Y& u) U9 i0 k
[root@ceph01 ~]# systemctl enable rbd-target-api
: }( g7 F9 o9 ~) k p
, A; f8 \+ a- D6 x - % `* x2 X& y7 P/ ]/ R( z2 R7 _8 H$ |
9 L, h/ [% c; z# d, V* c" x$ D9 a[root@ceph01 ~]# systemctl start rbd-target-api# k2 B; \+ D" h* x' g/ M0 S
2 c% r) j8 k, M9 t
2 Z* L: ^8 e1 b; \0 L6 @2 {
3 @# q! o0 i. z/ C/ t6 \# R. z5 C[root@ceph01 ~]# systemctl status rbd-target-api0 k# O3 h) I9 A2 a
! ~ f7 u3 Y6 M& d, d- `
4 o1 E% A* u7 e4 T, k6 t9 Q
- n$ z& c3 i) k; h# Y● rbd-target-api.service - Ceph iscsi target configuration API
, q& m6 c, H" f5 _6 x/ a# y, \7 h, |
7 R2 F! [! c+ @ b8 R# O Q, ^0 \/ }2 V# t% g: L
Loaded: loaded (/usr/lib/systemd/system/rbd-target-api.service; enabled; vendor preset: disabled)- ^ {" G; @9 u- h( D ^ w. j
; q" u# D6 q0 c# N. ~8 P- - F/ E7 B9 m' t& c- Q5 Q
5 H8 e% k- P* d8 q
Active: active (running) since Thu 2018-05-31 11:35:04 CST; 4s ago6 C5 ]2 C" X$ x5 a
+ Z2 k- z6 G& k! f" K
- % D9 F' M# O4 ~9 y0 g
1 a* X3 E# W( M- w3 w Main PID: 25372 (rbd-target-api)
( R* z/ S: E1 T2 p7 Z. L- F3 q
% H t' z( _& c" j7 S
" f; P. i2 ? `/ Y7 x- M* B! _6 ?
, a: ~% ^3 X7 n8 ^6 A CGroup: /system.slice/rbd-target-api.service$ [) m/ H" _6 \! V' m; z P
7 |% P" E4 R8 w7 z9 L5 g- ' B) g# Q4 @+ T) H4 s
: |. `( j+ G o3 k └─25372 /usr/bin/python /usr/bin/rbd-target-api H) C6 |7 X& D0 t3 P
5 I' |! V1 w0 |5 l" F7 W1 v! @
2 e) s" w) J( N: F N
. [$ W! H" Z! ~
) X. _+ o8 V5 d5 E$ S9 L; b5 `$ s4 u0 Z
- 9 ]* m) E" D5 q) h
( c1 A' b* n9 F9 _1 F
May 31 11:35:04 ceph01 systemd[1]: Started Ceph iscsi target configuration API.0 A8 h7 r, u- h. e" g2 `& d+ p
# z O! J% r9 a$ H p9 B
- # I; d% i" ~9 G$ D
& I' N* z* m3 d5 c7 q8 Z+ b2 ZMay 31 11:35:04 ceph01 systemd[1]: Starting Ceph iscsi target configuration API...
- S4 w$ s1 X% }3 S/ e0 Z8 N# H1 J. t) Z9 c" p' r
. A% {; a9 s8 k* K% s5 K4 i6 F) p' k$ I# B3 A
May 31 11:35:05 ceph01 rbd-target-api[25372]: Started the configuration object watcher
/ O. j6 } M g2 \5 H6 _, D Z( @/ m( O; Y6 m, P1 r$ p9 q
- $ t, u, ?; Q9 j! d; ^5 {( W9 ?0 k. K n
3 F% e4 T$ Z$ J
May 31 11:35:05 ceph01 rbd-target-api[25372]: Checking for config object changes every 1s
" b4 G" x$ P# O$ }0 g' _! w0 f$ {" r0 ]% Q X, ~
- 1 C; ~0 G8 @! i3 r7 O6 L
5 k! m, f; C# T4 o0 B7 WMay 31 11:35:05 ceph01 rbd-target-api[25372]: * Running on http://0.0.0.0:5000/9 Q8 v, P }+ [, B( q$ T
: W8 m. s# {; k6 o7 D
4 U8 o& A( r; H& n J' c" y
三、配置ISCSI服务3.1 创建target- 1 f6 V! n/ r1 q- o- F6 K9 i" K
0 I Y: @% E9 V1 _% J, E
[root@ceph01 ~]# gwcli, e; H* d2 E$ [3 c( w
1 _' G, A+ p* w- Q F - }! a% V% `7 ?8 W e
$ s8 d! y+ O9 O& t5 }$ {0 E
/> cd iscsi-target
3 y2 _; x C7 C' x# w% i' w1 D
5 `- Z9 U, L" _$ I6 M! a
3 m- s" E: S( ?- `# Q% f" ?0 i% C9 b8 c4 U- }( z+ \
/iscsi-target> create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw6 F* n; j. q$ I( x k
- v# Q$ s* H0 V( E' c
o* r- Y$ G9 o3 {% v! h9 w
3.2 创建ISCSI网关- * w* U* N/ H. Y
( Q2 v) }. A/ \2 ^7 W, ~; J7 q0 m
/iscsi-target...-igw/gateways> create ceph01 192.168.120.81 skipchecks=true! V1 a. ^: [" s; w0 f8 A8 P
7 O' T3 S8 E: z/ q
$ o' [6 f% _1 b) @9 ~
1 h9 P% K( F% n7 q6 qOS version/package checks have been bypassed S, S I+ n& h0 p. q
7 D Z+ M1 g' R! V C9 q. `
- 1 O' ~ n( B2 t, |% o: g& f+ a
& z5 }0 r9 t* T! E6 Y1 p1 w# ~
Adding gateway, sync'ing 0 disk(s) and 0 client(s); ~" `1 o) F! Z1 ^- `3 H) H" x! D
( O5 V N5 r0 S& m( C2 F/ B' Z% t - 3 K9 z0 {+ z1 F' ?, r' K" z
8 a, O% Z; W1 l5 B) k& g9 K
ok8 x. |# G: _6 i) z! l
/ k/ w5 ^( Q# I2 _4 g; f# q
8 o8 ~; s4 k: L/ o1 n5 u
( o1 u, {! W6 [8 R/iscsi-target...-igw/gateways> create ceph02 192.168.120.82 skipchecks=true7 k- f5 o5 s$ a0 A+ C; _+ t8 G
, m" O6 L$ R* s4 `; H% p/ m8 T
- 3 t) r3 F$ ^7 K6 I4 ^" A
6 `. _9 L* m( m; ~* \OS version/package checks have been bypassed0 ?: D' T4 c( E& R3 E3 o! j i
; w8 n2 s/ S/ O+ G2 j
) g. Q. L8 @" J* I/ K/ J! b% D! [6 w8 g4 W4 Y+ V) f6 v# P+ C
Adding gateway, sync'ing 0 disk(s) and 0 client(s): C! C3 e3 ?+ b7 a
9 _" q- {. o( _0 O9 Y% O5 h$ P) K
3 m( ^2 ]- m. |3 L% B
( k3 @4 c8 ~: J* u' ~: S* Ook
+ L3 z7 r# i' Y# @8 k: i$ [* v; f" @3 S
- * y) W5 K3 k( Q+ X8 o
8 p/ a( Q D5 @2 B+ O y
/iscsi-target...-igw/gateways> create ceph03 192.168.120.83 skipchecks=true
% Y$ N- w4 f9 l# B) V; s2 Z) k! W5 d- l2 S( u$ k( L$ d# Q
- & z2 F5 ^) A4 z- n7 I
. |- `5 `: g0 aOS version/package checks have been bypassed
7 j5 U- ?! z& `% i E; U3 r* k1 `. i# ]
% H( Y6 T+ v$ q+ K+ t+ \
0 F' [" V6 q6 YAdding gateway, sync'ing 0 disk(s) and 0 client(s); R. ^4 m* T+ G
2 f$ |/ z# G- K2 ]0 G( \
! Z6 F1 m" ?, I' m: V
4 e& h/ B- z9 _. G( A5 E( x+ x' Qok4 k0 F7 C! \$ l [9 ^3 j$ E$ ]' j
0 z* s9 K: q/ t/ s
- / Q8 _1 A) j' a+ v2 J# f
2 \* G1 ^; U# y5 b
/iscsi-target...-igw/gateways> ls4 j4 a( V+ B% V2 J
6 ~- r. Y: @+ f: i# g
- 9 G9 z" j) B+ K8 I5 y X
$ n" g: }$ r% }o- gateways .................................................................................................. [Up: 3/3, Portals: 3]
$ S* N+ t0 t! {2 H9 V2 G/ f( e# v3 u( X( N0 X; ~
# u$ {2 Z1 u. \7 p" P% ]" k* A2 d+ c8 S ^) p$ `: {! k
o- ceph01 .................................................................................................. [192.168.120.81 (UP)]# u8 F3 {9 q0 T/ n V
' r8 {# ^6 ~* A1 d
( K7 x( ^- }! G& ^/ h. \
( D0 Q* M( `' T1 d! S o- ceph02 .................................................................................................. [192.168.120.82 (UP)]
9 r% J! n1 ` r. |% G: `$ {
# u* V6 E# z! w
/ F7 w* B9 v5 Z5 M# o
3 V; A3 t$ k. n7 a' A1 M$ h; a' i o- ceph03 .................................................................................................. [192.168.120.83 (UP)]
1 m6 R7 B, Y" r% `5 b
' z' V0 l W. S. p& \5 q T
. P: ~0 H2 U5 r8 ]0 k
如果操作系统非Centos或redhat，则需要加skipchecks=true参数。 3.3 创建RBD image
$ g3 R- k) ^# b7 V. R# W/ X% s9 l' C3 |. \$ h& B# N9 v
/iscsi-target...-igw/gateways> cd /disks
' D* k8 [0 e; ]. |- x7 N8 \$ p& X3 i
, |( Z2 u% h L* u: l- 8 ^/ ^' c* |" \& `, N) z1 A! |, X
: c$ U$ j5 a1 z8 k+ Z( b0 x/disks> create Oracle vol01 100G. Z0 j' Y) S7 _. F; [# Q+ l L* o
/ ?2 s& I. T% A3 c. E
0 x4 L; e/ B _! v- C/ Z- B
3 a1 ~' x8 \. `1 b8 D5 f6 w( Uok
/ Q* |2 G1 J$ X# @ h
9 E; [4 X+ ^8 B# o2 R- 7 h3 E9 I+ c0 r+ w t
3 h' H# ~* ^) c. A) B; R7 I
/disks> create Oracle vol02 300G$ e. H* |1 f9 f- k# o& r/ o
% T" c* u& E5 D9 h. G
, `+ i/ _5 v! Q. a# G& {
! q9 W5 p6 g# {) b0 Zok: n& H, q7 t0 j. ]
$ X( ~5 d( \. P# K7 A& t
+ V0 D6 r( L: k$ y
3.4 创建客户端名称Linux平台可以查看/etc/iscsi/initiatorname.iscsi文件获取InitiatorName。如果修改了默认的名称，必须重启iscsid服务，否则在登录iscsi服务端的时候会报错。 - + W) b' l( }; t) P, ~9 B5 ]; m
3 q3 B" T+ H/ D
/disks> cd /iscsi-target/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts
0 h9 h. \; P9 W9 x8 ]6 M5 ]
7 B/ D: A# N( \, I# T
~4 [8 V7 f4 [+ E
4 Y2 Z, f7 D, c- B4 W/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
& ?7 j$ S/ c& D4 w9 _6 _: W# v, _1 @$ D- Z% D! T: b9 K* r
- - e3 l1 v# ] F5 k
0 j0 V9 |$ J: j s2 i
ok. ^: ]( \* S- ^) Q+ C9 a9 J; [! s
7 Y; _) z2 d# R& {2 z) R
- 2 @7 ?: c2 t8 i: l U, C7 f8 n: R3 O
* ?! `! S3 r+ R7 B+ \5 Q. H/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:ccd061606e1:odb04% ]4 }; ]8 w ?9 }6 n
1 f4 |( ?+ [. {. _" b$ n
- , C' f- \2 T9 Q) p" o/ }& L' i
# F) |; r# u( ~: y* e8 s. qok1 R- Q( X7 Q' ^% D
& ^" }" {. ~: ^& q( {2 K
' ]! C8 F, {' c
3.5 设置客户端认证
# k" N' r' U* I1 ?0 a% }8 c/ K' H# A1 g4 J1 k7 _1 j% u: g& N
/iscsi-target...csi-igw/hosts> cd iqn.1988-12.com.oracle:3d93d2aa7f1:odb03
; r% ?! r$ p3 ~! `# |5 U. |1 P! J X% h8 c$ \6 u$ D* V2 W; c
- / L) I5 D6 {; \! x/ l* q1 t3 ?
5 _$ I1 f3 z, z( q" g, p, m/ X/iscsi-target...odb03> auth chap=admin/redhat
. g6 T1 O& W1 a. }5 }# ~: n8 s
3 w% x/ m$ K: h. D2 _7 A; b& C
. J6 N$ G* Q; P" { y
2 [! r k. C7 p5 J9 P/ }. X+ \& Q/iscsi-target...odb04> auth chap=admin/redhat
' c9 t0 Z' T$ R% L3 d4 a0 z6 n% q
- l. B+ l6 J+ I. P) g$ \ u
3 |8 F, n. X/ X: L% e
3.6 客户端映射磁盘- # z4 C5 T' \, j7 Q0 X
: ?3 k; _% U1 P% g- p0 i" L/iscsi-target...odb03> disk add vol01
, |0 H: O/ [# j( {8 L6 W$ J& F0 [( r2 ?* n1 ]3 | Q' p5 U! R
- : g6 h1 C, j* L3 i' z- y* |
5 {6 x) Y5 E! i; [2 i9 E0 b' z# o
ok
# [! C# s4 N+ t3 t6 O: W' _
: f. b' J) h; e
0 k7 X, A L' i7 Y; k, u$ o7 r' k8 g% V* a1 w- d# l% v" P
/iscsi-target...odb03> disk add vol021 N/ O- p5 v- L: P4 b0 q; y
( O+ X2 r0 f5 ~1 A
: L; K; N, ^& e# G$ n* ]2 X% }' r, T+ _; U
ok9 b& ~1 W2 N/ ]
' {3 Z1 F; H& Y8 i: M' P4 l8 |% k
- 9 f- y( s5 m( c; H. Z
& J3 x/ N- e W2 Y9 K, }8 A8 l' M
/iscsi-target...odb04> disk add vol01 `( v4 K# {( ?* x2 R d, [
; Z9 O$ o0 Y5 v( B6 j$ y - ! {$ i8 F6 T" A3 [ J# L! m
/ G5 p2 \4 x, b* Q. s z- Vok
. g0 d5 ~8 i; [) @0 j* |
) J$ A- l' s4 E8 Y
3 k( l: N- y* t5 G( Z& W4 ~/ V6 T3 t
/iscsi-target...odb04> disk add vol02
4 f2 J/ y# F2 P( c/ [: V8 T
. m4 F8 C6 e6 G- 2 c& C9 b; j& T3 Y- A- X$ r9 {
. l0 J; _/ f% F, ]2 I% B9 S/ Q- G( t# \! Gok0 O$ F/ r+ M0 O4 W2 X5 v
7 E! J% n$ y4 S H5 a8 @
6 C8 q g i! S* r2 Z- ~
最后的结果如下图所示：
+ F+ O6 A; X. H% \ |
发表于 2021-7-9 17:51:57