|
部署ISCSI服务2.1 安装软件 ceph相关的ISCSI软件包可以从redhat通过的源代码进行编译或者下载centos已经编译好的。
7 {- V5 L6 |2 k# l2 d/ ^$ \- y" W% H7 w$ _' b" y# U
[root@ceph01 ~]# yum install ceph-iscsi-cli tcmu-runner ceph-iscsi-tools + E, \( c6 D T
$ t" ]& W( H) Y. I- h l
- / W* i! Q% s0 r, f" J+ Z: v
5 X" {/ X: h9 K9 H7 g7 [
[root@ceph01 ceph]# ceph osd pool create rbd 150 150
, e9 s- h2 e% @# t' C" A. H$ Q; S* @/ l: Y' V, m
: r3 I- o7 f+ g6 I) b$ A5 V: n/ E4 G7 ]# K- W/ y
[root@ceph01 ceph]# ceph osd pool application enable rbd rbd --yes-i-really-mean-it
' R5 [) s3 V% A% B' o V& n
+ j4 x7 Y$ L8 `6 h) l# |6 r: l
' V: U9 b' E* J4 r9 X/ d6 ^
2.2 创建配置文件创建iscsi-gateway.cfg，此文件主要设置iscsi服务的网关。 - 7 }7 E/ Q' v% |8 l, Y! G" U
5 a: ^; o- o3 w: F% u6 Z9 Y[root@ceph01 ~]# vi /etc/ceph/iscsi-gateway.cfg
5 X3 u8 P, z% `/ @- i9 {: ?
0 R+ Q" [; J- y1 y0 c" ` - " t2 N# j+ \) ]3 P+ @
* V( e9 m/ a/ `6 t0 N' Z
[config]
8 p' i/ r4 v# R4 S
- s9 j o: L! Q; _+ w - 9 q0 a# r9 D7 I3 w; p7 q e
/ Y' Z2 o. j ^, ~. ^! x) B( D4 e# Name of the Ceph storage cluster. A suitable Ceph configuration file allowing& T# S4 P/ U- W+ {8 {
; U( F8 z b/ _
& u% D2 ~' g& L/ t* o3 P1 w' T7 j+ s6 m5 i8 a' a, `8 G2 g& c" c
# access to the Ceph storage cluster from the gateway node is required, if not
% B6 u# D) {, I4 u4 k2 f5 X* B& d! {1 D; t/ {- D1 R
% ~; ^$ U! y" ]3 K m$ `' c: b: J- U* e
# colocated on an OSD node.
$ n) A; ?) Q4 E( ^1 `+ r6 ? W, |8 O5 a: w* g5 h9 |! K
- + n' P* E$ |, {6 D' j/ E& g
8 U8 W" u6 Z% J5 w1 ?7 y# Q4 s
cluster_name = ceph0 s1 y' T7 c# D* l5 K4 G! P
1 B5 X5 |( _. T
, o4 c' g7 k$ P1 x, r$ A+ K' E: m0 `9 N
0 @" _, a5 _7 b; Y3 S% n4 L; V5 X/ i/ {! i% Y6 s
; J9 ?5 f! G. _0 k- o6 j! b. g; q$ k3 a: V5 i( A% f, ]1 @5 @
# Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph9 N: K1 `' {6 Q2 r% O1 k+ E- G
7 I( X. Y& g/ }9 ?) f( @2 O- 9 ]0 [) r) B( s1 l
- [3 o+ ^# Q6 P) G4 [0 I2 D# drectory and reference the filename here0 z9 K/ k F% O6 M* x0 p
* r6 z: |6 t7 d# f
- # b) A) Y" \9 H3 A
4 C4 L9 h; i! Agateway_keyring = ceph.client.admin.keyring
& T: R( A( J! ]! Z( h3 j) ^ p$ R! e* V- w6 F" C
. h% E! |0 D8 }6 G! G6 `( `/ r# d" d# I# s2 q6 @- D: Q
: ^# f% s0 z6 A. S7 n* k
; R @! ]- F( Q2 f9 K
& Z) ]9 r4 w: F( A; k3 N
. R3 a: b" I, b/ v& g# API settings.( ]& R! u$ X. \/ C! B
& S; Q9 ?5 W. I
$ z) v: \! l M% b. K+ E
: P" ?; M) @& ]5 ]1 Q# The API supports a number of options that allow you to tailor it to your
% Z+ p& O% v5 c7 b- ^- G
" S; r) X3 w/ \8 J! y: R- % o+ ?& @4 f7 s( z$ R/ \
/ n# T! P$ p' I. p/ L, j5 k# local environment. If you want to run the API under https, you will need to
% R* D M8 J% Z4 o
# j3 ^( F* g f, A1 O - 2 _/ T+ `# X4 T9 h8 a
* d" W0 v7 D8 e1 r. K' ]8 } j# create cert/key files that are compatible for each iSCSI gateway node, that is
. K* X$ Z( O0 U& T5 P& }# i8 x4 i2 s: [( p' c
- 1 C& o, B) O) c9 s6 x
, n. W X7 l, X- t$ a4 q: E7 D5 R# not locked to a specific node. SSL cert and key files *must* be called" k. |2 p4 E( Z3 `, |' _/ U
' h1 G' m% E8 b, P- R - * {6 k+ u* W& f0 `0 J: J
/ \2 P0 U& R& O( f$ @# 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory$ ?7 k: C8 k1 G# X# o
, d- c9 v& Z8 }
2 G7 h- w$ ^4 _/ l2 ~" O6 ]6 D8 B/ C( @9 W
8 G8 ], h# F$ |# M# on *each* gateway node. With the SSL files in place, you can use 'api_secure = true'
& h, u$ n* {2 H" N. S ]) w! N9 }* r$ ~# j5 z; t9 s5 n# r
- 9 l3 J. R& _8 r" z
( f: v+ @: w; \! q# to switch to https mode.
. y4 G: f) R0 p6 Q) K0 e8 N) r+ y
8 @: J& z4 h3 o ]9 ~( n1 \
, t9 h- z( S% v$ r* T' \6 l
& _; W+ s, b1 [7 c% B* R0 e
' |( Q) l; g- s4 |
* t7 ~6 M0 S/ i" l( @3 `
: D& A! X2 c& F: \/ V4 ~0 [6 D5 x) p% ?4 T% X+ O5 P
# To support the API, the bear minimum settings are:
/ e/ x& X; u2 c) D: g
0 j2 S7 r- H s- y
/ }, ^4 R0 `+ A7 [
' I- ]" i" ]4 Sapi_secure = false3 H3 t1 _% y, c; R4 |* ^2 j
/ ?0 u6 Q! _* o; c% p
) j# _* z3 J k ]+ Q# d; Z
6 @% c* U( S) s2 O+ P" [' u9 C& d
6 {, Y% Z( U7 D: Z. g x/ i6 f
7 E# K4 D/ c7 i) }0 x# x9 T0 ]6 a% S+ O3 c& c+ _
# Additional API configuration options are as follows, defaults shown.! ~; s7 m3 ^# a2 k2 H
7 p2 Z$ i: m( G6 S; ^$ o( T
6 w, ~% K& j6 _
! K, u3 z0 o. G, a# api_user = admin
. l/ ?" t+ l: g' v) J6 C% m' m! `$ w( R
9 |' D' m' L% t! N6 ] \$ D/ X) m3 \* i, i7 |# J! N: C6 N" S/ w2 P! w
# api_password = admin
; M' v% ?1 J# \3 E- r7 M# f S1 _, d: \
- 8 Z- y& r& Y& v) i5 A- W
0 ]# u/ k" p; H: ^7 D
# api_port = 5001) E$ `: m! v# y$ N
4 V% e% r" m' f: B, A% e3 Z( W
, e; j' I$ V C0 q) z2 q8 e
7 w- j( _0 V8 r# s& q. [trusted_ip_list = 192.168.120.81,192.168.120.82,192.168.120.83
* p7 Y( w$ O# o% ~6 Y% {2 N, p# \' J3 N* L7 O% q
" L4 P! Q7 x' i) T }" g* B
2.3 同步文件到其他节点- 4 {1 ]$ @/ G+ a& W; _# x
u. R- p' J, l0 P[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph02:/etc/ceph
/ C/ e: _! `1 o9 u
0 m4 M& R Y3 H5 i - " H$ q3 a# I" a7 @& M. P* ~$ t
3 S% F l% A9 P
[root@ceph01 ~]# scp /etc/ceph/iscsi-gateway.cfg ceph03:/etc/ceph% M6 u& N' F2 m' C7 ~
& K: |$ {! m3 m- C( b% j: W
- ^) f8 t4 ]3 v, v* s) R
2.4 启动API服务- $ e: H3 b7 [* _! V! o* K! E
# M& _4 Y* W: f& z1 a/ ]+ Q% _9 p[root@ceph01 ~]# systemctl daemon-reload
- x; O; F: c2 Q* m" r* D4 L* `+ H* q; F/ z& n2 e3 {9 P6 A
- 7 b2 S/ F: _ O! R& p$ H8 p
+ P: s' `! |* ]6 S2 n, O. I. _
[root@ceph01 ~]# systemctl enable rbd-target-api
" L1 a# q( [- ^ X/ e' `
6 r8 c1 {9 l% z; z4 ~! s
8 n; v/ j/ A1 b% _$ I% X' a" e* [5 a9 E! H3 o& N& u( O0 V. M
[root@ceph01 ~]# systemctl start rbd-target-api6 E% u/ c) s! r7 Z, M
0 Z E0 k) ]0 e' [& q
: _- z, ?- w! R4 I r P" S( ~" T& c ~. b
[root@ceph01 ~]# systemctl status rbd-target-api
+ m! t6 L5 y( C3 k. A9 u2 D) Q
8 g, m4 ?' S" e$ W/ s3 d. y2 r- 3 Q# b; V5 C9 C$ Q7 o: a
* T# m# @7 ~" K( ]- b1 R0 X1 y
● rbd-target-api.service - Ceph iscsi target configuration API
" `9 i4 A0 l* g6 M$ R: n& `# r* L& Y8 T: }; f* P
) O4 H7 O/ \7 {2 Z2 S+ I, P9 Z- r4 ?, P& A: g8 o; f5 C2 B; F
Loaded: loaded (/usr/lib/systemd/system/rbd-target-api.service; enabled; vendor preset: disabled)9 F/ A' }; s4 F `
% H! y$ R% w4 [7 l6 E% i6 u6 t- $ @4 r+ l/ g# E% o8 o- @( `6 t
0 S8 r' Y# |6 t5 N/ E! X/ _2 i Active: active (running) since Thu 2018-05-31 11:35:04 CST; 4s ago
9 O" W4 E; h0 J" ]9 p5 w Y6 q& R0 _ R6 l. D$ K* \
" N7 t, g p( v0 D3 s( ^3 I
7 S' F" ^. s4 s Main PID: 25372 (rbd-target-api)+ X) p+ k' p0 m
% B4 i8 m3 o" b- ) G+ ]$ x z/ e. D
9 H m+ f& b( E3 ~0 E: T1 q2 ?6 D, i, W0 H CGroup: /system.slice/rbd-target-api.service
0 g, L; H ~% z# Z, ~* q8 y4 l q* B& O
- 5 O4 ^3 O" ]' x [: ^& V/ g! E
3 C) q2 x N3 n0 T& \6 F3 E
└─25372 /usr/bin/python /usr/bin/rbd-target-api
5 e. v5 E0 H. w1 W
. y. s* h$ q* r" h
0 a" T2 @8 n, x( k2 m8 H1 o& m2 s& N8 g& P3 G
D$ C V: `( D+ a- R0 g/ e/ s# [) R0 V9 C7 Z. z( A( o$ c
; X8 _4 O4 q( {) D/ X7 B w( ]7 U3 ^( M
May 31 11:35:04 ceph01 systemd[1]: Started Ceph iscsi target configuration API.$ T8 ?, v$ K: @& ]+ w
K7 y7 r/ H/ ]7 Y- * h- F$ b- |. d2 g0 O1 S! X7 \
' O9 h, Q: W% M' F jMay 31 11:35:04 ceph01 systemd[1]: Starting Ceph iscsi target configuration API...
2 L/ d* B6 E6 h
9 }: U# T8 u( T7 c5 W/ Q& i - 5 D" d9 Z/ j7 Q$ u: c1 P
0 L" R q9 b8 `+ `% D; }. nMay 31 11:35:05 ceph01 rbd-target-api[25372]: Started the configuration object watcher3 o$ Z, g- R N. S3 r3 Q
5 z$ @& V2 T" }* |
- . y+ T! R# w% @' ]* N9 K: }% @% }! Q
* ~* M8 v6 y- W8 uMay 31 11:35:05 ceph01 rbd-target-api[25372]: Checking for config object changes every 1s$ ^% H; g7 [& X/ h
2 _" ?, l! S# K. F - o, P# B4 G4 E
+ J; I! G' A$ c9 OMay 31 11:35:05 ceph01 rbd-target-api[25372]: * Running on http://0.0.0.0:5000/
' B2 ] ]9 c+ s
4 O0 m' {2 J# h4 t, s3 d4 K6 K0 L% R
# K# z \2 n' w* ^' t
三、配置ISCSI服务3.1 创建target- " P+ ?/ X, u% z
- k8 @( D; B9 f* H0 h8 n[root@ceph01 ~]# gwcli
7 Q: V4 R" u9 r! M1 k" ]+ l) f1 |8 P' h c
- / D+ Z: n; A! }
+ i( M& g0 z( ?$ N
/> cd iscsi-target
" s# F/ i' j% ?8 h3 g
w4 `4 ]5 V, O6 C
7 ?& x: Y. {! o# r: \0 k5 x$ D, n8 y7 M ~/ J( F: e
/iscsi-target> create iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw
- v- o: @* `; w( Y. g/ A+ i, X) b9 O8 K' p
" M8 }( P, u% [0 b: T
3.2 创建ISCSI网关
0 L' [, ~ j$ ]4 G% C# e
, m4 \9 W6 Z( [$ w4 O8 Q T/ P9 N/iscsi-target...-igw/gateways> create ceph01 192.168.120.81 skipchecks=true
. ^% o8 @% i% {
& ]2 ] J j9 i- Y* _7 e- ! Q2 a& N4 b9 w& r; G% b' N
5 ~% A+ n; N- J# d' f; M" y
OS version/package checks have been bypassed$ \( J" A6 c: g4 w, b, d1 h
% V3 n# c6 p) S G- x; y
. Z' S6 C* T, y
; S8 y% Z" \' {$ F% z# y2 w, h5 }Adding gateway, sync'ing 0 disk(s) and 0 client(s)
$ W p. e3 |) m! f- T' _! H
# u+ O6 P6 j1 j- v- . I, w+ Y- W3 @7 H+ i' ^4 B
' H2 Y- @$ X' D9 }
ok
2 v- Q( @0 N& @- I H. k5 r U) W( Z1 U1 O* t( T: z8 U6 U5 z
- 7 D- L( S9 I3 ]" d
5 }4 Y( D0 b3 e; g/iscsi-target...-igw/gateways> create ceph02 192.168.120.82 skipchecks=true+ h* z: ^0 S! m- J' [1 A$ M
8 F# `, G8 p& ~8 n1 H+ }, a5 e
* G# H t( s6 w& r) c' {- v! v+ J& ]" G
OS version/package checks have been bypassed: M* C+ ^5 s, V; x
9 Q0 B b! Z+ c# C+ y- 8 F' ^- g/ h* o, g: o3 A* w
' g2 v! ?2 M, R0 U3 B1 Q/ Q
Adding gateway, sync'ing 0 disk(s) and 0 client(s)
' c2 `- h( v3 J' U# e
& g0 v5 M; ~! v1 f% u# ]
7 u. Z" j2 ]' N* s* ^! r7 E8 o8 j2 D
ok
; g! e1 L' @5 G6 b2 n; [8 O' ~# n9 Q2 M- }& O+ V. z2 D
" z$ `! u( P A+ N8 O
. W# I* ~/ j. G- Y. \/iscsi-target...-igw/gateways> create ceph03 192.168.120.83 skipchecks=true0 d1 D- w1 } a, q) ~2 R
& S) _' B4 |, b- k- 7 q# _9 g! h ~3 [3 g, w
( a4 [- e# x& bOS version/package checks have been bypassed9 T1 N% k( k) V
# a6 B3 { \8 |& e% V
- 3 P5 Z6 h' S C
3 K2 r& Q4 H( [' I ?
Adding gateway, sync'ing 0 disk(s) and 0 client(s)
- I* q8 S ]0 @4 e' `, B0 l- X# H/ a' C
- + g; t; o+ j1 @, W3 L! v
S4 ?4 T9 w" c7 e, U, E) F
ok
5 E( w+ Y: p; o# M/ g; v i8 t: ^$ a
- 4 Y6 C" F( F/ B4 U7 S- {' r/ W
5 J' i6 [/ T }# u/iscsi-target...-igw/gateways> ls8 K9 {/ K- B$ ?
! s, b2 X" [ Q; V S. y - 3 o& l0 {. ]: m9 L2 k5 R( `+ \
8 x& G2 i# j8 p: j
o- gateways .................................................................................................. [Up: 3/3, Portals: 3]
& L7 C/ F8 |9 @/ d1 z- V1 `# g3 X5 z
j6 ^; J H$ p) g/ V! B; f! }+ @3 ~- v1 X' l
o- ceph01 .................................................................................................. [192.168.120.81 (UP)]
+ h% D+ z1 H( h% g. g
/ E/ E C# u( r2 q3 @3 l K U
# m8 v( ]. O. [' g" O
! u8 I; F/ o* r8 F, r4 g' p o- ceph02 .................................................................................................. [192.168.120.82 (UP)]
1 j' L9 [1 x1 u0 a; t! W- [ G X3 K9 Y7 d1 |
* O+ t0 ?4 M; s3 c5 I8 Y2 X1 n" D ]: I- j0 z
o- ceph03 .................................................................................................. [192.168.120.83 (UP)]
" I1 S$ P* _4 F4 F+ b9 f- I# l, c# O! S1 Y0 ?: e7 r& z) W3 R Q
7 b. W9 }0 p, `* n0 f/ ?9 J6 y
如果操作系统非Centos或redhat，则需要加skipchecks=true参数。 3.3 创建RBD image- ! I1 _) `+ D. d% T3 O, W
# T* k+ P. h; z6 V4 k/iscsi-target...-igw/gateways> cd /disks5 [: d8 r- l* b7 @9 t6 _$ R0 D
' p3 V! v* g, Y! p8 m- |4 k& S7 u - ]+ |+ N- S% R5 h' e4 q
1 s$ d K/ e- o) e
/disks> create Oracle vol01 100G
! r* M d( g. D/ b6 @4 P4 B
8 b6 f- ?3 t1 T) \3 X3 P* V8 g - 4 P& |1 v4 A6 S, |8 R0 _; H
0 U+ n' G; U/ U, j0 Jok
5 q) t8 K; D8 e& @/ d% K( b2 S! g% b4 w7 ^" n- L. c
- 4 o& @: `/ q" ]: _$ s: C5 r
$ a) R/ y; D1 ?8 d4 M1 @8 F/disks> create Oracle vol02 300G
" i( w2 w) @" C; g3 B8 l* S6 [$ T8 Y, J/ o% y1 O
- " {- p* N$ ~1 m( e, Q, J! f. [# J) A9 Y
, M1 {/ }8 t1 |; Y- i
ok
1 R: |" F/ L; U' _7 ?1 G; x8 p' u7 O9 s) I
5 K7 E8 u8 ]4 u5 c7 Y# x
3.4 创建客户端名称Linux平台可以查看/etc/iscsi/initiatorname.iscsi文件获取InitiatorName。如果修改了默认的名称，必须重启iscsid服务，否则在登录iscsi服务端的时候会报错。 - / k N* l6 f7 X
5 m4 f p: G- q' E! q; P* U! A' \! P/disks> cd /iscsi-target/iqn.2003-01.com.redhat.iscsi-gw:iscsi-igw/hosts " N" e: p/ s" J" y, |. M
3 i9 m9 V7 |; f9 Y* A( R6 X
9 n3 |$ c* `) C, r6 x h/ c) L3 y+ O! S" P% }- F6 J& T$ s1 { E$ _
/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:3d93d2aa7f1:odb039 a- D' T! G, n! C3 T
! u, O. {2 a7 e% x) R( \: Z) H1 A
- ) p# G0 S6 y) B0 ~5 Z5 }; }
9 C. O6 I* ~$ \# v+ }
ok9 S7 z8 K4 [" Y1 k
6 @& B( O0 n: `& |9 s5 ?# U
0 ]# f7 e; c+ ]1 e/ E* X, G
2 `9 v* a) T: ^/iscsi-target...csi-igw/hosts>create iqn.1988-12.com.oracle:ccd061606e1:odb04
) u" D1 A. B) ~
* Z' f% Y1 N; a( h6 T1 n2 X
8 n4 z9 p6 v" h3 u _! w! _9 {, _
- U/ R3 e$ @4 k; F( [ok
( [+ I% J" z, ^5 {: Y; Z9 ?1 D
3 M$ m4 [) g. a+ U4 @6 \* D, `# o9 K0 I: b
3.5 设置客户端认证
3 Y. ^" M$ ]8 f, _* @- o. H
$ `3 [' g; L! o6 {/iscsi-target...csi-igw/hosts> cd iqn.1988-12.com.oracle:3d93d2aa7f1:odb033 m1 U% C$ I1 F0 W* M+ A( s# D
* c& I/ S" \' N3 {( \- {. G2 J
1 V2 o8 B% G% W4 u- U7 }% U4 G0 m) ]: z6 `8 L* z. s
/iscsi-target...odb03> auth chap=admin/redhat
9 _6 q! j, L3 q: g, R9 p6 `, L# P2 J" W5 `# k4 E
, P5 a: b' n0 ]* g/ i4 P
! X" C$ T1 u: D/iscsi-target...odb04> auth chap=admin/redhat
, k( e- E$ S/ Y$ |7 E. b) { Y. `8 T& a
& D% G( N. E1 i' M) r4 L
3.6 客户端映射磁盘- ; v/ ?# J6 g6 z# N+ J' E3 \
$ n* a1 [7 b- w9 ?9 b. ?- K/iscsi-target...odb03> disk add vol01
; h0 K7 `1 I1 C
- ^ [! d: f' [; r& a/ c' a, q+ ` - & Y5 D" x3 Q2 z; `+ S/ ]. q% Y9 E
* {$ a3 R' {+ L$ x, m1 Ook
! d" ~1 U9 \: E N. y" w. g
2 c" t3 q Y+ F2 P$ U
- J# R& M$ D2 F1 }( O; ^4 X
q% ~! y1 n7 o; S/ d/iscsi-target...odb03> disk add vol02+ l3 E3 v6 a6 ~+ L- u* _! a
8 x: y6 t3 _9 B4 h
. R: Q, T; v$ r- g
+ `0 I: L4 t3 H+ N# x1 Xok+ X) K& \5 D5 u& O0 z' E5 D
& I% a: b2 B- D) M+ H) y
- ! I) u6 s+ n0 x
6 y( H1 u: _) V, u
/iscsi-target...odb04> disk add vol01% ^/ o8 {6 \4 K. _5 i8 H% o
2 l' T8 ?4 r$ j# N7 u% l
- / t# B7 T5 m% `8 O1 s" r) p
% e7 `) }6 j4 J! }4 |- j* G9 M5 Vok* K/ f0 t3 Y, ]; k
0 o3 @2 x4 w) Y" n0 N/ p" L
2 N3 L- Q6 X% p8 I: ~# v
' L" M( \/ N5 i. @& t& Q9 e/iscsi-target...odb04> disk add vol021 p: m6 U- ~# \
8 y3 P) r; C8 C. k z0 d
. h _8 b& V3 e3 Q- O7 r& n
' t3 z* E G* r g6 z. [$ pok
* B0 m+ p3 h. H0 T' e+ f O& P5 P+ {% Y
' m$ s# U8 Q$ t8 g
最后的结果如下图所示：
u: m! {) g2 g- n2 Y |
发表于 2021-7-9 17:51:57