找回密码
 注册
查看: 685|回复: 7

手动方式部署cephfs文件系统

[复制链接]

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
发表于 2022-7-26 10:50:44 | 显示全部楼层 |阅读模式
FS使用 部署MDS
1)为MDS元数据服务器创建一个数据目录
# mkdir -p /var/lib/ceph/mds/ceph-{hostname}
[root@ceph-2 ~]# mkdir -p /var/lib/ceph/mds/ceph-`hostname -s`

* y8 t( r" a7 ?0 g; h- \
[root@ceph-2 mds]# chown -R ceph:ceph /var/lib/ceph/mds/ceph-ceph-2/+ h- J5 }2 v' P& u* n' Y
0 E0 t( P' _+ A0 I6 }; M. k
1 o( Z0 t/ ^, i3 ~. _
修改ceph集群的配置文件/etc/ceph/ceph.conf文件,在该文件中添加mds的相关内容。具体添加内容如下:
5 I6 k6 B' Z9 O) E[mds]
6 L& n# m3 i% w; ]+ q$ xmds data =/var/lib/ceph/mds/ceph-ceph-2/mds.$id
+ a: b* ~. w  f$ g# L3 Tkeyring = /etc/ceph/ceph.mds.$id.keyring  F9 b: q6 ]! W" P1 t% n: B& Z
[mds.0]( B% Y3 J; }/ B  G# P
host = {hostname}
% _2 ?6 m* `$ [, _  i! @7 q& x, D
[mds]) ]) w: `* ]! I8 G, a* P
msd_data = /var/lib/ceph/mds/mds.$id
  j7 B9 {" K4 K9 Jkeyring = /var/lib/ceph/mds/ceph-2/keyring( F9 `/ X2 r" P3 i/ W3 B1 Y( |' I
[mds.0]
, i3 U. [* |7 H8 f* e* |0 K% Yhostname = ceph-2

# E/ V% k0 g5 s, f) V9 T! ?+ |

& l+ @& o4 X" a, i1 ]# X
2) 创建MDS用户
# ceph auth get-or-create mds.{hostname} osd "allow rwx" mds "allow" mon "allow profile mds"
" N& i. g" e1 D/ c* z) p
ceph auth get-or-create mds.`hostname -s` osd "allow rwx" mds "allow" mon "allow profile mds"
[mds.ceph-2]2 W+ A% e" Z( T- C$ o! e
key = AQCyWt9ixp9UMxAAOItsnAobL+9U/kbOGZHLaw==
% m- E! g$ ]; K5 i4 a

0 [# A: c* J( o5 M! ]( l# r8 ]; o
3)获取MDS密钥并导入其数据目录
# ceph auth get mds.{hostname} | tee /var/lib/ceph/mds/ceph-{hostname}/keyring

1 B% H+ m6 U5 v/ J/ z* V
[root@ceph-2 ~]# ceph auth get mds.`hostname -s`  |tee /var/lib/ceph/mds/ceph-2/keyring
8 s2 [: M- P! G8 aexported keyring for mds.ceph-2
1 S$ f8 T  V+ Q4 o[mds.ceph-2]
2 T$ @5 D* T. U0 [9 j# O. U4 E key = AQCyWt9ixp9UMxAAOItsnAobL+9U/kbOGZHLaw==
! O0 D) D6 {$ `7 y caps mds = "allow". }4 a6 u' M* v2 w- c2 p
caps mon = "allow profile mds"
( t9 y: w. a5 m9 {1 |) X$ ?! ^- n2 D caps osd = "allow rwx"
  Y! f0 d5 y( {+ D. T
4)启动守护进程
0 d7 g6 i/ ^1 `% |/ o- h9 v$ d
#systemctl restart ceph-mds@{hostname} & systemctl enable ceph-mds@{hostname}

" ~+ k$ ^0 b+ P  w3 b

2 e+ I# j: U3 I
创建cephfs
1)创建
#ceph osd pool create cephfs_data <pg_num>
#ceph osd pool create cephfs_metadata <pg_num>
#ceph fs new cephfs cephfs_metadata cephfs_data
2)检查状态
#ceph mds stat
挂载cephfs
1)创建挂载目录
#mkdir /mnt/cephfs
2)查看管理员密钥
密钥存放于ceph.client.admin.keyring内,key值即为密钥
3)使用mount命令挂载
# mount ceph IP:6789/ /mnt/cephfs -o name=admin,secret={secretkey}
4)使用df -h命令查看挂载情况

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 11:31:27 | 显示全部楼层
[root@ceph-2 mds]# ceph auth get-or-create mds.ceph-2 mds 'allow' osd 'allow *' mon 'allow rwx'  >/etc/ceph/ceph.mds-ceph-2.keyring5 m! C% m: [8 @  r8 J  V) D! F
[root@ceph-2 mds]#

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 11:38:59 | 显示全部楼层
[root@ceph-2 mds]# mkdir -p /var/lib/ceph/mds/mds.07 ^$ t3 ^' o( t  X! B3 T1 @
[root@ceph-2 mds]# vim /etc/ceph/ceph.conf
& t( P" J3 u- z3 }" L. |[root@ceph-2 mds]# ceph auth get-or-create mds.0 mds 'allow' osd 'allow *' mon 'allow rwx'  >/etc/ceph/ceph.mds.0.keyring' u$ D, @! j9 x! x- J9 Y
# I' A$ b# w6 \+ A" {; D
[root@ceph-2 mds]# systemctl status ceph-mds@0
7 Q' {7 A4 H! v. i* n5 Dceph-mds@0.service - Ceph metadata server daemon
( X: j, z+ \% @" g! k/ w7 n   Loaded: loaded (/usr/lib/systemd/system/ceph-mds@.service; disabled; vendor preset: disabled)
7 e- k# o) x* y" f" N/ K8 t   Active: active (running) since Tue 2022-07-26 11:38:23 CST; 6s ago; v7 a, H+ r9 e0 b6 [1 Y9 _
Main PID: 37142 (ceph-mds)
% s, ?2 F" S- C8 G. d. ^   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@0.service/ _9 N! [  C* k+ J: v0 u; K
           └─37142 /usr/bin/ceph-mds -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
& V' p8 j0 i8 q  m# Y. I+ z" ~7 Q* [& }8 c) P: r% v7 O
Jul 26 11:38:23 ceph-2.novalocal systemd[1]: Started Ceph metadata server daemon.) j* z. z: @- W0 Y/ Y7 I$ X
Jul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: 2022-07-26 11:38:23.555502 7fb5f38ef240 -1 deprecatio...it.
, V' K5 d3 a& k& K) C% NJul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: starting mds.0 at -
& k+ d( @) u/ z" ~( f, xHint: Some lines were ellipsized, use -l to show in full.
& x0 R) C0 L* b* A. t[root@ceph-2 mds]# systemctl status ceph-mds@0.service
0 S- t; j" b, L2 e3 sceph-mds@0.service - Ceph metadata server daemon
0 I$ s3 [) ]- j2 y( C! M   Loaded: loaded (/usr/lib/systemd/system/ceph-mds@.service; disabled; vendor preset: disabled)
' u: l" [7 m& i   Active: active (running) since Tue 2022-07-26 11:38:23 CST; 10s ago! I2 i2 }, Y5 b" }  J$ Z
Main PID: 37142 (ceph-mds)2 ?/ M1 Y7 O! G* j" C/ V
   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@0.service
/ Y! @. X* J) j2 t" q8 \; A4 x0 k7 {           └─37142 /usr/bin/ceph-mds -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
4 l8 o( U. C* C' H: I
0 E9 h# X7 ^( R0 l  I2 oJul 26 11:38:23 ceph-2.novalocal systemd[1]: Started Ceph metadata server daemon.. Q; y, ~/ Z& N4 _
Jul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: 2022-07-26 11:38:23.555502 7fb5f38ef240 -1 deprecatio...it.9 b" X5 l2 t- j8 z
Jul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: starting mds.0 at -
# T+ A! ?7 |% T- G: @( H4 IHint: Some lines were ellipsized, use -l to show in full.
; S2 _: V% u2 l% s  p' y[root@ceph-2 mds]# systemctl enable ceph-mds@0.service   _9 G9 ^) r; G' k4 ]
Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@0.service to /usr/lib/systemd/system/ceph-mds@.service.
+ F* ^7 O0 P; a5 e( l# s1 a+ o[root@ceph-2 mds]#

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 11:49:53 | 显示全部楼层
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
9 _% D% D! s) [" k0 K[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create compute03
, S% g2 h9 e# h2 S[ceph_deploy.cli][INFO  ] ceph-deploy options:/ D3 L  b: a( K/ z. \* J, S% b
[ceph_deploy.cli][INFO  ]  username                      : None+ [+ r( q8 i8 @1 f4 ^; N* w
[ceph_deploy.cli][INFO  ]  verbose                       : False' n8 P  b& b( K& C0 F2 k. Q: [5 w
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False. e# z' v( M* t& ^, F2 u6 K% ~( }
[ceph_deploy.cli][INFO  ]  subcommand                    : create9 ~2 f# H9 ]! N5 Z3 c4 |: w
[ceph_deploy.cli][INFO  ]  quiet                         : False
  }/ @- O  ]0 T# G  h; o+ X[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f182680d878>% y6 ?9 I/ R$ B2 t( G: p
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
) ?2 o# _. W! F1 e, [[ceph_deploy.cli][INFO  ]  func                          : <function mds at 0x7f1826a5fed8>
3 d8 `' x8 k8 m0 T. P[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
6 O1 o3 z% f2 Y& R+ T4 W[ceph_deploy.cli][INFO  ]  mds                           : [('compute03', 'compute03')]# m  p( D8 i0 T2 h* r0 m3 [. e
[ceph_deploy.cli][INFO  ]  default_release               : False
* V  `; V8 I& E2 t* c; `6 x) F[ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts compute03:compute03! Z) K* {9 A+ ?! l
[compute03][DEBUG ] connected to host: compute03
% R, W+ L: x! s( g0 I[compute03][DEBUG ] detect platform information from remote host1 \: d4 M4 ^5 J
[compute03][DEBUG ] detect machine type
% I, m% g1 R$ Y& _6 p[ceph_deploy.mds][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
9 X6 D, b- ~+ J8 U- T[ceph_deploy.mds][DEBUG ] remote host will use systemd  O% R5 n9 n' K: |% U9 F
[ceph_deploy.mds][DEBUG ] deploying mds bootstrap to compute03# [2 ~7 V0 R4 K" n7 w* Y
[compute03][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf7 c2 s5 F" u1 S' J: ^4 E
[compute03][WARNIN] mds keyring does not exist yet, creating one
# O/ z' M& h7 |! m5 s& f[compute03][DEBUG ] create a keyring file% |8 V2 t, Q' u3 x9 g+ P
[compute03][DEBUG ] create path if it doesn't exist0 Z* C3 ^- D- v% m
[compute03][INFO  ] Running command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.compute03 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-compute03/keyring# f0 [, [, K/ o3 _
[compute03][INFO  ] Running command: systemctl enable ceph-mds@compute03; x7 X  Q; W% w) ]0 O4 ^5 ~
[compute03][WARNIN] Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@compute03.service to /usr/lib/systemd/system/ceph-mds@.service.3 q5 J- s0 t7 y  b, i5 e  d
[compute03][INFO  ] Running command: systemctl start ceph-mds@compute03
3 [7 ]6 X  @" t- A& |' t0 \[compute03][INFO  ] Running command: systemctl enable ceph.target

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 11:55:31 | 显示全部楼层
ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.ceph-2 osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-2/keyring
5 U! @  e" |% D( P1 G
; _9 T* n4 s$ E* t  |: ?[root@ceph-2 ceph]# systemctl enable ceph-mds@ceph-2.service
5 c! N8 d' v  S* |Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ceph-2.service to /usr/lib/systemd/system/ceph-mds@.service.& C; Z* p' b1 R& b* @8 |
[root@ceph-2 ceph]# systemctl start ceph-mds@ceph-2.service

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 14:35:06 | 显示全部楼层
MDS
; k5 i/ M9 v5 d! q& }! @- M  b8 \
5 |/ t: v* i! ]7 _28. 创建mds工作目录
- @8 r8 t* ]3 Q5 Jmkdir -p /var/lib/ceph/mds/ceph-admin
( t" \" W. D0 e3 E3 [; D9 I' K# X+ Q/ @3 a

1 x3 a; l- A  P+ g& z' m' v29. 注册mds的密钥# w( \. g( C, s6 k7 i' R
ceph auth get-or-create mds.ceph-admin mds 'allow *' osd 'allow rwx' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-admin/keyring
+ C$ Q: b5 X; i. W# J. i/ I* g7 \
( |3 F8 i$ I- j  D4 s$ f8 @9 z; _0 n. K% n. U1 [7 }
30. 启动mds进程" C& D) \  A, \' q* \* D
ceph-mds --cluster ceph --id ceph-admin --setuser root --setgroup root
+ y% [1 b0 V; g1 r0 Q
0 ?; e3 H& r% K

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 14:52:05 | 显示全部楼层
[root@ceph-1 ~]# mkdir -p /var/lib/ceph/mds/ceph-1
( ?% P# R0 Y: Y" `6 m[root@ceph-1 ~]# ceph auth get-or-create mds.ceph-1 mds 'allow *' osd 'allow rwx' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-1/keyring
3 G: V1 r1 E$ s  K4 B# C* k4 p2 g) {( z8 k2 a; o

5 U+ I9 [! u  X: w添加MDS 创建mds数据目录 mkdir -p /var/lib/ceph/mds/ceph-`h...
3 T3 _2 i+ ]2 q; Z% H添加MDS
9 S) k2 J6 v' P1 V1 N8 V1 G2 H6 L创建mds数据目录* x* n# F& }$ q: f8 ~2 K4 r
mkdir -p /var/lib/ceph/mds/ceph-`hostname -s`
; J; B: A4 |: O" B7 i- ^% o6 I- t chown -R ceph.ceph /var/lib/ceph/mds/ceph-`hostname -s`
1 I6 Y# o. x! T3 q  z- c+ t复制' V+ }5 s: c% [# m" i
创建keyring
1 \) y) X- r  `# @' l ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-`hostname -s`/keyring --gen-key -n mds.`hostname -s`+ j0 I! I2 b4 o! G/ N$ R: m
复制
: w) P( F4 k* s/ C& x2 F9 _ 导入keyring并设置权限
* K  @& `  N6 c. h, s4 T' T5 \ceph auth add mds.`hostname -s` osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-`hostname -s`/keyring
8 w/ S9 N0 {# l) R( J chown ceph:ceph /var/lib/ceph/mds/ceph-`hostname -s`/keyring
: l- p* X$ ]( K' f; a复制
& F4 `* l7 d1 x9 X 修改ceph.conf配置文件& }4 }- a6 Y3 J: p& {
cat >> /etc/ceph/ceph.conf <<EOF9 i$ C* x# }, ]6 n8 v4 B4 Q7 h( k/ k
[mds.node-1]
4 p9 a2 f+ j5 E host = node-1
& c* v' D$ A+ i2 {: I6 z [mds.node-2]2 \: d3 Z& {7 |' a6 \1 H( M0 @
host = node-2; i0 Q) A: R/ _+ H
[mds.node-3]* S. L8 c/ ?/ B
host = node-3. K5 _' ~" }3 B& m) s3 r. r
EOF1 L) e9 x/ \+ F5 n4 W# F
复制
% X, K, V* j! N 启动mds服务
2 P$ D; F6 x: V7 Z$ F3 [systemctl enable ceph-mds@`hostname -s` && systemctl start ceph-mds@`hostname -s`0 u' w- W: B1 n+ Q+ A/ U
复制
- y3 D5 ?) j( R' G 现在状态应该是这样的
+ n6 }8 q( u2 P( f
  o! U" i0 z  a0 @* d
# b3 a" Q+ K7 @6 m9 Y* B[root@ceph-1 mds]# systemctl start ceph-mds@ceph-1.service ; a% R( v3 O7 h3 i
[root@ceph-1 mds]# systemctl status ceph-mds@ceph-1.service # |2 x$ ]! E; }9 v
ceph-mds@ceph-1.service - Ceph metadata server daemon0 \" \. ?; s( S. k9 e9 T! |
   Loaded: loaded ([url=]/usr/lib/systemd/system/ceph-mds@.service[/url]; enabled; vendor preset: disabled)
' G( @& X& W8 g9 C( O" x- e5 y( E8 l   Active: active (running) since Tue 2022-07-26 14:51:01 CST; 5s ago
7 w" s; b* X: e4 m7 b- M Main PID: 48091 (ceph-mds)
8 [. W, J# x9 c" H( t   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@ceph-1.service4 ^  i  o! y0 p" F
           └─48091 /usr/bin/ceph-mds -f --cluster ceph --id ceph-1 --setuser ceph --setgroup ceph  e$ \/ v; e3 i$ o+ ]0 a+ A
Jul 26 14:51:01 ceph-1.novalocal systemd[1]: Started Ceph metadata server daemon., q/ D) l: _% R
Jul 26 14:51:01 ceph-1.novalocal ceph-mds[48091]: starting mds.ceph-1 at -, b6 {: z3 |+ G9 C. q' F9 p$ c
2 N' p6 x$ ~( g( w8 v. t0 T
$ {7 ~) z* f$ X) Y+ Y) r

  [. W) C# S. J$ Z' S  a1 P' I1 {0 [+ X* i/ s; k" E  p2 M

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-7-26 14:53:25 | 显示全部楼层
您需要登录后才可以回帖 登录 | 注册

本版积分规则

返回首页|Archiver|手机版|小黑屋|易陆发现技术论坛 ( 蜀ICP备2026014127号-1 )

GMT+8, 2026-6-12 00:05 , Processed in 0.021241 second(s), 23 queries .

Powered by Discuz! X5.0

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表