找回密码
 注册
查看: 683|回复: 7

手动方式部署cephfs文件系统

[复制链接]

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
发表于 2022-7-26 10:50:44 | 显示全部楼层 |阅读模式
FS使用 部署MDS
1)为MDS元数据服务器创建一个数据目录
# mkdir -p /var/lib/ceph/mds/ceph-{hostname}
[root@ceph-2 ~]# mkdir -p /var/lib/ceph/mds/ceph-`hostname -s`
$ o/ M2 Y9 p, x. C! Y0 v, |4 ?
[root@ceph-2 mds]# chown -R ceph:ceph /var/lib/ceph/mds/ceph-ceph-2/
% R$ Y- x9 }$ [3 P) y# \" _
8 E" w' u* P! K  B; X
6 z* E1 [. L4 K  k: N* x9 r8 K0 |
修改ceph集群的配置文件/etc/ceph/ceph.conf文件,在该文件中添加mds的相关内容。具体添加内容如下:
* r1 N$ L3 j0 A2 u9 S4 {2 ~7 ?7 p[mds]( j  a- m3 p$ R* j
mds data =/var/lib/ceph/mds/ceph-ceph-2/mds.$id6 N: |. m& X" \2 ?" R& \8 n
keyring = /etc/ceph/ceph.mds.$id.keyring
7 u& N# T. Y+ b6 b: {! O: J8 e[mds.0]( V) A/ M8 Y8 Z1 V
host = {hostname}+ e4 X7 T! @9 x8 `

3 F* q' @# A/ n5 Y$ F/ X* E! _
[mds]
+ p6 o1 j+ g8 F# a( Hmsd_data = /var/lib/ceph/mds/mds.$id
" B* m0 }, R# b. Y/ Vkeyring = /var/lib/ceph/mds/ceph-2/keyring
# T" I5 f0 P5 K2 Z: y3 j! ][mds.0]1 I' m8 j2 I. M; D0 K
hostname = ceph-2

/ q$ y) ~1 z" E& V1 O4 \: v: {

+ X6 j: z* X% J' O
2) 创建MDS用户
# ceph auth get-or-create mds.{hostname} osd "allow rwx" mds "allow" mon "allow profile mds"
. \* N9 w; {/ f
ceph auth get-or-create mds.`hostname -s` osd "allow rwx" mds "allow" mon "allow profile mds"
[mds.ceph-2]9 k0 O- Y1 S/ P  O- I* l
key = AQCyWt9ixp9UMxAAOItsnAobL+9U/kbOGZHLaw==

* D' i+ S0 h1 y9 s/ A' G  Z' r) C
8 n9 C3 v( ?, ~; X6 P* z* f- c, z
3)获取MDS密钥并导入其数据目录
# ceph auth get mds.{hostname} | tee /var/lib/ceph/mds/ceph-{hostname}/keyring

* Z% n2 n' M1 s4 y
[root@ceph-2 ~]# ceph auth get mds.`hostname -s`  |tee /var/lib/ceph/mds/ceph-2/keyring; n8 `; K$ \9 k" B; e
exported keyring for mds.ceph-2% B! K5 Q1 f% W, e
[mds.ceph-2]
4 `6 Z- |  p! L8 ` key = AQCyWt9ixp9UMxAAOItsnAobL+9U/kbOGZHLaw==
) E9 S) e9 a8 i9 \/ O  L6 f, ] caps mds = "allow"
, g- @! b7 K; a caps mon = "allow profile mds"3 m* d8 k3 c, _5 F. [
caps osd = "allow rwx"
' m. H$ ?5 r1 O3 o
4)启动守护进程

6 [; {- U. M3 V+ c
#systemctl restart ceph-mds@{hostname} & systemctl enable ceph-mds@{hostname}

) ?; ?. d; |0 X% \, j
  z5 ?, N2 k- ?; ^% y4 V2 l
创建cephfs
1)创建
#ceph osd pool create cephfs_data <pg_num>
#ceph osd pool create cephfs_metadata <pg_num>
#ceph fs new cephfs cephfs_metadata cephfs_data
2)检查状态
#ceph mds stat
挂载cephfs
1)创建挂载目录
#mkdir /mnt/cephfs
2)查看管理员密钥
密钥存放于ceph.client.admin.keyring内,key值即为密钥
3)使用mount命令挂载
# mount ceph IP:6789/ /mnt/cephfs -o name=admin,secret={secretkey}
4)使用df -h命令查看挂载情况

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 11:31:27 | 显示全部楼层
[root@ceph-2 mds]# ceph auth get-or-create mds.ceph-2 mds 'allow' osd 'allow *' mon 'allow rwx'  >/etc/ceph/ceph.mds-ceph-2.keyring
% A6 |0 @# [7 v( b, D5 p[root@ceph-2 mds]#

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 11:38:59 | 显示全部楼层
[root@ceph-2 mds]# mkdir -p /var/lib/ceph/mds/mds.0
& t, p9 l& Z) r% b! U* R[root@ceph-2 mds]# vim /etc/ceph/ceph.conf
; H; ]1 b! O6 T  ]/ s* o( J[root@ceph-2 mds]# ceph auth get-or-create mds.0 mds 'allow' osd 'allow *' mon 'allow rwx'  >/etc/ceph/ceph.mds.0.keyring9 N2 F/ M0 i! a+ p

+ S# u; s  u5 v& E9 `$ b" H; s[root@ceph-2 mds]# systemctl status ceph-mds@0. q0 ~0 @9 }; J0 D
ceph-mds@0.service - Ceph metadata server daemon
8 h! P/ t: y  D5 V6 k4 v% C   Loaded: loaded (/usr/lib/systemd/system/ceph-mds@.service; disabled; vendor preset: disabled)
" r- `- l" c0 Z' V/ D/ m/ h  g: R/ T   Active: active (running) since Tue 2022-07-26 11:38:23 CST; 6s ago
# x. V- L+ k7 G9 F4 o) T- R Main PID: 37142 (ceph-mds)
0 F# w* B9 G9 r# h8 A. L   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@0.service
% s# Y* c, T3 s  L7 R           └─37142 /usr/bin/ceph-mds -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
5 w. }' b$ c- a" M! Z, H/ v* s2 C/ e  l5 E
Jul 26 11:38:23 ceph-2.novalocal systemd[1]: Started Ceph metadata server daemon.
& ?% v; ]1 p. zJul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: 2022-07-26 11:38:23.555502 7fb5f38ef240 -1 deprecatio...it.
+ t: T6 R4 g3 p* Y/ mJul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: starting mds.0 at -
) h1 H& n9 c( S7 o5 k& n" lHint: Some lines were ellipsized, use -l to show in full.
% J7 |: @% B9 M9 P9 F) I: S$ h[root@ceph-2 mds]# systemctl status ceph-mds@0.service
9 Q2 c- ~# o- f; A8 `& x+ [# p& }# Vceph-mds@0.service - Ceph metadata server daemon- o  c+ g6 i5 h( N1 v7 g. T
   Loaded: loaded (/usr/lib/systemd/system/ceph-mds@.service; disabled; vendor preset: disabled)/ n6 U8 Y: Y# C; t' R1 r
   Active: active (running) since Tue 2022-07-26 11:38:23 CST; 10s ago0 R3 x/ \/ l4 h' m' x
Main PID: 37142 (ceph-mds)
. ]6 y+ A# G. ?  [/ n   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@0.service
# {, n" x4 i( D. d2 A: _+ x- S% Z           └─37142 /usr/bin/ceph-mds -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
8 R' O2 ~. B) l% O; O0 y% D) i& r
Jul 26 11:38:23 ceph-2.novalocal systemd[1]: Started Ceph metadata server daemon.# l+ P2 i3 p5 _3 D; Z: H
Jul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: 2022-07-26 11:38:23.555502 7fb5f38ef240 -1 deprecatio...it.
) y& o, m9 `9 ~2 q' Y" `Jul 26 11:38:23 ceph-2.novalocal ceph-mds[37142]: starting mds.0 at -6 A# @* C- f" R' [  n
Hint: Some lines were ellipsized, use -l to show in full./ T4 X2 i) w. K1 w
[root@ceph-2 mds]# systemctl enable ceph-mds@0.service $ S$ L" j, F$ \% `+ ~
Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@0.service to /usr/lib/systemd/system/ceph-mds@.service.& m5 l2 |. c- j) [9 ?2 }
[root@ceph-2 mds]#

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 11:49:53 | 显示全部楼层
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
5 U+ u4 b$ ~5 i( G[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create compute03' w, Z( q& J; ?: l# w
[ceph_deploy.cli][INFO  ] ceph-deploy options:
2 v  K% L* J3 ~$ k6 N9 B[ceph_deploy.cli][INFO  ]  username                      : None
1 V: u) _. R4 b8 R- p[ceph_deploy.cli][INFO  ]  verbose                       : False
0 l4 _, ]) d4 z1 |2 n7 S$ e/ l+ w7 _[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
+ J: d+ T* t0 D& f5 m% B4 l7 k[ceph_deploy.cli][INFO  ]  subcommand                    : create
* g! W0 P( ~0 v2 U) [+ i  m[ceph_deploy.cli][INFO  ]  quiet                         : False
( C% r( Y! O' W! f% d' e8 n  p[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f182680d878>
* T  D' j. p4 Q5 G[ceph_deploy.cli][INFO  ]  cluster                       : ceph) r1 E% W1 Q: s6 [4 {5 U1 E
[ceph_deploy.cli][INFO  ]  func                          : <function mds at 0x7f1826a5fed8>
; m! g4 u) |' ]4 T: g! \0 O[ceph_deploy.cli][INFO  ]  ceph_conf                     : None& ]& @1 ~/ ]! ~. s
[ceph_deploy.cli][INFO  ]  mds                           : [('compute03', 'compute03')]# @; a' Y) ?$ m7 T" ]: \  w$ V
[ceph_deploy.cli][INFO  ]  default_release               : False- X0 {& F) T( W. o) h4 M3 t
[ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts compute03:compute03
9 x  e4 {+ y$ R; v+ a! M[compute03][DEBUG ] connected to host: compute03 ! {0 X. l# f4 _8 j" d
[compute03][DEBUG ] detect platform information from remote host. L( W* u; u: [% z" X
[compute03][DEBUG ] detect machine type' l9 I5 ^9 C4 O" Q% h
[ceph_deploy.mds][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
  i! ?6 n1 M2 A5 c[ceph_deploy.mds][DEBUG ] remote host will use systemd+ C0 _* {7 I: f
[ceph_deploy.mds][DEBUG ] deploying mds bootstrap to compute03
# b: R2 o- ^* O& P. @[compute03][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf) Y. \7 s& V1 \5 X
[compute03][WARNIN] mds keyring does not exist yet, creating one
# d. k2 u( W! t7 x[compute03][DEBUG ] create a keyring file, v4 u! r6 z% _4 [
[compute03][DEBUG ] create path if it doesn't exist
5 x/ ^. }1 v& p: D- M$ Q[compute03][INFO  ] Running command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.compute03 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-compute03/keyring
+ X7 s1 E/ a. t3 _[compute03][INFO  ] Running command: systemctl enable ceph-mds@compute03
0 j& f4 c. A# n[compute03][WARNIN] Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@compute03.service to /usr/lib/systemd/system/ceph-mds@.service.
2 n: C* P" y  `' i) b. F% }- G[compute03][INFO  ] Running command: systemctl start ceph-mds@compute03
( L  F4 {! m$ X3 e! }( j  W+ |: B) ~[compute03][INFO  ] Running command: systemctl enable ceph.target

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 11:55:31 | 显示全部楼层
ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.ceph-2 osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-2/keyring
' k" D7 z* _$ \) T" R1 d
6 B& N2 e& o. \) p1 l' w[root@ceph-2 ceph]# systemctl enable ceph-mds@ceph-2.service
3 }7 H6 [4 X6 X+ r) e; e0 PCreated symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ceph-2.service to /usr/lib/systemd/system/ceph-mds@.service.) J9 ]8 Z4 V5 T/ ]
[root@ceph-2 ceph]# systemctl start ceph-mds@ceph-2.service

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 14:35:06 | 显示全部楼层
MDS
2 |+ {4 I$ ]! n; Z
3 R- `! H2 |4 \2 V3 J" p8 z( S28. 创建mds工作目录
) c% ?" Q: g8 J1 s9 Umkdir -p /var/lib/ceph/mds/ceph-admin- |) T. W; M9 c+ ?

, W7 H7 T/ S" p8 V
7 n* ^: j' n# `" G29. 注册mds的密钥3 G- C6 z0 E. S# K& k( u" v4 ^
ceph auth get-or-create mds.ceph-admin mds 'allow *' osd 'allow rwx' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-admin/keyring! ]# I  b; T) l3 w
3 S" @9 e8 p) i5 q" R4 i$ B
! B; y; U' o# N1 E+ t2 R& V+ r
30. 启动mds进程( y3 x* `2 l4 d  i( r
ceph-mds --cluster ceph --id ceph-admin --setuser root --setgroup root: B" ^' k$ B, x$ B, y! ?1 f
; O8 X, f; U. J/ m

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 14:52:05 | 显示全部楼层
[root@ceph-1 ~]# mkdir -p /var/lib/ceph/mds/ceph-1
+ W3 k: U7 V: P) x7 z[root@ceph-1 ~]# ceph auth get-or-create mds.ceph-1 mds 'allow *' osd 'allow rwx' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-1/keyring, C$ V9 ?; x. i  p

( \* e& T( r9 w& z' P: o/ A, W0 B( j; a$ |: `
添加MDS 创建mds数据目录 mkdir -p /var/lib/ceph/mds/ceph-`h...* ?( S2 O! C. X& N
添加MDS. B$ O! C- [) E
创建mds数据目录/ H7 t1 a) |5 ]
mkdir -p /var/lib/ceph/mds/ceph-`hostname -s`
' o$ m5 p( G3 _! y* w chown -R ceph.ceph /var/lib/ceph/mds/ceph-`hostname -s`) Y9 e0 e3 o/ t1 m
复制2 @% Z; `! F4 {
创建keyring
7 Q8 r7 a$ M5 N' _6 o ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-`hostname -s`/keyring --gen-key -n mds.`hostname -s`
  P% z# k6 H! S6 q4 S+ Q& V复制7 w8 K" f2 ^4 e) Z( i
导入keyring并设置权限
' w; f. I  t! [; iceph auth add mds.`hostname -s` osd "allow rwx" mds "allow" mon "allow profile mds" -i /var/lib/ceph/mds/ceph-`hostname -s`/keyring) ]( i* e4 f6 _! u& D- N, C
chown ceph:ceph /var/lib/ceph/mds/ceph-`hostname -s`/keyring' Y9 N# \0 H, M6 E
复制5 d5 w/ G- c6 X1 ]* M: Z9 s+ U( S
修改ceph.conf配置文件- h3 B* C( u8 h0 a2 A2 ]
cat >> /etc/ceph/ceph.conf <<EOF* }) D) v' W+ m% b2 M+ S! d$ u6 [
[mds.node-1]
+ P' h2 C2 c7 b1 l host = node-11 c' y4 \1 ^/ m
[mds.node-2]/ Q" V1 P) z7 _% c: t
host = node-2
: n" i+ K/ x: R [mds.node-3]
6 Z% y4 I, l$ R- f+ Y" Q  `3 r: |: o host = node-3( R6 a0 F) f; l
EOF
# U0 y& q8 D6 N) x: L; M5 k复制8 a$ f+ i/ v6 Y( o. m
启动mds服务
! {1 [# V% v2 tsystemctl enable ceph-mds@`hostname -s` && systemctl start ceph-mds@`hostname -s`* {% _1 M; q  U& S
复制
9 I$ l: c$ G- `% K* T5 v3 e) _ 现在状态应该是这样的
0 ^( R$ D' E- X$ |5 P, x+ O3 h4 l' T9 W* M: l# S3 ~; S. L8 }

1 ]# r! `. m+ J2 m! x) j6 A8 G+ C[root@ceph-1 mds]# systemctl start ceph-mds@ceph-1.service 2 ]; t$ D2 ?( ~9 s  |5 b! W) ]
[root@ceph-1 mds]# systemctl status ceph-mds@ceph-1.service 6 a* L& |' C0 W- D
ceph-mds@ceph-1.service - Ceph metadata server daemon
' B+ V& i. b! w7 D" Q   Loaded: loaded ([url=]/usr/lib/systemd/system/ceph-mds@.service[/url]; enabled; vendor preset: disabled)8 ~3 p6 h8 C: C4 L
   Active: active (running) since Tue 2022-07-26 14:51:01 CST; 5s ago
$ n& ~- D: u" r$ ]. s& }7 ~ Main PID: 48091 (ceph-mds)
/ F% J3 c# d' X! h/ g- ?   CGroup: /system.slice/system-ceph\x2dmds.slice/ceph-mds@ceph-1.service
& l, Y0 f) ^' W           └─48091 /usr/bin/ceph-mds -f --cluster ceph --id ceph-1 --setuser ceph --setgroup ceph
) y4 \# F8 p6 v8 K" t, EJul 26 14:51:01 ceph-1.novalocal systemd[1]: Started Ceph metadata server daemon.
* o% e. D8 k! t( ?4 n) jJul 26 14:51:01 ceph-1.novalocal ceph-mds[48091]: starting mds.ceph-1 at -7 b" O7 E+ G8 {5 ?) X0 K. q) F

7 O8 U( _# M. b9 _, j% B( P% C
! c% c+ _: ?) q0 C; G
) H5 O; e' q& J3 P7 b
/ f$ r6 n$ y7 s7 Q- ?9 d; U2 @- v- ?( u

0

主题

0

回帖

9

积分

管理员

积分
9
QQ
 楼主| 发表于 2022-7-26 14:53:25 | 显示全部楼层
您需要登录后才可以回帖 登录 | 注册

本版积分规则

返回首页|Archiver|手机版|小黑屋|易陆发现技术论坛 ( 蜀ICP备2026014127号-1 )

GMT+8, 2026-6-11 22:58 , Processed in 0.026860 second(s), 23 queries .

Powered by Discuz! X5.0

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表