|
|
停止有问题ceph的mon服务： W& S! K; ~. d5 B
[root@host11 ~]# systemctl stop ceph-mon@host11
/ q( h# ]. [' J$ Y6 M( ^, u/ s# e* h/ Z
删除有问题的mon数据：
$ k+ ^7 `0 I* E! p! ]6 y+ u, r. ?[root@host11 ~]# rm -rf /var/lib/ceph/mon/ceph-host11/*
" @2 A3 F% ]1 R% y, ^3 r
3 p! |* t* T2 O5 K2 i: s; v; n尝试使用rsync方式同步：0 t- R' j ^) r: h8 g6 s
[root@host11 ~]# rsync -avz root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
8 }% T; B% i% |+ w6 v-bash: rsync: command not found3 o% J" q& s l
[root@host11 ~]# dnf install -y rsync3 }/ s, f9 U1 f! {
-bash: dnf: command not found, o1 b; [& r% q" B' y
报错：无法使用rsync方式同步：% ?: s- Z# d, n. o. Q" b+ s7 ~
+ U: [" P, R. u' S3 c: Y4 [# t使用scp方式复制：
+ z, F- x+ Y& l/ w& f[root@host11 ~]# scp root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/* g0 [9 Z( p+ E) M6 F
scp: /var/lib/ceph/mon/ceph-host10: not a regular file% t @' g/ M% F: N) `) k$ n
4 f- m) j8 z: t' v6 b' Y9 u3 ]& a, s; G3 V% ?
N, L- e. h5 Z) g' }[root@host11 ~]# scp -r root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
3 k. r* G# q1 w7 p; s# e0 N! Fkv_backend 100% 8 9.3KB/s 00:00
4 C5 x9 G+ u- C- k( vLOCK 100% 0 0.0KB/s 00:00 I* u% {: D5 s: ], f/ E* x
CURRENT 100% 17 24.8KB/s 00:00 # |% c+ E3 ^; `+ A# R8 t3 d
IDENTITY 100% 37 2.7KB/s 00:00 - ]; l1 r0 }+ e
OPTIONS-9023316 100% 4943 5.9MB/s 00:00
2 v; O3 i* U* N2 L, o; HMANIFEST-9024281 100% 4822KB 49.2MB/s 00:00 ' c/ W. H$ u3 S9 \, r
OPTIONS-9024284 100% 4943 6.8MB/s 00:00
+ d% w9 @+ H3 \# v/ T* [9106767.log 100% 14MB 51.4MB/s 00:00 & C7 f' ?* G0 `' j. ]
9106769.sst 100% 57MB 55.0MB/s 00:01
9 [, p6 m) P3 lkeyring 100% 77 43.2KB/s 00:00 k$ [( Q0 P1 F) L \$ v- S
done 100% 0 0.0KB/s 00:00 ( D6 C0 F7 }. c% A
systemd 100% 0 0.0KB/s 00:00 - n3 C0 e2 f6 L" r7 @" V) k k
min_mon_release 100% 3 0.2KB/s 00:00 6 x: _) m' E* Y, G0 X% v# v: Q) w
/ c( S+ }4 V6 g$ c% B5 F* D
[root@host11 ceph-host10]# ls
# R/ X5 Z0 }) i/ f# B& Z3 h3 Ndone keyring kv_backend min_mon_release store.db systemd
0 t0 L+ q8 x( @3 w8 ~7 q[root@host11 ceph-host10]# mv * ..+ {) k6 d. O6 X8 j
[root@host11 ceph-host10]# ls
6 K' X4 b5 @: c/ p( O( h, m[root@host11 ceph-host10]# cd ... v# V1 K4 U2 u# M3 \
[root@host11 ceph-host11]# ls) U) x) A7 N# p8 E2 D! o8 L, S
ceph-host10 done keyring kv_backend min_mon_release store.db systemd
' d2 N2 t: Z6 z+ q
) K( b, ]8 c( _) u4 f[root@host11 ceph-host11]# ls4 Z0 o) n" }1 [2 T
done keyring kv_backend min_mon_release store.db systemd
, s/ D/ v% q+ S$ |2 n M" r: u[root@host11 ceph-host11]# ll
, o& l) N( V, f; ~, M/ L' i7 P* ?5 z ototal 12
, f& I% W6 [ f, [8 y& q-rw-r--r-- 1 root root 0 Mar 28 07:20 done. L7 Z- K1 J8 ^% i/ Z8 q: F
-rw------- 1 root root 77 Mar 28 07:20 keyring
- V. H k, @' h Y" l- L% E1 O2 v-rw------- 1 root root 8 Mar 28 07:20 kv_backend
% ?# e9 P7 D' ^6 J-rw------- 1 root root 3 Mar 28 07:20 min_mon_release
; m* p0 T8 E; U$ r8 Ydrwxr-xr-x 2 root root 157 Mar 28 07:20 store.db
}% ^7 d6 P0 T. k* F" I8 b-rw-r--r-- 1 root root 0 Mar 28 07:20 systemd
$ F. L& ^+ N- C0 w6 |0 j[root@host11 ceph-host11]# cd ..4 ^2 A2 K1 }# G$ K, W
[root@host11 mon]# ls
% }, p$ m- c0 Y! [, @ceph-host11
m/ C; h6 W/ h7 a& g[root@host11 mon]# ll
$ q7 d" p8 z. M" jtotal 0
- z) J: ~: b s; u+ Z+ N# P o2 }0 ndrwxr-xr-x 3 ceph ceph 105 Mar 28 07:20 ceph-host11- d# Z* c8 v O5 t8 K" J2 k4 |
[root@host11 mon]# chown -R ceph:ceph ceph-host11/4 m# w8 [: s0 z7 T) ~
[root@host11 mon]# cd ceph-host11/
$ F+ ^% \9 ?; R! B[root@host11 ceph-host11]# ls
9 E/ V0 ` L# U6 ^0 c2 ?5 Zdone keyring kv_backend min_mon_release store.db systemd
/ X" v6 }$ a7 _ G[root@host11 ceph-host11]# ll
4 ]7 I$ l6 P; Y! ?8 B- X; o1 Ytotal 12
0 ~3 q8 s; w! S$ a% N$ J2 P( J5 }+ ?-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 done3 g* I$ C4 l4 K+ T( H
-rw------- 1 ceph ceph 77 Mar 28 07:20 keyring1 x1 h% `0 H5 \4 l% _- e
-rw------- 1 ceph ceph 8 Mar 28 07:20 kv_backend' M W" x$ W# V' M3 C3 L
-rw------- 1 ceph ceph 3 Mar 28 07:20 min_mon_release
( O$ A% a* ]* w1 o! f) M( }0 udrwxr-xr-x 2 ceph ceph 157 Mar 28 07:20 store.db
$ @: t5 U: o; Y/ m b-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 systemd+ k" K8 h; o* ~8 k' c! E
* O( h7 |* y7 D5 g启动mon服务：- K4 {% N) D5 U9 k
[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service 3 w$ r+ n4 N6 Y+ H# \6 n( h
Job for ceph-mon@host11.service failed because start of the service was attempted too often. See "systemctl status ceph-mon@host11.service" and "journalctl -xe" for details.7 \* R! @, \. p2 [6 f
To force a start use "systemctl reset-failed ceph-mon@host11.service" followed by "systemctl start ceph-mon@host11.service" again.
; v; T& v$ e& E& Y) ~ k6 {& L' K/ Y8 H+ N- t, D a! T" R
根据提示修改：
+ y- T1 N" P+ o: y! z( @4 E[root@host11 ceph-host11]# systemctl reset-failed ceph-mon@host11.service" m& i1 T2 B; u$ E9 p f. s
( S2 a9 O& j+ V- ?! i: x) q( m$ s
再次启动：3 }7 O) ^# S" j& v1 d4 c& }
[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service
0 L* Y2 K% I2 Q9 y检查状态：1 ^* E5 F, `, j
[root@host11 ceph-host11]# systemctl status ceph-mon@host11.service
! E9 Y4 r8 U; T0 s, o● ceph-mon@host11.service - Ceph cluster monitor daemon( R6 [+ d, |% R* t$ N+ R6 \' ]
Loaded: loaded (/usr/lib/systemd/system/ceph-mon@.service; enabled; vendor preset: disabled)
/ N% h" t& t" B; Y& i Active: active (running) since Sat 2026-03-28 07:22:00 CST; 11s ago
9 t1 P9 V* m5 a' Z( a: c- R( Y Main PID: 68995 (ceph-mon)( p1 L/ A# `6 E4 b& L
CGroup: /system.slice/system-ceph\x2dmon.slice/ceph-mon@host11.service2 v0 ^9 b/ o/ k2 s! [1 c
└─68995 /usr/bin/ceph-mon -f --cluster ceph --id host11 --setuser ceph --setgroup ceph6 b0 m. A* T4 k9 g& F' `
% ]( j& s% y, R/ k5 P: ?- C" ~5 P8 ZMar 28 07:22:00 host11 systemd[1]: Started Ceph cluster monitor daemon.0 r# k) z X: I, _
Mar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.757 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''6 ?2 e y. @1 x" E4 @3 N
Mar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.793 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''
+ V- c8 O7 E7 j# j. c/ H EHint: Some lines were ellipsized, use -l to show in full.1 k! y* D- W: {
[root@host11 ceph-host11]#
4 \9 Y- s: l" D* [ Z [[root@host11 ceph-host11]# . O* T; b( s! K1 {: U, H
[root@host11 ceph-host11]# ceph -s
$ p- R3 s6 [8 H- V6 `6 I cluster:
7 G0 a" g) p. I- T id: 9d22e36a-2bdd-4d2d-8394-ead7772 L/ f' |. v/ _2 D) [
health: HEALTH_WARN2 S) l V& M0 }
3 nearfull osd(s)* u: n K+ b+ | p: J' ], v. H+ u
5 pool(s) nearfull
9 y( E) w% Y' ~; J f8 f 5 daemons have recently crashed* I& Z5 ^. Y( a. ~$ F
5 k- Q- P+ i: Z' J+ A8 B5 ~% a4 ] services:
- w1 ]; [* G. t- |8 [ mon: 3 daemons, quorum host09,host10,host11 (age 22s)
# }2 D) f/ b* c$ U$ S# s mgr: host09(active, since 6w), standbys: host11, host10' Z4 x$ \- c O- Z+ ]1 f' P q- P
osd: 40 osds: 40 up (since 6w), 40 in (since 6w)
7 T* y& s; E( f$ }' p" S. w8 Q+ s" ]- w% o8 @
data:
% b2 M( ]3 v. G7 z- [ pools: 16 pools, 3072 pgs
) u4 [3 | \4 S( u objects: 7.15M objects, 27 TiB! |$ ^7 ~3 B6 s |. o
usage: 65 TiB used, 83 TiB / 147 TiB avail
' u' J$ X+ ^" m4 ]3 R pgs: 3069 active+clean
0 d* g% c$ B" _. ~4 j! E5 m 3 active+clean+scrubbing+deep
6 ~% L# e6 A2 [7 d7 e) M7 Z4 e( `- S
io:
. J4 b/ B7 T, t; R" \8 w& M client: 16 MiB/s rd, 27 MiB/s wr, 339 op/s rd, 380 op/s wr2 p, c2 E- {1 d. W
! M' T) g) K* V( }4 Q- D' G
mon问题解决。6 c' s* Q$ V/ A; G l
2 r/ d2 M, o7 V
$ G1 Q: F% ^: S9 G |
|
发表于 2026-3-28 10:23:06