|
|
停止有问题ceph的mon服务：
9 F" C1 f+ A L) L% ?! Z. _- i[root@host11 ~]# systemctl stop ceph-mon@host11
4 G; K% C" s2 Q$ R g9 Q2 f, K4 Y
3 t' C( x8 W2 A5 g8 U$ v- c0 K' n删除有问题的mon数据：
: D0 k4 A& p4 ?, ]. }, K K' J7 S[root@host11 ~]# rm -rf /var/lib/ceph/mon/ceph-host11/*
' A1 x2 }9 U. Y, ?7 z5 t9 m8 g# l; U& [
尝试使用rsync方式同步：( J. V9 n, w; a
[root@host11 ~]# rsync -avz root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
. C: G: n3 e: ]& V. p) w* W-bash: rsync: command not found
1 `2 M4 y ?' j9 M2 x[root@host11 ~]# dnf install -y rsync
3 U" Y% v* d3 X3 z. L2 }5 ?-bash: dnf: command not found
; @3 f$ T, A3 c( b I2 [4 n; X; s2 J$ ~, }报错：无法使用rsync方式同步：
7 o" K# b; g$ b- M! p5 H2 a+ ~7 M; m+ Y& z
使用scp方式复制：" l2 B* l3 c- k1 @ v" n
[root@host11 ~]# scp root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
# \0 o, M9 U/ y) Kscp: /var/lib/ceph/mon/ceph-host10: not a regular file
6 @* V( m4 @ f, z2 M: a4 M! {6 J5 j' c2 b
: L; w, a) J* f
; m( g8 }: @) B* J* H2 @
[root@host11 ~]# scp -r root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
. K# p' V: X, I) U4 I! A4 t# @kv_backend 100% 8 9.3KB/s 00:00
# R& i' w$ \& G9 _) D. b3 \+ XLOCK 100% 0 0.0KB/s 00:00
5 \0 F/ e, ]$ X) a9 i* z& h7 [1 qCURRENT 100% 17 24.8KB/s 00:00
. u( {4 @! V/ |9 ]) \( Y3 _1 iIDENTITY 100% 37 2.7KB/s 00:00
# E) P& D! G: x; R3 U' jOPTIONS-9023316 100% 4943 5.9MB/s 00:00
! F+ l' F" J! ?7 U# W/ d X: p% W3 PMANIFEST-9024281 100% 4822KB 49.2MB/s 00:00 . [. p3 ]0 k" {5 f! J) P
OPTIONS-9024284 100% 4943 6.8MB/s 00:00 % E! ^3 ?3 {0 _) t
9106767.log 100% 14MB 51.4MB/s 00:00
c' Z* X0 w$ m7 }7 ~8 m0 i9106769.sst 100% 57MB 55.0MB/s 00:01 + {- u6 N# K* Y9 ~3 J
keyring 100% 77 43.2KB/s 00:00 & k/ D% H& m2 H# S) B5 `
done 100% 0 0.0KB/s 00:00
5 `- D. C/ S' w* T( ]0 Qsystemd 100% 0 0.0KB/s 00:00 * f6 p6 d( \ P; [0 {0 Y( M
min_mon_release 100% 3 0.2KB/s 00:00
Z- h$ k5 y8 ?
8 m3 q* R2 p9 G: Z8 Y[root@host11 ceph-host10]# ls
2 _5 e9 C; q4 C. hdone keyring kv_backend min_mon_release store.db systemd: _% @& s9 j! g) r1 L
[root@host11 ceph-host10]# mv * ..- V$ L# x8 U8 h0 S( \+ D! T7 i
[root@host11 ceph-host10]# ls
7 [" k$ a- V4 G8 C+ |5 l[root@host11 ceph-host10]# cd ..
) F' I" v: P2 s( G8 q[root@host11 ceph-host11]# ls
. y7 M. a- c( k: Q) F ]3 t- Mceph-host10 done keyring kv_backend min_mon_release store.db systemd
5 |4 C9 \3 _. |) \ Y- l$ x, }5 ]: m# s+ u6 z4 D. l# c$ B
[root@host11 ceph-host11]# ls+ ^( w3 n' L I, p/ j& j6 f
done keyring kv_backend min_mon_release store.db systemd7 S/ f' { z. B# P
[root@host11 ceph-host11]# ll
% v( J, g, n9 J( m. c3 ototal 122 d5 ] R( u# I. q
-rw-r--r-- 1 root root 0 Mar 28 07:20 done
; t/ j ]* e: k) h+ _4 `7 Q-rw------- 1 root root 77 Mar 28 07:20 keyring
M& \2 W" O" ]# G h* I- C-rw------- 1 root root 8 Mar 28 07:20 kv_backend3 \% ~0 e# G' ]* C
-rw------- 1 root root 3 Mar 28 07:20 min_mon_release
$ a$ a! J* C6 j0 e9 y( h% l$ t2 e" ?drwxr-xr-x 2 root root 157 Mar 28 07:20 store.db
) v# q2 x- A$ P( Q8 V+ v; _( o Y-rw-r--r-- 1 root root 0 Mar 28 07:20 systemd; N m4 Q( J* O! K D
[root@host11 ceph-host11]# cd ..* B5 x0 N! D# Z1 p+ D
[root@host11 mon]# ls
" J k: B- S! sceph-host11% @+ t5 ]4 M1 j; A
[root@host11 mon]# ll" c) r$ b; ?( ]7 u5 p, H% A
total 0) e+ ^( p' D: l2 `2 k9 y+ ~$ C
drwxr-xr-x 3 ceph ceph 105 Mar 28 07:20 ceph-host11
' M/ R3 |( c, O" M1 C[root@host11 mon]# chown -R ceph:ceph ceph-host11/# Y, j& i* S% G. x3 D, b' Y3 l0 s
[root@host11 mon]# cd ceph-host11/
' Z1 Z0 r9 ^9 I' _, ^: _[root@host11 ceph-host11]# ls7 v" y: s6 b% C R t+ I
done keyring kv_backend min_mon_release store.db systemd' p5 E% o* Y H# p
[root@host11 ceph-host11]# ll3 D, q, M6 P1 I% i7 F( F! H
total 12
; |% t4 ]6 R& f6 U [1 K$ I-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 done
6 X( Z5 t8 `% p* U; W1 m7 W$ ~-rw------- 1 ceph ceph 77 Mar 28 07:20 keyring, r4 R/ i4 c. a0 B# M8 X0 L
-rw------- 1 ceph ceph 8 Mar 28 07:20 kv_backend2 i0 e/ c/ a/ p3 ?8 d9 L
-rw------- 1 ceph ceph 3 Mar 28 07:20 min_mon_release( R- O% k) M2 S9 Y
drwxr-xr-x 2 ceph ceph 157 Mar 28 07:20 store.db# \) Q$ x5 b. C# Y
-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 systemd
" c; W! d! L2 r& N" f$ \0 q2 V! w4 X4 o
启动mon服务：
+ |- {- H! J# n6 X# U2 D* }[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service
- H' [0 |5 i& U+ aJob for ceph-mon@host11.service failed because start of the service was attempted too often. See "systemctl status ceph-mon@host11.service" and "journalctl -xe" for details.
) l/ E3 g* I6 U' Z$ zTo force a start use "systemctl reset-failed ceph-mon@host11.service" followed by "systemctl start ceph-mon@host11.service" again.
, J, k9 B) Z( M' s4 i, {
4 S# W, e& b: H. ]: b8 m根据提示修改：
9 \0 Y! G* l1 ~' ]+ G# u1 H[root@host11 ceph-host11]# systemctl reset-failed ceph-mon@host11.service9 E0 `6 B# V7 s$ m$ q! h2 T
6 z8 } Q8 s- r+ t$ e, \. a
再次启动：
' H7 _. l$ |& B. P; x3 Y/ g[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service
6 e. v3 O# R/ G/ n# B检查状态：
$ V4 [. n; c [. a& @2 v[root@host11 ceph-host11]# systemctl status ceph-mon@host11.service
5 n& ?) }' I& q$ K' ]: \● ceph-mon@host11.service - Ceph cluster monitor daemon U2 L: f- `0 l; B
Loaded: loaded (/usr/lib/systemd/system/ceph-mon@.service; enabled; vendor preset: disabled)' P# h4 d, H" S; f
Active: active (running) since Sat 2026-03-28 07:22:00 CST; 11s ago
# D& s5 G i* |$ |6 R8 V4 M# G5 I Main PID: 68995 (ceph-mon)
" s- M2 s1 V8 I; |* U CGroup: /system.slice/system-ceph\x2dmon.slice/ceph-mon@host11.service3 b3 ^' B" I$ j. [5 C
└─68995 /usr/bin/ceph-mon -f --cluster ceph --id host11 --setuser ceph --setgroup ceph. j+ x) F/ t# e# }, }* ?' q5 t
: Q9 ~. a. Q1 |' x s, ?
Mar 28 07:22:00 host11 systemd[1]: Started Ceph cluster monitor daemon.
% t, q( r' ~. PMar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.757 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''
]$ l! d% f% H* y' e% U! HMar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.793 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''
+ Z5 ^5 y' y/ e( G7 \Hint: Some lines were ellipsized, use -l to show in full.
7 `! t9 h+ Y8 Y) f) E[root@host11 ceph-host11]#
& e! v! ?5 [$ k. ]% }- g4 _; n[root@host11 ceph-host11]#
3 Q: M H9 v) [5 b/ j% J1 M. R. Z[root@host11 ceph-host11]# ceph -s
- L2 F# |& c( S* Z2 ?5 q! M* U cluster:
( k$ g; k& x& ]: E! L1 i Q id: 9d22e36a-2bdd-4d2d-8394-ead7770 g* H/ l( S2 ]+ W2 N
health: HEALTH_WARN
\# k% m% [; d1 Q; a 3 nearfull osd(s)
7 a: ?% v% K' L; p 5 pool(s) nearfull7 e! r5 E4 o v) Q `0 R2 y
5 daemons have recently crashed; p) c; p6 {) A
, O$ i) C4 d1 g5 m9 h" o% j+ }
services:5 ]" X* a3 X; t% H5 W1 h* N% L
mon: 3 daemons, quorum host09,host10,host11 (age 22s)1 T. D. K9 \3 _3 T
mgr: host09(active, since 6w), standbys: host11, host10 z g, w9 k7 A9 i/ [. Q/ x m
osd: 40 osds: 40 up (since 6w), 40 in (since 6w)5 T- e, }2 ]) w2 ~) Z2 {& O% g
+ J- _2 {- r+ X1 U' g* _' q9 c) g) L2 h( v data:5 `+ e! s4 Z3 n/ U1 Y
pools: 16 pools, 3072 pgs
* |: \1 K ~! ]) a) D4 _& w! b+ d objects: 7.15M objects, 27 TiB
6 P7 J8 G7 S! l/ ^5 W usage: 65 TiB used, 83 TiB / 147 TiB avail( `9 P# c' C8 y$ p
pgs: 3069 active+clean# U1 T# T- Z; Z8 ]: t5 q9 Z1 a
3 active+clean+scrubbing+deep
, i6 ?2 i& o3 t! ~- l! I: L
4 d2 Z& F, p5 `) L% s1 P io:
+ H b3 x. Q' v9 g3 o client: 16 MiB/s rd, 27 MiB/s wr, 339 op/s rd, 380 op/s wr) j) G# V: ]! J+ i( g9 i! R# R
) _% E' F+ K+ J- u* ?- @
mon问题解决。; O5 o9 q% A, T, Q2 D
& Z: B/ M* ^/ N. K
: k9 {' O+ u! y% o: k/ ^
|
|
发表于 2026-3-28 10:23:06