|
|
停止有问题ceph的mon服务：
% f, f8 \( J# K0 |, h9 i& g3 u[root@host11 ~]# systemctl stop ceph-mon@host11' E0 m- P+ H- s/ @1 Y$ y1 Y9 j
6 }2 `9 R; o$ I! c删除有问题的mon数据：. [+ `/ E1 r0 K3 U+ E1 \ @
[root@host11 ~]# rm -rf /var/lib/ceph/mon/ceph-host11/*
: F( i0 O# J6 V& J' c: ` E/ A( F1 P6 `0 D+ n1 z$ Z
尝试使用rsync方式同步：8 i3 t. l% v9 |5 j" r' R- K( L
[root@host11 ~]# rsync -avz root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/
0 @& @+ Q" ]5 ~+ }. \5 E: g-bash: rsync: command not found
* K2 T7 T# J5 a" C[root@host11 ~]# dnf install -y rsync5 b3 {/ z- A# r8 G
-bash: dnf: command not found
( l( B, I" Q- e ^报错：无法使用rsync方式同步：
% E/ t6 J" g% G' Y, Y- f3 |1 H: w u8 Y* k4 T$ N5 F3 N8 I
使用scp方式复制：
7 `% B4 o4 T8 e7 Y/ ]: L, h. A. y[root@host11 ~]# scp root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/5 {/ r% v( T- F# O4 f+ C& M. h
scp: /var/lib/ceph/mon/ceph-host10: not a regular file
% w! R+ S5 @( L. I/ E* [% ^) {% \" K* F, `3 @' H9 |
2 z/ j! e3 b# n: I$ }" S6 Y$ ~! q( @1 X. S2 B
[root@host11 ~]# scp -r root@host10:/var/lib/ceph/mon/ceph-host10/ /var/lib/ceph/mon/ceph-host11/5 G' |, o$ n( f. V
kv_backend 100% 8 9.3KB/s 00:00 ) V \1 \% n7 p
LOCK 100% 0 0.0KB/s 00:00
; n& T7 Q& W0 Y) H' `# ]CURRENT 100% 17 24.8KB/s 00:00
7 `. O$ f! h9 g* @. Y4 {IDENTITY 100% 37 2.7KB/s 00:00 / b8 t/ @$ x5 p8 {( I d. T
OPTIONS-9023316 100% 4943 5.9MB/s 00:00
) |/ ~& d- C# N: E% K# R; ^* JMANIFEST-9024281 100% 4822KB 49.2MB/s 00:00
: _& b8 ~5 z9 P# k9 i0 OOPTIONS-9024284 100% 4943 6.8MB/s 00:00
0 E# C5 x. ^7 |9 V! q3 m- \+ m8 R9106767.log 100% 14MB 51.4MB/s 00:00
( j) Y" q( C# o% o1 a# R9106769.sst 100% 57MB 55.0MB/s 00:01 * Z5 m. m' I0 j8 d+ W# @# D
keyring 100% 77 43.2KB/s 00:00
p% w3 h# h' \ c: c5 t3 Cdone 100% 0 0.0KB/s 00:00 . t" e) K, M {& U
systemd 100% 0 0.0KB/s 00:00
- g, X! ]8 V3 k" \$ }min_mon_release 100% 3 0.2KB/s 00:00 : c& G+ d$ m1 s" n
' ~, ?3 O( L, M, l[root@host11 ceph-host10]# ls6 L7 s! n8 a' o9 [
done keyring kv_backend min_mon_release store.db systemd, W; a# ^1 |" s2 V+ A
[root@host11 ceph-host10]# mv * ..
+ H& V+ w, }8 A& ]% a/ w+ L[root@host11 ceph-host10]# ls" n3 p& C; N3 E+ T3 ?" G% h
[root@host11 ceph-host10]# cd ..
2 h' r3 V3 c7 _ p5 a- x* L* M[root@host11 ceph-host11]# ls5 v8 I" B$ ]' P9 D$ _6 ]+ `
ceph-host10 done keyring kv_backend min_mon_release store.db systemd
, \+ v' M6 P {' e8 j- E8 v P9 Y0 q% y G
[root@host11 ceph-host11]# ls
7 b: n, M' S: J1 M) P7 f4 b5 b* d6 H8 Edone keyring kv_backend min_mon_release store.db systemd
4 `' w$ J% x) V8 j$ D[root@host11 ceph-host11]# ll7 U" d, z% c9 N F6 T
total 124 a' ]0 ^0 T3 Y
-rw-r--r-- 1 root root 0 Mar 28 07:20 done+ c" |# x6 Q$ e' ?& y9 H/ y; i
-rw------- 1 root root 77 Mar 28 07:20 keyring O8 ?0 O5 |; L5 q
-rw------- 1 root root 8 Mar 28 07:20 kv_backend# `* p& ~9 x+ R1 z4 m
-rw------- 1 root root 3 Mar 28 07:20 min_mon_release5 U% j6 N, t' g1 g; D& W
drwxr-xr-x 2 root root 157 Mar 28 07:20 store.db' [; i& z* n, [' d6 M2 X# @9 K" j |
-rw-r--r-- 1 root root 0 Mar 28 07:20 systemd
( \ |& A$ @9 n5 O[root@host11 ceph-host11]# cd ... Y; z" A" h# t8 x5 p+ d8 d/ |
[root@host11 mon]# ls; s+ c+ j' Z' c& K! p
ceph-host11, P# o2 e D; n& |$ x* v, C
[root@host11 mon]# ll" M2 J5 w$ m; T' ]) B1 Q ^- b. I
total 0
! C$ {' x! Q2 tdrwxr-xr-x 3 ceph ceph 105 Mar 28 07:20 ceph-host11+ q' j" z$ h. e4 d, \
[root@host11 mon]# chown -R ceph:ceph ceph-host11/
0 \, x' T( _) p, l0 |[root@host11 mon]# cd ceph-host11/9 J! a" h3 `3 [& `/ W2 F
[root@host11 ceph-host11]# ls4 ~* Q8 U! D) Q- u# ~$ U% k
done keyring kv_backend min_mon_release store.db systemd' A9 q+ z) z! G5 l0 L
[root@host11 ceph-host11]# ll
7 _+ z; X$ J1 B; V+ wtotal 12
3 a' s0 e5 F- [9 x w0 i' w5 V-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 done4 B* {; E7 b5 X9 c% u3 E# w
-rw------- 1 ceph ceph 77 Mar 28 07:20 keyring" E5 I6 y/ q. V
-rw------- 1 ceph ceph 8 Mar 28 07:20 kv_backend! ? q; E. Y% J& z' k
-rw------- 1 ceph ceph 3 Mar 28 07:20 min_mon_release
$ o, l6 n$ v% [+ n! s- adrwxr-xr-x 2 ceph ceph 157 Mar 28 07:20 store.db
4 x1 ^+ k* ^( p! v-rw-r--r-- 1 ceph ceph 0 Mar 28 07:20 systemd5 u- h& r# o/ X) {
) W/ }6 u) E: r- j9 E) a
启动mon服务：# n0 ]7 O( `- \$ \6 P
[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service ' S' W0 `" Z6 B' h0 R
Job for ceph-mon@host11.service failed because start of the service was attempted too often. See "systemctl status ceph-mon@host11.service" and "journalctl -xe" for details.
+ q3 a( k/ f3 @( ^+ K- z$ sTo force a start use "systemctl reset-failed ceph-mon@host11.service" followed by "systemctl start ceph-mon@host11.service" again.7 S2 k7 t4 l- |6 F0 P- B/ f+ _% K& N4 s
" }: P; M) J, X! V
根据提示修改：
c/ C! Y8 [9 D" J[root@host11 ceph-host11]# systemctl reset-failed ceph-mon@host11.service( M$ u5 i% y0 D" L6 i
9 q$ V# R+ Q7 P$ x/ l
再次启动：7 z2 P' p2 ?: Z! f) d2 N, r9 Q
[root@host11 ceph-host11]# systemctl start ceph-mon@host11.service 6 o4 {) K* m6 o3 f3 p6 u
检查状态：
5 |: ]0 {: P) D# S[root@host11 ceph-host11]# systemctl status ceph-mon@host11.service
- [8 [ a0 m: k2 K, i3 U1 y● ceph-mon@host11.service - Ceph cluster monitor daemon' k8 f5 ^1 X1 u( O9 }2 H& t
Loaded: loaded (/usr/lib/systemd/system/ceph-mon@.service; enabled; vendor preset: disabled)
1 m; b R0 l h( e6 ?$ i0 L8 ~ Active: active (running) since Sat 2026-03-28 07:22:00 CST; 11s ago+ n# z7 F4 R8 o3 { m# A) Q/ L
Main PID: 68995 (ceph-mon)
6 L9 z W6 J8 L7 @ CGroup: /system.slice/system-ceph\x2dmon.slice/ceph-mon@host11.service
( z n' m2 S" e- s* o └─68995 /usr/bin/ceph-mon -f --cluster ceph --id host11 --setuser ceph --setgroup ceph, ?/ ]* n' R+ X( y* }
; E$ C0 O ]8 Z0 w# vMar 28 07:22:00 host11 systemd[1]: Started Ceph cluster monitor daemon.
7 N6 Y/ m% T" k1 YMar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.757 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''9 ^4 X; D: p3 _ Y" l2 z: s K
Mar 28 07:22:06 host11 ceph-mon[68995]: 2026-03-28 07:22:06.793 7fc8eef35700 -1 mon.host11@2(electing) e3 failed to get devid for : udev_device_new_from_sub...iled on ''
$ P6 H7 G% T) d" }Hint: Some lines were ellipsized, use -l to show in full.! v" W: f- C( {9 b. {/ Q
[root@host11 ceph-host11]#
$ b5 t2 u5 A% K7 Y; Q0 u0 [/ i$ x# d7 b[root@host11 ceph-host11]# 1 m7 I( `8 l" u
[root@host11 ceph-host11]# ceph -s
. j% k5 }; x, M2 C+ e" `7 @ cluster:7 d; s$ }5 Z8 c0 r2 D3 ]3 W
id: 9d22e36a-2bdd-4d2d-8394-ead777
3 s; N0 D+ S4 y# U: ] health: HEALTH_WARN" k2 m, s! U x8 S: l* H4 Z; M3 P
3 nearfull osd(s)- D: N+ I) H' f! T1 z2 R4 Q
5 pool(s) nearfull
8 O9 ^3 Q3 l: l) l% g4 f 5 daemons have recently crashed
' \3 V0 F8 {- ~
% E9 f9 c# Y q4 |, T services:
/ w* v4 k: d4 n! g6 B1 q mon: 3 daemons, quorum host09,host10,host11 (age 22s)
, _ E$ s$ r' [5 c mgr: host09(active, since 6w), standbys: host11, host10
0 T0 ` q8 u: R1 I. T0 O osd: 40 osds: 40 up (since 6w), 40 in (since 6w)
`. V* ^. ]7 ^# U7 `1 P
& \, a: O# B! I F2 [0 G$ c data:1 K( S# z5 L% E9 J5 H3 S p
pools: 16 pools, 3072 pgs
: |& O/ c( L' X# m: \ X' R9 y objects: 7.15M objects, 27 TiB
/ [4 @/ E7 q! N0 k usage: 65 TiB used, 83 TiB / 147 TiB avail& a" k) n7 d1 M/ K% t, I
pgs: 3069 active+clean
# D/ F5 P. Y3 d 3 active+clean+scrubbing+deep
1 q. Z0 W7 K% t; B! ~+ J3 X4 p
0 e$ e4 j' [# V$ Q io:
# S6 ^( X) F; f( ~, } client: 16 MiB/s rd, 27 MiB/s wr, 339 op/s rd, 380 op/s wr- R$ Q+ j3 @/ N6 V5 l
/ _1 r9 B+ o- ^mon问题解决。! T }) ?8 b5 \4 x( n) K
; S. n) Y. n) K+ W1 j5 n8 v9 c% F9 a2 Q: ^
. z. w$ S+ G3 I& \8 A5 Z |
|
发表于 2026-3-28 10:23:06