找回密码
 注册
查看: 509|回复: 2

HEALTH_WARN 1/5 mons down ceph 分布式存储检查状态告警 解决过程

[复制链接]

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
发表于 2022-11-15 10:08:55 | 显示全部楼层 |阅读模式
HEALTH_WARN 1/5 mons down,HEALTH_WARN 1/5 mons down ceph 分布式存储检查状态告警 解决过程
& u) ?+ Q, ?5 _- b: Q8 g% V: q0 ?+ R% ?3 e+ M) @. B  n
[root@hostceph1 deploy]# ceph -s4 p1 Y2 ~; R2 i2 l3 {! S5 L2 a' z
  cluster:
* u6 v* Y( k5 O. K    id:     8daade9f-1db5-4e9e-a0ac-cbe1b3abafd11 c( Y* U" x" i0 \0 g/ Q
    health: HEALTH_WARN
  V- H0 k! t( M9 Y! {* p' J            1/5 mons down, quorum hostceph1,hostceph2,hostceph3,compute085 D3 B8 C- m' |
+ U! W9 `% }0 W
  services:
" I- I0 j: @4 W( T" ~* C    mon: 5 daemons, quorum hostceph1,hostceph2,hostceph3,compute08 (age 2w), out of quorum: compute06
6 s# z$ f  ]4 {) T7 ~/ _( x    mgr: hostceph2(active, since 5M), standbys: hostceph3, hostceph1, compute07, compute09, compute08, compute06
( s1 l% W0 {5 K, q    osd: 34 osds: 34 up (since 3M), 34 in (since 5M)# _. g- ~  m) l5 ^6 r& m" T, D
, K- |! Y8 z. r. S3 B, Q6 |5 {
  data:" y. ?6 m: F+ r
    pools:   9 pools, 8832 pgs
1 N' E6 S5 L, A! o. D* |    objects: 312.99k objects, 1.2 TiB* k+ y! M. E0 p& ]' ~
    usage:   8.3 TiB used, 118 TiB / 127 TiB avail
9 ]+ }7 k" v/ L+ Y6 u! b  c- J    pgs:     8831 active+clean9 _0 B9 C" R: d' x' K3 c" Q2 K: c; x4 L
             1    active+clean+scrubbing+deep
+ h0 K8 C- B3 W$ n* h$ U( R1 g* t! x) Z1 T* B
  io:
9 z$ @2 k" {7 w" }4 R* w( _/ [7 F    client:   114 KiB/s rd, 712 KiB/s wr, 138 op/s rd, 51 op/s wr
: f2 ^6 d; `# \6 O7 T8 ~3 w, a' h9 x0 ^
[root@hostceph1 deploy]# ceph health detail 9 v2 Y0 @1 k( I, T3 z: [/ y' Q
HEALTH_WARN 1/5 mons down, quorum hostceph1,hostceph2,hostceph3,compute086 G% r# F5 M" ?' f4 t; Y
MON_DOWN 1/5 mons down, quorum hostceph1,hostceph2,hostceph3,compute08! B8 m# I  k( E8 X
    mon.compute06 (rank 3) addr [v2:172.20.1.9:3300/0,v1:172.20.1.9:6789/0] is down (out of quorum)9 O9 X1 g' X3 m% ]  d$ l1 f
[root@hostceph1 deploy]# ping 172.20.1.92 f% Q8 ^8 W# s) j6 p$ g
PING 172.20.1.9 (172.20.1.9) 56(84) bytes of data.( z* y7 @" o5 F& ~: [# ]; b5 `' U
64 bytes from 172.20.1.9: icmp_seq=1 ttl=64 time=0.125 ms0 x; B0 o" u7 [' V3 G
64 bytes from 172.20.1.9: icmp_seq=2 ttl=64 time=0.099 ms. R  C& ~$ P1 b# ~& K7 y
^C
& k4 U) j% ]4 z2 ~0 }4 U. y--- 172.20.1.9 ping statistics ---$ i- [5 }6 b  m+ l
2 packets transmitted, 2 received, 0% packet loss, time 999ms+ C9 ~" _, A( D% Y" }2 D4 @# w8 L
rtt min/avg/max/mdev = 0.099/0.112/0.125/0.013 ms7 r: a& ?8 \4 L1 j8 V8 a
[root@hostceph1 deploy]# ls+ S# `5 E5 z, s' M
ceph.bootstrap-mds.keyring  ceph.bootstrap-osd.keyring  ceph.client.admin.keyring  ceph.conf.bak         ceph.mon.keyring  cmap.txt   osdcreate.sh  ssdcrub.sh& D3 I6 x. k( A. C: g$ ?
ceph.bootstrap-mgr.keyring  ceph.bootstrap-rgw.keyring  ceph.conf                  ceph-deploy-ceph.log  cmap              cubosd.sh  pool.sh       ssdosdcreate.sh! p# ^+ A" Y' r' y/ w9 c
[root@hostceph1 deploy]# vim ceph.conf9 v( m3 q0 I+ h4 Z  B
[root@hostceph1 deploy]#
5 v( t& S2 l2 c# U2 I$ g. i
2 P1 V/ ~1 x& @- ^& R) t, j5 \! _6 E3 y$ \- i
! V9 K' D+ a- b" W
检查发现:# Y' O$ m) a% {& ~
[root@compute06 ~]# vim /etc/ceph/ceph.conf 5 E, Q: q1 Q6 H/ ?8 V9 O1 T

+ s2 F- H( c# Z1 O9 T) N; k! @# |* o[global]
' k/ Q/ D/ \8 Sfsid = 8daade9f-1db5-4e9e-a0ac-cbe1b3abafd10 K* |  q# {3 A! G3 g
mon_initial_members = hostceph1, hostceph2, hostceph3, compute06, compute08! O6 D' D( c% ~6 d! A" x
mon_host = 172.20..13, 172.20..14, 172.20.x.15, 172.20.x.11, 172.20.x.13) {# W/ k0 C' e  s# L5 ~7 K
auth_cluster_required = cephx
$ K4 X, o4 g# Q- e2 R- xauth_service_required = cephx+ r/ }$ F0 D& m- B/ v  |) R
auth_client_required = cephx2 q' ?0 [4 j, E; ~
* G  q9 G& L5 I4 S1 ^
7 l; M& c' W% v8 n
没有06的ip地址:6 ?+ \. a; }) A% K9 j' R/ d$ J, \
5 T3 U$ C0 W( B: a) k
在deploy节点修改配置文件
  f  |8 h: O7 m
2 n( |' t# Y, S5 q, Rvim /etc/ceph/ceph.conf ' |6 L+ V) W4 n; X; J3 A# p+ c

; F: w* ]4 b" ?  V" f[global]
7 v$ v$ f& v8 ~- C1 Cfsid = 8daade9f-1db5-4e9e-a0ac-cbe1b3abafd1" j$ f0 R: M0 R
mon_initial_members = hostceph1, hostceph2, hostceph3, compute06, compute08
6 |, z1 H# S4 d) T0 M; @mon_host = 172.20..13, 172.20..14, 172.20.x.15, 172.20.x.11, 172.20.x.97 Y6 V/ T# T4 F' V! \
auth_cluster_required = cephx
& D3 W" G# L1 m% G0 [/ Bauth_service_required = cephx# L' y* n% O' G
auth_client_required = cephx% T5 G# w% s% v9 b: o) B) f

( r- V6 v/ r% f, P  v& W2 @5 O0 P( ?- R( D( j% v, @
8 o$ T+ c2 H2 j, q% k
9 P! N2 C8 o7 \: i1 {& {
使用ceph-deploy admin  将配置文件下发到其他节点:& a4 N. f2 b4 l; B1 ?
ceph-deploy --overwrite-conf admin hostceph{1,2,3} compute0{6,7,8,9}
) o4 I& m' P% |7 R( ~! s' P% q[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf5 F" Y4 q' L8 Z6 k, M9 R9 a
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf admin hostceph1 hostceph2 hostceph3 compute06 compute07 compute08 compute09- B/ ~. F7 v+ E( ]& |* e7 f! z5 E) e
[ceph_deploy.cli][INFO  ] ceph-deploy options:: M& k1 v1 A! X0 J+ z2 T1 t' p
[ceph_deploy.cli][INFO  ]  username                      : None
( w! L, U/ }! N/ W( E. {+ k[ceph_deploy.cli][INFO  ]  verbose                       : False
- Y4 T! u  V, t$ [! @+ C[ceph_deploy.cli][INFO  ]  overwrite_conf                : True- T: v, P! ~; \0 k& ~
[ceph_deploy.cli][INFO  ]  quiet                         : False
9 N- w3 D. U+ E! Z7 f5 ]: O[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fe87edcd290># I! Q5 b* \4 b% J
[ceph_deploy.cli][INFO  ]  cluster                       : ceph* e1 n1 |' M% y- z; Y3 M& h) @
[ceph_deploy.cli][INFO  ]  client                        : ['hostceph1', 'hostceph2', 'hostceph3', 'compute06', 'compute07', 'compute08', 'compute09']
2 c# Y7 G3 c7 _! ?/ W% G. o[ceph_deploy.cli][INFO  ]  func                          : <function admin at 0x7fe87f8de230>
: w8 Q) G  X: m# ^- S[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
0 X4 Y( |7 O1 }5 _5 G6 E[ceph_deploy.cli][INFO  ]  default_release               : False
" w' N0 J2 u" K[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to hostceph1+ O* N& a* p: c; J6 J# a2 k: ?* B
[hostceph1][DEBUG ] connected to host: hostceph1
1 L- O# k+ M, N, b% ^[hostceph1][DEBUG ] detect platform information from remote host2 V4 W1 U1 F  t$ B- ]. ]' F: y3 _

+ y) N1 G% G4 v6 O9 l7 a2 v' E' D; P7 U; i- b6 q; l

( G' @! p* g5 H! O
1 K& V7 h1 e0 u0 X8 a1 C$ b) K! \

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-11-15 10:10:36 | 显示全部楼层
ceph-deploy --overwrite-conf mon  destroy compute06 : g- {( n  P" S( u. W1 M
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
7 I0 o& u0 y8 G5 Y% Y% S[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf mon destroy compute067 k4 j; t2 w9 y6 x
[ceph_deploy.cli][INFO  ] ceph-deploy options:
7 ~4 f7 V7 X+ w! q5 H( f; p+ L[ceph_deploy.cli][INFO  ]  username                      : None& ?1 G0 m, _* V* Q' D
[ceph_deploy.cli][INFO  ]  verbose                       : False
7 h3 o( p1 o. v( U0 R[ceph_deploy.cli][INFO  ]  overwrite_conf                : True" K) R9 P. |) P& d# w* m! W
[ceph_deploy.cli][INFO  ]  subcommand                    : destroy. e! r+ L' H' o# W( |( }
[ceph_deploy.cli][INFO  ]  quiet                         : False
" s8 {) h* D" T6 S0 T[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f90d4005e60>
/ v( g! X# Q; N[ceph_deploy.cli][INFO  ]  cluster                       : ceph% f# O' C/ ]* f; s4 E- g# C( U; g
[ceph_deploy.cli][INFO  ]  mon                           : ['compute06']" r6 s2 N, ~2 e. H" y3 g" j
[ceph_deploy.cli][INFO  ]  func                          : <function mon at 0x7f90d4059410>
; O& N% C0 h" h3 H$ |# p[ceph_deploy.cli][INFO  ]  ceph_conf                     : None8 o  d& w  N1 @; V
[ceph_deploy.cli][INFO  ]  default_release               : False( i) ~4 M* |7 }& b
[ceph_deploy.mon][DEBUG ] Removing mon from compute06
9 r, }2 B+ k$ k# X[compute06][DEBUG ] connected to host: compute06 ; x: }9 L9 l+ Z( Y' K
[compute06][DEBUG ] detect platform information from remote host
3 R; l8 t* W7 q; y[compute06][DEBUG ] detect machine type' W* g) s( M$ l* W7 e4 d
[compute06][DEBUG ] find the location of an executable
# B. [0 x" N- {[compute06][DEBUG ] get remote short hostname
- r2 ?$ t% U' }& I% [+ @[compute06][INFO  ] Running command: ceph --cluster=ceph -n mon. -k /var/lib/ceph/mon/ceph-compute06/keyring mon remove compute065 w7 E; C$ ?" f" \5 ~8 T% l
[compute06][WARNIN] 2022-11-15 10:16:22.023 7fabc2ca4700 -1 auth: unable to find a keyring on /etc/ceph/ceph.mon..keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory( A' E& o0 j/ x; u
[compute06][WARNIN] 2022-11-15 10:16:22.023 7fabc2ca4700 -1 AuthRegistry(0x7fabbc065d78) no keyring found at /etc/ceph/ceph.mon..keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
1 C2 l& @; p5 m! }/ K8 O0 Q[compute06][WARNIN] removing mon.compute06 at [v2:172.20.1.9:3300/0,v1:172.20.1.9:6789/0], there will be 4 monitors. t3 I8 V2 t9 u7 ?7 _
[compute06][INFO  ] polling the daemon to verify it stopped4 i8 \9 s4 o2 F) R, h# X+ h& f' D
[compute06][INFO  ] Running command: systemctl stop ceph-mon@compute06.service% ?& [5 k& m( |; Q; N; O1 p# G

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2022-11-15 10:12:15 | 显示全部楼层
ceph-deploy --overwrite-conf mon  add compute06
* j. s9 H) w- F) ]3 H[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf0 o! f, U1 Q4 o" R+ ^3 a
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf mon add compute06
2 `5 W$ F, @' ]% R* H3 H[ceph_deploy.cli][INFO  ] ceph-deploy options:
, z. N% A0 I, n. o$ S* w[ceph_deploy.cli][INFO  ]  username                      : None
  e& a7 a7 m" f* `& H+ E[ceph_deploy.cli][INFO  ]  verbose                       : False
5 E% ^; h9 P- w" i' t2 f[ceph_deploy.cli][INFO  ]  overwrite_conf                : True$ c( s$ L- ^# H$ u- j# o
[ceph_deploy.cli][INFO  ]  subcommand                    : add
5 y7 d2 U' ?8 A0 Z[ceph_deploy.cli][INFO  ]  quiet                         : False
/ ^2 L; d- m6 ?# R[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fe3aef63e60>) F! W9 c0 l, ^7 T, }6 j% h
[ceph_deploy.cli][INFO  ]  cluster                       : ceph4 F) N+ J" o0 J, ^
[ceph_deploy.cli][INFO  ]  mon                           : ['compute06']
: U% h* x; a2 M[ceph_deploy.cli][INFO  ]  func                          : <function mon at 0x7fe3aefb7410>
6 n  R( k. M7 B  ~; ~[ceph_deploy.cli][INFO  ]  address                       : None
3 L. |6 a2 e2 M  u, L; o# i) w[ceph_deploy.cli][INFO  ]  ceph_conf                     : None5 S, p( p) `* J6 e/ C1 U1 y% A% `
[ceph_deploy.cli][INFO  ]  default_release               : False" K: _  o! S( C" _7 k: Q$ A( M( G) m6 n
[ceph_deploy.mon][INFO  ] ensuring configuration of new mon host: compute064 `4 p$ ?0 s+ ]  V5 w; r. f3 ?' n
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to compute06/ q+ o9 \: M7 L/ x: s+ Z1 E& i
[compute06][DEBUG ] connected to host: compute06
, {: r+ I0 K- l' f" ~5 S0 G$ q" u: M% I[compute06][DEBUG ] detect platform information from remote host, a$ w! i& V6 ?7 F0 B' K
[compute06][DEBUG ] detect machine type1 X- C9 f) I6 `, v" W
[compute06][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf( O: t' e: {: Y0 j& u0 [+ {
[ceph_deploy.mon][DEBUG ] Adding mon to cluster ceph, host compute068 S9 y& R; r" v6 W
[ceph_deploy.mon][DEBUG ] using mon address by resolving host: 172.20.1.9
, M4 ?* h; e+ J5 H[ceph_deploy.mon][DEBUG ] detecting platform for host compute06 ...
& r3 _% l( E6 ]8 r[compute06][DEBUG ] connected to host: compute06
9 d/ y" j) \- q, i[compute06][DEBUG ] detect platform information from remote host. ^* h' B0 O3 z
[compute06][DEBUG ] detect machine type
; C# D$ z$ w, e  T" A1 s[compute06][DEBUG ] find the location of an executable
7 q- g- n, u. t' v[ceph_deploy.mon][INFO  ] distro info: CentOS Linux 7.9.2009 Core
( h+ B! v: T( p/ {* c5 B# S1 D' |[compute06][DEBUG ] determining if provided host has same hostname in remote
& M5 W0 u$ I5 w, q[compute06][DEBUG ] get remote short hostname0 e1 d6 \& l' @6 {3 r
[compute06][DEBUG ] adding mon to compute06
, K9 u9 R+ V8 S7 }% X4 R3 ^% L[compute06][DEBUG ] get remote short hostname
- w% l5 ^' ]5 E" \% N1 S- a" H[compute06][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf- J4 w, t8 L$ ?  W9 Q& f
[compute06][DEBUG ] create the mon path if it does not exist
) t& p* `) i2 D6 k% ~[compute06][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-compute06/done# }* Y7 r. b) q1 j
[compute06][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-compute06/done
0 x& m$ ]; W& l# C6 Y/ f' s[compute06][INFO  ] creating keyring file: /var/lib/ceph/tmp/ceph-compute06.mon.keyring
1 v) T* h; v5 \% M  g[compute06][DEBUG ] create the monitor keyring file
, `* t( ]8 W- f' D[compute06][INFO  ] Running command: ceph --cluster ceph mon getmap -o /var/lib/ceph/tmp/ceph.compute06.monmap
" y0 z% w. v, ^& P2 T( r8 J$ [4 v# d1 g[compute06][WARNIN] got monmap epoch 6- k6 r( L0 g: C& w0 g
[compute06][INFO  ] Running command: ceph-mon --cluster ceph --mkfs -i compute06 --monmap /var/lib/ceph/tmp/ceph.compute06.monmap --keyring /var/lib/ceph/tmp/ceph-compute06.mon.keyring --setuser 167 --setgroup 167
- O7 t. l6 ?' O3 j& V8 w0 S[compute06][INFO  ] unlinking keyring file /var/lib/ceph/tmp/ceph-compute06.mon.keyring0 ~- C% _" f& @; l) N2 [
[compute06][DEBUG ] create a done file to avoid re-doing the mon deployment( P, E& A) C& Q
[compute06][DEBUG ] create the init path if it does not exist* J/ r9 X+ [/ b& v
[compute06][INFO  ] Running command: systemctl enable ceph.target
- Z! w3 p. H0 A* U6 E7 G[compute06][INFO  ] Running command: systemctl enable ceph-mon@compute069 k: I! d3 q1 T) B: U. k
[compute06][INFO  ] Running command: systemctl start ceph-mon@compute06
: D7 A1 j" _+ K0 H- `[compute06][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.compute06.asok mon_status3 V8 g* K/ u: x( j7 O  y3 f
[compute06][WARNIN] monitor compute06 does not exist in monmap1 E3 l  M. x4 z/ S$ f$ H/ L# W* f
[compute06][INFO  ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.compute06.asok mon_status
3 L  d# g' U1 d[compute06][DEBUG ] ********************************************************************************
; E- _$ Z! J7 \[compute06][DEBUG ] status for monitor: mon.compute06. k1 D8 i) O6 t# r
[compute06][DEBUG ] {
2 E6 N5 f+ T& ~# c" m3 T2 ~- e2 O( D[compute06][DEBUG ]   "election_epoch": 0,
8 \. ~" X( j- Y[compute06][DEBUG ]   "extra_probe_peers": [], ; V  E0 D* j; B- b6 u; l
[compute06][DEBUG ]   "feature_map": {
3 F  c0 H& r3 E* g6 Y! E$ D[compute06][DEBUG ]     "mon": [
7 w& W+ c4 L- G4 T- x6 w+ u[compute06][DEBUG ]       {$ A0 \9 v* e7 o% t& K! o- W' x
[compute06][DEBUG ]         "features": "0x3ffddff8ffacffff",
& ~$ r: R3 K7 @  k/ f[compute06][DEBUG ]         "num": 1,
3 V& V; m& I9 F8 t4 w5 O) B9 Q7 Q[compute06][DEBUG ]         "release": "luminous"
; j& Y4 y" i6 W3 z  ?" X[compute06][DEBUG ]       }
+ _4 b  R! g) D$ N! K) X) P[compute06][DEBUG ]     ]
( w3 `) s6 M  j/ I, e[compute06][DEBUG ]   },
( c; z9 \# Y. b' O0 U[compute06][DEBUG ]   "features": {. C; j9 w0 L9 ?  P7 s
[compute06][DEBUG ]     "quorum_con": "0",
( W$ C7 e0 T1 Q[compute06][DEBUG ]     "quorum_mon": [], : g6 ]; {& {1 v9 P2 P+ r
[compute06][DEBUG ]     "required_con": "2449958197560098820", & w8 p) w/ x( J7 }( z& R5 ?
[compute06][DEBUG ]     "required_mon": [4 T" G0 S& V* M  T. a% ]2 K
[compute06][DEBUG ]       "kraken",
/ O4 F, m7 Z3 j) z4 }( Z[compute06][DEBUG ]       "luminous",
$ K) l7 m' T1 x; m6 N3 E! ]2 y) z[compute06][DEBUG ]       "mimic",
9 F: q% K0 D: x3 p9 S[compute06][DEBUG ]       "osdmap-prune",
, E" ~. g, O! Y- c) Z" k0 r[compute06][DEBUG ]       "nautilus"
8 l1 `% Q! l* N" ^) p% V[compute06][DEBUG ]     ]4 ~9 b' J; d5 W3 ~  Q9 Z
[compute06][DEBUG ]   },
9 z, c( f8 U- n( a5 @: h- `5 A[compute06][DEBUG ]   "monmap": {1 R3 t! y2 A( b- N) O* {6 V2 T" Y
[compute06][DEBUG ]     "created": "2021-10-28 14:49:15.304143", 6 w' M! B% o( n& g3 ~6 i+ O  n
[compute06][DEBUG ]     "epoch": 6, ( T1 a; l& L) ~
[compute06][DEBUG ]     "features": {
8 ~5 S. y% u4 h5 l$ \3 L[compute06][DEBUG ]       "optional": [], 7 v) @2 s4 O/ U) a( L& ~
[compute06][DEBUG ]       "persistent": [( P' D' A6 b3 \9 U/ F: O" o5 }: d: w" s( g
[compute06][DEBUG ]         "kraken", ' L) _7 u& x  A1 b7 z4 U, Z
[compute06][DEBUG ]         "luminous",
8 g  k5 }" V5 z7 n[compute06][DEBUG ]         "mimic",
7 j5 ]" K( N0 [5 d) G[compute06][DEBUG ]         "osdmap-prune", 8 A$ }. f, ^- T( o. u8 x5 H6 F
[compute06][DEBUG ]         "nautilus"
# ?& m' e" e  ]% p[compute06][DEBUG ]       ]! J/ i8 }& a7 C5 J! }$ }8 E
[compute06][DEBUG ]     }, . n+ C" b1 \$ `
[compute06][DEBUG ]     "fsid": "8daade9f-1db5-4e9e-a0ac-cbe1b3abafd1", 7 p3 w" @& D1 }% i& k  V
[compute06][DEBUG ]     "min_mon_release": 14, 8 D; ?, ]" q5 i$ u! K
[compute06][DEBUG ]     "min_mon_release_name": "nautilus",
( w1 k8 [! Q1 {' {  }3 L7 S[compute06][DEBUG ]     "modified": "2022-11-15 10:16:22.469673",
  C0 A0 ], F( M; C9 N! N[compute06][DEBUG ]     "mons": [
% Y5 Y' r- \* O- D0 @: x[compute06][DEBUG ]       {
1 q& |" K5 b3 a[compute06][DEBUG ]         "addr": "172.20.1.13:6789/0",
# C( N- }. @( t( Z# x, y8 |2 A[compute06][DEBUG ]         "name": "hostceph1",
0 `2 E. N( m$ ?' m[compute06][DEBUG ]         "public_addr": "172.20.1.13:6789/0",
" d+ Z# C3 C" V  d( \[compute06][DEBUG ]         "public_addrs": {, z. o9 ^8 u, w
[compute06][DEBUG ]           "addrvec": [( u9 n5 ^, B% F) C
[compute06][DEBUG ]             {
& i' S6 l- h+ R[compute06][DEBUG ]               "addr": "172.20.1.13:3300", & V" b8 ?- o3 T% j+ r- s7 |
[compute06][DEBUG ]               "nonce": 0,
5 v, c" b$ N, u* N/ e+ J[compute06][DEBUG ]               "type": "v2"1 f. [  W+ d: Z8 Y
[compute06][DEBUG ]             },
& R- m6 d. `+ I[compute06][DEBUG ]             {& x1 E7 N- N& s# r$ C% ]" B
[compute06][DEBUG ]               "addr": "172.20.1.13:6789", ' U9 ?5 J2 S+ h- q
[compute06][DEBUG ]               "nonce": 0, 9 O" S# S; u& t
[compute06][DEBUG ]               "type": "v1"
: W' j% [& B. a[compute06][DEBUG ]             }
+ P1 X/ s$ H: k7 v& W[compute06][DEBUG ]           ]/ f4 h# S& w4 J2 p! B6 Y
[compute06][DEBUG ]         }, ' [6 k% A: E! t- L: L
[compute06][DEBUG ]         "rank": 0% V  D  \; P3 k" t& m9 @
[compute06][DEBUG ]       }, % v3 s5 A4 \  _# S
[compute06][DEBUG ]       {8 S9 [3 j3 y4 {) r; @" u% ?  J$ v  o
[compute06][DEBUG ]         "addr": "172.20.1.14:6789/0", - u* k2 w2 \* R( F/ ?) ^
[compute06][DEBUG ]         "name": "hostceph2", 9 r+ n! i1 D4 {, z/ n! g9 f
[compute06][DEBUG ]         "public_addr": "172.20.1.14:6789/0", , |4 u0 u9 r. S2 s0 F: O3 P5 n
[compute06][DEBUG ]         "public_addrs": {
# U" C& Q; g7 @+ F6 R/ t, p[compute06][DEBUG ]           "addrvec": [
+ F3 R5 V( m6 R4 n& D[compute06][DEBUG ]             {8 s. r% c2 j5 Z% G. C# B* H- b& j
[compute06][DEBUG ]               "addr": "172.20.1.14:3300", # y2 d1 ]. B- C' z
[compute06][DEBUG ]               "nonce": 0, # P& O: S5 I+ j0 c  d3 X+ L
[compute06][DEBUG ]               "type": "v2"& [' J) n' u) t
[compute06][DEBUG ]             }, 4 p8 P& C2 L, U/ F$ P+ ?
[compute06][DEBUG ]             {6 T" I' X6 W: V$ k3 {
[compute06][DEBUG ]               "addr": "172.20.1.14:6789",
" H# E3 t( @5 k5 x! e% X[compute06][DEBUG ]               "nonce": 0,
# U. J. T% F3 ^& U[compute06][DEBUG ]               "type": "v1"
+ l+ E- f7 p1 f5 t- A# ^" n" g[compute06][DEBUG ]             }4 r1 O3 l# r* J4 a& f% `% f, g1 M% Q+ B
[compute06][DEBUG ]           ]
3 u- Y/ b9 H, E3 f[compute06][DEBUG ]         },
- z1 }, M# V/ k[compute06][DEBUG ]         "rank": 1
* ?7 X8 e7 ~6 W. ?) M6 m4 q6 b[compute06][DEBUG ]       },
  c; ~; F7 _& E7 u$ w2 N* g[compute06][DEBUG ]       {" G6 f: }  o% A* ~* ?
[compute06][DEBUG ]         "addr": "172.20.1.15:6789/0", / V7 y) K5 D# b9 }' s' X& ~: S
[compute06][DEBUG ]         "name": "hostceph3", ; T5 P1 v1 `3 \, \
[compute06][DEBUG ]         "public_addr": "172.20.1.15:6789/0",
! c) T- ^$ c% I8 q6 Z/ Q[compute06][DEBUG ]         "public_addrs": {
8 a- e5 C6 X! B5 ?1 g2 b[compute06][DEBUG ]           "addrvec": [: [" \3 D; L& v" }9 D. M
[compute06][DEBUG ]             {4 |1 o) s% s" J+ O5 O
[compute06][DEBUG ]               "addr": "172.20.1.15:3300", . L5 R! J5 ?0 s1 o( ^5 q; z  K
[compute06][DEBUG ]               "nonce": 0,
1 U5 A+ N4 C! \6 H) T) n- _6 Q[compute06][DEBUG ]               "type": "v2"
1 k; B$ O' Y& |& L* g2 V  O[compute06][DEBUG ]             }, 0 u: l- S* A6 u$ I" l* _1 p' T
[compute06][DEBUG ]             {3 ^' H5 e. {) J' ^7 k: q" i. s
[compute06][DEBUG ]               "addr": "172.20.1.15:6789", - N: u9 D! A! h5 q
[compute06][DEBUG ]               "nonce": 0, 2 h3 Z! t  G) B+ `
[compute06][DEBUG ]               "type": "v1"7 e: I! h  p8 c0 m  J+ R
[compute06][DEBUG ]             }. }/ C; _7 G2 g2 U* y1 b
[compute06][DEBUG ]           ]  }6 b4 u% }6 c; L! [4 L% [1 J
[compute06][DEBUG ]         },
/ K7 c+ a# m3 @( c- ?; ]( Q[compute06][DEBUG ]         "rank": 2$ B# n+ A' s6 M- m' C% F
[compute06][DEBUG ]       }, 8 D! O: f& P7 W. W! R
[compute06][DEBUG ]       {8 P! k1 ]% |6 ~2 Z
[compute06][DEBUG ]         "addr": "172.20.1.11:6789/0", 0 ^4 M. ~; p/ [
[compute06][DEBUG ]         "name": "compute08", * N6 x( C( h2 ?- @; C% }
[compute06][DEBUG ]         "public_addr": "172.20.1.11:6789/0",
1 a" j( c8 }! O. o. w6 y[compute06][DEBUG ]         "public_addrs": {0 k- Y8 ^' y, r! u! i, D
[compute06][DEBUG ]           "addrvec": [7 r, G, J( p8 J1 U: O4 F
[compute06][DEBUG ]             {
+ c+ S; ?+ g* Y( u8 I3 I[compute06][DEBUG ]               "addr": "172.20.1.11:3300", - x" I8 u- ?/ R5 L7 x3 C9 `
[compute06][DEBUG ]               "nonce": 0, ) G7 r9 r  k3 C. J1 i
[compute06][DEBUG ]               "type": "v2"
$ i* {( k0 i* g2 r" k# U[compute06][DEBUG ]             },
9 y4 n" q4 g9 x% l6 l" u1 X& {[compute06][DEBUG ]             {0 n$ z" A) d5 }% E: {
[compute06][DEBUG ]               "addr": "172.20.1.11:6789", ( R7 q1 u0 g; C7 V" }" w
[compute06][DEBUG ]               "nonce": 0,
# l# G  w: t* c' S[compute06][DEBUG ]               "type": "v1"( V& G* @. d+ ?( R0 ~
[compute06][DEBUG ]             }
) y: w2 V- z/ ?[compute06][DEBUG ]           ]6 I0 R2 }: z7 {9 T( l: m
[compute06][DEBUG ]         }, 8 V  P; w4 v% A9 q' x
[compute06][DEBUG ]         "rank": 3
2 D" N5 y& z" H  t2 X1 @[compute06][DEBUG ]       }7 n+ k/ T- l' Q4 ~4 c" N7 P$ \% P7 ^
[compute06][DEBUG ]     ]. [! t$ @, \' P0 \  H6 j6 w
[compute06][DEBUG ]   }, 6 i1 F; d) m9 J) B
[compute06][DEBUG ]   "name": "compute06",
1 m, O; }  e1 Q6 R[compute06][DEBUG ]   "outside_quorum": [],
! S, @8 _) ]$ k6 M[compute06][DEBUG ]   "quorum": [], 2 o) Y; Z+ _8 R2 H4 t0 t
[compute06][DEBUG ]   "rank": -1,
+ m+ y6 R& u* w2 o( @[compute06][DEBUG ]   "state": "probing",
6 a: H/ i2 ]1 |3 a  Y: v( ^9 w[compute06][DEBUG ]   "sync_provider": []
, w" S. C7 `6 e: J[compute06][DEBUG ] }5 h; `6 l; U% @2 c1 T+ S2 |) J- V
[compute06][DEBUG ] ********************************************************************************
8 I: Z% z4 k0 _1 q( T4 x( V[compute06][INFO  ] monitor: mon.compute06 is currently at the state of probing& d; l5 q1 J  d* X" Y* p& g
[root@hostceph1 deploy]# ceph -s+ M5 J* [! o6 ]' q
  cluster:, w/ f& o, T- Z5 s# {9 l
    id:     8daade9f-1db5-4e9e-a0ac-cbe1b3abafd1
- O- z# K0 p( [    health: HEALTH_WARN7 A: D  g$ O: c% w
            15 daemons have recently crashed
( Q/ m* c( O" b- s ; g9 |5 j/ [/ t3 _6 @
  services:
6 ]$ C  k. s) @0 L    mon: 5 daemons, quorum hostceph1,hostceph2,hostceph3,compute08,compute06 (age 0.657523s)+ w3 T9 ?$ W* f
    mgr: hostceph2(active, since 5M), standbys: hostceph3, hostceph1, compute07, compute09, compute08, compute06
8 e( c5 s: E% F    osd: 34 osds: 34 up (since 3M), 34 in (since 5M)1 O1 m7 n; l/ z; L3 U" u
# U* q  g0 L. K
  data:; K- c9 A6 [8 _/ Z0 g
    pools:   9 pools, 8832 pgs
) J& u4 H2 l$ L9 @( Z5 S    objects: 312.99k objects, 1.2 TiB
. w& `% T/ l4 T7 z( t4 t, _' [    usage:   8.3 TiB used, 118 TiB / 127 TiB avail
" v4 ]/ Y; B7 Z    pgs:     8832 active+clean+ C1 {& ~* @5 \4 q. P1 M
" P/ \/ c: W+ y3 J
  io:8 [# T: ~  i& z$ H$ z) P7 N
    client:   27 KiB/s rd, 64 KiB/s wr, 13 op/s rd, 6 op/s wr
; S" m6 S( k! E3 A + b! {3 \" i" s  u8 ^  w

. Z" N8 ~/ [: k9 K/ Zmon的告警解除了。遇到了新的问题。
您需要登录后才可以回帖 登录 | 注册

本版积分规则

返回首页|Archiver|手机版|小黑屋|易陆发现技术论坛 ( 蜀ICP备2026014127号-1 )

GMT+8, 2026-6-12 00:05 , Processed in 0.017541 second(s), 21 queries .

Powered by Discuz! X5.0

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表