|
|
[root@controller1 ~]# ceph health detail " h4 d! o4 k1 m" b
HEALTH_WARN 1 daemons have recently crashed, v0 m1 B+ H2 g
RECENT_CRASH 1 daemons have recently crashed
4 G' h6 T1 H2 Y' ? osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z7 B/ G+ d. q" u! m( t
* W9 n g! Y1 G5 G8 w% E* E7 O
: M+ f i! w5 M2 |8 J[root@controller1 ~]# ceph health detail . |5 _/ t# |0 X5 `
HEALTH_WARN 1 daemons have recently crashed
# l b: O( B1 ?7 X2 dRECENT_CRASH 1 daemons have recently crashed% \% T- Z: d" Q3 o& L5 n- f' S( P8 X
osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z
8 V+ t1 I" f) k7 U f3 S[root@controller1 ~]# ceph crash ls
8 D" k3 z* w0 g* F6 lID ENTITY NEW ! p3 y4 X, m9 a9 [
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 *
# D& q, \' Y6 x1 {% N[root@controller1 ~]# ceph crash info 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
) c6 \% J; L1 \# |{
& u! [+ \$ F1 a7 P/ l I8 _ "os_version_id": "7", & j2 o, r8 k- M
"assert_condition": "r >= 0 && r <= (int)tail_read",
4 e" P; Y- |& W6 k( O1 C5 U "utsname_release": "3.10.0-1160.el7.x86_64",
9 U6 ~; |6 B. |5 q "os_name": "CentOS Linux", 0 W+ L+ }, @5 k
"entity_name": "osd.7", $ @/ z5 Q. l8 v
"assert_file": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc", 3 m3 l; L2 g3 [2 r5 ]% l
"timestamp": "2022-11-17 08:45:41.660614Z",
5 X. B9 d2 ? ]3 A, M( l "process_name": "ceph-osd", + V7 _) K8 M" h. b
"utsname_machine": "x86_64", 4 ^' U1 A: j7 T5 L8 n |* S
"assert_line": 12188,
! L& Z+ [# ^/ p/ \ "utsname_sysname": "Linux",
b. z$ a/ V6 `4 w) e/ @ "os_version": "7 (Core)", + Y4 T. w8 J: q7 e8 m
"os_id": "centos", / L( E5 p1 z* ^' i( P
"assert_thread_name": "tp_osd_tp", / R' d* Y- T/ ?( x4 Y! x% ?
"utsname_version": "#1 SMP Wed Nov 18 03:43:48 UTC 2020", 1 I* @8 O. ]6 u+ h# w( M
"backtrace": [
! i% J2 ?2 q" O) h% X& L "(()+0xf630) [0x7f553054b630]",
$ e+ a9 X$ `8 a$ e# k o "(gsignal()+0x37) [0x7f552f33e3d7]", , P; r6 K& V9 ]+ w/ l, \
"(abort()+0x148) [0x7f552f33fac8]", D" o6 a: D, x; P
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x199) [0x564513118704]", 4 W% U0 [1 M7 ^; A A
"(()+0x4cc87d) [0x56451311887d]", : d/ I$ J& H! o' m% S% \) z; T
"(BlueStore::_do_write_small(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)+0x2fc7) [0x564513663057]",
: O! @* N2 t f1 |5 O "(BlueStore::_do_write_data(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, BlueStore::WriteContext*)+0x22a) [0x56451366420a]",
+ v- C0 Q" k4 F "(BlueStore::_do_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0x54c) [0x56451366b3dc]",
) o/ d! b- M* n. a$ F$ b1 Q "(BlueStore::_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0xcb) [0x56451366c06b]", $ t; I( J' S8 H- v# w B! A
"(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x152c) [0x56451366f46c]",
4 b3 m. \8 l0 M# Q' G4 H3 u "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x370) [0x564513683910]", p! y* u6 z: g% s$ r; F
"(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x54) [0x5645133f1794]",
. b# c( U* u0 G "(ReplicatedBackend::do_repop(boost::intrusive_ptr<OpRequest>)+0xc4b) [0x5645134e894b]", * d/ E: t6 x' ?1 [. A& ~
"(ReplicatedBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x257) [0x5645134f6f57]",
* p! o8 o" L3 E "(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x4a) [0x564513407dea]", % J2 ~, C* S9 W# \9 L" b" w. U+ i6 p6 ?6 Q
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x5b3) [0x5645133b81d3]",
+ S, d- ]( @# r6 E; D; f "(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x362) [0x5645131f8b62]", ! @3 ^+ G0 K- z3 I7 N$ g
"(PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x564513487752]", 5 R( a/ d3 n7 S/ E1 _
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x564513213b5f]",
s! {7 [( O9 Z( G" m% Q" q "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5645137b7dd6]",
- W$ L2 N K4 h1 f "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5645137ba8f0]",
" t5 n6 a0 t& f% ?1 | "(()+0x7ea5) [0x7f5530543ea5]",
; |$ I8 a2 O" `& v' e w e$ E8 v "(clone()+0x6d) [0x7f552f4069fd]"
" t6 c: p ^. G3 O2 r ], - {0 _& V1 |: J# w4 T- F
"utsname_hostname": "host10", 8 X9 ^( T; E8 L! Y/ ~1 H
"assert_msg": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)' thread 7f550e34f700 time 2022-11-17 16:45:41.630549\n/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: 12188: FAILED ceph_assert(r >= 0 && r <= (int)tail_read)\n", / u2 S% S1 j4 T ]/ d; U1 \) C
"crash_id": "2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074",
) T- X( p, i) @" a0 V: d" {1 Y "assert_func": "void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)", - \6 o4 w3 U3 H0 v* q5 y
"ceph_version": "14.2.8-111.el7"! P9 y3 }1 Q% i( T& Q
} f) y) j: i* d( L
[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074: r: F7 x; Q9 K# }0 l4 P
[root@controller1 ~]# ceph crash ls 7 @, A3 q! q/ v& v0 v
ID ENTITY NEW D% g2 D& O. i/ A' p- `9 X
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
7 {; l* |* c% c0 A[root@controller1 ~]# ceph crash archive-all 2 f& R, X5 ]7 T# y
[root@controller1 ~]# ceph crash ls
* j9 B9 i$ y! z. b* y' E6 uID ENTITY NEW
- X; h, }0 F* c7 W7 L" s6 ]2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
$ {. }( f, d* q' X2 G[root@controller1 ~]# ceph crash ls-new' i/ @8 E6 O) e% ]/ Q0 A u/ v
[root@controller1 ~]# ceph crash ls
# l& s+ j; z: c# W' i, a; `3 t' p. LID ENTITY NEW
' `; a- U" t. A4 _8 O2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 . X& \" \/ B: {/ _
[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
% b( V* U b0 w! y' C5 L2 y[root@controller1 ~]# ceph crash ls
7 O t! F$ x; L5 a5 M' }ID ENTITY NEW
2 R' Z3 G' c0 k0 z# s( H2 ~2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
9 s5 H+ w2 e: K[root@controller1 ~]# ceph crash ls-new8 z* K, o8 i( w4 x4 m3 ~
[root@controller1 ~]# ceph crash archive-all
% b0 \5 I7 s0 }- _8 a[root@controller1 ~]# ceph crash archive-all / y2 ~( U& S3 T9 q8 u
[root@controller1 ~]# ceph -s5 t1 d8 n }7 f% h; H. g4 x
cluster:
$ w( y5 E# e( y, o id: 9d22e36a-2bdd-4d2d-8394-48af75ead777+ s& Y- k) W* Z% V* m+ Y
health: HEALTH_OK0 I' z. k! }8 `' m
. @# k/ g" \$ \/ o+ L0 {
services:
+ G) [) T# _& j mon: 3 daemons, quorum host09,host10,host11 (age 4M)
, [, ?9 V$ L* X1 { mgr: host09(active, since 18M), standbys: host10, host11! G* ?% f4 E% D1 O0 v
osd: 40 osds: 40 up (since 10d), 40 in (since 11M)
& M. B w' y/ o+ N( f- A9 e$ o rgw: 3 daemons active (host09, host10, host11)
, m# A; u* x- J6 V3 {- A( J, D2 r- i8 l- {
task status:
3 v; o4 F# T/ D& J2 D# c. k* V7 A( s% @/ \7 n. k
data:
& X F% ~% q! m pools: 16 pools, 3072 pgs4 d& ?0 w+ ]. W& W% U
objects: 4.07M objects, 16 TiB
* f0 o) c/ d! }. Y" }" { usage: 39 TiB used, 108 TiB / 148 TiB avail
5 Z' L5 b1 k' u5 P pgs: 3069 active+clean7 o& Y7 J0 n+ Q2 }. K; y4 s
3 active+clean+scrubbing+deep, n! E9 g# p# C4 L4 x
2 E. R. ?, ]0 C2 f% l! G io:5 d$ Q7 e( N! S. L- A
client: 1.0 MiB/s rd, 21 MiB/s wr, 648 op/s rd, 421 op/s wr2 q* i3 K# ]7 b5 C* n4 {
+ [: O8 Q9 r. H* F3 s; v5 K
[root@controller1 ~]# ceph config set mgr mgr/crash/warn_recent_interval 0+ i9 R' m* A& I& |1 E' P5 \7 Z
[root@controller1 ~]#
6 G* m$ P4 [1 z/ w1 U
+ L+ O* O5 k E1 s9 X% _9 z" ~" E* V; [9 m. g/ M4 B
" Q1 Y- P( G' D8 k# e2 b
http://wwww.bbs168x.cn/forum.php ... Brecently%2Bcrashed' W7 _2 N/ s, X( P. J
- D* X- F* h/ c2 h# B
|
|
发表于 2022-11-28 10:16:47