|
|
[root@controller1 ~]# ceph health detail 6 G1 k( R; i- n: k9 ?% q
HEALTH_WARN 1 daemons have recently crashed
8 ^3 Q# O% Q3 B# C' n7 URECENT_CRASH 1 daemons have recently crashed
1 {) r, n6 K: O5 X6 M4 g osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z
" h2 U9 W6 z$ r3 M% ~' ~* z
}* _4 _1 n3 R9 s: U6 n# [/ V$ Y `
4 \: m' o2 H0 S N/ B( w6 u+ K[root@controller1 ~]# ceph health detail
: l3 V, }1 N- R$ ZHEALTH_WARN 1 daemons have recently crashed& V* P6 Y9 U$ `1 u. p
RECENT_CRASH 1 daemons have recently crashed) C4 A- z5 }7 {7 o# b+ r
osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z1 W/ w3 `+ q1 a1 T/ l, n
[root@controller1 ~]# ceph crash ls
1 K" ^3 v6 S* A4 p3 F( c$ q1 uID ENTITY NEW , D8 d( _3 [5 L. l3 R1 J
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 * 2 u. {- q6 A4 W+ b# }1 Z: {5 \+ [
[root@controller1 ~]# ceph crash info 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
# i. ?; s1 Z. K# ]{8 W2 C$ L. S2 o6 H
"os_version_id": "7", $ o1 O) E3 k/ W! z0 Q
"assert_condition": "r >= 0 && r <= (int)tail_read",
. V' ^7 t8 h' h2 [5 O5 i* F "utsname_release": "3.10.0-1160.el7.x86_64", , R! R* L2 W B. F- S! A1 Q1 `
"os_name": "CentOS Linux", * T3 d% s8 f+ V4 ~4 D2 |! a7 K
"entity_name": "osd.7", ' w3 \6 M; o6 [2 y) o
"assert_file": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc",
7 y! y" m0 q2 ~( q "timestamp": "2022-11-17 08:45:41.660614Z", & ]/ p; z! }5 t! T! M7 [
"process_name": "ceph-osd", $ d& w& F2 c0 L: H# {) N
"utsname_machine": "x86_64", 6 Q( }( Z9 s* l4 Q8 P! H
"assert_line": 12188,
D, A& V( @& S1 X6 d "utsname_sysname": "Linux",
: i3 h. r/ W' c: F! O2 i "os_version": "7 (Core)",
- S) h) f! C4 G6 n* O) y0 f. k' V6 n "os_id": "centos", 1 Y2 I1 z% n+ Z' l9 u+ `
"assert_thread_name": "tp_osd_tp",
7 X- d! w3 J, k { "utsname_version": "#1 SMP Wed Nov 18 03:43:48 UTC 2020", # z P. R# ]. V: l8 w3 v. Z5 t7 U
"backtrace": [
. ~" A. d9 W2 S7 F "(()+0xf630) [0x7f553054b630]",
( T N( b l# ^/ c" Z "(gsignal()+0x37) [0x7f552f33e3d7]",
. G" `8 A2 X! P0 S+ w "(abort()+0x148) [0x7f552f33fac8]",
1 d! F1 X T C* z5 ~* w, N I "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x199) [0x564513118704]", 6 [7 D/ [8 h8 i: n# q9 j
"(()+0x4cc87d) [0x56451311887d]", & {; E, e( R, @' z
"(BlueStore::_do_write_small(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)+0x2fc7) [0x564513663057]",
* ?1 @( r4 {5 ~* ?9 s9 R4 ^5 b "(BlueStore::_do_write_data(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, BlueStore::WriteContext*)+0x22a) [0x56451366420a]",
' x" x& T! ^" O0 ` "(BlueStore::_do_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0x54c) [0x56451366b3dc]", " ~7 g) }5 d: ?: O6 D8 | I0 J$ ~
"(BlueStore::_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0xcb) [0x56451366c06b]",
+ C8 j1 d% R0 q E0 u0 ` "(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x152c) [0x56451366f46c]",
3 X1 P% r/ Q9 U5 X) q# s5 d/ D# c "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x370) [0x564513683910]", 8 v6 L! ?+ |& q, O z w
"(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x54) [0x5645133f1794]",
h7 x# n q$ t8 W% D% Y; ] "(ReplicatedBackend::do_repop(boost::intrusive_ptr<OpRequest>)+0xc4b) [0x5645134e894b]",
/ |4 ^/ T9 R; ^# d; d0 i "(ReplicatedBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x257) [0x5645134f6f57]", ! E; k5 x0 ~6 v* j1 b, C0 _
"(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x4a) [0x564513407dea]", & I7 d. P$ b1 u- R4 ^' v* l6 |* Y
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x5b3) [0x5645133b81d3]", 2 Z: d2 c6 m! D# C- U; n' L
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x362) [0x5645131f8b62]",
/ S1 `" b' G1 I" L7 ~ "(PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x564513487752]",
# V0 l7 b! O+ H "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x564513213b5f]",
4 x* p9 D& y, ~1 i "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5645137b7dd6]", D; u( `4 u+ {" g9 g7 x8 ]" M
"(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5645137ba8f0]", & q; s. G& S; F' E
"(()+0x7ea5) [0x7f5530543ea5]", " N+ B+ y, G: g+ x" B6 k
"(clone()+0x6d) [0x7f552f4069fd]"
Q# b3 c: B& _ ],
8 H1 G# Z2 l: q "utsname_hostname": "host10",
g& ]; G# g' n/ e/ m. z* M "assert_msg": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)' thread 7f550e34f700 time 2022-11-17 16:45:41.630549\n/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: 12188: FAILED ceph_assert(r >= 0 && r <= (int)tail_read)\n",
$ Y# Q. I7 d6 Y9 j: O9 _% ]4 w "crash_id": "2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074",
2 E$ Q6 e. @+ P6 K "assert_func": "void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)",
6 H; u6 v1 _8 }/ y+ R! I8 B "ceph_version": "14.2.8-111.el7"/ K3 x+ s9 _ _7 f& J/ l& T
}6 y' v" q& x$ ?) _ Y9 M
[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074: h3 Y+ O3 R) J: h4 }
[root@controller1 ~]# ceph crash ls " M0 D& _6 I6 h: `. }
ID ENTITY NEW 2 T9 ]& Q' s& Q+ Z! n$ p5 N5 b$ h
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 . G. D, ^* h5 M' L: |% v
[root@controller1 ~]# ceph crash archive-all " x$ Z. R6 `9 Q2 e$ b8 L5 B+ s j8 J
[root@controller1 ~]# ceph crash ls
+ r6 Q1 j' `0 w6 P" yID ENTITY NEW - v/ B0 @3 d! ]" x8 J
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
( N: a, C6 T6 z[root@controller1 ~]# ceph crash ls-new' C6 g) U3 |! `% H% p( P& E8 t* a6 b3 v
[root@controller1 ~]# ceph crash ls 7 \ b* n4 A! F5 u# i5 q# u0 i
ID ENTITY NEW
/ d, G" n* X3 N7 K% V2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 # A7 a3 ^$ n8 Z4 w. y* T
[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
& d3 B7 l& \7 ~& p' z* t4 S[root@controller1 ~]# ceph crash ls
/ E9 D& z/ B$ Q& dID ENTITY NEW $ a9 c( n- y' k1 a8 q0 n. Q' _
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
/ Z) `! x6 q A' d% b[root@controller1 ~]# ceph crash ls-new
3 a a' D/ z# y2 ~[root@controller1 ~]# ceph crash archive-all
* c; e0 L- J. U0 S[root@controller1 ~]# ceph crash archive-all 7 k X6 \* Z; h7 G0 K& _
[root@controller1 ~]# ceph -s+ m4 W' H& `0 W, V1 z
cluster:& \9 x% k# Y0 ^; x+ q* v& N# E* b
id: 9d22e36a-2bdd-4d2d-8394-48af75ead777
- E9 p" U" o- D' @5 X D health: HEALTH_OK5 K/ b( V) E1 c" `; b
' j, ]) K7 z- L: @
services:
1 w+ m2 |2 }" g0 p+ i6 ^ mon: 3 daemons, quorum host09,host10,host11 (age 4M)7 R3 p5 c( m5 f# _8 q
mgr: host09(active, since 18M), standbys: host10, host11
; q6 S/ p! s& W+ w- H6 h osd: 40 osds: 40 up (since 10d), 40 in (since 11M)% ?* d: `, R) m" ~9 j
rgw: 3 daemons active (host09, host10, host11)
: g" |( \+ j7 D! W7 @& p% P$ w7 b
' a7 W! O' b0 ^, C: v task status:) w& m; ]4 r9 q+ f
3 S7 o3 f, |' C! o0 o6 i data:
3 C) ?% q6 d8 I" X I pools: 16 pools, 3072 pgs9 W! {* k/ R/ e3 {1 N4 ^4 z4 p1 s5 U
objects: 4.07M objects, 16 TiB
x# |0 I8 O0 W5 _$ ^- g( G- O2 S* u usage: 39 TiB used, 108 TiB / 148 TiB avail7 {( k, l% m6 c
pgs: 3069 active+clean3 k9 _8 I2 [: H; e! P
3 active+clean+scrubbing+deep, R$ K0 l$ S8 f1 `( Y: j8 C: S
! r- S1 Q0 p1 S io:7 ~ R% `( A) N. `7 |
client: 1.0 MiB/s rd, 21 MiB/s wr, 648 op/s rd, 421 op/s wr
7 |% O! T2 p, j
" i% f. R, i4 ?. F$ _. {[root@controller1 ~]# ceph config set mgr mgr/crash/warn_recent_interval 0
) z* c0 i6 h1 g: N& L/ ?# H* Y1 ?[root@controller1 ~]#
+ i# n& j7 _& X/ D+ h( b, L: B) ~0 l& q- Q$ P T
! ^% F( s( w9 z6 o& L' X* j4 i5 |- `+ F5 y1 t
http://wwww.bbs168x.cn/forum.php ... Brecently%2Bcrashed
1 U0 E$ m1 W/ N' _3 u4 j4 O8 B
# |* y" ]6 r! ]- W. h |
|
发表于 2022-11-28 10:16:47