|
|
[root@controller1 ~]# ceph health detail 3 f& {) E" ~, F- { t7 d
HEALTH_WARN 1 daemons have recently crashed! s8 ^4 x( r- ?, p4 O# j* v; i
RECENT_CRASH 1 daemons have recently crashed
! L$ o# @- q( u' d' K/ s osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z5 ^; s' D$ q8 w- Q# `7 n( {
+ |+ b% c, l0 j7 V* G1 y' ~% H6 I! N/ ~# R, X5 E
[root@controller1 ~]# ceph health detail # j% u7 q5 k9 n% I: n/ E
HEALTH_WARN 1 daemons have recently crashed+ E- T6 O& M( Z/ d8 g; K/ o
RECENT_CRASH 1 daemons have recently crashed# j/ f0 r* s$ r9 f6 g& Q
osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z( {& e- M/ e# r, A* T
[root@controller1 ~]# ceph crash ls
9 L, N/ ^3 c; i) JID ENTITY NEW ; u: }* i5 z- |% P! O1 g% L
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 *
) \& D2 v" E* S6 `, Z[root@controller1 ~]# ceph crash info 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
- Y; r# i) O: P( L0 Q{5 I8 f+ b- o8 K. w" L4 ^. q2 N
"os_version_id": "7", 2 {' g) p! z6 u j+ Z. y
"assert_condition": "r >= 0 && r <= (int)tail_read", % {3 ^8 y8 G3 S0 [9 G( p3 ^
"utsname_release": "3.10.0-1160.el7.x86_64", : j& z i' c, @9 K! t S& `) e
"os_name": "CentOS Linux",
- ?0 U5 s$ @7 _( N "entity_name": "osd.7",
/ q( @, d. h' T8 p* m5 Y7 ? "assert_file": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc",
* H) {# q# O9 Q( |8 @2 ] "timestamp": "2022-11-17 08:45:41.660614Z",
! L3 _1 r0 f+ \( W5 t "process_name": "ceph-osd",
3 \( t- C0 B$ a% O# P3 E w% V9 ` "utsname_machine": "x86_64", . G4 W. ?) R: t& c4 b' C+ Y# Y
"assert_line": 12188,
+ t* o% h% P6 k3 N "utsname_sysname": "Linux",
9 |; x0 I' ~7 M) ^/ w "os_version": "7 (Core)", ( M$ r/ d; w1 F" s
"os_id": "centos", W; b8 i7 | |) A8 l+ [" m& }
"assert_thread_name": "tp_osd_tp",
6 z( s: J+ Y, V8 `* R7 |8 u "utsname_version": "#1 SMP Wed Nov 18 03:43:48 UTC 2020", ( m& G: s9 j; X( l
"backtrace": [
, y8 x9 t: ~6 d9 n "(()+0xf630) [0x7f553054b630]", 5 m8 {; x1 k5 Q
"(gsignal()+0x37) [0x7f552f33e3d7]",
0 u2 r2 R' V0 |5 d0 x5 H "(abort()+0x148) [0x7f552f33fac8]",
4 D, z m% {1 T8 D# | "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x199) [0x564513118704]", : Q5 i0 y0 L1 F1 |( e) a
"(()+0x4cc87d) [0x56451311887d]",
/ U- H, i9 p0 }; p7 Z! n) Q "(BlueStore::_do_write_small(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)+0x2fc7) [0x564513663057]", ) [2 D2 J" j! e" l( O# M" T) T V8 L
"(BlueStore::_do_write_data(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, BlueStore::WriteContext*)+0x22a) [0x56451366420a]",
( G& ^& f( B/ n, {. O "(BlueStore::_do_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0x54c) [0x56451366b3dc]",
, `2 h/ A" p3 U! h2 A3 S- U "(BlueStore::_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0xcb) [0x56451366c06b]",
5 s2 u, ]7 o: O6 y. q "(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x152c) [0x56451366f46c]",
9 T) {- B v4 p "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x370) [0x564513683910]", T8 L+ _2 Z+ t0 f: [; O
"(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x54) [0x5645133f1794]",
- y5 |3 \$ s! ]3 A" I( O "(ReplicatedBackend::do_repop(boost::intrusive_ptr<OpRequest>)+0xc4b) [0x5645134e894b]",
* \ d/ [. z" ~8 ? "(ReplicatedBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x257) [0x5645134f6f57]", 1 `( w' c; K `9 b6 |8 G& X1 ^
"(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x4a) [0x564513407dea]",
+ j( H$ E/ U7 h3 v: w "(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x5b3) [0x5645133b81d3]",
3 p: e2 s6 g9 O) P "(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x362) [0x5645131f8b62]", 3 h' |: t# l1 V Z
"(PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x564513487752]", ~. |9 m+ X& s$ [% \/ e) a# S
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x564513213b5f]", 2 Z. s' t1 \3 e% Z6 e* L. ]; |! x
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5645137b7dd6]",
7 W2 c p: \; s9 T3 E. Q "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5645137ba8f0]", y8 d/ K+ o5 Y. J
"(()+0x7ea5) [0x7f5530543ea5]",
; J. X+ C+ I+ n* r3 L5 o0 H+ t "(clone()+0x6d) [0x7f552f4069fd]"
# F/ W$ w" L$ G2 c, X( @2 c ],
) M7 k% Z" T0 e. F3 C# v+ q* y "utsname_hostname": "host10", $ h/ E9 J! O' @8 _
"assert_msg": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)' thread 7f550e34f700 time 2022-11-17 16:45:41.630549\n/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: 12188: FAILED ceph_assert(r >= 0 && r <= (int)tail_read)\n", ( j/ A/ n+ |" z9 [( O" Q( V
"crash_id": "2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074", 7 J# L" g4 X- S5 j3 \* X: @6 L
"assert_func": "void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)",
, O8 q( ^7 C7 Z S; D1 X "ceph_version": "14.2.8-111.el7"
2 T! K1 l+ N. v! r2 X}) d8 @+ {0 @9 V2 w
[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
5 Q3 ^. q. M2 A5 Z6 G# X9 D* W. [[root@controller1 ~]# ceph crash ls
% n+ g- D8 P7 {3 s; t0 ?: ]+ CID ENTITY NEW
3 u- n* {9 x L i( x8 D9 Q2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 7 c: T$ G# p3 U# ]% f4 S
[root@controller1 ~]# ceph crash archive-all
" P: ^/ g6 z6 S4 u! E[root@controller1 ~]# ceph crash ls ; B+ }% y. t( F, R( ?3 z
ID ENTITY NEW
u$ E0 k, W1 M* j" U, ]! y. z2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7 - r: U" L- X4 W! k3 m
[root@controller1 ~]# ceph crash ls-new% {) h5 w- l$ \( l6 a
[root@controller1 ~]# ceph crash ls 7 c6 v& d5 q8 P1 d! }+ h/ _
ID ENTITY NEW + ^2 ]) }7 |& T( U
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
: ?7 ?$ i9 {5 P c[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c0743 h' n1 c' h% l1 I3 n' b, Q
[root@controller1 ~]# ceph crash ls ; f8 _8 k. j7 K
ID ENTITY NEW 7 I/ N- u3 [/ Y$ P8 t
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7
' a2 ~+ ?2 b3 _7 @[root@controller1 ~]# ceph crash ls-new" p. i3 t" ?0 |8 x
[root@controller1 ~]# ceph crash archive-all 8 P! G) t2 S. B* o3 |2 P6 l
[root@controller1 ~]# ceph crash archive-all + ]' p! a/ E$ h# D8 K
[root@controller1 ~]# ceph -s
% V+ G1 `3 ?! Y4 M7 Y cluster:$ Z" c5 s# q! x! o+ z, G
id: 9d22e36a-2bdd-4d2d-8394-48af75ead777
& `; L- E, Q, @ health: HEALTH_OK
% J7 Q3 P: H; T/ D' @- l' b% G' C1 f! p
services:! q' U6 I# f) u. ?0 u7 l9 i
mon: 3 daemons, quorum host09,host10,host11 (age 4M)
% e0 X" z; N8 z* S; H4 [1 `% Q mgr: host09(active, since 18M), standbys: host10, host11
& I* l; L0 \6 R) D6 Q osd: 40 osds: 40 up (since 10d), 40 in (since 11M), j) L4 `7 @* e2 y. }
rgw: 3 daemons active (host09, host10, host11)
5 a, {5 n- c) U7 M% S9 C" I! I* K- f2 v% o
task status:& p# {) a$ S) c
) E, v! r5 r/ a% s
data:
$ Q6 W6 ]) t7 a z! V6 d0 V pools: 16 pools, 3072 pgs" `: N* E M6 m2 X+ a0 d2 I
objects: 4.07M objects, 16 TiB6 C! h2 G G- m* p* l
usage: 39 TiB used, 108 TiB / 148 TiB avail
% F' t; T7 g! m' W3 p7 ]) r5 z pgs: 3069 active+clean
" }7 k. a8 F( ~# b 3 active+clean+scrubbing+deep& U# l0 b: }9 a o
$ P: P; V! H) U6 M# E
io:9 R/ A) U! Y' k2 h
client: 1.0 MiB/s rd, 21 MiB/s wr, 648 op/s rd, 421 op/s wr w+ u1 `2 s: ^" w# V/ I9 Z2 p
6 N; S: _. }# j; O8 Z
[root@controller1 ~]# ceph config set mgr mgr/crash/warn_recent_interval 0; O; `: \$ g; S7 u$ C/ M
[root@controller1 ~]#
9 s' U" w" s5 v i' s( F
. A4 P8 Y0 I8 `* T8 s% f
3 E7 K8 `9 e' c6 \- @% y% B9 }
" j1 A% T: @ W" h! M! }$ }http://wwww.bbs168x.cn/forum.php ... Brecently%2Bcrashed- @ H5 b" L! h. P' t
( X4 q: N( M5 s$ h
|
|
发表于 2022-11-28 10:16:47