HEALTH_WARN 1 daemons have recently crashed

admin

[root@controller1 ~]# ceph health detail
* u( k: \8 f& eHEALTH_WARN 1 daemons have recently crashed
; \8 G5 F) ~2 H* u3 B  B% y6 |RECENT_CRASH 1 daemons have recently crashed
    osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z
4 e0 t, A/ h3 H7 f- N( @

[root@controller1 ~]# ceph health detail
HEALTH_WARN 1 daemons have recently crashed
RECENT_CRASH 1 daemons have recently crashed
    osd.7 crashed on host host10 at 2022-11-17 08:45:41.660614Z
[root@controller1 ~]# ceph crash ls
+ G7 V" h/ H- [  J+ U+ qID                                                               ENTITY NEW
! o  l7 K; e; w/ F" C2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7   *  
[root@controller1 ~]# ceph crash info 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
( V  N6 n, v+ J- B, {& N7 E{
    "os_version_id": "7",
4 a# b  k: E6 Y: G* ?  j" K    "assert_condition": "r >= 0 && r <= (int)tail_read",
    "utsname_release": "3.10.0-1160.el7.x86_64",
    "os_name": "CentOS Linux",
    "entity_name": "osd.7",
; L% `, D  N* ]    "assert_file": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc",
# Y- X+ D3 K: [" |, A5 d    "timestamp": "2022-11-17 08:45:41.660614Z",
! d  i2 y' v3 U    "process_name": "ceph-osd",
    "utsname_machine": "x86_64",
    "assert_line": 12188,
5 b; }- ?/ N7 q+ g    "utsname_sysname": "Linux",
    "os_version": "7 (Core)",
    "os_id": "centos",
    "assert_thread_name": "tp_osd_tp",
    "utsname_version": "#1 SMP Wed Nov 18 03:43:48 UTC 2020",
6 j! b% W4 S8 \1 i    "backtrace": [
        "(()+0xf630) [0x7f553054b630]",
0 p& [! ?' d! J7 }, e( S        "(gsignal()+0x37) [0x7f552f33e3d7]",
        "(abort()+0x148) [0x7f552f33fac8]",
' Y2 [: R" F" ?  h        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x199) [0x564513118704]",
        "(()+0x4cc87d) [0x56451311887d]",
6 S3 B$ A( O  {$ l" ?        "(BlueStore::_do_write_small(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)+0x2fc7) [0x564513663057]",
2 ?7 U+ Q7 @" y/ _; I, |9 B6 @        "(BlueStore::_do_write_data(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, BlueStore::WriteContext*)+0x22a) [0x56451366420a]",
3 w" [' Q# f# \        "(BlueStore::_do_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0x54c) [0x56451366b3dc]",
- t/ \6 N* z9 S* x% z5 P        "(BlueStore::_write(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int)+0xcb) [0x56451366c06b]",
4 L& E0 t' y5 ^! l. K: }        "(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x152c) [0x56451366f46c]",
  Q8 x1 {) I$ u8 r        "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x370) [0x564513683910]",
        "(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x54) [0x5645133f1794]",
. |9 y2 l1 T. O7 H/ `        "(ReplicatedBackend::do_repop(boost::intrusive_ptr<OpRequest>)+0xc4b) [0x5645134e894b]",
        "(ReplicatedBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x257) [0x5645134f6f57]",
        "(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x4a) [0x564513407dea]",
" |1 p5 H$ R- W& Q2 g        "(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x5b3) [0x5645133b81d3]",
        "(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x362) [0x5645131f8b62]",
        "(PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x564513487752]",
& r  {: C  s$ @; v7 l3 c! S' L% Q3 N        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x90f) [0x564513213b5f]",
/ C$ O0 L# R; a) }# T        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b6) [0x5645137b7dd6]",
& v' A: m" S3 B        "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5645137ba8f0]",
        "(()+0x7ea5) [0x7f5530543ea5]",
7 d; Q1 m+ p" ?) J% ^# q# a        "(clone()+0x6d) [0x7f552f4069fd]"
    ],
: N: U# M  u% K: i    "utsname_hostname": "host10",
3 D: |6 a6 B0 m& `% C& F9 Q1 P    "assert_msg": "/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)' thread 7f550e34f700 time 2022-11-17 16:45:41.630549\n/home/miles/rpmbuild/BUILD/ceph-14.2.8/src/os/bluestore/BlueStore.cc: 12188: FAILED ceph_assert(r >= 0 && r <= (int)tail_read)\n",
$ ]9 z8 t; ~! n3 x( C    "crash_id": "2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074",
    "assert_func": "void BlueStore::_do_write_small(BlueStore::TransContext*, BlueStore::CollectionRef&, BlueStore::OnodeRef, uint64_t, uint64_t, ceph::buffer::v14_2_0::list::iterator&, BlueStore::WriteContext*)",
( n7 s1 |7 ]- C9 t: M% [+ y, ^, R    "ceph_version": "14.2.8-111.el7"
}
* M8 N+ U. @8 \& [0 \, C9 H8 }[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
" ?) l- ~% g/ G& A3 w+ ~) {[root@controller1 ~]# ceph crash ls
ID                                                               ENTITY NEW
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7      
' ]) ?  {; z0 R# x1 u[root@controller1 ~]# ceph crash archive-all
! N. f& r& W; V0 W[root@controller1 ~]# ceph crash ls
ID                                                               ENTITY NEW
' R7 Q9 x! N( m% _! c) d7 E2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7      
' u- Z5 h+ ?8 ~9 `/ P5 @' E5 r[root@controller1 ~]# ceph crash ls-new
2 f, \, p# c6 K[root@controller1 ~]# ceph crash ls
ID                                                               ENTITY NEW
% x) ?0 D# G- _) C; T) g& T% k  t2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7      
1 B- N3 n/ g' g! \" j. D[root@controller1 ~]# ceph crash archive 2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074
1 N. a( l& ^% s8 b[root@controller1 ~]# ceph crash ls
0 s2 d4 m$ Z1 Z# E9 w; c  Y5 M4 TID                                                               ENTITY NEW
2022-11-17_08:45:41.660614Z_886d754b-9ae6-4f6c-84f9-62f9f0d9c074 osd.7      
* @$ J* v7 d, R[root@controller1 ~]# ceph crash ls-new
! e/ w  w, t! _# O[root@controller1 ~]# ceph crash archive-all
" v0 j6 Q- M* O. Y6 J[root@controller1 ~]# ceph crash archive-all
[root@controller1 ~]# ceph -s
# x; Y* g( [9 m: M  cluster:
    id:     9d22e36a-2bdd-4d2d-8394-48af75ead777
2 ^+ y) y& N# P! x1 R5 d: y    health: HEALTH_OK
! c4 L$ ]4 _# a& w' Q/ ^
  services:
    mon: 3 daemons, quorum host09,host10,host11 (age 4M)
' S  H. ]3 p0 w' R    mgr: host09(active, since 18M), standbys: host10, host11
    osd: 40 osds: 40 up (since 10d), 40 in (since 11M)
: B  S/ |7 Q: p# z$ W# Q0 p, y0 S    rgw: 3 daemons active (host09, host10, host11)

4 w0 z  u5 {+ v8 O; F  \- c& W  task status:
3 H0 _4 ?8 o, m, a) t
  data:
2 F/ l) r- N5 p+ ^    pools:   16 pools, 3072 pgs
+ r4 z4 ~" @9 D1 t% L6 a    objects: 4.07M objects, 16 TiB
4 m2 x2 a" k( \$ I6 R    usage:   39 TiB used, 108 TiB / 148 TiB avail
    pgs:     3069 active+clean
             3    active+clean+scrubbing+deep
9 P9 n% Z/ K4 e( _9 j! ~2 m1 G
$ T& T8 R$ l, c  io:
    client:   1.0 MiB/s rd, 21 MiB/s wr, 648 op/s rd, 421 op/s wr

8 G; b) _+ K) o0 l1 U, ~8 M[root@controller1 ~]#  ceph config set mgr mgr/crash/warn_recent_interval 0
0 a$ Q* T: S- Q8 ]7 e[root@controller1 ~]#

  ]& ]) H% T+ S$ n2 R3 m

( V5 d+ a7 Q& m9 v9 [http://wwww.bbs168x.cn/forum.php ... Brecently%2Bcrashed
' ^8 o% M6 Z! t( j