浏览此站
联系我们相册切换到窄版 Language

 找回密码
 注册
易陆发现论坛»论坛 › 云计算开源区 › ceph分布式存储 › ceph 分布式存储同步过程中需要一次采坑过程，因为时间 ...
返回列表 发新帖
查看: 3223|回复: 1

ceph 分布式存储同步过程中需要一次采坑过程，因为时间不同步导致

[复制链接]
admin

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
发表于 2021-8-31 15:00:09 | 显示全部楼层 |阅读模式
[root@compute01 deploy]# ceph status  
1 s3 s; [) Y$ N/ T  cluster:
9 Y% J  o6 |/ k  P- ~' I6 w    id:     31403b11-8a1e-432f-876e-5a2c852f9dcc
: ~( q6 M3 U4 E7 t9 Y2 [    health: HEALTH_WARN
* I0 Y( ]& v. K' J& `6 j            Degraded data redundancy: 13701/41103 objects degraded (33.333%), 432 pgs degraded, 640 pgs undersized
) I  V! s2 y6 P, E6 @            13269 slow ops, oldest one blocked for 8178 sec, mon.compute02 has slow ops
( \4 n2 G0 G2 t+ \            clock skew detected on mon.compute02
$ d& F  [# R# J
8 U+ Y6 R4 D; N; M  services:3 ^' M3 m/ O/ d5 ~4 v. C! ?; Q
    mon: 3 daemons, quorum compute01,compute02,compute03 (age 2h)  H/ d( c0 Z% ]9 N/ [
    mgr: compute01(active, since 4h), standbys: compute03, compute02
" }  A4 O7 z  L: }4 D    osd: 3 osds: 2 up (since 4h), 2 in (since 2h)
' B; A) i% X& } + r: F  M+ s8 @# \* w4 Z5 s
  data:, ^2 T/ w$ O. D# ]& Z  `( x: a9 m
    pools:   6 pools, 640 pgs3 Q$ \& Y7 ^! n
    objects: 13.70k objects, 11 GiB& f( l9 a& u6 J9 B" J
    usage:   24 GiB used, 2.7 TiB / 2.7 TiB avail( k4 C( ?3 d* J1 q; @6 [
    pgs:     13701/41103 objects degraded (33.333%)
6 t* B& S$ A% n, _             432 active+undersized+degraded
( p) n1 L  H: E7 H* T" k             208 active+undersized
* @, B' G. G  w& i0 Z5 }) Z) E , M; u3 S# x" E7 A+ w
[root@compute01 deploy]# ceph osd crush remove osd.02 B* x+ m7 h* A2 O  [
removed item id 0 name 'osd.0' from crush map
5 o3 ?( Z0 v! h[root@compute01 deploy]# ceph osd rm osd.08 Y- f9 d; S' j- x9 e
removed osd.0
, }4 O$ W2 _, s# K8 e; s[root@compute01 deploy]# ceph auth del osd.02 h3 y9 I( t; `# P
updated# t, F; z: m0 i
[root@compute01 deploy]# history |grep sdb# \0 \) c. \4 E7 w0 ?
1023  history |grep sdb: H8 ~7 U" q; }  J1 c; [  q- _
[root@compute01 deploy]# history |grep ceph-deploy
4 L4 N0 d" I, B/ C 1024  history |grep ceph-deploy) u+ `$ l) [$ U- V4 }* o% A  o
[root@compute01 deploy]#  ceph-deploy osd create compute02 --data /dev/sdb
$ @' B! r1 N7 G9 c8 ~[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf+ m& a! q4 P. z2 O3 c2 y
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
( u: t1 N. `2 o' p* s) Y$ H7 w[ceph_deploy.cli][INFO  ] ceph-deploy options:
1 o3 E1 X! k4 E! N' Y[ceph_deploy.cli][INFO  ]  verbose                       : False
* F# n5 ]4 |: ?% M7 R6 w: h[ceph_deploy.cli][INFO  ]  bluestore                     : None
2 {2 b  Y" m2 }) I# m7 |[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fbd39643690>
( X' ~! R5 [- d3 e) T6 F% i3 [[ceph_deploy.cli][INFO  ]  cluster                       : ceph6 h7 {# W9 h: {+ Q  p6 a/ X9 `
[ceph_deploy.cli][INFO  ]  fs_type                       : xfs
. @# g" S; ~- J; |" g" y6 a3 I[ceph_deploy.cli][INFO  ]  block_wal                     : None" g5 P7 _; Q; ~8 ~" R$ O+ w
[ceph_deploy.cli][INFO  ]  default_release               : False( [0 T0 c/ B# z" \  L* @
[ceph_deploy.cli][INFO  ]  username                      : None
% |0 Z- D  B% d' t, K/ ?[ceph_deploy.cli][INFO  ]  journal                       : None
2 F$ ?4 N2 K  ^$ ^" y) g: I+ Y[ceph_deploy.cli][INFO  ]  subcommand                    : create; Q# ^; j1 Z: H; j% z) c
[ceph_deploy.cli][INFO  ]  host                          : compute02' E' c8 s4 b' f
[ceph_deploy.cli][INFO  ]  filestore                     : None
6 P1 b4 U; J7 Z; ?5 Z: X) j[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7fbd396711b8>
9 o/ P* m' L( j9 ?" L# s, n- G) e[ceph_deploy.cli][INFO  ]  ceph_conf                     : None/ k* m" c- q& b5 C5 }) J. @
[ceph_deploy.cli][INFO  ]  zap_disk                      : False
# \  i5 a+ ?+ ]8 @[ceph_deploy.cli][INFO  ]  data                          : /dev/sdb8 A( ~4 t$ W+ e. a; J; G, ~/ M9 R
[ceph_deploy.cli][INFO  ]  block_db                      : None
" g. k' d2 Q/ E, k[ceph_deploy.cli][INFO  ]  dmcrypt                       : False
" k$ y, ~4 j2 e[ceph_deploy.cli][INFO  ]  overwrite_conf                : False% W; ?$ `) c% c6 g
[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys( ]5 ~2 U! s- e7 h
[ceph_deploy.cli][INFO  ]  quiet                         : False  o" X4 k. v0 r( t3 E) Q
[ceph_deploy.cli][INFO  ]  debug                         : False
3 ^# O3 N2 n0 s6 D7 _4 h# {6 V[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb7 ?# D8 V/ x. t& x% f
[compute02][DEBUG ] connected to host: compute02
6 ]% a3 H% G  v5 N& t9 M[compute02][DEBUG ] detect platform information from remote host
, @) [" K# J2 r6 {  ^[compute02][DEBUG ] detect machine type
2 {) w2 _' J& X[compute02][DEBUG ] find the location of an executable' x3 a+ Q8 v4 |, e+ A( y$ c* g
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
' H& H, v8 F  H, [# \% ~5 d" e- D[ceph_deploy.osd][DEBUG ] Deploying osd to compute020 Z0 F0 n; X; y1 f' i$ k# G/ V
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
. E( W- f8 f! _6 Y[compute02][DEBUG ] find the location of an executable
+ N) u% x% \7 K0 c5 e$ G[compute02][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb7 @* F  y- i" j# ?3 }9 m
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
$ X* A% v& S& y# Z4 o[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 8bb22eb2-d2e1-4ae4-aa52-cc96422b25bb0 h, x+ f; x1 B

4 ^$ T" r) `( s( A; C8 c; B9 J' Z^CKilled by signal 2.
. q+ ?& b7 A0 L5 A/ H4 n& H1 \[ceph_deploy][ERROR ] KeyboardInterrupt
: I) X7 q+ K  V( r: }[root@compute01 deploy]# ls
( I! H( s' I, _0 y) f; zceph.bootstrap-mds.keyring  ceph.bootstrap-osd.keyring  ceph.client.admin.keyring  ceph-deploy-ceph.log
/ o3 @0 }4 r6 w4 r( @4 O$ O  E* t- C6 Qceph.bootstrap-mgr.keyring  ceph.bootstrap-rgw.keyring  ceph.conf                  ceph.mon.keyring: I3 E! N" v# Y  E- F
[root@compute01 deploy]# ' y) Q' m, q9 [7 x+ J8 B) ?; Q
[root@compute01 deploy]#
1 y, R# T; E( |8 U* ^+ V: x[root@compute01 deploy]# pwd
5 G/ S1 m/ c1 t7 H1 Q/ceph/deploy
8 g. [- _! _, R8 R$ L1 V" K[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb  ^C$ _$ ~- \& E# p5 Z, M
[root@compute01 deploy]#  ceph-deploy disk zap compute02 /dev/sdb- v  ^- h( J; B. D# z* b& O# j
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf# n, A2 p* E6 B/ F& U3 S# v% `
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb( L2 T1 z( `0 e
[ceph_deploy.cli][INFO  ] ceph-deploy options:
" d* O$ @/ t1 z6 A+ u; u[ceph_deploy.cli][INFO  ]  username                      : None$ k5 c, n0 Q. f$ d9 X) I, [
[ceph_deploy.cli][INFO  ]  verbose                       : False! {5 `- a+ Z- m/ e+ d0 l
[ceph_deploy.cli][INFO  ]  debug                         : False: Y( h* r; ]6 W: @5 ?- p
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
! f1 n; x( k; H+ M2 y[ceph_deploy.cli][INFO  ]  subcommand                    : zap
: o6 n) O1 b* K, }5 m0 f# p[ceph_deploy.cli][INFO  ]  quiet                         : False+ }" M* S4 @5 g
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f0db9e1dad0>
) Q8 u* A7 B, m( g[ceph_deploy.cli][INFO  ]  cluster                       : ceph
2 e- \& u: Q# Z1 Y+ M! r[ceph_deploy.cli][INFO  ]  host                          : compute022 x% Z6 l+ D" {
[ceph_deploy.cli][INFO  ]  func                          : <function disk at 0x7f0db9de3230>
1 l7 |; _5 U* B! |8 j[ceph_deploy.cli][INFO  ]  ceph_conf                     : None6 b7 o; [6 _4 |2 z2 j. w6 T
[ceph_deploy.cli][INFO  ]  default_release               : False
8 M) S0 o# W; }2 O! d8 G% a: f2 C- }[ceph_deploy.cli][INFO  ]  disk                          : ['/dev/sdb']
' T) h- l$ q. M1 c) g[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute029 ~; z( I; E7 H# B
[compute02][DEBUG ] connected to host: compute02 % K2 T1 r* T2 ]- g! F7 Y' p+ g, K
[compute02][DEBUG ] detect platform information from remote host
8 l: u8 V. {2 b$ r4 }, h6 ]& {) K[compute02][DEBUG ] detect machine type& G- E/ J2 L; @" @( i( Z
[compute02][DEBUG ] find the location of an executable; W/ D9 E6 v- G" ^# Y
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
. q1 z- i0 g9 |$ E: {' T$ h0 G6 {[compute02][DEBUG ] zeroing last few blocks of device2 C/ c0 q$ }4 }! y+ b0 e
[compute02][DEBUG ] find the location of an executable
4 H6 a. {' M1 l) M2 J# w[compute02][INFO  ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb, v9 l, z) s: d/ [  u8 b
[compute02][WARNIN] --> Zapping: /dev/sdb6 Z0 \6 A# D: w5 K& B8 r
[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table# V0 V' |  l2 c1 T
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync* Q* z) ~  m" |
[compute02][WARNIN]  stderr: 10+0 records in( s1 \4 n6 ~' _# L* y
[compute02][WARNIN] 10+0 records out  \) X% \2 v  f" T
[compute02][WARNIN] 10485760 bytes (10 MB) copied1 h" j9 I1 L6 B" ?7 j, C
[compute02][WARNIN]  stderr: , 0.0850991 s, 123 MB/s
, O+ e' `1 Z9 c4 I3 g[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>" X& C% Q# I( j! Z# B* J) ?
[root@compute01 deploy]#  ceph-deploy osd create compute02 --data /dev/sdb/ _: k0 n) m& g$ v! J. f1 U  X
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf2 ]9 p, b, A# |; C- n# S
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
1 E( W1 A) ?: V( p+ z. U[ceph_deploy.cli][INFO  ] ceph-deploy options:
. }3 E! j9 a4 v[ceph_deploy.cli][INFO  ]  verbose                       : False
, e7 ~( L( [# V/ a8 N* i. T8 U[ceph_deploy.cli][INFO  ]  bluestore                     : None
6 x' ?& q- R) F1 A( c2 p[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f45a0a92690>
( ?/ r2 d9 H/ `  ^3 I0 T; v[ceph_deploy.cli][INFO  ]  cluster                       : ceph
3 k/ L  D  m: z7 n1 ][ceph_deploy.cli][INFO  ]  fs_type                       : xfs4 ?. C) Q2 P0 |& w5 u
[ceph_deploy.cli][INFO  ]  block_wal                     : None
5 E9 p+ Q) {: Y+ b3 l) Z+ V[ceph_deploy.cli][INFO  ]  default_release               : False
% w; v# f6 g, b* R. z2 b* d9 x[ceph_deploy.cli][INFO  ]  username                      : None5 ?( Z4 r; f; l1 Z/ X7 t4 ]
[ceph_deploy.cli][INFO  ]  journal                       : None
3 N4 D  ?+ p- F9 F5 }[ceph_deploy.cli][INFO  ]  subcommand                    : create
) `; k9 V( I1 H4 s9 \4 a3 \% R4 G6 h[ceph_deploy.cli][INFO  ]  host                          : compute025 ~( P; S6 g3 H9 t" @! B
[ceph_deploy.cli][INFO  ]  filestore                     : None
8 G2 w3 @- ?* u* J5 ~9 \; y[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7f45a0ac01b8>. k, a" Z; K8 F- Z% }
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
' t' l, |. `5 o. {[ceph_deploy.cli][INFO  ]  zap_disk                      : False3 w# T' {* S8 K: b3 |# C
[ceph_deploy.cli][INFO  ]  data                          : /dev/sdb) g, R, Y7 J- j" K# M% B) @
[ceph_deploy.cli][INFO  ]  block_db                      : None0 V3 V/ c) [% C
[ceph_deploy.cli][INFO  ]  dmcrypt                       : False3 Y" o6 C* O1 F
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False6 h! F2 }& ~8 m$ q: B* l
[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys5 A4 ~$ b3 b% [6 r' V' d/ p
[ceph_deploy.cli][INFO  ]  quiet                         : False
1 J# z" F" M3 ^  ~/ C) |[ceph_deploy.cli][INFO  ]  debug                         : False
1 b3 y. s8 l' w) b( J[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
! a# E$ P. h" g0 Q# n( _[compute02][DEBUG ] connected to host: compute02 % y- c) |" m* V+ L- ]1 S. U
[compute02][DEBUG ] detect platform information from remote host) @- S/ d: H' e
[compute02][DEBUG ] detect machine type
2 _6 H! K+ M8 T6 M4 z4 _[compute02][DEBUG ] find the location of an executable
& P# Z. d' i$ U4 R) x[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
  U! F6 E' X' f* V( H( L[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
# i7 V  t* f" z* ?: {[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
3 A0 H, y& P. @, l2 I[compute02][DEBUG ] find the location of an executable
& o0 J6 ?) g4 _! n: ?[compute02][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
( V3 z; x3 P: g# t0 v- j[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key5 n: s) F/ {* K8 ~8 \  [
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9f1fca57-1a1c-4d30-bad8-2d4578f88e96
* S! G1 C: u. ^* z; l* v( H# ]^CKilled by signal 2.
9 T4 w) t0 v! E[ceph_deploy][ERROR ] KeyboardInterrupt! L* c  {( S" j  _3 b$ h" u
[root@compute01 deploy]# ls5 y3 p3 A, }$ ~; G! y" J. t' [
ceph.bootstrap-mds.keyring  ceph.bootstrap-osd.keyring  ceph.client.admin.keyring  ceph-deploy-ceph.log
% X  \* `  V: G  bceph.bootstrap-mgr.keyring  ceph.bootstrap-rgw.keyring  ceph.conf                  ceph.mon.keyring
. L/ l4 e7 s2 J( `+ A[root@compute01 deploy]# 7 w' w; i# a8 x+ i3 k% @: Y
[root@compute01 deploy]# 1 @/ c# d4 a9 }4 v4 v, K' P2 _
[root@compute01 deploy]# 1 n% b4 @3 m/ h: O1 I
[root@compute01 deploy]# ceph-de$ Q: a8 t% Q1 U6 k6 h2 B* I
ceph-dencoder  ceph-deploy    1 P$ _+ p/ K8 s: \- p: j  u
[root@compute01 deploy]# ceph-deploy
/ o! N% v* A: _4 L# K0 h6 O* Y- zusage: ceph-deploy [-h] [-v | -q] [--version] [--username USERNAME]
/ Y* B% P4 T! e4 E  T8 A                   [--overwrite-conf] [--ceph-conf CEPH_CONF]
* S, ?6 q  h6 X% d' y1 h. Y/ o                   COMMAND ...6 J9 l  P! C# H( @
Easy Ceph deployment: `9 ?9 j9 A; Z" I) d# V9 U0 k4 [
    -^-. T: ]5 K! X5 I, i
   /   \
7 C1 Y  U6 z% U/ n6 N. `   |O o|  ceph-deploy v2.0.18 c1 m% e9 O/ I1 C6 P! F
   ).-.(/ h/ X& s9 b- S% a7 N: e
  '/|||\`/ P: S' o6 ]( \( M
  | '|` |
9 ~2 R' g8 M) k: W" Y    '|`. B6 J; K4 ]4 ]+ @4 q# d# ~
Full documentation can be found at: http://ceph.com/ceph-deploy/docs
( E+ G+ @. b$ J: Y. E0 Q4 K4 T: koptional arguments:
4 M  Z5 c* b+ y& U" i& i2 F/ b  -h, --help            show this help message and exit
6 U. b, [: U! r. D! p( c  -v, --verbose         be more verbose
! Y+ H9 l# c1 x& C  -q, --quiet           be less verbose
5 d3 I& N; l2 M; L& o  --version             the current installed version of ceph-deploy+ K' P/ t: x9 G7 i1 A% z/ M
  --username USERNAME   the username to connect to the remote host2 D* M7 Q) s5 ]  R% a: `
  --overwrite-conf      overwrite an existing conf file on remote host (if
5 S, a, @' K. c$ K7 p* B$ ^7 I& \                        present)
" f) D) T( D* I! `' o9 @0 H  --ceph-conf CEPH_CONF, r/ i+ j1 a! d) m" t
                        use (or reuse) a given ceph.conf file
% |6 l& E* K" w) J& O. T* y/ ncommands:4 z8 ]& x, r$ x8 J. p
  COMMAND               description
) _8 z) p! u; j# Z3 v( J# \    new                 Start deploying a new cluster, and write a0 C+ q2 A- Q0 U; B% m
                        CLUSTER.conf and keyring for it.
+ j  l3 X: e( S" G0 ~7 |8 k    install             Install Ceph packages on remote hosts.
" h9 C! }+ C7 t- _+ k4 q* ^9 z    rgw                 Ceph RGW daemon management  `' a4 ]: Y( _" f
    mgr                 Ceph MGR daemon management; B2 e! }0 @$ S. i5 n
    mds                 Ceph MDS daemon management( ^9 @2 ?" L/ f$ C+ ?. D
    mon                 Ceph MON Daemon management
/ D0 z2 u+ F! O) x5 S/ e' p    gatherkeys          Gather authentication keys for provisioning new nodes.
' ]( C! J- y# s" o' l: Y' T    disk                Manage disks on a remote host.; D, @' u* u* H4 S# k4 E! a
    osd                 Prepare a data disk on remote host.6 N$ m, g! s. R. O" b8 O
    repo                Repo definition management
4 O$ x6 f4 r, j. A3 g0 z    admin               Push configuration and client.admin key to a remote
' M# V; d6 r! _) _- V                        host.
, E0 T1 K* D2 r7 U7 L3 W    config              Copy ceph.conf to/from remote host(s)
+ v; m  {, a# M8 s; N& \    uninstall           Remove Ceph packages from remote hosts.
7 Z- \" K2 ~! r: j1 D1 d    purgedata           Purge (delete, destroy, discard, shred) any Ceph data
5 ]4 t9 K3 [) P! B& S: Y2 m; X/ |                        from /var/lib/ceph
9 R3 Z" Q: t; ~! M4 Z- _    purge               Remove Ceph packages from remote hosts and purge all- X. C% A" j- s* M+ H7 f
                        data.& k, X: U9 V- l/ K9 Z* S& x
    forgetkeys          Remove authentication keys from the local directory.
9 B4 Q; J! d; y, X( R0 L' }; Y    pkg                 Manage packages on remote hosts., v) X. n4 [' ~& V! z2 t/ \2 h
    calamari            Install and configure Calamari nodes. Assumes that a  j/ Z, L/ n) o' ]* T$ `
                        repository with Calamari packages is already
- R  V6 L/ u) \7 ~9 Q+ M                        configured. Refer to the docs for examples
3 R1 O, I! C4 J" Q. v; v                        (http://ceph.com/ceph-deploy/docs/conf.html)7 Q# d$ E, |/ Q  C
See 'ceph-deploy <command> --help' for help on a specific command
( G" b: u' |: F! E; W# E% y1 [4 ~[root@compute01 deploy]#  ceph-deploy disk zap compute02 /dev/sdb  N4 g7 M" b" m) W, r9 Z3 j) Q) V
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf+ Y, x1 o" w* d3 V* m# X# @: |
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb
4 w5 y- W+ t' {  t( f3 X' p[ceph_deploy.cli][INFO  ] ceph-deploy options:
/ B- E. A" s* e[ceph_deploy.cli][INFO  ]  username                      : None& T8 L0 o/ R. q
[ceph_deploy.cli][INFO  ]  verbose                       : False
8 D0 P  T% p  L2 [: c[ceph_deploy.cli][INFO  ]  debug                         : False
) e, x# x4 @9 A# S[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
3 V8 O" I5 r8 D1 q: \1 ^. j/ |[ceph_deploy.cli][INFO  ]  subcommand                    : zap
; u/ [, Z  j; U[ceph_deploy.cli][INFO  ]  quiet                         : False
4 e6 x+ }' M3 I2 G( Q5 Q[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f1c11596ad0>
8 Y3 ?- H' M" l  R' M& E" h" D[ceph_deploy.cli][INFO  ]  cluster                       : ceph
* r+ x7 z( u( }6 q- h3 ^[ceph_deploy.cli][INFO  ]  host                          : compute02& S- _$ Q  X3 U4 ^, z; L& C- p
[ceph_deploy.cli][INFO  ]  func                          : <function disk at 0x7f1c1155c230>
2 q* x2 J4 ?7 }[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
$ j( {# A, v8 _. n+ H( f( ~' }[ceph_deploy.cli][INFO  ]  default_release               : False
% i- e$ j+ W0 \[ceph_deploy.cli][INFO  ]  disk                          : ['/dev/sdb']' u! w; ^7 Z9 L8 J( ^2 U
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
+ ~8 s$ h1 Y2 M7 J& C: v[compute02][DEBUG ] connected to host: compute02 ) J; C8 W' V; G1 E. y
[compute02][DEBUG ] detect platform information from remote host
+ s. c5 n; r  E& K[compute02][DEBUG ] detect machine type
$ X  B5 s- ~' y' ]* V. S+ w: z! p[compute02][DEBUG ] find the location of an executable! R4 G! `' Z* m( R5 Y# H; \. h+ [
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
2 j$ H# O$ ?2 g0 i+ q4 n[compute02][DEBUG ] zeroing last few blocks of device
  E/ B8 `( i. \6 T1 Q[compute02][DEBUG ] find the location of an executable6 q) N0 n  H  U; m% K
[compute02][INFO  ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
3 g2 C$ D9 {* _- k$ G[compute02][WARNIN] --> Zapping: /dev/sdb9 R) D: i9 r" g8 K- `! a( E& L$ R
[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table. E# `$ J6 d0 p! U8 I# g% f: X+ c
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync4 x( `' S0 V6 j1 i# Z; x0 o+ B
[compute02][WARNIN]  stderr: 10+0 records in6 ?' k( ~2 _) \4 G& p; _1 b( j
[compute02][WARNIN] 10+0 records out5 P2 C1 L( F+ t, `8 J! O
[compute02][WARNIN] 10485760 bytes (10 MB) copied
" t: p& r+ O5 }" o  a  H[compute02][WARNIN]  stderr: , 0.0591334 s, 177 MB/s3 C4 k$ [4 V5 p4 q  m: l
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
+ ]& o( T5 B* U" \* |  {[root@compute01 deploy]#  ceph-deploy osd create compute02 --data /dev/sdb
: v) |- T5 o& U2 u1 H" ?[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
) Q! ?0 s" U% o8 S9 y[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
9 m9 B7 U! R, b$ C" Q2 m& S/ {- u[ceph_deploy.cli][INFO  ] ceph-deploy options:  d0 j/ }1 g, k, H$ h
[ceph_deploy.cli][INFO  ]  verbose                       : False
7 y2 m  b0 C) b6 T1 h7 S8 N[ceph_deploy.cli][INFO  ]  bluestore                     : None
9 M! I5 R) a4 M$ V[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f23c40b9690>* f5 S; W# x+ k" M
[ceph_deploy.cli][INFO  ]  cluster                       : ceph8 A$ R% }$ w/ j
[ceph_deploy.cli][INFO  ]  fs_type                       : xfs) v! v2 |: w7 j' f
[ceph_deploy.cli][INFO  ]  block_wal                     : None
, g3 @" ]+ Z8 H$ y4 q% \. O9 [6 f7 Z[ceph_deploy.cli][INFO  ]  default_release               : False
6 R# F8 r& ^9 l& _+ Z[ceph_deploy.cli][INFO  ]  username                      : None
; i5 u* l6 q9 v# F/ \/ S[ceph_deploy.cli][INFO  ]  journal                       : None) X& Q3 ]/ Z$ S
[ceph_deploy.cli][INFO  ]  subcommand                    : create  H5 r* `1 h2 M* C
[ceph_deploy.cli][INFO  ]  host                          : compute02. e0 ~9 n/ ^" y" x
[ceph_deploy.cli][INFO  ]  filestore                     : None
* u/ _8 o8 c4 [  k, m  P[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7f23c40e71b8>
  {  s8 v. {3 u3 J0 F# o1 ]% z  d' U[ceph_deploy.cli][INFO  ]  ceph_conf                     : None5 Q, q: F" g+ \, g
[ceph_deploy.cli][INFO  ]  zap_disk                      : False
  c3 l- @$ J3 c9 q: v  {% O[ceph_deploy.cli][INFO  ]  data                          : /dev/sdb. k1 v+ h3 j4 V2 z  o2 ?
[ceph_deploy.cli][INFO  ]  block_db                      : None
  H" L- c! z" {) O# w[ceph_deploy.cli][INFO  ]  dmcrypt                       : False2 @  r( O/ q$ Q
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
* F) E( a9 e: ], W9 s[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys- B# Q3 {0 o$ v6 @6 J: }
[ceph_deploy.cli][INFO  ]  quiet                         : False1 |. ^1 r) ~7 I$ X
[ceph_deploy.cli][INFO  ]  debug                         : False' _7 l# S! v1 p1 U* G0 m1 O
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
" L1 }% V+ {1 K" n7 p: v9 I& K[compute02][DEBUG ] connected to host: compute02 9 w: O# G' b/ A
[compute02][DEBUG ] detect platform information from remote host
  S) X7 t! D! {% X" B[compute02][DEBUG ] detect machine type
! P* C( S8 f2 f# b[compute02][DEBUG ] find the location of an executable
( Y- `1 e* l, X! X: W1 g[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core6 z: x: u3 M: U8 [
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
( @4 {. y3 _7 y* k[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf+ z: B; w& n& h/ v
[compute02][DEBUG ] find the location of an executable5 u5 F9 _1 Y, w- h
[compute02][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
8 c! `4 `9 g4 f9 o[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key6 }* ]& _3 t! U! F6 [1 o) u
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new fab4f939-9a55-4e45-94d5-2434dbb24766: M4 ]* ^' A1 w5 ^& b( m
^CKilled by signal 2.
. I9 A% P8 j2 `, ?( b7 x0 j) [[ceph_deploy][ERROR ] KeyboardInterrupt# J& S  ?9 X2 z; _" _, R9 m: i
[root@compute01 deploy]# ls" R6 t& Q* e. P
ceph.bootstrap-mds.keyring  ceph.bootstrap-osd.keyring  ceph.client.admin.keyring  ceph-deploy-ceph.log  q. Y$ U5 k8 f" f& c0 k$ N6 s
ceph.bootstrap-mgr.keyring  ceph.bootstrap-rgw.keyring  ceph.conf                  ceph.mon.keyring
( a& ^2 A2 ?" P$ V8 Q[root@compute01 deploy]# vim /etc/ceph/ceph.conf
, p4 K/ @) G' Q6 e, F[root@compute01 deploy]# vim ceph.conf ) P* D3 H+ I" @; _6 c
[root@compute01 deploy]# vim ceph9 N( l: U3 @  F& L9 T( a+ {
ceph.bootstrap-mds.keyring  ceph.bootstrap-osd.keyring  ceph.client.admin.keyring   ceph-deploy-ceph.log        
7 g, u/ k! u6 ~3 D1 Iceph.bootstrap-mgr.keyring  ceph.bootstrap-rgw.keyring  ceph.conf                   ceph.mon.keyring            
( ~/ L3 d! j' p! u. e[root@compute01 deploy]# vim ceph.mon.keyring - A1 N. D% ?9 e! c4 y# u: u7 [
[root@compute01 deploy]# vim ceph.client.admin.keyring
  Q9 k' J& o) P2 u[root@compute01 deploy]#  ceph-deploy disk zap compute02 /dev/sdb
; q& `& l2 i2 {  s[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf' |8 f! D) y" a! h
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb% @% f  c$ f8 T! o( y
[ceph_deploy.cli][INFO  ] ceph-deploy options:  g0 ^' W9 I' X) g
[ceph_deploy.cli][INFO  ]  username                      : None: S' Z' N6 b9 J* M% c& X# O
[ceph_deploy.cli][INFO  ]  verbose                       : False
* a3 k6 {; `* p: M[ceph_deploy.cli][INFO  ]  debug                         : False) S: l- Q9 ?7 S) u3 I- o$ K, T
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False) X" d/ r  E) }( P! S" g& N6 f' s
[ceph_deploy.cli][INFO  ]  subcommand                    : zap- f" z/ ~" h; N* A1 P* C) M
[ceph_deploy.cli][INFO  ]  quiet                         : False9 X& W  T, |% i
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fa10739bad0>
. }# A% ^5 l7 D" K[ceph_deploy.cli][INFO  ]  cluster                       : ceph0 G$ E; H, {8 L
[ceph_deploy.cli][INFO  ]  host                          : compute027 s7 ^: R# v/ ]  o
[ceph_deploy.cli][INFO  ]  func                          : <function disk at 0x7fa107361230>
1 v! H7 v6 J; [9 T8 t1 b3 n3 C+ J[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
0 `' h  m6 Z) q4 S$ ~4 V2 s" q[ceph_deploy.cli][INFO  ]  default_release               : False# K+ |% ], i$ R
[ceph_deploy.cli][INFO  ]  disk                          : ['/dev/sdb']/ [* v" |0 n3 d9 V2 m4 [
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
! N* T* e( ]0 d3 D9 O9 c1 t[compute02][DEBUG ] connected to host: compute02
5 O) T" }5 S/ \- N[compute02][DEBUG ] detect platform information from remote host0 ?7 J  [0 q8 f- o$ g+ g) ~
[compute02][DEBUG ] detect machine type# O% d& Q; T. x3 v) H
[compute02][DEBUG ] find the location of an executable, o) l3 S6 z1 c1 q
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
( d7 \) \0 O7 ~. P  Q# z' r% i[compute02][DEBUG ] zeroing last few blocks of device/ }8 |. U9 a. E: l% T$ _9 l8 d
[compute02][DEBUG ] find the location of an executable
3 J3 s# h4 |+ q[compute02][INFO  ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
  z6 G& o2 x3 g/ e  ][compute02][WARNIN] --> Zapping: /dev/sdb/ J: D. h$ ^8 d0 j8 m
[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table
. E% n! _. d+ z! P: x" z/ C[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
4 q- a+ Z* s1 f, c& ~) V[compute02][WARNIN]  stderr: 10+0 records in& I& l8 b" M6 @( m4 x& @; n
[compute02][WARNIN] 10+0 records out  K, G4 {( n+ e8 A
[compute02][WARNIN] 10485760 bytes (10 MB) copied
% N' T, @- e/ N[compute02][WARNIN]  stderr: , 0.0869923 s, 121 MB/s
9 W) I$ L. ~5 S4 C[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
' n  r" |3 e0 f[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
: f' e+ s4 E/ x[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf! R4 u% }5 x; |( e
[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb( e5 d, ?& f: M  P+ f3 g6 R
[ceph_deploy.cli][INFO  ] ceph-deploy options:
5 O' c* S% R- ^9 z. \4 y# o[ceph_deploy.cli][INFO  ]  verbose                       : False
! F$ z' b, u4 i5 |[ceph_deploy.cli][INFO  ]  bluestore                     : None
; w6 `6 ], f6 C0 K) _* ^( Z1 G[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f13835dd690>* P! g! n3 W# Y' v( }6 C5 c
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
4 Y7 F9 I/ _$ H1 T- P/ ][ceph_deploy.cli][INFO  ]  fs_type                       : xfs
9 I3 A- y1 T* e5 d8 n3 l, K" k[ceph_deploy.cli][INFO  ]  block_wal                     : None
  V, K* \7 V3 b, y$ o! B3 T[ceph_deploy.cli][INFO  ]  default_release               : False! V. {4 Y6 [" {3 W1 g! P7 G
[ceph_deploy.cli][INFO  ]  username                      : None
' z+ W0 H8 ~2 A4 v. W( `$ F[ceph_deploy.cli][INFO  ]  journal                       : None1 y+ O) Z* ]& V9 U
[ceph_deploy.cli][INFO  ]  subcommand                    : create" `2 T4 {' m- ?9 ]) [3 W
[ceph_deploy.cli][INFO  ]  host                          : compute02
- V5 z6 c1 j9 B$ L[ceph_deploy.cli][INFO  ]  filestore                     : None
9 u# o1 D# X2 B! L; }# `6 q[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7f138360b1b8>
2 `$ k; r1 E& z[ceph_deploy.cli][INFO  ]  ceph_conf                     : None' r6 {1 @" t0 u% }# }  Y7 k% c5 p
[ceph_deploy.cli][INFO  ]  zap_disk                      : False
' Z4 c: C$ ^) y/ N[ceph_deploy.cli][INFO  ]  data                          : /dev/sdb
3 N. G' Z- n' f; o- N* s1 m[ceph_deploy.cli][INFO  ]  block_db                      : None7 ~' W$ O" m; q- {' h
[ceph_deploy.cli][INFO  ]  dmcrypt                       : False$ e3 K0 m) n$ L  n9 H5 `" y' J8 \: D
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False, S1 h: i8 L) t6 B
[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys
( G8 P  L+ o0 U+ l# g[ceph_deploy.cli][INFO  ]  quiet                         : False4 u3 w" Z4 c2 w- b! ~( h% }
[ceph_deploy.cli][INFO  ]  debug                         : False
0 W/ W! b1 C0 X. b5 k+ N% {" t3 l  v6 v[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
" F" u& W6 m0 ]! S- s; G[compute02][DEBUG ] connected to host: compute02
; z* v$ R  V0 N- d( H* ^[compute02][DEBUG ] detect platform information from remote host0 o# h6 O' ^8 A$ m
[compute02][DEBUG ] detect machine type: X* _2 H9 {$ E$ j
[compute02][DEBUG ] find the location of an executable
7 \- q, t" ?: u  P4 g  `  e: z[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core0 o2 N, w5 ^' r) K1 R
[ceph_deploy.osd][DEBUG ] Deploying osd to compute023 B" `' p. A" P) k6 _4 ~
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf) ?! C$ b  G$ B6 f2 S% `6 J
[compute02][DEBUG ] find the location of an executable  X$ o1 Z6 j0 W: D' R
[compute02][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
: v9 V$ [, U: v  G) j[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key2 \' m3 k" h8 }+ L
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new c71415c4-fdaa-4955-8c26-71d160c6121e
5 X$ C* o2 {9 ]. x' }" K[compute02][WARNIN] No data was received after 300 seconds, disconnecting...
( T$ c! N! p9 Z) s' v8 x[compute02][INFO  ] checking OSD status...
0 x* w6 Q8 Z  y' ^1 ^# e[compute02][DEBUG ] find the location of an executable8 |) D5 y* y- ?5 y/ p6 M# E
[compute02][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json2 F1 H" T4 {+ V; j2 z* ^9 k4 v
[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.; h6 _4 C* w' p) N8 V) \. {
[root@compute01 deploy]# ceph osd tree
+ [! r' X+ j/ BID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF 3 R( @. L. L/ L, A) j3 d
-1       2.00000 root default                              
) F1 w% k+ Q. l-5       1.00000     host compute01                         / F+ T# t$ _6 T5 O
1   hdd 1.00000         osd.1          up  1.00000 1.00000
& O0 M! u6 v& ^% K, v0 J-3             0     host compute02                         ( ^5 \( Z: M6 b6 w
-7       1.00000     host compute03                         # `1 C6 W/ C' k( M2 @; ]
2   hdd 1.00000         osd.2          up  1.00000 1.00000
0 L4 r- I  L$ T2 u1 I 0             0 osd.0                down        0 1.00000
1 Q- W+ ?7 o8 _0 \) f 3             0 osd.3                down        0 1.00000 . j3 Y$ s$ L$ y" E" |4 k1 S, f( \

- ?4 T% R) z( d: s/ B( f: G在另一台节点上：: ~. S* r  p% M, ~6 x8 H6 ~
[root@compute02 ~]# ceph osd tree
4 y1 h. r6 M) [) ~9 L; R  |( `; T8 B9 d* k

3 t8 ~. M3 M  A4 ^- ]1 {d^CInterrupted
$ G0 T/ `) r& z# }! `Traceback (most recent call last):
0 Q$ U2 H8 c) x, }  File "/usr/bin/ceph", line 1269, in <module>' |- ?# r/ D+ q7 z6 w
    retval = main()
8 a: ^- O( G2 _2 @0 ]9 \  File "/usr/bin/ceph", line 1200, in main6 S+ S3 q1 G2 k7 J( u, z% w% a
    verbose)/ E- k9 N( m( [1 \$ e
  File "/usr/bin/ceph", line 622, in new_style_command* _; F" [  [+ f( Y/ _8 I
    ret, outbuf, outs = do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose)
! K+ m- r& g* b  w8 V  File "/usr/bin/ceph", line 596, in do_command% k" E: @" q  X9 B" l4 z
    return ret, '', ''
8 f0 P/ x7 O3 s6 H) o& ~UnboundLocalError: local variable 'ret' referenced before assignment" \2 k, Y0 q6 m7 ?; @: S/ Z
[root@compute02 ~]# date
- h; x( G) K% Z  G- v
- f* ^: Q* s( ?3 [% w1 B. R连osd tree都不能查看：5 d# z. _  W. G9 t; p2 j
没办法，只能查看日志：
$ l0 ?: O: P! D+ [, K3 `; A9 |8 b6 T/ Y4 R8 b
% d9 u& ~; Z& X# w/ p( I
ceph的log日志视乎没有任何帮助，和ceph -s的状态一样。
/ l% {& m. }% E[root@compute02 ~]# tail -f /var/log/ceph/ceph.log
: w; m& ^2 W2 d* }) P# W& Q( c- v2021-08-31 14:13:37.918792 mgr.compute01 (mgr.49274115) 15887 : cluster [DBG] pgmap v8821: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)) V2 L) |* _7 H
2021-08-31 14:13:41.694764 mon.compute01 (mon.0) 717737 : cluster [WRN] Health check update: 528 slow ops, oldest one blocked for 477 sec, mon.compute02 has slow ops (SLOW_OPS)' i- \1 b+ {; v' S0 m9 G2 n. y
2021-08-31 14:13:46.695723 mon.compute01 (mon.0) 717740 : cluster [WRN] Health check update: 535 slow ops, oldest one blocked for 482 sec, mon.compute02 has slow ops (SLOW_OPS)
0 v# }: e& Z; g$ t, U2021-08-31 14:13:39.919835 mgr.compute01 (mgr.49274115) 15888 : cluster [DBG] pgmap v8822: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)1 u5 s9 w. j, p2 ]6 x3 ]
2021-08-31 14:13:41.921160 mgr.compute01 (mgr.49274115) 15892 : cluster [DBG] pgmap v8823: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)2 _0 n- p9 _- d) G
2021-08-31 14:13:43.925789 mgr.compute01 (mgr.49274115) 15893 : cluster [DBG] pgmap v8824: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
1 i0 v+ S8 `( B5 o% U2021-08-31 14:13:45.927393 mgr.compute01 (mgr.49274115) 15894 : cluster [DBG] pgmap v8825: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)
3 i; G+ Q; i0 M# Z' O2021-08-31 14:13:47.930528 mgr.compute01 (mgr.49274115) 15895 : cluster [DBG] pgmap v8826: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)
- I2 `9 Z/ ]( K2021-08-31 14:13:51.696665 mon.compute01 (mon.0) 717744 : cluster [WRN] Health check update: 544 slow ops, oldest one blocked for 487 sec, mon.compute02 has slow ops (SLOW_OPS)( Q/ q9 z, O3 v+ [1 B6 h) A1 g+ u  v; c
2021-08-31 14:13:56.697487 mon.compute01 (mon.0) 717766 : cluster [WRN] Health check update: 547 slow ops, oldest one blocked for 492 sec, mon.compute02 has slow ops (SLOW_OPS)6 j+ t! E4 j  Y9 R5 E* v, j6 w- a2 _
2021-08-31 14:13:49.932275 mgr.compute01 (mgr.49274115) 15899 : cluster [DBG] pgmap v8827: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%). \4 Y; t0 D2 K+ `6 U
2021-08-31 14:13:51.934115 mgr.compute01 (mgr.49274115) 15900 : cluster [DBG] pgmap v8828: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)7 Q, y$ V# s: h( s/ s
2021-08-31 14:13:53.938702 mgr.compute01 (mgr.49274115) 15901 : cluster [DBG] pgmap v8829: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)3 r1 U& @$ m+ p' D$ d
2021-08-31 14:13:55.939684 mgr.compute01 (mgr.49274115) 15905 : cluster [DBG] pgmap v8830: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)* P0 H; d) f' Q
2021-08-31 14:13:57.944025 mgr.compute01 (mgr.49274115) 15906 : cluster [DBG] pgmap v8831: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)8 I/ p, F# T% J/ C4 n& y0 V+ K) O- R
2021-08-31 14:14:01.698441 mon.compute01 (mon.0) 717769 : cluster [WRN] Health check update: 554 slow ops, oldest one blocked for 497 sec, mon.compute02 has slow ops (SLOW_OPS); _- W9 c0 B5 U* e7 d% E
^C, I* s% _+ e2 j+ o
/ G) g! _. i/ x, v; Z* P

" U; Q6 E, G; K, U4 ]查看ceph-osd.0.log时发现  osd.0 553 unable to obtain rotating service keys; retrying，意思就是时间不同步，不停的重试中：
8 z! u/ t9 k  Q, j5 `( V4 v+ v8 v/ Q
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.; S  y1 I" ], d; l# Y! [! k
ceph-osd.0.log              ceph-osd.0.log-20210826.gz  ceph-osd.0.log-20210828.gz  ceph-osd.0.log-20210830.gz  ceph-osd.1.log- |6 X* a. E: d# |$ U
ceph-osd.0.log-20210825.gz  ceph-osd.0.log-20210827.gz  ceph-osd.0.log-20210829.gz  ceph-osd.0.log-20210831.gz  ceph-osd.1.log-20210721.gz
! @4 C8 C4 Q$ B% ]7 \$ t[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
) h+ R1 C) \: t) B; d' i  C) x" Hceph-osd.0.log              ceph-osd.0.log-20210826.gz  ceph-osd.0.log-20210828.gz  ceph-osd.0.log-20210830.gz  ceph-osd.1.log
+ S- s/ J: |6 A& wceph-osd.0.log-20210825.gz  ceph-osd.0.log-20210827.gz  ceph-osd.0.log-20210829.gz  ceph-osd.0.log-20210831.gz  ceph-osd.1.log-20210721.gz
& w+ w  o6 o: h) {  _[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log
0 p  l9 R& c; T" ^* v. G% h2021-08-31 11:25:00.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
2 I7 W; B  P, r( j6 ~! j4 w2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
- a) k: M. \0 U" A3 A6 d, t0 |2021-08-31 11:25:30.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
  @0 ?' O- t/ T: w2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
) r, G9 G- D3 {9 ]  Q2021-08-31 11:26:00.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
( \: P& h8 s) c2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying- e+ a8 {+ _& N) _) o
2021-08-31 11:26:30.809 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 301 A$ B( H: a# `, `0 S& V' I
2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying4 `. G7 c: d% T5 v) z/ ?
2021-08-31 11:27:00.809 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
' _$ ?/ k) j  w" Y6 }4 t( R$ d0 s% }2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
) H8 J7 {% Z( p0 x, m# C- C- ]1 R' P3 \& c* p! R' i7 g
^C$ w+ z( E) ?  J3 p  O1 c1 g
" o0 R: K. o9 ]8 i
) p8 }0 W% i0 e6 g

- p2 G5 K6 ~% G/ P9 {# }^C^C
! l2 l0 U$ |5 o/ W[root@compute02 ~]# ^C
/ Y4 w) Z' j8 l; n* U& g[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
. X7 |+ u1 z1 z( b/ Rceph-osd.0.log              ceph-osd.0.log-20210826.gz  ceph-osd.0.log-20210828.gz  ceph-osd.0.log-20210830.gz  ceph-osd.1.log
3 W$ Y% |/ q8 v4 O% p- uceph-osd.0.log-20210825.gz  ceph-osd.0.log-20210827.gz  ceph-osd.0.log-20210829.gz  ceph-osd.0.log-20210831.gz  ceph-osd.1.log-20210721.gz1 t/ e6 l8 Q7 o6 D0 b* f
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log1 L2 A3 Y3 ^* N2 S! H& W
2021-08-31 11:25:00.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
$ i$ {! `+ _: n- \2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
  H. o) |6 s  J6 y- M$ ^2021-08-31 11:25:30.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
- I: A9 _$ p) ]. |% g1 }& [2 ]2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
  H9 p. k' c( H; a4 q2021-08-31 11:26:00.808 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
8 L5 @" p2 C% Y# f1 o( }# n2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
% q$ U! j7 w, V5 l8 u2021-08-31 11:26:30.809 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30% |- A+ B9 V  o) B: G# i3 b
2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying: s  [5 C% x0 Y0 b& z7 \
2021-08-31 11:27:00.809 7f80e5e47a80  0 monclient: wait_auth_rotating timed out after 30
: e4 }$ \+ |* t! Q3 j2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying) E5 w' Z4 R! c: h* y
^C& e& X; ?2 R* l" h0 F/ d

& m+ m5 k! N' E7 ~( z4 f4 T9 o$ `3 @
同步时间吧：虽然安装的chrony时间同步，但我们同步的时候，还是使用ntpdate比较快些：
9 t2 t4 {) `1 i& T
7 v2 h8 m, L: X4 ^/ i# R/ g5 L$ C[root@compute02 ~]# ntpdate -u compute01$ O* z- [; v  b
31 Aug 14:16:21 ntpdate[6103]: step time server 192.168.0.75 offset -14.901748 sec
( d4 {) m$ u6 h7 x# x$ X6 g0 W& E+ c$ k* G1 Y
同步完成之后，发现osd tree 命令好使了：  I6 c' V0 U& S9 ?0 n* I% r

1 S% |' D# q# b* A* J" ?[root@compute02 ~]# ceph osd tree0 ]9 q( f9 p4 ^7 K! o
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF / d) H3 f3 s& M; J4 W5 |
-1       2.00000 root default                              
2 J/ j" s0 W: q, _+ U' [-5       1.00000     host compute01                         ' N8 C7 |% K* \6 }( `
1   hdd 1.00000         osd.1          up  1.00000 1.00000
% S7 @7 ^, B6 m2 e- O-3             0     host compute02                         / V1 u9 P7 N3 ~
-7       1.00000     host compute03                         : w3 `) }( N) h; k. E% r1 v
2   hdd 1.00000         osd.2          up  1.00000 1.00000 * N/ m% K6 Y8 N' C) l  H+ o
0             0 osd.0                down        0 1.00000
) m  ]; }8 F3 m" F- m- M0 \+ |' a 3             0 osd.3                down        0 1.00000
" I9 u( z: `. l- A, t$ R! m  s' @; ^4 p4 G
把down状态的osd清理掉：! `8 W" m; Q% i/ @9 J/ d9 J6 y

2 k! c" S. F! k* M[root@compute02 ~]# ceph osd out osd.3( k, ^2 n  e# E
osd.3 is already out.
* t5 K2 Y8 ^: e' S0 [[root@compute02 ~]# ceph osd rm osd.3
4 b) ~0 [: B4 {  I+ W! W4 Gremoved osd.39 B! w2 p" Y0 Y3 I6 m* q5 P2 k
[root@compute02 ~]# ceph osd crush remove osd.3) _8 d. Z1 X/ b/ o8 G9 q. o
device 'osd.3' does not appear in the crush map9 y! \# c  d4 @; @( Q+ i4 ?2 i2 P
[root@compute02 ~]# ceph osd rm osd.3
$ f( \( S8 b. [7 s2 R" Bosd.3 does not exist.
8 ^. T- j+ H9 U  w: f- {' k[root@compute02 ~]#  ceph auth del osd.33 S9 ?# f% ^* M$ g2 D' W
updated- h+ d4 V. a" I3 |! y, x) [; ^  q( o
0 I* f0 y# I& T7 ]% x2 a& T/ V, Q- G
[root@compute02 ~]# ceph osd in osd.0
; T3 P! y  m; a+ B8 R) R! O  Nmarked in osd.0.
& k# v0 D, O! j) }9 ^0 d) W  Y[root@compute02 ~]# ceph osd tree2 k  A7 |  a5 |- |9 n* `7 g( c1 Y
ID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF
# m6 M9 O" W+ z: F) A. P- E  ]% q-1       2.00000 root default                               / x! _# ^7 @& H$ M5 K) I1 T4 \: S
-5       1.00000     host compute01                        
3 O8 _7 y6 Q; e9 H( D, ^  k 1   hdd 1.00000         osd.1          up  1.00000 1.00000
+ h3 \( g& ?* {# a-3             0     host compute02                        
% i% L1 w5 x& }% {-7       1.00000     host compute03                         3 m, R- k) r3 r( L3 ?% y
2   hdd 1.00000         osd.2          up  1.00000 1.00000
1 m+ l5 E) C" O$ `% {6 [ 0             0 osd.0                down  1.00000 1.00000 2 E9 X" B7 g" G+ W( F$ z0 [
[root@compute02 ~]# systemctl restart ceph-osd@ 5 @$ j( o; O, J7 i5 l
Display all 337 possibilities? (y or n)! G2 F. _3 W1 ?
[root@compute02 ~]# systemctl restart ceph-osd@ ' C7 ?* l) O: x" |, X
Display all 337 possibilities? (y or n)
  S7 O1 q/ V/ l5 x+ s' G: _[root@compute02 ~]# systemctl restart ceph-osd@0
; x: v3 L( _* V6 ~+ V6 e2 y/ c( O& E8 nJob for ceph-osd@0.service failed because the control process exited with error code. See "systemctl status ceph-osd@0.s
) d3 c2 k  r3 p3 @% v5 D2 n
% z) P/ P' G5 G& U. `0 Q5 p- e& J, a

1 t8 Y6 ~. l4 P$ s/ X+ a重新添加ceph磁盘osd：
; C6 E) w6 m6 r) \; \/ s2 Q) s7 m+ h9 Q  F. z2 d8 C) r
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb $ x) @" v+ L6 m) h0 X( _
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
9 J9 V, J! T! X[ceph_deploy.cli][INFO  ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb! d* `& P" v" ^. e3 w  T
[ceph_deploy.cli][INFO  ] ceph-deploy options:
! x4 d# p0 S$ U' G) S1 |( G[ceph_deploy.cli][INFO  ]  verbose                       : False
* O& |  c6 _5 C' S0 D0 N% B. \0 M) Q  e[ceph_deploy.cli][INFO  ]  bluestore                     : None
4 @: u/ S, D: S' h* r/ C3 [[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f518f889690>2 v- O! V% r9 H0 j
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
8 W" l' g9 t! X' f  [6 X[ceph_deploy.cli][INFO  ]  fs_type                       : xfs( P* m" I1 @  L
[ceph_deploy.cli][INFO  ]  block_wal                     : None
) y8 A5 }+ c/ i2 M0 ]5 p[ceph_deploy.cli][INFO  ]  default_release               : False
. B8 e! f- h) s[ceph_deploy.cli][INFO  ]  username                      : None
% C3 j& J5 T* G2 n. t[ceph_deploy.cli][INFO  ]  journal                       : None1 o+ f; C' j6 M. y4 V. L$ _
[ceph_deploy.cli][INFO  ]  subcommand                    : create
! u3 B! X2 f3 k[ceph_deploy.cli][INFO  ]  host                          : compute02
/ H  k7 @  ]) R/ x/ \[ceph_deploy.cli][INFO  ]  filestore                     : None
  h: R9 ?  O% |3 x6 `' s% Z2 Q[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7f518f8b71b8>* F9 |) f& }0 o, [: V/ ?
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
: H/ i7 ^; f0 [* z- `5 j1 Z[ceph_deploy.cli][INFO  ]  zap_disk                      : False
, G! ^0 D: h2 V, E3 b& b1 t[ceph_deploy.cli][INFO  ]  data                          : /dev/sdb. W) V0 ?1 H* b0 A
[ceph_deploy.cli][INFO  ]  block_db                      : None( V; G; S/ Z, M6 p, C0 F! Z
[ceph_deploy.cli][INFO  ]  dmcrypt                       : False. E, f0 y9 b+ B
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False- t2 M2 H, T2 N3 Y, s% o
[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys
% X4 K, t- S5 m8 t/ W[ceph_deploy.cli][INFO  ]  quiet                         : False. U2 R2 F0 F5 W7 }5 W* |. a
[ceph_deploy.cli][INFO  ]  debug                         : False
  [7 i! p1 O6 {) Z+ v[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb( J  c# V0 M) |" S( N9 o/ k+ ^5 F+ e
[compute02][DEBUG ] connected to host: compute02
! t; E) j. f+ i1 R& A[compute02][DEBUG ] detect platform information from remote host
0 E+ t# F  ~2 B, P[compute02][DEBUG ] detect machine type
! o0 `3 w3 O/ K3 _[compute02][DEBUG ] find the location of an executable6 T1 P% q$ S6 B4 r& a
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.9.2009 Core
' r( Z1 g% T5 V+ U[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
$ J8 A7 p$ Q  H" L4 M( l7 k[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
# ?4 {9 y8 l2 T( v& l0 T[compute02][DEBUG ] find the location of an executable  {' M; G/ }6 v' g' B0 }
[compute02][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
# y% }: ~# U. f[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
/ L$ x. s! I, Q) P3 Q! p[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new bbfd988d-bb2e-44f2-a366-17501a8bbada+ Q, S0 R- a+ R5 I
[compute02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l 100%FREE -n osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463
- u: M4 t: c' G5 r( L; g[compute02][WARNIN]  stdout: Logical volume "osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada" created.% p2 b& J" H. q: r6 d
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key5 i+ `5 q4 x" q+ n
[compute02][WARNIN] Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0
& U# z0 c  N# N2 ]7 j[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada1 z; ?" `' i' ~
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
4 H# t+ j5 G: w9 T& t3 n, ^8 P[compute02][WARNIN] Running command: /usr/bin/ln -s /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block  W) R) F* r4 U% Z
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap) `3 P8 ]/ j- S# D5 _
[compute02][WARNIN]  stderr: 2021-08-31 14:21:34.978 7fb58c080700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
- U5 {! Z9 N- n' q, E0 a4 p[compute02][WARNIN] 2021-08-31 14:21:34.978 7fb58c080700 -1 AuthRegistry(0x7fb584065de8) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
4 Q3 R; H5 E% o7 C# H7 O, B[compute02][WARNIN]  stderr: got monmap epoch 3
% I* U$ s) v  k3 G; a$ s& Z4 M[compute02][WARNIN] Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==
+ V5 n4 t( o! @3 l! K. d[compute02][WARNIN]  stdout: creating /var/lib/ceph/osd/ceph-0/keyring& r. L) n. M& @( J9 G4 R  y, [3 K
[compute02][WARNIN] added entity osd.0 auth(key=AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==)
2 M! F. t! r( F+ F: l9 c$ v- O9 W[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring6 R) W2 s- [$ w! o2 W( g% }% \: F" G
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/- m' g- \" v; A) g) V
[compute02][WARNIN] Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid bbfd988d-bb2e-44f2-a366-17501a8bbada --setuser ceph --setgroup ceph% r! _; O0 t- @( D) m& K, ^
[compute02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdb
1 w, f6 y; C/ a0 C5 o3 e. w& k[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0$ C# v4 y0 l  X+ u
[compute02][WARNIN] Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada --path /var/lib/ceph/osd/ceph-0 --no-mon-config1 m) Z1 r8 M6 d3 p. f. ~' T
[compute02][WARNIN] Running command: /usr/bin/ln -snf /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block3 R" _7 N$ a/ [2 r. W8 l
[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block3 \7 ]% y4 y0 |" w3 S
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
# j. ~5 W5 R9 z" O- G  M[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
$ ?8 x1 M3 W2 F6 c# i3 v+ p[compute02][WARNIN] Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada# Q* W7 U2 u2 G1 Q9 O
[compute02][WARNIN]  stderr: Created symlink from [url=]/etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada.service[/url] to [url=]/usr/lib/systemd/system/ceph-volume@.service[/url].0 D6 T; T# A7 N# t5 w
[compute02][WARNIN] Running command: /usr/bin/systemctl enable --runtime ceph-osd@0
& X2 y  x) X4 `5 F: t  U[compute02][WARNIN]  stderr: Created symlink from [url=]/run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service[/url] to [url=]/usr/lib/systemd/system/ceph-osd@.service[/url]., ]$ S; ?0 o3 C) _4 E# x& H* `/ e
[compute02][WARNIN] Running command: /usr/bin/systemctl start ceph-osd@0$ d, T+ S  n" L8 S7 l
[compute02][WARNIN] --> ceph-volume lvm activate successful for osd ID: 03 @4 _! X/ q3 f6 T5 X
[compute02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdb" L! Y3 x& X/ x- n
[compute02][INFO  ] checking OSD status...
5 Y! N) r+ P  i" ?, l3 b) C$ R[compute02][DEBUG ] find the location of an executable# x5 Q5 g; K) A& A
[compute02][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
# c% A5 p- f/ b( v[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use./ H% m0 ?, W9 v5 J
" R5 @1 B9 k8 T3 W4 x; H5 U8 r
; q5 p- k' a0 Y* u. f) V% `0 \4 ~1 i% ]
成功创建。这个坑确实有点过分。ceph存储一定要先解决时间不同步问题，否则有很多意想不到的惊喜：
% n- J6 X$ R: k% e- C% k' x5 ?1 r6 {
' }" b& i: O& j0 ^" B
/ P* i. B' k& n# f[root@compute01 deploy]# ceph osd status  9 R' o5 e! k' K+ G5 I: I( e
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
# {" ~* n7 ^. x/ E( V9 S| id |    host   |  used | avail | wr ops | wr data | rd ops | rd data |   state   |# J1 T: E- M* m5 D
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
$ G5 [3 U# g6 F1 W: \| 0  | compute02 | 1096M |  557G |    0   |     0   |    0   |     0   | exists,up |2 s! e4 F# I* j1 k
| 1  | compute01 | 12.1G | 2223G |    0   |     0   |    0   |     0   | exists,up |
& |' d. `3 R* G| 2  | compute03 | 12.1G |  546G |    0   |     0   |    0   |     0   | exists,up |9 i; _' e4 z# l; U' n; x
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
% q3 ~$ n( ]+ `; o9 C* e4 _3 U[root@compute01 deploy]# - ^9 g7 x* I( ^" p- d& F3 h  T1 p. k3 W
[root@compute01 deploy]# & r( ?0 V  w8 Y- s, Z* }! P
[root@compute01 deploy]#
' S9 H- n$ G. v! d% t2 o[root@compute01 deploy]# ceph osd status  ) S1 g6 ]4 m& g8 q! b
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+$ J# L( ]4 v) B+ X2 i
| id |    host   |  used | avail | wr ops | wr data | rd ops | rd data |   state   |: W* n# A' ~2 e1 G6 T3 A) _
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
) q5 ~' L3 N( O% ?* z6 `0 \1 m| 0  | compute02 | 1096M |  557G |    0   |     0   |    0   |     0   | exists,up |
4 z6 h* C2 R% p( t7 l1 r0 m- {| 1  | compute01 | 12.1G | 2223G |    0   |     0   |   10   |   327   | exists,up |+ Y* M3 l" L- o) {/ f3 O1 W  T
| 2  | compute03 | 12.1G |  546G |    0   |     0   |    9   |   266   | exists,up |
2 n* e$ _3 Q. ]5 A$ I& J+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
* O$ k! S6 Z* |* o7 C) t8 l3 C6 s[root@compute01 deploy]# ceph osd status  + F& B$ s6 A$ S; g: B- o; W
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
7 y" O  Q" W# R( J) y+ _/ a2 e| id |    host   |  used | avail | wr ops | wr data | rd ops | rd data |   state   |- G. L0 `1 j/ n6 F0 x: y8 A
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+6 K% @, _. Y+ Y2 V
| 0  | compute02 | 1096M |  557G |    0   |     0   |    0   |     0   | exists,up |# A. C, v( N( J% t9 _5 \! L0 Z* J
| 1  | compute01 | 12.1G | 2223G |    0   |     0   |   10   |   327   | exists,up |
* H$ k/ P* V  y5 \  w1 S( ~! @8 u0 J| 2  | compute03 | 12.1G |  546G |    0   |     0   |    9   |   266   | exists,up |' X& Y% u0 q/ V! L# X; E
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
% J8 a: r7 Q0 V( d8 t; H[root@compute01 deploy]# ceph osd status  
/ G7 ?: A$ A" K0 x, U8 Z- ]+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
1 E2 C# m- a! x  o6 r; U| id |    host   |  used | avail | wr ops | wr data | rd ops | rd data |   state   |6 g9 \8 w9 s3 m6 `7 o7 y6 K: P* M
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
5 ]  ~6 B! J: j3 r| 0  | compute02 | 1096M |  557G |    0   |     0   |    0   |     0   | exists,up |" c; y% V; p/ }
| 1  | compute01 | 12.1G | 2223G |    0   |     0   |    0   |     0   | exists,up |, k- B* O8 @) u+ R
| 2  | compute03 | 12.1G |  546G |    0   |     0   |    0   |     0   | exists,up |' C; y/ b# y. y9 @0 U
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
8 ^- [# n  k/ v8 ~: W- Y5 v[root@compute01 deploy]# ceph osd status  
: p5 p4 v% n. Y% @; ]) t4 R+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
5 S. f5 e" F# x! i6 u' X5 o| id |    host   |  used | avail | wr ops | wr data | rd ops | rd data |   state   |
1 Y) X- m0 \" y; Q6 w/ A- L+----+-----------+-------+-------+--------+---------+--------+---------+-----------+; t6 E$ k6 ^8 u( F- F. _, q
| 0  | compute02 | 1096M |  557G |    0   |     0   |    0   |     0   | exists,up |$ b2 E( S6 P7 t- n( A: L& B
| 1  | compute01 | 12.1G | 2223G |    0   |     0   |    0   |     0   | exists,up |0 H% v$ ]$ \  f* E1 x
| 2  | compute03 | 12.1G |  546G |    0   |     0   |    0   |     0   | exists,up |
. q" E" ?8 U9 L! }* u+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
! E; {8 y  g9 s6 X6 H
回复

举报

admin

1

主题

0

回帖

12

积分

管理员

积分
12
QQ
 楼主| 发表于 2021-8-31 15:23:15 | 显示全部楼层
[root@compute01 ~]# ceph osd tree
& r$ V. ?) t2 c# Q; B6 lID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF * Z2 S1 N* G6 {- w, |; |
-1       2.00000 root default                               # o% o2 }- x  Q7 P: l
-5       1.00000     host compute01                         % c1 S& \4 y% S  h. k' @9 k
1   hdd 1.00000         osd.1          up  1.00000 1.00000
: w8 y4 m+ v) j0 r! ^1 V-3             0     host compute02                         5 ~5 v9 B  B3 _) W
-7       1.00000     host compute03                         ) S; x: X3 H2 H( n/ W9 C4 R( d  l
2   hdd 1.00000         osd.2          up  1.00000 1.00000
( \: t: O0 y6 K8 p6 m2 D* e- h 0   hdd       0 osd.0                  up  1.00000 1.00000
6 |( t8 M: I/ [- E5 r& F' l- u! [8 L, d; Y  v
[root@compute01 ~]# ceph osd crush set osd.0 1.00000 host=compute02$ V. d. X$ D" z% ^# P1 B! ^
set item id 0 name 'osd.0' weight 1 at location {host=compute02} to crush map6 H4 L5 z! L2 O5 z
[root@compute01 ~]# ceph osd tree
) _; X6 X; k6 U% M4 {; pID CLASS WEIGHT  TYPE NAME          STATUS REWEIGHT PRI-AFF - P* F9 k+ D. T1 a# P7 w
-1       3.00000 root default                              
( j+ N/ E* L( C) x4 b3 D! ^-5       1.00000     host compute01                         ; x7 |7 ^0 i, s+ _) W/ d( w# Z
1   hdd 1.00000         osd.1          up  1.00000 1.00000 & q# H$ a/ `+ K; o' m& `
-3       1.00000     host compute02                        
3 I  }+ g/ }8 E. _1 t$ E 0   hdd 1.00000         osd.0          up  1.00000 1.00000
9 l+ i9 f: ~  ]* C1 M! y-7       1.00000     host compute03                         3 B7 N5 ^+ `% A5 R
2   hdd 1.00000         osd.2          up  1.00000 1.00000 % J; s& s5 D& J* d8 f
[root@compute01 ~]#
5 A9 x3 K1 C2 B9 I1 r5 |6 P9 o
( O) u' }( C) V2 o9 X/ G$ E0 f( P$ X% ]7 @6 g# k
回复

举报

返回列表 发新帖
高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 | 注册

本版积分规则

返回首页|Archiver|手机版|小黑屋|易陆发现技术论坛 ( 蜀ICP备2026014127号-1 )

GMT+8, 2026-6-12 00:01 , Processed in 0.019941 second(s), 22 queries .

Powered by Discuz! X5.0

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表