|
|
[root@compute01 deploy]# ceph status
* R+ U+ o. ^ @7 l cluster:
' V1 r$ C2 A5 k( R1 t9 e& @ id: 31403b11-8a1e-432f-876e-5a2c852f9dcc
1 _% S6 Z0 j! j health: HEALTH_WARN e4 ?; s5 ^* p% m4 B
Degraded data redundancy: 13701/41103 objects degraded (33.333%), 432 pgs degraded, 640 pgs undersized- S* q# z/ F# u4 e
13269 slow ops, oldest one blocked for 8178 sec, mon.compute02 has slow ops) I3 d) i0 w' R2 c/ u5 X5 i
clock skew detected on mon.compute02
$ w0 j; D# u6 p( B: W; u- s; Y ' P+ R. v! D; y4 ]
services:) S& X3 m- j O2 R1 s, Z
mon: 3 daemons, quorum compute01,compute02,compute03 (age 2h)
) Q8 K+ W! ?" q9 Z" C3 _- F+ L mgr: compute01(active, since 4h), standbys: compute03, compute02
: ~% V2 m& _' N, p# C9 x9 q5 i osd: 3 osds: 2 up (since 4h), 2 in (since 2h)
1 ?7 r4 _' S! j+ F0 Y7 d, V
6 p9 ~; F) C4 Z, v; t) X data:
4 K V7 J* M! k v. z6 Y pools: 6 pools, 640 pgs
1 r. p1 | v6 s7 U5 K! v% ~ objects: 13.70k objects, 11 GiB' b& k" Z8 l) }' h" r7 m- t
usage: 24 GiB used, 2.7 TiB / 2.7 TiB avail
' V5 t; p% B) C/ X' X9 { pgs: 13701/41103 objects degraded (33.333%)
5 [: L" k, x5 D- e ~; w- ] 432 active+undersized+degraded
3 @& ` g+ t! }& N4 M; S$ J 208 active+undersized
, N) k S2 m, B. v
1 D( n3 k3 U) j/ V7 `6 b[root@compute01 deploy]# ceph osd crush remove osd.0
2 w- f$ [$ C. k5 xremoved item id 0 name 'osd.0' from crush map
8 z, D/ g& B- P! k7 H; b& B[root@compute01 deploy]# ceph osd rm osd.00 A% q( K h1 u
removed osd.0. V$ @: O2 l0 s4 d
[root@compute01 deploy]# ceph auth del osd.08 y9 o) J! B1 c! w
updated
9 f% b* T0 h( H% A% _# {* X4 z; C[root@compute01 deploy]# history |grep sdb/ T! v4 w7 @2 o9 w) E! a# U
1023 history |grep sdb
! T6 }8 e* T- P' @* \[root@compute01 deploy]# history |grep ceph-deploy0 H, \1 Z3 G/ v i* V
1024 history |grep ceph-deploy
5 q0 `' z1 u: Q# k[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb' W. |6 G" m* ~% c3 E6 d
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
6 w% H; c5 C/ Y/ x- _[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
) A2 E0 G- M: N# [1 i6 [# p[ceph_deploy.cli][INFO ] ceph-deploy options:- J# L( X& y' P, Q" b
[ceph_deploy.cli][INFO ] verbose : False
1 V6 n% W' j; w2 s& \' [$ _+ H: n[ceph_deploy.cli][INFO ] bluestore : None- |: O: [- ^6 ~7 H
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fbd39643690>; \% m$ M) _) ]3 g3 E- }) }
[ceph_deploy.cli][INFO ] cluster : ceph
7 O: c; z3 F* p% L[ceph_deploy.cli][INFO ] fs_type : xfs
2 P( d8 _3 e5 B[ceph_deploy.cli][INFO ] block_wal : None4 R& f& q% T' K6 Z; O- a0 q8 y$ s
[ceph_deploy.cli][INFO ] default_release : False# S: |6 I1 `# r3 s9 V
[ceph_deploy.cli][INFO ] username : None
) c( _. T. t# O% ^6 A/ l[ceph_deploy.cli][INFO ] journal : None k' g6 O, H" @% r
[ceph_deploy.cli][INFO ] subcommand : create w; D* p( V) x
[ceph_deploy.cli][INFO ] host : compute024 u* s% B8 D3 l; s, q# m0 f+ L
[ceph_deploy.cli][INFO ] filestore : None
- Z- K9 W. x" l8 K) _+ y* x* ^, t[ceph_deploy.cli][INFO ] func : <function osd at 0x7fbd396711b8>; v/ ^6 X6 a! U- T+ B1 q- y1 i; }
[ceph_deploy.cli][INFO ] ceph_conf : None
! ^9 ~/ ^7 w/ E7 B7 D, k[ceph_deploy.cli][INFO ] zap_disk : False8 C0 {, }) V0 X* D! O6 p& X6 s1 N7 P
[ceph_deploy.cli][INFO ] data : /dev/sdb
) o6 Z* ]# D* s* Q' ~* X* A. W( U[ceph_deploy.cli][INFO ] block_db : None, M0 V2 f* m+ n' \
[ceph_deploy.cli][INFO ] dmcrypt : False, m' S$ t9 X( |
[ceph_deploy.cli][INFO ] overwrite_conf : False8 ]2 R3 R6 C8 |) `8 `8 E5 N
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
. y* ~- v- R* d: k[ceph_deploy.cli][INFO ] quiet : False; b+ s/ w5 g3 \% ]$ ~7 T
[ceph_deploy.cli][INFO ] debug : False
1 F9 t, V2 y& A7 [2 |6 {[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
6 W0 m4 |2 [2 v- ^- V$ O$ h[compute02][DEBUG ] connected to host: compute02 4 q4 h' {2 s1 Q s
[compute02][DEBUG ] detect platform information from remote host$ Z0 v. _ ~) K9 @. u! |
[compute02][DEBUG ] detect machine type
+ v- a+ ] y8 {% ?, L u[compute02][DEBUG ] find the location of an executable
H9 p# }& M5 z: @9 A[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
0 E' n' F6 `) W- x: ][ceph_deploy.osd][DEBUG ] Deploying osd to compute02
/ K3 Y i Z6 D& ~% v6 @$ t2 V[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf2 r S |* [$ D: Y* n
[compute02][DEBUG ] find the location of an executable* g( a6 I8 l/ K! `8 Y n: c
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
# C, y7 B0 U/ n[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key, A: d- a+ P% ]( X, l
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 8bb22eb2-d2e1-4ae4-aa52-cc96422b25bb
& ]/ }! l# H* B$ O9 w/ `* _
6 }% c- h' f, [7 ^) G( g^CKilled by signal 2.# l7 |( F# u" L: s! q; \% \
[ceph_deploy][ERROR ] KeyboardInterrupt. F6 B$ w- n" |: s
[root@compute01 deploy]# ls. A1 ]1 _6 P) f9 ^& j
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log. f' J% Y, R7 `( ]# F7 K& W' m
ceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring) H( a! U+ |3 h' e1 @; q3 j
[root@compute01 deploy]#
& X0 O7 g2 _7 g[root@compute01 deploy]#
! o2 @7 x$ k2 U# |3 Z$ A[root@compute01 deploy]# pwd
, b3 I; t0 E) ~' c* W& r8 K/ceph/deploy7 x* V% W) ^- e- `
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb ^C3 R$ I& k7 _8 V# U0 {' r
[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb- w' g, z- h1 \% y# e7 @0 C
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
! ^2 ^" B: l, J" o9 |; h: i[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb) l" t A. \) Y9 I( f! ?
[ceph_deploy.cli][INFO ] ceph-deploy options:# Q3 i5 r! k% y. F1 U
[ceph_deploy.cli][INFO ] username : None- \3 L( k# D T Q0 G4 n6 }' w* h
[ceph_deploy.cli][INFO ] verbose : False X: k, o0 n# s4 p3 K+ L$ S
[ceph_deploy.cli][INFO ] debug : False
V1 q" `, B; B; Z! Z[ceph_deploy.cli][INFO ] overwrite_conf : False
# ^# j9 \8 {/ ~, W1 G w[ceph_deploy.cli][INFO ] subcommand : zap% C" c9 t8 b- h5 O1 m' W3 f& f) B
[ceph_deploy.cli][INFO ] quiet : False
. a+ L; d, F3 @# m[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f0db9e1dad0>, }1 w, ^- D4 S+ O" v
[ceph_deploy.cli][INFO ] cluster : ceph2 y( K1 W& P6 v( H( T. r5 C5 D
[ceph_deploy.cli][INFO ] host : compute02, ^/ F' |3 [" j+ J" D, e
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f0db9de3230>
5 b' s3 {$ v) s) \. H) {# V% t[ceph_deploy.cli][INFO ] ceph_conf : None
+ v* {1 W' t- X7 l9 O[ceph_deploy.cli][INFO ] default_release : False/ l& G4 K, [; v" z
[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']
) n3 F7 ]# ^# K' ?8 }5 s[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02! v4 a- Q* I# w4 _9 m, U+ d
[compute02][DEBUG ] connected to host: compute02
- T1 I7 n, B1 p- W$ K# {[compute02][DEBUG ] detect platform information from remote host+ g1 o Q0 F5 J
[compute02][DEBUG ] detect machine type: N# O% o6 X% p+ [5 H6 s: w
[compute02][DEBUG ] find the location of an executable8 Q) S: [3 ]4 Y/ J3 ]) F
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core1 M! l- [2 y; Q6 A/ @
[compute02][DEBUG ] zeroing last few blocks of device
& L$ h# A2 h8 n9 N[compute02][DEBUG ] find the location of an executable+ p N& f4 x1 {4 d" j
[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb9 p0 h1 K- @ g5 P
[compute02][WARNIN] --> Zapping: /dev/sdb
5 c$ e8 R* R& T9 d[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table' I( b8 U0 t. |) l4 ^6 P
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
$ k5 a; W0 A% E7 Z: B4 l- k[compute02][WARNIN] stderr: 10+0 records in
8 F, b& D9 R5 L( |5 p9 A: ]& h[compute02][WARNIN] 10+0 records out
. D& w; J+ g$ V; V[compute02][WARNIN] 10485760 bytes (10 MB) copied
) |9 t* J% ]% @5 ~) |1 P[compute02][WARNIN] stderr: , 0.0850991 s, 123 MB/s0 G7 c- |/ M, A. ]4 T5 ~9 ^' A( ]
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>: q% b- _: K; I! @ Y
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
3 s; [8 s" x2 [/ w0 P1 i[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf2 Y5 Q3 \5 X7 J- E# y) y
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
& }: s/ o! d) n& g: U+ N$ O[ceph_deploy.cli][INFO ] ceph-deploy options:
( @' @. o5 J! L7 Z[ceph_deploy.cli][INFO ] verbose : False9 N2 r7 e/ K/ x4 Z: s! s
[ceph_deploy.cli][INFO ] bluestore : None g) s7 a1 w; r8 z
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f45a0a92690>0 r4 G7 H0 p! B, j
[ceph_deploy.cli][INFO ] cluster : ceph
5 D/ x4 J% U9 B) U* R Q[ceph_deploy.cli][INFO ] fs_type : xfs
. i( ?4 n9 `1 _) a[ceph_deploy.cli][INFO ] block_wal : None
% C: O2 o' Q. I. n! H4 Q7 e- T[ceph_deploy.cli][INFO ] default_release : False
! y/ o/ F0 b n- U! v" o[ceph_deploy.cli][INFO ] username : None. `! u6 R$ y/ a7 V. C6 R) q
[ceph_deploy.cli][INFO ] journal : None0 J% b/ j W \. d% \" Q1 M
[ceph_deploy.cli][INFO ] subcommand : create" I u7 A0 D) Z% s
[ceph_deploy.cli][INFO ] host : compute02# O* a2 E2 Z2 s' r8 x& _& A
[ceph_deploy.cli][INFO ] filestore : None
! @/ y) D1 T1 h6 o6 E! f" H[ceph_deploy.cli][INFO ] func : <function osd at 0x7f45a0ac01b8>4 U* [3 p3 H$ }- d0 n4 x
[ceph_deploy.cli][INFO ] ceph_conf : None
: A. l$ G/ C+ {[ceph_deploy.cli][INFO ] zap_disk : False
/ E, s+ W: w8 s# T4 G[ceph_deploy.cli][INFO ] data : /dev/sdb' M3 I2 W2 z) n
[ceph_deploy.cli][INFO ] block_db : None
/ |% U) Q6 x& W+ q[ceph_deploy.cli][INFO ] dmcrypt : False
1 p7 T0 Q3 B G1 H[ceph_deploy.cli][INFO ] overwrite_conf : False* v% @& v5 B: h. A7 t
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
L( |4 { w& `[ceph_deploy.cli][INFO ] quiet : False4 O* r% l' ] j
[ceph_deploy.cli][INFO ] debug : False
~9 H+ c) G) k6 u% E[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
8 I' ~; S/ R* d( I[compute02][DEBUG ] connected to host: compute02 4 D8 `' B6 t( _# J, a3 R" T) ~
[compute02][DEBUG ] detect platform information from remote host
8 ^1 l' L% Q2 B: \3 E[compute02][DEBUG ] detect machine type
* O" Z8 f2 i- C) @" h0 k0 g[compute02][DEBUG ] find the location of an executable
% V/ @/ Z1 w. G8 o. L6 k/ r[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core* ]# a- C- J2 @- n0 _6 `% j
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02; E% V* C5 J% f" N3 _7 T
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
+ j4 o7 m' z0 M2 W[compute02][DEBUG ] find the location of an executable
- X) [$ v. J0 F6 O6 G[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb+ k% K, g' r( p" ^1 ]
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
( h6 j6 |& J, u* E9 J. I. y6 e4 C# i[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9f1fca57-1a1c-4d30-bad8-2d4578f88e96) j$ n, g7 f9 ?2 \0 N% A9 {+ x! \1 h
^CKilled by signal 2.
8 Y+ m% u% m+ h# m" Y[ceph_deploy][ERROR ] KeyboardInterrupt
, W1 s; T4 ]) O% H- e/ P( M3 I[root@compute01 deploy]# ls1 ^1 V! h% F5 E9 |! |
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
8 ?+ c S6 t& ? rceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring
; o0 _+ d( r3 x4 z' s% ?& o5 q9 p[root@compute01 deploy]#
; O% Z! C; A2 X5 d* H[root@compute01 deploy]# ' l+ x6 Y. }7 m( G/ o+ C& `
[root@compute01 deploy]#
8 Y: E* Y- |. C7 C% S/ n2 U4 _[root@compute01 deploy]# ceph-de
( a7 K, a" M6 A" y' f" r0 o' qceph-dencoder ceph-deploy
, v& Y' |$ H% V[root@compute01 deploy]# ceph-deploy
4 A: ?' ?* n) T- G* i1 X2 Eusage: ceph-deploy [-h] [-v | -q] [--version] [--username USERNAME]- v( [" d8 V% F" K. H# q' X* ]
[--overwrite-conf] [--ceph-conf CEPH_CONF]0 ^, i1 K) m0 a0 ]
COMMAND ..., l* d; v+ Q; i
Easy Ceph deployment
# p7 \8 U1 D3 W -^-
6 i. e3 I5 g! s! f3 L. H / \, d9 h9 n( z& g$ |0 Q
|O o| ceph-deploy v2.0.1
$ H' X6 n& z B1 P/ F! D ).-.(
/ v* \1 z3 ]- t$ R '/|||\`% x9 b4 O$ h4 ?: ^+ y E
| '|` |. ?2 Z- Q& M8 i5 r6 _
'|`1 j0 T) l' Z2 N4 m
Full documentation can be found at: http://ceph.com/ceph-deploy/docs7 s1 U$ R6 N. G5 E
optional arguments:' n. u; Z+ Q! [& v- E, K* C
-h, --help show this help message and exit! b9 m, R/ a/ D6 L5 n) k8 o
-v, --verbose be more verbose
# X% g( z/ R) J3 p7 V6 i -q, --quiet be less verbose
! e0 t) f3 k4 S --version the current installed version of ceph-deploy
- u u) b( U5 w6 ]2 ~. g/ ]1 J --username USERNAME the username to connect to the remote host
+ y+ R2 y4 H C' ? --overwrite-conf overwrite an existing conf file on remote host (if, @1 T8 k* a2 f! X5 Q
present)
4 B9 i2 a, |, h0 U) d --ceph-conf CEPH_CONF5 L- @$ U1 t3 w* b" n5 q5 O3 l
use (or reuse) a given ceph.conf file
: J4 C h( ]6 d V+ [) Bcommands:
& R& X1 D% x3 E COMMAND description2 b0 o% w# r* Z/ @& I$ b% Y
new Start deploying a new cluster, and write a
& Q& b9 l9 `) w CLUSTER.conf and keyring for it.0 Q8 I1 s* D9 Z
install Install Ceph packages on remote hosts.. U) @3 X' u7 X# Z6 H
rgw Ceph RGW daemon management' A2 Q( O' j! n7 G! o% t. z
mgr Ceph MGR daemon management/ m3 {5 [/ r" W( F4 p8 j& H- a
mds Ceph MDS daemon management1 J1 R& _" v5 m4 b: K
mon Ceph MON Daemon management
3 Z. n, }5 K& n) n gatherkeys Gather authentication keys for provisioning new nodes.$ W1 [7 F1 c' s& e( T8 A( L
disk Manage disks on a remote host.$ t! T+ v, A" Z- @* g8 Q
osd Prepare a data disk on remote host.
`, K( q5 X H repo Repo definition management
2 p8 m5 L( E% t) I* R admin Push configuration and client.admin key to a remote
6 s% X6 |2 S0 i+ m4 \# P7 e8 v host.; `4 ~* S# y5 V) h. A1 j: z3 z
config Copy ceph.conf to/from remote host(s)% U R* l% Y# ]; M
uninstall Remove Ceph packages from remote hosts.2 s. b+ K- G" g2 D7 ]; S
purgedata Purge (delete, destroy, discard, shred) any Ceph data
/ h3 ^" O0 W- T4 ~; c! p/ p from /var/lib/ceph5 N! U2 ^( Q* _7 H" _1 W" u7 D
purge Remove Ceph packages from remote hosts and purge all
7 }( p0 l* y7 n. Y. l7 G data.2 G4 ?: f7 {" T
forgetkeys Remove authentication keys from the local directory.2 p7 \2 s1 L, }% W; T/ r9 O
pkg Manage packages on remote hosts.# x) _% g! w8 C# ^; P, V7 V
calamari Install and configure Calamari nodes. Assumes that a" G) l. O- N* E9 N# \% m$ j# w
repository with Calamari packages is already
) R5 Y n. ^. Y4 b$ d5 c- f6 O configured. Refer to the docs for examples
% P; x t" y/ L (http://ceph.com/ceph-deploy/docs/conf.html): }$ s# V4 j7 ^2 u& \- e
See 'ceph-deploy <command> --help' for help on a specific command' g5 d8 ?# g/ z1 a9 G9 z
[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb7 \& u6 o; N* n9 V# t
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
" \/ x3 P8 T7 A! |7 _; g[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb) j/ ^ j+ S4 ?' y) B. E. C) j$ [
[ceph_deploy.cli][INFO ] ceph-deploy options:
& `) ~& X# k% k. u9 {7 x8 ^[ceph_deploy.cli][INFO ] username : None; k& v- m, ~1 s' I
[ceph_deploy.cli][INFO ] verbose : False
% e2 r6 S/ @0 {' F F[ceph_deploy.cli][INFO ] debug : False
/ o' G& h l4 V: `) M6 [ L[ceph_deploy.cli][INFO ] overwrite_conf : False
' X6 O' f6 N- t[ceph_deploy.cli][INFO ] subcommand : zap; S9 F* [; Z, C' Y9 h, ?# D
[ceph_deploy.cli][INFO ] quiet : False' i5 [1 V2 X1 R* \3 h" S
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f1c11596ad0>
& B5 D+ a4 ` h! j* Y. v[ceph_deploy.cli][INFO ] cluster : ceph
% l3 t' }1 y c- r& L3 c6 l) R[ceph_deploy.cli][INFO ] host : compute02$ J$ z+ d& B, N' m* o
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f1c1155c230>
& N: T7 |' K0 \) ]3 s[ceph_deploy.cli][INFO ] ceph_conf : None
* k$ o( | U' r- _[ceph_deploy.cli][INFO ] default_release : False
) `! H% P8 \9 Z+ E6 p# k[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']
1 p: D" m% N u+ @& B" ^; e+ p7 h' Q[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
( H+ h0 P: }0 k4 o% F2 Q[compute02][DEBUG ] connected to host: compute02 - V: |4 O1 q$ d+ V+ j
[compute02][DEBUG ] detect platform information from remote host
2 r ^! ?% p3 L1 q[compute02][DEBUG ] detect machine type' J& I% r! [8 F R# c- w+ ]5 _
[compute02][DEBUG ] find the location of an executable& N+ Z. O1 m a3 C5 p$ d
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
u: L# [; a- D8 V, N* k9 Z[compute02][DEBUG ] zeroing last few blocks of device& t( b; b$ U7 u) D- T
[compute02][DEBUG ] find the location of an executable
/ a& Q" r2 x! Q! _; V5 q[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb: B: ~( I+ K! i g2 u- D
[compute02][WARNIN] --> Zapping: /dev/sdb
# g& X. c* [. x4 e3 U1 ^[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table# C3 s8 R# S& G" }% y. [ ]( V
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
5 z: n/ v' o! W( h9 j[compute02][WARNIN] stderr: 10+0 records in: h7 r4 t! ~5 ~7 E k
[compute02][WARNIN] 10+0 records out* Q# T# n- d) `8 s6 N+ v% l" d
[compute02][WARNIN] 10485760 bytes (10 MB) copied/ W# v) k3 E2 Y# o/ _1 k9 ?: f+ l% [
[compute02][WARNIN] stderr: , 0.0591334 s, 177 MB/s3 N6 a, K7 w( x& m
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>, f% P: l) q0 c$ N- V4 g0 K# S0 x' y
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb3 H7 j; c' e5 ~7 K
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf* t# _* j# N7 Y8 M; b8 ^
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb( V6 B. M- y; x% G! B0 Q; D" s _$ X, ~
[ceph_deploy.cli][INFO ] ceph-deploy options:
* L& A B2 A) x3 @/ e; f' F: a[ceph_deploy.cli][INFO ] verbose : False+ o% t* k, u! x+ X
[ceph_deploy.cli][INFO ] bluestore : None0 c, w% }" k$ w p/ N9 n# P* O; D
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f23c40b9690>
6 |5 Q5 q# E Y& D[ceph_deploy.cli][INFO ] cluster : ceph
9 X) ? i/ c5 b+ N0 K[ceph_deploy.cli][INFO ] fs_type : xfs
! a. H5 s8 ~2 u[ceph_deploy.cli][INFO ] block_wal : None) D8 g v2 q4 l! ?+ @3 z
[ceph_deploy.cli][INFO ] default_release : False( O1 Z( t! P/ a& f6 y
[ceph_deploy.cli][INFO ] username : None
7 x3 j3 @5 I3 t' ?[ceph_deploy.cli][INFO ] journal : None9 B: i/ [ i4 \- ]. y
[ceph_deploy.cli][INFO ] subcommand : create
4 r1 a1 }2 v6 D# H0 k: B[ceph_deploy.cli][INFO ] host : compute023 y s3 V0 ?6 Y: z0 P8 k% Z
[ceph_deploy.cli][INFO ] filestore : None
% Q |' w) ]/ T% w+ e' W' N[ceph_deploy.cli][INFO ] func : <function osd at 0x7f23c40e71b8>
* p' P. L1 D& e8 X[ceph_deploy.cli][INFO ] ceph_conf : None
9 S/ j0 ]" q# K7 L" {( Q$ f/ c[ceph_deploy.cli][INFO ] zap_disk : False
; }4 v+ f, G) k1 n. y2 ^[ceph_deploy.cli][INFO ] data : /dev/sdb, d3 g" _+ h/ ~% X, y
[ceph_deploy.cli][INFO ] block_db : None/ X6 m" t! m/ K' j3 h6 X9 j
[ceph_deploy.cli][INFO ] dmcrypt : False
3 D8 U }/ {; B# ~[ceph_deploy.cli][INFO ] overwrite_conf : False
5 W) {2 b. o/ {9 @( p/ V( R[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys$ V. q/ C6 A+ F5 Q+ V) O' z
[ceph_deploy.cli][INFO ] quiet : False7 h, I# e6 ~! N$ t% B
[ceph_deploy.cli][INFO ] debug : False
' H" u; R" M& W A* ~1 p S[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
0 _% S9 U& l( a; i: A[compute02][DEBUG ] connected to host: compute02 & T& R. a5 |4 M, N1 t5 |% `5 V
[compute02][DEBUG ] detect platform information from remote host' j; X* ]) c; I2 e5 A+ K
[compute02][DEBUG ] detect machine type
- W8 ~, F5 x/ F9 W[compute02][DEBUG ] find the location of an executable' j( _1 b- A5 ^7 N2 u. _
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core, X5 V {+ y) @0 D
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02: S2 J( ]2 p$ e l$ Q+ a! I
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
# H: L" P+ @7 [9 R' K[compute02][DEBUG ] find the location of an executable
9 H$ h# C9 o$ a( V[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
* s; w0 S3 q, d/ L' e/ v[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
& X6 d2 a2 ^0 }+ ?9 f C: o: b[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new fab4f939-9a55-4e45-94d5-2434dbb24766
( B! W P3 b+ a^CKilled by signal 2.
& N' _; q4 e2 W! [: i[ceph_deploy][ERROR ] KeyboardInterrupt
) Y( A8 S. x h" C3 `9 C[root@compute01 deploy]# ls
$ I7 a) u5 w; }& l! @ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
* d/ x2 L* D k9 N1 n' L2 M% Rceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring; g4 J; i1 I6 z1 P0 `3 e3 F4 V) p" n. ~
[root@compute01 deploy]# vim /etc/ceph/ceph.conf
h3 D4 |& a- T[root@compute01 deploy]# vim ceph.conf 0 U! J( l' Q n/ G# g0 L
[root@compute01 deploy]# vim ceph
: w: l7 V9 p6 U5 Z. |9 W5 q% lceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
6 C0 ^% y" [$ o/ ^# C( m7 m! Vceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring & L, N' w% K: C3 ^
[root@compute01 deploy]# vim ceph.mon.keyring
S# i' q) Y7 P2 R[root@compute01 deploy]# vim ceph.client.admin.keyring - v# I4 P8 y" ]0 v" E1 V. R8 d
[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb
/ z9 i4 {( v( L5 G0 c% E[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
) E2 X9 m7 O$ r- [ ]! N[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb
" t. Z. R! v* [6 z/ ]! `, w5 M$ N[ceph_deploy.cli][INFO ] ceph-deploy options:
7 G' b0 S5 m( p3 {/ n[ceph_deploy.cli][INFO ] username : None
5 g) g9 D& g3 k! ^[ceph_deploy.cli][INFO ] verbose : False# p0 \; b. V4 \8 D. l; u! I. G
[ceph_deploy.cli][INFO ] debug : False( S. O) _$ q2 B3 ~/ u# V ?5 ?7 F
[ceph_deploy.cli][INFO ] overwrite_conf : False
/ G, h! r/ A, Y7 x[ceph_deploy.cli][INFO ] subcommand : zap
4 k3 s+ @; Y; k8 J' @[ceph_deploy.cli][INFO ] quiet : False' Y, X) I) a* \1 |; Y/ t8 X- y% x
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fa10739bad0>
; w' M8 G b: [* L% a* Y[ceph_deploy.cli][INFO ] cluster : ceph
) K% u$ x- g5 W1 ?) F[ceph_deploy.cli][INFO ] host : compute028 B# V$ G# B9 V& S7 G
[ceph_deploy.cli][INFO ] func : <function disk at 0x7fa107361230>
1 ]+ [' g: L1 r, N1 \[ceph_deploy.cli][INFO ] ceph_conf : None! j, \' u6 l6 U, C' {: c& C+ P
[ceph_deploy.cli][INFO ] default_release : False
0 D; C- f( r7 X4 c[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']9 u: ?& M1 Z$ `+ @8 {
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute024 n1 C, m% p; s
[compute02][DEBUG ] connected to host: compute02
+ ?1 x* J% M' z[compute02][DEBUG ] detect platform information from remote host- G- ?8 `$ i2 H* W( j* w3 d
[compute02][DEBUG ] detect machine type
, X/ s" I9 o; y[compute02][DEBUG ] find the location of an executable
u2 b" m- e+ [( u; _$ g; @2 d7 \[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core2 }7 |$ e; |' c6 V L8 P; }2 q8 U8 P
[compute02][DEBUG ] zeroing last few blocks of device
5 P- J. J0 L+ y+ [[compute02][DEBUG ] find the location of an executable9 v- p# O K6 ?, z' {5 L
[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb' q2 ~# @5 i2 Z
[compute02][WARNIN] --> Zapping: /dev/sdb
1 Y: c& c2 b- [: J8 K[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table2 z! F3 r+ n! g/ z* R3 K. w h+ Q- ?
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
; k) |# `% G/ g; `/ [[compute02][WARNIN] stderr: 10+0 records in5 H8 k( n2 w# Z8 s+ X' I
[compute02][WARNIN] 10+0 records out
9 n5 V, W( f4 }( i[compute02][WARNIN] 10485760 bytes (10 MB) copied
9 G' V0 ]* Q+ H& A: }% ?[compute02][WARNIN] stderr: , 0.0869923 s, 121 MB/s
6 w: f* y$ i3 Q/ [5 S" T' C! S[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
; ^6 ?& h. R g- O[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
( \2 l& c1 C/ r[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf# v0 x8 @) h) n& O0 Z" W# L+ C2 `/ [
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb- {' F: J- n* I( m; Z
[ceph_deploy.cli][INFO ] ceph-deploy options:
2 j( g% A( J# M% h6 ~- M; m[ceph_deploy.cli][INFO ] verbose : False3 F4 H7 R2 E- D3 e6 I
[ceph_deploy.cli][INFO ] bluestore : None! y1 J u. a, m# p- o+ V/ C
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f13835dd690>; y; L: e" C! k9 H# n0 |! V
[ceph_deploy.cli][INFO ] cluster : ceph: u) F' ~' V, ]
[ceph_deploy.cli][INFO ] fs_type : xfs
6 }) {% T# b4 A5 h+ s: E[ceph_deploy.cli][INFO ] block_wal : None8 S, j- H* a* |# u% A
[ceph_deploy.cli][INFO ] default_release : False
5 ?( E7 x5 [5 u& [9 o* C[ceph_deploy.cli][INFO ] username : None
% @! s# T% E6 s[ceph_deploy.cli][INFO ] journal : None: w- ^( ^2 ?9 s3 B( @
[ceph_deploy.cli][INFO ] subcommand : create) C! g: Z8 z* I1 W/ U$ O- _
[ceph_deploy.cli][INFO ] host : compute02
5 v) ~5 t: M6 T[ceph_deploy.cli][INFO ] filestore : None
7 w9 G) P, x9 y: ^9 q3 t5 J, F[ceph_deploy.cli][INFO ] func : <function osd at 0x7f138360b1b8> M( b0 n$ u& B ]9 o1 W
[ceph_deploy.cli][INFO ] ceph_conf : None7 f, q# i0 w8 t+ N1 X8 u: o
[ceph_deploy.cli][INFO ] zap_disk : False5 z" H- X1 \7 |
[ceph_deploy.cli][INFO ] data : /dev/sdb0 R. W: p+ W! a5 A
[ceph_deploy.cli][INFO ] block_db : None2 O q7 U' d7 o% k, a4 N- V
[ceph_deploy.cli][INFO ] dmcrypt : False u6 P$ l/ M! d# I
[ceph_deploy.cli][INFO ] overwrite_conf : False
9 _6 s' {/ s+ _( @9 U) S: h% T n) z# M[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
- O1 f K3 M7 ][ceph_deploy.cli][INFO ] quiet : False
" @# C/ R* b. u9 G; l6 h' v; f[ceph_deploy.cli][INFO ] debug : False
! R+ x7 X1 H6 {/ E[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
( M* ?) U! w* i# }+ e2 d# R, c[compute02][DEBUG ] connected to host: compute02 # j1 F. {; O+ J7 @
[compute02][DEBUG ] detect platform information from remote host! E& b) U& P: K0 U( Z0 v1 l
[compute02][DEBUG ] detect machine type6 J, c# s8 _1 |6 k6 J# E
[compute02][DEBUG ] find the location of an executable. r* ?' ?* \; h, `
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
( J. b2 f2 ^2 p[ceph_deploy.osd][DEBUG ] Deploying osd to compute028 \( b! q8 ^9 Y1 P
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf3 O) H8 u! r6 v0 b0 a5 M
[compute02][DEBUG ] find the location of an executable
* Z+ }! {8 U7 Z$ Z" v& r( _[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb* g* y. m( L# U+ l
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
1 \$ v& _5 y6 c9 V+ L[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new c71415c4-fdaa-4955-8c26-71d160c6121e
) C' b1 z# d( x7 }) ~8 P# c4 E& X[compute02][WARNIN] No data was received after 300 seconds, disconnecting...
, y* j5 |* r8 U" V[compute02][INFO ] checking OSD status...
2 _5 g4 o V2 [1 I* e1 B[compute02][DEBUG ] find the location of an executable, q) U4 A- u# M- ^9 X
[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
, C: G1 n J) L3 N! L[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.+ K2 P, T; h; U- B/ {
[root@compute01 deploy]# ceph osd tree9 W! B3 ]! b0 W+ `9 d# W
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
! R# t5 \7 G5 e) J-1 2.00000 root default % f# h0 K' Y" |, a: S, t) Q
-5 1.00000 host compute01
3 |+ C5 r! J# d 1 hdd 1.00000 osd.1 up 1.00000 1.00000 6 ?& I) F2 i; _4 [
-3 0 host compute02
# q g: M3 ?1 _/ x5 d# @3 M-7 1.00000 host compute03
" q! |8 I5 \, }& U2 e7 d4 r7 ^ 2 hdd 1.00000 osd.2 up 1.00000 1.00000 $ S* P( ^! i0 C( g
0 0 osd.0 down 0 1.00000 ( l' T+ W9 K6 |5 f) W8 V6 y
3 0 osd.3 down 0 1.00000
2 R4 h& S6 E: E* I5 y
7 `- f+ p3 g5 e在另一台节点上：6 M' { u4 Z- ^# _- @! r$ V
[root@compute02 ~]# ceph osd tree0 Y3 u3 ^6 q! W
! c$ X9 y4 C9 _
/ f' L8 q+ ?; O
d^CInterrupted
5 g% p' S9 e( h" `2 A. k5 jTraceback (most recent call last):
8 {( h L4 c6 q' w9 E File "/usr/bin/ceph", line 1269, in <module>" M) T. K0 ?; U& n$ F- I7 u
retval = main()
& t# k- @* }3 h6 R( A File "/usr/bin/ceph", line 1200, in main
% B! M; p4 S& ^" h! i8 m5 O0 v- D verbose)
. U2 V5 e5 j4 Z9 r3 o5 S# ~+ h File "/usr/bin/ceph", line 622, in new_style_command
3 | O' e5 r+ x. w ret, outbuf, outs = do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose)6 r" E$ G7 u% D. X2 Q
File "/usr/bin/ceph", line 596, in do_command
) E! v B' Q; Z) E) t* O+ A return ret, '', ''' j9 H! i/ [4 p) T
UnboundLocalError: local variable 'ret' referenced before assignment- ?( E* F( e. Z( K% D2 G
[root@compute02 ~]# date
* B; ~( Y, d# [, i9 g3 k1 H9 B) u+ `2 S T O1 @- V* f, D
连osd tree都不能查看：6 q. \/ y0 k' H- L/ M4 z
没办法，只能查看日志：) J8 {" z3 A, o9 f7 Y, Y" g
. ?9 ^4 E$ a( I |" A% k) [
1 ?, A9 | b4 Z% Q# X
ceph的log日志视乎没有任何帮助，和ceph -s的状态一样。, v- T0 l$ V, n+ @6 S
[root@compute02 ~]# tail -f /var/log/ceph/ceph.log. x- z) @5 y U" i# {( X
2021-08-31 14:13:37.918792 mgr.compute01 (mgr.49274115) 15887 : cluster [DBG] pgmap v8821: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)8 }# E5 w& H( \9 O$ b# G
2021-08-31 14:13:41.694764 mon.compute01 (mon.0) 717737 : cluster [WRN] Health check update: 528 slow ops, oldest one blocked for 477 sec, mon.compute02 has slow ops (SLOW_OPS)' h6 v/ B% |8 ^* R& |: @
2021-08-31 14:13:46.695723 mon.compute01 (mon.0) 717740 : cluster [WRN] Health check update: 535 slow ops, oldest one blocked for 482 sec, mon.compute02 has slow ops (SLOW_OPS)
/ z6 C7 K- x; h3 j2021-08-31 14:13:39.919835 mgr.compute01 (mgr.49274115) 15888 : cluster [DBG] pgmap v8822: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%) G6 g" n( l! }6 O$ L% V
2021-08-31 14:13:41.921160 mgr.compute01 (mgr.49274115) 15892 : cluster [DBG] pgmap v8823: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)1 U; G; y6 z; Z1 r
2021-08-31 14:13:43.925789 mgr.compute01 (mgr.49274115) 15893 : cluster [DBG] pgmap v8824: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)" R4 ]' G& q9 \
2021-08-31 14:13:45.927393 mgr.compute01 (mgr.49274115) 15894 : cluster [DBG] pgmap v8825: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)
* j3 i$ R4 c. e2 x' U% l+ s2021-08-31 14:13:47.930528 mgr.compute01 (mgr.49274115) 15895 : cluster [DBG] pgmap v8826: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)0 ?6 z B S9 ]4 C# z' Q
2021-08-31 14:13:51.696665 mon.compute01 (mon.0) 717744 : cluster [WRN] Health check update: 544 slow ops, oldest one blocked for 487 sec, mon.compute02 has slow ops (SLOW_OPS)
8 f: g% c+ V" N4 o* p1 h4 u- U9 e2021-08-31 14:13:56.697487 mon.compute01 (mon.0) 717766 : cluster [WRN] Health check update: 547 slow ops, oldest one blocked for 492 sec, mon.compute02 has slow ops (SLOW_OPS)
^3 M) v3 R; S2 O2021-08-31 14:13:49.932275 mgr.compute01 (mgr.49274115) 15899 : cluster [DBG] pgmap v8827: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
) J" l" w. B, n* J2021-08-31 14:13:51.934115 mgr.compute01 (mgr.49274115) 15900 : cluster [DBG] pgmap v8828: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
k3 l9 d4 P. j# W/ ~! Z2021-08-31 14:13:53.938702 mgr.compute01 (mgr.49274115) 15901 : cluster [DBG] pgmap v8829: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)7 N8 Q& V4 d* R
2021-08-31 14:13:55.939684 mgr.compute01 (mgr.49274115) 15905 : cluster [DBG] pgmap v8830: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
) h, I# V9 u/ n2021-08-31 14:13:57.944025 mgr.compute01 (mgr.49274115) 15906 : cluster [DBG] pgmap v8831: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
; R+ F; |7 x& J! Q8 B2021-08-31 14:14:01.698441 mon.compute01 (mon.0) 717769 : cluster [WRN] Health check update: 554 slow ops, oldest one blocked for 497 sec, mon.compute02 has slow ops (SLOW_OPS)2 _) K2 ?4 o f8 `$ Z; A+ J
^C2 |. ]; M( I! h5 m8 b4 i
5 |2 Q0 h/ y8 ^ \( B
2 b2 o" g. H m7 i# l7 D2 ?' f查看ceph-osd.0.log时发现 osd.0 553 unable to obtain rotating service keys; retrying，意思就是时间不同步，不停的重试中：
/ G4 L3 Y$ h4 H/ P/ l5 e
[0 ]. H( _; e1 U) {- k[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.$ I% A0 u2 d0 l+ _8 d$ C$ q( c( c
ceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log
9 u; |' A1 F& \! _ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz- o: f8 f3 w1 ^# E- U' ?) _6 N
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
$ S$ Q& ?* t+ Q7 m6 ^4 T! Zceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log
* _& |- l5 s* sceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz
3 j: n2 D+ h) N9 n# {- {5 {[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log; H5 r) k6 x! f( n# i
2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
5 v g" ?7 Q. [* ^2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
# U8 U( c7 i! V u2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
8 {- f9 g x2 [2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying; ^6 x* w1 S* X3 B+ r z
2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
! M0 R2 p! i6 o$ k2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
+ v$ E: g. t2 |* X2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30/ r1 v8 y9 G8 L1 [/ d
2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying4 K" \" N8 R4 P3 @0 n
2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
1 y/ {+ f/ k! O0 d( R) Q2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying! V' m. g+ K9 N" ^# w( m
- r3 p6 l7 [9 ]1 u0 A% M^C- R- I1 r: U) y; @# `) p
8 D. t% w5 u/ J" E
! h- p* M7 |' E5 t: z9 L7 o9 C) v' `5 Z- ?: M+ M
^C^C3 v `- M" r$ x" |$ q4 a* ?
[root@compute02 ~]# ^C
! X$ [& j4 v) l8 t1 ^" o0 c" y[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.8 F* B1 {. w l- r: F
ceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log& Y6 r8 C) V# t, l0 L# X) n
ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz
% C6 t7 r. t' ^8 Q Y[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log
9 M$ A3 ^( U- c* F2 d) f' q2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
9 l/ Z# R I* Z" ?2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying, o. k2 s; U$ f7 x, s, M' m8 Q! u
2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
) w8 t4 K' Z! v/ Y1 i0 D( u6 [2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
" }$ `( d& G. I0 q, m, U" p5 r2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
; y- R* U2 r, L% n: l2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying* ~8 }9 U F& |; J5 o; \
2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30+ [0 e' R+ ^8 L% O# N! x5 S# I
2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
3 g L8 _( e- x7 _$ V5 r: v2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30% e0 w! N! U! P* G4 ]( v
2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
f: O5 ]2 z, b/ F^C
4 N1 k$ y6 g3 n" l8 n9 Z! `
6 u/ |+ W1 P7 k# |! f
* a2 F: O5 k' c, b" H# b' D同步时间吧：虽然安装的chrony时间同步，但我们同步的时候，还是使用ntpdate比较快些：
/ b/ z1 f5 E& L* v7 U! A2 b8 M, H- L* F0 | \8 R/ p- f
[root@compute02 ~]# ntpdate -u compute01. y1 b5 ]4 b+ r
31 Aug 14:16:21 ntpdate[6103]: step time server 192.168.0.75 offset -14.901748 sec
+ v! t- b; ?; g5 g/ C) m! P: Z7 `4 s$ j+ K
同步完成之后，发现osd tree 命令好使了：
3 u' T! w* k: E. [' R) b
) v, w. t# h8 Q2 b2 i" K% {% v: Y$ H[root@compute02 ~]# ceph osd tree
a) A0 c& p1 j7 WID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
' T" d) U8 I# _# q/ z- z: p-1 2.00000 root default 3 T! T1 ?" Y" X' `1 W4 x# K
-5 1.00000 host compute01
/ A6 d+ c7 ^- W z 1 hdd 1.00000 osd.1 up 1.00000 1.00000
" f- b( P7 W" x G' _-3 0 host compute02 ) F. r+ A5 Q- v
-7 1.00000 host compute03
# p1 `" [3 x& w& `1 v* s: l 2 hdd 1.00000 osd.2 up 1.00000 1.00000
|+ N; C( d. j7 u( Z 0 0 osd.0 down 0 1.00000 , U* {7 _5 D# L9 z5 _: U' x
3 0 osd.3 down 0 1.00000 / n3 I9 W* ~9 ~" V$ U s! W* H
, E- m0 D: i: Q; w* p# t
把down状态的osd清理掉：; \; o6 i# P, m8 o! J" @" w
$ T8 E6 c% | z" i( |: v% y. I[root@compute02 ~]# ceph osd out osd.3
+ X& t0 l1 w0 x( e9 cosd.3 is already out. - \" @ R9 w" [5 N3 c3 B9 W
[root@compute02 ~]# ceph osd rm osd.3
8 ^ Y; c5 M; o% `removed osd.3/ K& {( v7 u3 V$ X
[root@compute02 ~]# ceph osd crush remove osd.3
! W$ ?2 M# M ~5 n& i$ m- Jdevice 'osd.3' does not appear in the crush map
7 o: S+ e: C- r, @[root@compute02 ~]# ceph osd rm osd.3
+ O$ ]$ e( }. K) mosd.3 does not exist.
$ ^( P# W, H! V N4 Y h[root@compute02 ~]# ceph auth del osd.34 `& w( w+ d, W
updated' G- U e4 F1 U- e
4 t8 ^" G7 q; d$ t4 L8 s
[root@compute02 ~]# ceph osd in osd.0' X/ k3 G' J, k( o( [
marked in osd.0. 4 e; n7 {% z- {4 f
[root@compute02 ~]# ceph osd tree) J3 N$ F8 p, j6 H
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF 2 l' ]( _" N5 N8 J* ~
-1 2.00000 root default
9 ~4 ^4 ?3 }2 o$ {7 v-5 1.00000 host compute01
/ D* d# T: H1 A 1 hdd 1.00000 osd.1 up 1.00000 1.00000
; e- b \$ ^$ }8 y$ s! a+ A2 X-3 0 host compute02 $ M5 a/ J! Y* v3 z" L4 ~
-7 1.00000 host compute03
9 g2 z; B& A0 D/ C( q 2 hdd 1.00000 osd.2 up 1.00000 1.00000 + ^( H5 ^* k% D8 s9 O1 G7 F
0 0 osd.0 down 1.00000 1.00000
; a6 I( E" u& [# M[root@compute02 ~]# systemctl restart ceph-osd@
! z5 X/ J" P3 }: A9 yDisplay all 337 possibilities? (y or n)
5 K1 Z, f D6 k/ r A8 Z6 `[root@compute02 ~]# systemctl restart ceph-osd@ ! v6 y2 R$ ~8 X7 v
Display all 337 possibilities? (y or n)
& f/ Q7 r9 W* u& y7 x' B[root@compute02 ~]# systemctl restart ceph-osd@0
1 w4 F( L7 M& oJob for ceph-osd@0.service failed because the control process exited with error code. See "systemctl status ceph-osd@0.s4 a3 X a. i9 ]( i$ Z9 ^5 C
9 M* y$ Z& H5 j: a4 n
, W& Z1 R/ q# N' t, H* F
4 g1 f9 k0 |9 I4 j8 [
重新添加ceph磁盘osd：
y3 W4 Q6 Y8 s5 C+ z* S4 K2 c" r& t# ^. E
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
6 Z& r( Z# }) s[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
/ H2 C7 X9 e* h* R: D- I3 O, {[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
' k3 V2 l r" s4 Y: y! G[ceph_deploy.cli][INFO ] ceph-deploy options:
" N8 X% T% E1 v+ P9 {[ceph_deploy.cli][INFO ] verbose : False
- E V- u4 `( B H! H[ceph_deploy.cli][INFO ] bluestore : None3 b' d n8 m& E" y0 b
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f518f889690>
5 Q% Q9 X) h' Z- s1 \[ceph_deploy.cli][INFO ] cluster : ceph) V0 F$ t: H; z5 {1 C) N2 n* L% D
[ceph_deploy.cli][INFO ] fs_type : xfs
/ \+ \- |( b" x# H0 W[ceph_deploy.cli][INFO ] block_wal : None9 d; s) |2 I) ]
[ceph_deploy.cli][INFO ] default_release : False
$ I: O: Y' @3 P3 |6 R( p[ceph_deploy.cli][INFO ] username : None1 e! A0 J9 }4 d4 B; [1 k( S
[ceph_deploy.cli][INFO ] journal : None
( V8 g; @& B, |, Q6 i1 [[ceph_deploy.cli][INFO ] subcommand : create
; Y" K. f, s! q( ?' ~, V7 n[ceph_deploy.cli][INFO ] host : compute02' W, s4 M) c& I1 C& r4 W3 U" W7 j% s
[ceph_deploy.cli][INFO ] filestore : None
- O6 w+ A: E3 @ N8 z* r[ceph_deploy.cli][INFO ] func : <function osd at 0x7f518f8b71b8>& ^. d1 P3 H, K+ G% i
[ceph_deploy.cli][INFO ] ceph_conf : None
% |. v2 v% t G; y" q4 s[ceph_deploy.cli][INFO ] zap_disk : False. V' p* G" [, K7 n
[ceph_deploy.cli][INFO ] data : /dev/sdb
5 i9 J7 R$ x0 k4 E8 g0 B% V[ceph_deploy.cli][INFO ] block_db : None
5 l- R2 \! t, o6 B8 a$ J[ceph_deploy.cli][INFO ] dmcrypt : False. M' } t! V9 P% N0 m4 |8 a
[ceph_deploy.cli][INFO ] overwrite_conf : False, i! r6 a1 D2 J* U
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys! S" G8 a* M1 r/ l1 h8 V
[ceph_deploy.cli][INFO ] quiet : False
& l& R e! R. Z, J[ceph_deploy.cli][INFO ] debug : False6 N' Z( o8 f/ j8 e
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb3 ~; L2 O6 M7 b' p
[compute02][DEBUG ] connected to host: compute02
1 N0 ]4 j: @7 |6 S% c[compute02][DEBUG ] detect platform information from remote host, r1 d+ |3 F+ `" G/ L
[compute02][DEBUG ] detect machine type! T2 j* h- n/ N2 @. q
[compute02][DEBUG ] find the location of an executable
' ^% g* [0 w ][ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
, v4 F8 p; a9 _' M5 }[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
4 }, A" O, M- I! g9 J- d! x0 p/ L[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf, d7 N* U: C6 ^- i0 @' d+ a( o
[compute02][DEBUG ] find the location of an executable
0 g9 E7 X. `* v8 Z5 C7 Z[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb4 C" I% [6 ` k9 t7 `
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key8 G9 G9 N: W8 ~& O) V3 @. M5 ^# i) d, T
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new bbfd988d-bb2e-44f2-a366-17501a8bbada3 l$ T! @7 B4 ^' [4 Q
[compute02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l 100%FREE -n osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463
4 A9 {( y6 F' F+ s/ Y3 A[compute02][WARNIN] stdout: Logical volume "osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada" created.$ h' k/ [" v8 g5 h4 t* }+ M
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
8 [6 ^* T; L& c* t7 T: L1 ?3 @[compute02][WARNIN] Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0( |4 \* t9 q W, j' u0 [0 [0 r9 M
[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada4 a7 @0 t8 B6 [4 g! z
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
2 [ k0 |, D' [1 a# B, z; D[compute02][WARNIN] Running command: /usr/bin/ln -s /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block7 Z8 @. Z& l% u6 h8 c* b
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
, I# g( x( v, _: k9 Q[compute02][WARNIN] stderr: 2021-08-31 14:21:34.978 7fb58c080700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
( y V5 m; Q5 L; k5 M7 J) \! `[compute02][WARNIN] 2021-08-31 14:21:34.978 7fb58c080700 -1 AuthRegistry(0x7fb584065de8) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx, G0 \9 }# r. R' w1 K2 V9 @# X
[compute02][WARNIN] stderr: got monmap epoch 3
) C7 h: v' l4 j/ U3 o[compute02][WARNIN] Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==
4 T, |+ x2 R; \' s: H[compute02][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-0/keyring
3 @7 s5 u' A6 \. M$ |[compute02][WARNIN] added entity osd.0 auth(key=AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==)
; N# V3 Z+ F/ w+ y8 P& |) ^[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring
. T8 m% y6 W L[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/5 }+ y! L, t8 y/ k& e& ?
[compute02][WARNIN] Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid bbfd988d-bb2e-44f2-a366-17501a8bbada --setuser ceph --setgroup ceph
! g2 @+ Y+ x! a1 ^" Y. c[compute02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdb, y( w* {! P, V( R/ S
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
6 U6 b2 ]8 _5 U2 m[compute02][WARNIN] Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada --path /var/lib/ceph/osd/ceph-0 --no-mon-config3 m5 G0 @: V; w- g5 X" u& V
[compute02][WARNIN] Running command: /usr/bin/ln -snf /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block
7 R9 e) M( z; I3 Y" k( y @[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block5 Z# | s% U, z
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2( Y# {# y: m# D$ ?3 L
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
- {' E5 a; `, L: E5 p; _9 `* Y[compute02][WARNIN] Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada
; e! p" l. l# F$ z+ a" n3 }5 Q[compute02][WARNIN] stderr: Created symlink from [url=]/etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada.service[/url] to [url=]/usr/lib/systemd/system/ceph-volume@.service[/url].- f5 j! l5 G6 [! k* L3 ^
[compute02][WARNIN] Running command: /usr/bin/systemctl enable --runtime ceph-osd@0
; |+ B: N8 T0 R j5 @[compute02][WARNIN] stderr: Created symlink from [url=]/run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service[/url] to [url=]/usr/lib/systemd/system/ceph-osd@.service[/url].
/ ]- ~/ w. Y: H% x[compute02][WARNIN] Running command: /usr/bin/systemctl start ceph-osd@0
( P% V- @* p. e. _5 v7 \3 ~, j) ?[compute02][WARNIN] --> ceph-volume lvm activate successful for osd ID: 0
' }1 O( K( t' v5 _$ Q! m3 {: X% X[compute02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdb
2 y k0 R9 r$ Z; W( u" s[compute02][INFO ] checking OSD status...
5 J9 t4 [9 X# @# s! E[compute02][DEBUG ] find the location of an executable2 E8 _# `6 n# n+ z- B+ J5 K
[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
5 h1 C1 m: U! N0 b[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.
1 X# N) E0 G: \# o9 ~ F. [0 t( l
% i0 ?7 d; f7 _+ ^( {8 c0 v9 P9 |3 q
成功创建。这个坑确实有点过分。ceph存储一定要先解决时间不同步问题，否则有很多意想不到的惊喜：
. z- o/ `& v H1 S. ^1 a* i1 p0 q2 d2 e4 d1 g8 Y2 P
% V# e4 u3 ^8 o% A( Y[root@compute01 deploy]# ceph osd status
, c. U& Z8 F, u# D+ f+----+-----------+-------+-------+--------+---------+--------+---------+-----------++ Q f1 q8 p/ F% M$ R" S* [9 o6 o0 J
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |% U8 e" S$ Y: `* h
+----+-----------+-------+-------+--------+---------+--------+---------+-----------++ h& Q4 Z! _: g9 t% x
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |+ D5 P/ ~5 Q S: K' v1 y( r/ J7 j
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |! k" s" D, c* p, _$ O
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |, i% q7 L. H* w3 }
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+) t D Z, V5 e( C! M
[root@compute01 deploy]#
% H2 O, R, }* U[root@compute01 deploy]# 5 r0 b; p' u$ w
[root@compute01 deploy]# 6 [, W/ t( x+ R$ P
[root@compute01 deploy]# ceph osd status
2 W e/ I* o' Y8 U+----+-----------+-------+-------+--------+---------+--------+---------+-----------+, n8 F# P5 J, b3 y
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |4 |0 t$ D V- c7 I
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+7 d, G: `( i7 M+ q0 z
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
2 _: R/ P1 n1 H; D7 G% b- n1 z1 ]| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |- w+ n' V% h0 D- L5 \
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |+ {2 z% u" w& n$ l* p! _0 U8 w5 q+ S
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+6 K; R5 l' A! ^3 @# [7 e/ c( K
[root@compute01 deploy]# ceph osd status 0 F* Q6 q! ~* n) n7 S% j3 ^
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+9 [# Z- u0 G) g0 T* k/ W
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |! E0 s( Y/ e6 m
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+* S4 S/ t' h" v' L$ k" _, `2 F
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |' X' R* C8 Z) r$ p1 a' `4 C7 g
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |& @& g) M* }' h, M8 d5 D" W
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |" x! W. W- w; U, c2 M) C3 B4 U
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
5 b+ Q' b0 Y$ J" A[root@compute01 deploy]# ceph osd status , Z; Z4 ?/ V: Q( D) b2 e
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+" b2 ^6 a ^" V) c
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |0 S2 w# q1 ]6 n: m$ g: e0 \) h" G& Q
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
H. M" a/ E) }% T: s4 G$ W1 v| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
/ U2 g2 A6 i: ^5 i9 ?| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |% I( X$ ], D! w3 N
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
! R; a+ c# C3 W, d+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
' C4 m4 w) {- J8 {- O( H0 W[root@compute01 deploy]# ceph osd status 2 K3 K' G1 ~3 ?2 `# a
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
# | U, H* K3 t2 ^| id | host | used | avail | wr ops | wr data | rd ops | rd data | state | t4 Q& g& e4 a& j; g0 I
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
& {: v. A$ k6 \& L; n| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |- I! Z/ S2 U- j5 P1 v8 s" ~) T
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |
) m6 p: M# o) n1 {- I| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |' [, A, y+ u1 Y. V# o9 `
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+! W% c& H3 T" O; b t
|
|
发表于 2021-8-31 15:00:09