|
|
[root@compute01 deploy]# ceph status
, |- v( v' {; x$ z6 ?" R5 R cluster:
% W) ]4 {# F& ]) v" R7 n id: 31403b11-8a1e-432f-876e-5a2c852f9dcc
7 m1 M8 F/ q3 r7 l' u3 ? health: HEALTH_WARN
8 G0 ~5 }. n) Y1 W Degraded data redundancy: 13701/41103 objects degraded (33.333%), 432 pgs degraded, 640 pgs undersized
3 X# m. a% I! g 13269 slow ops, oldest one blocked for 8178 sec, mon.compute02 has slow ops
( w+ r" H5 o- z! I' k" m! I clock skew detected on mon.compute02) V5 O9 K% ^9 p8 O
/ w; ?, e5 u8 ?7 z
services:
, n* ?$ L6 w3 z& A mon: 3 daemons, quorum compute01,compute02,compute03 (age 2h)
& A6 C( c7 u+ d3 O% ^* | mgr: compute01(active, since 4h), standbys: compute03, compute02; i+ P, Z2 r: A. m% A! ^$ D
osd: 3 osds: 2 up (since 4h), 2 in (since 2h)
" I4 }$ g) K- d7 u
0 C2 f( `) ?' {: d data:
6 a: s2 i% B+ K [7 A- { pools: 6 pools, 640 pgs; f# i; n0 ^6 Y- L9 n
objects: 13.70k objects, 11 GiB* D. ~6 L; q: ]0 ^. X
usage: 24 GiB used, 2.7 TiB / 2.7 TiB avail
, ^' k* L7 w N( q, t pgs: 13701/41103 objects degraded (33.333%)
$ y. D# a2 S5 v( z; g% l6 Y 432 active+undersized+degraded
& `/ y6 }1 e3 B5 R2 p; q# |; F 208 active+undersized6 P( e K' Q+ g/ ^ E" p* c0 {
# U$ }; F* A1 t8 E9 L2 @" R[root@compute01 deploy]# ceph osd crush remove osd.0
. p3 f8 V5 [ U3 e& |# X+ f/ a) ~: tremoved item id 0 name 'osd.0' from crush map0 [* h ~2 ?5 z3 ]# i) ~' z
[root@compute01 deploy]# ceph osd rm osd.06 {, J- w/ F' Y* b' C3 ^, D& A
removed osd.09 B2 H6 i" K# }5 d0 R6 q
[root@compute01 deploy]# ceph auth del osd.0, Z7 T& t" _ p# X6 J0 y2 l
updated
8 t! c& U3 B1 t, f1 B8 ^[root@compute01 deploy]# history |grep sdb- V" T3 z0 R: q2 i0 c
1023 history |grep sdb) Q! X% d4 H, @- \
[root@compute01 deploy]# history |grep ceph-deploy
( ?2 e3 T1 d5 h: l5 U/ Q! E 1024 history |grep ceph-deploy
! l1 J& h# q1 X& l[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
& X& z/ I' E a! M1 G& _- q[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
+ O9 Y" a) e. k6 [8 F[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb/ d% \% j1 Q G3 [# W! n$ s+ X
[ceph_deploy.cli][INFO ] ceph-deploy options:: M- a- H7 x; d' a8 c6 x& m, D
[ceph_deploy.cli][INFO ] verbose : False8 F5 S: M6 T3 e/ ]3 x
[ceph_deploy.cli][INFO ] bluestore : None$ @, F) y- U3 y% G2 Z1 I' s5 M
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fbd39643690>
. R0 B, J3 B1 K! s[ceph_deploy.cli][INFO ] cluster : ceph6 Y5 N1 f. Z" Q5 J, b
[ceph_deploy.cli][INFO ] fs_type : xfs0 ~* L9 x( K/ _5 Y$ u
[ceph_deploy.cli][INFO ] block_wal : None) `/ p1 N, Z; `# U" d$ Q
[ceph_deploy.cli][INFO ] default_release : False" y6 G* \; n6 D8 M- }. q
[ceph_deploy.cli][INFO ] username : None; D3 O+ f; m7 ?. S" C; m I! }' h
[ceph_deploy.cli][INFO ] journal : None& t% J! \; A0 S8 g0 T
[ceph_deploy.cli][INFO ] subcommand : create& {, ^- {8 q7 M) P3 @6 q, o
[ceph_deploy.cli][INFO ] host : compute02
7 W5 E( k2 l, r) ?6 S) a( ^[ceph_deploy.cli][INFO ] filestore : None
; N ?7 ?0 o1 a5 F5 r& ~: B[ceph_deploy.cli][INFO ] func : <function osd at 0x7fbd396711b8>1 l; p F: ~" S
[ceph_deploy.cli][INFO ] ceph_conf : None C! {3 _- @! ]; w9 ]4 u9 f
[ceph_deploy.cli][INFO ] zap_disk : False2 g- e( U. |- H$ T$ B1 A
[ceph_deploy.cli][INFO ] data : /dev/sdb
; ]1 N, v% c+ H( ~# g[ceph_deploy.cli][INFO ] block_db : None
" Y* A- ]* `; w9 j# l6 F( K[ceph_deploy.cli][INFO ] dmcrypt : False1 M* d/ M$ O' s0 ]# q9 u9 i
[ceph_deploy.cli][INFO ] overwrite_conf : False) }& V, B; [. P* R
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
1 B5 u. ^, S+ Q3 R7 {[ceph_deploy.cli][INFO ] quiet : False
2 I/ E8 f. m' M- `' s[ceph_deploy.cli][INFO ] debug : False
/ t* u+ L: `0 q8 a q[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb) ^; B# E! }$ P. w, A Q7 B# E
[compute02][DEBUG ] connected to host: compute02
P3 B( O7 {& C0 U0 e/ O! q[compute02][DEBUG ] detect platform information from remote host
X$ i# I) R \( o) Z[compute02][DEBUG ] detect machine type! e% g1 f" k) `5 Z {
[compute02][DEBUG ] find the location of an executable; r0 S C1 E! |$ H% B
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
; N, o* _' r+ ]1 y[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
; C! [3 O) h9 N+ X[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf! `' |" ?6 @' C7 H
[compute02][DEBUG ] find the location of an executable; r* o& N" _! ]6 ^0 J7 _, y
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb! v( N* Y! k1 Z9 ~4 ]4 v0 [
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
, W) j) @/ M! [: M+ x" z[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 8bb22eb2-d2e1-4ae4-aa52-cc96422b25bb* L$ ?+ T5 z9 H* \0 J
5 N7 {+ N; C1 x& _& D; Q% b
^CKilled by signal 2.
l% _* ?* I( D; k; K[ceph_deploy][ERROR ] KeyboardInterrupt
7 [! [, I( H3 [7 U {- s3 ~. [7 H[root@compute01 deploy]# ls
1 w) u7 T2 q3 X T+ Dceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
' m5 Q$ @9 S( P: x' s* Gceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring' x. B' i' b7 e4 k4 I
[root@compute01 deploy]#
( w% I" Y: M. I9 Z( u" v; H& d: e+ h[root@compute01 deploy]#
/ \) w' K; |9 c3 E5 {5 H[root@compute01 deploy]# pwd
/ n. g( V p8 V& i' C2 c0 j" j, }/ceph/deploy
* Y8 u' \$ y% F6 G[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb ^C0 B; g p3 E. v6 B5 c1 b( p& H
[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb% f3 G7 z X0 _7 W7 A) m
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
}$ O" N/ d' |9 d; X[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb4 w' H2 E) y0 V/ b
[ceph_deploy.cli][INFO ] ceph-deploy options:
% }* }2 |3 k, D; I[ceph_deploy.cli][INFO ] username : None
/ |4 U" w% F5 S; F' F5 K7 I/ [[ceph_deploy.cli][INFO ] verbose : False
. O: w& M6 m( p, }[ceph_deploy.cli][INFO ] debug : False
* c/ A. K' |) I[ceph_deploy.cli][INFO ] overwrite_conf : False
5 Q. ^& U& C+ l5 H2 j( r* _[ceph_deploy.cli][INFO ] subcommand : zap
7 Q: [4 u) r/ d, h[ceph_deploy.cli][INFO ] quiet : False
! U; c8 Q: i4 |+ k& y[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f0db9e1dad0>
2 c. q- R% W8 k' S( }[ceph_deploy.cli][INFO ] cluster : ceph
$ t; j. L: p" c: p3 }( v[ceph_deploy.cli][INFO ] host : compute02
# d( u: X8 W+ G[ceph_deploy.cli][INFO ] func : <function disk at 0x7f0db9de3230>
0 h! X; P: ]/ H3 r' m: W, \; f[ceph_deploy.cli][INFO ] ceph_conf : None$ K$ X( l) N1 O( z
[ceph_deploy.cli][INFO ] default_release : False
/ g9 V" Y0 c7 v) s# S" ?8 t$ W( H[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']0 t* N4 [4 X D7 B0 E
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
3 q+ T& b- p6 x( Y% g8 Q( J[compute02][DEBUG ] connected to host: compute02
; R2 V) @- @4 @" |[compute02][DEBUG ] detect platform information from remote host1 p, w9 h8 \3 R Y: y
[compute02][DEBUG ] detect machine type( Q# w8 i! {2 r1 L. ]
[compute02][DEBUG ] find the location of an executable
/ Q @0 O9 g! G( i3 m/ @4 W[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core( f0 \5 X1 G1 ~0 K# @7 q
[compute02][DEBUG ] zeroing last few blocks of device$ k: c: Y' i1 @
[compute02][DEBUG ] find the location of an executable+ _& }$ ~- d" R' J6 `7 }
[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb2 O u6 U7 C0 N' M: X* p- w
[compute02][WARNIN] --> Zapping: /dev/sdb
* r! k% h% |' V7 {[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table
1 o& v# Y! s* G4 K[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync8 Y) Q9 w! [9 P9 C. d. D7 C8 a' ^# v9 R
[compute02][WARNIN] stderr: 10+0 records in' }6 N) }! ^- E& M, R% l
[compute02][WARNIN] 10+0 records out
4 L2 D' O1 T+ I* x: ^$ o[compute02][WARNIN] 10485760 bytes (10 MB) copied' f- I. [3 ]( d" T1 ]; g
[compute02][WARNIN] stderr: , 0.0850991 s, 123 MB/s
0 G- v1 l; R# Z/ d0 W7 ^& @[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
) ^" |; ~" G X$ S" x. ^9 }[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb6 _: e9 J5 W% r! H; K O- j
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
; W) x M% {; G1 M* i2 T% I2 [[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb. |3 o3 B( ?# Z$ z7 n1 q: R4 i+ w
[ceph_deploy.cli][INFO ] ceph-deploy options:
' m5 i( `2 Q; Z[ceph_deploy.cli][INFO ] verbose : False
& j# W3 S" \! G6 a. `[ceph_deploy.cli][INFO ] bluestore : None T: L( `6 Q z1 e7 d9 {
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f45a0a92690>' P- z ^- j3 X! p' w
[ceph_deploy.cli][INFO ] cluster : ceph+ |9 f" d+ ]) J
[ceph_deploy.cli][INFO ] fs_type : xfs
8 I0 i& I1 l5 P8 m[ceph_deploy.cli][INFO ] block_wal : None
9 P6 _0 L. O9 r% N[ceph_deploy.cli][INFO ] default_release : False+ {# P: l" y4 y# e+ ~) q, W
[ceph_deploy.cli][INFO ] username : None& E d& v! C& r d, B' f
[ceph_deploy.cli][INFO ] journal : None
, P0 Y, a4 Z* k, M[ceph_deploy.cli][INFO ] subcommand : create
, S1 x& U) i6 t2 G. q[ceph_deploy.cli][INFO ] host : compute02+ g% J3 Q( M, K
[ceph_deploy.cli][INFO ] filestore : None( T/ P: g: M8 [' r t" F: b0 f
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f45a0ac01b8>9 O7 l6 A+ G! p. x
[ceph_deploy.cli][INFO ] ceph_conf : None; `& B/ O& Q3 v- _3 y
[ceph_deploy.cli][INFO ] zap_disk : False
& ]1 z1 H6 o% U, Z8 O& A[ceph_deploy.cli][INFO ] data : /dev/sdb* w+ S! F. p! d+ ~$ I. d; @1 |
[ceph_deploy.cli][INFO ] block_db : None& M, z S3 X$ \. G( z
[ceph_deploy.cli][INFO ] dmcrypt : False
1 G1 i2 C2 [' T2 d[ceph_deploy.cli][INFO ] overwrite_conf : False$ Y5 d# z* i3 u- }: O: u; u# W0 b
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys; ~' Q9 B# K) B$ ]8 K# ~
[ceph_deploy.cli][INFO ] quiet : False$ L0 S( ?& s9 d- h3 H; T
[ceph_deploy.cli][INFO ] debug : False) P: s; u6 v: u3 q* s8 q& v
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
, A( p# N" L' z) V7 c/ M3 E d[compute02][DEBUG ] connected to host: compute02
$ w+ h: Y4 E1 @& k/ K, ^[compute02][DEBUG ] detect platform information from remote host+ r( J" n* H* C$ E2 ^ v7 K6 H2 g2 P
[compute02][DEBUG ] detect machine type( [# L9 ?- Q9 b% V* @
[compute02][DEBUG ] find the location of an executable0 J/ ^& x* P- O( j0 E
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
) l1 D7 V) b& T$ D4 X# x[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
- Y& Q* ]4 ?2 D7 f) X/ \; B% t[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
9 w1 O5 h% F1 t! {) `& t[compute02][DEBUG ] find the location of an executable
0 j9 @* u% h1 O' d! J[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb) b, ~; k, f( I
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key0 M5 O: Q5 f8 a
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9f1fca57-1a1c-4d30-bad8-2d4578f88e96
* K# D4 [! ~4 u h/ s) _4 e^CKilled by signal 2.
( v" x$ r) s3 B5 j, b3 Z, V6 ^7 w[ceph_deploy][ERROR ] KeyboardInterrupt4 j1 s4 V; d9 Z7 L) F
[root@compute01 deploy]# ls
/ Z* ~1 J! x7 z- s' W# Z/ Xceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log. S* Q2 k- U) j" c( l& e. q
ceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring
. `% v; ]. x" O) Y. V[root@compute01 deploy]#
+ x) X9 G7 \' _( o[root@compute01 deploy]# 5 P: s0 u+ b+ N& x5 m, Y0 D- x
[root@compute01 deploy]# 9 t8 G: J1 q( e) u7 b
[root@compute01 deploy]# ceph-de
! J* a, S! i( P$ iceph-dencoder ceph-deploy
; J/ ^( U: t4 t P3 u2 r: ~- V7 {& J0 G[root@compute01 deploy]# ceph-deploy + z A# d2 v7 T' z
usage: ceph-deploy [-h] [-v | -q] [--version] [--username USERNAME]
+ p+ K& B7 }8 \& Y4 K [--overwrite-conf] [--ceph-conf CEPH_CONF]* e* _# V4 c0 v
COMMAND ...
' i% y: y+ m* ]* M( fEasy Ceph deployment
' U0 N3 ]% e& o% u9 W -^-0 Q$ E, Y6 F) e
/ \
1 |1 o2 |* d9 M; b5 A |O o| ceph-deploy v2.0.1, D. g+ O5 e- _
).-.(. ?/ ?& K" I! w$ j( p! }7 n
'/|||\`
2 Z+ b+ u' z/ R1 W* A) |6 j | '|` |+ n% F/ ?' r6 S& ~0 f. d
'|`8 i2 r8 S) H- ~' z, w' d; s* n
Full documentation can be found at: http://ceph.com/ceph-deploy/docs
8 q+ F& K% z5 `; e( q8 loptional arguments:
% B8 n. h0 y1 i5 [ -h, --help show this help message and exit
: p: y# c* |. V4 S -v, --verbose be more verbose
& Y/ |0 a6 z5 U2 d7 u l -q, --quiet be less verbose
3 `' |) Z' U s4 Y+ }: F --version the current installed version of ceph-deploy/ T$ H/ |" h+ V3 J5 B- x
--username USERNAME the username to connect to the remote host% c0 l1 {7 G9 R
--overwrite-conf overwrite an existing conf file on remote host (if
; {0 J% }' m2 d' N% q present): T' g5 }- @7 p1 w; h
--ceph-conf CEPH_CONF
% @) B. \5 q# c8 W use (or reuse) a given ceph.conf file' O/ Q$ ]$ h0 f& S+ u$ p& w# z
commands:
3 D7 j. U# w; l) r COMMAND description6 Z- B9 m: Z1 T+ o N: g' m9 t
new Start deploying a new cluster, and write a' `1 r$ E+ w4 h3 e! q5 K4 s+ {
CLUSTER.conf and keyring for it.( P' I" `0 p) [6 q1 J
install Install Ceph packages on remote hosts.' S! P7 {7 ~9 x
rgw Ceph RGW daemon management; [3 J. U! _. s+ V; T! ~& }
mgr Ceph MGR daemon management
; y( B& O# h* Z7 a- M+ n* A4 D mds Ceph MDS daemon management
$ |( w+ ^! P0 m9 |4 ~# @- ~ mon Ceph MON Daemon management
& F, n5 Z& B' ` gatherkeys Gather authentication keys for provisioning new nodes.% [2 v9 s! k3 M
disk Manage disks on a remote host.
+ a ~9 Q% L. z8 p1 ?8 l8 c osd Prepare a data disk on remote host.
* F0 I& _5 S2 q8 \: M/ ?( M repo Repo definition management
# i, ~8 Z' S4 H+ `. h admin Push configuration and client.admin key to a remote/ h4 t5 {8 \( J! Y; H
host.* m/ t- w) b" z0 }
config Copy ceph.conf to/from remote host(s)
: o' }* k0 M' v uninstall Remove Ceph packages from remote hosts.
I& y w( @( G: Q purgedata Purge (delete, destroy, discard, shred) any Ceph data
- M' Z3 ]# V' a from /var/lib/ceph
% i2 K8 p$ l# [" U: c# u: j* c purge Remove Ceph packages from remote hosts and purge all$ A) Q# V$ @$ r+ U) ]
data.
% E7 C, t( [4 w C' Q7 C forgetkeys Remove authentication keys from the local directory.
) a' u# D/ I# `7 a pkg Manage packages on remote hosts.
* J( V# |4 U% e2 O/ V/ W! L8 |) ] calamari Install and configure Calamari nodes. Assumes that a
; u# x- ]1 i7 ~" X" W( X" F repository with Calamari packages is already' ]0 K9 x2 T: w% x3 e# J0 l' {
configured. Refer to the docs for examples
- j( M+ R6 v( N8 P (http://ceph.com/ceph-deploy/docs/conf.html)
: ~5 V3 `8 H' X; ASee 'ceph-deploy <command> --help' for help on a specific command
2 O( E4 ^# D8 q8 N+ i[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb
( m+ q& Q" [8 T, E5 k9 V[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
7 n4 ]0 b& E8 d5 \4 G3 f- A7 q/ J, U[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb9 }+ p6 v+ l( O/ N
[ceph_deploy.cli][INFO ] ceph-deploy options:6 m, v: z2 @( p! b2 R; T, j9 t0 f
[ceph_deploy.cli][INFO ] username : None# B4 p! r o( B2 r3 l- I
[ceph_deploy.cli][INFO ] verbose : False
' L# h% i0 ?' s3 T$ h[ceph_deploy.cli][INFO ] debug : False
6 x) I; c$ ^3 P( P \3 p' s[ceph_deploy.cli][INFO ] overwrite_conf : False3 j5 I7 Y7 t, x0 ]; R, o$ S
[ceph_deploy.cli][INFO ] subcommand : zap3 N* [0 G( I: f' O
[ceph_deploy.cli][INFO ] quiet : False; X0 s; l# |. V$ c& G, b
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f1c11596ad0>
@$ G4 i& C, N0 E5 d8 h5 o[ceph_deploy.cli][INFO ] cluster : ceph
, H; q4 O7 J7 ?: y[ceph_deploy.cli][INFO ] host : compute02. J/ F' V% r) x" F
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f1c1155c230>5 W. Q& q$ {4 E4 G& l0 C
[ceph_deploy.cli][INFO ] ceph_conf : None
3 }4 A! w; i1 Z2 {[ceph_deploy.cli][INFO ] default_release : False% y2 W9 _% J8 `3 \7 v* @
[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']. S5 [5 n9 f. K9 S4 U) C
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
* _! g: f$ i* z& n9 A; s: G[compute02][DEBUG ] connected to host: compute02 3 U$ z8 u; `5 r& Z: X P
[compute02][DEBUG ] detect platform information from remote host1 v* c+ @' f1 q* ^, r% [6 Y0 u
[compute02][DEBUG ] detect machine type
% t$ X" L, r4 S9 X+ Y# d* M0 @[compute02][DEBUG ] find the location of an executable% K K4 N+ J5 H" P, z$ v
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core' z6 Z7 f3 F! M8 W4 P% B9 ]! m
[compute02][DEBUG ] zeroing last few blocks of device4 L9 u% K$ _7 b* g* E# _2 G
[compute02][DEBUG ] find the location of an executable
& R0 Z) @0 O1 C! a5 O$ x9 t, s[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb* p" A$ j2 Z1 g5 q* D) |( B
[compute02][WARNIN] --> Zapping: /dev/sdb
( ~; e$ A" q: f3 x1 M1 ][compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table# ~, O& A9 F! C! f) Q' Q3 h
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync; G9 x7 Y% G5 A0 s
[compute02][WARNIN] stderr: 10+0 records in7 F; A: e1 m' O1 l# @3 c
[compute02][WARNIN] 10+0 records out0 R/ W* W, X5 r) J5 `2 a: r
[compute02][WARNIN] 10485760 bytes (10 MB) copied( I C: w" S' q U) V
[compute02][WARNIN] stderr: , 0.0591334 s, 177 MB/s
t& n& F( K% o3 j6 }[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>) b9 w2 X) |" ]4 l( ?1 F8 c
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
6 S* W, G) k& k: E& @! @, Z: J, e) k8 C[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
8 w5 F& {9 \4 X: k% K[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb/ k* {2 E# W% c' r
[ceph_deploy.cli][INFO ] ceph-deploy options:
" ]" w5 n1 ^( m5 z7 C+ N5 Q2 r[ceph_deploy.cli][INFO ] verbose : False
/ U4 C) P/ _+ m4 w. w! t[ceph_deploy.cli][INFO ] bluestore : None
5 l! K3 h! T4 I[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f23c40b9690>4 c( X3 x! `& s: T3 m9 E
[ceph_deploy.cli][INFO ] cluster : ceph: h n) N6 C% h9 x8 i. O& x9 e" Y
[ceph_deploy.cli][INFO ] fs_type : xfs
. m2 u; A* R, _$ `2 D, k* Q$ U[ceph_deploy.cli][INFO ] block_wal : None% @8 u% J& A# y1 ^5 e
[ceph_deploy.cli][INFO ] default_release : False
, L7 S5 _2 ~4 C" x4 U( u[ceph_deploy.cli][INFO ] username : None' {0 F. A& G2 w$ j
[ceph_deploy.cli][INFO ] journal : None
1 R4 U% w( V# n# w[ceph_deploy.cli][INFO ] subcommand : create& ]4 e, ~ p2 A8 B! Q; E! m2 P
[ceph_deploy.cli][INFO ] host : compute02+ ]! @, |6 J9 b. C. x
[ceph_deploy.cli][INFO ] filestore : None9 `6 s# i% T$ ]
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f23c40e71b8>) H U2 g/ D3 a! Z$ g u; C1 K. I& d
[ceph_deploy.cli][INFO ] ceph_conf : None
: \. C9 V" p* @' x; }4 ~[ceph_deploy.cli][INFO ] zap_disk : False. E" n$ f- a8 _* I" E! F
[ceph_deploy.cli][INFO ] data : /dev/sdb4 g) O( D& Z; \
[ceph_deploy.cli][INFO ] block_db : None
. d8 i! _4 H1 ~1 D[ceph_deploy.cli][INFO ] dmcrypt : False
# f" V: s. x* ?[ceph_deploy.cli][INFO ] overwrite_conf : False7 O* a0 U) o \ \, x! L
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys7 {! L! H% j0 V4 Z) t/ K- A4 [! w: K
[ceph_deploy.cli][INFO ] quiet : False
! `5 [' B2 e% q t1 {) [8 D8 x) U[ceph_deploy.cli][INFO ] debug : False
; ~4 J. D! d% L* `[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
7 H$ ~' S9 q* ^9 A# U9 d# a[compute02][DEBUG ] connected to host: compute02 % o) h, L$ E; ?) s; E4 c
[compute02][DEBUG ] detect platform information from remote host
$ l( P7 o7 l) f z[compute02][DEBUG ] detect machine type# }- s: G! j/ K: u5 k3 A$ \
[compute02][DEBUG ] find the location of an executable
# s1 R& H4 ?# k5 C5 ?[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core. i( u# _* S: F# j6 s
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02% g' C8 o+ ?' \6 ^
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf$ Y+ d; R0 i+ k: |
[compute02][DEBUG ] find the location of an executable
! [, _4 ^4 e- E5 _- o. z$ A[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
0 f! X1 D$ h" M1 H- T* P[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
0 s: {/ \# p+ c/ ?/ |9 z7 F- C[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new fab4f939-9a55-4e45-94d5-2434dbb247662 S# A" Z, |$ S# k$ s
^CKilled by signal 2.
& e+ L" X& C6 D/ f8 i( t" G2 l* q[ceph_deploy][ERROR ] KeyboardInterrupt
) Y* P* m4 A& u0 \0 j[root@compute01 deploy]# ls7 F& t% w V. d5 i3 X/ x j
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
: N9 k* r2 Y+ ^2 I, `ceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring
, b: _+ ]( b/ T8 [, ]5 P! d4 d# y[root@compute01 deploy]# vim /etc/ceph/ceph.conf
8 f0 T4 r2 D' K9 L) ?: ~[root@compute01 deploy]# vim ceph.conf # U& c3 H H+ G5 F
[root@compute01 deploy]# vim ceph) S6 ~# ]$ |6 a1 ^1 B# D, H
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
, x/ t$ O, ]8 R. k2 A" ?" nceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring 3 l h! l8 @' ?3 x$ J" s
[root@compute01 deploy]# vim ceph.mon.keyring
* y* f! [" N" z9 E( h1 x[root@compute01 deploy]# vim ceph.client.admin.keyring
& W X! A. O% u+ V( i" w[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb7 }/ l: m2 H* p/ }
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
' S S! J" G' I2 f, I Y[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb
7 I2 I1 \; u. ]. v[ceph_deploy.cli][INFO ] ceph-deploy options:
, F& t) a2 D4 v% d[ceph_deploy.cli][INFO ] username : None
. O- i9 W- T$ D: i* c% ?[ceph_deploy.cli][INFO ] verbose : False
+ O" }# U6 \# r! s0 k[ceph_deploy.cli][INFO ] debug : False, R8 V- d8 C( I- w# t" R# _3 r1 }; K
[ceph_deploy.cli][INFO ] overwrite_conf : False
" ? D* m( o- F6 i* H[ceph_deploy.cli][INFO ] subcommand : zap) [& I8 {: ~5 G- r8 U
[ceph_deploy.cli][INFO ] quiet : False6 I( y, z% u' ]3 k4 J
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fa10739bad0>8 n1 g+ [, `) P4 x- ?
[ceph_deploy.cli][INFO ] cluster : ceph
' ?; E8 M; k' I" |! X[ceph_deploy.cli][INFO ] host : compute02
4 e& _. q8 e2 _[ceph_deploy.cli][INFO ] func : <function disk at 0x7fa107361230>
" P4 K! U, c" u( u; d! ^2 E, ^[ceph_deploy.cli][INFO ] ceph_conf : None* V( J$ T, I1 q) O
[ceph_deploy.cli][INFO ] default_release : False
: q- u3 q4 w0 s4 C+ K[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']
9 `; r6 N' p: E, i[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
$ o! S" A% c ~% a5 ? A" K) Y[compute02][DEBUG ] connected to host: compute02 - }! {" O k3 Z" e
[compute02][DEBUG ] detect platform information from remote host
1 c/ P+ H+ Y% i& s6 k( O[compute02][DEBUG ] detect machine type
8 @0 { j0 I- _6 | t9 k% ]; t[compute02][DEBUG ] find the location of an executable
+ w- B& p4 t4 M2 M* y[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
% {/ Y' C* v/ `% B[compute02][DEBUG ] zeroing last few blocks of device c% |9 `! L- n/ ]' m0 G
[compute02][DEBUG ] find the location of an executable1 l1 ?2 }% K* ~9 A- ?0 O
[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
6 f: X/ c9 L3 v" [[compute02][WARNIN] --> Zapping: /dev/sdb
, Q/ q/ @' |; Z[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table
0 K6 e3 r% L8 n& q+ U& J# }7 P7 ?[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
+ Z* t5 S9 i. ?, l# j% r[compute02][WARNIN] stderr: 10+0 records in9 g/ r5 A4 I6 ^4 x6 R z4 s
[compute02][WARNIN] 10+0 records out
( l8 ?9 `, U& g% r% L+ ?, B: z[compute02][WARNIN] 10485760 bytes (10 MB) copied- ` t) k) |( u
[compute02][WARNIN] stderr: , 0.0869923 s, 121 MB/s
0 i7 i+ {/ S' l9 S[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>
: b7 P0 ~/ C% w2 R- X[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
5 d+ m- o0 \6 k9 x) I+ O[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf- S) T' A* \1 Q/ X3 d; V) I
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb8 H ?7 H, x$ T4 g
[ceph_deploy.cli][INFO ] ceph-deploy options:
1 P7 i8 L6 ]+ P! z3 n i) p0 L5 d4 Z[ceph_deploy.cli][INFO ] verbose : False
) k$ U) v1 U7 m% b) \$ i k[ceph_deploy.cli][INFO ] bluestore : None
# A7 K* b6 v+ |' ][ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f13835dd690>$ `+ _0 d! l" q, B
[ceph_deploy.cli][INFO ] cluster : ceph
' J, Y: c* w- V[ceph_deploy.cli][INFO ] fs_type : xfs
8 k1 a2 q& W _6 I6 T) \% y[ceph_deploy.cli][INFO ] block_wal : None
$ P, a4 z! E, Z7 e& y1 j8 K1 ?[ceph_deploy.cli][INFO ] default_release : False
# K/ s+ b$ T) J[ceph_deploy.cli][INFO ] username : None& U: @7 n7 O% G% w! ^& O* F0 J
[ceph_deploy.cli][INFO ] journal : None
; Z f0 p$ j* R( |: B" F[ceph_deploy.cli][INFO ] subcommand : create
, p0 n7 R8 U' }) ?6 D8 w' r- ?0 ^[ceph_deploy.cli][INFO ] host : compute02& N: n' T6 R- [- M" r2 M0 S
[ceph_deploy.cli][INFO ] filestore : None5 E" I2 I% [9 K
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f138360b1b8>
. K, N4 Z% E- k) M5 t _[ceph_deploy.cli][INFO ] ceph_conf : None: u+ z' S2 E3 z
[ceph_deploy.cli][INFO ] zap_disk : False' U, w/ n' q! z
[ceph_deploy.cli][INFO ] data : /dev/sdb
& m7 J) Z- Z6 ^3 X. f0 s[ceph_deploy.cli][INFO ] block_db : None
: O1 E4 V5 t6 a# G% f; Y( ?9 w/ J% ][ceph_deploy.cli][INFO ] dmcrypt : False; G( j1 h1 q# }6 e/ S
[ceph_deploy.cli][INFO ] overwrite_conf : False- z1 U& a$ V v$ _7 C. B8 w$ [
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
" \; H. A& S# w: L F) m[ceph_deploy.cli][INFO ] quiet : False
0 ^" G( X; \, R" p- V[ceph_deploy.cli][INFO ] debug : False
8 h) k& l4 _/ |5 g: s+ \ E I9 b* @# n[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb0 Z) u* I9 L$ o' {1 u
[compute02][DEBUG ] connected to host: compute02
: E; J" [) `& H[compute02][DEBUG ] detect platform information from remote host
6 R: {) J1 e: U1 m5 c[compute02][DEBUG ] detect machine type
+ ^3 y1 m, R: X" f) g[compute02][DEBUG ] find the location of an executable* a7 d# }. x1 ]$ {4 `' ~% r {
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core9 f; j7 _% \3 g+ o5 q2 X7 N0 ]
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02( A( ?+ B8 T, Z5 Z, v8 t1 [+ r
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf! s- M* Z. D0 C) }7 v
[compute02][DEBUG ] find the location of an executable: t8 C8 K& R1 b! u0 V) r, l6 l
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
9 O3 A( N. X7 l) P[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
, h6 s% w9 c/ h[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new c71415c4-fdaa-4955-8c26-71d160c6121e
" b2 E/ a' R: S7 ?/ `# h/ W6 P# h[compute02][WARNIN] No data was received after 300 seconds, disconnecting...
0 Y2 r3 a3 j* K6 @& W, l- i, x[compute02][INFO ] checking OSD status...0 t# r& _' D5 W* E7 ?2 r
[compute02][DEBUG ] find the location of an executable- A+ x; \* ?: m0 Y6 J
[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json: W; K* F" e# W1 `* c* M
[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.
9 m6 f$ X% O! A8 E. w6 n[root@compute01 deploy]# ceph osd tree
. y1 J l9 ]6 G0 ], O/ X0 EID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
`% ]8 |5 p L+ k$ ?2 {-1 2.00000 root default $ s' d# f9 g# @7 G6 N. h
-5 1.00000 host compute01
* q. y, ^1 d2 K 1 hdd 1.00000 osd.1 up 1.00000 1.00000 4 K+ G$ B6 I0 ^3 J F" u# p6 Z
-3 0 host compute02 2 q9 D6 N* K6 [+ j& W6 y& N2 y
-7 1.00000 host compute03
+ f) f! p9 j5 y- d; n8 C 2 hdd 1.00000 osd.2 up 1.00000 1.00000 4 p' X2 E# t3 L6 u: S9 z7 I$ s
0 0 osd.0 down 0 1.00000 3 v# T4 [" K8 b
3 0 osd.3 down 0 1.00000 2 c* R9 r2 `( B4 _# R7 Q( Q: E
& T/ e: N# A4 G) n在另一台节点上：5 t" d$ |9 q Z% H
[root@compute02 ~]# ceph osd tree3 h& C! m8 F+ S# H5 S# Y
3 W M2 w$ r$ e" W% V; r0 O# E& U) j8 c$ z2 Q) O
d^CInterrupted2 ]& _. d, D/ m4 O: Z% h
Traceback (most recent call last):: z2 K" P' ]- M" A% B; ]
File "/usr/bin/ceph", line 1269, in <module>
! V; a" j% q' f) B retval = main()! ]) ^& g8 Q! v3 f
File "/usr/bin/ceph", line 1200, in main
( V$ B# V6 `5 Q! S( Q. o verbose)2 \: f3 l' B$ h* G: _( x% f; R
File "/usr/bin/ceph", line 622, in new_style_command
1 r" U- w6 ^' j ret, outbuf, outs = do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose)
" B0 \" g$ q1 A; \& S File "/usr/bin/ceph", line 596, in do_command
' }5 [: E/ q8 |" i$ ? return ret, '', ''$ |0 r p/ g3 D: d- l" X
UnboundLocalError: local variable 'ret' referenced before assignment
: b i& @5 G, |4 K[root@compute02 ~]# date
4 w1 m: Z5 S/ f! K
5 L- ~6 a0 D! t& t. g4 z) c连osd tree都不能查看：
. o* Y, [( @: X没办法，只能查看日志：! T: K6 {1 c7 c# r
0 Y, K3 W- E' [7 l$ g m
5 B6 g* [2 O- f( a- c4 t
ceph的log日志视乎没有任何帮助，和ceph -s的状态一样。
6 L+ \7 V, m! Y/ X+ _# L[root@compute02 ~]# tail -f /var/log/ceph/ceph.log
( F5 _' T9 f+ _* Z) g5 c9 }- C2021-08-31 14:13:37.918792 mgr.compute01 (mgr.49274115) 15887 : cluster [DBG] pgmap v8821: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
9 J2 V3 ], m* F2 t8 r+ @- ^6 g. J/ A2021-08-31 14:13:41.694764 mon.compute01 (mon.0) 717737 : cluster [WRN] Health check update: 528 slow ops, oldest one blocked for 477 sec, mon.compute02 has slow ops (SLOW_OPS)0 n' Z5 v( z( W, T3 h
2021-08-31 14:13:46.695723 mon.compute01 (mon.0) 717740 : cluster [WRN] Health check update: 535 slow ops, oldest one blocked for 482 sec, mon.compute02 has slow ops (SLOW_OPS)4 Q& Y4 j s) g5 ]' X7 k0 Y
2021-08-31 14:13:39.919835 mgr.compute01 (mgr.49274115) 15888 : cluster [DBG] pgmap v8822: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
% n2 f" K! Z, ?! p3 p/ `2021-08-31 14:13:41.921160 mgr.compute01 (mgr.49274115) 15892 : cluster [DBG] pgmap v8823: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
! v% J5 _; e: J# B( r9 s* u6 E0 x- m2021-08-31 14:13:43.925789 mgr.compute01 (mgr.49274115) 15893 : cluster [DBG] pgmap v8824: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)% }3 Q3 k5 @- }
2021-08-31 14:13:45.927393 mgr.compute01 (mgr.49274115) 15894 : cluster [DBG] pgmap v8825: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)
4 U* D/ }+ `9 I, J( s/ ?, P2021-08-31 14:13:47.930528 mgr.compute01 (mgr.49274115) 15895 : cluster [DBG] pgmap v8826: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)3 X; a h6 t; z! v) o3 v5 I
2021-08-31 14:13:51.696665 mon.compute01 (mon.0) 717744 : cluster [WRN] Health check update: 544 slow ops, oldest one blocked for 487 sec, mon.compute02 has slow ops (SLOW_OPS)
3 X, |/ z! q* Y7 Z' n' a4 \2021-08-31 14:13:56.697487 mon.compute01 (mon.0) 717766 : cluster [WRN] Health check update: 547 slow ops, oldest one blocked for 492 sec, mon.compute02 has slow ops (SLOW_OPS)+ L! s& Z6 x4 z, l, s7 W8 C
2021-08-31 14:13:49.932275 mgr.compute01 (mgr.49274115) 15899 : cluster [DBG] pgmap v8827: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
% M% V, @( S) W: E! ^% a2021-08-31 14:13:51.934115 mgr.compute01 (mgr.49274115) 15900 : cluster [DBG] pgmap v8828: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
5 U6 z: t7 K5 {7 W2021-08-31 14:13:53.938702 mgr.compute01 (mgr.49274115) 15901 : cluster [DBG] pgmap v8829: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
' s1 j# m0 ?, a2021-08-31 14:13:55.939684 mgr.compute01 (mgr.49274115) 15905 : cluster [DBG] pgmap v8830: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
, B z4 L7 @: D( Q1 a2021-08-31 14:13:57.944025 mgr.compute01 (mgr.49274115) 15906 : cluster [DBG] pgmap v8831: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
6 t/ s0 D o" Q6 Q' V$ }8 l8 G# }2021-08-31 14:14:01.698441 mon.compute01 (mon.0) 717769 : cluster [WRN] Health check update: 554 slow ops, oldest one blocked for 497 sec, mon.compute02 has slow ops (SLOW_OPS)
( S, ]* V' N# @6 c, [3 V3 `! b^C
4 T0 z6 w: E# j- e) A+ t* ~
v# R& @! i5 Y0 A" G* ]7 D, b: D$ V$ F: A4 ^3 |5 y
查看ceph-osd.0.log时发现 osd.0 553 unable to obtain rotating service keys; retrying，意思就是时间不同步，不停的重试中：
/ F) Y/ q+ N0 K6 M' V% o8 Q
% N: M6 j! m5 L" M. X0 b. m; b, f6 I5 q/ n[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
2 n( T* b. g+ Z* E! hceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log
: P- U$ o# I8 f- ~/ |" Iceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz: C `* n( e y
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd., u2 m* ^9 W/ A& A: `6 |4 ^
ceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log
6 d& F( X/ \/ ~3 q5 A4 [ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz: u+ g9 z* R4 _
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log
% E9 F9 V/ r3 ~/ E' R2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
# e/ v& `4 q1 d; v2 l# U2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying/ y, i; R4 l+ Y; v
2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
2 B$ N* Q" r- x1 r4 D9 z2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying+ W/ c5 S# s; B% l3 t u6 G
2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30. e) u) e: P: f: j5 e
2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
6 r; k; Z l5 P) x; C7 m2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
" u; ]+ z+ k' {# h# r, {# ]1 g2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
5 ~* f. H, I( ~$ k2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
8 k* ]6 J9 Y, x9 q. b2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying1 _* I+ p/ o3 g
3 g( i$ S; c% A; ?) z9 M
^C$ r0 c& L8 n2 I0 |3 W2 l
/ |2 U3 ]8 m9 m( N9 g
6 f" N3 k @3 G) t. E) s' _: c0 s t
/ }' S3 L4 Q- P: k" G5 Q5 R^C^C
; H! c) s2 M% @: x! _2 Q/ t# i[root@compute02 ~]# ^C
3 u9 V& }% \9 E, m2 b3 a[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.. o' S1 ]3 D6 D+ J1 N8 o3 N) v
ceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log5 e% M) w0 A4 s3 a5 t" ^
ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz
0 I/ i. X, ]( {[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log
$ k7 f$ y- N6 w) c6 x1 ]2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
+ {- V( ?# `, }: N2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying, v5 |' ^6 c" Y" z: Q
2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
$ L! c/ ~9 L: [9 B( Y/ d. z( `2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying8 r" |) I" i6 w( `
2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30- D' O$ x0 E; v& ~
2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying/ R4 m, `; ?) F" u- g# ~
2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
$ @. x: z8 Z; x4 ]6 u/ h2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying! j s- l5 b1 y t
2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
# l' ^' V) e0 M) _& \2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying2 d) H; b# _& N
^C6 ^+ ]* v! v1 |
7 p3 C! o9 N. l) h
6 [/ p, H( m- r. r4 A P
同步时间吧：虽然安装的chrony时间同步，但我们同步的时候，还是使用ntpdate比较快些：9 V$ }$ j8 \% q5 J! B3 `9 S
. x$ Y9 U$ W' F! j. ]2 u[root@compute02 ~]# ntpdate -u compute01
2 o' y4 i4 B! r) [+ f31 Aug 14:16:21 ntpdate[6103]: step time server 192.168.0.75 offset -14.901748 sec
, n0 ?+ W4 x. |, H
, ]3 ^( B4 w$ \' b+ I同步完成之后，发现osd tree 命令好使了：
! |3 a! s% A' ], r
; E2 w: t, b2 Z2 J- m[root@compute02 ~]# ceph osd tree
( }) v2 X1 F& e/ JID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
3 {' r; e, [! ]; n j0 q0 F4 L-1 2.00000 root default 7 R+ M% F$ n' l/ F& V
-5 1.00000 host compute01 , }/ J% F7 }; {9 u. [
1 hdd 1.00000 osd.1 up 1.00000 1.00000
1 o, g# G( n+ W, x-3 0 host compute02
5 V h" X7 m9 n-7 1.00000 host compute03
- S- {0 }7 Y# Y0 W7 @& ]& ` 2 hdd 1.00000 osd.2 up 1.00000 1.00000 + L* i- C, M: K" t6 M" }6 Y' A
0 0 osd.0 down 0 1.00000
, l: i, Z8 g+ E1 c; f 3 0 osd.3 down 0 1.00000 ( Y( F' l1 w0 q$ M$ U& u2 X
# f0 i1 t9 l7 G% u/ B, Q把down状态的osd清理掉： v8 l# M7 e/ U: c% }, f
# c! `; G& p/ T1 W& X2 L6 n[root@compute02 ~]# ceph osd out osd.3
K. Z$ v0 m: _5 M }# r- u7 F, Q+ ?osd.3 is already out.
2 F7 b! _' K" |) H[root@compute02 ~]# ceph osd rm osd.3
5 Q5 z! t+ ], S5 w, Kremoved osd.3; A3 d0 a9 B7 d1 N6 ^# x
[root@compute02 ~]# ceph osd crush remove osd.3
, B9 b( ]# j% U) V% I* ydevice 'osd.3' does not appear in the crush map
! ^4 Z+ p# k! c% D8 g& u6 i[root@compute02 ~]# ceph osd rm osd.3
6 S4 Q9 ]) R: o! n$ yosd.3 does not exist.
* F. y, P5 [6 }8 l- N$ j/ h6 K[root@compute02 ~]# ceph auth del osd.3 u) ]$ t( ` U* E% t8 b0 P
updated3 e7 Q% @- G( [! u. @# G
i* v6 {. d8 p# @& P3 |0 {
[root@compute02 ~]# ceph osd in osd.0
' M, I- g/ m% _' u8 l; Q+ \/ omarked in osd.0.
4 ]) F) {. _2 U4 W# F[root@compute02 ~]# ceph osd tree
+ f- S% r. H1 sID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF 6 n& G/ e' B% P# U: c
-1 2.00000 root default
0 J/ x! k8 j4 O: r-5 1.00000 host compute01
1 d: U' E. S2 D& g0 d" @2 Q 1 hdd 1.00000 osd.1 up 1.00000 1.00000 6 s8 a: M' z% T4 F! E
-3 0 host compute02 : U7 G* `* _8 R8 w' D/ k
-7 1.00000 host compute03 " }2 u/ i' R* ]9 k
2 hdd 1.00000 osd.2 up 1.00000 1.00000 1 I c9 P) r v- b
0 0 osd.0 down 1.00000 1.00000
/ d; J3 ]/ _# P: O+ ]; A[root@compute02 ~]# systemctl restart ceph-osd@ 4 X5 D: U6 c* E
Display all 337 possibilities? (y or n)3 ?4 i; J$ U+ Y/ I
[root@compute02 ~]# systemctl restart ceph-osd@
" c3 j) j _+ \, O: J) ^1 kDisplay all 337 possibilities? (y or n)8 O& Y9 d: ?6 j6 J: k5 U' T( r
[root@compute02 ~]# systemctl restart ceph-osd@0' B" a1 Z) o7 G( B; R1 d$ c) K: k
Job for ceph-osd@0.service failed because the control process exited with error code. See "systemctl status ceph-osd@0.s' g5 L$ X u3 a$ F- K' H' g0 x
1 ]% K. [: b8 K3 {; T
- V' s X' z+ }6 V7 j2 d
. S2 M2 I |9 D2 e! ~重新添加ceph磁盘osd：
" w! X. [, I1 ^, T! E
2 ?( y' w; j( b- U[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
- t& o9 U3 j' d& I! l[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
) P, ^; }9 y, W[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
7 k5 {* i; r5 m4 f: d( ~! I[ceph_deploy.cli][INFO ] ceph-deploy options:* i% s C% k% w M6 h8 g0 P- h
[ceph_deploy.cli][INFO ] verbose : False
8 Y: S' _( }+ f4 ]2 g[ceph_deploy.cli][INFO ] bluestore : None
# S: b b. s; U7 U9 q9 J7 g[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f518f889690>
/ o$ V" W7 ^5 H {[ceph_deploy.cli][INFO ] cluster : ceph% a6 P( n1 V7 W" B9 k+ Z$ r$ i
[ceph_deploy.cli][INFO ] fs_type : xfs+ @; b* q0 \2 `1 b1 w( g
[ceph_deploy.cli][INFO ] block_wal : None
1 E: t7 Y8 m9 C0 _[ceph_deploy.cli][INFO ] default_release : False( D* F9 F7 o4 V; _
[ceph_deploy.cli][INFO ] username : None& y: d. G0 @0 i4 f; e6 D8 c
[ceph_deploy.cli][INFO ] journal : None
: w% b8 v# } ?3 ?. x( W+ K[ceph_deploy.cli][INFO ] subcommand : create% ?" S: E& ]- S% e
[ceph_deploy.cli][INFO ] host : compute02' P# }3 _9 ]7 f+ y7 T C& G d
[ceph_deploy.cli][INFO ] filestore : None
0 R/ V. u9 P% R2 f6 B: ^( X[ceph_deploy.cli][INFO ] func : <function osd at 0x7f518f8b71b8>
8 _/ S, ~' E) q& s[ceph_deploy.cli][INFO ] ceph_conf : None/ N2 @4 V u3 h- s! {
[ceph_deploy.cli][INFO ] zap_disk : False* N( z! ^0 t3 B! [- }
[ceph_deploy.cli][INFO ] data : /dev/sdb8 Y }# U" |# l' S! c
[ceph_deploy.cli][INFO ] block_db : None
6 T1 c" t* H% |8 n[ceph_deploy.cli][INFO ] dmcrypt : False" d# P% E1 u: ^0 Y' c
[ceph_deploy.cli][INFO ] overwrite_conf : False3 _) L4 u8 z1 y
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
; o+ [8 e U- ]' h. w; I3 }; {, n[ceph_deploy.cli][INFO ] quiet : False
& f, k0 C5 A6 p/ L& k[ceph_deploy.cli][INFO ] debug : False
: e4 f! V' h" G! b6 |% T$ }[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
; C: B7 O, P4 P {[compute02][DEBUG ] connected to host: compute02 ) i2 y, C% v3 N7 v
[compute02][DEBUG ] detect platform information from remote host
! {$ J3 j% Y3 B/ \9 p3 B3 C. c: u[compute02][DEBUG ] detect machine type
4 \8 a$ b d: w' t% T O3 U1 u[compute02][DEBUG ] find the location of an executable
! p: y0 Y- N; J9 q7 u1 D, K[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
# P: j" K2 v) c; M, u[ceph_deploy.osd][DEBUG ] Deploying osd to compute029 X1 W, e' C0 s% I8 a' Z" H
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf. V/ z6 j6 e$ a; y" i
[compute02][DEBUG ] find the location of an executable/ M8 z+ ^- W, Q& ^% I( H
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
% {/ A) I9 T9 B/ ^, F! d; G7 b& O[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key% I$ y& ~8 V C; s, w
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new bbfd988d-bb2e-44f2-a366-17501a8bbada
; f8 h* C) G1 t" i$ v$ Y5 m[compute02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l 100%FREE -n osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463, j6 E: k. Z, O: j
[compute02][WARNIN] stdout: Logical volume "osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada" created.
7 `" f2 W" Z2 s) S( m[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key, `6 c$ s% o; B) g. V" h3 m x
[compute02][WARNIN] Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0
: T ]% d# h% W t$ l[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada
; z" o6 W9 t$ z[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
4 i) u+ F* A; G2 {$ w' c# G. K- g[compute02][WARNIN] Running command: /usr/bin/ln -s /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block2 x9 C/ X) i7 ?6 V9 t" p5 }8 J
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
; S/ o0 y0 I% k[compute02][WARNIN] stderr: 2021-08-31 14:21:34.978 7fb58c080700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
8 h4 K! }1 k" o7 f) E[compute02][WARNIN] 2021-08-31 14:21:34.978 7fb58c080700 -1 AuthRegistry(0x7fb584065de8) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
3 W1 F/ B( ?5 d. v+ G: r[compute02][WARNIN] stderr: got monmap epoch 3
$ Q% m4 G5 z6 h. |! n8 V3 R% D[compute02][WARNIN] Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==
* s g5 R- B6 U5 @[compute02][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-0/keyring
& q1 C9 Q' u! |8 ?) L[compute02][WARNIN] added entity osd.0 auth(key=AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==)
8 M; Z$ A; o+ F$ b% h1 l[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring
" {' C7 o& E8 W! V) y4 P( v[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/2 W% x! h6 \0 t, K5 T/ T
[compute02][WARNIN] Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid bbfd988d-bb2e-44f2-a366-17501a8bbada --setuser ceph --setgroup ceph+ r7 X4 d/ ~2 T9 r1 V" l' p
[compute02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdb
8 S+ W! \4 k) y- U, F8 Y[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
( X( m- x6 U B) I1 O[compute02][WARNIN] Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada --path /var/lib/ceph/osd/ceph-0 --no-mon-config: b8 M2 G8 C0 ]" W( c% @
[compute02][WARNIN] Running command: /usr/bin/ln -snf /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block2 M8 q) y# @! G' C1 F6 o0 I% g
[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block
2 _5 z. ~% G" E[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2# i0 L x( w2 }3 C+ }$ y; U4 T
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
+ X! Q( }, r k/ N- b, E) C[compute02][WARNIN] Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada6 ?% g0 j9 i4 n/ m7 I& X3 U
[compute02][WARNIN] stderr: Created symlink from [url=]/etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada.service[/url] to [url=]/usr/lib/systemd/system/ceph-volume@.service[/url].$ \! B, W; }9 D3 [
[compute02][WARNIN] Running command: /usr/bin/systemctl enable --runtime ceph-osd@0
) y- P: k% X8 } { m4 Z3 s8 y[compute02][WARNIN] stderr: Created symlink from [url=]/run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service[/url] to [url=]/usr/lib/systemd/system/ceph-osd@.service[/url]., u5 [! n4 p; `, }2 r" w9 `) p
[compute02][WARNIN] Running command: /usr/bin/systemctl start ceph-osd@0; a# ^& \& |" O& @* W' m3 Z
[compute02][WARNIN] --> ceph-volume lvm activate successful for osd ID: 0
7 j% ~7 i, G7 H( S! |4 w[compute02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdb
/ T8 b/ U/ E6 Y: i9 O[compute02][INFO ] checking OSD status...
" ^2 d7 L a4 q" Y$ p: P" `[compute02][DEBUG ] find the location of an executable
, b; }1 A, p& y- H& l[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json; O6 e; \: u9 q
[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use. x3 v( I" B7 F
# b0 b Z. k5 x9 t+ \
( F- g- T/ }: L/ |1 a成功创建。这个坑确实有点过分。ceph存储一定要先解决时间不同步问题，否则有很多意想不到的惊喜：4 e" U5 `# u Z" H3 ]5 p
, }4 x& m B( @( J- S. j
' L! w; |( b/ i3 m1 X+ w[root@compute01 deploy]# ceph osd status
9 O* O: J* T/ j" k5 g7 z" ~, l+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
/ }" @ s$ ~2 o" ^| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |) ^% F& d* Q3 ~: c+ N+ r" s- j
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+& t, M' R% O' h; T
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |5 V% ]! R. l J; @- q- T2 \- r" H
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |9 u/ H5 r) t! R/ r* h5 S8 o, e
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
& `7 q) k* b0 Q+----+-----------+-------+-------+--------+---------+--------+---------+-----------+8 C& [( k8 p! Z. d8 x) n
[root@compute01 deploy]# - m) w8 ]: w8 K& |8 Z
[root@compute01 deploy]#
! z: D& ^) G; E& e[root@compute01 deploy]# # k% M# {# n0 n' N, W
[root@compute01 deploy]# ceph osd status
1 z. ?2 l, X7 G* G+ T4 r+----+-----------+-------+-------+--------+---------+--------+---------+-----------+! ^- e, y2 V* p: G7 J
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
7 s( C! q( ?- { R# O+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
6 {7 I/ g" E0 q+ ?| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |0 W, S5 n- N$ V. O5 G6 w# b
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |) E! [. ?+ q+ |; Z
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |9 J! ]) X& F \
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+( a* V$ |. n, A( o/ G9 X' S
[root@compute01 deploy]# ceph osd status - Z; D: \; ]% C7 y- I
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
, A- Q6 q" Q' f- W, C" z, a" h| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
7 R7 V+ e& G+ q( M: [2 F' f7 }- X+----+-----------+-------+-------+--------+---------+--------+---------+-----------+$ X" M3 U% n1 E' H: F% Q
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |' T ^1 y9 S/ C
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |
% p2 c. O! C6 n I+ D8 K0 R1 H| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |! p' | ~% m2 l8 |/ `- J4 `
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+: k/ E; S! t! k+ ]4 s: m
[root@compute01 deploy]# ceph osd status " f2 y2 e, [& j7 a' B" I/ P( @6 N
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+0 @1 P; h+ `' v
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
0 b2 Y# C$ R9 w5 c( \3 k+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
2 B* I, H8 I9 z ?/ J7 d4 r. x| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
3 M7 Y7 ]+ }0 p, D| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |& J" F, A8 [) r9 Y; t" v
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
& m( V4 k3 ~) ^ z; o/ N8 W& x$ A, Q+----+-----------+-------+-------+--------+---------+--------+---------+-----------+1 ^/ \! C p" F4 U( v# y
[root@compute01 deploy]# ceph osd status ( O3 h2 o. k" O
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
9 Q( [# K5 B4 p& L! r| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |, g% }% w% d2 ^* _
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
# `- k. b3 E2 z! a! I| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
8 ?6 e! M9 P6 d- s, X2 {6 M| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |
6 d! U3 F! F+ e# w2 g' }" o: z| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
$ U) o2 @- ]( b A; |0 F+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
- x+ Q! C+ Z8 T5 X9 o |
|
发表于 2021-8-31 15:00:09