|
|
[root@compute01 deploy]# ceph status . c$ L+ {1 N1 T" _) D5 u" L0 _1 \
cluster:
' C2 {4 t" g: t1 | id: 31403b11-8a1e-432f-876e-5a2c852f9dcc
. ?9 q# L: R6 f6 R) ` health: HEALTH_WARN
; P( V( X0 v" C5 f5 j Degraded data redundancy: 13701/41103 objects degraded (33.333%), 432 pgs degraded, 640 pgs undersized3 n& }5 ~' k7 z G
13269 slow ops, oldest one blocked for 8178 sec, mon.compute02 has slow ops- t. w o. q. q1 X& m' K! Y
clock skew detected on mon.compute02
$ D2 M4 P f# S: X+ @ 1 Z. U5 Y( i \$ S
services:
- i7 k5 `! a5 G% Y mon: 3 daemons, quorum compute01,compute02,compute03 (age 2h)
( f3 u* ?1 }7 t5 N7 z7 | mgr: compute01(active, since 4h), standbys: compute03, compute02
% [# C; [7 I" L6 |. [ osd: 3 osds: 2 up (since 4h), 2 in (since 2h)( {1 i, p& _; u# `( K5 o5 y
5 T0 l( G. W) N& R' w
data:
8 r" o; [+ N% w4 x pools: 6 pools, 640 pgs7 A9 y9 h% D) Q1 t& q
objects: 13.70k objects, 11 GiB' \3 S4 s% [* ~* e7 t2 z
usage: 24 GiB used, 2.7 TiB / 2.7 TiB avail
6 m2 D( g' v/ j. z pgs: 13701/41103 objects degraded (33.333%)
/ Y' l/ a/ F& }* u; L; K: G! b3 @ 432 active+undersized+degraded
. ^* k- s% c$ A 208 active+undersized
, Y4 d3 d' {) x6 }; q' b
; P x+ E, K! c( Z3 p' R[root@compute01 deploy]# ceph osd crush remove osd.0
. `% A" T2 `/ v8 B3 Y9 ~removed item id 0 name 'osd.0' from crush map; u' O5 v; w1 Z1 S- Q4 r8 A e
[root@compute01 deploy]# ceph osd rm osd.0
" y9 y8 n* x( `removed osd.0
- M; y6 p' t# {+ N[root@compute01 deploy]# ceph auth del osd.0
4 s& k q% r% u% Mupdated
9 w' l. i1 b7 r( x O3 ]/ U5 a[root@compute01 deploy]# history |grep sdb( k. R' e8 O8 T6 m
1023 history |grep sdb
$ [7 d! b' h# T[root@compute01 deploy]# history |grep ceph-deploy3 x) t8 V; F- J7 c* i! m2 M
1024 history |grep ceph-deploy
# Y- q+ R( r, z9 z& F[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
) k: t* a. L4 E$ q) ?& s# v' F0 C[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf% b: l8 @* Z$ C& O4 r0 f( W1 s
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
& ?5 A- G( {- R+ i[ceph_deploy.cli][INFO ] ceph-deploy options:
( I4 ^6 W' e# k+ \. v6 N[ceph_deploy.cli][INFO ] verbose : False
/ r) c+ \ V" t3 S% r, y[ceph_deploy.cli][INFO ] bluestore : None
2 ?, Y4 B1 z- N3 P1 O$ r9 |, n[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fbd39643690>
' a) @# S# d! p! ?4 I* _: w5 W[ceph_deploy.cli][INFO ] cluster : ceph
4 Y$ m: y' T1 F/ `, R; r[ceph_deploy.cli][INFO ] fs_type : xfs
& J4 [, U. P( S9 z; X7 n! U: i6 [; K[ceph_deploy.cli][INFO ] block_wal : None( v& e0 {0 G' O
[ceph_deploy.cli][INFO ] default_release : False8 U# v% m+ P4 B
[ceph_deploy.cli][INFO ] username : None# e2 }( N" n( X% p6 D( i6 ]2 {/ d' _& Q
[ceph_deploy.cli][INFO ] journal : None
8 B+ N$ y. x7 \0 T[ceph_deploy.cli][INFO ] subcommand : create2 `, y$ h9 u7 T! W$ r$ N: @) \
[ceph_deploy.cli][INFO ] host : compute02
0 k! m$ I1 u- r- U7 x[ceph_deploy.cli][INFO ] filestore : None
, W& d( {- d6 P# _[ceph_deploy.cli][INFO ] func : <function osd at 0x7fbd396711b8>& o7 z2 Z2 g. T8 x' z: @% o
[ceph_deploy.cli][INFO ] ceph_conf : None
7 O* s" d" W" R/ L) Y: d! i6 K% t/ Y[ceph_deploy.cli][INFO ] zap_disk : False
: m9 c; T5 {0 L. a[ceph_deploy.cli][INFO ] data : /dev/sdb
. F7 ^' x3 I% g/ _3 G# A0 }# i[ceph_deploy.cli][INFO ] block_db : None
" y' I! H, ]* z/ ?/ \/ p8 V[ceph_deploy.cli][INFO ] dmcrypt : False
4 v+ P- s, S' o$ m: d6 m& I0 h) s8 G[ceph_deploy.cli][INFO ] overwrite_conf : False: {. }) V( g% B) q$ U% p6 z; W
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys9 ?% y0 @* \4 h) _
[ceph_deploy.cli][INFO ] quiet : False0 R0 a1 b/ g0 s; R, m) A
[ceph_deploy.cli][INFO ] debug : False
! ?, ^2 ]# Y* u! m% _$ E[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
3 t3 d: F9 [7 D) m6 A% K[compute02][DEBUG ] connected to host: compute02 9 w ^* k ?% l+ ` y9 i$ ?
[compute02][DEBUG ] detect platform information from remote host
! n( J8 Z% A% H# j/ k[compute02][DEBUG ] detect machine type
9 X9 H% D3 g& t- S4 _& d8 C8 {9 C[compute02][DEBUG ] find the location of an executable( f, Y3 ~# V# b6 x. h, J
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core5 v& Y9 ?5 R- O; {. `
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
/ n8 K Q1 |( @[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
, n; Y6 u8 A$ k# G I[compute02][DEBUG ] find the location of an executable
1 @7 C, t6 i3 p9 ?0 s+ G[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
$ V" j- V# l# r0 l[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key6 P# O) i6 U5 ^3 {! E3 H D
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 8bb22eb2-d2e1-4ae4-aa52-cc96422b25bb
% p3 K2 H9 k# o& T" Q* A' B) q$ j5 H, |4 s% A0 h& }* d" s/ A- x
^CKilled by signal 2. o, ^, [8 u. y$ R; F8 U
[ceph_deploy][ERROR ] KeyboardInterrupt
# ^- Z' G( S) l# N) z[root@compute01 deploy]# ls
0 L' ~3 Z5 i& P/ |ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
- Y2 ~# ]& u; Kceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring
4 D' d) ?9 v0 _/ [3 t0 k[root@compute01 deploy]#
& E( @$ M& t+ d5 Q. F+ X2 X[root@compute01 deploy]#
& A6 A6 Z9 U8 W' |[root@compute01 deploy]# pwd9 ]- i ?8 K7 }7 C
/ceph/deploy. ?. n; e7 K- D- a d, T/ U* y
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb ^C
9 n! z; x! Y. z- ^[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb8 [0 b1 j5 C4 }9 \. c& B! k
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf; W/ {+ a- J* l, g, W% n1 t
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb
$ q/ }" J3 ~' ?3 E2 {[ceph_deploy.cli][INFO ] ceph-deploy options: ^9 x! \# J& M1 _0 V
[ceph_deploy.cli][INFO ] username : None! Q2 k; ~' ?' L( b
[ceph_deploy.cli][INFO ] verbose : False2 ~, ]( Q: L9 \5 K
[ceph_deploy.cli][INFO ] debug : False
$ }3 g( v, k9 n1 m; U9 P[ceph_deploy.cli][INFO ] overwrite_conf : False/ B% X2 M* q: a. @. s& ^
[ceph_deploy.cli][INFO ] subcommand : zap
% R* K) N' D. E5 F/ p5 ][ceph_deploy.cli][INFO ] quiet : False
, [ v5 I% n5 u; ^" O[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f0db9e1dad0>
/ o( q0 H. u! K3 Q[ceph_deploy.cli][INFO ] cluster : ceph7 }. ?, H/ t7 w2 e. f- D4 O
[ceph_deploy.cli][INFO ] host : compute02$ J* j4 `5 g- n; U( y F9 E4 M
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f0db9de3230>
) E$ a% ~% T" y[ceph_deploy.cli][INFO ] ceph_conf : None
& R! w: R3 f- R' E5 S& p[ceph_deploy.cli][INFO ] default_release : False" q- C. N/ C1 N4 d
[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']# f1 G; l; ?( s( Z; t8 Y8 a l
[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02# Y. j7 ^& ^/ d4 l6 h0 E X1 P
[compute02][DEBUG ] connected to host: compute02 + [7 ~' ]! o& J+ @" a; D
[compute02][DEBUG ] detect platform information from remote host
! I: u9 f( g D2 l[compute02][DEBUG ] detect machine type# r- b6 W6 C, o. q. D
[compute02][DEBUG ] find the location of an executable
5 Q2 H2 _4 N- a9 S* C[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
2 }1 l$ v) }% Y. j[compute02][DEBUG ] zeroing last few blocks of device
; b4 H: x" f9 d6 T+ t$ W[compute02][DEBUG ] find the location of an executable
& C& ^, P! Y! C# P3 G# U' G t' Z[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb( S; ]" |7 M" K; @: |" e
[compute02][WARNIN] --> Zapping: /dev/sdb* _( ]+ R8 Z2 w9 t- O! U$ R
[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table- Z2 P6 k( a* A% e# o( D
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync
9 j% x6 ]+ k' Z1 I7 z9 f[compute02][WARNIN] stderr: 10+0 records in& W" H, A7 p: R' j2 I4 Z4 c
[compute02][WARNIN] 10+0 records out" {: v9 f0 s7 U" }
[compute02][WARNIN] 10485760 bytes (10 MB) copied* X* |* H6 s( Y) N9 L' G9 ^
[compute02][WARNIN] stderr: , 0.0850991 s, 123 MB/s# v7 u1 u$ v5 g* p/ @/ U& E* G
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>) q: q. m: @4 O; j' f0 H% B
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb9 n5 N( {/ \2 a' Q% J
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
' R6 g# Z) M2 ~% T( t; K[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb
% q( F9 y4 [. E- p[ceph_deploy.cli][INFO ] ceph-deploy options:: G/ {) `) G1 b' C) b, k! m, a
[ceph_deploy.cli][INFO ] verbose : False
a0 C7 _( ?4 Q3 H- _' T[ceph_deploy.cli][INFO ] bluestore : None
2 ]$ @6 I2 O! X1 i[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f45a0a92690>4 i( f1 g7 e/ U
[ceph_deploy.cli][INFO ] cluster : ceph+ @8 c o1 f) _ S0 K
[ceph_deploy.cli][INFO ] fs_type : xfs
2 l7 `; _8 Y1 O" K' U% d* r( m: Z[ceph_deploy.cli][INFO ] block_wal : None+ w, X o6 r9 f
[ceph_deploy.cli][INFO ] default_release : False
" J( U( G2 `: t" `" |[ceph_deploy.cli][INFO ] username : None
C' q1 v1 _9 c1 t) L. E& p[ceph_deploy.cli][INFO ] journal : None
9 v) M8 ^6 }8 g$ g, @3 R( n[ceph_deploy.cli][INFO ] subcommand : create, `4 H/ o+ ^* R! y* Y) ]
[ceph_deploy.cli][INFO ] host : compute02% P4 j8 y: f N8 A7 W; r/ D k
[ceph_deploy.cli][INFO ] filestore : None( I% E1 y$ U& g. E8 j3 z, [
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f45a0ac01b8>
1 \' e+ w; x( G- y+ U+ y[ceph_deploy.cli][INFO ] ceph_conf : None: N/ T" W( _/ [5 J! r1 V
[ceph_deploy.cli][INFO ] zap_disk : False
" Z+ j: E% ~3 X1 J5 Q0 J, n[ceph_deploy.cli][INFO ] data : /dev/sdb7 S3 q5 `' ]0 p, x. j9 k
[ceph_deploy.cli][INFO ] block_db : None$ r6 {4 i" F+ T0 K- H; z
[ceph_deploy.cli][INFO ] dmcrypt : False6 g9 d; w w* X9 U, P& ?- e. D
[ceph_deploy.cli][INFO ] overwrite_conf : False2 ?5 V6 Y5 `* ~' Y
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
6 ^& z+ t. h% h0 \; ?[ceph_deploy.cli][INFO ] quiet : False! n) N' m B) g" p% a% C. l$ q
[ceph_deploy.cli][INFO ] debug : False$ N9 T4 N7 i1 E8 C# f
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
& d! o+ k3 _8 l[compute02][DEBUG ] connected to host: compute02 0 |; K! A2 @7 m; R( U( D
[compute02][DEBUG ] detect platform information from remote host
! K" E' N3 G7 u* J[compute02][DEBUG ] detect machine type
, Z9 {3 M% V# R9 c5 x+ g! T[compute02][DEBUG ] find the location of an executable( o; J4 R: i. j; u
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
0 I1 ^& I8 }8 P# x, f9 n( J[ceph_deploy.osd][DEBUG ] Deploying osd to compute02: s$ p6 c2 }8 }, T7 J7 Y' Q
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf' o4 l s- t9 _
[compute02][DEBUG ] find the location of an executable
2 I0 g, G, r1 g, g3 o- x: @[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb. ~7 C) s2 `% z7 k9 ] Q
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key* t. @/ w! {8 K+ `
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9f1fca57-1a1c-4d30-bad8-2d4578f88e96
& H! Z0 Y4 I# y1 k) f* x/ x^CKilled by signal 2." z. v8 x% E$ T1 z' e% f |. d
[ceph_deploy][ERROR ] KeyboardInterrupt
: K c3 u( R1 n9 Y3 j[root@compute01 deploy]# ls4 b+ X" w5 Z! C' d
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
; \& O- m* |5 s! B: R/ G% Hceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring8 ]+ w8 q+ n( Z& D$ @: i" h; x
[root@compute01 deploy]#
9 K! R: q8 K: }[root@compute01 deploy]# ' k- V# }- S! W' g
[root@compute01 deploy]# ! A1 v) h* R& d |
[root@compute01 deploy]# ceph-de* ?8 [: f/ r4 ]7 U3 }9 r) @! z8 b
ceph-dencoder ceph-deploy
n' k9 H# w1 O: A[root@compute01 deploy]# ceph-deploy
6 n/ i2 y1 R4 D8 e4 y. }usage: ceph-deploy [-h] [-v | -q] [--version] [--username USERNAME]
0 v% _5 M% a. D [--overwrite-conf] [--ceph-conf CEPH_CONF]
9 k" G& s* i/ q3 n# u; I+ c6 X COMMAND ...
3 \7 P. h0 k/ F' n( R/ QEasy Ceph deployment
( F& i$ R/ I5 V0 i) k- `" A -^-
- i) I/ E: J( R' M / \
! V" N$ X" }9 e! M" O* Q' T# n |O o| ceph-deploy v2.0.1
$ V/ T g8 g) M& _6 C) n( p ).-.(( }/ V7 F- U9 w+ X4 R: A
'/|||\`" q0 e+ y5 S3 n- R: I( o
| '|` |2 ~# Y# r3 X8 z; [' I' |9 r9 k
'|`
0 {1 w* s- u! J2 L& DFull documentation can be found at: http://ceph.com/ceph-deploy/docs1 @' d2 }# ?9 Q& q! |7 S, z
optional arguments:
2 `6 _ i" A; b4 W. D0 Q! P7 } -h, --help show this help message and exit
1 c! Y) m; O0 A+ @) H -v, --verbose be more verbose
9 X) {: H2 |& |8 F" r9 w' ^) r- k -q, --quiet be less verbose
3 q$ Z& r* z0 I' J* D% [0 }2 m8 ~ --version the current installed version of ceph-deploy
' `3 V: b' U# {4 }; [, C+ w --username USERNAME the username to connect to the remote host
* o7 u2 c+ ^% G0 @) ]! H --overwrite-conf overwrite an existing conf file on remote host (if( O" |) t+ _1 y: K
present)3 q; Q N+ F% M9 x" L# z- V0 w
--ceph-conf CEPH_CONF. p7 V' I! S5 D0 S. ~
use (or reuse) a given ceph.conf file; Z: | U) q4 c# k- r. |. a
commands:
- D7 V5 b+ @% }/ f3 d7 K1 s8 }9 K COMMAND description
2 Z7 ^% d& i7 R' J# H( q new Start deploying a new cluster, and write a
+ f+ ]" W/ `5 D; H- ?! c CLUSTER.conf and keyring for it.# S; i: N. c1 b" q
install Install Ceph packages on remote hosts.$ t) w1 P, h1 \0 `+ o& i
rgw Ceph RGW daemon management( v9 y, H0 ^6 r4 ^5 e+ o% L4 h) j8 y
mgr Ceph MGR daemon management
7 B2 @% n- b. l mds Ceph MDS daemon management3 W" U6 b3 u2 f
mon Ceph MON Daemon management6 N6 L% Q( g6 Q7 F# v+ Y5 h
gatherkeys Gather authentication keys for provisioning new nodes.
/ y) C2 j" S; e4 @ R' V1 ~$ ` disk Manage disks on a remote host.
" X+ a; C, I6 U D/ \ osd Prepare a data disk on remote host.
. x3 h3 d) ~$ Y8 | repo Repo definition management4 k" Z8 c( p3 @, u; t* F
admin Push configuration and client.admin key to a remote
0 e8 Q2 W+ s% z. s; x host.8 J8 q+ t7 u. V* ~5 x& N3 L! _
config Copy ceph.conf to/from remote host(s)
4 f% u: w) f9 p i6 e: e uninstall Remove Ceph packages from remote hosts.
6 u$ Y1 W5 p6 r, Q$ ? purgedata Purge (delete, destroy, discard, shred) any Ceph data7 n, c8 g s: \& {. P/ H" i
from /var/lib/ceph
) V3 e2 w( P& j: ?' ]4 f purge Remove Ceph packages from remote hosts and purge all4 H7 q$ e8 {: m7 o- x
data.2 b. o; s( m. v! s) X- i v
forgetkeys Remove authentication keys from the local directory.
" L1 l) |4 p9 T7 V pkg Manage packages on remote hosts.
$ N' s# D8 A/ l( N! A calamari Install and configure Calamari nodes. Assumes that a4 V$ S' b& f/ S; @) x
repository with Calamari packages is already
$ B1 A5 t0 P6 p. N configured. Refer to the docs for examples" V" ^, J9 V1 t; t
(http://ceph.com/ceph-deploy/docs/conf.html)
. D$ Q! N. S9 QSee 'ceph-deploy <command> --help' for help on a specific command7 k* |/ T' W9 o3 C' O
[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb
. F+ \' I4 V, a4 W- h' v! s[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
* m' f$ a, j( z! `6 v7 N; y8 q[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb# g, e0 F1 a* X8 E6 R
[ceph_deploy.cli][INFO ] ceph-deploy options:" }, H* L0 N3 u* u& {, c i
[ceph_deploy.cli][INFO ] username : None1 j' \5 J4 p/ T" d" X9 l6 U
[ceph_deploy.cli][INFO ] verbose : False7 b+ a- z M9 S) y9 F0 V: b! X2 H7 b, o
[ceph_deploy.cli][INFO ] debug : False2 Y8 z+ t6 y7 t5 @! x& p
[ceph_deploy.cli][INFO ] overwrite_conf : False
, o2 d3 [1 A/ t[ceph_deploy.cli][INFO ] subcommand : zap, f" c) s5 w$ D* ?# K
[ceph_deploy.cli][INFO ] quiet : False" e" a" P. P5 T {
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f1c11596ad0>+ G; A7 _6 a. X6 H0 [
[ceph_deploy.cli][INFO ] cluster : ceph( J/ [: k( H0 @+ Q2 p
[ceph_deploy.cli][INFO ] host : compute025 c b9 _$ N9 ]' C( {" h
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f1c1155c230>+ i8 S2 ? W8 i, y, g4 Z2 [* i
[ceph_deploy.cli][INFO ] ceph_conf : None
: a: |9 i: D; I+ W! \" v[ceph_deploy.cli][INFO ] default_release : False9 b3 t; b* n) ~" P
[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']
/ \! x$ _) d; K- l[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02
( G" m/ S3 J; {) Z/ v; R[compute02][DEBUG ] connected to host: compute02 4 Y. v: c8 y- M- {- y* W
[compute02][DEBUG ] detect platform information from remote host. @) I) e9 r( ]' y: N
[compute02][DEBUG ] detect machine type7 _( B; O5 l& E
[compute02][DEBUG ] find the location of an executable7 |! x+ k# `$ [3 e+ N
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core" E- l& C) G) ^& W$ ~* K4 m
[compute02][DEBUG ] zeroing last few blocks of device1 W) [0 ~( t! v
[compute02][DEBUG ] find the location of an executable4 b4 I% _ D0 ^, c
[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
% t2 R' B$ O) A% m[compute02][WARNIN] --> Zapping: /dev/sdb
; U3 Y2 X4 U& h8 O0 q5 I[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table+ T& f$ d3 K1 O
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync' A( h/ G( T. Y. }) l
[compute02][WARNIN] stderr: 10+0 records in
$ y" N% }& ^* O# N/ y[compute02][WARNIN] 10+0 records out: c0 F# s/ A9 D6 w; D+ m
[compute02][WARNIN] 10485760 bytes (10 MB) copied' C% y0 E, D8 E H! V
[compute02][WARNIN] stderr: , 0.0591334 s, 177 MB/s! y( U, x* Z1 D0 `
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>! V2 C- ^( V0 ~' V' |
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
' T; V7 ~) ~' b$ K$ s }. \# F[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
7 @ e% B# q( V$ y/ x: D[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb+ k2 m; B% v. u% o, E* ?" Y2 ^
[ceph_deploy.cli][INFO ] ceph-deploy options:
* O, l X' J, e5 l[ceph_deploy.cli][INFO ] verbose : False
" K+ W" k3 P) U4 O$ _[ceph_deploy.cli][INFO ] bluestore : None1 y5 l5 d, b* j+ H& Q0 Y
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f23c40b9690>& p# ^, E9 Z3 A% _
[ceph_deploy.cli][INFO ] cluster : ceph' f* S7 H3 V( g. H" R2 o" G
[ceph_deploy.cli][INFO ] fs_type : xfs
6 F0 f; h, s9 N" i, m$ q[ceph_deploy.cli][INFO ] block_wal : None
8 [" Z* [$ O# [* g2 P[ceph_deploy.cli][INFO ] default_release : False
6 U8 U! I7 x* O; E6 C% ?. O[ceph_deploy.cli][INFO ] username : None+ M; n6 t. }* b5 [: m$ z
[ceph_deploy.cli][INFO ] journal : None
' b! H) i# x) | V# {[ceph_deploy.cli][INFO ] subcommand : create
0 y+ y+ |: V ~# T[ceph_deploy.cli][INFO ] host : compute02) e- z, |) n) I
[ceph_deploy.cli][INFO ] filestore : None$ l/ z, L* L/ I# v6 ]
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f23c40e71b8>3 u, z4 E# F% B" X
[ceph_deploy.cli][INFO ] ceph_conf : None6 [3 p/ Y2 E4 @9 M- ^; \3 R! a( Q
[ceph_deploy.cli][INFO ] zap_disk : False
$ D1 {( o- ~; P3 g1 m% j: }$ ~[ceph_deploy.cli][INFO ] data : /dev/sdb' b/ }7 s0 H0 M/ X& D
[ceph_deploy.cli][INFO ] block_db : None: d( v+ r) \) {6 P" Z; x0 ?* F
[ceph_deploy.cli][INFO ] dmcrypt : False: o# k% O/ [( m( @
[ceph_deploy.cli][INFO ] overwrite_conf : False
5 a4 z- ~' Y ^[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
5 _3 E- M% u" _& |[ceph_deploy.cli][INFO ] quiet : False n Y+ I% O- s4 E+ x% U) h
[ceph_deploy.cli][INFO ] debug : False0 u1 i% I ]( m& V5 z9 D( ?
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb8 M$ W$ d( ^1 C4 @7 A. ^
[compute02][DEBUG ] connected to host: compute02 . O; p/ _" S) z" F" P3 Q; }
[compute02][DEBUG ] detect platform information from remote host
, T7 U/ G0 r8 p: w: c% O$ z[compute02][DEBUG ] detect machine type+ D( n/ Q+ V9 K, I, w
[compute02][DEBUG ] find the location of an executable
' F, k. S% }, {: v4 L[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core
3 _5 O1 V! ~% L[ceph_deploy.osd][DEBUG ] Deploying osd to compute02# Q7 S$ m& U$ y: b
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
3 S; r) }, F# l[compute02][DEBUG ] find the location of an executable- _3 j% l/ q# L
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb/ a$ }9 p+ s" O1 Z1 ~; g% O7 y
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key" c# H* _( Q% N! i! k
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new fab4f939-9a55-4e45-94d5-2434dbb24766- G0 K1 H1 U6 |( G! O, s! f$ I
^CKilled by signal 2.
# f3 [7 d# N* m5 C# _2 J[ceph_deploy][ERROR ] KeyboardInterrupt5 A' Q: v M M @) X' y% C
[root@compute01 deploy]# ls- V, r3 P. p9 b" t! G" H9 `
ceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log Q' n7 ]! t* p$ `3 y- Y; l( S$ a
ceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring, _5 n" E6 f- _$ }! l- T! o
[root@compute01 deploy]# vim /etc/ceph/ceph.conf
& |. D( y7 w7 q- L. q[root@compute01 deploy]# vim ceph.conf
5 |0 e1 S+ L& b8 O+ l+ M* n/ R[root@compute01 deploy]# vim ceph
3 h* [" g6 B1 b7 iceph.bootstrap-mds.keyring ceph.bootstrap-osd.keyring ceph.client.admin.keyring ceph-deploy-ceph.log
/ |5 d+ O% o8 n- Bceph.bootstrap-mgr.keyring ceph.bootstrap-rgw.keyring ceph.conf ceph.mon.keyring ! n9 ~0 S3 w3 h6 t
[root@compute01 deploy]# vim ceph.mon.keyring
8 y! Z& s5 S* z1 q: ^/ e1 p[root@compute01 deploy]# vim ceph.client.admin.keyring
( J) D" a W8 m" K[root@compute01 deploy]# ceph-deploy disk zap compute02 /dev/sdb
3 _7 i& ?, T/ Z% E[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf! O' B/ X& e1 z* x& H) ^
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk zap compute02 /dev/sdb
4 h R0 w0 h) W! |[ceph_deploy.cli][INFO ] ceph-deploy options:
0 T* \% V2 K. A! O9 F; A& a j[ceph_deploy.cli][INFO ] username : None
- w- n5 ]% P* H- Q# Q6 F# B[ceph_deploy.cli][INFO ] verbose : False8 |& n2 x4 r1 ?
[ceph_deploy.cli][INFO ] debug : False8 @2 |( `$ D; [+ V
[ceph_deploy.cli][INFO ] overwrite_conf : False5 _: R( j1 u6 [4 E4 n8 M7 s, Y
[ceph_deploy.cli][INFO ] subcommand : zap
! { r1 ^ t" t* k' ^+ k[ceph_deploy.cli][INFO ] quiet : False: f) B. n, m! |( q$ }
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fa10739bad0>
+ r! m8 a! F! J, T[ceph_deploy.cli][INFO ] cluster : ceph2 \( j+ W# r$ x' E& R& K
[ceph_deploy.cli][INFO ] host : compute02
; D8 j# w& |1 t: Z% W) t[ceph_deploy.cli][INFO ] func : <function disk at 0x7fa107361230>
3 e8 d0 s# V7 o/ D4 ]/ Z8 ?7 G. d[ceph_deploy.cli][INFO ] ceph_conf : None
0 J& V/ J- F% n% ]" F$ ~[ceph_deploy.cli][INFO ] default_release : False
; n; J% H: e' O, u. K[ceph_deploy.cli][INFO ] disk : ['/dev/sdb']
8 ]; E# d' q" Q3 k9 E" K3 r( J( T[ceph_deploy.osd][DEBUG ] zapping /dev/sdb on compute02- P0 }) C% `% D6 N
[compute02][DEBUG ] connected to host: compute02 * S/ s) U7 ?8 ]
[compute02][DEBUG ] detect platform information from remote host
) x; |' a# ?% Y3 Z6 x9 W" J8 A[compute02][DEBUG ] detect machine type
6 y% h6 D4 D; C# n[compute02][DEBUG ] find the location of an executable8 H) a; @8 p# y& m/ v8 _: R
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core8 {7 k( @+ p2 e9 E
[compute02][DEBUG ] zeroing last few blocks of device, ~7 H$ z/ ~3 O6 c' F% B& Q$ ?
[compute02][DEBUG ] find the location of an executable
L9 V8 t2 X! P8 K# Q2 ?[compute02][INFO ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb( w2 z2 Y! V& n) I0 ~
[compute02][WARNIN] --> Zapping: /dev/sdb" s( b9 v4 U' G( t6 ]1 Z$ O' V
[compute02][WARNIN] --> --destroy was not specified, but zapping a whole device will remove the partition table1 p2 V3 t4 n% P* |3 l- q8 Y
[compute02][WARNIN] Running command: /usr/bin/dd if=/dev/zero of=/dev/sdb bs=1M count=10 conv=fsync ]# a& T5 @+ J! @- F8 ^
[compute02][WARNIN] stderr: 10+0 records in* H; V! T" C9 V1 H
[compute02][WARNIN] 10+0 records out' r% \, J1 u6 a. S0 ~
[compute02][WARNIN] 10485760 bytes (10 MB) copied1 P' {; [$ `/ T. G' ^9 ?
[compute02][WARNIN] stderr: , 0.0869923 s, 121 MB/s. U' [5 g/ l- b" F, x
[compute02][WARNIN] --> Zapping successful for: <Raw Device: /dev/sdb>: G. J. c3 s3 X. g" r ^
[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb & i' T# x2 t/ C8 C. W0 Z( Y
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf3 ]& l- ?2 S( M, e/ T; L9 f& q! M
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb+ G- ~0 s; x$ @; P2 z# T
[ceph_deploy.cli][INFO ] ceph-deploy options:
9 S! }& S0 H4 o& i D[ceph_deploy.cli][INFO ] verbose : False8 \# h* `* D5 n! A
[ceph_deploy.cli][INFO ] bluestore : None: |+ d7 G6 ]( G' a2 q
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f13835dd690>9 F" b2 c0 t* T4 J& E; {% J$ R7 E
[ceph_deploy.cli][INFO ] cluster : ceph/ p" C! b( T2 G4 k4 l2 G
[ceph_deploy.cli][INFO ] fs_type : xfs# k! f" s1 y3 H% k% [5 _
[ceph_deploy.cli][INFO ] block_wal : None
' N& S6 h) {% W7 I# ][ceph_deploy.cli][INFO ] default_release : False) W, O: j0 M) Q k
[ceph_deploy.cli][INFO ] username : None' A3 J n' H8 d! O. t( H5 x0 h8 M
[ceph_deploy.cli][INFO ] journal : None
1 O( o4 `' } J8 `[ceph_deploy.cli][INFO ] subcommand : create4 X+ n& `. m2 ?7 k( w% v; |
[ceph_deploy.cli][INFO ] host : compute02
3 @3 b; Z3 y# @: e[ceph_deploy.cli][INFO ] filestore : None
" L# R {+ z7 V" {2 |[ceph_deploy.cli][INFO ] func : <function osd at 0x7f138360b1b8>
- t( Q) d- B$ R" g[ceph_deploy.cli][INFO ] ceph_conf : None6 j! {7 T5 F- C, W# G; J
[ceph_deploy.cli][INFO ] zap_disk : False
, J3 e7 V% O% f[ceph_deploy.cli][INFO ] data : /dev/sdb
|5 w( {3 s* H1 k/ A5 p' u7 \3 R[ceph_deploy.cli][INFO ] block_db : None
: M7 t/ h8 U! S4 J+ q" n& T[ceph_deploy.cli][INFO ] dmcrypt : False
& W% S4 I, |% n, Z- U1 [6 T1 R[ceph_deploy.cli][INFO ] overwrite_conf : False% D& j5 P1 N) {- z( v3 }0 O" [
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
- e: ?6 d9 d2 `, T+ P) [[ceph_deploy.cli][INFO ] quiet : False
! t! b. _1 l! ~, e[ceph_deploy.cli][INFO ] debug : False
( h. K4 w0 l0 L j9 `) L" ^7 {9 _[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
3 ~7 @0 R0 c4 I5 P[compute02][DEBUG ] connected to host: compute02
4 g0 T: ^3 d7 E' r6 {0 ~% _[compute02][DEBUG ] detect platform information from remote host0 d' f1 `( {2 S6 h6 l" Q( l
[compute02][DEBUG ] detect machine type- m6 p! T1 M# ?1 G, h& J$ G% e/ E
[compute02][DEBUG ] find the location of an executable9 R" I0 K4 h! Q/ M3 x7 k' j
[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core! ~$ C4 n8 {( |
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02
. \- T$ Y( _2 R% K$ E# `[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf/ h# V! N$ a" F: W
[compute02][DEBUG ] find the location of an executable
) ?* B q( e! W) `: ~- m C[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb9 g9 a) |- B9 c J" c
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
, ~$ i7 Y0 a1 m5 M' x( C/ v[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new c71415c4-fdaa-4955-8c26-71d160c6121e) a" v3 |* R4 D
[compute02][WARNIN] No data was received after 300 seconds, disconnecting...6 {# u, ?( r, S& E
[compute02][INFO ] checking OSD status...
) t5 C6 o. X) a% a6 \1 E[compute02][DEBUG ] find the location of an executable
3 N2 s- c' T) g' j& ]& l! c[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
2 H5 G) |0 n: q4 l1 N[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.
2 ]+ V7 A- h7 g& O9 H2 c$ e[root@compute01 deploy]# ceph osd tree* d' V4 H- X5 T! }9 }+ r3 ?
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
6 Z6 O* s% j* ?. b-1 2.00000 root default
, w2 U% M5 h% \-5 1.00000 host compute01
0 n! T5 [ h. t% W 1 hdd 1.00000 osd.1 up 1.00000 1.00000
" t+ R7 ?- J" ]' I1 [, F& E-3 0 host compute02 % z& Y# |* v* H, O7 g# f6 P
-7 1.00000 host compute03
' h" _7 X/ C# e! K 2 hdd 1.00000 osd.2 up 1.00000 1.00000
+ n' r- v2 g3 B" Q! ? 0 0 osd.0 down 0 1.00000
6 }8 k5 x9 [1 W! p! @9 r 3 0 osd.3 down 0 1.00000 0 b0 }4 ^- E0 J
1 U ~& Y+ x" d- y& @" D, T; P }( F
在另一台节点上：0 e! h6 M9 }- t" E7 w4 s; g& |
[root@compute02 ~]# ceph osd tree
) \+ R( M$ m; e9 j
/ S: Z- V1 a' Q
; x! N$ ~8 {0 V; p' G7 Q; u! Td^CInterrupted. d, @' J6 J: K4 {
Traceback (most recent call last):- s: z! @" @$ t5 l: A
File "/usr/bin/ceph", line 1269, in <module>+ l% y% S( P, | F. A$ {8 A
retval = main()# z$ E4 [8 P5 P/ H; e- i8 l
File "/usr/bin/ceph", line 1200, in main2 c$ L; W* {4 q# Z
verbose)
" \6 [+ l/ k; W4 a" w. t File "/usr/bin/ceph", line 622, in new_style_command
$ d; @+ F. n: Q$ W4 t) Z0 _ ret, outbuf, outs = do_command(parsed_args, target, cmdargs, sigdict, inbuf, verbose). E& |0 x* v0 p# `5 G
File "/usr/bin/ceph", line 596, in do_command
, S2 U v' n, o% P return ret, '', ''
' N% k: V( ]6 d7 _1 s% J) {' pUnboundLocalError: local variable 'ret' referenced before assignment3 j9 \7 z) F7 w# x1 w
[root@compute02 ~]# date( H, _8 s( b& Z0 J
$ _9 t u, o: v连osd tree都不能查看：3 S( r+ w/ ]0 z$ ~' u) ?
没办法，只能查看日志：. D) t9 m) ^- P
; n- k* ~: b" r) J
/ s0 J7 E. \2 C3 V1 W
ceph的log日志视乎没有任何帮助，和ceph -s的状态一样。
2 [: d, x. j9 W- W[root@compute02 ~]# tail -f /var/log/ceph/ceph.log. p# k9 O' O& M! ?& c. u0 Q; r
2021-08-31 14:13:37.918792 mgr.compute01 (mgr.49274115) 15887 : cluster [DBG] pgmap v8821: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
. v8 F; C' o' h; P2021-08-31 14:13:41.694764 mon.compute01 (mon.0) 717737 : cluster [WRN] Health check update: 528 slow ops, oldest one blocked for 477 sec, mon.compute02 has slow ops (SLOW_OPS)/ G: ^' p% d. _& K: A
2021-08-31 14:13:46.695723 mon.compute01 (mon.0) 717740 : cluster [WRN] Health check update: 535 slow ops, oldest one blocked for 482 sec, mon.compute02 has slow ops (SLOW_OPS); d% @& o, n9 @. @9 |4 H
2021-08-31 14:13:39.919835 mgr.compute01 (mgr.49274115) 15888 : cluster [DBG] pgmap v8822: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
; t4 N" F# [3 ^% i3 D2021-08-31 14:13:41.921160 mgr.compute01 (mgr.49274115) 15892 : cluster [DBG] pgmap v8823: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%): @3 T+ D; ~) p1 w3 Z. |& w
2021-08-31 14:13:43.925789 mgr.compute01 (mgr.49274115) 15893 : cluster [DBG] pgmap v8824: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 34 KiB/s rd, 41 op/s; 13701/41103 objects degraded (33.333%)
# w1 i" r0 e: j1 I& V4 y. e& @2021-08-31 14:13:45.927393 mgr.compute01 (mgr.49274115) 15894 : cluster [DBG] pgmap v8825: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)% Y3 c @3 ~8 ^' p
2021-08-31 14:13:47.930528 mgr.compute01 (mgr.49274115) 15895 : cluster [DBG] pgmap v8826: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 7.7 KiB/s rd, 9 op/s; 13701/41103 objects degraded (33.333%)" a- }7 N( l- z k( I3 u
2021-08-31 14:13:51.696665 mon.compute01 (mon.0) 717744 : cluster [WRN] Health check update: 544 slow ops, oldest one blocked for 487 sec, mon.compute02 has slow ops (SLOW_OPS)
7 ]; y3 C' e2 R' P7 H2021-08-31 14:13:56.697487 mon.compute01 (mon.0) 717766 : cluster [WRN] Health check update: 547 slow ops, oldest one blocked for 492 sec, mon.compute02 has slow ops (SLOW_OPS)" [1 P$ }0 ^& ?: g' u* i
2021-08-31 14:13:49.932275 mgr.compute01 (mgr.49274115) 15899 : cluster [DBG] pgmap v8827: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)% l" N. l+ R& r: i% f- n
2021-08-31 14:13:51.934115 mgr.compute01 (mgr.49274115) 15900 : cluster [DBG] pgmap v8828: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)0 e4 z/ p* s! C, L8 c
2021-08-31 14:13:53.938702 mgr.compute01 (mgr.49274115) 15901 : cluster [DBG] pgmap v8829: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)0 E/ R: b; D/ E7 p8 }4 p
2021-08-31 14:13:55.939684 mgr.compute01 (mgr.49274115) 15905 : cluster [DBG] pgmap v8830: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
2 d% q4 J7 [' \: Z7 D- E/ F5 @2021-08-31 14:13:57.944025 mgr.compute01 (mgr.49274115) 15906 : cluster [DBG] pgmap v8831: 640 pgs: 432 active+undersized+degraded, 208 active+undersized; 11 GiB data, 22 GiB used, 2.7 TiB / 2.7 TiB avail; 13701/41103 objects degraded (33.333%)
2 x3 X: |5 f) x( [: I( q2021-08-31 14:14:01.698441 mon.compute01 (mon.0) 717769 : cluster [WRN] Health check update: 554 slow ops, oldest one blocked for 497 sec, mon.compute02 has slow ops (SLOW_OPS) N9 x: O1 L* N, m, l# |8 J! r
^C+ }9 \: L& W- c; d }* t0 m6 Z q
* g: w5 Q* g9 o5 @: b3 c( o
, @! e$ H Q1 N: l& p查看ceph-osd.0.log时发现 osd.0 553 unable to obtain rotating service keys; retrying，意思就是时间不同步，不停的重试中：
; |5 _' n, ^6 C
: I. X: `/ L4 [9 o m" L5 g[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
! x- @- J2 v. hceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log8 r; G1 ` h, M- `' O0 l6 K& B
ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz" V* m( H' t7 h, `( V) V# \
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.
" z6 L# D$ f, n) l! w3 ?' fceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log8 b$ U" u# j# ~/ o* d
ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz+ l: H7 K. Z! d3 r
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log. C; a% A2 m! T1 ?: b
2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
( T( x. t) p+ S: X2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
, S2 {5 d8 p% l2 ?2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 307 w1 X" C Y# i$ U F" Y
2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
c, C/ `8 S" { K7 B1 x2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
: h3 [! X: `: I' Z1 w2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
]1 {: u! A7 c G8 k2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30, r! w7 z: u# ~8 e/ [. e0 [# p5 @
2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying* I5 X* X) b+ r) V/ p0 _
2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30+ }. M. j. b. P" g, T
2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
# ?# O- W% ?& a: J
7 @/ `! ?7 g" j- ?^C
$ S9 K' x* p$ X6 {7 `+ L( d& L4 r. Z
- B# [# M, a$ \% @& n
9 q% u& w; l! H2 n! z
& r# h! i0 T/ w+ q! [" d: V/ [^C^C/ e0 y" k$ b$ X. k# `6 v$ h
[root@compute02 ~]# ^C' J) K1 ^ H5 ]: w2 i3 ~8 Y; x `8 w
[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.5 O( [' |/ h; f0 w. T8 V
ceph-osd.0.log ceph-osd.0.log-20210826.gz ceph-osd.0.log-20210828.gz ceph-osd.0.log-20210830.gz ceph-osd.1.log$ Z' r6 ^( A% ~6 `, s% Q" w
ceph-osd.0.log-20210825.gz ceph-osd.0.log-20210827.gz ceph-osd.0.log-20210829.gz ceph-osd.0.log-20210831.gz ceph-osd.1.log-20210721.gz
: |$ r, q9 d1 m- H, q[root@compute02 ~]# tail -f /var/log/ceph/ceph-osd.0.log
4 K7 C* o' x0 A7 r; U2021-08-31 11:25:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 307 R" }4 p6 H/ N7 A% d2 I
2021-08-31 11:25:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying( I7 D, f! h) f1 Z. p. Z
2021-08-31 11:25:30.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30* l" ]' F/ r' L9 K0 b
2021-08-31 11:25:30.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying4 u/ u" D2 k. ~ a, S3 J1 E
2021-08-31 11:26:00.808 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
; x, t6 F5 C" T; V4 _. i/ O2021-08-31 11:26:00.808 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying9 A! Z/ D* q% w% M2 s
2021-08-31 11:26:30.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
E B4 L- [! ^$ n% f+ U8 U2021-08-31 11:26:30.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying
3 R1 o: E8 s% z V2021-08-31 11:27:00.809 7f80e5e47a80 0 monclient: wait_auth_rotating timed out after 30
8 W6 h( y& U, ~# N9 L# N( |7 r2021-08-31 11:27:00.809 7f80e5e47a80 -1 osd.0 553 unable to obtain rotating service keys; retrying( I* Q& |. Y3 j6 ?" y$ d/ P
^C
2 ?4 W" p2 X% H5 ~8 X* q6 C$ j
( p7 t; h- Z; M& R
9 ^1 m9 A$ M" Z- n$ o+ h& M同步时间吧：虽然安装的chrony时间同步，但我们同步的时候，还是使用ntpdate比较快些：
' [ ?$ T/ ~. A& c( F; ^) {
- o" f1 I% \8 @+ M" X, v[root@compute02 ~]# ntpdate -u compute01
6 I! n& W- t# R1 c7 e& i2 B. F' N31 Aug 14:16:21 ntpdate[6103]: step time server 192.168.0.75 offset -14.901748 sec3 O8 x! c3 A; S% [7 I
( s! ^3 @7 S. N% @. B2 v1 \
同步完成之后，发现osd tree 命令好使了：
7 J& K# J1 W3 p* n# D" Q/ H8 y1 y
5 F% M! p6 Z" @6 S8 e+ k[root@compute02 ~]# ceph osd tree
# I) Z, B) O( N* b" G1 LID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF : f q4 ], U' z7 j j/ e5 h5 v
-1 2.00000 root default
2 G' b6 R0 V" l% o9 B( L) W-5 1.00000 host compute01 7 g& {& R, B( C8 Z! Q
1 hdd 1.00000 osd.1 up 1.00000 1.00000
. c; z* N2 H$ ]% J! A V6 }9 {) T-3 0 host compute02 ) O) I1 ^7 [% J$ I6 W3 E: m: q* s
-7 1.00000 host compute03
; ?( c, p. }4 w4 R, K( l, `3 b 2 hdd 1.00000 osd.2 up 1.00000 1.00000
- T) I' y! N3 M3 f+ v0 } 0 0 osd.0 down 0 1.00000
+ e4 S- Q9 u# n/ I9 e! } 3 0 osd.3 down 0 1.00000 9 A7 b" G! Q) ?! f9 x. L, Z
3 ~$ e8 a$ a# m w
把down状态的osd清理掉：4 U; k- u# }1 V, J$ W8 P
" f* v3 ~8 K& D ^) L1 R[root@compute02 ~]# ceph osd out osd.36 l0 N$ b6 Q5 _5 o+ z& _3 p
osd.3 is already out.
3 H1 J5 T8 j V' q[root@compute02 ~]# ceph osd rm osd.3% S8 o! M4 i3 M0 O
removed osd.3
: a+ l, X/ _# T$ f2 G) F0 j7 ^[root@compute02 ~]# ceph osd crush remove osd.3
5 u/ y. f' U( Gdevice 'osd.3' does not appear in the crush map { p. x, Y+ {; W' E5 r8 d6 }3 O
[root@compute02 ~]# ceph osd rm osd.3
6 ?; ]3 _/ }9 G" @. [+ | |osd.3 does not exist.
* _, ]/ S# k; @( r# o2 ~# b1 ~[root@compute02 ~]# ceph auth del osd.3
7 n0 k6 w: y+ N( Mupdated
1 T% S, R5 H. P' o$ U8 ~6 r' F
* [! ~1 ~* m7 u/ r[root@compute02 ~]# ceph osd in osd.0: N* @, r' d) X$ ^8 a5 L! t
marked in osd.0.
8 J/ d( P$ ^1 `/ }, v[root@compute02 ~]# ceph osd tree
/ r. D. X$ ?7 H, N# T7 EID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF ! x2 j# F$ y4 D$ Y- P7 T7 N
-1 2.00000 root default
$ y, w) K0 J$ P! W& W-5 1.00000 host compute01 ; ]- a, j( W9 t9 G( ?6 w
1 hdd 1.00000 osd.1 up 1.00000 1.00000 4 d' n( m0 z8 A3 B( n& c* q' J( c
-3 0 host compute02
" r$ q! E; u& \+ C$ c1 @; w-7 1.00000 host compute03
" ?( u, o" w% A 2 hdd 1.00000 osd.2 up 1.00000 1.00000 $ _3 e' O% |2 X7 O% b0 j
0 0 osd.0 down 1.00000 1.00000 4 d) \; X% Z8 g! ~9 U) s( J
[root@compute02 ~]# systemctl restart ceph-osd@
- L: J3 Y* q" o' E5 h' J3 ~7 A* s8 \Display all 337 possibilities? (y or n)
, B. Z4 {1 Y' O y; y$ O& e: f( k[root@compute02 ~]# systemctl restart ceph-osd@ 0 m0 e6 l9 Z/ B: P6 I
Display all 337 possibilities? (y or n)( X9 n$ ?$ _! I9 c
[root@compute02 ~]# systemctl restart ceph-osd@00 t% c1 `2 p' O* g4 f
Job for ceph-osd@0.service failed because the control process exited with error code. See "systemctl status ceph-osd@0.s& q, N5 b6 G5 Q0 b: T& j; Q" t
% L1 ^: P8 t. K/ Q6 K0 I
& p4 B) C# l" m$ V g* V8 Q' ~2 |9 D
重新添加ceph磁盘osd：/ h. U. L+ V/ `! p% e5 K& Z% G
. q" B/ J* p2 Y) ^& s% @5 w[root@compute01 deploy]# ceph-deploy osd create compute02 --data /dev/sdb
; v$ f4 P! T) l3 ?[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf+ q0 U5 V0 E0 A4 R
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy osd create compute02 --data /dev/sdb" M _- b7 i5 z: Q! b
[ceph_deploy.cli][INFO ] ceph-deploy options:
/ [9 F: U3 Q7 ^: ~+ S2 ~[ceph_deploy.cli][INFO ] verbose : False7 I3 w) J4 A9 D4 M! A+ B9 H- R
[ceph_deploy.cli][INFO ] bluestore : None
9 ?( a h3 r- k0 c9 b% H[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f518f889690>
% _# k6 V0 x: T# T[ceph_deploy.cli][INFO ] cluster : ceph6 N. N8 R0 w5 {% z: K
[ceph_deploy.cli][INFO ] fs_type : xfs$ f, w/ [$ p" ?" q
[ceph_deploy.cli][INFO ] block_wal : None% X0 r& q) G4 P5 z0 ] `
[ceph_deploy.cli][INFO ] default_release : False
, h% \2 y6 v( t. e[ceph_deploy.cli][INFO ] username : None! R3 M- [! E; a }+ S
[ceph_deploy.cli][INFO ] journal : None$ ~+ B q- g. v, |' w4 y* V. I
[ceph_deploy.cli][INFO ] subcommand : create
, u9 E3 e% A" a" Q8 Z9 H[ceph_deploy.cli][INFO ] host : compute02" R+ z5 g0 b$ [1 w, \
[ceph_deploy.cli][INFO ] filestore : None+ j* ^6 V, g+ W7 Y" M7 u
[ceph_deploy.cli][INFO ] func : <function osd at 0x7f518f8b71b8>
- u6 \+ a/ G. p. z[ceph_deploy.cli][INFO ] ceph_conf : None# i3 s! y7 ~1 V
[ceph_deploy.cli][INFO ] zap_disk : False8 Z8 X' G# T$ v
[ceph_deploy.cli][INFO ] data : /dev/sdb
4 p' A3 ~5 l4 P5 \2 ?+ d[ceph_deploy.cli][INFO ] block_db : None8 b; m' A( H' k1 f2 L. {! |
[ceph_deploy.cli][INFO ] dmcrypt : False! W( e z) J8 `% \' E1 x
[ceph_deploy.cli][INFO ] overwrite_conf : False
3 J- ^/ r/ [9 I* j[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
1 j$ P/ a; d* ]$ R9 l* B6 h$ }/ H[ceph_deploy.cli][INFO ] quiet : False
; u8 s- `# p% o: e/ j[ceph_deploy.cli][INFO ] debug : False. `7 a2 [. k- N& G2 T1 J
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
: ^: L3 O& P- ? y, P[compute02][DEBUG ] connected to host: compute02 , a5 K6 `7 r2 q
[compute02][DEBUG ] detect platform information from remote host
5 [( F& Y2 H( w; c) F* v[compute02][DEBUG ] detect machine type
- V( J& h# ]+ ~ K[compute02][DEBUG ] find the location of an executable
' a$ X% [/ D4 t[ceph_deploy.osd][INFO ] Distro info: CentOS Linux 7.9.2009 Core9 _' N3 t( c5 z3 r
[ceph_deploy.osd][DEBUG ] Deploying osd to compute02, O |' j; b7 q( ~: F+ {* ]' G
[compute02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
- W$ d' x9 O/ u) d$ S[compute02][DEBUG ] find the location of an executable4 {5 T, G1 I# e4 u8 e4 X3 [, m
[compute02][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb, M! [( K% w% L
[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key0 K) Z g7 m# s; A+ y) C
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new bbfd988d-bb2e-44f2-a366-17501a8bbada( U8 }8 U5 j4 c8 A2 m
[compute02][WARNIN] Running command: /usr/sbin/lvcreate --yes -l 100%FREE -n osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463
9 Y( D) R/ w. R7 V/ s- p6 A; S[compute02][WARNIN] stdout: Logical volume "osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada" created.
1 j% G; G, \+ i[compute02][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
5 u) c" }1 X' D7 k5 }8 a1 w% A[compute02][WARNIN] Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0: ~' e& | S+ o* m" ]! Y# p* [% b
[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada
- }) a/ s! {! x+ V4 [[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
4 V6 ~& X: E7 d" l: u# q$ M[compute02][WARNIN] Running command: /usr/bin/ln -s /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block) J4 [7 v- c! w7 ? a$ Z
[compute02][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap
4 }, T0 x) H) n- l6 M3 s7 b& [[compute02][WARNIN] stderr: 2021-08-31 14:21:34.978 7fb58c080700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory, [- D7 j* s8 @ i+ j/ o+ `( _/ {
[compute02][WARNIN] 2021-08-31 14:21:34.978 7fb58c080700 -1 AuthRegistry(0x7fb584065de8) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
. b' ^; w9 \5 c U+ j[compute02][WARNIN] stderr: got monmap epoch 3
( d& e: N; z" k[compute02][WARNIN] Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==" @' A+ H: B$ \! T. l; g
[compute02][WARNIN] stdout: creating /var/lib/ceph/osd/ceph-0/keyring
8 [' ~0 [# g5 A/ ^2 K8 I[compute02][WARNIN] added entity osd.0 auth(key=AQBsyi1hTYl7OhAAMzTvAS8eKANTVRi882qXxw==)* [, H/ d/ \! [' E$ n6 C# l, Q; u
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring+ P1 Z3 q) u r. `- W
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/' R2 J' K8 f0 h( N, s9 r9 T
[compute02][WARNIN] Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid bbfd988d-bb2e-44f2-a366-17501a8bbada --setuser ceph --setgroup ceph( Z# \3 K/ x% s) R+ T" J
[compute02][WARNIN] --> ceph-volume lvm prepare successful for: /dev/sdb4 Q6 S% c, s1 V b$ e! p3 m
[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
+ X/ ]" |) V- f6 m) K1 @[compute02][WARNIN] Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada --path /var/lib/ceph/osd/ceph-0 --no-mon-config$ O! k& q2 p3 V7 s" \
[compute02][WARNIN] Running command: /usr/bin/ln -snf /dev/ceph-0f6fa318-810d-4df6-8d52-afeac6f9a463/osd-block-bbfd988d-bb2e-44f2-a366-17501a8bbada /var/lib/ceph/osd/ceph-0/block
! @% p% u" S8 j2 t9 H$ u5 c" a[compute02][WARNIN] Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block
Q$ K) B" S. Q2 t) p) N[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2
- _) M$ l. j& n[compute02][WARNIN] Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0: }& m, p. ]# H! R* f% @
[compute02][WARNIN] Running command: /usr/bin/systemctl enable ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada7 q3 r& q. L/ d' [1 i1 L, S8 \0 T F
[compute02][WARNIN] stderr: Created symlink from [url=]/etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-0-bbfd988d-bb2e-44f2-a366-17501a8bbada.service[/url] to [url=]/usr/lib/systemd/system/ceph-volume@.service[/url].# M; `! f5 q5 \' @* H# G R+ J
[compute02][WARNIN] Running command: /usr/bin/systemctl enable --runtime ceph-osd@0* O. \ o. d7 f3 c; \
[compute02][WARNIN] stderr: Created symlink from [url=]/run/systemd/system/ceph-osd.target.wants/ceph-osd@0.service[/url] to [url=]/usr/lib/systemd/system/ceph-osd@.service[/url].$ L8 k. j) j' }% x
[compute02][WARNIN] Running command: /usr/bin/systemctl start ceph-osd@0! y6 p; X5 @+ l- e; S. L
[compute02][WARNIN] --> ceph-volume lvm activate successful for osd ID: 0- U8 o1 d4 M' Y! d. ^. S
[compute02][WARNIN] --> ceph-volume lvm create successful for: /dev/sdb! I r6 U: F, d+ r& x& ]7 j9 Q
[compute02][INFO ] checking OSD status...
0 F% G5 e6 Y4 t[compute02][DEBUG ] find the location of an executable4 y0 q% F! x( k6 _% c& h8 o$ O) x
[compute02][INFO ] Running command: /bin/ceph --cluster=ceph osd stat --format=json2 J* L: K( o0 o! l$ Q8 u$ u
[ceph_deploy.osd][DEBUG ] Host compute02 is now ready for osd use.5 E# H. _" p7 S, B- i- P+ d
: Q; g1 f0 Q2 L/ ?
* E9 E. y2 o& K* ? W1 @成功创建。这个坑确实有点过分。ceph存储一定要先解决时间不同步问题，否则有很多意想不到的惊喜：
: Q* `% u3 k7 i- A9 _( P0 `+ l6 |3 K
. e4 c5 E! d$ B/ U1 L0 z( Q
[root@compute01 deploy]# ceph osd status / H6 D# z# o1 c1 G E7 o
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
7 {7 ^5 g% n3 K+ }/ f' X' N8 z| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
# a; G8 q; C* e+ O3 [+----+-----------+-------+-------+--------+---------+--------+---------+-----------+# Y, G( a3 z3 Y' v7 ^! Z4 _2 w- @
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |2 c; c; `* c7 e8 ?7 r" A
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |/ M8 a, j, }# j, P
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
9 {" M4 h) E! ?; g) ^! o' o* S+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
" t) C+ W7 r% s: d% {! r& a, r$ a* _[root@compute01 deploy]#
|8 X, A. E( C$ C[root@compute01 deploy]# + L; t8 a+ ^. ?- {; ?
[root@compute01 deploy]#
7 v) a5 h3 e! M) s) x[root@compute01 deploy]# ceph osd status ) C6 Z8 \7 n: Z/ S, K: m3 L
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+4 T( y+ O# p+ l. I$ H; q: x
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
8 N2 o9 I% s5 { \+----+-----------+-------+-------+--------+---------+--------+---------+-----------+6 l1 t/ f0 e5 {" X9 D! a
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |. V. G' i. o; g+ M+ l# E6 ^
| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |
7 v& R M) g( A9 H9 C| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |$ H) P; r6 a7 ?% G5 N' k" ~
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
* P# r7 Z ^( ?7 I3 N: L[root@compute01 deploy]# ceph osd status
; O7 ^- Q% j5 I% y) T" N9 Y+----+-----------+-------+-------+--------+---------+--------+---------+-----------++ g1 f/ E5 A6 _
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
m3 V* P1 U# c, x; J+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
4 l7 K+ S5 u0 B| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
}$ j$ Y4 ~' R3 l4 L6 x% I! I| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 10 | 327 | exists,up |
" \( T) J: d' q" B* K' j! X V| 2 | compute03 | 12.1G | 546G | 0 | 0 | 9 | 266 | exists,up |) x! ]3 u: S3 _
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
0 X, ]$ V7 J) X& k! L[root@compute01 deploy]# ceph osd status 4 A5 c5 Z, t) E& c3 K* T g3 y+ F8 w
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+% Q5 @# R" ~2 Z- T
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |- z* c4 l9 w y' a' B
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+- ?' s8 m4 G4 K% X& G% z; t
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
0 J9 Q* M3 P) K0 K6 |6 ?* V| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |
+ q6 P9 m/ r: V0 [5 R| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |; P1 u3 P2 m. j! G. B. A, Y
+----+-----------+-------+-------+--------+---------+--------+---------+-----------+8 r0 N9 A* C% `6 W' X7 ~+ Q
[root@compute01 deploy]# ceph osd status
. C9 D7 Q' K# _8 |+----+-----------+-------+-------+--------+---------+--------+---------+-----------+ b4 e L; i" V" F. i
| id | host | used | avail | wr ops | wr data | rd ops | rd data | state |
$ m) p# x! ^- r. q% U$ @( E/ {, Z2 h+----+-----------+-------+-------+--------+---------+--------+---------+-----------+, J! r* z. D0 H& D" Y4 i; N- ^
| 0 | compute02 | 1096M | 557G | 0 | 0 | 0 | 0 | exists,up |
- W. Y* d1 n" @' m" o+ Q- b7 p| 1 | compute01 | 12.1G | 2223G | 0 | 0 | 0 | 0 | exists,up |' i1 g, U( t9 H
| 2 | compute03 | 12.1G | 546G | 0 | 0 | 0 | 0 | exists,up |
% J& f* Z4 L1 p+----+-----------+-------+-------+--------+---------+--------+---------+-----------+
3 o; O$ o$ f5 n |
|
发表于 2021-8-31 15:00:09