- 积分
- 11013
在线时间 小时
最后登录1970-1-1
|

楼主 |
发表于 2023-1-12 19:52:38
|
显示全部楼层
启动chronyd服务,检查服务7 ~+ s. j- ~8 f$ i* i" d
[root@tp266-1 ~]# ceph orch host add tp266-1
# L$ b; _7 Z; B2 kError EINVAL: New host tp266-1 (tp266-1) failed check: ['podman|docker (/bin/podman) is present', 'systemctl is present', 'lvcreate is present', "No time sync service is running; checked for ['chrony.service', 'chronyd.service', 'systemd-timesyncd.service', 'ntpd.service', 'ntp.service', 'ntpsec.service']", 'Hostname "tp266-1" matches what is expected.', 'ERROR: No time synchronization is active']% M1 g) i3 W' U; `& b
[root@tp266-1 ~]# systemctl status chronyd.service
! I1 ]- [5 c! D4 L● chronyd.service - NTP client/server2 I: H- Z/ `4 P/ Z7 {" l$ C6 N
Loaded: loaded (/usr/lib/systemd/system/chronyd.service; disabled; vendor preset: enabled)6 B8 g3 N1 m9 p
Active: inactive (dead)
' m* l& ?% E! X9 n5 f5 s! @- y& Q Docs: man:chronyd(8)
" X! \' I- r* |9 P4 N$ r8 { T& Q( L man:chrony.conf(5)4 a0 k c5 A. b
[root@tp266-1 ~]# systemctl enable chronyd.service
5 N: B7 \# k4 j$ k2 a7 fCreated symlink /etc/systemd/system/multi-user.target.wants/chronyd.service → /usr/lib/systemd/system/chronyd.service.
+ f# ?5 U9 X5 n9 ?, @! {- _[root@tp266-1 ~]# systemctl start chronyd.service
9 r2 t( D1 {& X% j8 Z0 b[root@tp266-1 ~]# ceph orch host add tp266-1/ H3 H+ d. E# \! R0 d# ^% w5 E
Added host 'tp266-1'
I* i8 P; {5 d0 h[root@tp266-1 ~]# ceph orch host ls& I: b- b+ P6 V5 |
HOST ADDR LABELS STATUS ! Z$ y' x. r0 L" d0 G- z& }3 D) G
gm268-1 gm268-1 mon mgr mds
; K9 t/ Z$ L: @4 v [. dgm268-2 gm268-2 mon mgr mds % b2 k1 L: _ x# F
gm268-3 gm268-3 mon mgr mds . o% Y' ^) h; b4 w2 S U
gm268-4 gm268-4 Y9 Y( j, u# k, \1 z
gm268-5 gm268-5
T i( @& ^, ^# c2 ^tp266-1 tp266-1 4 q- P8 c8 \$ U8 _) P5 D) N: T9 M% {' o
[root@tp266-1 ~]# ceph osd tree
T. w. Q* a; {7 sID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
) x9 |1 G$ B6 Y B -1 494.79141 root default , ^/ D9 b( z$ k6 U u- Y
-3 101.86882 host gm268-1 6 f( H& S6 ^; C, t
0 hdd 14.55269 osd.0 up 1.00000 1.00000
2 R: s- p+ v# M/ H 5 hdd 14.55269 osd.5 up 1.00000 1.00000
, c+ A- z6 c5 B" q4 r 6 hdd 14.55269 osd.6 up 1.00000 1.00000
6 F/ R. _! ]7 i2 J 7 hdd 14.55269 osd.7 up 1.00000 1.000007 x. [8 `+ F! S# U) x/ G
8 hdd 14.55269 osd.8 up 1.00000 1.00000
" V$ u4 k: U1 _# ~* C 9 hdd 14.55269 osd.9 up 1.00000 1.00000
, l1 J0 B* e8 o. O$ v+ X1 Q( Z* D; h 30 hdd 14.55269 osd.30 up 1.00000 1.00000
, `7 K" h" i" R% O6 X$ R6 f$ c2 Z: H -5 101.86882 host gm268-2
" G- C/ g" K3 ^7 G* i( u 1 hdd 14.55269 osd.1 up 1.00000 1.00000
8 y5 x- Q* ]8 i/ i% D( J 10 hdd 14.55269 osd.10 up 1.00000 1.000004 }/ F7 ?0 D; D4 ~8 Y8 o5 v+ }- N
11 hdd 14.55269 osd.11 up 1.00000 1.00000, p. v4 ]+ |; H# {5 U
12 hdd 14.55269 osd.12 up 1.00000 1.00000 w' `6 R1 P- }5 x
13 hdd 14.55269 osd.13 up 1.00000 1.00000
* M8 g' j( R+ I( @5 r+ H7 z$ O 14 hdd 14.55269 osd.14 up 1.00000 1.00000
4 q7 l( K! P2 e+ ` 31 hdd 14.55269 osd.31 up 1.00000 1.00000$ X* c9 _: ^0 a8 G% F, t
-7 101.86882 host gm268-3
) M1 S; C2 ~) L, p 2 hdd 14.55269 osd.2 up 1.00000 1.00000
# Z l) L/ Q. ^; k5 q' I) O 15 hdd 14.55269 osd.15 up 1.00000 1.00000
; D* R/ G: j1 |3 v' E 16 hdd 14.55269 osd.16 up 1.00000 1.00000
' ]/ C- N7 ^4 F8 w) d1 V3 n 17 hdd 14.55269 osd.17 up 1.00000 1.00000* ?: `# w w0 m$ o
18 hdd 14.55269 osd.18 up 1.00000 1.00000! _: z0 t; g6 {+ T8 ~9 p9 ~6 N. _
19 hdd 14.55269 osd.19 up 1.00000 1.000003 |5 S( G [" @4 ]7 C: C
32 hdd 14.55269 osd.32 up 1.00000 1.00000
* S* L7 s1 @8 a( X) J -9 101.86882 host gm268-4
, t) c: q: o( z5 A* ? 3 hdd 14.55269 osd.3 up 1.00000 1.00000
, T/ ]% x" X! H 20 hdd 14.55269 osd.20 up 1.00000 1.00000
2 ^. V1 F3 P! x- d: ^' q' k 21 hdd 14.55269 osd.21 up 1.00000 1.00000
0 ?; ~4 c% w! i' W4 h! M 22 hdd 14.55269 osd.22 up 1.00000 1.00000
$ E% Z' o# b5 K6 ] 23 hdd 14.55269 osd.23 up 1.00000 1.00000
, L8 v8 r# V& V2 R 24 hdd 14.55269 osd.24 up 1.00000 1.00000
( d+ x3 J: |+ @ G, j7 Y8 O* Q 33 hdd 14.55269 osd.33 up 1.00000 1.00000. i+ Q A7 E4 Q, z) E0 S
-11 87.31613 host gm268-5 # N' u, ]* J) K( S
4 hdd 14.55269 osd.4 up 1.00000 1.00000
, u7 ?# [% {4 S 25 hdd 14.55269 osd.25 up 1.00000 1.00000
7 d- N$ I+ Z) w, @% F% R( e( h 26 hdd 14.55269 osd.26 up 1.00000 1.00000& ~5 m8 y( o6 @2 ]& O, u& v' p
27 hdd 14.55269 osd.27 up 1.00000 1.00000
/ g$ G) f1 V: R0 e: v2 L 28 hdd 14.55269 osd.28 up 1.00000 1.000003 F6 w& p: i7 B# Z1 k5 H
29 hdd 14.55269 osd.29 up 1.00000 1.00000" n0 d4 ^% l. L8 x
[root@tp266-1 ~]# ceph orch device
8 N; I' g: D& O$ q+ X' kls zap ) O7 y. ~ h1 H- q
[root@tp266-1 ~]# ceph orch device ls
( n6 k- D5 Q1 ]. S1 A9 n' }& s- DHostname Path Type Serial Size Health Ident Fault Available 6 @2 l0 u; D5 p0 [
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No & g8 g" v& ]" l9 Q& ^$ @6 B
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No & e) I! ~- f( e
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
6 g/ A0 J P1 P$ H# L( agm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No $ T$ d$ c! I4 z/ f# y3 @3 Q6 j- r
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No ) C* A) h: g- Z: y
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No $ j* ]2 y- N9 ^6 H2 ~1 C) f
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
1 v" N2 H8 O( R2 Y, v4 \( Wgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
, s3 ?/ s& ~3 z) ]! N# Ygm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
+ K- Y T+ Z) m+ B0 a. Wgm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 3 G( c2 z0 M) F! P! Y
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No # W0 U* j& U' Y0 L4 B* N4 o, N
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No ' l4 L' a' b7 B5 T7 D/ m
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No . N! |5 Z1 N4 @* N
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No / h' \3 y$ Q$ n6 ?4 Q+ K
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No / }9 Y. _: Z( B0 K% ] \7 ]
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
( u }5 A" ?/ f5 ]gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
1 ^- r( E& f3 ]6 C1 ngm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No # _1 z$ L& I- V5 E1 {! H
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No C4 M! L3 e& e- q
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
/ x# M/ ^1 H' ?# W/ Sgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
; I; c& ]! B: _& D4 qgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No ( P" q+ c. d) _5 Z
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
: e+ K, J$ u6 H) d2 ]: e* f+ w cgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No ( D# s( }. v5 W# ^3 Y7 a) q& g
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
8 j3 p8 e2 y' {' q! I( B! qgm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
( X5 [5 _; P0 R) H6 cgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
7 A( A! V, v' N! `gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No * |; X6 C( T- W
gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No 0 k2 r; m7 p" L' B$ P; O, R6 l: o1 J
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No 9 G6 s ~6 Y& q
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No " q8 a: G( H3 F( O$ y. W$ a. k
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
" Y2 \( U% h$ _gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
+ ?" e3 U/ z ogm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No 4 C ]) B$ N* w2 c6 s
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No & G- ]# w2 Y0 o7 s
[root@tp266-1 ~]# ceph orch device ls 6 u+ y( v, c/ l! a$ j) z3 s/ Z# x
Hostname Path Type Serial Size Health Ident Fault Available
3 ~1 Q; K+ _! [; _) \1 [gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No / p2 u1 w+ @5 o+ u9 }
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No $ c" C9 f* {& a8 M. S, d
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No $ P3 Q( k! m, E( c. y
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
, {+ X2 x* i: n1 J3 Ggm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
$ G% o* O9 P) f) l3 dgm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No ; h( G: D9 c5 T f4 g u
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
3 |6 z3 Z; z2 v, u1 d' Mgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
' |: J" M. A( Z8 G* ~3 i: K6 Jgm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
0 p) a7 n; \5 I$ Ogm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
4 E* z2 g0 D5 b* ?gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
$ P9 s0 q2 E( x7 K0 |+ Q( ugm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
3 k2 f+ y8 ]- V, y2 q. qgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
& T P. n7 q' M1 H" Fgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
0 d# q9 b- r1 U: H& R( Z. z# Tgm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No " a- c k _4 c' i8 K" v* Y
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
8 V/ ^4 W. O) X/ I5 h" j2 @gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No 6 i: Q5 r0 s3 J% }3 u$ L# D
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No 1 c7 P- W ]% T
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No 3 i9 M9 W: V8 t5 E
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
3 \) h& ^7 b8 D; L8 ~8 I sgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
* s& c; v; F) G0 t Bgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No 9 I* h p% l7 P' q; j* n
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
6 n% |8 T) Q) j. ugm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No * l3 ?/ W6 S/ i" h! L3 R
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
# ~8 ]. A p( Qgm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
2 f: C0 b! I }gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No 8 f1 g0 A2 s% h, p. v& S
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
- ]: W, \% v) n( Q# u/ [ C7 _gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
* g" k- i4 K! N$ Bgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No ) [% S7 C4 U- p- s( X% h6 [% D
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
5 p0 M* L& ^. ~( W% P1 b# hgm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
s) l, ~+ r R! {gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No 6 ~" ?% e2 E/ j! j& h: O
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
; ~! K8 N2 B4 ?+ Kgm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No 1 ]: @2 Y Z8 }; j. J# ^# {! {8 n
[root@tp266-1 ~]# ceph orch device ls
+ a6 n9 h, `! z. |+ J' n1 a8 v; @Hostname Path Type Serial Size Health Ident Fault Available 1 P1 f, Y# l1 x) H0 k
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
9 o' Y- F1 v6 d$ _0 ~gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No 1 t8 c% S6 I! y, T" k' v
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No ; ^+ Q+ _& o$ P4 l( ^/ Y. ^
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No ; V( t* D) z! `* k: ^0 C/ }
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
5 P: w: r ?! z; Bgm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
& x' o2 |/ @3 C3 Lgm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No ; r5 W. ~3 O4 h* k
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No ( s0 f2 v! K2 F$ P
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No ; T+ b/ n$ ~+ N6 v3 P; M
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 7 u4 T3 d! i; b; [
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No : \/ K/ \0 e2 a
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
" O- K" h# S! v. L! D) _6 Ugm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
/ o' l( |! [, y$ q$ jgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 2 N! x/ G$ M8 U# d
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
: I$ N& d$ {4 a0 }. F# A- v2 [0 w# |gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No 3 R V( h/ m; ]7 Q
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
# ]+ j; `- m) h) S5 M& {gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No 7 [2 n$ M) N! A/ w" K' b
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No 3 ?( Q% U% G+ L
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
$ t1 G7 X( O# p, M; e' f2 r' E( Cgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No 8 j) y/ C q% M% ]0 p G
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No & N" k; L# ?3 w4 F* a6 ^
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
2 q9 V# h" |- N) r1 W9 Egm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No , H+ r' o- I5 o) h/ r& {( M) z
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
* O5 l4 |. i1 Z1 Y. v! Pgm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No ! v5 D( p2 n1 E% @4 U8 u; F+ R
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No $ V$ q' j9 q+ w
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
2 {, O( r4 G2 W+ A5 agm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
! z5 V9 a! B1 j- Ugm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
. u; o5 f# W2 \6 Egm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
* S9 n& _$ Y2 Bgm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
' u" s& y/ @. y+ w% ggm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
6 z4 [" t( G& K7 Ngm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
\, G- H5 N( N0 }' Egm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No * E! {% V) N" e8 ~' k
[root@tp266-1 ~]# ceph orch device ls
" P' s0 W1 v9 W# `) Z( y2 rHostname Path Type Serial Size Health Ident Fault Available 1 q: e9 A7 J* ` h6 k3 Q! i
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
# s1 a& H- `& Dgm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No + z% s; a9 t4 r3 f
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
: h. q, q0 |& @" n, \3 egm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
) ?+ v9 o7 G! b1 ^1 y0 zgm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
0 @# u$ G# b9 q2 P2 w+ qgm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No ( R( ?, j! L" D6 B% h8 R9 Q
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No / q& l0 E1 d$ @* c% W! K& _; ~
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 6 j' S \) o/ k0 f+ G
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No " m; a" J. R- R! w8 O) i
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No . U% ^3 {1 E# n4 z- L
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
9 b$ y4 b; t, F; \0 r% }% k* ygm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
+ \: `6 ~: B7 n7 C1 G! lgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
3 f9 c/ |+ M" ?9 W- @' Y9 a! ]7 jgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
6 [) U, p1 [2 K0 O- ugm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No . G# h+ y% O8 o, g; M
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No ' D3 |* ]9 A( e- \ S7 d
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No ! c3 t" U% [' P9 O7 s' m0 \6 b
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No 3 g+ H& T( |! e! ?
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
( c3 K5 W$ W0 g! bgm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No + f7 I$ Q* x6 ~8 d' t1 \
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No ; u, @ o# I- L1 f& O. T
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
' L( v! N* t: o; X3 @gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
+ l+ ^8 |* s( j' ~6 p+ pgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No - t+ J5 H0 U- q$ j1 ]1 Y1 }" I
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
8 n; m- D) K- u& \5 o2 f, s6 Z& |gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
3 d$ t9 {0 \8 a9 v2 Tgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
- b1 o0 G: H% R" F x6 P) e @gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
/ ]/ U% H, \8 s C; @/ b4 p/ ogm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
. {7 _5 }. b! W kgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
5 h: Q. ?- V2 S1 R3 _& jgm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No & g2 l) i# X# v0 R/ t
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No 3 k* v; @7 U# H2 H, Q- F
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No ) V. I" x! K- ~. c( n% g7 y
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
( W( ~9 {+ G/ ]( Z: w) M$ cgm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No
/ l) W0 @ b& d& x0 `$ _% g" s, W[root@tp266-1 ~]# ceph orch device ls . `' P% A$ {: h( l) w n3 c9 G( x
Hostname Path Type Serial Size Health Ident Fault Available 2 ?# Y" v! U4 |1 p2 c! A( [
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
9 `7 r4 f A* A" sgm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
`+ O/ s+ ~/ o( ]- I* e& _8 s4 Hgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No * G9 C* R$ o$ u6 N
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No ! a( X/ z8 I. C" a
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No 9 a. _9 c5 X/ D0 ?
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
: I2 q4 A( T* Z0 @) V/ v1 @gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
% I; n0 a$ X6 c+ j" Cgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 5 E% T7 \, {$ y+ k+ Y5 j3 ?$ t
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No * b! j) @9 \8 A/ h; V" C( Z
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
3 V# }; y; y7 K" V2 d/ ngm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
) `6 C8 C7 C1 h: C/ K3 C: \9 |: Egm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
8 u8 c; {) ~) n+ qgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
3 D1 Y" H" y; w5 s' rgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 0 c1 D o, A) ?4 ~2 Q
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No 4 w0 N: R6 ?+ ?4 i0 t" n1 |( k
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No 5 \2 v" O% B& H$ p% y$ ^
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No ( O8 D) N& ~4 \8 T2 K
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No 2 e0 _1 n: ]# L6 m; @7 w( _$ I$ m. K% |
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
6 k( H; A/ L; O7 f; ~3 |gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
- S. x8 y+ [" Z! n5 Sgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No # o4 l2 d+ m. Q+ G. W
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No , Y8 [' p0 `: Y- I2 i* o& x. w
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No 1 }: H# l. F; Q8 N/ m
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No % n$ f) H" C4 U, @. J
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
# U% y; e% f/ F. I1 @gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
6 U/ s3 W' S, h) j6 E1 Fgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
: I/ L' X' H2 `' f. ngm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No 7 p, O$ N, }& N! y! p7 `
gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
2 E( T6 R- z% f+ mgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
2 E$ A$ Y" V7 a0 k* b+ cgm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
! B- p2 k3 f& l# b% @gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No 9 ~7 F# v- a/ G4 p
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No ' ? V" x/ H S( i
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
" h5 _. H2 s/ V% q z0 b9 {gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No 6 @" ]9 a( t) j
[root@tp266-1 ~]# ceph orch host ls
2 b4 h5 u5 a4 _" y( w. cHOST ADDR LABELS STATUS
& P6 J, O3 u- Q: z2 q! Rgm268-1 gm268-1 mon mgr mds
, ^% h e" q$ F3 F$ @+ s! sgm268-2 gm268-2 mon mgr mds 7 q5 c7 \) s4 W# ^! W+ o" w/ q
gm268-3 gm268-3 mon mgr mds 5 d. i8 P5 O& U. a6 {2 s
gm268-4 gm268-4
, ?+ \/ w! c5 `; R$ ygm268-5 gm268-5
2 n" \; ~5 N6 f( Wtp266-1 tp266-1
# v" `; E$ o, w4 r+ S) i4 r, _[root@tp266-1 ~]# podman ps1 P) W1 a$ Q2 P4 W
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
$ B5 Q3 E. p% N; f& c& D[root@tp266-1 ~]# ceph orch ls : r$ @( M# T5 V9 o
NAME RUNNING REFRESHED AGE PLACEMENT IMAGE NAME IMAGE ID
! B+ f; S8 w: c$ y- U* R ralertmanager 1/1 72m ago 3d count:1 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f
# W$ x* Y3 c7 z. P0 V1 s1 d7 ~crash 5/6 72m ago 3d * quay.io/ceph/ceph:v15 93146564743f
3 ?) R7 N ?& w' q. K5 K) Dgrafana 1/1 72m ago 3d count:1 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646
( _7 S9 j8 d4 C' vmds.cephfs 3/3 72m ago 3d label:mds quay.io/ceph/ceph:v15 93146564743f
4 W) N" G4 `4 }3 e4 i4 vmgr 3/3 72m ago 3d label:mgr quay.io/ceph/ceph:v15 93146564743f ) _/ J( t! p, Q6 v+ V/ V; P
mon 3/3 72m ago 3d label:mon quay.io/ceph/ceph:v15 93146564743f
, m7 [! Q. b7 P- pnode-exporter 5/6 72m ago 3d * quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf
0 `% _. a% W( z% F1 x; d% Z3 A, mosd.None 34/0 72m ago - <unmanaged> quay.io/ceph/ceph:v15 93146564743f
( q9 ~: I( Y, q, H/ ~+ l9 }6 kprometheus 1/1 72m ago 3d count:1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 $ \; t* }4 m3 }% o7 `
[root@tp266-1 ~]# ceph
" w% y$ l/ o2 H7 z: L: M/ U4 oalerts config-key features insights mgr osd rbd telegraf versions
* g `+ t2 X; R7 u$ b1 P; Xauth crash fs iostat mon pg report telemetry zabbix; Q; u. J# X0 g9 x h1 [8 z6 Q3 j: G
balancer dashboard fsid k8sevents nfs progress restful tell $ t3 o% @# X- V
cephadm device health log node prometheus service test_orchestrator % d; Q/ }6 k4 d3 |/ F5 \* J3 s
config df influx mds orch quorum_status status time-sync-status $ V2 Q- X; Y) ?/ K# E0 `
[root@tp266-1 ~]# ceph 8 }$ Y0 S: [" T/ o5 {+ L, y
alerts config-key features insights mgr osd rbd telegraf versions
; _% q& G' C2 g/ Tauth crash fs iostat mon pg report telemetry zabbix( C" n- `6 ]1 T
balancer dashboard fsid k8sevents nfs progress restful tell
$ K8 F- r, e6 }- H9 Ucephadm device health log node prometheus service test_orchestrator 3 n. y6 s6 v& `2 P7 l; I) I
config df influx mds orch quorum_status status time-sync-status
' N, w( I) p. K2 Y/ u8 f/ x[root@tp266-1 ~]# ceph orch host ls- j3 z9 x: E4 y4 D0 |. o2 [7 p
HOST ADDR LABELS STATUS 0 ]8 a1 I0 u0 U* z8 N( l' i. C
gm268-1 gm268-1 mon mgr mds
' y$ E1 M% A- H8 R9 Rgm268-2 gm268-2 mon mgr mds " X) | {7 B5 e9 T
gm268-3 gm268-3 mon mgr mds
* B; l9 r; K2 K. egm268-4 gm268-4
" x1 t) _6 W7 L" L$ _6 xgm268-5 gm268-5 . }8 A6 v9 B. @0 Y. Q2 u ^" L; p$ l
tp266-1 tp266-1
! w4 H& Y6 c5 ^[root@tp266-1 ~]# ceph orch device ls
. V4 c' K% ?2 Z7 a% CHostname Path Type Serial Size Health Ident Fault Available
# b2 t f& ]7 }2 ]6 d0 I( [gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
8 V8 P, w6 g; s R" ~gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No 6 B" K! @4 _* ]
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
) G3 ]' E0 h9 G+ W, `; dgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
' S$ v/ |; b: B, Ogm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No 1 R7 m' G k: H3 |6 B
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No 3 A) m8 _ H$ Y" N+ X
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
8 |" {4 F7 W9 g( K' w3 Ogm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 6 n* Y) f q& o+ @1 l
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No & N: w5 F0 W% \- R" @7 q7 M. [
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
# B. c; {+ G( T& J6 igm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No ( m5 s: F% ^# ~1 @' i
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
# p H! p( k1 O4 I! S9 `$ D( igm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
+ ]! M0 _1 r0 s4 P1 [gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No * m9 K3 C/ B' n$ _
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
0 @, I3 Y: D, i8 t2 M) t+ ugm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
( X: Z" A5 F- y: Mgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
3 W% _8 l1 d$ u# L+ C/ l, p- G: \4 A% \gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
$ t' G0 e3 t6 |2 j* ggm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
! B7 _# g' V# O" S) q0 N p5 ~gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
, Y( O% `" O4 c' X7 I+ r4 `) _6 Cgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No 3 R5 e: Q, H9 T6 ]) u0 H
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No 2 N- b6 b/ s) X% |9 H: Y1 R9 D
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No , ` G3 }& ]& a r. |
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
2 m/ U: r$ Y9 @" T1 r7 |) igm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
# y# J1 z! A1 J2 dgm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
7 Z/ T8 U" o1 c' O) K4 egm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No ) K5 w; D Z e
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
' t# A a" Z: C6 ^6 w9 M' ^0 dgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
. ^% z! G- ~6 F; k9 c% p, C6 jgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No 6 y- N% }# r d: {
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No 9 l" e5 \- [% w3 w
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
3 u) ~, H( {, p& ~$ g& O }2 Qgm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
# d- }3 D0 Y/ Y+ i! n- i, |gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
$ }0 F' G u9 t) mgm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No , i, C5 k8 l: I- d
[root@tp266-1 ~]# ceph orch device ls |grep tp266-1
! s f7 u& g4 _5 P[root@tp266-1 ~]# ceph orch host ls
9 G( M! `) ~- f2 i, e$ AHOST ADDR LABELS STATUS
3 ^* X( a% h1 Y* @' h: b0 t' Ugm268-1 gm268-1 mon mgr mds
7 M( f9 v2 V0 h/ A0 @gm268-2 gm268-2 mon mgr mds 5 X6 }9 A2 ^1 {$ U
gm268-3 gm268-3 mon mgr mds ; _6 [. x, q8 E
gm268-4 gm268-4 $ g9 M1 F6 b( @/ r; R5 W, W
gm268-5 gm268-5 + Z2 ]7 r6 P5 y& E
tp266-1 tp266-1 ; J1 |# R/ `% H% G: G) ~0 A
[root@tp266-1 ~]# ceph orch apply , }* u9 c3 t) j
alertmanager grafana mds mon node-exporter prometheus rgw
8 w+ O+ X" O' A) `/ g! N0 dcrash iscsi mgr nfs osd rbd-mirror
+ {4 |& i* X! b& I7 Q E& a[root@tp266-1 ~]# ceph orch apply
" O! s( _# @' K5 r @) {0 kalertmanager grafana mds mon node-exporter prometheus rgw 7 f& H( n/ ]: R4 E
crash iscsi mgr nfs osd rbd-mirror
- X1 r# b% Z# |5 p# I1 |# N[root@tp266-1 ~]# ceph orch apply
7 f; N4 x! U+ I. m6 ialertmanager grafana mds mon node-exporter prometheus rgw
1 }/ v" w$ h9 f; R! L, u% m7 o* z: ecrash iscsi mgr nfs osd rbd-mirror
! Q7 ]# `9 G3 _[root@tp266-1 ~]# ceph orch apply node-exporter * T5 O; z6 Z& V# w0 G- u
#Update the size or placement for a service or apply a large yaml spec& E) E2 e0 f J: ^8 ~& _. h- y4 @) ^
[root@tp266-1 ~]# ceph orch apply node-exporter 8 a) p2 q6 b# n r, O
#Update the size or placement for a service or apply a large yaml spec
4 b- \8 u/ s; F! Z7 o' w[root@tp266-1 ~]# ceph orch apply node-exporter
; `3 l! E$ U( l+ w9 W: ~/ V #Update the size or placement for a service or apply a large yaml spec
3 ?! G: {; t9 C% }# v/ h {6 p[root@tp266-1 ~]# ceph orch apply node-exporter tp266-1
9 e9 h# |5 c" ~0 e1 J3 JScheduled node-exporter update...
# H5 C) K& |4 D/ _" X[root@tp266-1 ~]# ceph orch apply tp266-1 tp266-1
$ Z3 w+ [& T, H" n- k1 Dtp266-1 not valid: tp266-1 not one of 'true', 'false'0 k) k# @4 ]- x
Invalid command: unused arguments: ['tp266-1']
9 b1 R* N0 t7 r. d% Q3 X# Sorch apply [mon|mgr|rbd-mirror|crash|alertmanager|grafana|node-exporter|prometheus] [<placement>] [--dry-run] [plain|json|json-pretty|yaml] [--unmanaged] : Update the size or placement for a service or apply a large yaml spec. X: S K! @1 x% ~; O
Error EINVAL: invalid command
# g+ @7 ?8 B! Q5 _% D/ Z( s[root@tp266-1 ~]# ceph orch apply crash tp266-1
' X8 z8 ^9 T" h7 p6 Z0 TScheduled crash update...% z Z1 p0 J& d
[root@tp266-1 ~]# podman ps
" s# Y3 P0 T: g7 v) ?1 dCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
. f) e8 P/ N" I; Z[root@tp266-1 ~]# podman ps
/ }8 C! y$ [% `( B( o" {CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
! R/ s/ z0 D6 X' B. ]8 ~[root@tp266-1 ~]# ceph orch host ls
5 j0 r5 ?8 |. e; v" HHOST ADDR LABELS STATUS
' p: M( _ o; \/ b" Rgm268-1 gm268-1 mon mgr mds 2 I9 V4 _' }/ m1 L
gm268-2 gm268-2 mon mgr mds
1 v/ r) h- ]) ^gm268-3 gm268-3 mon mgr mds 5 O6 @: w/ E: ~. e4 x* g
gm268-4 gm268-4
* t- U/ @% J, Q/ Y: ^gm268-5 gm268-5 3 }! P! u# ^4 `
tp266-1 tp266-1 * E( L O6 _4 G( Y2 @( {
[root@tp266-1 ~]# podman ps
, E: [' o! G2 S* k4 ySocket error Event: 32 Error: 10053.
# [% T2 t* b0 g+ A/ W0 k( yConnection closing...Socket close.: a2 W- z/ ^# ^# a% N- G: o7 c
* s. y& X* l% W |. QConnection closed by foreign host.
% H' X$ y( {! Y0 t- f5 [% x# e+ ^4 M' r5 V. V+ Y* {) r
Disconnected from remote host(gm268-5 192.168.54.59) at 17:12:41.$ h; H+ E2 K- m. u
) x @5 Q& h$ [/ a& y
Type `help' to learn how to use Xshell prompt.
5 L8 Y: l5 o- {2 X( A[C:\~]$ , j0 e9 b$ y) c z0 |9 J2 `- H
6 c: a7 l; s: s' T- r1 ZConnecting to 192.168.54.59:22685...5 M0 |* d$ L9 t( a- g
Connection established.. q! G' _- |/ s% i" Q$ H
To escape to local shell, press 'Ctrl+Alt+]'.
3 z z8 `7 @- L' S% m O/ x. {- G6 d& e- Q. d) z
Activate the web console with: systemctl enable --now cockpit.socket
) q; t: B; \7 B9 {5 c# d) i( \6 a' z1 h9 ?& u) _: v0 ?$ n
Last login: Thu Jan 12 17:11:13 2023 from 192.168.42.712 Y0 D! |. A# |% G
[root@gm268-5 ~]# ssh tp266-1
4 j6 l0 L9 X5 T/ z6 D3 v- o8 OActivate the web console with: systemctl enable --now cockpit.socket
' k; Q8 z) U- E# E9 g) [4 `, p, Y" F
" \( i. ?! `7 x. |+ n9 {+ S$ O/ pLast login: Thu Jan 12 17:04:20 2023 from 10.12.5.2
$ m; B; R1 ~# i4 p4 \[root@tp266-1 ~]# podman ps* X3 [7 t* n" [0 K
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b0 A0 q/ d+ m9 U& B3 A" ?6 [[root@tp266-1 ~]#
' g Y2 t, D0 l% _* b4 a5 k[root@tp266-1 ~]#
; y {0 K& n% i0 X[root@tp266-1 ~]#
3 m5 p5 n2 W6 h[root@tp266-1 ~]#
' d& h" t! l& G/ s W/ Z; f6 G[root@tp266-1 ~]# podman ps" a _+ `% K G" i. d
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
9 i! A3 Q6 ^: Q) a; ]% x5 h/ O1 w[root@tp266-1 ~]# ceph orch device ls 2 L3 I) B3 w0 T1 u) S1 g9 ~. @) W
Hostname Path Type Serial Size Health Ident Fault Available
& p" j$ E" ~0 W; Dgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
3 z6 H6 m9 b' y0 `7 Agm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No : G6 @0 ?3 i7 @! X
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No / y, S/ v* L" E& _7 y
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No 6 l; w+ u2 p$ ?7 c7 C6 m r; i+ ?
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
- J- J; ^; i: f g# Q4 M; {" a" \gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
3 Z. m8 C" Z2 x+ ~8 o5 y3 z# |gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
: g7 G9 p2 {' U( C3 b, fgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 1 L/ L" @4 \/ l$ J+ q- D
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No ; L1 y* A @; X7 G: X
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 1 ~8 \) `8 d: e3 j8 ~
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
1 t- }$ O) W7 q% K4 wgm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
& d1 j- R* h6 [ ]6 t% Igm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
3 r$ z8 r3 T' mgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 4 I9 `7 K* B# M( D' d- E
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
: C; g7 r# V$ w" lgm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
2 |! R& ^6 }" s0 J, z% N+ K+ lgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No - X% V) y4 [+ B8 V4 p) M
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No ( v3 e5 D/ J F5 d
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
3 j( T' y1 Z5 w' r6 M& m- c1 N7 zgm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No 9 w% e' M/ h( U/ z
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
! E" M0 k0 Y6 Q! d" fgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No % K1 r! _2 l" F" j: h5 d1 j
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No / q& e' X& \8 ~' D7 l/ @" H$ j
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No 2 n* U, M' g+ P: L- e) [) X3 q
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
6 D+ x( D* R/ d# K# H. G7 d! ]gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No ?2 k, ?; O1 A- r4 C, K$ g8 K: @8 h
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No 7 v1 M# f. W6 F" I; G, I6 X0 Y
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
7 t0 \( H7 `2 i% tgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
1 V% ? a6 {1 {: }' hgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No 3 B6 m5 R* b0 s* z% f, f9 f9 Y
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
, A- N+ Z' ?% y, }- t' D z1 dgm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
& H' s# j4 @$ R% Z( j8 P# Vgm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No , Q' m& I& J) n; S, ~; R, |
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
+ k' K @( ^. y Mgm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No $ x {' @/ U, R$ \( M
[root@tp266-1 ~]# ceph orch device ls
' g2 c7 c8 S# y. sHostname Path Type Serial Size Health Ident Fault Available
3 j ]8 `* ~ O, s) M+ @6 G$ I+ Mgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No ' F0 s" ?6 r3 _/ W6 N* X2 q
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
0 ~+ r( h/ h3 h# T5 ^! lgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
0 q- \# h2 y* a( r6 Fgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
+ C7 i) K) H% l# @gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
0 ]/ J: {: Z6 B# {9 w' L* s0 ngm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
. V# B* p8 d+ p3 @4 }! U( W" `gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No ! @5 |6 ?) H5 D! z8 p. M3 P" ^& w4 v
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
9 B k* e1 q! }. tgm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No 1 S0 e% [2 K& t: w" F$ N
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 4 d+ G! c" S/ x7 ~' Y$ y! b$ x
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No " p0 R( \" G; y% L, B
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No + T+ z' F8 \# a9 m
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No 4 m5 t/ U- G+ G# r( |
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
" z/ W! \. \& z, L# y1 M) e# F/ N$ Lgm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
8 b# D8 n+ c3 e" ^# j/ Y& A3 x& \gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No , B2 x( E! b9 k( m/ n
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
8 r/ ]$ m6 ^9 ?8 W/ Egm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
8 \4 Y0 N( A" K0 o K* o* B- Ygm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
' V; g6 L! E2 E+ s4 X1 s" s4 A6 g) b; rgm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No & P! ^* Y; j' N6 b, M
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
s' I( K4 ~- P Z1 r# Ygm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
8 a7 j* u0 u. ?3 s% h& Pgm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
# l. }; S# r5 v" M ugm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
0 x0 L2 r5 q7 p4 n5 igm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No ; @0 I5 _3 E# ~) B0 j
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No 8 A4 f6 Z6 ?' [1 @
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
& W4 L$ v- Y$ y o9 O+ f$ O9 w& Ogm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
% z" l- U; x# L x ?8 b% @- Dgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No 9 ?3 E5 C3 j5 N, |: S
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No 6 M, ~" M: F! |+ K$ F
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
( a( s" g; V; Z' k+ ygm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No 4 d; U6 ^0 j# h; k9 P; A
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No 9 M5 l% A! A! \% M8 S( D
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No . Q1 `9 r. n( D* j" h
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No 5 y- A9 F; V0 q
[root@tp266-1 ~]# ceph orch device ls
( ~* V7 I) k9 sHostname Path Type Serial Size Health Ident Fault Available % U" Y/ D- p2 a9 S6 K: x0 C/ E
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No 6 a' C$ n [$ V$ ~7 o+ h
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
0 q- |" T+ U Y( Q" q- Q; Sgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No , z0 _/ \( K/ y. s
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No ; k) g2 `5 G- n2 c& P6 B5 Q$ x9 A6 W
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
! _+ {$ s" I- e; b# f& mgm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No & ^: {$ S# o) Q/ A1 ]
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
& g9 R0 _+ k. A/ rgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
5 s8 W* R7 v; E& j/ s/ T) Fgm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
" A" ~) J& b& ]gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No / T& [3 T c* O- Y8 G
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
7 k3 V+ m1 j8 V* Q. Z- kgm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No 5 A& m6 }$ |' p& ^2 Q9 v. ^6 {& Q; l
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
! t( K+ q' u# P2 Igm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No * x/ G7 S- T& B& [
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
# r- ]0 f5 `# k5 Dgm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No 8 d; m5 |5 D$ f/ b& `) w/ ?
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
( C8 Q- n6 X/ k! \- J! q1 ggm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No / E6 g+ R$ ]& v' p/ }
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No : E* V4 K- ^, }% M! X' t7 `) p
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No : }1 s8 B5 b6 X" V3 K
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
% |$ A* e6 I$ E, h1 v7 q5 Wgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No ) I" n3 m' Z+ E
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
* r, _: t! g4 s0 Hgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No % @& q" D+ L, K5 P
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No / c; ~" m- U0 H0 D0 F$ _
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
( X. F6 ]$ n, @/ R; H- Kgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No 4 D# O, J! \" d8 e
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
& W4 G4 C. L4 `4 O: |gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No 2 G0 x% X& ?0 a/ S1 V
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
+ @! q* t1 F3 q* m; ]gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
, D7 a. `7 V0 C y Y% ogm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
/ t$ s& Z; r8 e4 n/ e8 g: agm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
1 j# f' K' ^2 |7 D$ n5 y* c* W2 Ugm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
; k: p3 J$ _* ` Ygm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No , E+ e& I. ~( w* R6 v1 M
[root@tp266-1 ~]# ceph orch host ls * c |# w( d! d; {2 ]* v5 |) D
HOST ADDR LABELS STATUS
7 ~" ?, I5 V+ s) e& C* Tgm268-1 gm268-1 mon mgr mds
+ p9 r" r5 x9 v+ C- ggm268-2 gm268-2 mon mgr mds 3 V+ E; M7 S$ _/ y- b" N
gm268-3 gm268-3 mon mgr mds ; u. q# u- i1 D8 A' j+ r" c. x
gm268-4 gm268-4 % X& S8 I% R, H6 N& k
gm268-5 gm268-5
+ H, c! V; O+ s( ^% c m2 ?4 t$ ktp266-1 tp266-1
( ~2 w+ K3 n: i8 ~[root@tp266-1 ~]# cephadm shell' M6 b9 o! y. v
Using recent ceph image quay.io/ceph/ceph@sha256:de7016bf6da2dcbb73796f66ef2776419b2c5a3ee1378835e8791a14c2c3e5b1
\! M1 Z0 }/ t) B& g; |[ceph: root@tp266-1 /]# cephadm
/ ?7 A+ ]8 y: y. dNo command specified; pass -h or --help for usage9 a0 X/ j/ v) ]- c
[ceph: root@tp266-1 /]# exit
7 k6 S* M) k9 V% A' N7 G: [1 Iexit3 d7 O$ n' h6 u9 B1 a: z" I: r! f- s; F
[root@tp266-1 ~]# ceph orch host ls
" ?, C0 G$ o( b0 |- F6 dHOST ADDR LABELS STATUS
( A9 s8 B3 {1 B) ugm268-1 gm268-1 mon mgr mds
; A$ p& W D; B2 E5 D: Wgm268-2 gm268-2 mon mgr mds $ H+ t1 r' d$ a0 g9 n: L4 R7 u
gm268-3 gm268-3 mon mgr mds # z1 R' U) x0 `6 \4 ^5 I
gm268-4 gm268-4
4 B; S g7 g: \" ggm268-5 gm268-5 ) ^& q0 ~ W/ x) { G9 a
tp266-1 tp266-1
5 e! u1 i+ \2 i# ?% r6 t, g[root@tp266-1 ~]# podman ps" g- s$ W! f, x. c
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2 M* Q. z3 }8 H v: }# ~ a[root@tp266-1 ~]# exit; A7 D$ |3 q2 N6 X4 ^1 A! r
logout) W1 m% w0 N3 d ]" T' p; G
Connection to tp266-1 closed.
+ t" B b& i) I3 R" p0 r0 s[root@gm268-5 ~]# podman
& ?" m- V) T# h; I: N7 Dattach cp export image inspect logs play push save system unshare
* x7 T; ?- y; n5 ^* j$ @8 ?auto-update create generate images kill manifest pod restart search tag untag 3 I7 W( l7 O: j9 V
build diff healthcheck import load mount port rm start top version
5 X& j2 L6 g+ T8 J p& Qcommit events help info login network ps rmi stats unmount volume : y+ a; T& s' w
container exec history init logout pause pull run stop unpause wait
% L, n. k2 K# \& s' D[root@gm268-5 ~]# podman ps 5 A4 R+ \) @( v
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
$ z4 R) w ~- d7 y% g79ab1651e550 quay.io/ceph/ceph:v15 inventory --forma... 2 hours ago Up 2 hours ago strange_kirch% d G* k, N8 U* t
a48e0b56a516 quay.io/ceph/ceph:v15 -n osd.27 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.27
3 D& ]( _' J6 ~9 r. Kc16fff33a250 quay.io/ceph/ceph:v15 -n osd.26 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.269 ~: A9 s2 Q+ O% j$ H
6761927d89c6 quay.io/ceph/ceph:v15 -n osd.25 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.25
& ^4 s7 d* A% M! z3158677090f6 quay.io/ceph/ceph:v15 -n osd.4 -f --set... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.4
$ ^3 d" a: \" N7 B2 W1a58b23cb4b2 quay.io/ceph/ceph:v15 -n osd.29 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.29
# r7 l8 d* }# [e800611062b1 quay.io/ceph/ceph:v15 -n osd.28 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.280 m& I+ X3 ?# r8 I
5c5454782321 quay.io/prometheus/node-exporter:v0.18.1 --no-collector.ti... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-node-exporter.gm268-5
! x: f' D/ Q6 V06f85b89a143 quay.io/ceph/ceph:v15 -n client.crash.g... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-crash.gm268-5
. v/ U) l' u" Y9 ~* q[root@gm268-5 ~]# df -Th
# ]% `) d, g! U) D; V7 j |
|