|
|
楼主 |
发表于 2023-1-12 19:52:38
|
显示全部楼层
启动chronyd服务,检查服务
: u! j" \) L! w[root@tp266-1 ~]# ceph orch host add tp266-1
, q/ [# A: r4 H0 H7 f. [Error EINVAL: New host tp266-1 (tp266-1) failed check: ['podman|docker (/bin/podman) is present', 'systemctl is present', 'lvcreate is present', "No time sync service is running; checked for ['chrony.service', 'chronyd.service', 'systemd-timesyncd.service', 'ntpd.service', 'ntp.service', 'ntpsec.service']", 'Hostname "tp266-1" matches what is expected.', 'ERROR: No time synchronization is active']! A5 V4 e9 c* T! U3 p
[root@tp266-1 ~]# systemctl status chronyd.service + h) Z! s. [$ p7 H) O( O+ e! J7 N o
● chronyd.service - NTP client/server
( Y7 }5 J" |, s* M C Loaded: loaded (/usr/lib/systemd/system/chronyd.service; disabled; vendor preset: enabled)' e n% z* j. b7 e
Active: inactive (dead)& m7 O& B% v+ ]! V. k
Docs: man:chronyd(8)- L- f9 g5 C" K5 U! @- v
man:chrony.conf(5)
% Y* [8 A& H" _, m/ P9 w" d[root@tp266-1 ~]# systemctl enable chronyd.service
3 \( U2 Z0 g* G6 h4 UCreated symlink /etc/systemd/system/multi-user.target.wants/chronyd.service → /usr/lib/systemd/system/chronyd.service.
5 L3 x, A* {; r# `[root@tp266-1 ~]# systemctl start chronyd.service
8 J0 G7 r; g F) i% ?4 o[root@tp266-1 ~]# ceph orch host add tp266-1
4 T. o2 s! g1 {0 K, G8 M* sAdded host 'tp266-1'# v5 g+ y! g/ ~. w3 i, \8 T
[root@tp266-1 ~]# ceph orch host ls5 G$ Q5 q' k! B- i/ Q2 K
HOST ADDR LABELS STATUS - s3 m' l5 l7 N3 E+ V6 z; }6 ?, B
gm268-1 gm268-1 mon mgr mds
, F+ Z& G( V+ _+ ]+ Hgm268-2 gm268-2 mon mgr mds
3 z0 b9 K3 a1 e; k H6 m5 {4 a2 wgm268-3 gm268-3 mon mgr mds
- r) R. Q4 W! p+ ?gm268-4 gm268-4
0 h; P8 C+ a, J; igm268-5 gm268-5 ; ~: R' G# M, L6 z
tp266-1 tp266-1 $ W1 Y' p3 P' j
[root@tp266-1 ~]# ceph osd tree
. W6 K7 Q, y3 n! B% F: MID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
7 r9 q4 g# s8 x/ F -1 494.79141 root default
/ [# q! L5 ?! h: p -3 101.86882 host gm268-1
4 g& _' t9 l: x7 K5 l 0 hdd 14.55269 osd.0 up 1.00000 1.00000
5 o2 z% R" z' @3 u, L 5 hdd 14.55269 osd.5 up 1.00000 1.000009 H3 m' {% d8 b; [# H% g' A
6 hdd 14.55269 osd.6 up 1.00000 1.00000
! U1 Z9 L/ @4 J6 w/ J2 V+ m# V2 J' k 7 hdd 14.55269 osd.7 up 1.00000 1.00000
& e; ^* F" P3 {6 A0 X2 x 8 hdd 14.55269 osd.8 up 1.00000 1.00000
- c, f/ O" x, J* p( A 9 hdd 14.55269 osd.9 up 1.00000 1.00000
( H/ l' w( u/ g& x8 \ e* W7 G" | 30 hdd 14.55269 osd.30 up 1.00000 1.00000
) P/ _! \0 d/ U' N' s9 V -5 101.86882 host gm268-2 ( d1 V" W; Q0 n: R
1 hdd 14.55269 osd.1 up 1.00000 1.00000! c0 s. A" f' ?9 ^! @
10 hdd 14.55269 osd.10 up 1.00000 1.00000
5 l& j# @. s$ ]. U, p6 Q) {7 v' C 11 hdd 14.55269 osd.11 up 1.00000 1.000008 E! u$ M6 `5 K0 g
12 hdd 14.55269 osd.12 up 1.00000 1.00000
4 W! I, B3 L( t 13 hdd 14.55269 osd.13 up 1.00000 1.00000
7 {/ X% Z, P0 Q( e) P* M/ } 14 hdd 14.55269 osd.14 up 1.00000 1.00000
9 B% n0 G. I! x 31 hdd 14.55269 osd.31 up 1.00000 1.00000
) w; x* I# I) D+ L0 s& U -7 101.86882 host gm268-3 3 l2 }. L+ J8 ^4 z/ N4 u: [2 F+ y
2 hdd 14.55269 osd.2 up 1.00000 1.00000
d& z' ^( ]/ m; o L 15 hdd 14.55269 osd.15 up 1.00000 1.00000
3 W2 _8 Q, G2 P, \7 e- z7 u 16 hdd 14.55269 osd.16 up 1.00000 1.00000
2 Q" S4 u3 o k: d" f4 ~ 17 hdd 14.55269 osd.17 up 1.00000 1.00000
: w O6 r6 {1 l5 J% j& c8 m 18 hdd 14.55269 osd.18 up 1.00000 1.00000
# _; x" n6 T3 w1 E 19 hdd 14.55269 osd.19 up 1.00000 1.00000# L, F3 D- Q5 D2 l# C
32 hdd 14.55269 osd.32 up 1.00000 1.00000
5 K# e% h& B& a. T5 U, [5 P' \ -9 101.86882 host gm268-4 Y* {: S0 D: H$ V( ~* H( l
3 hdd 14.55269 osd.3 up 1.00000 1.00000
7 L z; k4 S. r; L0 w; Q" b. @ 20 hdd 14.55269 osd.20 up 1.00000 1.00000
9 F# X7 y, S `" b( ^( q8 }; { 21 hdd 14.55269 osd.21 up 1.00000 1.00000
- G6 b& _# v# h6 f" g 22 hdd 14.55269 osd.22 up 1.00000 1.00000
* m+ p, h$ Z1 T; ^ 23 hdd 14.55269 osd.23 up 1.00000 1.000000 i" {0 a2 e. `; ]" V% y
24 hdd 14.55269 osd.24 up 1.00000 1.00000
5 g# O: _+ V0 ~( C8 v( l 33 hdd 14.55269 osd.33 up 1.00000 1.000000 s, E- i4 G8 p
-11 87.31613 host gm268-5
6 V9 @* G2 u: K 4 hdd 14.55269 osd.4 up 1.00000 1.00000
8 ~9 t6 i+ f" z; h+ v 25 hdd 14.55269 osd.25 up 1.00000 1.00000+ }8 X, W4 m1 e$ Z7 H* @: b
26 hdd 14.55269 osd.26 up 1.00000 1.000006 d" |% ?8 V( J. ` L
27 hdd 14.55269 osd.27 up 1.00000 1.00000
4 r: b1 E; x5 n7 X6 m 28 hdd 14.55269 osd.28 up 1.00000 1.00000: A- x* M. c: p% }
29 hdd 14.55269 osd.29 up 1.00000 1.00000; S4 p: m" Q% E- s A( O! ]* S
[root@tp266-1 ~]# ceph orch device
C+ I: @% r s+ W" }, [: z. ~ Sls zap 9 p/ F! ~9 A1 s6 f3 G. P
[root@tp266-1 ~]# ceph orch device ls ( } t5 U* ?1 ]# F4 B" ~5 g
Hostname Path Type Serial Size Health Ident Fault Available 7 p0 ?4 U* B0 R f1 m+ M
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
8 v( t" O9 |; o* I4 F* i# tgm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No 1 G4 p! N7 _* u. p# t" d
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
# Q! W$ f9 ?. w6 c7 y5 Agm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No / E$ L5 }- E2 [. I c( y6 y
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No % f' p* w8 a9 y2 `% d+ a4 Q
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
0 y# ^+ T( U% s' F# W% Vgm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
7 b% d7 S2 P/ W; U0 Fgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 9 T. v$ V! Z* e8 g3 {& T$ i8 o- T
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
9 j8 b7 H7 N! I+ l& a9 s3 kgm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 0 D3 z; l/ |- Q! i, U/ {
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
+ [" Z8 U- H0 ~' j4 Igm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
/ \6 ~; ^% A* e, ygm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
$ L2 v4 O# Q9 w* ]! P3 Ygm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
0 z- z: ` a! s9 e$ x) pgm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No {) w# i/ E3 h8 c* R7 w$ |0 C5 a3 D
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
1 l- K! j) b; m; Q1 b U6 Pgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
% e7 ~) @& B* e) L# H* m* ]gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No " I; {5 W ?$ {" O" p
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No 5 M" G! Y# G. C' Y
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No ) T- I; e7 u$ r+ c6 C
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No / M. L! a8 ]$ \' E# o2 B# `
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
1 Z" i' l6 L- z Q# z9 i' Cgm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No # D4 R$ Y& f- T( e+ N: \" s* s
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
; ]9 W/ b8 ?' j5 J: M% Dgm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No 8 H9 i) z/ w9 I) f3 B4 _1 \
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
( Z3 p- _0 t: {/ }* [gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No : s7 p7 \, l% L( M' b& n4 e' k; G2 T
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No * Q, R0 _2 B ]0 n; q4 K
gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
% ~7 c' a- r) I) M& f) P- Wgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No n+ A2 s* j5 \" ?, ]
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No 6 B$ ~4 A1 J) f) F; @8 W3 x& T
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
9 h1 R0 h6 d8 S, Ogm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No 0 o0 G7 L0 {) C) Q
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No 2 t$ S$ w5 E" f D/ T+ q
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No ) S/ \2 w/ Z1 i4 ]! C4 [: ~+ \
[root@tp266-1 ~]# ceph orch device ls
( w0 w& a/ A( v; d5 _- l FHostname Path Type Serial Size Health Ident Fault Available
0 Z$ l* t `, b7 }& O# dgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No ) O& G" U+ i% \4 p9 D, H! U
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No - k2 v) h7 [3 L4 J
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
: s8 i w9 I3 j4 f+ T8 u0 Tgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
+ ~+ s8 t& [* ^2 dgm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
, d% C- R# y0 agm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No 5 C; }( V1 `9 e* x' p
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No " M! ~' C2 _ x$ N9 k
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
4 S2 J* y7 q7 y, _% mgm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No ( Y9 q: s1 k' m( e5 d0 h
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No 5 y: c" [3 D0 s# y1 A( `* q0 P
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
- g+ n/ M8 b8 {% G) d# ^gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No 0 ?4 m6 X" p* N Y' `' A
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No $ c- `6 |' b+ m# z( W0 Z
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No $ y) d) C. Z7 W1 H
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No % \) B' u2 D4 I% s
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
% G9 F4 B! x% M/ jgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
0 F* L$ ~2 e) u. ?0 [/ ]7 Ygm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
* k& G; A! D2 A. V8 tgm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No % a4 q N, D; P$ A
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No 0 d4 z0 v/ P# E8 W# z. u
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
- C+ m0 {2 e7 I3 a1 hgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
6 w; w3 Q b! }gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No . m4 ^4 o9 p, S2 E$ K: H' V. |% d$ w( A
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
2 z& l) C& z* M# F2 l) Z7 Dgm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No 6 m8 H. v* U! [' q3 X3 E
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No : @! p# n% i( U+ _! U: a
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No - S5 F" p0 ?% e; M; o- c
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
" Y! u6 n1 {' Z9 lgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No $ |/ L; U& i# o1 M" \, g) P" P2 l
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
; \, O% |; V) h( ^( P" n2 t' Sgm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No 5 @: m- s- j* c& D$ ` u5 R
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
. } [% S3 V2 @) G! j% Kgm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
$ F4 k! k3 A) E* s+ R+ E# wgm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No 8 t! F5 f0 U+ z3 @. I: y
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No
- G4 t+ g* D+ I* y' h2 [# g[root@tp266-1 ~]# ceph orch device ls : q- S. P( L' F
Hostname Path Type Serial Size Health Ident Fault Available
: h' P' x; f; [) Y7 _' J, q% r2 h; l) o: r' kgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
8 b7 L1 S" g6 Y8 [gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No ; D. d* _4 U6 {' Q3 \
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No + z8 ^% Y* s& e, \
gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No ( E& m M6 k$ N
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No 4 r; D* K1 h+ t# C3 {6 k
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
* x3 w" Q9 M9 ^% N, Wgm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No & h" C8 o- d, m- j+ _2 W" I
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
+ l3 H+ e6 i6 g! X- q3 j* R, Jgm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
9 `7 H3 G! Y! R1 |- k5 ^gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No - C/ `0 N1 d1 @* X/ V- i' |! N* V! z
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
# d! q7 m1 e* ^* ~gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
. `! J0 Z; \7 P0 Tgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
! j8 W4 w& }9 f3 y) H4 s; l6 egm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No * c) l% h. [- l v
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No & M7 H, I, h4 F5 z% R* p
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
1 ^! f3 N$ P, B2 M) s/ Egm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No 6 P+ M! S; C5 L0 ?7 J
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No * `1 \9 N$ T% m- E; Q
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No 1 F, f% r7 C }
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No : Z; Q2 f& H. V& Q u* r
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No 7 q+ k- ?5 b1 E; F X3 x# e
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No / M' G% L" ?9 H r4 B, [8 A
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No - A) |" a9 B1 X" ?, e
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
/ T; `( M) p' a4 s$ H6 p. V8 o" h/ f rgm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No 2 Z# Q* ^$ r/ H
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
$ n% k8 X/ ~6 Wgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No 1 b# w; `( [, m2 G5 @
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
1 O( n3 d2 u* C/ C8 Lgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
$ u* [- X4 y$ l/ K* y4 Ngm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
/ a0 @# C) k, A! `gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No ; X N5 i& o' X, v& Y# A
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No / T# f8 Z% h6 b. V3 z
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No & d9 D* u- F5 m9 a, L/ }
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No ) d& K! H/ a5 x; b& h
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No ! {9 A" k8 t0 T0 t7 K) G
[root@tp266-1 ~]# ceph orch device ls 2 s$ i p9 K( E( U6 L
Hostname Path Type Serial Size Health Ident Fault Available
) [& Z3 Q% K8 I9 C: d% N9 ygm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
4 o3 r' p) e6 ~( T, M. ugm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No 1 D3 _6 v* ^7 i% j/ \1 _( |
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
3 E4 f; d) b- M: z. T' T1 Y) o% ?& @gm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
) Y# R4 ]. o4 R+ D, f4 x9 Fgm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No % y' m7 E- J) U
gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No 3 ?' k9 i D* r1 Z! r3 ]
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No / @, x; e/ j9 B# ? \
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
. }9 q' w+ t ]6 _+ @gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No , [) X! I! f( [6 b8 C8 y
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
0 _5 T0 M. R/ x% C5 x: Q9 egm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No 9 x8 I7 ^" q. l) l
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No 3 J3 {: Y: \! ^/ q
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No 4 q, W" n3 d4 t% [
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
) G* Y' d+ [ H$ j# }gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
* |$ x" x+ _# O( Q9 X& G8 N) _gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
& e. {) A$ f0 T- q4 }8 q& Zgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
k! b3 r1 p0 D6 n( r& A, Ogm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No " ?7 s% @' H4 r A
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
2 [; M3 S9 U4 v1 ngm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
5 ~$ S* K' F0 z' ]# A% n: F7 ygm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No ) \3 T6 L/ A- [5 X# J* A* _* I9 m
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
9 n; s; _$ Q t) lgm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No ) H: [7 ~5 {+ t' Y" d6 n# l
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
6 V2 L, B1 H- z: @( Z0 _. ^gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No 1 \7 v" A K$ \! S6 u9 B* M
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
8 O5 o: ^7 m$ Igm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No * M% U* D" m" C( v9 }/ Y, y7 U
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
f7 s! n8 c9 v+ m, cgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No
8 _+ D5 ~8 b- \" [# qgm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No ! s: Y+ n* P2 @5 k. x& g4 |
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No + `6 h9 F3 F, `& ~/ c* A8 O& P& J
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
: M! T& G' e- }0 W2 A1 R2 ?7 P, Lgm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No ; X$ G# B& T; X5 N* M2 s$ D6 F2 s
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No ) D. W' ^0 ^( [3 X. @* a7 S
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No # S' j8 a5 O" a5 s E/ x
[root@tp266-1 ~]# ceph orch device ls
) V$ u- E% V9 ^Hostname Path Type Serial Size Health Ident Fault Available
! a0 `! F" t" y: ^1 r: ^5 _/ Kgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No 0 z2 K, ~+ q; D3 A
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
9 G/ l+ v& c- k8 _( y3 [/ Qgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
# q2 Q2 v, ~/ _$ B1 H) Ugm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
/ _. S+ @, H9 c8 Bgm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
' N, v2 v1 @5 v, _4 |gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
# |- u1 k2 p. G4 N) Lgm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No ; A4 p; @+ F6 v: Q
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No _8 v- l/ Q5 @
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No " F* |7 H# Z. G5 b; M
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No # c) s" t) O& J2 K4 l! } `" a0 u7 g
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No 7 O: i$ g2 m1 N2 U0 R5 z1 B
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No 1 E3 t6 ^6 l$ _7 I* l3 I
gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
0 R5 D2 v2 b6 n/ f0 Dgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 7 @8 w; ?) n+ S# D, T. j6 [# h
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No ! z ]' T; B. r* h
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No # \0 v- G1 S. r0 K5 W/ |) B% N
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No - {, q8 A- @ M0 U* V' \2 E
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
5 J5 L9 s2 g2 y% Agm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
5 W, a" A: ?6 H% ygm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
7 I- m: r! q2 Ygm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No ! ]+ C% A6 I, {/ ~7 T( q4 a
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No P) f) H4 ~+ M
gm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No $ n* E0 T- b3 ~9 o2 f4 H! Q
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No / A I- f* }% t. U
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No - E+ k" n, d7 h- H+ N+ P! |
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
* E, B" @( U" \8 }" r# m$ Q, Ygm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
+ t: @# r) ]: P4 b7 @ G7 fgm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
; {$ A8 K% o1 d. n I- [gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No + t! p0 _9 c$ `- [1 @: j, A
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No % B' x) K: D9 C8 E0 i
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No ) F/ H7 j) \( D5 P& E4 B. V/ ~
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No - |) W+ m/ [/ j3 x
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
5 H( }4 H$ M& I+ P+ bgm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
$ b, H o6 L! Z2 b- p* |7 `9 k4 zgm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No 3 r; C# e6 Q- J1 x
[root@tp266-1 ~]# ceph orch host ls
6 @! e7 T+ `: ?4 F4 s7 BHOST ADDR LABELS STATUS
. \1 A6 X' M' u* k6 W- \) Ugm268-1 gm268-1 mon mgr mds $ A. i& ^7 M) s. N, P$ Y& a
gm268-2 gm268-2 mon mgr mds # S/ v; [( D- F( a& U
gm268-3 gm268-3 mon mgr mds
$ }: b8 x0 j' {* ?4 Lgm268-4 gm268-4 " Q7 q' x9 `: N& @: A
gm268-5 gm268-5
) z8 A! s- r; g& \; f3 g5 O" mtp266-1 tp266-1
- w1 n$ w8 K- q1 A' v: ]/ j[root@tp266-1 ~]# podman ps/ m0 a, k4 ` e' S* u* [+ e
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES6 l Y: a3 ^2 g- x
[root@tp266-1 ~]# ceph orch ls * Q: b2 N4 q! U* q+ s! k- \9 p
NAME RUNNING REFRESHED AGE PLACEMENT IMAGE NAME IMAGE ID
0 l8 A {2 E+ E5 z) E7 Kalertmanager 1/1 72m ago 3d count:1 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f
: i5 V, _& k k' H0 o& Ucrash 5/6 72m ago 3d * quay.io/ceph/ceph:v15 93146564743f 2 F7 m1 P* ^# }0 C. ]/ `
grafana 1/1 72m ago 3d count:1 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 # B8 M5 \. j6 C
mds.cephfs 3/3 72m ago 3d label:mds quay.io/ceph/ceph:v15 93146564743f
2 @# j- A* B4 X, A2 jmgr 3/3 72m ago 3d label:mgr quay.io/ceph/ceph:v15 93146564743f
, S7 S0 d& \! j( o Zmon 3/3 72m ago 3d label:mon quay.io/ceph/ceph:v15 93146564743f
1 G2 R6 r& e7 i1 T& c% lnode-exporter 5/6 72m ago 3d * quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf ; Z/ y6 G; V& |" `+ C; _
osd.None 34/0 72m ago - <unmanaged> quay.io/ceph/ceph:v15 93146564743f
& _! x2 K1 B* C' W8 R6 `( jprometheus 1/1 72m ago 3d count:1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 2 I) f' J: p5 |; o. L0 f2 T
[root@tp266-1 ~]# ceph
' p' Q! y9 g0 R& y3 talerts config-key features insights mgr osd rbd telegraf versions: O6 F2 X4 I& c1 ?% ^
auth crash fs iostat mon pg report telemetry zabbix
% k' H% `. n+ J% U5 e! ^3 sbalancer dashboard fsid k8sevents nfs progress restful tell 7 p0 r; `, F+ }. `, H: ?$ [
cephadm device health log node prometheus service test_orchestrator ( m5 _% V0 Y+ a8 f f# r( R( Z
config df influx mds orch quorum_status status time-sync-status , Z& C+ x/ G! B- \# l( H8 S
[root@tp266-1 ~]# ceph
! d- n# K8 _' {$ }- ]) g5 `/ D+ ~alerts config-key features insights mgr osd rbd telegraf versions
9 O1 S+ y3 }. j& p9 Y3 Fauth crash fs iostat mon pg report telemetry zabbix
: S4 i6 q2 `* Qbalancer dashboard fsid k8sevents nfs progress restful tell
4 X/ j2 ~/ A( N( c0 Icephadm device health log node prometheus service test_orchestrator
1 J2 L4 P% Y" _, G- y" h, |config df influx mds orch quorum_status status time-sync-status , Q% G- z7 g5 _7 u! m' X
[root@tp266-1 ~]# ceph orch host ls
, v4 p. x1 f n, j5 T, ?6 hHOST ADDR LABELS STATUS
) T" j4 w; T! P6 Ygm268-1 gm268-1 mon mgr mds
& j/ F, a. w7 H( j0 B9 Ogm268-2 gm268-2 mon mgr mds 6 p3 }1 M# e0 [0 t# }, X
gm268-3 gm268-3 mon mgr mds
6 u* ]; Y( n" Dgm268-4 gm268-4 * {2 s& L. r0 z0 ^7 J6 _
gm268-5 gm268-5 ( N0 n4 ]: U% \- K; t" [" O( c
tp266-1 tp266-1 7 Z' b5 {' x* j! Z5 r: ~
[root@tp266-1 ~]# ceph orch device ls 1 d; U5 r" q5 Y5 s1 I
Hostname Path Type Serial Size Health Ident Fault Available + n$ t/ v0 w2 o3 A5 g5 J
gm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No 6 w* M5 w4 Q8 ?; M. u
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
& p6 d# o# }1 kgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
) C, K: p4 A5 p0 } r9 Bgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No 0 S. T0 I/ w. u
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
" @' ^1 m0 @& y" _* _gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
: ?8 H$ L# L, p8 H' R# Dgm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
n l8 b4 [/ I2 v @gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No
( S, `/ E) ?- j; j/ ggm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No
, {/ z8 _! `( [2 C3 kgm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
, J: p$ ]$ b! L6 O/ Wgm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No
$ A. ]8 u+ g4 v- E$ @* _& m: vgm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
& z0 k/ r g+ N/ G. Ggm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No ) x& [6 w3 [4 v# k9 B5 v
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 7 b4 U8 \' C" ` q5 f# M
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No 7 s8 U" O: i$ F4 W5 y
gm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
3 |: L/ |" {- A% N, \gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No % i8 F' Y( r& b$ ?; ~$ L
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
" Z% X/ i9 e0 p; a% ogm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No 3 I. K/ W. O" ~6 ]! K5 {+ o
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No * Z- Q. L; D6 T9 ], }! A
gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
& d; @+ x, F% l I9 o1 }" Wgm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
8 k2 }1 }, u S5 {) c* @6 Agm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
+ b) r6 _; r1 v4 Bgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
. p& C, h3 v) d+ a# Agm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
; y* k- A6 ^0 j, D+ k9 U- J% C2 hgm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No 8 V. S, |; p, t& H' }/ Z3 G. O4 K, ~
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No # m5 r e' P& {7 y
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
1 k) W$ m8 m" G c6 agm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No ; k& q8 Q3 P1 \8 O6 s2 f
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
4 ~' ^' R9 U' X- m" H& j" R! a7 l. W/ p& agm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
m5 }" n0 T* o4 a9 c" o: ygm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No ; Y- q/ `6 c$ K% u; Z
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No 6 T2 {7 y, s0 J5 H
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No / s" y0 B5 ?1 a8 Z
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No R. O# `* d9 S- m( h$ S- T6 C S
[root@tp266-1 ~]# ceph orch device ls |grep tp266-1 _7 r0 z7 G( d0 S- ]! \7 N
[root@tp266-1 ~]# ceph orch host ls
6 k4 m8 @* K$ z; b u9 h3 ~HOST ADDR LABELS STATUS 3 P* ?& F' J; @4 I: H/ `
gm268-1 gm268-1 mon mgr mds
1 z9 u: ]1 i8 C V6 E6 o" Sgm268-2 gm268-2 mon mgr mds
1 Z* S- c* b) S- X M6 Wgm268-3 gm268-3 mon mgr mds
; P1 M$ o. d- Vgm268-4 gm268-4
2 C Q D% `! N r/ y) \4 |gm268-5 gm268-5 7 t4 J0 }, H" A' _8 K* v
tp266-1 tp266-1 , i( r0 ]- L6 i: {& m6 V8 j5 k$ Q
[root@tp266-1 ~]# ceph orch apply
8 }9 R9 s7 B" D; _( c1 ^: oalertmanager grafana mds mon node-exporter prometheus rgw
6 N3 R1 Q) V- G6 H ^crash iscsi mgr nfs osd rbd-mirror + f. P U4 |# k( g* s
[root@tp266-1 ~]# ceph orch apply
2 Z8 B3 T# j+ j" O0 o; S! A2 v8 kalertmanager grafana mds mon node-exporter prometheus rgw
+ E& X9 j* Y3 R( y! I. r2 D2 n7 Mcrash iscsi mgr nfs osd rbd-mirror + W9 _) D# ~6 _/ N. g
[root@tp266-1 ~]# ceph orch apply
0 T. u6 T0 X' l/ Z9 V4 Oalertmanager grafana mds mon node-exporter prometheus rgw
2 ~: ~5 t& e( F+ ?2 J) y {crash iscsi mgr nfs osd rbd-mirror
) L5 T" w( ^- N! N4 [; p) g2 o. j K[root@tp266-1 ~]# ceph orch apply node-exporter
1 k8 ]/ S( s* Y: K0 k #Update the size or placement for a service or apply a large yaml spec# P* C$ o& I2 `
[root@tp266-1 ~]# ceph orch apply node-exporter % T8 u; j- C* m: @: k6 }
#Update the size or placement for a service or apply a large yaml spec
. v. u1 T: j6 ?% }; J1 S; _[root@tp266-1 ~]# ceph orch apply node-exporter * M- c/ L( _% ]# Y
#Update the size or placement for a service or apply a large yaml spec2 q$ M* X. e- N* w+ L
[root@tp266-1 ~]# ceph orch apply node-exporter tp266-1
" V0 Q0 v1 H5 y. S, YScheduled node-exporter update...
: l8 h) z+ r+ `[root@tp266-1 ~]# ceph orch apply tp266-1 tp266-1
6 v& t" ] U% Q8 ]" @! M: btp266-1 not valid: tp266-1 not one of 'true', 'false'
0 ~8 V0 ?* y3 U) X5 m6 N, EInvalid command: unused arguments: ['tp266-1']8 h2 c+ D3 ^4 ]+ g, Q
orch apply [mon|mgr|rbd-mirror|crash|alertmanager|grafana|node-exporter|prometheus] [<placement>] [--dry-run] [plain|json|json-pretty|yaml] [--unmanaged] : Update the size or placement for a service or apply a large yaml spec
: ?1 R4 m! [3 A* P: u6 {9 lError EINVAL: invalid command) ~0 U- S! Y) N- P4 e
[root@tp266-1 ~]# ceph orch apply crash tp266-1- a. V8 q% V2 Z4 T
Scheduled crash update..." ]) r8 N# @1 K' u
[root@tp266-1 ~]# podman ps
3 J& _. {7 s% }& a4 ECONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES! e1 b; |- ?/ {1 C7 b. E' H
[root@tp266-1 ~]# podman ps
5 D- x4 @8 ]0 B4 f+ L R9 a& VCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES. M9 v9 Z; f3 [& w( e' B5 [
[root@tp266-1 ~]# ceph orch host ls: k% v6 } I& W; s3 t: ?. a6 K
HOST ADDR LABELS STATUS : c& u! l3 M9 T6 b# _
gm268-1 gm268-1 mon mgr mds 4 b1 Y% M" w, s) J6 }( Y3 b
gm268-2 gm268-2 mon mgr mds
/ c7 x5 t! f6 B" ^gm268-3 gm268-3 mon mgr mds 7 r4 e1 T$ U! `* j4 Y% [! T1 h
gm268-4 gm268-4
; j6 C; F I4 F9 }6 n7 wgm268-5 gm268-5
: }2 f5 X2 D' wtp266-1 tp266-1 4 Z- ]3 o5 ?$ ]% Q
[root@tp266-1 ~]# podman ps) T* r$ R' j$ m4 X# j
Socket error Event: 32 Error: 10053.
1 w. w4 q1 ~, kConnection closing...Socket close.
: \6 z6 `- U- ?: }$ @1 p1 C, @$ \2 V. D
' D4 k& g J$ E9 T0 J6 `( b2 lConnection closed by foreign host.8 u5 W( s) T% J; ^* k- a
+ ]8 x5 x* w. }9 V7 l7 G: _. d* \
Disconnected from remote host(gm268-5 192.168.54.59) at 17:12:41.
) E: K: n8 L. X. ~! M
) B/ K/ m8 M/ o8 H% j% GType `help' to learn how to use Xshell prompt.
2 u' O8 l' A: f4 F4 g[C:\~]$
$ C" F' c4 l% d. L7 P* P
# R, e( ^& Y' Z& M YConnecting to 192.168.54.59:22685... L' ?! [1 ~0 O. \- s
Connection established.5 j, E" x L% z
To escape to local shell, press 'Ctrl+Alt+]'.% B* F; k4 U6 N6 H8 [
5 U) N* i( L, f- r
Activate the web console with: systemctl enable --now cockpit.socket; c$ m8 Q( s' Q7 L" a
$ j: P( C! w5 n2 P! B3 p* `. G/ e
Last login: Thu Jan 12 17:11:13 2023 from 192.168.42.710 X/ U' R# ]9 O; c4 ~6 J# \5 s
[root@gm268-5 ~]# ssh tp266-1 7 q8 R! ^) L/ F! ^* N7 E5 ?
Activate the web console with: systemctl enable --now cockpit.socket B6 \6 ~ j+ F
3 v( N6 G3 r1 R; {+ A J) f# b6 C
Last login: Thu Jan 12 17:04:20 2023 from 10.12.5.2/ f! J# w# s5 P$ q! K3 l" a
[root@tp266-1 ~]# podman ps/ b" l( b! v: }/ Y
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
; u8 V9 Y7 o3 Y7 u M[root@tp266-1 ~]#
; c7 {& i2 ~1 z! I1 r[root@tp266-1 ~]# . C5 {: V6 Q& B4 B5 b
[root@tp266-1 ~]#
5 o- ], M: x) w( ^[root@tp266-1 ~]# 3 v; F& ?& P8 Y' \+ y% R" {
[root@tp266-1 ~]# podman ps
: Z k$ M4 e# d' N, I+ U$ ^CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES, l! K. `' M( m8 n$ c
[root@tp266-1 ~]# ceph orch device ls 9 i6 b5 F" \& B( o4 O& l
Hostname Path Type Serial Size Health Ident Fault Available
: G+ `) N. m9 Y5 B. M6 i/ ngm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
/ [# w3 Q4 I: {! {( _gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No " f5 w; E% ]# a* H
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
C$ b2 X9 F M: `: ]- c7 P$ jgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No . C' r8 b0 k9 A4 `+ J2 l
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
4 h* K a8 b2 ~. i1 ]( L! Agm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No / |) m& z3 y% k$ u
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No 3 }9 j" c2 M" a8 B& v
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No % n, Z# i9 s/ q W8 {" H" x
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No : S- ~1 [1 G) H. @
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No ! B8 K# z3 F# G, A
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No ( Y8 e& Y; Z: K8 }# t: ~+ U! Y
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
9 b0 b/ Q- e3 P: _# A. s, l0 ~gm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
* }' H0 c" c/ i$ v# I/ R5 Zgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No ) k* n7 o8 i, {" ^7 V4 P' q6 a
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
/ M6 U, d/ Q0 a U) w* pgm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No
! Y( b8 {! h2 Q- _+ xgm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
- W- z" P+ n% R) Y" Sgm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
! p3 a7 x3 O9 j! t& A: u3 F5 hgm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
$ y3 d; M0 Y8 @: Ogm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
4 G; |4 h; [8 b L8 hgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
6 L/ Q- p* v8 O5 p: ]( D, ~9 Egm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
/ }+ F4 ]) v7 _- C; { Pgm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No 4 |' e( _( g3 r9 H4 a
gm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No ! Q# a+ K j* b5 `* Q! D1 e
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No # W5 q0 d: _, X' S
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No
6 }* ?, e& l0 W( `' H( Q" hgm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No
2 k( X, t M9 M; w% M- igm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No ! h, m* U' f0 K4 |
gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No % _! S& S2 K6 \, c5 e2 t2 @7 ]
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
5 Y0 _, z4 T E+ z1 Y5 sgm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No
% R! q2 O e+ y4 m$ bgm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No
" O" y; } I5 _) k1 I/ e+ ], U% S, Cgm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
$ D8 g3 t6 r, I6 T! _; U' H. ngm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
0 k+ R. l& C( b X3 r& ^9 S1 {gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No 3 u. }6 [. t; i# r4 Z8 r X
[root@tp266-1 ~]# ceph orch device ls ( I& g7 Z/ L6 m. x# n2 q# q% V+ q) r5 b
Hostname Path Type Serial Size Health Ident Fault Available
) P P' o" O; z4 J5 qgm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No
& A3 W: |. W# n ygm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No
' e3 \! ?/ G$ |7 u- ?' |1 hgm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
4 \8 M D, l1 w6 ]- H' d1 f) dgm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No & L1 ?. \/ y/ ~* N) W8 t
gm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
; G( J0 E+ |2 E3 }$ C* jgm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No
0 N7 \9 E1 |/ p3 z7 ggm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No 4 P- x- c# ]7 p% q
gm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No / h+ {' o" Q! j5 ^0 U- N- C1 ?
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No * I- |3 E* a. i* A) W! g
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No
! {7 Y+ O- {" y3 T; D% t+ T Qgm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No 5 E6 x/ b. s! b: I3 O
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
' @+ x2 w8 A: e; q! O2 s2 lgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No
+ J# L2 T. ]5 V8 @+ wgm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No
. z; H& a- t( n) Ygm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
, P2 d2 }: H L/ Bgm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No ) }4 j+ @5 U x. k
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No $ w3 m6 L5 S1 I( i% h
gm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No ) j2 N/ R% Z7 X, o
gm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No : b) _ x& \. ?6 ^, h
gm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
( B2 n" Q& }5 ?2 J$ _gm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No
% c& `5 i0 ~" u# G5 h5 s& agm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
( U- ^; m2 ^ R# c/ a0 w( _% agm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
+ {* V9 h. Z9 M9 s# } o* kgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No
" }0 j+ p: t& j4 [ j! i, _gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No
- ^5 N& f }6 x5 X* {gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No ( p# d9 @7 o5 P
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No 9 X( g r: V' ?
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No
! S( `8 t- y7 B( k4 H7 U2 vgm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No ; d5 Y) i0 y: L! }; |3 N( N$ E2 i
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No
- P, N5 x% Y! c- m4 H" @2 q5 ^ zgm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No M- [, f" x. C0 I: g; O0 w
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No e; e) `' ]1 X
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No
& o4 E3 A- R4 q H5 j+ |2 I9 o* F- a3 ggm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No
: E- ~* }" t( \& W' C5 G4 U- ggm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No
9 y5 O p! E1 _" I T. n[root@tp266-1 ~]# ceph orch device ls " R; _1 t( N; _ c" V* d
Hostname Path Type Serial Size Health Ident Fault Available
4 J1 j Q$ x- c( `% e% ~* z Egm268-1 /dev/sdb hdd ZR57NZTL 16.0T Unknown N/A N/A No ! P z) V" n$ u
gm268-1 /dev/sdc hdd ZR5C2ZJH 16.0T Unknown N/A N/A No 4 ?* M: V* h$ c6 E c4 R7 n
gm268-1 /dev/sdd hdd ZR60WPTL 16.0T Unknown N/A N/A No
* ]8 l7 ^* G! ^. E" B3 a+ igm268-1 /dev/sde hdd ZR70RHVC 16.0T Unknown N/A N/A No
9 |4 b" w7 h; m Z- ugm268-1 /dev/sdf hdd ZR5BZP22 16.0T Unknown N/A N/A No
' y( S/ b6 E2 B6 D, ^gm268-1 /dev/sdg hdd ZR70S24Q 16.0T Unknown N/A N/A No 4 u- C; v2 a; `
gm268-1 /dev/sdh hdd ZR70SC6Z 16.0T Unknown N/A N/A No
) N9 x# X2 A2 K; x: u$ Hgm268-2 /dev/sdb hdd ZR60VY28 16.0T Unknown N/A N/A No 4 ?0 |+ b" u% Y2 p
gm268-2 /dev/sdc hdd ZR60VMD7 16.0T Unknown N/A N/A No " H; ~' n- f% k7 \9 g" ]5 Q
gm268-2 /dev/sdd hdd ZR60WP1Q 16.0T Unknown N/A N/A No " A5 h2 b- Y& N+ u: T1 x
gm268-2 /dev/sde hdd ZR5BV70V 16.0T Unknown N/A N/A No 0 m6 d9 H$ A+ [6 E
gm268-2 /dev/sdf hdd ZR70RD1H 16.0T Unknown N/A N/A No
, {0 i3 |: V* z0 R' w) s) W1 fgm268-2 /dev/sdg hdd ZR56SX4N 16.0T Unknown N/A N/A No 3 f& x% P W. k8 P
gm268-2 /dev/sdh hdd ZR60WHVQ 16.0T Unknown N/A N/A No 6 I, u: l- m/ Y; L, g4 c
gm268-3 /dev/sdb hdd ZR70S1H2 16.0T Unknown N/A N/A No
' g' v! Q$ g; j8 [/ mgm268-3 /dev/sdc hdd ZR5C05T9 16.0T Unknown N/A N/A No 2 r7 R2 e' n1 {: e: A1 y
gm268-3 /dev/sdd hdd ZR60W0VD 16.0T Unknown N/A N/A No
1 F1 j) E& f& f& Hgm268-3 /dev/sde hdd ZR70S8ZB 16.0T Unknown N/A N/A No
, X. f; `2 F% R9 Tgm268-3 /dev/sdf hdd ZR5BQ7DG 16.0T Unknown N/A N/A No
& s0 _9 L6 X4 V0 N% e+ w( H; x! Zgm268-3 /dev/sdg hdd ZR60WAM3 16.0T Unknown N/A N/A No
* s$ E# @7 x& h' Bgm268-3 /dev/sdh hdd ZR5BVZ1L 16.0T Unknown N/A N/A No . e: U7 ? y5 b4 m! L+ X" r* S
gm268-4 /dev/nvme0n1 ssd S4EWNF0MB07783D 1000G Unknown N/A N/A No
6 o; s0 } n8 Egm268-4 /dev/sdb hdd ZR5BQAQ8 16.0T Unknown N/A N/A No
) w" O% p. }( w5 |9 Qgm268-4 /dev/sdc hdd ZR60WSB1 16.0T Unknown N/A N/A No - N- ^: b7 {0 J
gm268-4 /dev/sdd hdd ZR60GAFJ 16.0T Unknown N/A N/A No 5 w8 {! o& u! o" y }
gm268-4 /dev/sde hdd ZR70S2MC 16.0T Unknown N/A N/A No * C k/ e) J* x+ o* v
gm268-4 /dev/sdf hdd ZR70RR2Z 16.0T Unknown N/A N/A No * Y+ v, ]9 Z2 Z- P
gm268-4 /dev/sdg hdd ZR70S9BA 16.0T Unknown N/A N/A No , c. t* ?' m$ Y# G
gm268-4 /dev/sdh hdd ZR5C0BSJ 16.0T Unknown N/A N/A No 2 u6 ~- e& [$ f! ^2 Z
gm268-5 /dev/sdb hdd ZR70SD4K 16.0T Unknown N/A N/A No + o3 h, N/ y1 l; K" [* Y# I8 z
gm268-5 /dev/sdc hdd ZR5BSKHS 16.0T Unknown N/A N/A No & b6 T) _( b" k1 O' ~ z" ?! g6 |
gm268-5 /dev/sdd hdd ZR5C3JGK 16.0T Unknown N/A N/A No / j8 I* s7 s. l: [# f1 V
gm268-5 /dev/sde hdd ZR70S1S0 16.0T Unknown N/A N/A No . A: M: U) l7 N6 z1 o0 d8 K
gm268-5 /dev/sdf hdd ZR70H12V 16.0T Unknown N/A N/A No % D( k" j" v4 I+ V. |8 P% T
gm268-5 /dev/sdg hdd ZR70S3AV 16.0T Unknown N/A N/A No
3 s" m+ U4 K, ?( @3 U, |[root@tp266-1 ~]# ceph orch host ls
, s$ X* d- C" V2 @HOST ADDR LABELS STATUS " m, ^: B# j6 }9 H+ u' ~
gm268-1 gm268-1 mon mgr mds
+ i" q8 \. B0 _) u7 b: v6 e# Igm268-2 gm268-2 mon mgr mds
; v, ?' i; V! o7 I* o+ a9 Hgm268-3 gm268-3 mon mgr mds ) R3 H: K1 P6 y, j$ j: A
gm268-4 gm268-4 + @5 g( q% q. r% G2 U6 o3 {+ r
gm268-5 gm268-5
* e0 e: B; Z; g7 ?2 n7 btp266-1 tp266-1
* F' \. o1 F5 z, o# c[root@tp266-1 ~]# cephadm shell7 s- g/ R! A3 d% l
Using recent ceph image quay.io/ceph/ceph@sha256:de7016bf6da2dcbb73796f66ef2776419b2c5a3ee1378835e8791a14c2c3e5b1
% _& K' Y) H, V3 ]7 g4 P! c/ |[ceph: root@tp266-1 /]# cephadm ; e8 s' L( B" [2 P
No command specified; pass -h or --help for usage
4 w- ^9 E" l I A1 U% a" {[ceph: root@tp266-1 /]# exit7 Z2 a9 c1 @4 A1 U5 B' q
exit
H( x0 L& r$ q[root@tp266-1 ~]# ceph orch host ls
' b/ V* W; W; o6 `: ?/ CHOST ADDR LABELS STATUS
+ r+ V- W1 Y2 f& v/ l( j2 [* t( E, |gm268-1 gm268-1 mon mgr mds
6 U. d% b* \1 Q5 i! n- Ogm268-2 gm268-2 mon mgr mds ' i9 m( M# v, \5 c! ~" v1 W
gm268-3 gm268-3 mon mgr mds
3 }* l/ w, c; zgm268-4 gm268-4
" u; A( L, {" ], D# Z/ m" P% ogm268-5 gm268-5
8 A7 N+ n0 [9 |8 R$ x9 t; ztp266-1 tp266-1
, V( C. m- {$ n, H( p0 n[root@tp266-1 ~]# podman ps
$ U$ [( `1 J6 ]6 }; `5 P7 Z) ]3 F# ]CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
" y' p1 s9 v7 K9 Z[root@tp266-1 ~]# exit
& f2 L4 y% i! P2 ]/ R% P* hlogout
2 m7 F; ^4 p& v+ @7 L( G% H" eConnection to tp266-1 closed.
* }0 A. m6 ?* T[root@gm268-5 ~]# podman
0 V" ^6 ~& P* u& D4 W U9 c2 M; {3 Kattach cp export image inspect logs play push save system unshare
* G h" o& t" j, S: Fauto-update create generate images kill manifest pod restart search tag untag 6 j3 E9 u1 L; \! ^6 n- W
build diff healthcheck import load mount port rm start top version
" O8 g3 V3 l1 C, w3 F) X6 \commit events help info login network ps rmi stats unmount volume . ^0 n7 s" G- p& X
container exec history init logout pause pull run stop unpause wait
( H9 B! Z$ ]6 F$ ? ~5 i5 g[root@gm268-5 ~]# podman ps
, I" k! p# B o7 LCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES4 k8 j5 ~, J. W# |
79ab1651e550 quay.io/ceph/ceph:v15 inventory --forma... 2 hours ago Up 2 hours ago strange_kirch
S% n# n9 T8 W5 ]' ca48e0b56a516 quay.io/ceph/ceph:v15 -n osd.27 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.270 A! S/ m$ t( D8 l
c16fff33a250 quay.io/ceph/ceph:v15 -n osd.26 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.26: M% p% x3 P' p1 K" l
6761927d89c6 quay.io/ceph/ceph:v15 -n osd.25 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.255 {5 h5 W& |5 @8 S0 @& j S6 A
3158677090f6 quay.io/ceph/ceph:v15 -n osd.4 -f --set... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.4
0 s0 ~9 O/ s; u) T2 w% Y, ~1a58b23cb4b2 quay.io/ceph/ceph:v15 -n osd.29 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.29' U! q; K2 B" e8 `2 t2 D5 E
e800611062b1 quay.io/ceph/ceph:v15 -n osd.28 -f --se... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-osd.28" t% r0 M. V& j
5c5454782321 quay.io/prometheus/node-exporter:v0.18.1 --no-collector.ti... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-node-exporter.gm268-5
+ I. w$ }, [9 e8 N06f85b89a143 quay.io/ceph/ceph:v15 -n client.crash.g... 3 days ago Up 3 days ago ceph-ce68aab8-8f46-11ed-88c0-ac1f6b3a30b9-crash.gm268-59 w% P2 b: Y& r* H
[root@gm268-5 ~]# df -Th1 V* T. T8 b4 I ^
|
|