|
|
检查ceph 状态:
- M/ ^4 M: D& j+ g8 o[root@compute01 ~]# ceph -s
7 y4 e1 r7 k/ D8 L4 J cluster:
7 J5 X% C7 D+ T1 w; k id: a6d0359e-8295-4e62-8620-54273462e79f! o% v+ l7 n6 k: C
health: HEALTH_WARN7 P# x4 E/ s4 R2 q
noout,nobackfill,norecover flag(s) set% C( J' N, i) |% k/ W. L8 O# x3 q
Degraded data redundancy: 3793/96483 objects degraded (3.931%), 876 pgs degraded, 512 pgs undersized
9 ` q! F. O+ T& F$ E; w+ [ 6 pgs not deep-scrubbed in time" L+ x0 G& U- M6 O8 `
clock skew detected on mon.compute02, mon.compute039 n) Q# M1 ~7 H1 f: _8 a$ ?; n" P
. _, b+ m- @7 F! Q
services:
5 G6 C% n1 E) D4 }' `$ h) c5 r mon: 3 daemons, quorum compute01,compute02,compute03 (age 22m)) Y6 a/ x5 s" x2 O+ C, d. t
mgr: compute02(active, since 4d), standbys: compute03, compute01, k) o+ l- |# H9 B
osd: 6 osds: 6 up (since 22m), 6 in (since 2w); 512 remapped pgs
4 X$ Y. s$ F. ~" G6 S; u; N; Z: H% A flags noout,nobackfill,norecover
( ^* V( v2 d3 V% h! {" A. N+ e5 W$ _& i9 n c% A5 l
data:2 Y0 S4 X3 D7 k
pools: 5 pools, 1408 pgs) m% x3 a v4 ^0 x) B5 M5 t
objects: 32.65k objects, 132 GiB4 g! u4 p- e4 P- O
usage: 381 GiB used, 4.9 TiB / 5.2 TiB avail" F! n' z# r6 R+ K( R
pgs: 3793/96483 objects degraded (3.931%)
6 c( X) o5 b- ~/ E 450 active+recovery_wait+undersized+degraded+remapped
# N M% N! j0 a6 H6 H/ q 444 active+clean
3 e) r( m+ u# d) N: c5 j `9 b 362 active+recovery_wait+degraded* b7 ?% F$ S3 L! t; t6 t6 e
87 active+recovery_wait
' |5 h8 O, h U 62 active+undersized+degraded+remapped+backfill_wait @; L+ ]5 B+ L
2 active+recovering+degraded, H l. @9 C* B) d! e
1 active+recovering
5 q) }5 K9 d0 A. y7 ^, d# }: ~5 J! @- l7 ]: f, ]/ W
io:
# H g6 U- w p1 f client: 156 KiB/s rd, 78 KiB/s wr, 14 op/s rd, 8 op/s wr" r3 {& x5 e" H! P- I. o( y
2 }9 q( E G, E; C$ e: }
% c8 h/ o R. y6 y设置ceph osd 开启recover和覆盖同步配置:
f! C; D& G- J5 O' M! c. w! G3 A$ d4 |0 f
, n8 X2 a2 }( _) H
[root@compute01 ~]# ceph osd unset norecover
9 u( S k% L0 d/ K5 O0 p9 Mnorecover is unset4 ]# l5 r l* | d) b" n+ S. ~
[root@compute01 ~]# ceph osd unset nobackfill
2 T# g8 h/ A0 t. {; y( ^' pnobackfill is unset
) ~7 {3 K- ?6 g" J( ?; R) ~3 H# @) p
查看状态:% C: m2 P: ~+ q7 s
[root@compute01 ~]# ceph -s
- O* U) }, f5 K! W6 e1 \ cluster:
- t. i6 H* R* f7 @7 u3 J: c3 Y/ i! i id: a6d0359e-8295-4e62-8620-54273462e79f1 L( Z- w& [, V) R* U5 c
health: HEALTH_WARN8 ~* M: S* r% k, j( m! f) d
noout flag(s) set
) |6 V* Q4 E. c8 ` Degraded data redundancy: 3537/96486 objects degraded (3.666%), 705 pgs degraded, 512 pgs undersized
- c4 H6 F% }3 V3 g 6 pgs not deep-scrubbed in time5 x& y: G' R: P- \- e
clock skew detected on mon.compute02, mon.compute03+ T) a; N* J. k0 C8 \% g
: x0 o7 f* U# s" S3 } services:- L& d5 ? e/ i2 Z7 P7 z0 u' a
mon: 3 daemons, quorum compute01,compute02,compute03 (age 23m)
: v' q+ B7 L/ u* C+ V mgr: compute02(active, since 4d), standbys: compute03, compute01
) [4 Q; y% q/ W5 L8 X osd: 6 osds: 6 up (since 23m), 6 in (since 2w); 512 remapped pgs
" S' A$ U3 X$ e' w, R% a flags noout
$ y# \8 l7 {% g1 j1 A( x
) V7 d: [! k3 b( g5 z! U data:3 B" P) n0 V$ y/ d8 i
pools: 5 pools, 1408 pgs
( v# f& j9 M4 M8 P. @ objects: 32.65k objects, 132 GiB
* v/ T' u; f( m usage: 382 GiB used, 4.9 TiB / 5.2 TiB avail
; D c, M3 B3 l5 C$ [: t8 D1 k! g pgs: 3537/96486 objects degraded (3.666%)
5 a- v$ g7 h' L% N. s/ z1 ~ 649 active+clean
. G$ G7 Y0 n" k$ J! Q 450 active+recovery_wait+undersized+degraded+remapped8 N# _2 f2 H% c4 T7 h
191 active+recovery_wait+degraded% f' L* i7 w+ u/ |4 u4 U
62 active+undersized+degraded+remapped+backfill_wait: L* q0 M+ w% V2 h: v( d; Y
54 active+recovery_wait
5 X# Z0 s: ~: d- A 2 active+recovering+degraded
( y* c& N8 i0 p4 J7 R
6 W6 j+ S! [$ P" J io:3 w4 h( a* I5 R6 t) d& p
client: 204 KiB/s rd, 797 KiB/s wr, 147 op/s rd, 42 op/s wr6 V; O/ Z4 R8 _* O2 P
recovery: 89 MiB/s, 21 objects/s
% a! Z& _: Z% a! p! A) D1 D" f 上面看到数据开始在同步了,覆盖了。! ^% e1 S" n( z- s* p+ j
: Z( D9 o; @7 h# d1 f8 G查看osd是否都正常:! T4 a: y& S6 P/ m/ U* {/ ^
[root@compute01 ~]# ceph osd tree
: n% _# Z4 T: xID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF 6 t' p$ g. f& T. E4 Z1 |9 I
-11 0 host ssd-compute05 / |- k' B% g @9 j9 t) q
-3 6.00000 root ssd
2 Q" H0 }8 B1 v( v6 M7 ] -4 2.00000 host ssd-compute01
* t- J2 T: D7 T: A 0 ssd 1.00000 osd.0 up 1.00000 1.00000 / R. U* u7 f( n2 J$ ~) Z: Q- |
1 ssd 1.00000 osd.1 up 1.00000 1.00000
$ v. m( X& h; ~" N7 k0 [ -5 2.00000 host ssd-compute02
. [) ]* x& h2 C) h8 q0 [ 2 ssd 1.00000 osd.2 up 1.00000 1.00000 0 N% D( n# \: u' f" y G
3 ssd 1.00000 osd.3 up 1.00000 1.00000 1 T0 C) J. `& u) b$ M3 I
-6 2.00000 host ssd-compute03
6 V( f$ z7 A& R' b3 S2 a 5 ssd 1.00000 osd.5 up 1.00000 1.00000 8 i/ S! E2 r4 r/ u+ ~- j
6 ssd 1.00000 osd.6 up 1.00000 1.00000
8 ~7 k- J; C/ x1 D& V -1 0 root default $ Q$ s' S) M0 \$ n
+ X8 j! R, k4 K& x- o) O# R6 L
[root@compute01 ~]# ceph -s
% f }, t& m& s7 ~1 b cluster:
' U% c' P* {) y0 r- P$ ^ id: a6d0359e-8295-4e62-8620-54273462e79f
4 {2 x; q) a$ z# O health: HEALTH_WARN
( ?- A1 d V6 w3 C/ l noout flag(s) set
/ [; v" r3 ]/ Y; l2 S Reduced data availability: 3 pgs peering
* W# `$ N/ r$ \" T Degraded data redundancy: 2173/96486 objects degraded (2.252%), 125 pgs degraded, 133 pgs undersized5 u, B- R( c! D6 O& m
6 pgs not deep-scrubbed in time% d/ L2 q" n- N: V9 e
clock skew detected on mon.compute02, mon.compute03
7 \- q, f: {6 V: R1 ]5 [- W' g- i& W/ `; D8 E- W$ Q. v
services:
! l9 U/ ?" B) T& R( Z mon: 3 daemons, quorum compute01,compute02,compute03 (age 24m)
9 b! q' F Y3 j( B. l# B mgr: compute02(active, since 4d), standbys: compute03, compute010 Q$ s5 h5 b$ A# S s% F
osd: 6 osds: 6 up (since 24m), 6 in (since 2w); 104 remapped pgs
( x% P& ~0 X- K0 n flags noout
, C; l6 H( P3 M/ Z2 _5 S
9 T9 Y( C1 O0 c& [6 | data:4 _% ~& `: v2 p) p1 D R9 N/ J
pools: 5 pools, 1408 pgs2 F4 F! p- G) r
objects: 32.65k objects, 132 GiB
! y0 C3 ?% B+ F% j$ j+ a5 z+ ^2 K/ Z usage: 383 GiB used, 4.9 TiB / 5.2 TiB avail
& M$ n( x) H6 q! d" B: o pgs: 0.568% pgs not active
H {' P$ z# S9 w, c9 m 2173/96486 objects degraded (2.252%)/ O: n8 ^5 `1 h: n6 S
1267 active+clean$ s J6 b& @ C& [1 k. l
62 active+undersized+degraded+remapped+backfill_wait3 y9 l+ {/ `* _
62 active+recovery_wait+undersized+degraded+remapped- D! |1 k: }. ~) q: ]
8 active+undersized+remapped$ z, ?$ j, b$ D1 J* w; F, B2 `: e
8 peering
+ m: z: y5 r" u* } 1 active+recovering+undersized+degraded+remapped
4 n3 ^7 b: b- B$ u I& n* t$ \. X0 q8 N* C
io:( C3 Q d) G2 c0 I3 B. L) Y) v1 f
client: 407 KiB/s rd, 792 KiB/s wr, 34 op/s rd, 52 op/s wr
# w) H9 K! p9 N) w, J recovery: 72 MiB/s, 2 keys/s, 22 objects/s! d s" Z9 ^ N+ c+ v! a. `
; }2 p# |' Y& T* i! ]) x7 a. Y% ?" r( p' U& K/ |: c+ J! q
查看时间:6 Q/ j# E. y/ N0 P/ U5 g0 ~+ w% K
[root@compute01 ~]# date& q7 u6 f( h ]2 T. G4 u8 }2 U
Mon Dec 20 10:32:23 CST 2021
9 D7 m3 m. T0 [2 g5 |[root@compute01 ~]# date
$ ^" t9 w/ \" P, _8 IMon Dec 20 10:32:38 CST 2021
4 z) |, C4 P2 c9 I[root@compute01 ~]# date+ P6 L. h8 |5 v: J* y0 c8 Q) a$ l
Mon Dec 20 10:33:00 CST 2021
5 e( z. _. l7 X
+ N% ?! u& {9 U$ O[root@compute01 ~]# chronyc sources
( k" n; Y- T3 k- i2 u/ E! O210 Number of sources = 26 t4 [$ D7 L1 Q6 W" H
MS Name/IP address Stratum Poll Reach LastRx Last sample
8 i1 _; c M$ f7 c# N, V1 M3 u===============================================================================
, w& t# E: p/ i! ^: r2 M^? compute01 0 9 0 - +0ns[ +0ns] +/- 0ns
+ d! u+ [. ~/ \+ Q3 J^? compute02 0 9 0 - +0ns[ +0ns] +/- 0ns
1 e* A4 [9 y6 q! T; d- G; l0 z[root@compute01 ~]# vim /etc/chrony.conf + V8 v2 [. Z2 z# y7 y% M
0 ^4 x' G( ~* p, r, V
[root@compute01 ~]# systemctl restart chronyd.service + a7 u* T& X5 m& |% }
[root@compute01 ~]# chronyc sources 0 ?3 Z4 M+ J( c& k( J: ~
210 Number of sources = 2
6 b K! T4 t x" JMS Name/IP address Stratum Poll Reach LastRx Last sample
& b* D3 g2 T3 e===============================================================================
( Z: g4 m# Z& m" \- j) v: k* l^* compute01 10 6 17 3 +2667ns[-7752ns] +/- 28us
& F, F& J* }) i$ j- N^? compute02 0 6 0 - +0ns[ +0ns] +/- 0ns
' z8 q3 G3 y1 y) _! O. z# k[root@compute01 ~]#
, k. n4 a Z0 f- u7 Z[root@compute01 ~]# & T( c2 D" a) V, u3 G r& ^
[root@compute01 ~]# vim /etc/chrony.conf 0 r+ I* j$ n+ L8 K
[root@compute01 ~]# date, ~5 `& k" Q F( j
Mon Dec 20 10:36:29 CST 2021' d. _ c$ c% e* W% [8 w* v
- A2 z2 H$ R( { A, [2 F1 V$ C, L3 a5 k' d& _+ a1 J
时间同步即可:; X2 h* d/ C- M: K! F* Q+ D
0 S! p; N4 K; |[root@compute01 ~]# ceph -s0 e3 p9 a/ V D
cluster:* r# l8 m: k T. s7 b
id: a6d0359e-8295-4e62-8620-54273462e79f/ s( F& N4 R" N) i
health: HEALTH_WARN# m9 m3 L& n4 @- @
noout flag(s) set
+ A h+ D2 B: _0 b; s$ B 1 pgs not deep-scrubbed in time" I% K7 g+ p- z& R# }$ a! Q* X; m
4 q. A; a8 {/ H- Z: B; h7 w
services:
: W0 m& ]+ f- j5 p+ r mon: 3 daemons, quorum compute01,compute02,compute03 (age 48s)0 m( J0 j! [- K9 X- C
mgr: compute02(active, since 4d), standbys: compute03, compute01: ^" [) { Y! Z- d# q6 c; d
osd: 6 osds: 6 up (since 28m), 6 in (since 2w)
! |: M3 G0 m3 V ~8 W2 E9 a flags noout; O9 P4 V* @" N" }
# v1 ^5 M% e0 ~& Q- C data:0 \4 g8 L9 x7 e; I4 ]1 O
pools: 5 pools, 1408 pgs
V' u. F W% w2 `% u- C7 S* B0 K+ Y2 | objects: 32.65k objects, 132 GiB
' n* C2 @/ H# e8 d) k' v+ P usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail+ S, S- s- [ ~4 b( m# c) B* y/ \
pgs: 1407 active+clean
9 @. V( c# [+ n5 p' N 1 active+clean+scrubbing+deep
* d, m- j2 c; N& Q1 [6 M" B! S# Z; v
6 K5 Z& K5 y5 Y) M/ \. _ io:4 `( W3 R9 b3 n9 \- u1 I* K
client: 20 KiB/s rd, 499 KiB/s wr, 4 op/s rd, 18 op/s wr2 b& A% T7 y6 Y/ D0 ?9 P4 O
, y, y* q1 o! V, }
查看ceph存储是否正常:5 ~4 }; l) T. b2 U
* M$ y9 P- D0 W' }# K: C[root@compute01 ~]# ceph -s
: {2 _/ G8 q7 r1 K1 a cluster:
* k" q) y& Z7 M0 c8 C id: a6d0359e-8295-4e62-8620-54273462e79f
- l Y2 I# S& k1 V health: HEALTH_WARN
: Q5 K" J, l, N# E' p noout flag(s) set8 M' u0 ^0 u% m4 U0 L) t8 K. i" l
1 pgs not deep-scrubbed in time
" K8 m `" d$ n) H) X2 u# D
0 E0 e1 ?3 m$ j. l6 E! G services:
5 q& X& c" ]' e) j( E& O! C" u mon: 3 daemons, quorum compute01,compute02,compute03 (age 60s)
* F- l; ?# i }+ i' D: H mgr: compute02(active, since 4d), standbys: compute03, compute01
* v( ]4 Q4 h/ r osd: 6 osds: 6 up (since 28m), 6 in (since 2w)! w I& m0 O3 O- R ^9 m( A8 d* O
flags noout- j8 ? o& f! z5 m2 Z
& {! a: K: P+ t( n' C% H* U) s data:
5 i' ?6 G5 U/ N$ W+ ~ pools: 5 pools, 1408 pgs
$ i5 `& V7 M6 g8 ~3 r2 ~ objects: 32.65k objects, 132 GiB4 Q0 b9 o/ N7 |9 e- B% k2 H
usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail
9 F5 t0 [0 \2 j( ` pgs: 1407 active+clean1 B# A6 n3 G, ]. a
1 active+clean+scrubbing+deep
; j7 Y) s" i0 ?- j9 m9 B
1 R/ l" c5 M( i- c( g: r; G# U io:) j- d$ _6 _4 }1 [( J
client: 120 KiB/s rd, 595 KiB/s wr, 13 op/s rd, 33 op/s wr
: q1 U; t* i1 }( T" g& P3 D* W! |8 U* Q
, e6 R1 t. l3 v: t$ z9 J3 Y! P[root@compute01 ~]# ceph osd unset noout
$ b. G( j8 f% h& K5 b/ O: J2 W% s* gnoout is unset* E+ ?2 |9 d' s, t. m7 d4 M2 E
. i- g: r% s. N+ G8 X m( G
检查状态:
3 q, @1 P3 n" j5 X. Q$ R) ~[root@compute01 ~]# ceph -s3 z) ~% v. |; N) `; i7 C
cluster:1 r6 p* K! M+ N, U
id: a6d0359e-8295-4e62-8620-54273462e79f) Z6 f% L: ?5 s/ ~* {6 ~2 ^, C
health: HEALTH_OK
3 g" G) d3 L$ u5 s* w" l$ C) C9 U5 N7 J7 f
services:
' T! K: Q |2 Z9 T B7 t" G mon: 3 daemons, quorum compute01,compute02,compute03 (age 100s)
$ s7 }' {; g0 ?, b mgr: compute02(active, since 4d), standbys: compute03, compute01* E& q! O( M& h" C5 b: p
osd: 6 osds: 6 up (since 29m), 6 in (since 2w)
; U" ~4 g& W! a7 F
. x3 y c2 Z, ?- E$ t0 ?. w data:$ X9 ]- U9 S, h: \$ S/ `' h9 V0 p
pools: 5 pools, 1408 pgs
) B$ H+ o n& H5 t1 H7 C objects: 32.65k objects, 132 GiB- @6 W9 i7 i* o3 z8 k' o1 j
usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail
9 j* Y0 U p6 b- j, P pgs: 1408 active+clean3 N8 Q$ b$ l6 `5 F! p: l' J
- E: X1 }$ N& V io:
5 J; f7 W" `( }- |: i. V8 |# t# M client: 539 KiB/s rd, 285 KiB/s wr, 191 op/s rd, 26 op/s wr
6 F2 ^0 L; L4 f. ?
" u+ l; P! C4 v( k[root@compute01 ~]# ceph -s
" `# w4 w& Q# K cluster:4 g* G& u0 R0 N& O' G
id: a6d0359e-8295-4e62-8620-54273462e79f: k. k t( J& }% J+ w, F7 |
health: HEALTH_OK
, ^- C" U- d+ G/ o# [. ]
! L6 d( ^2 @$ w8 |- H" B* O% F services:
# r5 o. V* d8 A! q% c8 f mon: 3 daemons, quorum compute01,compute02,compute03 (age 102s)
+ p& R( k1 a) L; `7 Q- m* K mgr: compute02(active, since 4d), standbys: compute03, compute01
. }* A: v* n" B# Y osd: 6 osds: 6 up (since 29m), 6 in (since 2w)
+ ?0 K( v6 z8 }
4 G1 K2 L2 E2 [- e% { data:* ^. s0 f9 k! h
pools: 5 pools, 1408 pgs
/ t; j1 {( J* ?' V" { objects: 32.65k objects, 132 GiB
( \/ B+ E4 G1 H& d5 @8 e+ k usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail! ~% p+ s' H% S3 J% \" ^: a
pgs: 1407 active+clean
0 t: H: _& ^8 [' j 1 active+clean+scrubbing+deep
6 I+ \. L! @& O2 {3 ?8 z& P" S2 S3 U/ u; H
io:( z4 d2 y. L* h8 q2 T$ }
client: 110 KiB/s rd, 196 KiB/s wr, 12 op/s rd, 18 op/s wr% v, P, i! ]7 s$ Q4 M
* ^' U- ^& ?, k[root@compute01 ~]# ceph -s
! z# z# M$ \$ v1 b S cluster:+ v- v6 z6 \6 V/ c) U( b$ b l
id: a6d0359e-8295-4e62-8620-54273462e79f
0 Q0 ~5 F8 |; N) W# U6 L: C health: HEALTH_OK8 Y. X* P( [% l- ]4 n( {& w! o
! x- m1 G* I7 l4 E, V( M4 c services:/ B2 q& n2 d n! j& ]& [
mon: 3 daemons, quorum compute01,compute02,compute03 (age 105s) E9 Z1 l3 o% z. {, |
mgr: compute02(active, since 4d), standbys: compute03, compute01+ ~% K* z# M' x! z; }
osd: 6 osds: 6 up (since 29m), 6 in (since 2w)& p$ o V8 q: b) }& k# J% W" V
0 i! v" P; ?# s# @% ]+ Y0 `
data:' p" ^$ ^4 n# I7 d
pools: 5 pools, 1408 pgs
p: r! L G8 @' ?3 C4 N+ N F2 l objects: 32.65k objects, 132 GiB; m; }# D$ Z) D" z
usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail1 V. u6 P ?( T
pgs: 1407 active+clean
' P0 q9 _- j7 d6 C" w 1 active+clean+scrubbing+deep
, [( E) O& D9 _' J1 @0 ?5 y9 I" v, c$ {! s: s+ P7 V
io:
( Y# c$ g/ E, E5 p" _$ H client: 74 KiB/s rd, 193 KiB/s wr, 10 op/s rd, 18 op/s wr7 \6 \8 Z3 x9 s* K; Y
5 E& c$ K5 G/ B+ y( ~/ o2 \- y, G[root@compute01 ~]# ceph -s$ ]9 i+ V$ e5 `$ {, q/ h0 D
cluster:; b0 Z _# h) |8 `
id: a6d0359e-8295-4e62-8620-54273462e79f: D# U3 E& g. J2 E
health: HEALTH_OK
0 s* [' @4 r9 h8 y; }* I$ r y. l3 f5 F2 a o$ W0 q" V
services:8 O0 A$ J Z+ i
mon: 3 daemons, quorum compute01,compute02,compute03 (age 3m)
; m' @& r+ l1 V2 i9 E: N mgr: compute02(active, since 4d), standbys: compute03, compute01
9 } b* y* h4 a osd: 6 osds: 6 up (since 30m), 6 in (since 2w)0 y8 u0 `& B) i( l8 T, R
8 v! S' k! m( v- s2 A3 ~8 ]
data:
0 K' S& h4 ~0 l: c0 o. K pools: 5 pools, 1408 pgs- n) q9 J/ J4 B1 t
objects: 32.65k objects, 132 GiB/ S$ B1 l2 H: P0 h& Z' o
usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail
; X0 H0 ^0 O$ }1 m pgs: 1408 active+clean8 Y/ |6 h7 W3 q4 F! r0 G" T' i8 H
" ~2 d2 @6 S9 [7 A, v. {8 v! a
io:
$ j9 }5 ?- ]2 q5 y( [1 T' I client: 840 KiB/s rd, 1.4 MiB/s wr, 63 op/s rd, 43 op/s wr8 J! \, l) R1 v# G
/ q8 U$ R( V, \
' _3 r- o' ?; j2 G% c+ x" b% V
5 Q' t8 }1 E& W& G% t& ]1 D9 t' h$ q+ m, ?" s3 @# ~
总结,此过程中出现的问题,一是时间同步服务器有问题,二是ceph 设置了不剔除osd,不同步,不分配问题。
/ R( v9 d/ f7 V" L5 n/ K2 n
; P' b3 a! [9 y/ D* |( O0 M7 @2 G0 v8 G! p; K
|
|