|
|
检查ceph 状态：
, h$ X& p0 ]: q# y3 ~" u4 [7 b% y% x# n3 z[root@compute01 ~]# ceph -s8 V5 f) D9 f& h6 M( f* k3 C4 h
cluster:
% p- [2 |9 q/ G) G2 I id: a6d0359e-8295-4e62-8620-54273462e79f! p2 u4 \' B% E; c2 [9 }/ d
health: HEALTH_WARN& o2 S4 t' ~; R. B
noout,nobackfill,norecover flag(s) set
7 a3 Y: D1 \9 E* W8 e Degraded data redundancy: 3793/96483 objects degraded (3.931%), 876 pgs degraded, 512 pgs undersized2 z. E& Q0 W3 b, d# U" A
6 pgs not deep-scrubbed in time @7 S( y0 n/ e/ b
clock skew detected on mon.compute02, mon.compute03& T3 P9 u) d7 R
; e1 _" m. i, o% o services:; m8 s, G* M/ w
mon: 3 daemons, quorum compute01,compute02,compute03 (age 22m)
8 ]# D" `9 m: |& d mgr: compute02(active, since 4d), standbys: compute03, compute015 K8 o4 |) _0 R5 \
osd: 6 osds: 6 up (since 22m), 6 in (since 2w); 512 remapped pgs
6 B# w9 A. X- ]; z0 R$ Q1 b2 V flags noout,nobackfill,norecover2 Y S. r. v: S* R+ m2 |
- G( B1 y+ K" k( `- }4 t data:# X. r2 F( C+ p+ G" ]
pools: 5 pools, 1408 pgs0 z; g) v2 F6 G" g7 v1 t
objects: 32.65k objects, 132 GiB4 _$ H; ` e$ H' w4 P
usage: 381 GiB used, 4.9 TiB / 5.2 TiB avail
, ~$ O1 K9 d1 s pgs: 3793/96483 objects degraded (3.931%) [ { _& @5 r% }9 ]0 J' V' Y, Y
450 active+recovery_wait+undersized+degraded+remapped* z! j0 H+ W* P8 ^+ c6 [
444 active+clean4 x/ x& }% w3 B& i
362 active+recovery_wait+degraded
' j% g3 e- r6 A* t" v 87 active+recovery_wait
/ h, f- K6 r2 H$ E- l 62 active+undersized+degraded+remapped+backfill_wait
$ i+ g3 J3 p8 ~/ G f! M 2 active+recovering+degraded6 p' N. A' J t- z$ q9 h: I% `; P2 |) A
1 active+recovering
: }* Y8 g3 U2 ?0 H/ M# O3 x, k
* j. E& [: T$ L. B* L8 ~ io:
' W; X, a0 ]/ g3 t J- A, p client: 156 KiB/s rd, 78 KiB/s wr, 14 op/s rd, 8 op/s wr- K& Q6 \9 O1 S" ~5 N1 F5 U
/ A, l! E8 a6 D/ @9 V- C" m
, `( F3 V/ b' y1 L! t( o+ g设置ceph osd 开启recover和覆盖同步配置：
+ a& u- O5 Z% z9 V* W% m9 x# `9 W8 {) k& ]$ H7 |" w: C# O
0 d3 y. u4 Y, ~/ B" n5 I! z[root@compute01 ~]# ceph osd unset norecover 5 Y& \4 X R& t6 f7 e
norecover is unset- H# L7 n- }7 k3 h; I; \5 c: [
[root@compute01 ~]# ceph osd unset nobackfill
6 v ~1 ` ]1 D, O: ? f5 Gnobackfill is unset
3 t: w+ g1 G; f% ~
; {" h/ w9 F4 Z查看状态：0 t; h7 }' U' n4 [( d; o
[root@compute01 ~]# ceph -s% f3 U& W W! u
cluster:3 h4 `, d6 v; l' q
id: a6d0359e-8295-4e62-8620-54273462e79f5 C1 @, {; e- r* Z
health: HEALTH_WARN; N1 D# n* {' k
noout flag(s) set
$ i4 m0 G4 F1 w( L- L9 ? g Degraded data redundancy: 3537/96486 objects degraded (3.666%), 705 pgs degraded, 512 pgs undersized- Q: u( G2 ?2 s0 t$ P: ?
6 pgs not deep-scrubbed in time8 g Y2 `8 X+ e% j% _
clock skew detected on mon.compute02, mon.compute037 P" J# i# k" F; N
6 S" v9 O2 j9 G! Y: N
services:
' a# c, R# G' I4 x( ]" d( D mon: 3 daemons, quorum compute01,compute02,compute03 (age 23m)7 O f D; W+ Z, u, B) _
mgr: compute02(active, since 4d), standbys: compute03, compute01
' V/ e) G6 L+ r/ {" f, N; Y9 i7 a osd: 6 osds: 6 up (since 23m), 6 in (since 2w); 512 remapped pgs
1 i4 P) z g k( P+ | ~ flags noout# y, W& V1 O# N2 i# h5 e0 _
: v. y2 }! W9 V s2 ? A) \" T
data:
. w3 t L' {9 d, J \ pools: 5 pools, 1408 pgs/ S- N/ _8 D4 d$ ?0 K: o/ I4 r4 j
objects: 32.65k objects, 132 GiB
1 g% J4 t; z8 u& Z; P. V3 W3 ? usage: 382 GiB used, 4.9 TiB / 5.2 TiB avail
" ]7 W6 q7 j1 V) j6 L1 }% D pgs: 3537/96486 objects degraded (3.666%)
* t4 V9 g+ K. u- J 649 active+clean# `; _% u, A! G4 H X9 @
450 active+recovery_wait+undersized+degraded+remapped
6 \$ F1 L, Q+ I: Z: Q 191 active+recovery_wait+degraded
; C5 g6 C% D$ R. g 62 active+undersized+degraded+remapped+backfill_wait
' @7 m- {8 Q. J# [/ t" _6 W 54 active+recovery_wait c! z4 @( q' M. G5 A9 s% b
2 active+recovering+degraded
: R; u& |. G1 X3 r3 A6 }& X$ t( ? ^1 W) T8 G+ Z- d
io:
3 I# s7 f, G' Q; ^( t1 |0 D6 U7 I, L client: 204 KiB/s rd, 797 KiB/s wr, 147 op/s rd, 42 op/s wr, M2 z2 {* n5 |. P T
recovery: 89 MiB/s, 21 objects/s4 Z5 G: \% ]. A3 _6 B% t& {0 l+ E
上面看到数据开始在同步了，覆盖了。: N& T! H' _ Q* N! z
5 t4 _, M$ D+ h0 P5 v
查看osd是否都正常：
" S$ }9 b1 D9 F0 J[root@compute01 ~]# ceph osd tree8 |1 b$ G; r- z! b; @3 S2 M
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
( Q* Z5 G% V0 d# j1 f! M) E$ i-11 0 host ssd-compute05 3 a" |$ b9 ]. R
-3 6.00000 root ssd
" @. A$ \# L" u+ j -4 2.00000 host ssd-compute01
8 v6 S+ S; o' s+ n/ q 0 ssd 1.00000 osd.0 up 1.00000 1.00000 1 r- ^6 e' ]! v9 F
1 ssd 1.00000 osd.1 up 1.00000 1.00000
1 f6 l7 R7 E% f' n& v) h -5 2.00000 host ssd-compute02 9 M+ a5 S( n: S' H, z* R1 j
2 ssd 1.00000 osd.2 up 1.00000 1.00000
$ e v* o1 n: o5 C 3 ssd 1.00000 osd.3 up 1.00000 1.00000 7 A# ?7 H% r4 m, f# K" z! J
-6 2.00000 host ssd-compute03 & T9 S! F; }9 Z
5 ssd 1.00000 osd.5 up 1.00000 1.00000 * \+ P# T* r5 V
6 ssd 1.00000 osd.6 up 1.00000 1.00000
7 z( J, \, g/ Q# B: i3 }9 U* y -1 0 root default
. F) V b5 N& G# B0 K! P4 t2 B/ G# b, o8 a$ J a' [4 H
[root@compute01 ~]# ceph -s* R- A4 s: J. J" x. ]3 R* S
cluster:
0 u/ M0 e8 C' n5 p6 [. s3 }; r. n7 x id: a6d0359e-8295-4e62-8620-54273462e79f* T4 ^4 k i1 @* k& K
health: HEALTH_WARN
8 S+ U- A! |" S( M9 m noout flag(s) set
1 K5 e# ~5 A/ `% K% T: m Reduced data availability: 3 pgs peering/ ~( {" E' m" P
Degraded data redundancy: 2173/96486 objects degraded (2.252%), 125 pgs degraded, 133 pgs undersized
1 W- r( z, @6 A8 }5 P 6 pgs not deep-scrubbed in time
( v: l0 A0 K* C Y/ d7 v clock skew detected on mon.compute02, mon.compute03
8 Z \4 S; m" B5 Q- ?' Y
- G3 }9 F* K8 m7 @1 H* b" l3 A services:# p8 R2 N9 a5 H1 I$ B' S! q4 X1 {
mon: 3 daemons, quorum compute01,compute02,compute03 (age 24m)/ H: V, ^/ @ g0 b8 L
mgr: compute02(active, since 4d), standbys: compute03, compute015 }5 Q$ h% _1 j4 o2 f4 V
osd: 6 osds: 6 up (since 24m), 6 in (since 2w); 104 remapped pgs; w1 r; f7 e" ^$ Q. l8 X
flags noout t( J( \. t5 Y+ H$ G. p
; V. o, m, w' e$ o3 U! h4 } data:
' W6 X1 a: Z& z+ ]% l, R/ k pools: 5 pools, 1408 pgs
, C4 {8 C, |) a% r" W4 O objects: 32.65k objects, 132 GiB- ]+ U" E+ b, H" S% o
usage: 383 GiB used, 4.9 TiB / 5.2 TiB avail: ]' n7 `& f8 F4 q$ o* q, z1 F
pgs: 0.568% pgs not active
" R" K" w9 p4 X1 I 2173/96486 objects degraded (2.252%)
9 s' |, R! T+ D( c 1267 active+clean
/ a# a1 p: t3 a4 B; s 62 active+undersized+degraded+remapped+backfill_wait" j4 }% w0 p1 s4 u) S
62 active+recovery_wait+undersized+degraded+remapped
( _. g) x% z+ u7 N) F' a 8 active+undersized+remapped
7 U& A) E8 H0 ^9 l8 @" _ 8 peering
6 H7 Q5 x, o& p: d5 u% m 1 active+recovering+undersized+degraded+remapped
9 I* ], X# @! H" J q# _
. p1 R! M7 }0 U' j/ p# G& P' y6 N! d io:( V, g7 J) t- d7 }3 p0 E( u4 U
client: 407 KiB/s rd, 792 KiB/s wr, 34 op/s rd, 52 op/s wr
; p+ _6 ^- e1 L4 t8 c) K recovery: 72 MiB/s, 2 keys/s, 22 objects/s% P* q7 x* j5 J
- f' z. l% l9 |# m0 V8 \4 v8 q* [: m' u& Q( W; K
查看时间：2 C' p9 A/ d2 a% Z
[root@compute01 ~]# date
# \, Z, Q- Q. A- w+ E3 hMon Dec 20 10:32:23 CST 2021
: T8 ~5 Y$ q e3 B8 \9 ]; O[root@compute01 ~]# date4 q- n- c( M9 u+ F4 c% ^
Mon Dec 20 10:32:38 CST 2021; @7 _4 w8 G' c
[root@compute01 ~]# date
c7 F9 ]+ Q' D) e, @- G6 e. IMon Dec 20 10:33:00 CST 2021
& P: d3 H6 y& F. \8 A# j1 P# @: e0 j( X5 j, }2 c
[root@compute01 ~]# chronyc sources
* k" S6 x! V0 i210 Number of sources = 2: l$ z, Q# d4 u- }
MS Name/IP address Stratum Poll Reach LastRx Last sample / R. A5 W% N9 f& Q/ D
===============================================================================% Z, e$ u* P: V0 W$ g* ~5 ^ L
^? compute01 0 9 0 - +0ns[ +0ns] +/- 0ns ~* ^3 S5 k$ D5 b! @2 H4 G p
^? compute02 0 9 0 - +0ns[ +0ns] +/- 0ns
0 K- q2 v$ q/ {. B d[root@compute01 ~]# vim /etc/chrony.conf
; Q, m$ K/ m( D2 ?. R; e( y+ d/ |# r8 R5 C0 l" X! n( ?
[root@compute01 ~]# systemctl restart chronyd.service 9 ?/ ] H3 |! d+ j0 E; E
[root@compute01 ~]# chronyc sources
% z$ m8 z! e% E4 Z' z210 Number of sources = 2
5 Z& M4 r9 j# M6 @" B1 D( VMS Name/IP address Stratum Poll Reach LastRx Last sample
( I+ a% r$ J" P8 t3 U===============================================================================
$ b7 D" C+ q9 i, B* i1 o- m^* compute01 10 6 17 3 +2667ns[-7752ns] +/- 28us( J4 `3 b+ X& [# ~
^? compute02 0 6 0 - +0ns[ +0ns] +/- 0ns& H6 o8 t/ Y7 |& l
[root@compute01 ~]# 1 c! g3 A. `5 E0 x+ c
[root@compute01 ~]#
2 }/ @ k* y) J[root@compute01 ~]# vim /etc/chrony.conf
$ H( N2 u6 s! T# |6 U* ^% O[root@compute01 ~]# date
9 {/ j9 m' u* s, n- ~Mon Dec 20 10:36:29 CST 20210 a# j# k( _9 u1 E' c5 F( j
. N5 Y* h" f) Q7 n. d& c& T& `7 K; t, x7 j8 p/ X% W6 v9 m
时间同步即可：$ n6 K; w- B# {% }& n9 Q
5 m/ O' D2 o! |$ W+ X[root@compute01 ~]# ceph -s
$ `* O, d0 S# L- g cluster:
1 a* C3 ]% V. G+ O7 s id: a6d0359e-8295-4e62-8620-54273462e79f) E+ @% Y$ t/ q( \0 S' Z- I
health: HEALTH_WARN/ Z) b( f* D# |
noout flag(s) set9 O1 s- _( X. W0 f6 O# a
1 pgs not deep-scrubbed in time
( U3 v$ I! Y( z' n( _& a& f) K; o8 u1 `( w
services:
6 W) ^- w: V( B4 D k6 L3 x9 r mon: 3 daemons, quorum compute01,compute02,compute03 (age 48s)
0 ?2 f' p* G8 J% L5 j! o" ^- n mgr: compute02(active, since 4d), standbys: compute03, compute01
( t6 x- x/ a7 o/ K+ n osd: 6 osds: 6 up (since 28m), 6 in (since 2w)2 T* P/ [" G6 A8 P, C5 M. ]; g( s
flags noout% `1 V. S" B8 h8 V3 N' z/ W5 w
+ F& ]$ {" Q/ Q8 k* h' } data:
9 x7 p' k& I! G" I6 y+ k pools: 5 pools, 1408 pgs
$ F h9 g% _) p- B: { objects: 32.65k objects, 132 GiB( X; i1 l$ c4 c7 t, B/ _6 |
usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail) c2 g1 I5 f6 C& A* F
pgs: 1407 active+clean
6 \$ Y& V7 T+ x0 O 1 active+clean+scrubbing+deep
. t% e( f4 C3 \" G+ W$ `) ^
/ @3 o \& U& V' b9 F io:
, V+ z+ f# [' _6 g$ `0 R: R+ ? client: 20 KiB/s rd, 499 KiB/s wr, 4 op/s rd, 18 op/s wr* {8 j6 T0 _' V0 {
: v- n* d7 Z7 t; e7 S 查看ceph存储是否正常：6 H8 z& S' D1 W+ R3 I
5 [- J2 q+ U J$ Y' ?
[root@compute01 ~]# ceph -s
4 y9 J5 V6 z C cluster:0 T; i/ e. O$ p8 t: u. _6 Y0 Q b
id: a6d0359e-8295-4e62-8620-54273462e79f
6 [: |% G$ X4 c1 m health: HEALTH_WARN4 t8 T% a) s) k- h, E" z$ a' e( D
noout flag(s) set, ~ p( Z- g$ L7 T+ h' S% Z2 t
1 pgs not deep-scrubbed in time; J0 c+ n% C9 T% C8 ?
- A; ~4 c5 ~0 `! o# X
services:0 R% [4 X7 ^, s# P- h( N) c
mon: 3 daemons, quorum compute01,compute02,compute03 (age 60s)+ i. |! v: t7 P
mgr: compute02(active, since 4d), standbys: compute03, compute017 c I+ Q9 C% S `; }
osd: 6 osds: 6 up (since 28m), 6 in (since 2w)+ Y6 [9 f3 a& B3 K
flags noout
0 S6 Z% U0 Q6 M( I0 l# A1 _. f" U, \8 ?, r" k5 k
data:
6 k( D# e! u- F' C. T4 k4 |" g pools: 5 pools, 1408 pgs, e( P; j. W! Y5 v# P! \
objects: 32.65k objects, 132 GiB
a% C) Y6 l9 N' z& K1 f usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail+ I: b: B X% S# Y% w* P$ \+ _8 d
pgs: 1407 active+clean! n; K! ?1 ]/ ?* K D
1 active+clean+scrubbing+deep! ?" F6 \" e8 ?# h; i7 C
' r6 [" N4 @! |
io:
( }* {' \+ \6 U2 B& H1 d2 Q client: 120 KiB/s rd, 595 KiB/s wr, 13 op/s rd, 33 op/s wr5 ]! [7 \% b1 P: @* b
! c: @2 b# B- x# N
! ~! _2 l% K7 k: ]( Y( ]2 F[root@compute01 ~]# ceph osd unset noout; p" I& t2 E( U; W$ H
noout is unset& U& t% I4 x! x* h1 O& a5 v
" \7 L+ q3 I9 p; {
检查状态：! k3 c; E* S" y+ Z
[root@compute01 ~]# ceph -s o* Y7 _6 d4 z) J: \
cluster:$ T; V* I3 d3 \, R% T+ I. ^
id: a6d0359e-8295-4e62-8620-54273462e79f
# ]& H, W3 L& `( F; ]9 ]2 V% [+ T' Q health: HEALTH_OK
5 P4 E$ o& K+ v1 M( N! n1 m X5 `# R* D1 J
services:2 P* c7 p/ b1 Z0 S! ^
mon: 3 daemons, quorum compute01,compute02,compute03 (age 100s)- V& e; r1 V$ w/ p; }
mgr: compute02(active, since 4d), standbys: compute03, compute01
( G9 {7 K. d+ M8 i$ Q osd: 6 osds: 6 up (since 29m), 6 in (since 2w)
0 a# D; n4 I- W4 T3 d& H0 D. @ t: ]; {4 N4 n6 b& v
data:, d) ?* }0 I4 ^" W# r
pools: 5 pools, 1408 pgs; s! S" m) _6 S
objects: 32.65k objects, 132 GiB
& o" @' }/ B- v usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail
* I" D- h9 D9 D2 ` n pgs: 1408 active+clean
* b7 \/ G0 U; Y* u& a% s/ a) T7 }
io:
* |7 u8 i5 N- O& P client: 539 KiB/s rd, 285 KiB/s wr, 191 op/s rd, 26 op/s wr
- U. w/ M3 h* k+ Q" w# M i* r: r$ s7 P9 N- j) q
[root@compute01 ~]# ceph -s" U% c3 @7 Z6 j9 i# p
cluster:
3 }4 a# j1 ?- G t5 j id: a6d0359e-8295-4e62-8620-54273462e79f7 m8 y! l6 |' W9 {' U
health: HEALTH_OK
+ [ n- U( Y1 o
. m/ d' e! U k1 D) y services:! g; \# S2 M. D+ U M0 W# }
mon: 3 daemons, quorum compute01,compute02,compute03 (age 102s)
3 c+ H$ [& n! S" ~ mgr: compute02(active, since 4d), standbys: compute03, compute018 G i( y) I: }9 O
osd: 6 osds: 6 up (since 29m), 6 in (since 2w)$ N7 Z- O6 M) k* G
& v9 f% D) I6 t. R [- c
data:
+ @# t, h. N% Z" I, K pools: 5 pools, 1408 pgs! d$ n( A8 b' a
objects: 32.65k objects, 132 GiB
- H# B- M7 b8 n( d* O& f usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail4 Y) Y, h/ L' Y) l
pgs: 1407 active+clean
! }/ D+ A. W: G; N7 b: |8 W" y 1 active+clean+scrubbing+deep
8 I3 b7 s* l' a1 y, _. F7 q- c& s1 I+ w4 x
io:
# D1 H( N& e( m& Z2 Y% I client: 110 KiB/s rd, 196 KiB/s wr, 12 op/s rd, 18 op/s wr
7 Z; H [0 Q- n! Y$ h$ }5 a; `: y; z0 P t
[root@compute01 ~]# ceph -s
( Y* b" S9 s, J. r0 z2 i9 x cluster:
- ~- _+ s6 h/ r; S- f f id: a6d0359e-8295-4e62-8620-54273462e79f) s) ^& _' Z. e1 f
health: HEALTH_OK7 D$ m1 M/ o+ N8 s' j% Z
* V6 ?% r7 k" s; B) Z services:0 p. M0 R& K7 R* ~4 {- b4 L
mon: 3 daemons, quorum compute01,compute02,compute03 (age 105s); P. I+ W# T' x' _8 g9 ~
mgr: compute02(active, since 4d), standbys: compute03, compute01$ O/ _4 H/ G2 V
osd: 6 osds: 6 up (since 29m), 6 in (since 2w)
6 G4 J, _: [( O) }. S
) T c& t' V. O# I$ D# b data:7 d+ b6 U8 J1 K9 b& A
pools: 5 pools, 1408 pgs
4 T2 ^! q2 ?- b; z( q1 o objects: 32.65k objects, 132 GiB
+ J+ F) K8 i: \4 ]( t$ q1 D8 f; `( G usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail1 b- J+ ~! l5 A; l H |, q. D
pgs: 1407 active+clean- [& d# C7 m2 o: b' F
1 active+clean+scrubbing+deep; I/ v. J0 @, C7 U, ]
" T H7 J* V# `3 C+ n. f/ E/ k io:
" s5 ?# e' o! j8 h6 e2 } client: 74 KiB/s rd, 193 KiB/s wr, 10 op/s rd, 18 op/s wr( |$ l4 j7 c) J( g+ `- q
+ B# t6 y" Y2 t7 {, E6 h3 g% q8 q& p[root@compute01 ~]# ceph -s' K8 C0 L! C5 p$ V
cluster:
) ^4 K( `; ]% x1 ?) y: x id: a6d0359e-8295-4e62-8620-54273462e79f
' f4 o- \" r. d4 z# R0 l health: HEALTH_OK
, |, P6 x9 `7 b' q$ M5 s5 K; E/ W2 g1 Q6 k! F. \ a4 y, l
services:
* z: n/ P9 S4 U- o1 D v% G' N mon: 3 daemons, quorum compute01,compute02,compute03 (age 3m)
, r0 }- h5 `5 V" `( ~5 } mgr: compute02(active, since 4d), standbys: compute03, compute01
7 i4 Z! q G) }* L/ B J/ d osd: 6 osds: 6 up (since 30m), 6 in (since 2w)1 G( _# ]# O1 Y+ S
! L j+ R5 z& b
data:
% b8 u. Q7 p4 }# H0 g pools: 5 pools, 1408 pgs, ]( _! M4 n. G! _; T
objects: 32.65k objects, 132 GiB
4 @( I$ G6 q. z/ P0 e6 O usage: 384 GiB used, 4.9 TiB / 5.2 TiB avail' ]0 T+ |7 b1 Q2 t
pgs: 1408 active+clean
' R3 U' P' e& A5 w
- X3 N, J7 M( |. a io:- r: t, S# n o u
client: 840 KiB/s rd, 1.4 MiB/s wr, 63 op/s rd, 43 op/s wr
% ?! \% O# k7 V0 {6 l6 I7 s3 h$ M3 Q, L: _( h4 L5 U( W
" q) }5 |, C2 h5 _7 U$ a% u
, y' v, M2 S- Z+ ?
- l* E* F0 N" T# i总结，此过程中出现的问题，一是时间同步服务器有问题，二是ceph 设置了不剔除osd，不同步，不分配问题。; o, [2 s9 O' @
: D/ }7 A# B, f4 f* p8 Q1 K% P! p2 `! V! D9 O
|
|
发表于 2021-12-20 11:00:13