- 积分
- 11892
在线时间 小时
最后登录1970-1-1
|

楼主 |
发表于 2022-8-23 09:53:54
|
显示全部楼层
index pool的 large omap 处理 x6 ?9 t' Y4 `% m4 p! u
向单个bucket压测2000W个object,默认设置shard数为16,压测到1800W出现large omap,介绍一下错误定位和如何处理。
1 ^; E0 k/ s' p5 A. h
' P D6 B3 x* o W/ B异常定位
, t/ I& o! S+ @9 ]+ N集群状态如下. `, Y F, w) A6 M
+ A, S- g3 _! O$ y* Q
[root@demo123 cephuser]# ceph health detail
# W. P/ ?- H% u* }# F0 ?: LHEALTH_WARN 16 large omap objects
$ l, s' O0 H. C$ C; \( B+ vLARGE_OMAP_OBJECTS 16 large omap objects" A& C5 f B7 [4 N/ {$ m7 s
16 large objects found in pool 'cn-bj-test2.rgw.buckets.index'# x. C8 R- u# B9 c- p5 F; s
Search the cluster log for 'Large omap object found' for more details.
y* Y& u) h$ P2 z7 n' O% G* ]复制! a* }$ D, M p# x4 Q
通过脚本找到对应的pg信息,脚本请查看之前一篇omap large处理的文章。
) X- O9 K0 X% f! h- M6 g- d# `
4 M1 G/ e# u/ J+ h[root@demo123 cephuser]# python large_omap.py
4 y. K! u! ^4 I9 PLarge omap objects poolname = cn-bj-test2.rgw.buckets.index! T9 V; a4 ~6 x: d
pgid=13.1f OSDs=[78, 9, 59] num_large_omap_objects=1
3 T+ d! B- t4 l* D" b# Ypgid=13.33 OSDs=[59, 79, 19] num_large_omap_objects=1& r0 F6 \. D7 S! V4 U; |. q U+ D5 ~
pgid=13.3c OSDs=[49, 29, 78] num_large_omap_objects=1
+ S% D6 b9 Q+ {pgid=13.3d OSDs=[48, 69, 9] num_large_omap_objects=1/ E* z) O# o, `
pgid=13.45 OSDs=[88, 39, 28] num_large_omap_objects=1- d8 @' Y! {' ? c' A
pgid=13.4d OSDs=[38, 29, 89] num_large_omap_objects=1
6 s! `, z$ g; k, k+ Lpgid=13.50 OSDs=[68, 19, 59] num_large_omap_objects=1
3 C" a+ W1 ^( j# n) w9 S* xpgid=13.6b OSDs=[39, 79, 8] num_large_omap_objects=14 f5 Q# {8 }9 r1 K3 g" i
pgid=13.8e OSDs=[38, 9, 78] num_large_omap_objects=1
; B2 {/ y( b+ J7 y, \4 A1 Hpgid=13.d1 OSDs=[9, 88, 38] num_large_omap_objects=1
) A8 v: }3 y! U7 m7 g6 c; L" lpgid=13.d2 OSDs=[59, 88, 28] num_large_omap_objects=1: E/ R7 j% B% i8 H2 w# C3 ^
pgid=13.e1 OSDs=[19, 88, 49] num_large_omap_objects=1
! X5 F& H' t( }6 d' U4 |% @pgid=13.e4 OSDs=[38, 19, 89] num_large_omap_objects=1! g% r, e* N5 N9 U V+ g
pgid=13.e7 OSDs=[19, 89, 38] num_large_omap_objects=1; I- R8 W# X( `. `7 W5 c( C' r
pgid=13.ec OSDs=[89, 28, 48] num_large_omap_objects=1& O6 c" w' S) C5 A, n: S
pgid=13.f5 OSDs=[38, 88, 19] num_large_omap_objects=1+ j4 P7 ]) _# l" ^; w$ V) ~& c
复制
% F b+ `! [ r, b+ I& O# B9 v: }查找OSD日志,确定object名称(".dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.11"),发现omap条目数达到了2378492,超过默认告警值
s/ u( \6 W6 e- R
, ^! i2 H7 K" W( F; S% T[root@demo123 cephuser]# zcat /var/log/ceph/ceph-osd.19.log-20181231.gz |grep "omap"4 @; U; a+ d$ y' ]
2018-12-30 23:00:42.334766 7f6583f44700 0 log_channel(cluster) log [WRN] : Large omap object found. Object: 13:87443b2d:::.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.11:head Key count: 2378492 Size (bytes): 491722758
( z- G/ v' q& i. V: N复制
! k6 ~9 _* b# ?0 p8 Z默认告警值为2000000,2378492>2000000,不建议去修改这个默认值,因为改得过大会加大集群出现异常的风险,属于掩耳盗铃。
* P6 e" x/ a4 I' @ Z% I* E
+ w F9 O" G+ Q3 j) L w% I L% e, O[root@demo123 cephuser]# ceph daemon /var/run/ceph/ceph-osd.19.asok config show |grep large. C! {* f1 A# d6 Q
"osd_bench_large_size_max_throughput": "104857600",) |" N. {/ X, u% F! f+ i9 N5 ~- N
"osd_deep_scrub_large_omap_object_key_threshold": "2000000",
0 Q% C n4 Z" ]) g' h0 v! O$ [1 f "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824",, h4 I k2 [' R; n& p
复制/ E! Q. |" O+ d/ Y$ h
查看一下发生omap过大的bucket,确定相关信息2 ]- @, C, x G
% s2 B; T. ^* @% Z" g) {3 |9 B+ ~
[root@demo123 cephuser]# radosgw-admin bucket stats --bucket=demo1
0 W" t1 |& G# ]4 ^; r2 k{" _* n q4 X7 f
"bucket": "demo1",
0 k( Q8 C* W S) a- ]$ V, A7 z- a "zonegroup": "68f1dcf5-0470-4a48-8cd2-51c837a2cafb",; z3 w6 l9 A# Y4 |: r% R5 \6 x9 q K% L
"placement_rule": "default-placement",5 Q; C5 B' O( A6 y8 r
"explicit_placement": {
3 u$ J1 d ]# R "data_pool": "",- t- Z/ ]8 A$ {2 k6 {9 p
"data_extra_pool": "",
& w' z# x7 I' ^ r: u "index_pool": ""
" h6 a J: _) r( {/ v/ b3 M },
0 W J$ I2 r) Z- ]' p "id": "afd874cd-f976-4007-a77c-be6fca298b71.34209.1", #当前bucket instance ID,
; A% ]0 q- |. P' K "marker": "afd874cd-f976-4007-a77c-be6fca298b71.34209.1",6 _- |6 t' ]6 Q) a0 Y
"index_type": "Normal",
- }7 c7 O0 l. R/ z5 ]* b "owner": "s3test",! m( R% z+ _' g; d
"ver": "0#2638037,1#2637965,2#2632835,3#2632869,4#2632799,5#2632597,6#2633289,7#2633175,8#2637227,9#2637609,10#2637997,11#2632455,12#2631337,13#2631624,14#2631983,15#2632359",: o# s8 T3 _. ]9 T$ K
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0,11#0,12#0,13#0,14#0,15#0", #16个shard" S' g1 B3 C1 e/ c
"mtime": "2018-11-28 16:47:45.560039",
- C7 T+ ~( Q* {2 M3 K "max_marker": "0#00002638036.2638608.5,1#00002637964.2638536.5,2#00002632834.2649479.5,3#00002632868.2633634.5,4#00002632798.2633370.5,5#00002632596.2633168.5,6#00002633288.2633860.5,7#00002633174.2633747.5,8#00002637226.2637798.5,9#00002637608.2638181.5,10#00002637996.2638569.5,11#00002632454.2633026.5,12#00002631336.2631914.5,13#00002631623.2632195.5,14#00002631982.2632554.5,15#00002632358.2632930.5",& b9 a. f! i+ J1 A' h h7 C
"usage": {5 I9 M. s. O8 A, N$ @3 D7 l
"rgw.main": {) `4 Q" Y& Q. r6 n( |- B
"size": 1975757355553,
2 Q. T, v3 ? ~6 C7 v5 O "size_actual": 2047893610496,2 W, a2 K- n/ W ~2 Q& b( c
"size_utilized": 1975757355553,0 Y, ^- X* k B. ]8 w" r, P
"size_kb": 1929450543,7 j( S8 f9 I; _. o; N3 q" w
"size_kb_actual": 1999896104,
2 J+ W2 d: ], W: a2 B "size_kb_utilized": 1929450543,
! c# J1 X( Q+ A' ?+ o W. C "num_objects": 19998962 #近2000Wobject' m0 p# O1 Q) L. @) z( z4 ~# _$ h
}$ [3 K U3 ^( L2 G! Z( l6 T
},
/ d6 g# Z2 p/ ~' c8 r3 G "bucket_quota": {* I( n1 B. _2 C/ H
"enabled": false,3 U+ C2 e( @# `( z
"check_on_raw": false,
# V, w2 p0 f& s. P" m$ H "max_size": -1,
6 H( [% t/ m! H9 a1 P "max_size_kb": 0,
0 K4 w; e# S# l! F, E1 u "max_objects": -15 y8 @# l: n6 W# N. R6 t0 J( o0 x
}
d. p1 }0 N. ?$ w- K1 N5 q}
* P4 V! k* b% d! W+ c复制
8 y) d. J: t0 A3 M异常处理
$ Z5 ~# `9 S& h通过bucket reshard操作,将原来的bucket 重新划分shard,shard数量从16->64。注意reshard有风险,最好停掉客户端的读写操作以后再进行,同时如果你使用了multisite,请根据官方说明立即关闭Dynamic resharding特性。! ?# x9 w. |7 T
" l; U* k1 ]5 C
Dynamic resharding 说明: http://docs.ceph.com/docs/mimic/radosgw/dynamicresharding/
o' h$ Y2 ?5 \% v9 L, ^; ~& C# {2 u4 e4 E3 I
做完reshard需要手工删除之前的索引数据,工具也提示了下面的内容。
7 r# Q6 ?* M5 t' \
x m( G. i3 }) u[root@demo123 cephuser]# radosgw-admin bucket reshard --bucket demo1 --num-shards 64
- ^0 e$ _; _5 ~: m" @# D*** NOTICE: operation will not remove old bucket index objects ***
, E" b% U! N! V+ K7 {0 z6 h*** these will need to be removed manually ***
1 h; R! A6 ~+ E% b+ d. _+ ctenant:) t8 z M) K( U; Y* |' B' t( K
bucket name: demo1# R V. C3 Z; e" b0 B* u
old bucket instance id: afd874cd-f976-4007-a77c-be6fca298b71.34209.1- ^! [! t: M) ]
new bucket instance id: afd874cd-f976-4007-a77c-be6fca298b71.45786.1
1 g0 z7 c& e: n" q3 ~& Btotal entries: 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000 11000 19998962
* z/ W% P( a5 {! k9 B% J* i) n& K2019-01-03 11:42:33.741314 7f74d15c6dc0 0 WARNING: RGWReshard::add failed to drop lock on demo1:afd874cd-f976-4007-a77c-be6fca298b71.34209.1 ret=-2
7 l2 M# H! `4 g/ g, j复制: ]% M& ^5 b: [+ D* Z8 L/ S, o
检查reshard结果
0 B ?6 j3 s- b9 O& F" z
% r" e% [' j) C# N2 p+ t5 [3 g[root@demo123 cephuser]# radosgw-admin bucket stats --bucket=demo1
6 r: d! g3 L. A3 q{
}7 \$ F' @9 Y+ E+ n' H# r4 S "bucket": "demo1",
$ b: N" r/ T5 E A# q "zonegroup": "68f1dcf5-0470-4a48-8cd2-51c837a2cafb",
1 d( [% a8 f5 B: _3 U* x6 U& L "placement_rule": "default-placement",. j0 y7 d' A2 i Y3 C$ i
"explicit_placement": {9 U, {" w7 c: u/ b# Q2 P
"data_pool": "",
3 g5 z- Q: E$ X) T' ? "data_extra_pool": "",
# w+ i: w5 ]. b0 V8 B "index_pool": ""$ T" }% k' ~. x& D/ P. V
},
9 Y0 y3 {5 \- k& ?0 s9 ?% I "id": "afd874cd-f976-4007-a77c-be6fca298b71.45786.1", #bucket instance ID发生变化& n% v+ A! F6 J3 c, M$ l, e
"marker": "afd874cd-f976-4007-a77c-be6fca298b71.34209.1",' z& I4 h* M3 ]& g
"index_type": "Normal",
- L( }- t. U& X) w3 G "owner": "s3test",
2 w6 g4 k W, K/ I$ y+ i% c "ver": "0#4920,1#4920,2#4883,3#4877,4#4882,5#4883,6#4885,7#4880,8#4882,9#4880,10#4878,11#4883,12#4923,13#4883,14#4882,15#4874,16#4878,17#4880,18#4884,19#4881,20#4882,21#4881,22#4876,23#4922,24#4883,25#4887,26#4881,27#4879,28#4879,29#4879,30#4882,31#4884,32#4880,33#4879,34#4917,35#4876,36#4883,37#4885,38#4884,39#4879,40#4883,41#4880,42#4880,43#4882,44#4884,45#4877,46#4879,47#4877,48#4881,49#4880,50#4881,51#4881,52#4883,53#4876,54#4880,55#4884,56#4881,57#4885,58#4882,59#4881,60#4881,61#4881,62#4883,63#4882",#shard 数量变为了649 n0 h7 F; v9 Q% \
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0,11#0,12#0,13#0,14#0,15#0,16#0,17#0,18#0,19#0,20#0,21#0,22#0,23#0,24#0,25#0,26#0,27#0,28#0,29#0,30#0,31#0,32#0,33#0,34#0,35#0,36#0,37#0,38#0,39#0,40#0,41#0,42#0,43#0,44#0,45#0,46#0,47#0,48#0,49#0,50#0,51#0,52#0,53#0,54#0,55#0,56#0,57#0,58#0,59#0,60#0,61#0,62#0,63#0",
6 z1 d6 I+ S/ G6 } "mtime": "2019-01-03 11:32:50.349905",- R8 H1 }9 X+ ?4 U: m4 V& P! A
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#,8#,9#,10#,11#,12#,13#,14#,15#,16#,17#,18#,19#,20#,21#,22#,23#,24#,25#,26#,27#,28#,29#,30#,31#,32#,33#,34#,35#,36#,37#,38#,39#,40#,41#,42#,43#,44#,45#,46#,47#,48#,49#,50#,51#,52#,53#,54#,55#,56#,57#,58#,59#,60#,61#,62#,63#",& U- j" |, N# w/ Y" S9 ~3 Z6 Q
"usage": {# {+ k$ t1 ?5 s3 o
"rgw.main": {
6 X T" B6 e2 t0 j2 Q! h4 ^+ t* ~ "size": 1975757355553,
. G: L# c+ d q "size_actual": 2047893610496,
" `3 a; x% \" k, ^" D3 l& x6 v' Y "size_utilized": 1975757355553,: J) z, S& H. _, V7 r( t0 v
"size_kb": 1929450543,# a; z" J0 p# l! Y3 o+ Y2 {$ @0 M( }
"size_kb_actual": 1999896104,
9 b& i* G* K+ p9 p3 e "size_kb_utilized": 1929450543,
& I# o/ ]% m) l$ y9 v E: a0 T "num_objects": 19998962
" h2 S, i' q. `& l. c, Z }7 h2 G0 e6 I# ^" Z" l
},
0 S+ N# |) D3 R; K L2 J "bucket_quota": {: _7 w, c7 t/ H+ k4 w3 ^) E: i R
"enabled": false,
' C) L" S2 s6 g* L4 K "check_on_raw": false,: s9 v$ U+ } Q& m @/ \ c0 i
"max_size": -1,
$ Z3 j G+ m& h2 |3 y+ e. | "max_size_kb": 0,* C2 Q) D5 w$ }) @ `7 X! |
"max_objects": -1
' H) ^8 Q. }4 e' m: t6 L }! ?2 t9 U% G1 }. l7 {/ _. ?( @* m+ W
}
- o# L. H9 k8 ]- H- ]复制
8 ?# |# X C' \' P! V回收旧数据
1 |/ _6 l0 ]- ~; c( v% y根据之前工具的提示需要回收index和meta两个pool里面的残留数据
8 b- o/ z9 x$ ?: A# g2 o% Q3 j
4 u8 P; q, G0 r$ @4 L9 Y回收index pool数据
5 `2 d# Z1 ]6 a$ H! q' o/ _: f# V! Y, ?7 H7 R, L0 K! D
[root@demo123 cephuser]# rados ls -p cn-bj-test2.rgw.buckets.index|grep "afd874cd-f976-4007-a77c-be6fca298b71.34209.1"
2 ~9 ~+ @" N5 n# j& o& F, E$ r.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.5 S4 r; _4 A0 r: b7 }
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.15
, E! h% c/ \- x.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.2- h B0 @/ j R# x* [
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.1, U. T G! P: @$ w# |6 a% a
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.05 u6 V0 x5 Q4 j% r/ f- _
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.48 o" x4 C q. R" M3 F! G( H
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.11 p# D: o: o' ]7 |
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.13
$ w# Z* l; L3 v1 f.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.6
# Y7 x1 S$ n. \6 R$ ?9 j( _/ @2 ?.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.3
! w9 B! i1 b5 b" f W/ T.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.7
% v4 L# F+ x9 |. `6 W4 ^1 s# o/ \.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.9- p& D, W0 \; j7 A8 ~% J: r. s. `* w
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.14
1 k+ J9 V2 K/ n. z% W.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.103 ^6 l3 h1 l! ]
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.12- e3 U( S1 y0 M+ d' H5 n, e
.dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.8
2 I/ C: ?7 J, v( v2 v- ?9 C# r复制' \3 S4 V; g6 D' J; R& {) g
使用rados rm命令删除数据/ ]# R( U' c# x3 q! ~2 B
- o2 X. X) O, q3 @: Y
[root@demo123 supdev]# rados ls -p cn-bj-test2.rgw.buckets.index|grep "afd874cd-f976-4007-a77c-be6fca298b71.34209.1"|awk '{print "rados rm -p cn-bj-test2.rgw.buckets.index "$1}'|sh -x
9 k4 S: `; h- l+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.5
, M7 [" T9 ?* h+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.15
. d3 ~& V( U3 b5 [6 h+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.2
7 a6 q1 c8 L: g1 R1 c! H+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.1
3 w8 P. Y* r: @" o9 |2 p+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.06 }4 w3 ~+ V8 C0 F1 J* p
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.44 ?: }; q ~1 K; p' [
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.11# N' ~! p n& ] t# v* p9 m1 Q& ^
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.13
5 c! t9 v. m! r* T4 Z+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.6
4 r0 ~$ {! n1 {: d, J/ ^. R4 y: |+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.3
6 ~5 N) S6 ]6 C# y! S$ l+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.7
# t4 e! c4 a3 u& a) A+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.9' x9 r( C9 b+ v9 T& ^% T' n, x. `7 G
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.14
, W7 P7 c" b' I5 N6 r- U) q9 O1 v, f+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.10/ T5 o5 y9 ^& A9 }, p6 D/ H
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.122 d# P! D# \4 J' P+ c6 f( ]
+ rados rm -p cn-bj-test2.rgw.buckets.index .dir.afd874cd-f976-4007-a77c-be6fca298b71.34209.1.8$ ~8 ^" k' v& i; i* w- n% ] c
复制 x; @& u7 z9 b: M- O
回收meta pool的数据
8 o4 H( n Q: v5 K7 a. k
p K% R/ p8 u. B/ p[root@demo123 cephuser]# rados ls -p cn-bj-test2.rgw.meta --all
+ P7 K6 {$ H. w# G( Hroot demo1
( d7 R* u6 G% T9 H: i$ Hroot .bucket.meta.demo1:afd874cd-f976-4007-a77c-be6fca298b71.45786.1& Z5 Y( ~, T$ T: x' p( d
root .bucket.meta.demo1:afd874cd-f976-4007-a77c-be6fca298b71.34209.1 #残留. K' y8 [0 x+ @ o: Y, B4 N
root my-new-container_segments
$ f) L( X# ~3 qroot .bucket.meta.demo2:afd874cd-f976-4007-a77c-be6fca298b71.34353.1
. n( O" ]8 E2 Y7 i) Eroot .bucket.meta.my-new-container:afd874cd-f976-4007-a77c-be6fca298b71.7991.1
9 O5 `, v2 S: i1 M- Q1 k8 q' u( uusers.uid s3test.buckets
1 E4 G p% S: z, j& Y* qusers.uid swiftuser7 ^6 w5 A% L) o
users.swift swiftuser:swiftuser1 N5 v A3 f7 Q* a* O7 Y" ?% d
users.keys SNACA4LX9DS21NGMSRX4* P' r* d% @1 ^9 o
root .bucket.meta.my-new-container_segments:afd874cd-f976-4007-a77c-be6fca298b71.7991.4, L' {( H! k! h: O+ M4 J, f/ @. W
users.uid s3test
, d# j- t4 H1 m" Zroot demo22 Z4 {/ P. q' V9 i( c2 r6 C
users.keys XP8E2452AB6EBU3RPD0C L( W" C0 F1 Q) x9 v
root my-new-container
3 {* K6 Q* l5 n- s* c9 E" Ausers.uid swiftuser.buckets
8 J) `0 ]3 f5 L$ N- @4 ^" yusers.uid synchronization-user' ~* C. x/ @- h. S! O3 x2 g
复制
. |8 b U/ N) t G注意这里用的ceph L版本,使用了namespace,所以要指定namespace才能删除
7 C; D7 L/ s8 t k2 ^
# N# n( _0 s' _& p6 l( S# ~6 r5 b- e[root@demo123 cephuser]# rados rm -p cn-bj-test2.rgw.meta .bucket.meta.demo1:afd874cd-f976-4007-a77c-be6fca298b71.34209.1 --namespace=root: L: f5 N B* G4 j& G! D
[root@demo123 cephuser]# rados ls -p cn-bj-test2.rgw.meta --all* T" a( D( s, K% \9 Y2 k
root demo1
! b* H0 ]$ a2 qroot .bucket.meta.demo1:afd874cd-f976-4007-a77c-be6fca298b71.45786.1
5 j7 V' V: K; T3 ^root my-new-container_segments- K3 M# n) i" y' z, R4 H8 x3 V7 @
root .bucket.meta.demo2:afd874cd-f976-4007-a77c-be6fca298b71.34353.1
3 w4 T1 s2 K) Q* W) _root .bucket.meta.my-new-container:afd874cd-f976-4007-a77c-be6fca298b71.7991.1
( F$ P+ Q" q+ ~) g! Q2 xusers.uid s3test.buckets$ c7 g0 i: M5 C4 b" D( k: A
users.uid swiftuser
: Q" m+ }7 N; S, O- d$ ^users.swift swiftuser:swiftuser1, \0 ^# B; I' S, N" L! V
users.keys SNACA4LX9DS21NGMSRX4- ?& w \" U s5 u; b, Z
root .bucket.meta.my-new-container_segments:afd874cd-f976-4007-a77c-be6fca298b71.7991.4
' R# K0 |& L! a- N' lusers.uid s3test+ F0 k6 d; Z o6 w( H/ f1 M
root demo2
$ x6 @9 @; ~) S# z7 N/ Dusers.keys XP8E2452AB6EBU3RPD0C
7 t/ H" X. y$ i4 @ ^# oroot my-new-container
, N y- o9 D4 T6 O$ @9 Q. X8 H8 ]users.uid swiftuser.buckets8 L9 F! n. u9 x3 e3 i
users.uid synchronization-user# Y% h& T5 O: t( r' a
复制; y+ B: G1 R. r1 C/ v
清除large omap告警
2 C- R, B x) e$ R删完了object并不会恢复告警,需要手工对相应的pg进行deep-scrub操作,具体如下) H* x) ], r" s6 R6 N
3 u6 M8 [' u' c' B g: z
[root@demo123 cephuser]# python large_omap.py
( g8 z% P7 H4 U1 @ KLarge omap objects poolname = cn-bj-test2.rgw.buckets.index
& s1 K' v# P( X- o* N! ppgid=13.33 OSDs=[59, 79, 19] num_large_omap_objects=1
% a3 c# B* V1 b! ^& fpgid=13.3c OSDs=[49, 29, 78] num_large_omap_objects=1
6 s6 I/ _1 F0 i1 Z& rpgid=13.3d OSDs=[48, 69, 9] num_large_omap_objects=1: {! Q" C- b( @3 H ^8 |9 w2 w
pgid=13.45 OSDs=[88, 39, 28] num_large_omap_objects=1- b% d" K9 N+ O1 o& N0 g
pgid=13.4d OSDs=[38, 29, 89] num_large_omap_objects=1
4 k3 @- k3 ^: ipgid=13.50 OSDs=[68, 19, 59] num_large_omap_objects=1 H/ J; y" p8 o. v* ~4 _# A
pgid=13.6b OSDs=[39, 79, 8] num_large_omap_objects=14 l' r0 F' k& T0 G/ B
pgid=13.8e OSDs=[38, 9, 78] num_large_omap_objects=1
* F* f% ~+ E6 t, [6 Ypgid=13.d1 OSDs=[9, 88, 38] num_large_omap_objects=1
; g% P- `; n8 b0 j9 _7 B: hpgid=13.d2 OSDs=[59, 88, 28] num_large_omap_objects=1 }# d! T2 Q9 _% k8 K
pgid=13.e1 OSDs=[19, 88, 49] num_large_omap_objects=1
# N! y. ^2 L2 X4 h; Y2 B" f$ h$ G, d4 kpgid=13.e4 OSDs=[38, 19, 89] num_large_omap_objects=1; L! m% [+ \; V# |4 D6 K/ u
pgid=13.e7 OSDs=[19, 89, 38] num_large_omap_objects=1
2 m/ u2 w! _" Z9 M2 gpgid=13.ec OSDs=[89, 28, 48] num_large_omap_objects=1) v' s) u5 o5 w) _% a* o& M
pgid=13.f5 OSDs=[38, 88, 19] num_large_omap_objects=1' s4 a( v- L, @ Z
[root@demo123 cephuser]# ceph pg deep-scrub 13.33
0 t/ e. T9 Q. l5 [% S2 s% w7 x' y) Jinstructing pg 13.33 on osd.59 to deep-scrub
3 O3 \0 V$ K3 k1 R! s1 _ R! y6 Y复制- @$ q2 U. f( I; e4 _4 F' c M
操作完可以看到有pg进行dep-scrub,之后状态恢复6 s2 l8 F) U& R, A5 T5 w. v; k c
q, N, M x9 L% `
[root@demo123 cephuser]# ceph -s4 k0 U' ?- Q! C5 R2 C: L
cluster:
6 s& T3 ^4 n3 ^/ O- ~0 o2 A& { id: 21cc0dcd-06f3-4d5d-82c2-dbd411ef0ed9
- U) }/ [5 }/ n6 P health: HEALTH_WARN, O7 D! o2 E4 A: X( c: o6 y: F
16 large omap objects
9 T8 |6 D; q0 c" n/ H2 k7 B L2 Y! h5 u' i( H
services:1 {9 r, {. s$ D- Y; J% S
mon: 3 daemons, quorum demo122,demo131,demo141% s- t9 D1 Q( m: N2 E
mgr: demo141(active)$ l# p9 \+ W: a& r5 q" m
osd: 90 osds: 90 up, 90 in" e/ I1 i* B0 q( l, s i
rgw: 1 daemon active5 P5 k7 ? u0 q6 F* ?2 b
0 o2 N% n4 K" o4 g
data: c+ ^+ o. n$ A. ~8 u
pools: 7 pools, 3712 pgs
3 C0 k D4 Y5 e9 l7 w+ B objects: 20.13M objects, 1.80TiB
4 \' D8 {9 H6 q" Z usage: 7.28TiB used, 408TiB / 415TiB avail; j5 d7 [" a9 w3 g6 b
pgs: 3711 active+clean
; Y& r/ B# M) t" i9 f+ t 1 active+clean+scrubbing+deep #开始deep scrub
" a& g# s: [2 {; i+ }) Z% F( [& ?$ _3 x9 x
io:
2 O2 ^, C" ]$ v' G* q7 _, y client: 5.29MiB/s rd, 935B/s wr, 69op/s rd, 28op/s wr; w+ q( ]. a. d4 G3 ?) @8 Y, R
/ g: r% w" u* _, M9 ?& r! h7 L[root@demo123 cephuser]# ceph -s1 ]0 l& _ ]- G0 L0 X
cluster:) f! E, |5 A0 w, r) u6 S
id: 21cc0dcd-06f3-4d5d-82c2-dbd411ef0ed9. @4 Y8 X- b+ A, H- |: v$ Q3 @& d
health: HEALTH_WARN8 k5 @0 J4 n% s5 z" T e. P! b
15 large omap objects #减少了1个
" h* l& K2 {9 |4 F% n% l
0 }1 i3 B Z: b8 M4 _9 @ services:, U* _7 B* A: l S: h
mon: 3 daemons, quorum demo122,demo131,demo141' k' N0 X& A: |
mgr: demo141(active)/ V& d' a [3 W1 V% I. {4 [* T
osd: 90 osds: 90 up, 90 in0 ~. u& a$ |6 o: [1 h; x
rgw: 1 daemon active" |- P3 n# ^+ \, S2 @8 d/ v
7 P. Z/ D, T6 [& ?. r
data:
* n/ o5 K: V- r/ O pools: 7 pools, 3712 pgs
8 Y' R8 Q: ?# s8 T* V6 u objects: 20.13M objects, 1.80TiB
1 ?4 e% N! i- U: l usage: 7.28TiB used, 408TiB / 415TiB avail
/ y* ?! s8 |2 Q& d8 ~ pgs: 3712 active+clean
/ g6 F1 W3 G: \9 n* t2 |
/ t0 p3 h5 j8 U; i2 f) \' L& ~ io:
: {3 c) F3 p g9 y3 ~8 p client: 5.33MiB/s rd, 680B/s wr, 36op/s rd, 6op/s wr; c* h3 P/ |+ F: c L- T1 O. B
复制
/ D3 G$ i2 x5 B4 c* S总结
. ^0 l% o/ a9 z9 K0 xindex pool的omap告警一般就分为两类:
+ h% Z7 s* J! o; x$ n4 b
; r% L5 P/ S& ?/ }5 o% b5 P- e8 W一类是object条目数过多,导致对应的index 元数据条目数过多,可以用上面的方法处理。
9 K _* s P( R! d5 n5 d另外一类是bilog过多,这里的方法就不适用了,需要手工进行bilog清理,关于bilog后续会有详细章节介绍。9 L2 Y' r% @4 [1 Y" i8 B/ `
' ]( q+ `" A2 l: A5 v
|
|