ceph存储backfills值和recover速度设置

admin

ceph daemon osd.2 config show|grep osd_max_backfills
4 u2 b9 s0 }/ T4 [4 l, Z! i3 ^ceph daemon osd.2 config show|grep osd_recovery_max_active
5 D2 f* r$ M! x* c1 I
. J5 B: o8 C7 Y; N0 t# k) |ceph daemon osd.2 config show|grep osd_recovery_sleep
* F4 {1 u! f0 o# w$ T; v
5 }& l8 l  w. X# M" |$ h
. v( e3 f; G+ d. fceph tell osd.* injectargs "--osd_recovery_sleep_hdd=0"
ceph tell osd.* injectargs "--osd_max_backfills=10"
, |4 K0 G1 m: w7 d1 Sceph tell osd.* injectargs "--osd_recovery_max_active=15"

/ C( S3 U4 P- G8 X2 ]; x6 Y
同步速度就会上去
% _7 |) P& v; ~$ c- q

admin

官方建议的PG使用数量：
" f0 r$ [+ e; g: t9 j( M
集群中小于5个OSD，则设置PG的数量为128。
- R# d: ?7 P$ ^1 K4 H0 s* H集群有5-10个OSD时，设置PG的数量为512。
4 d. y7 e$ |; j! |3 e集群中有10-50个OSD时，设置PG的数量为1024.
6 V- b3 r- d2 E; |当集群中超过50个OSD时，需要权衡PG的数量，有一种公式：

! T+ w. }' S1 N% Z! C& f) H( M! P​ (osd_number * 100)

+ d" M' ~" G* ~pg_num = ------------------------------

​ pool size

: f7 D( ^0 A1 @) p例如集群有90个OSD，Pool设置的副本数为3：(90*100)÷3=3000
& j$ }3 h3 }" m! B/ u' b% Z
PGP的数量要和PG的数量保持一致。
  j& V7 d% I- i6 k( `+ ]
* H) ]% r( A# j9 }) |- {; `注意这个PG数量是集群所能承担的总PG数量，每个资源池分配多个PG，还需要通过总的PG数乘以资源池占整个集群数据比例，最终拿到一个资源池应该分配多少个PG的数量。

admin

控制数据恢复及回填速度
) |. Q; m5 |4 W; m1 P# q9 k# 将用于数据恢复操作的优先级提到最高级别
0 E, O6 Y% s/ r3 M5 c7 _- iceph tell osd.* injectargs "--osd_recovery_op_priority=63"
$ ]0 y. I/ d8 K- G
% n, q: A" w  O) t# 将用于client I/O操作的优先级降到3
) e: J3 _) Z$ F6 P7 C! hceph tell osd.* injectargs "--osd_client_op_priority=3"

# 将每个osd上用于回填并发操作数由默认的1调整到50
ceph tell osd.* injectargs "--osd_max_backfills=50"

& W- |& S+ i/ o3 Z3 r2 O# 将每个osd上用于恢复的并发操作数由默认的3调整到50
ceph tell osd.* injectargs "--osd_recovery_max_active=50"

  e5 Z# X) [* @1 |6 i# 将每个osd上用于执行恢复的线程数由默认的1调整到10
ceph tell osd.* injectargs "--osd_recovery_threads=10"

admin

[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_max_backfills
% }3 k7 M$ B8 n0 O  G+ Q! M5 t( b$ I    "osd_max_backfills": "1",
6 @2 z1 A: u& v  R+ p! [! `6 V[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_recovery_max_active
: M2 k1 S5 j  H6 Q: `7 f    "osd_recovery_max_active": "3",
! f7 o5 l$ e% h4 z[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_recovery_sleep
* A  x9 h. I$ X% R    "osd_recovery_sleep": "0.000000",
    "osd_recovery_sleep_hdd": "0.100000",
* C& E) d/ }, Y7 d; g    "osd_recovery_sleep_hybrid": "0.025000",
+ K/ ]( ^$ f3 E- ~5 E5 G1 ]    "osd_recovery_sleep_ssd": "0.000000",
[root@hostceph1 ~]# ceph daemon osd.2 config show|grep max_backfills
    "osd_max_backfills": "1",
[root@hostceph1 ~]# ceph daemon osd.2 config show|grep recovery_max_active
    "osd_recovery_max_active": "3",
[root@hostceph1 ~]# ceph daemon osd.2 config show|grep recovery_threads
- a6 G5 O3 A" e7 _0 T0 S[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_recovery_sleep_hdd=0"
- n) i) B- z9 n9 z/ Mosd.0: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.1: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
4 m% q- \. R. u  s0 kosd.2: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.3: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.4: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
: ~; H- Z; F- a# qosd.5: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.6: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
+ ?6 g% W2 I# C0 G8 u& p- l' Eosd.7: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.8: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
8 g2 u6 b; P4 a  t( J! F: f, \osd.9: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
. {" u) b/ M. ~9 q! R# }osd.10: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.11: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.12: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
3 Y. u6 e& S% u* f" n8 X& U4 L. B( sosd.13: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.14: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.15: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.16: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.17: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.18: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.19: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
) G, m) ]8 f) \. J# M" `8 ]osd.20: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.21: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.22: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
- f9 K1 c' D' b- h7 Posd.23: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
) c% n% ?* j. U0 E5 K0 Gosd.24: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
: z) n* @, ~- w2 Qosd.25: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
: w1 O& G+ y9 J; ^. p& Qosd.26: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.27: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
4 Q: `/ i! P% j5 v+ h% nosd.28: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
2 ^0 o) g. |1 v9 d. C4 eosd.29: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.30: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
8 v: c5 O$ S- ~" `. V$ D) nosd.31: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.32: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
  A7 H6 ^& S3 _" p; tosd.33: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_max_backfills=10"
osd.0: osd_max_backfills = '10'
( W' g5 z; t/ Vosd.1: osd_max_backfills = '10'
/ _% P  i( i! c3 `osd.2: osd_max_backfills = '10'
0 h5 N4 y+ Q' M9 v* W4 c# N* [5 C* k* uosd.3: osd_max_backfills = '10'
osd.4: osd_max_backfills = '10'
osd.5: osd_max_backfills = '10'
1 f+ O. q6 E4 Iosd.6: osd_max_backfills = '10'
osd.7: osd_max_backfills = '10'
8 J/ Z  o; {9 G. hosd.8: osd_max_backfills = '10'
9 ^$ I- E% @! Vosd.9: osd_max_backfills = '10'
osd.10: osd_max_backfills = '10'
osd.11: osd_max_backfills = '10'
$ N$ M8 e. f6 A  e7 Z) fosd.12: osd_max_backfills = '10'
' z5 ?- Z  q  C& ?' \8 z5 R# ]$ zosd.13: osd_max_backfills = '10'
osd.14: osd_max_backfills = '10'
osd.15: osd_max_backfills = '10'
# L# c* J4 c8 ?/ josd.16: osd_max_backfills = '10'
osd.17: osd_max_backfills = '10'
$ }/ C* N6 G+ u) l4 K* G0 wosd.18: osd_max_backfills = '10'
osd.19: osd_max_backfills = '10'
osd.20: osd_max_backfills = '10'
osd.21: osd_max_backfills = '10'
osd.22: osd_max_backfills = '10'
% A6 \* ]4 w4 @: T" q5 b5 Xosd.23: osd_max_backfills = '10'
osd.24: osd_max_backfills = '10'
osd.25: osd_max_backfills = '10'
osd.26: osd_max_backfills = '10'
* F7 L% F; z/ Zosd.27: osd_max_backfills = '10'
( L% g$ z' @! w2 Y+ ?- fosd.28: osd_max_backfills = '10'
6 \3 W8 ^) p! h# z' R3 [, Josd.29: osd_max_backfills = '10'
osd.30: osd_max_backfills = '10'
osd.31: osd_max_backfills = '10'
1 r% E! X% p* M+ dosd.32: osd_max_backfills = '10'
osd.33: osd_max_backfills = '10'
, Y/ c$ x; {* R) d[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_recovery_max_active=10"
osd.0: osd_recovery_max_active = '10' (not observed, change may require restart)
* ?5 H1 m! ^' k" [4 C  ~osd.1: osd_recovery_max_active = '10' (not observed, change may require restart)
, }2 {: V+ Q5 e" J9 R8 ]+ xosd.2: osd_recovery_max_active = '10' (not observed, change may require restart)
7 V- n# q& |7 m( o. aosd.3: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.4: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.5: osd_recovery_max_active = '10' (not observed, change may require restart)
  q3 `1 a) m$ @" r. sosd.6: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.7: osd_recovery_max_active = '10' (not observed, change may require restart)
' i" D# J/ u& ]. u$ Q+ Rosd.8: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.9: osd_recovery_max_active = '10' (not observed, change may require restart)
& e+ v; x, m* `9 S! {0 Mosd.10: osd_recovery_max_active = '10' (not observed, change may require restart)
6 ?, K' V5 F9 T" j$ ^osd.11: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.12: osd_recovery_max_active = '10' (not observed, change may require restart)
7 }( y; ^  h3 ^/ ^% Oosd.13: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.14: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.15: osd_recovery_max_active = '10' (not observed, change may require restart)
! d6 d5 V- c* m3 |  N' I7 q5 Vosd.16: osd_recovery_max_active = '10' (not observed, change may require restart)
( |7 p- e! Z& V2 uosd.17: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.18: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.19: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.20: osd_recovery_max_active = '10' (not observed, change may require restart)
# g2 A* @6 I! J3 dosd.21: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.22: osd_recovery_max_active = '10' (not observed, change may require restart)
. G) B- b$ H, M! U9 \osd.23: osd_recovery_max_active = '10' (not observed, change may require restart)
* E9 F# G; x( C8 _2 Y. k3 s  B2 _osd.24: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.25: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.26: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.27: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.28: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.29: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.30: osd_recovery_max_active = '10' (not observed, change may require restart)
5 P  U  s) o/ O' P4 B2 Eosd.31: osd_recovery_max_active = '10' (not observed, change may require restart)
2 }4 f3 k* `! L+ r  u' Q: r1 s; zosd.32: osd_recovery_max_active = '10' (not observed, change may require restart)
# H1 p0 j# O$ z2 X+ q: d4 `: m- ?osd.33: osd_recovery_max_active = '10' (not observed, change may require restart)
9 ]  f. @# Q% ]- F  R, s# \$ a[root@hostceph1 ~]# ceph -s
  cluster:
9 h7 f& @5 b$ b8 J9 A) _9 @1 r    id:     29046cc0-0682-496b-98b1-912e59964282
% a1 Q# [: W$ g, n: d  h& I7 b    health: HEALTH_WARN
            Reduced data availability: 24 pgs inactive
- Y! J: g/ g# A: q/ E# e2 t( L1 P            24 pgs not deep-scrubbed in time
            24 pgs not scrubbed in time

( T" ~3 n% a9 ?! _9 y6 z6 p2 e  services:
: I) c3 q  [7 E% f  P7 t* O    mon: 3 daemons, quorum hostceph1,hostceph2,hostceph3 (age 3M)
* n8 `/ g; F3 Y: N# r    mgr: hostceph2(active, since 3M), standbys: hostceph1, hostceph3
    osd: 34 osds: 34 up (since 3M), 34 in (since 3M)

% `7 ]( d, \- y* S$ Z! L8 A4 L; B  data:
7 A9 P- ^, q1 H  p+ e    pools:   9 pools, 5760 pgs
    objects: 3.11M objects, 12 TiB
9 m5 F5 q% i. D" W+ j: l3 \- J    usage:   33 TiB used, 92 TiB / 126 TiB avail
5 i- x+ H9 X$ n3 E    pgs:     0.417% pgs unknown
             5735 active+clean
             24   unknown
3 V6 I3 c4 h' z4 ?" e             1    active+clean+scrubbing+deep
" p4 T! M# y/ z: {( M3 D" ^9 U; Z
  io:
% o/ n* z1 U) S8 w5 t8 q7 @, t    client:   212 KiB/s rd, 20 MiB/s wr, 258 op/s rd, 647 op/s wr
) y7 J$ d1 t: A
# S- A5 V4 ]% j[root@hostceph1 ~]# ceph -s
  cluster:
    id:     29046cc0-0682-496b-98b1-912e59964282
    health: HEALTH_WARN
7 A5 X9 b+ G3 H            Reduced data availability: 24 pgs inactive
0 s# {2 |6 p' w  ~( y            24 pgs not deep-scrubbed in time
( B) ?. i. Q9 J+ E4 Q  p2 K1 t6 m            24 pgs not scrubbed in time
! k$ P+ H. \% l7 n
: f; ~# ~/ D! v$ b$ U2 p  services:
+ x& C, |3 ]! I1 A5 G, D8 T    mon: 3 daemons, quorum hostceph1,hostceph2,hostceph3 (age 3M)
    mgr: hostceph2(active, since 3M), standbys: hostceph1, hostceph3
    osd: 34 osds: 34 up (since 3M), 34 in (since 3M)
% |# `/ u. [+ S& x! T) @
  data:
! j; E( a8 U5 W/ p. f    pools:   9 pools, 5760 pgs
    objects: 3.11M objects, 12 TiB
. d. Y1 m. W; A5 }' c    usage:   33 TiB used, 92 TiB / 126 TiB avail
! I& k5 J+ |! v! }    pgs:     0.417% pgs unknown
# a& ?/ B7 u! F2 H$ o+ ]. C$ c             5735 active+clean
5 Z0 w3 a7 M4 B  c6 r. @) A             24   unknown
             1    active+clean+scrubbing+deep
. P! O# X4 \- l! |) k4 Y
  io:
4 R! i: w  z8 q6 e9 s% M    client:   9.7 KiB/s rd, 20 MiB/s wr, 12 op/s rd, 625 op/s wr