ceph存储backfills值和recover速度设置

admin

ceph daemon osd.2 config show|grep osd_max_backfills
ceph daemon osd.2 config show|grep osd_recovery_max_active

ceph daemon osd.2 config show|grep osd_recovery_sleep
5 T0 W7 b8 ?, V% K$ @

ceph tell osd.* injectargs "--osd_recovery_sleep_hdd=0"
ceph tell osd.* injectargs "--osd_max_backfills=10"
ceph tell osd.* injectargs "--osd_recovery_max_active=15"
1 ^9 m6 @$ [7 k" l

4 _) e& C2 W  N' }+ l同步速度就会上去
8 d* j2 ~/ {4 {1 K

admin

官方建议的PG使用数量：

4 Q% p" T: i" c& h& d! ~集群中小于5个OSD，则设置PG的数量为128。
0 O6 P7 n* {  X6 C集群有5-10个OSD时，设置PG的数量为512。
$ P5 w" I& T' h+ |8 ^) z8 C集群中有10-50个OSD时，设置PG的数量为1024.
' }1 D  H0 R5 V' E6 n当集群中超过50个OSD时，需要权衡PG的数量，有一种公式：
  u- j3 I* ]/ {' a' @: t: n6 s
: v/ o1 u$ x" ]1 a* m6 ]; u" F& f' ~- v​ (osd_number * 100)

& P& c8 r  X$ G, s3 @" apg_num = ------------------------------

& n+ _7 u' g; T​ pool size
9 t) w# @0 ^0 E& x9 J
例如集群有90个OSD，Pool设置的副本数为3：(90*100)÷3=3000
! }/ F! c  k% G& `7 @
PGP的数量要和PG的数量保持一致。

& g, j' y0 U# }" L! D注意这个PG数量是集群所能承担的总PG数量，每个资源池分配多个PG，还需要通过总的PG数乘以资源池占整个集群数据比例，最终拿到一个资源池应该分配多少个PG的数量。

admin

控制数据恢复及回填速度
) j" q+ n, U2 F) f+ T/ `8 W( e# 将用于数据恢复操作的优先级提到最高级别
' R( d1 m$ K$ U' y* S  vceph tell osd.* injectargs "--osd_recovery_op_priority=63"

# 将用于client I/O操作的优先级降到3
ceph tell osd.* injectargs "--osd_client_op_priority=3"
6 f4 v" p7 A3 E) K
# 将每个osd上用于回填并发操作数由默认的1调整到50
; s4 I+ W1 b) P* f' ]0 jceph tell osd.* injectargs "--osd_max_backfills=50"
& |% a" u- Z  T% q% u
# 将每个osd上用于恢复的并发操作数由默认的3调整到50
ceph tell osd.* injectargs "--osd_recovery_max_active=50"

" P7 D0 x+ E) J. T$ V# 将每个osd上用于执行恢复的线程数由默认的1调整到10
ceph tell osd.* injectargs "--osd_recovery_threads=10"

admin

[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_max_backfills
    "osd_max_backfills": "1",
[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_recovery_max_active
    "osd_recovery_max_active": "3",
[root@hostceph1 ~]# ceph daemon osd.2 config show|grep osd_recovery_sleep
  p' P6 d4 U( `    "osd_recovery_sleep": "0.000000",
    "osd_recovery_sleep_hdd": "0.100000",
, @5 C: C& Z: P' ]    "osd_recovery_sleep_hybrid": "0.025000",
; A! t6 j% h+ b, C$ \    "osd_recovery_sleep_ssd": "0.000000",
4 O/ H- N5 Z7 f# {3 H[root@hostceph1 ~]# ceph daemon osd.2 config show|grep max_backfills
    "osd_max_backfills": "1",
# b) ?. o1 I' x3 Z, w0 A- R[root@hostceph1 ~]# ceph daemon osd.2 config show|grep recovery_max_active
    "osd_recovery_max_active": "3",
. }% {9 t" D4 U9 V7 F( u[root@hostceph1 ~]# ceph daemon osd.2 config show|grep recovery_threads
' |( p) d+ G2 ?7 x[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_recovery_sleep_hdd=0"
. O8 s; b3 I- t/ |osd.0: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.1: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.2: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.3: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
9 Z: J: Z8 g+ S/ `0 W  qosd.4: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
' ^* ~' v. w# {% r3 cosd.5: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
) M; ~8 l# {: W' ^osd.6: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.7: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.8: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.9: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
0 x4 I/ G/ C' w: R1 W7 j) @5 }' Kosd.10: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
+ |! x+ e$ X( z" qosd.11: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.12: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.13: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
: M$ A, P) Y+ E' B2 Uosd.14: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.15: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.16: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.17: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.18: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.19: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.20: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
& T8 p% s6 \; `. Q& v  D1 dosd.21: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.22: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
- F- l( A" h6 v0 e, F  E; kosd.23: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.24: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
% u2 C0 W  ^7 x7 x  ?3 R: u4 Zosd.25: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.26: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
, g7 v; x9 _% S/ Kosd.27: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
* X7 y7 o( P& U9 F. c: Yosd.28: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
7 y, \, _- y( V; R9 zosd.29: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
2 t0 U% I" e3 ~& a5 ~) @, rosd.30: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
' ^( {' t- S4 i! ]osd.31: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.32: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
osd.33: osd_recovery_sleep_hdd = '0.000000' (not observed, change may require restart)
! Z2 L& b0 F/ U+ l[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_max_backfills=10"
osd.0: osd_max_backfills = '10'
( c; C+ i/ z3 F6 Z- ~osd.1: osd_max_backfills = '10'
osd.2: osd_max_backfills = '10'
1 L4 ?) a8 I% p; _; d- q. p8 D6 g; Kosd.3: osd_max_backfills = '10'
' d/ i) T. F; A8 nosd.4: osd_max_backfills = '10'
osd.5: osd_max_backfills = '10'
osd.6: osd_max_backfills = '10'
osd.7: osd_max_backfills = '10'
osd.8: osd_max_backfills = '10'
osd.9: osd_max_backfills = '10'
- {8 |( n+ d; f  _5 Rosd.10: osd_max_backfills = '10'
osd.11: osd_max_backfills = '10'
) d: ?5 Z' A  rosd.12: osd_max_backfills = '10'
osd.13: osd_max_backfills = '10'
; I, S5 m, M) F6 w( Josd.14: osd_max_backfills = '10'
osd.15: osd_max_backfills = '10'
osd.16: osd_max_backfills = '10'
osd.17: osd_max_backfills = '10'
osd.18: osd_max_backfills = '10'
  D% y* \  ]$ z6 O$ P+ vosd.19: osd_max_backfills = '10'
8 ?& W, ~* A  B! g4 H* w+ M7 nosd.20: osd_max_backfills = '10'
osd.21: osd_max_backfills = '10'
osd.22: osd_max_backfills = '10'
: Z9 }! e& `" Q% P+ d! Vosd.23: osd_max_backfills = '10'
osd.24: osd_max_backfills = '10'
  ~- z* B$ A7 e. Posd.25: osd_max_backfills = '10'
osd.26: osd_max_backfills = '10'
! H( Z* o) z9 a( |" uosd.27: osd_max_backfills = '10'
0 \% t7 ^+ P( I# m0 e4 `1 Nosd.28: osd_max_backfills = '10'
osd.29: osd_max_backfills = '10'
8 y6 `, H. K# T( `osd.30: osd_max_backfills = '10'
$ ~4 G1 W- k; m8 _, i6 josd.31: osd_max_backfills = '10'
$ l! `" }) y2 j8 |. |osd.32: osd_max_backfills = '10'
& c! N- v# o& I" p% j6 Losd.33: osd_max_backfills = '10'
[root@hostceph1 ~]# ceph tell osd.* injectargs "--osd_recovery_max_active=10"
/ W% Q) A6 t& i' R, g! mosd.0: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.1: osd_recovery_max_active = '10' (not observed, change may require restart)
4 M4 Y5 G, k: G: \osd.2: osd_recovery_max_active = '10' (not observed, change may require restart)
& @! _! k& U* u( ~( h8 G; Gosd.3: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.4: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.5: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.6: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.7: osd_recovery_max_active = '10' (not observed, change may require restart)
8 N$ E) |8 I. E0 ~osd.8: osd_recovery_max_active = '10' (not observed, change may require restart)
7 \7 i4 X$ x# t  Z" ^osd.9: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.10: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.11: osd_recovery_max_active = '10' (not observed, change may require restart)
* V4 C9 E5 A" x0 Iosd.12: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.13: osd_recovery_max_active = '10' (not observed, change may require restart)
# u& j  ~$ k' V+ w' h6 Sosd.14: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.15: osd_recovery_max_active = '10' (not observed, change may require restart)
5 }3 W6 f$ R; s' Rosd.16: osd_recovery_max_active = '10' (not observed, change may require restart)
( A. N" K3 d( I/ a* zosd.17: osd_recovery_max_active = '10' (not observed, change may require restart)
3 ?9 Y! ~5 _! Z# s, wosd.18: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.19: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.20: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.21: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.22: osd_recovery_max_active = '10' (not observed, change may require restart)
* O' [7 C& `9 Vosd.23: osd_recovery_max_active = '10' (not observed, change may require restart)
% W8 q4 R5 \# B* _) losd.24: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.25: osd_recovery_max_active = '10' (not observed, change may require restart)
# Y- d$ S1 ?* x' x# d/ q) Tosd.26: osd_recovery_max_active = '10' (not observed, change may require restart)
8 p8 ^' c1 z! z! T' O0 L2 josd.27: osd_recovery_max_active = '10' (not observed, change may require restart)
5 |& r3 Z! t+ _0 w8 Rosd.28: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.29: osd_recovery_max_active = '10' (not observed, change may require restart)
% ~/ t1 @2 ]. E8 s/ Z! f+ C9 q6 }osd.30: osd_recovery_max_active = '10' (not observed, change may require restart)
osd.31: osd_recovery_max_active = '10' (not observed, change may require restart)
2 H+ U- m8 N: t2 [* _0 o5 }' K, Yosd.32: osd_recovery_max_active = '10' (not observed, change may require restart)
- O/ ?1 f$ Z6 M( R- Zosd.33: osd_recovery_max_active = '10' (not observed, change may require restart)
[root@hostceph1 ~]# ceph -s
6 p5 b/ y' ~# D" Y& M  X4 l  cluster:
' _4 A  f9 p1 O4 `    id:     29046cc0-0682-496b-98b1-912e59964282
    health: HEALTH_WARN
            Reduced data availability: 24 pgs inactive
+ c5 g1 Q' M5 p" i. o7 `; O            24 pgs not deep-scrubbed in time
            24 pgs not scrubbed in time
6 ~) P1 {% ?* V2 P$ e* \4 f) r
/ p( D* H' y/ E9 s: d6 [! c  services:
    mon: 3 daemons, quorum hostceph1,hostceph2,hostceph3 (age 3M)
    mgr: hostceph2(active, since 3M), standbys: hostceph1, hostceph3
    osd: 34 osds: 34 up (since 3M), 34 in (since 3M)
! h1 {3 z1 [7 j% W% ~" x. `! s
  data:
* }* L5 I9 {& b- h' \( Y2 c    pools:   9 pools, 5760 pgs
    objects: 3.11M objects, 12 TiB
    usage:   33 TiB used, 92 TiB / 126 TiB avail
( _7 [: ?) {, Q. Q# P    pgs:     0.417% pgs unknown
+ G: T9 b/ r" ~  a             5735 active+clean
' M1 N# F5 h  B  M/ f. j/ ]# G; |             24   unknown
* y- d* |5 d3 l  |# D& p* w' P             1    active+clean+scrubbing+deep
- F3 d9 L6 Y' f& d; ]
  io:
5 ^: D" {4 n( |$ R. W    client:   212 KiB/s rd, 20 MiB/s wr, 258 op/s rd, 647 op/s wr
+ T5 W8 A; k1 m6 x& y/ F
[root@hostceph1 ~]# ceph -s
  cluster:
0 W5 d" t) M6 T" v    id:     29046cc0-0682-496b-98b1-912e59964282
    health: HEALTH_WARN
6 A& ^6 N  e! W/ v( i6 [" I            Reduced data availability: 24 pgs inactive
7 T* W; u* a: v5 G            24 pgs not deep-scrubbed in time
            24 pgs not scrubbed in time

  services:
    mon: 3 daemons, quorum hostceph1,hostceph2,hostceph3 (age 3M)
    mgr: hostceph2(active, since 3M), standbys: hostceph1, hostceph3
: g6 q% `0 G( S. D% Z* \4 z    osd: 34 osds: 34 up (since 3M), 34 in (since 3M)
; a. R% n1 J" C- P+ ]* K
6 a7 s; l7 k7 o! p: _/ Q  data:
, }3 q7 W% q. S8 _3 A! ~2 a8 h7 b    pools:   9 pools, 5760 pgs
    objects: 3.11M objects, 12 TiB
    usage:   33 TiB used, 92 TiB / 126 TiB avail
    pgs:     0.417% pgs unknown
             5735 active+clean
             24   unknown
             1    active+clean+scrubbing+deep
4 M, v/ |; l7 p0 A  W% N* p0 u! ]
  io:
    client:   9.7 KiB/s rd, 20 MiB/s wr, 12 op/s rd, 625 op/s wr