ceph fs status报错解决 AttributeError: ‘NoneType’ object has no attribute ‘get’

admin

问题描述：
[root@mon1 ~]# ceph fs status
/ Q( r, v% B0 w. f% FError EINVAL: Traceback (most recent call last):
File “/usr/lib64/ceph/mgr/status/module.py”, line 310, in handle_command
return self.handle_fs_status(cmd)
File “/usr/lib64/ceph/mgr/status/module.py”, line 176, in handle_fs_status
mds_versions[metadata.get(‘ceph_version’, “unknown”)].append(info[‘name’])
8 y4 M, i& {7 lAttributeError: ‘NoneType’ object has no attribute ‘get’
解决：
  I* E9 X& J7 F2 W6 T8 n2 ~报错的原因可能176行是获取的 metadata 为空；
1 \4 l& k1 Q" c! O[root@mon1 ~]# ceph mds metadata
/ U, E* [' U! j4 F" N- i- ?[
/ K4 U  ?( b, l- w  G* U{
“name”: “mon1”,
“addr”: “12.10.1.24:6800/583896531”,
“arch”: “x86_64”,
9 \$ K. S% v* f8 f“ceph_version”: “ceph version 12.2.5 (cad919881333ac92274171586c827e01f554a70a) luminous (stable)”,
“cpu”: “Intel® Xeon® CPU E5-2670 v2 @ 2.50GHz”,
0 a+ j8 R4 v3 g; [% p“distro”: “centos”,
  U$ M9 H" Q0 W6 ~“distro_description”: “Redhat Linux 7 (Core)”,
“distro_version”: “7”,
“hostname”: “mon1”,
3 K2 J% A( w' Y/ {5 t8 ]“kernel_description”: “#1 SMP Thu Nov 30 09:35:20 EST 2017”,
5 U8 a& }' X1 ~- k5 W“kernel_version”: “4.14.3-1.el7.elrepo.x86_64”,
( {; k, b7 [5 E“mem_swap_kb”: “20971516”,
“mem_total_kb”: “65923428”,
“os”: “Linux”
  T' q4 k* T2 r) i& o},
{
"name": "mon2"
},
% F, S" t% b2 N% G, j8 M3 ?{
“name”: “mon3”,
1 G- A- y3 y. F* C( ?7 o“addr”: “12.10.1.25:6800/3412998711”,
“arch”: “x86_64”,
“ceph_version”: “ceph version 12.2.5 (cad919881333ac92274171586c827e01f554a70a) luminous (stable)”,
! J- {% @' P' n: @“cpu”: “Intel® Xeon® CPU E5-2697 v4 @ 2.30GHz”,
8 E/ M: J1 `4 f) D7 ^“distro”: “centos”,
“distro_description”: “Redhat Linux 7 (Core)”,
“distro_version”: “7”,
“hostname”: “mon3”,
“kernel_description”: “#1 SMP Wed May 9 18:05:47 UTC 2018”,
“kernel_version”: “3.10.0-862.2.3.el7.x86_64”,
7 i/ s' x' v( Z7 m6 V) Q“mem_swap_kb”: “20479996”,
“mem_total_kb”: “197975904”,
* l9 J  I! r3 C" b6 k2 ^( n“os”: “Linux”
}
]

/ D9 g! h, K4 s查询发现有的mds只有name，
. j# |) Z; n. c- N) [2 L
这个时候只要把有问题的mds服务重启下即可
3 ?! e1 ^2 \( h- _/ N' [systemctl restart ceph-mds@mon2.service

，问题解决。


- J5 W- p5 x8 U: u0 {, k$ D4 R7 b

admin

[root@ceph-2 ~]# ceph mds metadata
[
    {
        "name": "ceph-3",
2 D6 d! a( L1 X5 C+ N        "addr": "192.168.120.31:6800/2291079756",
        "arch": "x86_64",
        "ceph_version": "ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)",
1 d  [- C; l% n; {        "cpu": "Intel Xeon E3-12xx v2 (Ivy Bridge, IBRS)",
        "distro": "rhel",
0 k' ]$ F$ h" J( ~* V. L        "distro_description": "Red Hat Enterprise Linux Server 7.9 (Maipo)",
3 Z4 w. y! w5 [0 U. O) s        "distro_version": "7.9",
        "hostname": "ceph-3.novalocal",
        "kernel_description": "#1 SMP Tue Aug 18 14:50:17 EDT 2020",
6 o0 \: m% ]1 N. o% ~0 w        "kernel_version": "3.10.0-1160.el7.x86_64",
        "mem_swap_kb": "0",
        "mem_total_kb": "3880160",
8 d8 E$ o1 c3 ?+ I% N. B* h        "os": "Linux"
    },
    {
        "name": "ceph-2",
        "addr": "192.168.120.16:6800/463439748",
4 c4 W1 r9 ~& z! Z        "arch": "x86_64",
        "ceph_version": "ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)",
        "cpu": "Intel Xeon E3-12xx v2 (Ivy Bridge, IBRS)",
        "distro": "rhel",
        "distro_description": "Red Hat Enterprise Linux Server 7.9 (Maipo)",
        "distro_version": "7.9",
        "hostname": "ceph-2.novalocal",
        "kernel_description": "#1 SMP Tue Aug 18 14:50:17 EDT 2020",
        "kernel_version": "3.10.0-1160.el7.x86_64",
+ Z$ l/ w5 {% K* c        "mem_swap_kb": "0",
        "mem_total_kb": "3880160",
        "os": "Linux"
    },
    {
7 V) `, x- S7 T( D0 s1 Q$ _! e8 T1 j        "name": "ceph-1",
        "addr": "192.168.120.23:6800/4181220800",
" O* L5 M# j6 M: ^        "arch": "x86_64",
        "ceph_version": "ceph version 12.2.8-52.el7 (3af3ca15b68572a357593c261f95038d02f46201) luminous (stable)",
        "cpu": "Intel Xeon E3-12xx v2 (Ivy Bridge, IBRS)",
$ U5 j! i3 [. E6 C" k        "distro": "rhel",
        "distro_description": "Red Hat Enterprise Linux Server 7.9 (Maipo)",
+ m. b! u% o* q- u# {; D$ f        "distro_version": "7.9",
& l: A! ^3 K( y# e7 F+ t        "hostname": "ceph-1.novalocal",
        "kernel_description": "#1 SMP Tue Aug 18 14:50:17 EDT 2020",
  \2 x0 }9 e5 x. E        "kernel_version": "3.10.0-1160.el7.x86_64",
! i; ?, j! x  {, e; e5 _        "mem_swap_kb": "0",
; ?/ B( }; c4 F        "mem_total_kb": "3880168",
; v1 u7 R6 d, Y7 [3 j9 u9 y" [        "os": "Linux"
3 Y& C& F  u% P$ ]1 z( K, p    }
]

admin

ceph集群提供元数据服务，则客户端可以提挂载ceph文件系统。客户端访问数据时，则在客户端和元数据服务器中都缓存相应的数据。元数据服务器会和客户端inode占用情况来消减缓存。当客户端响应太慢，则会报错“failing to respond to cache pressure” or MDS_HEALTH_CLIENT_RECALL。若确实是客户端负荷较大，是正常读写操作，可以考虑增大mds_recall_warning_decay_rate参数的值（默认为60s），从而消除警告。
# c+ [3 b) U- d$ |: R) u, G
可以查询ceph客户端的ID号及其使用inode数（num_caps的值）。

7 s1 v. S# A3 Z7 `0 s! I* }4 s5 gceph tell mds.0 session ls
谨慎使用如下命令踢出目标客户端或全部客户端。
! g- E1 S0 L5 T6 f1 [6 A
ceph tell mds.0 session evict id=11134635
3 l7 E/ W+ p. [% \* Gceph tell mds.0 session evict
! W: {- a+ {$ {踢出客户端是将客户端加入了黑名单，可以使用如下命令查看黑名单信息或移出黑名单。虽然移出黑名单，可能还不能让客户端正常挂载ceph文件系统，因此需要谨慎处理。

1 |5 r. c+ W; lceph osd blacklist ls
6 {4 _1 _; Y9 I* p# e! aceph osd blacklist rm 192.168.20.1:0/1498586492
ceph osd blacklist clear
: B  V4 h: K7 c9 _+ \6. Reduced data availability: 4 pgs inactive, 4 pgs incomplete
2 o% A1 }- L: d% ]. j+ S2 V" Y当有pgs出现incomplete时，表明pgs对应的OSDs存活数量少于最小副本数。因此，其对应的数据无法读写，处于reduced状态，会导致MDS服务出问题，提示如下报错信息，示例：

3 MDSs report slow metadata IOs
2 MDSs report slow requests
) i- U6 c2 f0 y- n* M# f3 [2 MDSs behind on trimming
+ I* Z# B5 I4 @, l4 \Reduced data availability: 4 pgs inactive, 4 pgs incomplete

- H& f/ v7 n- g; S) ~8 M) zpg 5.6de is incomplete, acting [254,356,222,352,111,247,100,133,351,206] (reducing pool cephfs_data min_size from 8 may help; search ceph.com/docs for 'incomplete')
pg 5.6e9 is incomplete, acting [276,244,357,358,221,321,311,229,314,351] (reducing pool cephfs_data min_size from 8 may help; search ceph.com/docs for 'incomplete')
pg 5.73b is incomplete, acting [186,279,351,247,293,354,359,220,181,283] (reducing pool cephfs_data min_size from 8 may help; search ceph.com/docs for 'incomplete')
4 `8 R2 j9 U' Z3 R3 _pg 5.eda is incomplete, acting [164,157,120,227,353,351,295,269,95,354] (reducing pool cephfs_data min_size from 8 may help; search ceph.com/docs for 'incomplete')
& X" n" W1 L' {5 J9 o此时，需要修复pgs。
" N# O  ~. x* @4 y* C! ^
- [5 Z* Z. k9 Y/ t9 G# 查询pg信息（pg id 为 5.6de）
3 y; q: z) e: E$ ~ceph pg 5.6de query

# 强行重建pg
9 S8 i4 ?/ F% ~7 Oceph osd force-create-pg 5.6de --yes-i-really-mean-it
  o; `/ j( r: ?9 \* d  {