自动化kolla-ansible部署openstack+GPU透传方法

admin

1. CentOS7.x-8.x系列为虚拟机配置GPU直通
复制代码
( v) T0 \( W/ N" `0 N1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
( S+ k, a8 n8 Z- d$ {vfio
2 Y7 O% j5 Y* Z! @  C6 j9 rvfio_iommu_type1
vfio_pci
0 l+ Q; d' i: \9 Okvm
kvm_intel
* B5 d2 |+ G" Q( R. O
) l# t  G4 a% n7 p% P' n: A. f2. 在KVM主机上启用IOMMU
#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
: u* x; d  Z) C5 ?1 t) \#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"

vim /etc/default/grub

5 ~, J( \$ B1 N0 ^GRUB_TIMEOUT=5
, u" g: O1 r2 q1 K* y/ h2 v; |GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
" J8 y# I. ^- w2 Q$ }) r) |GRUB_DISABLE_SUBMENU=true
+ f0 o( P& w+ O, K/ w; ^; `GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet intel_iommu=on"
GRUB_DISABLE_RECOVERY="true"
7 i$ m8 g! p  I7 f2 Y) j2 C0 G8 r
   3.  重新生成grub
+ |1 C/ E  k: V' J   EFI
   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
1 e6 b0 {9 E6 {) t4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
. S3 t; W9 ]% ~2 [: cvim  /etc/modprobe.d/blacklist.conf
" V" Q5 s, ~/ S* r4 D5 |" Kblacklist snd_hda_intel
blacklist amd76x_edac
" L  q6 J+ r" F1 \9 [1 w; Oblacklist vga16fb
blacklist nouveau
blacklist rivafb
8 }9 `8 u+ M3 i7 S& wblacklist nvidiafb
blacklist rivatv
9 ]1 |: k, r8 m2 O  Y& L6 fblacklist nvidia
/ \$ l4 C: H0 ^% C1 g
! P. k1 Q6 f  ?! C5.  查找显卡的Product ID 以及 Vendor ID：
) `6 {0 N) D8 r9 C  Kyum install pciutils -y
lspci -nn | grep NVIDIA
+ N+ N7 T* p8 d3 o如下：
! D6 ~" [1 ~5 ]. k: [[root@stein-a ~]#
5 B% z$ t6 a: d, |03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
- }2 S/ h. B: V) T5 V4 D03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

6.  编辑
vim /etc/modprobe.d/vfio.conf
# create new: for [ids=***], specify [vendor-ID:device-ID]
options vfio-pci ids=10de:1bb1,10de:10f0
  J5 A# }; S6 p8 m
' D3 S: T7 z. w' ?7.  写入到系统启动项
echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf

8.  重新生成initramfs
mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
% Q3 s: \. A! i) X- K+ ldracut -v /boot/initramfs-$(uname -r).img $(uname -r)

9.  重启系统
reboot

10. 验证
+ M1 D2 D* a; v6 U' m% elspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
[root@stein-a ~]# lspci -nnk -d 10de:1bb1
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
' s8 t! q( s# |/ `! I: o        Subsystem: NVIDIA Corporation Device [10de:11a3]
# X& r0 ^1 v9 T, L+ F( X# y        Kernel driver in use: vfio-pci
        Kernel modules: nouveau
0 Z: n: P. ^' W[root@stein-a ~]# dmesg | grep -i vfio
[    2.503115] VFIO - User Level meta-driver version: 0.3
[    2.515645] vfio_pci: add [10de:1bb1[ffff:ffff]] class 0x000000/00000000
) g; Q$ ?3 j, `& C. Q) |5 h+ o/ c[    2.515752] vfio_pci: add [10de:10f0[ffff:ffff]] class 0x000000/00000000
; B5 G9 I' r! j! Q4 `$ A[root@stein-a ~]#
复制代码
' ^7 _) _5 J& H  s: a7 x( D& ^
* r9 R; G5 U, ~+ g2. Ubuntu18.04系列为虚拟机配置GPU直通
复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
vfio
& n/ B! f9 \* y7 j# ovfio_iommu_type1
9 R; i8 k2 e/ y: wvfio_pci
- B  Z  f( U0 _; a, ?" ekvm
' a) V' x& r  K. a( |! Ukvm_intel
: t5 D# O, J; ]4 q  d) I$ N% B
3 w* D$ ^; r4 n7 m8 X2. 在KVM主机上启用IOMMU
3 {  }* Y; c1 v% L#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
" u* t* _3 U$ c#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"
1 [1 j7 {$ @5 I0 ?
vim /etc/default/grub
+ Q' o. k' e3 J3 k  l
GRUB_DEFAULT=0
6 c) T  \5 O+ A. q& }9 RGRUB_TIMEOUT_STYLE=hidden
$ z: H0 |, p6 h$ vGRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
GRUB_CMDLINE_LINUX=""

   3.  重新生成grub
   EFI
$ N. O2 ~" d6 Q) s- y: I   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
# B9 P& |2 R; _4 S  q* J   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
3 o9 c% p8 T$ ]/ f8 [1 F8 Ivim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
blacklist amd76x_edac
blacklist vga16fb
blacklist nouveau
blacklist rivafb
blacklist nvidiafb
blacklist rivatv
blacklist nvidia
$ p4 Y% E; T* h; w/ B5 C& Y8 W
. f+ @  I4 b) D, j3 l( y2 E. j5.  查找显卡的Product ID 以及 Vendor ID：
apt install pciutils -y
- b& x9 K4 p2 }lspci -nn | grep NVIDIA
$ C' e! k1 J6 f1 @如下：
; [, \6 p/ C# y: h# [) u, M[root@stein-a ~]# lspci -nn | grep NVIDIA
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

6 i: a& O9 ]# e/ C6.  编辑
! ^6 d7 v- X' E1 v0 P. Nvim /etc/modprobe.d/vfio.conf
( E6 H# s- y/ M6 Y( g, E" [* Y# create new: for [ids=***], specify [vendor-ID:device-ID]
options vfio-pci ids=10de:1bb1,10de:10f0
( p7 U& y0 r0 k& W+ P9 c/ |* |
7.  写入到系统启动项
7 A+ }: X- C) ]: }* K4 Gecho 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
  N- r# H9 i, g' P. i" O3 H
8.  重新生成initramfs
- j  p9 j- @2 u4 T1 gdracut -v /boot/initramfs-$(uname -r).img $(uname -r)

9.  重启系统
reboot
' R, P4 A+ v4 J. w  |6 }
10. 验证
lspci -nnk -d 10de:1bb1
8 I0 L) f9 h; D( j; R' cdmesg | grep -i vfio
root@kvm:~# lspci -nnk -d 10de:1bb1
, x: M) E/ ?: U7 C  H) d0 Vdmesg | grep -i vfio
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
6 \& E% M- o% [% o7 |/ R0 Y        Subsystem: NVIDIA Corporation GP104GL [Quadro P4000] [10de:11a3]
        Kernel driver in use: vfio-pci
        Kernel modules: nvidiafb, nouveau
- G" J7 D1 a" o8 Z7 a- Zroot@kvm:~# dmesg | grep -i vfio
[    3.838714] VFIO - User Level meta-driver version: 0.3
[    3.846238] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
[    3.866370] vfio_pci: add [10de:1bb1[ffffffff:ffffffff]] class 0x000000/00000000
4 r) z. u3 J( D# {7 ]' f0 ?3 `[    3.886375] vfio_pci: add [10de:10f0[ffffffff:ffffffff]] class 0x000000/00000000
复制代码
7 M1 j8 J/ v! t7 J( p9 F$ A
+ J7 g' Z8 w# K9 Q  I6 Y复制代码
. X' [( p: e4 H5 A! M#如果你单机部署的，在单机下配置。
: G' i4 }8 t; j! \. Z* d# g$ H: w#如果你是高可用部署的，在三台控制节点配置
2 x2 _0 `2 S  U- K! c5 N1. 添加pci
vim /etc/kolla/config/nova/nova-compute.conf
) M  Y, P0 j1 B2 b2 l9 l0 s6 r5 G[libvirt]
inject_password=true
  i6 H1 d* a+ y! O$ tcpu_mode=host-passthrough
virt_type = kvm
[pci]
passthrough_whitelist: { "vendor_id": "10de", "product_id": "1bb1" }
; Y$ C! q  e3 S5 y7 M) k2 v
& \  i" _5 D- x3 N2. 修改nova.conf
vim /etc/kolla/config/nova.conf
[DEFAULT]
! R7 Z. ]. Q* A7 q! z8 vservice_down_time = 120
+ y+ z$ b/ J, N# I. G4 P- Ocpu_allocation_ratio = 4.0   
disk_allocation_ratio=1.0
& Q; k9 {: w( i7 Tram_allocation_ratio = 1.0   
- I+ Z: r% o$ ?7 R1 \# q  vreserved_host_disk_mb = 4096
reserved_host_memory_mb = 4096
allow_resize_to_same_host = True   
" }+ f0 B& k, \& S! j1 V( Cremove_unused_base_images = False
/ u3 g- J( U) d. c  Dimage_cache_manager_interval = 0
( R+ m1 D& z/ J" Gresume_guests_state_on_host_boot = True

* F! k2 r5 h( p, n( m6 l' N[PCI]
alias: { "vendor_id":"10de", "product_id":"1bb1", "device_type":"type-PCI", "name":"quadro-p4000" }
6 O4 S  s, D# q( {9 W[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
available_filters = nova.scheduler.filters.all_filters
" _" \( B& H! _
3. GPU 类型实例创建  
7 c: w# B5 h, I. v. popenstack flavor create --vcpus 4 --ram 8192 --disk 30 --property "pci_passthrough:alias"="quadro-p400:1" g1.4c.8m.p400
9 X: [4 c0 n  T* `复制代码
3 N4 ]# @1 p/ F' _) J- K
9 X/ x" r' u# ~3. CentOS7.x系列 安装显卡驱动
, a5 x; _% ~* n: {5 K* E3 X复制代码
1.  查看是否含有英伟达显卡
* ^9 d. q; x% S# g9 ]& Qlspci | grep -i NVIDIA
8 v$ e; \& O- z$ D. `#下面说明有1块英伟达的显卡
$ l* n0 P! }8 u* ~7 a" M9 ?[root@train-all ~]#  lspci | grep -i NVIDIA
& M9 i6 @0 w1 _04:00.0 VGA compatible controller: NVIDIA Corporation GP104GL [Quadro P4000] (rev a1)
04:00.1 Audio device: NVIDIA Corporation GP104 High Definition Audio Controller (rev a1)
/ o  {0 F1 o- A[root@train-all ~]#
( e. B  X: x+ D" W3 y
; a( r5 B- r) o2 h; @) i: Y. X2.  添加ELRepo源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
) y! j8 l- g: Y. ^  I1 F5 K
* _, l: u- W! r9 X3.  安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm

4.  安装nvidia-detect
% N! j0 F, Z* e, c& uyum install nvidia-detect -y

5.  运行nvidia-detect
nvidia-detect -v

$ }" h' w$ d! D2 D* J7 {% |: j" K6.  查找驱动程序
# f" _$ ~! s$ g/ r/ ~/ L" byum search kmod-nvidia

7.  安装驱动程序
/ O1 {7 @8 o# F  ^9 W+ F1 Gyum install kmod-nvidia.x86_64 -y

8.  查看禁用Nouveau
4 n6 J! x" f; K" v: @0 P+ [! vlsmod | grep nouveau
#若没有输出 则说明禁用成功，否则执行下面的命令

9.  在/etc/modprobe.d/blacklist-nouveau.conf中创建一个文件，其内容如下：
vi /etc/modprobe.d/blacklist-nouveau.conf
. [- @. y6 f: E. Q0 P) t+ P/ H1 H( u添加
blacklist nouveau
options nouveau modeset=0

10. 重新生成内核initramfs
0 h" h# f# ?- u5 o0 n" ?" Adracut --force
9 Q1 n, y+ q3 i& o" F& O- E6 V/ ?
, I* |3 i0 e0 U! M5 I: ~* y9 o11.  重启系统
& O% b) @# v2 ^. z5 p2 zreboot
* t$ i- V4 C  V
7 a' D6 H' ]% R12.  测试
$ b8 }& _8 a: Z& J" W9 l' K" \nvidia-smi
, E' ?$ ^% w, M3 z; j& ^& M2 w