自动化kolla-ansible部署openstack+GPU透传方法

admin

1. CentOS7.x-8.x系列为虚拟机配置GPU直通
# I* |! z0 C) E0 C3 K: @复制代码
! u5 n+ x6 C! i  x3 b1. 编辑文件vim  /etc/modules， 添加以下内容：
pci_stub
vfio
$ ]' ]# h! D" }: ~3 Hvfio_iommu_type1
vfio_pci
kvm
, e# d( Z& Z4 [! ?0 Ikvm_intel
  l$ K" U, t% h* d3 X2 u8 u* t9 y
% y& g# d7 _8 J# g$ k2. 在KVM主机上启用IOMMU
; x: l+ ~" C- c: v#对于Intel芯片：
: t# H! ?& ?8 B! gGRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
$ ?. [- J. Y. t, O, N#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"

vim /etc/default/grub

/ ?4 d5 @0 @% M- n& }: _/ N9 ~GRUB_TIMEOUT=5
* n6 A1 y0 e4 Z% `9 K+ v+ j3 _$ sGRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
+ i+ r" D$ @: O( m0 k/ d5 J# _GRUB_DISABLE_SUBMENU=true
) ^! e2 ~2 [1 W9 V; [5 c' fGRUB_TERMINAL_OUTPUT="console"
2 O7 h) a- _$ ~* c0 QGRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet intel_iommu=on"
6 t) r, b  u1 G. a# c: QGRUB_DISABLE_RECOVERY="true"
4 v5 f1 v7 t$ l* I# [& b
; Z, o: E0 n3 J/ |$ B   3.  重新生成grub
   EFI
   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
' T- ~0 K* M) O4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
blacklist amd76x_edac
+ ~+ ]  w. j3 }5 s; Y+ c  @blacklist vga16fb
blacklist nouveau
+ ]9 |% g9 O: p3 Z% Q3 w$ z8 Y2 h$ g! bblacklist rivafb
4 |# }2 Z, N2 X2 p# |# Iblacklist nvidiafb
blacklist rivatv
blacklist nvidia
; l4 ]" y) Z0 Q3 E
5.  查找显卡的Product ID 以及 Vendor ID：
yum install pciutils -y
0 l2 k& I. q, D9 }lspci -nn | grep NVIDIA
如下：
1 z4 R; m' j  \  r/ a( g[root@stein-a ~]#
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)
, Z5 m4 |/ ^5 r  j9 U, E- j
6.  编辑
3 h. L7 x# p3 [. Dvim /etc/modprobe.d/vfio.conf
# create new: for [ids=***], specify [vendor-ID:device-ID]
, c- L" J# ?# B- m4 z* Q5 p5 uoptions vfio-pci ids=10de:1bb1,10de:10f0

7.  写入到系统启动项
) w5 L( n2 x9 ^$ r! q. Zecho 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf

9 @  D; l2 E- U- C$ H" Q8.  重新生成initramfs
) M, h. q* m4 G9 M7 a/ \3 omv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
dracut -v /boot/initramfs-$(uname -r).img $(uname -r)

9.  重启系统
reboot

10. 验证
& s! c$ W2 u3 @9 y3 mlspci -nnk -d 10de:1bb1
' E2 ~/ y: w  {: G- J/ T! ?dmesg | grep -i vfio
2 \) E. l9 c  Y& T5 o& i* f[root@stein-a ~]# lspci -nnk -d 10de:1bb1
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation Device [10de:11a3]
( f( Y3 l: n. b        Kernel driver in use: vfio-pci
4 {0 z& k1 @7 j% \" |1 U        Kernel modules: nouveau
& N% N& m( G3 [  C4 Q2 W[root@stein-a ~]# dmesg | grep -i vfio
& n4 |$ t8 L; S. |% d+ q1 X% X[    2.503115] VFIO - User Level meta-driver version: 0.3
" t' L( I7 H. D8 D$ r[    2.515645] vfio_pci: add [10de:1bb1[ffff:ffff]] class 0x000000/00000000
[    2.515752] vfio_pci: add [10de:10f0[ffff:ffff]] class 0x000000/00000000
" Q! K" g  l- U5 C$ a8 ^; E[root@stein-a ~]#
6 ]1 U$ E" ?* \# \' u; I- d( P复制代码
6 a% Q% e$ w8 k- r9 b% v
% G; @8 m, G0 \! J0 x( z! j8 q2. Ubuntu18.04系列为虚拟机配置GPU直通
复制代码
  H# W8 h% R: J0 j1 ?1. 编辑文件vim  /etc/modules， 添加以下内容：
6 L+ u8 R0 z- i; |( Bpci_stub
vfio
% j) a3 p' c# b& `+ O0 ?0 dvfio_iommu_type1
vfio_pci
kvm
kvm_intel
/ S! C  k  p4 E& K* g
5 i% P0 i8 P' @1 {% E/ _2. 在KVM主机上启用IOMMU
4 d6 t) w, D* H' \& i8 P; m#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
# m7 s* h3 F' P) a$ H#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"

0 U! @/ J) w( yvim /etc/default/grub
, |: v9 i6 D5 C$ m1 d0 ^
( @" `+ V& s, f/ ?% ?3 VGRUB_DEFAULT=0
* \. o$ |7 G0 x8 T2 qGRUB_TIMEOUT_STYLE=hidden
+ g: s* }" P+ \' U( j, t' c  aGRUB_TIMEOUT=0
3 ?+ j' {" W0 j+ H/ BGRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
& {! ?: ^8 O, L8 d) d& U" oGRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
* @) v/ j" |2 b! H: g8 kGRUB_CMDLINE_LINUX=""
( O3 h. n' G6 K( ~- }$ Z
   3.  重新生成grub
   EFI
6 i! M! [- @: y( `6 k% s   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
* |9 S8 j" C% ]5 t   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
1 u) B9 }& D+ ]/ j3 q' I' J8 q3 d5 F4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
blacklist snd_hda_intel
5 C+ j& i: h1 g' z) ~( m( rblacklist amd76x_edac
& t# Q; t0 ]3 B( h3 x& ~; Dblacklist vga16fb
blacklist nouveau
2 K5 l( [/ `6 u* r: a4 Wblacklist rivafb
' D3 N% d0 C; f  y9 mblacklist nvidiafb
blacklist rivatv
blacklist nvidia

5.  查找显卡的Product ID 以及 Vendor ID：
' `$ Q) p% D& j2 G, kapt install pciutils -y
lspci -nn | grep NVIDIA
如下：
[root@stein-a ~]# lspci -nn | grep NVIDIA
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

6.  编辑
vim /etc/modprobe.d/vfio.conf
# create new: for [ids=***], specify [vendor-ID:device-ID]
options vfio-pci ids=10de:1bb1,10de:10f0

3 k! c4 b+ K3 }7.  写入到系统启动项
echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf
6 H$ J- c" j/ T7 N
8.  重新生成initramfs
9 J8 j, d& c9 M; J5 {$ Gdracut -v /boot/initramfs-$(uname -r).img $(uname -r)
  u6 `( E! y7 l+ \
9.  重启系统
# C( h% ^% Q% l1 y, D# p- O! ureboot

! Y" x- E. O4 @5 u10. 验证
lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
root@kvm:~# lspci -nnk -d 10de:1bb1
5 c7 O/ Z" L& wdmesg | grep -i vfio
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation GP104GL [Quadro P4000] [10de:11a3]
        Kernel driver in use: vfio-pci
        Kernel modules: nvidiafb, nouveau
root@kvm:~# dmesg | grep -i vfio
  @7 H$ [( c) ^! ~[    3.838714] VFIO - User Level meta-driver version: 0.3
[    3.846238] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
[    3.866370] vfio_pci: add [10de:1bb1[ffffffff:ffffffff]] class 0x000000/00000000
7 |9 H& ^' D& M8 N2 }8 g/ x6 ~5 z: ?[    3.886375] vfio_pci: add [10de:10f0[ffffffff:ffffffff]] class 0x000000/00000000
, D+ `& ~! E' w; N+ `复制代码

# x& ^' |0 W  D! h) [5 e复制代码
#如果你单机部署的，在单机下配置。
#如果你是高可用部署的，在三台控制节点配置
$ E  I  [0 V0 x0 W3 |1. 添加pci
) ?  u" j$ ~; r3 X  v2 [2 Cvim /etc/kolla/config/nova/nova-compute.conf
[libvirt]
inject_password=true
cpu_mode=host-passthrough
! ^4 \6 i8 [( l/ avirt_type = kvm
[pci]
passthrough_whitelist: { "vendor_id": "10de", "product_id": "1bb1" }

2. 修改nova.conf
vim /etc/kolla/config/nova.conf
$ e6 d+ T" o7 Y5 U( G: @; j6 y, o[DEFAULT]
service_down_time = 120
( B, A3 n8 R8 `5 i  O$ D8 E9 ccpu_allocation_ratio = 4.0   
disk_allocation_ratio=1.0
/ A, y, T& b6 {* A% eram_allocation_ratio = 1.0   
reserved_host_disk_mb = 4096
+ f5 |6 h3 ~3 u8 T$ w5 {! hreserved_host_memory_mb = 4096
allow_resize_to_same_host = True   
! I  R  X8 L: ^+ d- T) gremove_unused_base_images = False
image_cache_manager_interval = 0
# k5 `8 Q: ^% Wresume_guests_state_on_host_boot = True

! F  D& {4 d' u! g0 r1 G5 b- w' i[PCI]
) K8 ~5 ]& e  U3 K5 @4 h' S2 H: ialias: { "vendor_id":"10de", "product_id":"1bb1", "device_type":"type-PCI", "name":"quadro-p4000" }
[filter_scheduler]
4 K/ f) R1 r8 x1 Uenabled_filters = RetryFilter, AvailabilityZoneFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
9 y- n( `; E$ V) q, R$ R* y; Vavailable_filters = nova.scheduler.filters.all_filters

3. GPU 类型实例创建  
# ~" l. _  |& _" Topenstack flavor create --vcpus 4 --ram 8192 --disk 30 --property "pci_passthrough:alias"="quadro-p400:1" g1.4c.8m.p400
复制代码
$ j+ M& \' P4 |; V  `- {
+ y1 u3 z5 L& _" p$ a2 s8 h) Q3. CentOS7.x系列 安装显卡驱动
) c0 @' i9 X7 o- {8 @+ T  k复制代码
. b' |- [  {* d% y+ v1.  查看是否含有英伟达显卡
lspci | grep -i NVIDIA
2 j9 U7 r3 v6 x( x; y7 J#下面说明有1块英伟达的显卡
[root@train-all ~]#  lspci | grep -i NVIDIA
04:00.0 VGA compatible controller: NVIDIA Corporation GP104GL [Quadro P4000] (rev a1)
: p6 L% b- H, P6 p04:00.1 Audio device: NVIDIA Corporation GP104 High Definition Audio Controller (rev a1)
+ ?' @- I& j1 P& U# [( h  C1 f[root@train-all ~]#

2.  添加ELRepo源
; ^9 r: \1 C' f. Nrpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

2 T9 H- {* B  J; m. k3.  安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
2 U0 P0 ]5 |: g, [2 x& Y' E
4 Y% T5 v4 L0 ^, s# d4.  安装nvidia-detect
' R% L2 F4 U& B" Q8 z3 }& u1 x3 Gyum install nvidia-detect -y
( R" R0 {  `4 E: j
5.  运行nvidia-detect
! e; @9 X: z/ H$ {. {nvidia-detect -v
- B. J* n2 n$ w! F  s  Q
6.  查找驱动程序
) j! ~3 `4 h4 }. h" w- t9 `yum search kmod-nvidia

7.  安装驱动程序
yum install kmod-nvidia.x86_64 -y
: G0 Q2 Y2 e" `" j
! O1 M2 c- _" d# u8.  查看禁用Nouveau
lsmod | grep nouveau
#若没有输出 则说明禁用成功，否则执行下面的命令
0 N' K" p1 o' r! r, g9 F5 U: G
3 J6 Z0 O0 g  F6 _  i: J/ }& x9.  在/etc/modprobe.d/blacklist-nouveau.conf中创建一个文件，其内容如下：
vi /etc/modprobe.d/blacklist-nouveau.conf
. t" _. A' {( H3 I- `4 D' }! |, I添加
blacklist nouveau
- ^" _# Q% k- f* j7 zoptions nouveau modeset=0

10. 重新生成内核initramfs
2 b2 G( ?' V( Wdracut --force

11.  重启系统
5 a- r* I, z; X& _! t7 y# J8 E( ~reboot

# Y9 {* j! g8 f# w' F! I1 L12.  测试
nvidia-smi
& E& ~4 i, u0 T