自动化kolla-ansible部署openstack+GPU透传方法

admin

1. CentOS7.x-8.x系列为虚拟机配置GPU直通
! \" C8 ]7 b8 U8 e+ N复制代码
1. 编辑文件vim  /etc/modules， 添加以下内容：
" u8 d, ^- d0 ?8 d7 g/ ]8 K6 h) mpci_stub
# n. f* j% O5 Uvfio
vfio_iommu_type1
vfio_pci
kvm
# `2 Z6 v% J, @8 F) |, kkvm_intel

5 q/ _* ?  `2 o7 T2. 在KVM主机上启用IOMMU
, F$ g8 W, k  e5 I1 x, ^' X! H+ {* ^#对于Intel芯片：
7 ]6 p4 `) K5 [GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
#对于AMD芯片：
1 z; z; s; e+ HGRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"
0 A9 @7 h4 |" G8 g) b% T  |
vim /etc/default/grub

2 [! A2 S8 v& B( A1 WGRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
# b" t; X" O) b7 a6 _2 a4 CGRUB_DISABLE_SUBMENU=true
0 P4 T2 Y% ^/ d) b3 W% tGRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rhgb quiet intel_iommu=on"
7 u: V7 M/ x9 b9 |GRUB_DISABLE_RECOVERY="true"

   3.  重新生成grub
. Q/ b) V; v9 j* g% x/ w$ m8 [( A   EFI
% a; ~. N) C. v0 ?' F9 r4 n7 V   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
) W2 P6 j: |5 P9 Q: U   非EFI
% l' C9 S, Y3 V2 }   grub2-mkconfig -o /boot/grub2/grub.cfg
4 }0 l  Q' n2 z$ K* }5 V4 h4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
$ L8 e- B' {, U# x6 U( Z" S5 vblacklist snd_hda_intel
blacklist amd76x_edac
# B. ?. N  f# f1 A$ c# D; hblacklist vga16fb
blacklist nouveau
- e( n: Z3 x( j0 a6 t6 ?blacklist rivafb
blacklist nvidiafb
% @5 K  k: l) G7 ~  ~blacklist rivatv
blacklist nvidia

5.  查找显卡的Product ID 以及 Vendor ID：
* G4 y+ F3 B5 u% |% d. Gyum install pciutils -y
lspci -nn | grep NVIDIA
如下：
  s5 W; o  Q8 X8 G! c& f, b0 b[root@stein-a ~]#
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)
. v' R4 i* ]5 m4 p+ f, ^; w% J
6.  编辑
vim /etc/modprobe.d/vfio.conf
7 R1 b$ c5 C: H, u: ]% a& w# create new: for [ids=***], specify [vendor-ID:device-ID]
options vfio-pci ids=10de:1bb1,10de:10f0
  @& N9 g1 X7 K6 L/ A/ w
7.  写入到系统启动项
  M. U+ H/ b/ d8 o  ~( `3 Hecho 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf

8.  重新生成initramfs
mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
dracut -v /boot/initramfs-$(uname -r).img $(uname -r)

: O6 C' N* q3 b& P& w5 ?+ t& @9.  重启系统
# d# I: _* I1 Q0 p4 preboot

10. 验证
lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
[root@stein-a ~]# lspci -nnk -d 10de:1bb1
5 i& O' }5 y; T1 J" `! z03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation Device [10de:11a3]
! Q' e. l% O  g/ _8 [        Kernel driver in use: vfio-pci
, p: n- ~8 A# K8 E. Y$ A& r        Kernel modules: nouveau
[root@stein-a ~]# dmesg | grep -i vfio
[    2.503115] VFIO - User Level meta-driver version: 0.3
) g+ Z9 Z( c8 g4 h% C) ~. x[    2.515645] vfio_pci: add [10de:1bb1[ffff:ffff]] class 0x000000/00000000
& `! m" a6 q$ p4 I[    2.515752] vfio_pci: add [10de:10f0[ffff:ffff]] class 0x000000/00000000
[root@stein-a ~]#
$ r: ~+ O7 v6 h* x; ]复制代码
8 b$ J! u& N' t8 J
# O8 y+ |) q% L6 N2. Ubuntu18.04系列为虚拟机配置GPU直通
' R" H9 K. d! ^$ P0 Y. g+ _( E+ E复制代码
, s' T- s' y: f8 I2 V% {1. 编辑文件vim  /etc/modules， 添加以下内容：
0 w" i3 i. m* i$ fpci_stub
" z2 C9 C) I" B; s8 Avfio
vfio_iommu_type1
vfio_pci
kvm
kvm_intel
7 T3 L/ Y6 q3 R4 m5 _; k
+ f, `1 g1 A- m& Q" a2. 在KVM主机上启用IOMMU
$ b9 v5 V( p6 z* Q% l#对于Intel芯片：
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
7 V7 U9 D# j+ w  c4 A! f/ w& M#对于AMD芯片：
GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt iommu=1"

vim /etc/default/grub
! }( p1 [3 T# b9 e
& }8 v( g" X7 xGRUB_DEFAULT=0
GRUB_TIMEOUT_STYLE=hidden
* T& M# b  t* j: x4 jGRUB_TIMEOUT=0
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
$ |7 p* k6 F$ h" `$ {% gGRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on"
GRUB_CMDLINE_LINUX=""
/ c9 G1 G2 f& L; `
   3.  重新生成grub
: \; Z' H, R3 {  H   EFI
0 t0 V: X4 b  D# ^) L   grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg
: U$ e! f" `  e4 f4 q5 T* y. [* i   非EFI
   grub2-mkconfig -o /boot/grub2/grub.cfg
4.  将下列内容加入到blacklist中以避免被宿主机占用，编辑文件
vim  /etc/modprobe.d/blacklist.conf
9 J" ~6 V( r( ?9 k- `& A- xblacklist snd_hda_intel
- @8 ^* a9 ]; [& ablacklist amd76x_edac
blacklist vga16fb
- z# |2 c0 p% m6 Ablacklist nouveau
blacklist rivafb
blacklist nvidiafb
' b/ F! E* l, O5 }8 cblacklist rivatv
blacklist nvidia

5.  查找显卡的Product ID 以及 Vendor ID：
apt install pciutils -y
: f- ]6 p  F5 K( Flspci -nn | grep NVIDIA
* K+ G6 Z5 T7 R5 X6 X  d+ ?1 x/ {如下：
( N& t& K3 O6 |) q[root@stein-a ~]# lspci -nn | grep NVIDIA
# E+ U( I; N7 {) V, [03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
; b" P; e- r% [# C4 r( n; Q, d03:00.1 Audio device [0403]: NVIDIA Corporation GP104 High Definition Audio Controller [10de:10f0] (rev a1)

  s" ]( `0 T; ]. E# X6.  编辑
vim /etc/modprobe.d/vfio.conf
3 `# D! Z+ c& A4 w# create new: for [ids=***], specify [vendor-ID:device-ID]
options vfio-pci ids=10de:1bb1,10de:10f0

7.  写入到系统启动项
$ {2 W( H  @" Z+ e0 ]( G% ^$ Xecho 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf

7 g+ ~- C7 k( X5 l$ U& C. k' w1 k8.  重新生成initramfs
1 D1 c4 D; p; H) sdracut -v /boot/initramfs-$(uname -r).img $(uname -r)

4 G3 G# E! Q) s/ D9.  重启系统
reboot

2 @( S. J  U& `/ F' \10. 验证
lspci -nnk -d 10de:1bb1
2 T  c3 g5 D* m- h% e4 t; admesg | grep -i vfio
6 |* _+ h" |' b1 c# e; Iroot@kvm:~# lspci -nnk -d 10de:1bb1
dmesg | grep -i vfio
03:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP104GL [Quadro P4000] [10de:1bb1] (rev a1)
        Subsystem: NVIDIA Corporation GP104GL [Quadro P4000] [10de:11a3]
        Kernel driver in use: vfio-pci
        Kernel modules: nvidiafb, nouveau
6 \4 `+ x- L- b" O) y4 l2 Oroot@kvm:~# dmesg | grep -i vfio
& ~% a# c$ t7 E7 |* _[    3.838714] VFIO - User Level meta-driver version: 0.3
  d, j2 b0 G5 K0 X[    3.846238] vfio-pci 0000:03:00.0: vgaarb: changed VGA decodes: olddecodes=io+mem,decodes=io+mem:owns=none
0 k- n6 ?" I1 K$ [' L: }1 @" Y[    3.866370] vfio_pci: add [10de:1bb1[ffffffff:ffffffff]] class 0x000000/00000000
[    3.886375] vfio_pci: add [10de:10f0[ffffffff:ffffffff]] class 0x000000/00000000
复制代码
, r" f% F, T1 i) [
, I& K/ z  ^% i2 R: e3 D复制代码
% }: ]! K6 c- I/ e* n#如果你单机部署的，在单机下配置。
#如果你是高可用部署的，在三台控制节点配置
1. 添加pci
vim /etc/kolla/config/nova/nova-compute.conf
[libvirt]
inject_password=true
. q7 P9 A6 }0 K4 ncpu_mode=host-passthrough
5 S* v  O; E* D* q9 Q; Gvirt_type = kvm
[pci]
" C8 F* a: @' G* l2 }passthrough_whitelist: { "vendor_id": "10de", "product_id": "1bb1" }
+ S$ T! j! O! i% u. e' ^) z
* c! z. H! ~% ~" t; A2. 修改nova.conf
$ `  C4 G# P" ]) M% O3 A4 E' t# uvim /etc/kolla/config/nova.conf
[DEFAULT]
' N+ T, S9 V3 t1 B& Yservice_down_time = 120
cpu_allocation_ratio = 4.0   
5 E; m/ v0 D* ]0 Idisk_allocation_ratio=1.0
; k1 |1 F# h! V# r% B+ mram_allocation_ratio = 1.0   
$ J7 ^' o9 h$ ?8 P1 Nreserved_host_disk_mb = 4096
4 O3 m8 X$ S* A3 hreserved_host_memory_mb = 4096
allow_resize_to_same_host = True   
remove_unused_base_images = False
6 P  S7 @0 W9 nimage_cache_manager_interval = 0
# \3 `1 E9 o+ _/ W4 j0 {3 gresume_guests_state_on_host_boot = True

4 R' O; _/ t  D+ P& ]) H# j[PCI]
alias: { "vendor_id":"10de", "product_id":"1bb1", "device_type":"type-PCI", "name":"quadro-p4000" }
[filter_scheduler]
enabled_filters = RetryFilter, AvailabilityZoneFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
available_filters = nova.scheduler.filters.all_filters
+ |$ G/ W0 g5 H9 c' V9 b
* t7 l0 e* b9 p- f0 ^6 t3. GPU 类型实例创建  
9 }9 ], e: m  H/ o# `, ?openstack flavor create --vcpus 4 --ram 8192 --disk 30 --property "pci_passthrough:alias"="quadro-p400:1" g1.4c.8m.p400
复制代码

3. CentOS7.x系列 安装显卡驱动
$ ]1 g* v, C' o9 ~: x  x复制代码
1.  查看是否含有英伟达显卡
9 z! |+ p4 |6 a; w9 r' _& w& clspci | grep -i NVIDIA
#下面说明有1块英伟达的显卡
[root@train-all ~]#  lspci | grep -i NVIDIA
04:00.0 VGA compatible controller: NVIDIA Corporation GP104GL [Quadro P4000] (rev a1)
04:00.1 Audio device: NVIDIA Corporation GP104 High Definition Audio Controller (rev a1)
[root@train-all ~]#

2.  添加ELRepo源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
6 X  V+ ?- A5 g8 l! a4 i, ?0 @
3.  安装ELRepo
# A$ R2 Q. d4 R% Frpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
) h( j# W0 ~1 t, C5 t( W9 o  x
0 t" E/ h- }5 e2 [. a5 J) \* J4.  安装nvidia-detect
yum install nvidia-detect -y

5.  运行nvidia-detect
nvidia-detect -v
0 j- u# S4 b% x" ^( Z% B. g8 }8 k
6.  查找驱动程序
yum search kmod-nvidia
" l, J- b& U+ Z) `( m/ `. R5 A
3 O( _0 V, z3 c0 Z/ t6 Y8 B7.  安装驱动程序
yum install kmod-nvidia.x86_64 -y

* [: J  C: ~/ S2 b8.  查看禁用Nouveau
lsmod | grep nouveau
3 x5 U" K: l! L* r9 l) U% v#若没有输出 则说明禁用成功，否则执行下面的命令
- L* D, b$ L5 Z& `  z9 l# m3 p
, z9 ^- p3 @: y& y5 r9.  在/etc/modprobe.d/blacklist-nouveau.conf中创建一个文件，其内容如下：
vi /etc/modprobe.d/blacklist-nouveau.conf
添加
blacklist nouveau
. ?8 {3 v( }: N( g) F* Hoptions nouveau modeset=0

10. 重新生成内核initramfs
dracut --force
; U/ U+ h. Q. r6 f. Q& d
4 j2 A5 h9 t; L5 q$ B11.  重启系统
reboot
1 i: \0 l6 j2 o2 j3 Y% Z
4 {6 E8 I, q( w; f2 ]12.  测试
, o! W8 |- z; E1 xnvidia-smi
6 Z  U0 ~, ]/ ]2 o