openstack 相关gpu配置

admin

OpenStack的相关配置
1. 配置nova-scheduler （controller节点），编辑文件 /etc/nova/nova.conf:
' }- o1 a5 B$ Q  Q' ~3 g  n, Q[DEFAULT]
scheduler_default_filters = RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter
scheduler_available_filters = nova.scheduler.filters.all_filters
/ N- `! m( _7 ]4 W" c9 d% j8 \重启nova-scheduler服务
* c$ |2 @7 k$ D, V4 F' T, N[root@controller ~]# systemctl restart openstack-nova-scheduler.service
" H0 W& V# A! @' H" m0 N
! n" z4 g; R+ ^/ a$ U) d[root@controller ~]# systemctl status openstack-nova-scheduler.service
" z: x8 [( i  o5 G6 ^& Z, y● openstack-nova-scheduler.service - OpenStack Nova Scheduler Server
   Loaded: loaded (/usr/lib/systemd/system/openstack-nova-scheduler.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2022-06-10 21:50:56 CST; 22s ago
Main PID: 105509 (nova-scheduler)
4 g' y4 j0 y# {4 t3 d    Tasks: 9 (limit: 100963)
   Memory: 276.0M
  F3 K' D: n% ^9 |# q   CGroup: /system.slice/openstack-nova-scheduler.service
           ├─105509 /usr/bin/python3 /usr/bin/nova-scheduler
( L; t/ S' d. f. S; d  l4 c           ├─105528 /usr/bin/python3 /usr/bin/nova-scheduler
           ├─105529 /usr/bin/python3 /usr/bin/nova-scheduler
" U3 ^7 @. _. z: x( `) Q           ├─105530 /usr/bin/python3 /usr/bin/nova-scheduler
6 J0 |5 t5 ?8 j           ├─105531 /usr/bin/python3 /usr/bin/nova-scheduler
           ├─105532 /usr/bin/python3 /usr/bin/nova-scheduler
           ├─105533 /usr/bin/python3 /usr/bin/nova-scheduler
           ├─105534 /usr/bin/python3 /usr/bin/nova-scheduler
           └─105535 /usr/bin/python3 /usr/bin/nova-scheduler
Jun 10 21:50:52 controller systemd[1]: Starting OpenStack Nova Scheduler Server...
Jun 10 21:50:56 controller systemd[1]: Started OpenStack Nova Scheduler Server.


2. 配置nova-api （controller节点），编辑文件 /etc/nova/nova.conf:
, w/ ?3 f& r0 r& s0 D! f) G[pci]
& u8 l, _+ m& P- ?; `& q# }alias = { "name": "nvidia1080", "product_id": "1b06", "vendor_id": "10de", "device_type": "type-PCI" }
% I4 |9 D2 Z6 ?/ M- k0 t! e- ~[pci]
9 n" z9 u! R* N3 Z# Z+ D  nalias = { "name": "nvidiaGF119","product_id": "104a","vendor_id": "10de","device_type": "type-PCI" }

8 `* N/ Y# `0 Q# x# t# G4 h( p9 [" f重启nova-api服务

[root@controller ~]# systemctl restart openstack-nova-api.service
- g0 ^" s. O( _$ p8 t
3. 配置nova-compute（compute 节点），编辑文件/etc/nova/nova.conf:
[pci]
passthrough_whitelist = { "vendor_id": "10de", "product_id": "104a" }
alias = {
       "name": "nvidiaGF119",
8 I, j" m  X5 H: O3 k! m: a       "product_id": "104a",
       "vendor_id": "10de",
       "device_type": "type-PCI"
& r; p  T% Z+ }( Y}
) Q; w- e* I- Q3 S, v
6 n' x8 e0 h$ `1 U
[pci]
9 Z  o/ n3 l3 H# l. A9 {( ypassthrough_whitelist = { "vendor_id": "10de", "product_id": "104a" }
alias = { "name": "nvidiaGF119", "product_id": "104a", "vendor_id": "104a", "device_type": "type_PCI" }
8 ]: {- o* B; @: _9 x
重启nova-compute服务
2 V1 S3 j9 Y3 A! A9 ]% c[root@compute01 ~]# systemctl restart openstack-nova-compute.service
. N  W9 W* ~/ G* C# m0 {. ?+ k' I
[root@compute01 ~]# systemctl restart openstack-nova-compute.service
[root@compute01 ~]# tail -f /var/log/nova/
nova-compute.log    privsep-helper.log  
: [) `2 i( z1 k2 z( C" ?! O[root@compute01 ~]# tail -f /var/log/nova/nova-compute.log
2022-06-10 22:10:51.891 12258 INFO oslo.privsep.daemon [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] Spawned new privsep daemon via rootwrap
5 R7 N4 U  H$ F4 b1 ~2022-06-10 22:10:51.796 12299 INFO oslo.privsep.daemon [-] privsep daemon starting
2022-06-10 22:10:51.800 12299 INFO oslo.privsep.daemon [-] privsep process running with uid/gid: 0/0
: V2 K) E5 _; a. j+ m2022-06-10 22:10:51.804 12299 INFO oslo.privsep.daemon [-] privsep process running with capabilities (eff/prm/inh): CAP_NET_ADMIN/CAP_NET_ADMIN/none
+ l# C' ]1 V. r- H2022-06-10 22:10:51.804 12299 INFO oslo.privsep.daemon [-] privsep daemon running as pid 12299
2022-06-10 22:10:52.437 12258 INFO os_vif [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] Successfully plugged vif VIFBridge(active=True,address=fa:16:3e:bf:2a:4e,bridge_name='qbr24719437-3e',has_traffic_filtering=True,id=24719437-3ee6-469b-af02-c1fcea041be2,network=Network(b83e2ffc-eaad-455f-b299-18e09d58be32),plugin='ovs',port_profile=VIFPortProfileOpenVSwitch,preserve_on_delete=False,vif_name='tap24719437-3e')
2022-06-10 22:10:52.459 12258 INFO os_vif [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] Successfully plugged vif VIFBridge(active=True,address=fa:16:3e:fe:4c:d1,bridge_name='qbr58f2e526-38',has_traffic_filtering=True,id=58f2e526-386b-43da-9818-208b6a34b6e8,network=Network(5eb067d8-cd9b-4eec-ac0b-b5982752e75d),plugin='ovs',port_profile=VIFPortProfileOpenVSwitch,preserve_on_delete=False,vif_name='tap58f2e526-38')
2022-06-10 22:10:52.478 12258 INFO os_vif [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] Successfully plugged vif VIFBridge(active=True,address=fa:16:3e:bd:8b:42,bridge_name='qbr24c6e701-e5',has_traffic_filtering=True,id=24c6e701-e5b4-4277-9895-cc67a4097280,network=Network(5eb067d8-cd9b-4eec-ac0b-b5982752e75d),plugin='ovs',port_profile=VIFPortProfileOpenVSwitch,preserve_on_delete=False,vif_name='tap24c6e701-e5')
2022-06-10 22:10:52.481 12258 INFO nova.compute.manager [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host
/ v. z) Q9 ~* y  I2022-06-10 22:10:54.740 12258 INFO nova.virt.libvirt.host [req-e600b0bc-1cc4-4e85-a406-4d0c094560ee - - - - -] kernel doesn't support AMD SEV
/ }: t) l  u$ ?( n4 J
& X8 p3 t( ?  Z6 a& C
三 验证
1. 创建设置flavor：
openstack flavor create --public --ram 2048 --disk 20 --vcpus 2 m1.large
! P; a( Y& [, I+ |& Fopenstack flavor set m1.large --property pci_passthrough:alias='nvidia1080:2'
. n, Q! S% i' ~, q. F# S, c- J/ Unvidia1080 即为alias中的那么， 2为GPU的数量。
2. 创建instance：
openstack server create --flavor m1.large --image cirros-0.3.5-x86_64-uec --wait test-pci
3. 在cirros下查看GPU信息如下：
4 n7 \4 p+ j- U( F/ Z$ lspci -k
# W0 b, y/ A7 W! c4 g5 T...
" V$ S# H1 H. \# G" s$ y00:05.0 Class 0300: 10de:1b06
00:06.0 Class 0300: 10de:1b06
...
四 NVIDIA显卡的问题
  ?( o" b& Q# r0 P9 F因为NIVIDIA显卡的驱动会检测是否跑在虚拟机里，如果在虚拟机里驱动就会出错，所以我们需要对显卡驱动隐藏hypervisor id。在OpenStack的Pile版本中的Glance 镜像引入了img_hide_hypervisor_id=true的property，所以可以对镜像执行如下的命令隐藏hupervisor id：
* y* A( i) C/ j) j" f3 }4 a; O' T; j$ openstack image set IMG-UUID --property img_hide_hypervisor_id=true
) a( x! N4 c1 i( W7 j% K, w* L通过此镜像安装的instance就会隐藏hypervisor id。
) G  F" j7 V7 U如果是Pike之前的版本， 可以参考Consumer-grade GPUs in an OpenStack system (NVIDIA GPUs)这篇文章的做法。
可以通过下边的命令查看hypervisor id是否隐藏：
+ q7 u$ g8 }, b  ~! |$ cpuid | grep hypervisor_id
   hypervisor_id = "KVMKVMKVM   "
   hypervisor_id = "KVMKVMKVM   "
上边的显示结果说明没有隐藏，下边的显示结果说明已经隐藏：
' A- O2 c$ |% g! Y' Q0 c$ D$ cpuid | grep hypervisor_id
. m& [$ b; E' U  \: [1 |   hypervisor_id = "  @  @    "
   hypervisor_id = "  @  @    "


+ q; K4 W% P( Z8 l1 b
9 N$ m* P# A- k! d# y
* d' X# Y( p# M8 d

admin

[root@controller ~]# openstack flavor create --public --ram 2048 --disk 20 --vcpus 2 m1.large
+----------------------------+--------------------------------------+
| Field                      | Value                                |
" n. E/ J. w; ]; g3 f5 h% _1 T) v+----------------------------+--------------------------------------+
| OS-FLV-DISABLED:disabled   | False                                |
| OS-FLV-EXT-DATA:ephemeral  | 0                                    |
| description                | None                                 |
! Y* c+ {& }$ r  ]* G+ n| disk                       | 20                                   |
. z) i8 c0 Q* w! k' k$ N; C| id                         | a56773dd-2ab1-453b-ab94-95c559334567 |
- C1 g5 Y& g" x- S) e7 W| name                       | m1.large                             |
& w+ O, l9 ~! m0 p| os-flavor-access:is_public | True                                 |
: ?% B7 d8 a- c| properties                 |                                      |
" Q# Y! [" }* u$ {; y: i| ram                        | 2048                                 |
' O4 J* w9 t: K* c  p2 ~| rxtx_factor                | 1.0                                  |
, P! L0 Q$ o0 O8 K* S6 D| swap                       |                                      |
| vcpus                      | 2                                    |
* I( ]' V$ h& ]/ a' w& K+----------------------------+--------------------------------------+
( D  G  P( q# a4 |' c[root@controller ~]#  openstack flavor set m1.large --property pci_passthrough:alias='nvidia1080:2'

admin

[root@controller ~]#  openstack flavor set m1.large --property pci_passthrough:alias='nvidiaGF119:1'

% O% H9 Q: l; n% @0 g3 Z0 L) L这里的值必须和nova.conf中的值一样  
否则报错