Top
测试硬件直通,参考文章:
bhyve PCI Passthrough
GPU Passthrough in Bhyve(如此文所说,显卡直通后,虚拟机中显示设备报43错误)


首先要在/boot/loader.conf文件中加入一行:
vmm_load="YES"
然后重启系统。
使用pciconf -lv命令查看系统中有哪些pci设备:
# pciconf -lv
hostb0@pci0:0:0:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1450 subvendor=0x1022 subdevice=0x1450
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Root Complex'
    class      = bridge
    subclass   = HOST-PCI
amdiommu0@pci0:0:0:2:   class=0x080600 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1451 subvendor=0x1022 subdevice=0x1451
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) I/O Memory Management Unit'
    class      = base peripheral
    subclass   = IOMMU
hostb1@pci0:0:1:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
pcib1@pci0:0:1:1:       class=0x060400 rev=0x00 hdr=0x01 vendor=0x1022 device=0x1453 subvendor=0x1022 subdevice=0x1453
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) PCIe GPP Bridge'
    class      = bridge
    subclass   = PCI-PCI
pcib2@pci0:0:1:3:       class=0x060400 rev=0x00 hdr=0x01 vendor=0x1022 device=0x1453 subvendor=0x1022 subdevice=0x1453
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) PCIe GPP Bridge'
    class      = bridge
    subclass   = PCI-PCI
hostb2@pci0:0:2:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb3@pci0:0:3:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
pcib8@pci0:0:3:1:       class=0x060400 rev=0x00 hdr=0x01 vendor=0x1022 device=0x1453 subvendor=0x1022 subdevice=0x1453
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) PCIe GPP Bridge'
    class      = bridge
    subclass   = PCI-PCI
hostb4@pci0:0:4:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
hostb5@pci0:0:7:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
pcib9@pci0:0:7:1:       class=0x060400 rev=0x00 hdr=0x01 vendor=0x1022 device=0x1454 subvendor=0x1022 subdevice=0x1454
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B'
    class      = bridge
    subclass   = PCI-PCI
hostb6@pci0:0:8:0:      class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1452 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge'
    class      = bridge
    subclass   = HOST-PCI
pcib10@pci0:0:8:1:      class=0x060400 rev=0x00 hdr=0x01 vendor=0x1022 device=0x1454 subvendor=0x1022 subdevice=0x1454
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B'
    class      = bridge
    subclass   = PCI-PCI
intsmb0@pci0:0:20:0:    class=0x0c0500 rev=0x59 hdr=0x00 vendor=0x1022 device=0x790b subvendor=0x1458 subdevice=0x5001
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'FCH SMBus Controller'
    class      = serial bus
    subclass   = SMBus
isab0@pci0:0:20:3:      class=0x060100 rev=0x51 hdr=0x00 vendor=0x1022 device=0x790e subvendor=0x1458 subdevice=0x5001
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'FCH LPC Bridge'
    class      = bridge
    subclass   = PCI-ISA
hostb7@pci0:0:24:0:     class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1460 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 0'
    class      = bridge
    subclass   = HOST-PCI
hostb8@pci0:0:24:1:     class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1461 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 1'
    class      = bridge
    subclass   = HOST-PCI
hostb9@pci0:0:24:2:     class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1462 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 2'
    class      = bridge
    subclass   = HOST-PCI
hostb10@pci0:0:24:3:    class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1463 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 3'
    class      = bridge
    subclass   = HOST-PCI
hostb11@pci0:0:24:4:    class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1464 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 4'
    class      = bridge
    subclass   = HOST-PCI
hostb12@pci0:0:24:5:    class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1465 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 5'
    class      = bridge
    subclass   = HOST-PCI
hostb13@pci0:0:24:6:    class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1466 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 6'
    class      = bridge
    subclass   = HOST-PCI
hostb14@pci0:0:24:7:    class=0x060000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1467 subvendor=0x0000 subdevice=0x0000
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 7'
    class      = bridge
    subclass   = HOST-PCI
nvme0@pci0:1:0:0:       class=0x010802 rev=0x01 hdr=0x00 vendor=0x1d97 device=0x1d97 subvendor=0x1d97 subdevice=0x1d97
    vendor     = 'Shenzhen Longsys Electronics Co., Ltd.'
    class      = mass storage
    subclass   = NVM
xhci0@pci0:2:0:0:       class=0x0c0330 rev=0x02 hdr=0x00 vendor=0x1022 device=0x43b9 subvendor=0x1b21 subdevice=0x1142
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'X370 Series Chipset USB 3.1 xHCI Controller'
    class      = serial bus
    subclass   = USB
ahci0@pci0:2:0:1:       class=0x010601 rev=0x02 hdr=0x00 vendor=0x1022 device=0x43b5 subvendor=0x1b21 subdevice=0x1062
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'X370 Series Chipset SATA Controller'
    class      = mass storage
    subclass   = SATA
pcib3@pci0:2:0:2:       class=0x060400 rev=0x02 hdr=0x01 vendor=0x1022 device=0x43b0 subvendor=0x1b21 subdevice=0x0201
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'X370 Series Chipset PCIe Upstream Port'
    class      = bridge
    subclass   = PCI-PCI
pcib4@pci0:3:0:0:       class=0x060400 rev=0x02 hdr=0x01 vendor=0x1022 device=0x43b4 subvendor=0x1b21 subdevice=0x3306
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = '300 Series Chipset PCIe Port'
    class      = bridge
    subclass   = PCI-PCI
pcib5@pci0:3:2:0:       class=0x060400 rev=0x02 hdr=0x01 vendor=0x1022 device=0x43b4 subvendor=0x1b21 subdevice=0x3306
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = '300 Series Chipset PCIe Port'
    class      = bridge
    subclass   = PCI-PCI
pcib6@pci0:3:3:0:       class=0x060400 rev=0x02 hdr=0x01 vendor=0x1022 device=0x43b4 subvendor=0x1b21 subdevice=0x3306
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = '300 Series Chipset PCIe Port'
    class      = bridge
    subclass   = PCI-PCI
pcib7@pci0:3:4:0:       class=0x060400 rev=0x02 hdr=0x01 vendor=0x1022 device=0x43b4 subvendor=0x1b21 subdevice=0x3306
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = '300 Series Chipset PCIe Port'
    class      = bridge
    subclass   = PCI-PCI
xhci1@pci0:4:0:0:       class=0x0c0330 rev=0x00 hdr=0x00 vendor=0x1b21 device=0x1343 subvendor=0x1458 subdevice=0x5007
    vendor     = 'ASMedia Technology Inc.'
    device     = 'ASM1143 USB 3.1 Host Controller'
    class      = serial bus
    subclass   = USB
igb0@pci0:5:0:0:        class=0x020000 rev=0x03 hdr=0x00 vendor=0x8086 device=0x1539 subvendor=0x1458 subdevice=0xe000
    vendor     = 'Intel Corporation'
    device     = 'I211 Gigabit Network Connection'
    class      = network
    subclass   = ethernet
vgapci0@pci0:8:0:0:     class=0x030000 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1f08 subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 [GeForce RTX 2060 Rev. A]'
    class      = display
    subclass   = VGA
hdac0@pci0:8:0:1:       class=0x040300 rev=0xa1 hdr=0x00 vendor=0x10de device=0x10f9 subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 High Definition Audio Controller'
    class      = multimedia
    subclass   = HDA
xhci2@pci0:8:0:2:       class=0x0c0330 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1ada subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 USB 3.1 Host Controller'
    class      = serial bus
    subclass   = USB
none0@pci0:8:0:3:       class=0x0c8000 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1adb subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 USB Type-C UCSI Controller'
    class      = serial bus
none0@pci0:9:0:0:       class=0x130000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x145a subvendor=0x1022 subdevice=0x145a
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Zeppelin/Raven/Raven2 PCIe Dummy Function'
    class      = non-essential instrumentation
none1@pci0:9:0:2:       class=0x108000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1456 subvendor=0x1022 subdevice=0x1456
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) Platform Security Processor'
    class      = encrypt/decrypt
xhci2@pci0:9:0:3:       class=0x0c0330 rev=0x00 hdr=0x00 vendor=0x1022 device=0x145c subvendor=0x1458 subdevice=0x5007
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) USB 3.0 Host Controller'
    class      = serial bus
    subclass   = USB
none2@pci0:10:0:0:      class=0x130000 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1455 subvendor=0x1022 subdevice=0x1455
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Zeppelin/Renoir PCIe Dummy Function'
    class      = non-essential instrumentation
ahci1@pci0:10:0:2:      class=0x010601 rev=0x51 hdr=0x00 vendor=0x1022 device=0x7901 subvendor=0x1458 subdevice=0xb002
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'FCH SATA Controller [AHCI mode]'
    class      = mass storage
    subclass   = SATA
hdac0@pci0:10:0:3:      class=0x040300 rev=0x00 hdr=0x00 vendor=0x1022 device=0x1457 subvendor=0x1458 subdevice=0xa182
    vendor     = 'Advanced Micro Devices, Inc. [AMD]'
    device     = 'Family 17h (Models 00h-0fh) HD Audio Controller'
    class      = multimedia
    subclass   = HDA
注:此系统中有一块2060显卡,被识别为TU106(上面褐色粗体字部分)。

以下尝试将该2060显卡直通给Windows虚拟机。
编辑/boot/loader.conf文件,加入一行:
pptdevs="8/0/0 8/0/1 8/0/2 8/0/3"
pptdevs后面的参数可以设置128个字符,若有更多设备需要处理,可以加更多行:
pptdevs2="2/0/0 4/0/0"
重启系统后再次查看pci设备,则显卡对应的设备变为:
ppt0@pci0:8:0:0:        class=0x030000 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1f08 subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 [GeForce RTX 2060 Rev. A]'
    class      = display
    subclass   = VGA
ppt1@pci0:8:0:1:        class=0x040300 rev=0xa1 hdr=0x00 vendor=0x10de device=0x10f9 subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 High Definition Audio Controller'
    class      = multimedia
    subclass   = HDA
ppt2@pci0:8:0:2:        class=0x0c0330 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1ada subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 USB 3.1 Host Controller'
    class      = serial bus
    subclass   = USB
ppt3@pci0:8:0:3:        class=0x0c8000 rev=0xa1 hdr=0x00 vendor=0x10de device=0x1adb subvendor=0x1462 subdevice=0x3755
    vendor     = 'NVIDIA Corporation'
    device     = 'TU106 USB Type-C UCSI Controller'
    class      = serial bus
当以上设置完成后,宿主机将无法进入图形界面,会提示找不到显示器。这是因为显卡已经不再由宿主机管理。
使用vm pass命令查看可直通的设备:
# vm pass
DEVICE     BHYVE ID     READY        DESCRIPTION
hostb0     0/0/0        No           Family 17h (Models 00h-0fh) Root Complex
amdiommu0  0/0/2        No           Family 17h (Models 00h-0fh) I/O Memory Management Unit
hostb1     0/1/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
pcib1      0/1/1        No           Family 17h (Models 00h-0fh) PCIe GPP Bridge
pcib2      0/1/3        No           Family 17h (Models 00h-0fh) PCIe GPP Bridge
hostb2     0/2/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
hostb3     0/3/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
pcib8      0/3/1        No           Family 17h (Models 00h-0fh) PCIe GPP Bridge
hostb4     0/4/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
hostb5     0/7/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
pcib9      0/7/1        No           Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B
hostb6     0/8/0        No           Family 17h (Models 00h-1fh) PCIe Dummy Host Bridge
pcib10     0/8/1        No           Family 17h (Models 00h-0fh) Internal PCIe GPP Bridge 0 to Bus B
intsmb0    0/20/0       No           FCH SMBus Controller
isab0      0/20/3       No           FCH LPC Bridge
hostb7     0/24/0       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 0
hostb8     0/24/1       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 1
hostb9     0/24/2       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 2
hostb10    0/24/3       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 3
hostb11    0/24/4       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 4
hostb12    0/24/5       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 5
hostb13    0/24/6       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 6
hostb14    0/24/7       No           Family 17h (Models 00h-0fh) Data Fabric: Device 18h; Function 7
nvme0      1/0/0        No           -
xhci0      2/0/0        No           X370 Series Chipset USB 3.1 xHCI Controller
ahci0      2/0/1        No           X370 Series Chipset SATA Controller
pcib3      2/0/2        No           X370 Series Chipset PCIe Upstream Port
pcib4      3/0/0        No           300 Series Chipset PCIe Port
pcib5      3/2/0        No           300 Series Chipset PCIe Port
pcib6      3/3/0        No           300 Series Chipset PCIe Port
pcib7      3/4/0        No           300 Series Chipset PCIe Port
xhci1      4/0/0        No           ASM1143 USB 3.1 Host Controller
igb0       5/0/0        No           I211 Gigabit Network Connection
ppt0       8/0/0        Yes          TU106 [GeForce RTX 2060 Rev. A]
ppt1       8/0/1        Yes          TU106 High Definition Audio Controller
ppt2       8/0/2        Yes          TU106 USB 3.1 Host Controller
ppt3       8/0/3        Yes          TU106 USB Type-C UCSI Controller
none0      9/0/0        No           Zeppelin/Raven/Raven2 PCIe Dummy Function
none1      9/0/2        No           Family 17h (Models 00h-0fh) Platform Security Processor
xhci2      9/0/3        No           Family 17h (Models 00h-0fh) USB 3.0 Host Controller
none2      10/0/0       No           Zeppelin/Renoir PCIe Dummy Function
ahci1      10/0/2       No           FCH SATA Controller [AHCI mode]
hdac0      10/0/3       No           Family 17h (Models 00h-0fh) HD Audio Controller
可以看到,BHYVE ID栏的8/0/0、8/0/1、8/0/2、8/0/3这三个设备的名称已经变更为ppt0~ppt3,READY栏也变变成了Yes。
BHYVE ID的三段为:bus/slot/function(总线/插槽/功能)。
使用vm config编辑对应的虚拟机配置文件,加入如下内容:
passthru0="8/0/0"
passthru1="8/0/1"
passthru2="8/0/2"
passthru3="8/0/3"
对于有些amd系统,在使用vm start vmname命令时,可能无法启动,日志中会记录如下内容:
fatal; pci passthrough not supported on this system (no VT-d or amdvi)
需要在/boot/loader.conf文件中加入一行,并重启系统:
hw.vmm.amdvi.enable=1

对于USB直通,需要针对各个USB接口对应的BHYVE ID进行测试,找到适当的usb接口进行直通。