diff --git a/README.md b/README.md index 5b154a84..d1d386f2 100644 --- a/README.md +++ b/README.md @@ -1,930 +1,125 @@ -# NVIDIA Linux Open GPU Kernel Module Source - -This is the source release of the NVIDIA Linux open GPU kernel modules, -version 550.76. - - -## How to Build - -To build: - - make modules -j$(nproc) - -To install, first uninstall any existing NVIDIA kernel modules. Then, -as root: - - make modules_install -j$(nproc) - -Note that the kernel modules built here must be used with GSP -firmware and user-space NVIDIA GPU driver components from a corresponding -550.76 driver release. This can be achieved by installing -the NVIDIA GPU driver from the .run file using the `--no-kernel-modules` -option. E.g., - - sh ./NVIDIA-Linux-[...].run --no-kernel-modules - - -## Supported Target CPU Architectures - -Currently, the kernel modules can be built for x86_64 or aarch64. -If cross-compiling, set these variables on the make command line: - - TARGET_ARCH=aarch64|x86_64 - CC - LD - AR - CXX - OBJCOPY - -E.g., - - # compile on x86_64 for aarch64 - make modules -j$(nproc) \ - TARGET_ARCH=aarch64 \ - CC=aarch64-linux-gnu-gcc \ - LD=aarch64-linux-gnu-ld \ - AR=aarch64-linux-gnu-ar \ - CXX=aarch64-linux-gnu-g++ \ - OBJCOPY=aarch64-linux-gnu-objcopy - - -## Other Build Knobs - -NV_VERBOSE - Set this to "1" to print each complete command executed; - otherwise, a succinct "CC" line is printed. - -DEBUG - Set this to "1" to build the kernel modules as debug. By default, the - build compiles without debugging information. This also enables - various debug log messages in the kernel modules. - -These variables can be set on the make command line. E.g., - - make modules -j$(nproc) NV_VERBOSE=1 - - -## Supported Toolchains - -Any reasonably modern version of GCC or Clang can be used to build the -kernel modules. Note that the kernel interface layers of the kernel -modules must be built with the toolchain that was used to build the -kernel. - - -## Supported Linux Kernel Versions - -The NVIDIA open kernel modules support the same range of Linux kernel -versions that are supported with the proprietary NVIDIA kernel modules. -This is currently Linux kernel 3.10 or newer. - - -## How to Contribute - -Contributions can be made by creating a pull request on -https://github.com/NVIDIA/open-gpu-kernel-modules -We'll respond via GitHub. - -Note that when submitting a pull request, you will be prompted to accept -a Contributor License Agreement. - -This code base is shared with NVIDIA's proprietary drivers, and various -processing is performed on the shared code to produce the source code that is -published here. This has several implications for the foreseeable future: - -* The GitHub repository will function mostly as a snapshot of each driver - release. - -* We do not expect to be able to provide revision history for individual - changes that were made to NVIDIA's shared code base. There will likely - only be one git commit per driver release. - -* We may not be able to reflect individual contributions as separate - git commits in the GitHub repository. - -* Because the code undergoes various processing prior to publishing here, - contributions made here require manual merging to be applied to the shared - code base. Therefore, large refactoring changes made here may be difficult to - merge and accept back into the shared code base. If you have large - refactoring to suggest, please contact us in advance, so we can coordinate. - - -## How to Report Issues - -Problems specific to the Open GPU Kernel Modules can be reported in the -Issues section of the https://github.com/NVIDIA/open-gpu-kernel-modules -repository. - -Further, any of the existing bug reporting venues can be used to communicate -problems to NVIDIA, such as our forum: - -https://forums.developer.nvidia.com/c/gpu-graphics/linux/148 - -or linux-bugs@nvidia.com. - -Please see the 'NVIDIA Contact Info and Additional Resources' section -of the NVIDIA GPU Driver README for details. - -Please see the separate [SECURITY.md](SECURITY.md) document if you -believe you have discovered a security vulnerability in this software. - - -## Kernel Interface and OS-Agnostic Components of Kernel Modules - -Most of NVIDIA's kernel modules are split into two components: - -* An "OS-agnostic" component: this is the component of each kernel module - that is independent of operating system. - -* A "kernel interface layer": this is the component of each kernel module - that is specific to the Linux kernel version and configuration. - -When packaged in the NVIDIA .run installation package, the OS-agnostic -component is provided as a binary: it is large and time-consuming to -compile, so pre-built versions are provided so that the user does -not have to compile it during every driver installation. For the -nvidia.ko kernel module, this component is named "nv-kernel.o_binary". -For the nvidia-modeset.ko kernel module, this component is named -"nv-modeset-kernel.o_binary". Neither nvidia-drm.ko nor nvidia-uvm.ko -have OS-agnostic components. - -The kernel interface layer component for each kernel module must be built -for the target kernel. - - -## Directory Structure Layout - -- `kernel-open/` The kernel interface layer -- `kernel-open/nvidia/` The kernel interface layer for nvidia.ko -- `kernel-open/nvidia-drm/` The kernel interface layer for nvidia-drm.ko -- `kernel-open/nvidia-modeset/` The kernel interface layer for nvidia-modeset.ko -- `kernel-open/nvidia-uvm/` The kernel interface layer for nvidia-uvm.ko - -- `src/` The OS-agnostic code -- `src/nvidia/` The OS-agnostic code for nvidia.ko -- `src/nvidia-modeset/` The OS-agnostic code for nvidia-modeset.ko -- `src/common/` Utility code used by one or more of nvidia.ko and nvidia-modeset.ko -- `nouveau/` Tools for integration with the Nouveau device driver - - -## Nouveau device driver integration - -The Python script in the 'nouveau' directory is used to extract some of the -firmware binary images (and related data) encoded in the source code and -store them as distinct files. These files are used by the Nouveau device -driver to load and communicate with the GSP firmware. - -The layout of the binary files is described in nouveau_firmware_layout.ods, -which is an OpenDocument Spreadsheet file, compatible with most spreadsheet -software applications. - - -## Compatible GPUs - -The NVIDIA open kernel modules can be used on any Turing or later GPU -(see the table below). However, in the __DRIVER_VERION__ release, GeForce and -Workstation support is considered to be Beta quality. The open kernel modules -are suitable for broad usage, and NVIDIA requests feedback on any issues -encountered specific to them. - -For details on feature support and limitations, see the NVIDIA GPU driver -end user README here: - -https://us.download.nvidia.com/XFree86/Linux-x86_64/550.76/README/kernel_open.html - -For vGPU support, please refer to the README.vgpu packaged in the vGPU Host -Package for more details. - -In the below table, if three IDs are listed, the first is the PCI Device -ID, the second is the PCI Subsystem Vendor ID, and the third is the PCI -Subsystem Device ID. - -| Product Name | PCI ID | -| ----------------------------------------------- | -------------- | -| NVIDIA TITAN RTX | 1E02 | -| NVIDIA GeForce RTX 2080 Ti | 1E04 | -| NVIDIA GeForce RTX 2080 Ti | 1E07 | -| Quadro RTX 6000 | 1E30 | -| Quadro RTX 8000 | 1E30 1028 129E | -| Quadro RTX 8000 | 1E30 103C 129E | -| Quadro RTX 8000 | 1E30 10DE 129E | -| Quadro RTX 6000 | 1E36 | -| Quadro RTX 8000 | 1E78 10DE 13D8 | -| Quadro RTX 6000 | 1E78 10DE 13D9 | -| NVIDIA GeForce RTX 2080 SUPER | 1E81 | -| NVIDIA GeForce RTX 2080 | 1E82 | -| NVIDIA GeForce RTX 2070 SUPER | 1E84 | -| NVIDIA GeForce RTX 2080 | 1E87 | -| NVIDIA GeForce RTX 2060 | 1E89 | -| NVIDIA GeForce RTX 2080 | 1E90 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1025 1375 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08A1 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08A2 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08EA | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08EB | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08EC | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08ED | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08EE | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 08EF | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 093B | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1028 093C | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 8572 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 8573 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 8602 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 8606 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 86C6 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 86C7 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 87A6 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 103C 87A7 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1043 131F | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1043 137F | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1043 141F | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1043 1751 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 1660 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 1661 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 1662 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 75A6 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 75A7 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 86A6 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1458 86A7 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1462 1274 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1462 1277 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 152D 1220 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1558 95E1 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1558 97E1 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1A58 2002 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1A58 2005 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1A58 2007 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1A58 3000 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1A58 3001 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1E90 1D05 1069 | -| NVIDIA GeForce RTX 2070 Super | 1E91 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 8607 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 8736 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 8738 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 8772 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 878A | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 103C 878B | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1043 1E61 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 1511 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 75B3 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 75B4 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 76B2 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 76B3 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 78A2 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 78A3 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 86B2 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1458 86B3 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1462 12AE | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1462 12B0 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1462 12C6 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 17AA 22C3 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 17AA 22C5 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1A58 2009 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1A58 200A | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 1A58 3002 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1E91 8086 3012 | -| NVIDIA GeForce RTX 2080 Super | 1E93 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1025 1401 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1025 149C | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1028 09D2 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 8607 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 86C7 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 8736 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 8738 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 8772 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 87A6 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 103C 87A7 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 75B1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 75B2 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 76B0 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 76B1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 78A0 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 78A1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 86B0 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1458 86B1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1462 12AE | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1462 12B0 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1462 12B4 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1462 12C6 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1558 50D3 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1558 70D1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 17AA 22C3 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 17AA 22C5 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1A58 2009 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1A58 200A | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1A58 3002 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1E93 1D05 1089 | -| Quadro RTX 5000 | 1EB0 | -| Quadro RTX 4000 | 1EB1 | -| Quadro RTX 5000 | 1EB5 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1025 1375 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1025 1401 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1025 149C | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1028 09C3 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8736 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8738 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8772 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8780 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8782 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8783 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 103C 8785 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1043 1DD1 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1462 1274 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1462 12B0 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1462 12C6 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 17AA 22B8 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 17AA 22BA | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1A58 2005 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1A58 2007 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1A58 2008 | -| Quadro RTX 5000 with Max-Q Design | 1EB5 1A58 200A | -| Quadro RTX 4000 | 1EB6 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 1028 09C3 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8736 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8738 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8772 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8780 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8782 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8783 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 103C 8785 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 1462 1274 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 1462 1277 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 1462 12B0 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 1462 12C6 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 17AA 22B8 | -| Quadro RTX 4000 with Max-Q Design | 1EB6 17AA 22BA | -| NVIDIA GeForce RTX 2070 SUPER | 1EC2 | -| NVIDIA GeForce RTX 2070 SUPER | 1EC7 | -| NVIDIA GeForce RTX 2080 | 1ED0 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1025 132D | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1028 08ED | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1028 08EE | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1028 08EF | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 103C 8572 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 103C 8573 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 103C 8600 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 103C 8605 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1043 138F | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 1043 15C1 | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 17AA 3FEE | -| NVIDIA GeForce RTX 2080 with Max-Q Design | 1ED0 17AA 3FFE | -| NVIDIA GeForce RTX 2070 Super | 1ED1 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 1025 1432 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 103C 8746 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 103C 878A | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 1043 165F | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 144D C192 | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 17AA 3FCE | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 17AA 3FCF | -| NVIDIA GeForce RTX 2070 Super with Max-Q Design | 1ED1 17AA 3FD0 | -| NVIDIA GeForce RTX 2080 Super | 1ED3 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 1025 1432 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 1028 09D1 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 103C 8746 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 103C 878A | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 1043 1D61 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 1043 1E51 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 1043 1F01 | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 17AA 3FCE | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 17AA 3FCF | -| NVIDIA GeForce RTX 2080 Super with Max-Q Design | 1ED3 17AA 3FD0 | -| Quadro RTX 5000 | 1EF5 | -| NVIDIA GeForce RTX 2070 | 1F02 | -| NVIDIA GeForce RTX 2060 | 1F03 | -| NVIDIA GeForce RTX 2060 SUPER | 1F06 | -| NVIDIA GeForce RTX 2070 | 1F07 | -| NVIDIA GeForce RTX 2060 | 1F08 | -| NVIDIA GeForce GTX 1650 | 1F0A | -| NVIDIA GeForce RTX 2070 | 1F10 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1025 132D | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1025 1342 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08A1 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08A2 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08EA | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08EB | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08EC | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08ED | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08EE | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 08EF | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 093B | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1028 093C | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 103C 8572 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 103C 8573 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 103C 8602 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 103C 8606 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1043 132F | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1043 136F | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1043 1881 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1043 1E6E | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 1658 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 1663 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 1664 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 75A4 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 75A5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 86A4 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1458 86A5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1462 1274 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1462 1277 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1558 95E1 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1558 97E1 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1A58 2002 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1A58 2005 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1A58 2007 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1A58 3000 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1A58 3001 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1D05 105E | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1D05 1070 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 1D05 2087 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F10 8086 2087 | -| NVIDIA GeForce RTX 2060 | 1F11 | -| NVIDIA GeForce RTX 2060 | 1F12 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 1028 098F | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 103C 8741 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 103C 8744 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 103C 878E | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 103C 880E | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 1043 1E11 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 1043 1F11 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 1462 12D9 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 17AA 3801 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 17AA 3802 | -| NVIDIA GeForce RTX 2060 with Max-Q Design | 1F12 17AA 3803 | -| NVIDIA GeForce RTX 2070 | 1F14 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1025 1401 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1025 1432 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1025 1442 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1025 1446 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1025 147D | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1028 09E2 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1028 09F3 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 8607 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 86C6 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 86C7 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 8736 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 8738 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 8746 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 8772 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 878A | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 878B | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 87A6 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 103C 87A7 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1043 174F | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 1512 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 75B5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 75B6 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 76B4 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 76B5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 78A4 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 78A5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 86B4 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1458 86B5 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1462 12AE | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1462 12B0 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1462 12C6 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1558 50D3 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1558 70D1 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1A58 200C | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1A58 2011 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F14 1A58 3002 | -| NVIDIA GeForce RTX 2060 | 1F15 | -| Quadro RTX 3000 | 1F36 | -| Quadro RTX 3000 with Max-Q Design | 1F36 1028 0990 | -| Quadro RTX 3000 with Max-Q Design | 1F36 103C 8736 | -| Quadro RTX 3000 with Max-Q Design | 1F36 103C 8738 | -| Quadro RTX 3000 with Max-Q Design | 1F36 103C 8772 | -| Quadro RTX 3000 with Max-Q Design | 1F36 1043 13CF | -| Quadro RTX 3000 with Max-Q Design | 1F36 1414 0032 | -| NVIDIA GeForce RTX 2060 SUPER | 1F42 | -| NVIDIA GeForce RTX 2060 SUPER | 1F47 | -| NVIDIA GeForce RTX 2070 | 1F50 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 1028 08ED | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 1028 08EE | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 1028 08EF | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 103C 8572 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 103C 8573 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 103C 8574 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 103C 8600 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 103C 8605 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 17AA 3FEE | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F50 17AA 3FFE | -| NVIDIA GeForce RTX 2060 | 1F51 | -| NVIDIA GeForce RTX 2070 | 1F54 | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F54 103C 878A | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F54 17AA 3FCE | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F54 17AA 3FCF | -| NVIDIA GeForce RTX 2070 with Max-Q Design | 1F54 17AA 3FD0 | -| NVIDIA GeForce RTX 2060 | 1F55 | -| Quadro RTX 3000 | 1F76 | -| Matrox D-Series D2450 | 1F76 102B 2800 | -| Matrox D-Series D2480 | 1F76 102B 2900 | -| NVIDIA GeForce GTX 1650 | 1F82 | -| NVIDIA GeForce GTX 1630 | 1F83 | -| NVIDIA GeForce GTX 1650 | 1F91 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 103C 863E | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 103C 86E7 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 103C 86E8 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1043 12CF | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1043 156F | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1414 0032 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 144D C822 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1462 127E | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1462 1281 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1462 1284 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1462 1285 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1462 129C | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 17AA 229F | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 17AA 3802 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 17AA 3806 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 17AA 3F1A | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F91 1A58 1001 | -| NVIDIA GeForce GTX 1650 Ti | 1F95 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1025 1479 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1025 147A | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1025 147B | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1025 147C | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 103C 86E7 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 103C 86E8 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 103C 8815 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1043 1DFF | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1043 1E1F | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 144D C838 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1462 12BD | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1462 12C5 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1462 12D2 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 17AA 22C0 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 17AA 22C1 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 17AA 3837 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 17AA 3F95 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1A58 1003 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1A58 1006 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1A58 1007 | -| NVIDIA GeForce GTX 1650 Ti with Max-Q Design | 1F95 1E83 3E30 | -| NVIDIA GeForce GTX 1650 | 1F96 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F96 1462 1297 | -| NVIDIA GeForce MX450 | 1F97 | -| NVIDIA GeForce MX450 | 1F98 | -| NVIDIA GeForce GTX 1650 | 1F99 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1025 1479 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1025 147A | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1025 147B | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1025 147C | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 103C 8815 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1043 13B2 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1043 1402 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1043 1902 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1462 12BD | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1462 12C5 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1462 12D2 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 17AA 22DA | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 17AA 3F93 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F99 1E83 3E30 | -| NVIDIA GeForce MX450 | 1F9C | -| NVIDIA GeForce GTX 1650 | 1F9D | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1043 128D | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1043 130D | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1043 149C | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1043 185C | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1043 189C | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 12F4 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 1302 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 131B | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 1326 | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 132A | -| NVIDIA GeForce GTX 1650 with Max-Q Design | 1F9D 1462 132E | -| NVIDIA GeForce MX550 | 1F9F | -| NVIDIA GeForce MX550 | 1FA0 | -| NVIDIA T1000 | 1FB0 1028 12DB | -| NVIDIA T1000 | 1FB0 103C 12DB | -| NVIDIA T1000 | 1FB0 103C 8A80 | -| NVIDIA T1000 | 1FB0 10DE 12DB | -| NVIDIA DGX Display | 1FB0 10DE 1485 | -| NVIDIA T1000 | 1FB0 17AA 12DB | -| NVIDIA T600 | 1FB1 1028 1488 | -| NVIDIA T600 | 1FB1 103C 1488 | -| NVIDIA T600 | 1FB1 103C 8A80 | -| NVIDIA T600 | 1FB1 10DE 1488 | -| NVIDIA T600 | 1FB1 17AA 1488 | -| NVIDIA T400 | 1FB2 1028 1489 | -| NVIDIA T400 | 1FB2 103C 1489 | -| NVIDIA T400 | 1FB2 103C 8A80 | -| NVIDIA T400 | 1FB2 10DE 1489 | -| NVIDIA T400 | 1FB2 17AA 1489 | -| NVIDIA T600 Laptop GPU | 1FB6 | -| NVIDIA T550 Laptop GPU | 1FB7 | -| Quadro T2000 | 1FB8 | -| Quadro T2000 with Max-Q Design | 1FB8 1028 097E | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8736 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8738 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8772 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8780 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8782 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8783 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 8785 | -| Quadro T2000 with Max-Q Design | 1FB8 103C 87F0 | -| Quadro T2000 with Max-Q Design | 1FB8 1462 1281 | -| Quadro T2000 with Max-Q Design | 1FB8 1462 12BD | -| Quadro T2000 with Max-Q Design | 1FB8 17AA 22C0 | -| Quadro T2000 with Max-Q Design | 1FB8 17AA 22C1 | -| Quadro T1000 | 1FB9 | -| Quadro T1000 with Max-Q Design | 1FB9 1025 1479 | -| Quadro T1000 with Max-Q Design | 1FB9 1025 147A | -| Quadro T1000 with Max-Q Design | 1FB9 1025 147B | -| Quadro T1000 with Max-Q Design | 1FB9 1025 147C | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8736 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8738 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8772 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8780 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8782 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8783 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 8785 | -| Quadro T1000 with Max-Q Design | 1FB9 103C 87F0 | -| Quadro T1000 with Max-Q Design | 1FB9 1462 12BD | -| Quadro T1000 with Max-Q Design | 1FB9 17AA 22C0 | -| Quadro T1000 with Max-Q Design | 1FB9 17AA 22C1 | -| NVIDIA T600 Laptop GPU | 1FBA | -| NVIDIA T500 | 1FBB | -| NVIDIA T1200 Laptop GPU | 1FBC | -| NVIDIA GeForce GTX 1650 | 1FDD | -| NVIDIA T1000 8GB | 1FF0 1028 1612 | -| NVIDIA T1000 8GB | 1FF0 103C 1612 | -| NVIDIA T1000 8GB | 1FF0 103C 8A80 | -| NVIDIA T1000 8GB | 1FF0 10DE 1612 | -| NVIDIA T1000 8GB | 1FF0 17AA 1612 | -| NVIDIA T400 4GB | 1FF2 1028 1613 | -| NVIDIA T400 4GB | 1FF2 103C 1613 | -| NVIDIA T400E | 1FF2 103C 18FF | -| NVIDIA T400 4GB | 1FF2 103C 8A80 | -| NVIDIA T400 4GB | 1FF2 10DE 1613 | -| NVIDIA T400E | 1FF2 10DE 18FF | -| NVIDIA T400 4GB | 1FF2 17AA 1613 | -| NVIDIA T400E | 1FF2 17AA 18FF | -| Quadro T1000 | 1FF9 | -| NVIDIA A100-SXM4-40GB | 20B0 | -| NVIDIA A100-PG509-200 | 20B0 10DE 1450 | -| NVIDIA A100-SXM4-80GB | 20B2 10DE 1463 | -| NVIDIA A100-SXM4-80GB | 20B2 10DE 147F | -| NVIDIA A100-SXM4-80GB | 20B2 10DE 1622 | -| NVIDIA A100-SXM4-80GB | 20B2 10DE 1623 | -| NVIDIA PG509-210 | 20B2 10DE 1625 | -| NVIDIA A100-SXM-64GB | 20B3 10DE 14A7 | -| NVIDIA A100-SXM-64GB | 20B3 10DE 14A8 | -| NVIDIA A100 80GB PCIe | 20B5 10DE 1533 | -| NVIDIA A100 80GB PCIe | 20B5 10DE 1642 | -| NVIDIA PG506-232 | 20B6 10DE 1492 | -| NVIDIA A30 | 20B7 10DE 1532 | -| NVIDIA A30 | 20B7 10DE 1804 | -| NVIDIA A30 | 20B7 10DE 1852 | -| NVIDIA A800-SXM4-40GB | 20BD 10DE 17F4 | -| NVIDIA A100-PCIE-40GB | 20F1 10DE 145F | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 179B | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 179C | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 179D | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 179E | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 179F | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 17A0 | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 17A1 | -| NVIDIA A800-SXM4-80GB | 20F3 10DE 17A2 | -| NVIDIA A800 80GB PCIe | 20F5 10DE 1799 | -| NVIDIA A800 80GB PCIe LC | 20F5 10DE 179A | -| NVIDIA A800 40GB Active | 20F6 1028 180A | -| NVIDIA A800 40GB Active | 20F6 103C 180A | -| NVIDIA A800 40GB Active | 20F6 10DE 180A | -| NVIDIA A800 40GB Active | 20F6 17AA 180A | -| NVIDIA AX800 | 20FD 10DE 17F8 | -| NVIDIA GeForce GTX 1660 Ti | 2182 | -| NVIDIA GeForce GTX 1660 | 2184 | -| NVIDIA GeForce GTX 1650 SUPER | 2187 | -| NVIDIA GeForce GTX 1650 | 2188 | -| NVIDIA GeForce GTX 1660 Ti | 2191 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1028 0949 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 85FB | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 85FE | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 86D6 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 8741 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 8744 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 878D | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 87AF | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 103C 87B3 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1043 171F | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1043 17EF | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1043 18D1 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1414 0032 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 128A | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 128B | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 12C6 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 12CB | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 12CC | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 1462 12D9 | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 17AA 380C | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 17AA 381D | -| NVIDIA GeForce GTX 1660 Ti with Max-Q Design | 2191 17AA 381E | -| NVIDIA GeForce GTX 1650 Ti | 2192 | -| NVIDIA GeForce GTX 1660 SUPER | 21C4 | -| NVIDIA GeForce GTX 1660 Ti | 21D1 | -| NVIDIA GeForce RTX 3090 Ti | 2203 | -| NVIDIA GeForce RTX 3090 | 2204 | -| NVIDIA GeForce RTX 3080 | 2206 | -| NVIDIA GeForce RTX 3070 Ti | 2207 | -| NVIDIA GeForce RTX 3080 Ti | 2208 | -| NVIDIA GeForce RTX 3080 | 220A | -| NVIDIA CMP 90HX | 220D | -| NVIDIA GeForce RTX 3080 | 2216 | -| NVIDIA RTX A6000 | 2230 1028 1459 | -| NVIDIA RTX A6000 | 2230 103C 1459 | -| NVIDIA RTX A6000 | 2230 10DE 1459 | -| NVIDIA RTX A6000 | 2230 17AA 1459 | -| NVIDIA RTX A5000 | 2231 1028 147E | -| NVIDIA RTX A5000 | 2231 103C 147E | -| NVIDIA RTX A5000 | 2231 10DE 147E | -| NVIDIA RTX A5000 | 2231 17AA 147E | -| NVIDIA RTX A4500 | 2232 1028 163C | -| NVIDIA RTX A4500 | 2232 103C 163C | -| NVIDIA RTX A4500 | 2232 10DE 163C | -| NVIDIA RTX A4500 | 2232 17AA 163C | -| NVIDIA RTX A5500 | 2233 1028 165A | -| NVIDIA RTX A5500 | 2233 103C 165A | -| NVIDIA RTX A5500 | 2233 10DE 165A | -| NVIDIA RTX A5500 | 2233 17AA 165A | -| NVIDIA A40 | 2235 10DE 145A | -| NVIDIA A10 | 2236 10DE 1482 | -| NVIDIA A10G | 2237 10DE 152F | -| NVIDIA A10M | 2238 10DE 1677 | -| NVIDIA H100 NVL | 2321 10DE 1839 | -| NVIDIA H800 PCIe | 2322 10DE 17A4 | -| NVIDIA H800 | 2324 10DE 17A6 | -| NVIDIA H800 | 2324 10DE 17A8 | -| NVIDIA H20 | 2329 10DE 198B | -| NVIDIA H20 | 2329 10DE 198C | -| NVIDIA H100 80GB HBM3 | 2330 10DE 16C0 | -| NVIDIA H100 80GB HBM3 | 2330 10DE 16C1 | -| NVIDIA H100 PCIe | 2331 10DE 1626 | -| NVIDIA H100 | 2339 10DE 17FC | -| NVIDIA H800 NVL | 233A 10DE 183A | -| NVIDIA GH200 120GB | 2342 10DE 16EB | -| NVIDIA GH200 120GB | 2342 10DE 1805 | -| NVIDIA GH200 480GB | 2342 10DE 1809 | -| NVIDIA GeForce RTX 3060 Ti | 2414 | -| NVIDIA GeForce RTX 3080 Ti Laptop GPU | 2420 | -| NVIDIA RTX A5500 Laptop GPU | 2438 | -| NVIDIA GeForce RTX 3080 Ti Laptop GPU | 2460 | -| NVIDIA GeForce RTX 3070 Ti | 2482 | -| NVIDIA GeForce RTX 3070 | 2484 | -| NVIDIA GeForce RTX 3060 Ti | 2486 | -| NVIDIA GeForce RTX 3060 | 2487 | -| NVIDIA GeForce RTX 3070 | 2488 | -| NVIDIA GeForce RTX 3060 Ti | 2489 | -| NVIDIA CMP 70HX | 248A | -| NVIDIA GeForce RTX 3080 Laptop GPU | 249C | -| NVIDIA GeForce RTX 3060 Laptop GPU | 249C 1D05 1194 | -| NVIDIA GeForce RTX 3070 Laptop GPU | 249D | -| NVIDIA GeForce RTX 3070 Ti Laptop GPU | 24A0 | -| NVIDIA GeForce RTX 3060 Laptop GPU | 24A0 1D05 1192 | -| NVIDIA RTX A4000 | 24B0 1028 14AD | -| NVIDIA RTX A4000 | 24B0 103C 14AD | -| NVIDIA RTX A4000 | 24B0 10DE 14AD | -| NVIDIA RTX A4000 | 24B0 17AA 14AD | -| NVIDIA RTX A4000H | 24B1 10DE 1658 | -| NVIDIA RTX A5000 Laptop GPU | 24B6 | -| NVIDIA RTX A4000 Laptop GPU | 24B7 | -| NVIDIA RTX A3000 Laptop GPU | 24B8 | -| NVIDIA RTX A3000 12GB Laptop GPU | 24B9 | -| NVIDIA RTX A4500 Laptop GPU | 24BA | -| NVIDIA RTX A3000 12GB Laptop GPU | 24BB | -| NVIDIA GeForce RTX 3060 | 24C7 | -| NVIDIA GeForce RTX 3060 Ti | 24C9 | -| NVIDIA GeForce RTX 3080 Laptop GPU | 24DC | -| NVIDIA GeForce RTX 3070 Laptop GPU | 24DD | -| NVIDIA GeForce RTX 3070 Ti Laptop GPU | 24E0 | -| NVIDIA RTX A4500 Embedded GPU | 24FA | -| NVIDIA GeForce RTX 3060 | 2503 | -| NVIDIA GeForce RTX 3060 | 2504 | -| NVIDIA GeForce RTX 3050 | 2507 | -| NVIDIA GeForce RTX 3050 OEM | 2508 | -| NVIDIA GeForce RTX 3060 Laptop GPU | 2520 | -| NVIDIA GeForce RTX 3060 Laptop GPU | 2521 | -| NVIDIA GeForce RTX 3050 Ti Laptop GPU | 2523 | -| NVIDIA RTX A2000 | 2531 1028 151D | -| NVIDIA RTX A2000 | 2531 103C 151D | -| NVIDIA RTX A2000 | 2531 10DE 151D | -| NVIDIA RTX A2000 | 2531 17AA 151D | -| NVIDIA GeForce RTX 3060 | 2544 | -| NVIDIA GeForce RTX 3060 Laptop GPU | 2560 | -| NVIDIA GeForce RTX 3050 Ti Laptop GPU | 2563 | -| NVIDIA RTX A2000 12GB | 2571 1028 1611 | -| NVIDIA RTX A2000 12GB | 2571 103C 1611 | -| NVIDIA RTX A2000 12GB | 2571 10DE 1611 | -| NVIDIA RTX A2000 12GB | 2571 17AA 1611 | -| NVIDIA GeForce RTX 3050 | 2582 | -| NVIDIA GeForce RTX 3050 | 2584 | -| NVIDIA GeForce RTX 3050 Ti Laptop GPU | 25A0 | -| NVIDIA GeForce RTX 3050Ti Laptop GPU | 25A0 103C 8928 | -| NVIDIA GeForce RTX 3050Ti Laptop GPU | 25A0 103C 89F9 | -| NVIDIA GeForce RTX 3060 Laptop GPU | 25A0 1D05 1196 | -| NVIDIA GeForce RTX 3050 Laptop GPU | 25A2 | -| NVIDIA GeForce RTX 3050 Ti Laptop GPU | 25A2 1028 0BAF | -| NVIDIA GeForce RTX 3060 Laptop GPU | 25A2 1D05 1195 | -| NVIDIA GeForce RTX 3050 Laptop GPU | 25A5 | -| NVIDIA GeForce MX570 | 25A6 | -| NVIDIA GeForce RTX 2050 | 25A7 | -| NVIDIA GeForce RTX 2050 | 25A9 | -| NVIDIA GeForce MX570 A | 25AA | -| NVIDIA GeForce RTX 3050 4GB Laptop GPU | 25AB | -| NVIDIA GeForce RTX 3050 6GB Laptop GPU | 25AC | -| NVIDIA GeForce RTX 2050 | 25AD | -| NVIDIA RTX A1000 | 25B0 1028 1878 | -| NVIDIA RTX A1000 | 25B0 103C 1878 | -| NVIDIA RTX A1000 | 25B0 10DE 1878 | -| NVIDIA RTX A1000 | 25B0 17AA 1878 | -| NVIDIA RTX A400 | 25B2 1028 1879 | -| NVIDIA RTX A400 | 25B2 103C 1879 | -| NVIDIA RTX A400 | 25B2 10DE 1879 | -| NVIDIA RTX A400 | 25B2 17AA 1879 | -| NVIDIA A16 | 25B6 10DE 14A9 | -| NVIDIA A2 | 25B6 10DE 157E | -| NVIDIA RTX A2000 Laptop GPU | 25B8 | -| NVIDIA RTX A1000 Laptop GPU | 25B9 | -| NVIDIA RTX A2000 8GB Laptop GPU | 25BA | -| NVIDIA RTX A500 Laptop GPU | 25BB | -| NVIDIA RTX A1000 6GB Laptop GPU | 25BC | -| NVIDIA RTX A500 Laptop GPU | 25BD | -| NVIDIA GeForce RTX 3050 Ti Laptop GPU | 25E0 | -| NVIDIA GeForce RTX 3050 Laptop GPU | 25E2 | -| NVIDIA GeForce RTX 3050 Laptop GPU | 25E5 | -| NVIDIA GeForce RTX 3050 6GB Laptop GPU | 25EC | -| NVIDIA GeForce RTX 2050 | 25ED | -| NVIDIA RTX A1000 Embedded GPU | 25F9 | -| NVIDIA RTX A2000 Embedded GPU | 25FA | -| NVIDIA RTX A500 Embedded GPU | 25FB | -| NVIDIA GeForce RTX 4090 | 2684 | -| NVIDIA GeForce RTX 4090 D | 2685 | -| NVIDIA RTX 6000 Ada Generation | 26B1 1028 16A1 | -| NVIDIA RTX 6000 Ada Generation | 26B1 103C 16A1 | -| NVIDIA RTX 6000 Ada Generation | 26B1 10DE 16A1 | -| NVIDIA RTX 6000 Ada Generation | 26B1 17AA 16A1 | -| NVIDIA RTX 5000 Ada Generation | 26B2 1028 17FA | -| NVIDIA RTX 5000 Ada Generation | 26B2 103C 17FA | -| NVIDIA RTX 5000 Ada Generation | 26B2 10DE 17FA | -| NVIDIA RTX 5000 Ada Generation | 26B2 17AA 17FA | -| NVIDIA RTX 5880 Ada Generation | 26B3 1028 1934 | -| NVIDIA RTX 5880 Ada Generation | 26B3 103C 1934 | -| NVIDIA RTX 5880 Ada Generation | 26B3 10DE 1934 | -| NVIDIA RTX 5880 Ada Generation | 26B3 17AA 1934 | -| NVIDIA L40 | 26B5 10DE 169D | -| NVIDIA L40 | 26B5 10DE 17DA | -| NVIDIA L40S | 26B9 10DE 1851 | -| NVIDIA L40S | 26B9 10DE 18CF | -| NVIDIA L20 | 26BA 10DE 1957 | -| NVIDIA GeForce RTX 4080 SUPER | 2702 | -| NVIDIA GeForce RTX 4080 | 2704 | -| NVIDIA GeForce RTX 4070 Ti SUPER | 2705 | -| NVIDIA GeForce RTX 4070 | 2709 | -| NVIDIA GeForce RTX 4090 Laptop GPU | 2717 | -| NVIDIA RTX 5000 Ada Generation Laptop GPU | 2730 | -| NVIDIA GeForce RTX 4090 Laptop GPU | 2757 | -| NVIDIA RTX 5000 Ada Generation Embedded GPU | 2770 | -| NVIDIA GeForce RTX 4070 Ti | 2782 | -| NVIDIA GeForce RTX 4070 SUPER | 2783 | -| NVIDIA GeForce RTX 4070 | 2786 | -| NVIDIA GeForce RTX 4060 Ti | 2788 | -| NVIDIA GeForce RTX 4080 Laptop GPU | 27A0 | -| NVIDIA RTX 4000 SFF Ada Generation | 27B0 1028 16FA | -| NVIDIA RTX 4000 SFF Ada Generation | 27B0 103C 16FA | -| NVIDIA RTX 4000 SFF Ada Generation | 27B0 10DE 16FA | -| NVIDIA RTX 4000 SFF Ada Generation | 27B0 17AA 16FA | -| NVIDIA RTX 4500 Ada Generation | 27B1 1028 180C | -| NVIDIA RTX 4500 Ada Generation | 27B1 103C 180C | -| NVIDIA RTX 4500 Ada Generation | 27B1 10DE 180C | -| NVIDIA RTX 4500 Ada Generation | 27B1 17AA 180C | -| NVIDIA RTX 4000 Ada Generation | 27B2 1028 181B | -| NVIDIA RTX 4000 Ada Generation | 27B2 103C 181B | -| NVIDIA RTX 4000 Ada Generation | 27B2 10DE 181B | -| NVIDIA RTX 4000 Ada Generation | 27B2 17AA 181B | -| NVIDIA L2 | 27B6 10DE 1933 | -| NVIDIA L4 | 27B8 10DE 16CA | -| NVIDIA L4 | 27B8 10DE 16EE | -| NVIDIA RTX 4000 Ada Generation Laptop GPU | 27BA | -| NVIDIA RTX 3500 Ada Generation Laptop GPU | 27BB | -| NVIDIA GeForce RTX 4080 Laptop GPU | 27E0 | -| NVIDIA RTX 3500 Ada Generation Embedded GPU | 27FB | -| NVIDIA GeForce RTX 4060 Ti | 2803 | -| NVIDIA GeForce RTX 4060 Ti | 2805 | -| NVIDIA GeForce RTX 4060 | 2808 | -| NVIDIA GeForce RTX 4070 Laptop GPU | 2820 | -| NVIDIA RTX 3000 Ada Generation Laptop GPU | 2838 | -| NVIDIA GeForce RTX 4070 Laptop GPU | 2860 | -| NVIDIA GeForce RTX 4060 | 2882 | -| NVIDIA GeForce RTX 4060 Laptop GPU | 28A0 | -| NVIDIA GeForce RTX 4050 Laptop GPU | 28A1 | -| NVIDIA RTX 2000 Ada Generation | 28B0 1028 1870 | -| NVIDIA RTX 2000 Ada Generation | 28B0 103C 1870 | -| NVIDIA RTX 2000E Ada Generation | 28B0 103C 1871 | -| NVIDIA RTX 2000 Ada Generation | 28B0 10DE 1870 | -| NVIDIA RTX 2000E Ada Generation | 28B0 10DE 1871 | -| NVIDIA RTX 2000 Ada Generation | 28B0 17AA 1870 | -| NVIDIA RTX 2000E Ada Generation | 28B0 17AA 1871 | -| NVIDIA RTX 2000 Ada Generation Laptop GPU | 28B8 | -| NVIDIA RTX 1000 Ada Generation Laptop GPU | 28B9 | -| NVIDIA RTX 500 Ada Generation Laptop GPU | 28BA | -| NVIDIA RTX 500 Ada Generation Laptop GPU | 28BB | -| NVIDIA GeForce RTX 4060 Laptop GPU | 28E0 | -| NVIDIA GeForce RTX 4050 Laptop GPU | 28E1 | -| NVIDIA RTX 2000 Ada Generation Embedded GPU | 28F8 | +# NVIDIA Linux Open GPU with P2P support + +This is a fork of NVIDIA's driver with P2P support added for 4090's. + +`./install.sh` to install if that's all you want. + +You may need to uninstall the driver from DKMS. Your system needs large BAR support and IOMMU off. + +Not sure all the cache flushes are right, please file issues on here if you find any issues. + +NOTE: This is not a hack, this is using PCIe according to the spec. With cleanups, this could potentially be upstreamed. + +## How it works + +Normally, P2P on NVIDIA cards uses MAILBOXP2P. This is some hardware interface designed to allow GPUs to transfer memory back in the days of small BAR. It is not present or disabled in hardware on the 4090s, and that's why P2P doesn't work. There [was a bug in early versions](https://forums.developer.nvidia.com/t/standard-nvidia-cuda-tests-fail-with-dual-rtx-4090-linux-box/233202) of the driver that reported that it did work, and it was actually sending stuff on the PCIe bus. However, because the mailbox hardware wasn't present, these copies wouldn't go to the right place. You could even crash the system by doing something like `torch.zeros(10000,10000).cuda().to("cuda:1")` + +In some 3090s and all 4090s, NVIDIA added large BAR support. + +``` +tiny@tiny14:~$ lspci -s 01:00.0 -v +01:00.0 VGA compatible controller: NVIDIA Corporation AD102 [GeForce RTX 4090] (rev a1) (prog-if 00 [VGA controller]) + Subsystem: Micro-Star International Co., Ltd. [MSI] Device 510b + Physical Slot: 49 + Flags: bus master, fast devsel, latency 0, IRQ 377 + Memory at b2000000 (32-bit, non-prefetchable) [size=16M] + Memory at 28800000000 (64-bit, prefetchable) [size=32G] + Memory at 28400000000 (64-bit, prefetchable) [size=32M] + I/O ports at 3000 [size=128] + Expansion ROM at b3000000 [virtual] [disabled] [size=512K] + Capabilities: + Kernel driver in use: nvidia + Kernel modules: nvidiafb, nouveau, nvidia_drm, nvidia +``` + +Notice how BAR1 is size 32G. In H100, they also added support for a PCIe mode that uses the BAR directly instead of the mailboxes, called BAR1P2P. So, what happens if we try to enable that on a 4090? + +We do this by bypassing the HAL and calling a bunch of the GH100 methods directly. Methods like `kbusEnableStaticBar1Mapping_GH100`, which maps the entire VRAM into BAR1. This mostly just works, but we had to disable the use of that region in the `MapAperture` function for some reason. Shouldn't matter. + +``` +[ 3491.654009] NVRM: kbusEnableStaticBar1Mapping_GH100: Static bar1 mapped offset 0x0 size 0x5e9200000 +[ 3491.793389] NVRM: kbusEnableStaticBar1Mapping_GH100: Static bar1 mapped offset 0x0 size 0x5e9200000 +``` + +Perfect, we now have the VRAM mapped. However, it's not that easy to get P2P. When you run `./simpleP2P` from `cuda-samples`, you get this error. + +``` +[ 3742.840689] NVRM: kbusCreateP2PMappingForBar1P2P_GH100: added PCIe BAR1 P2P mapping between GPU2 and GPU3 +[ 3742.840762] NVRM: kbusCreateP2PMappingForBar1P2P_GH100: added PCIe BAR1 P2P mapping between GPU3 and GPU2 +[ 3742.841089] NVRM: nvAssertFailed: Assertion failed: (shifted >> pField->shift) == value @ field_desc.h:272 +[ 3742.841106] NVRM: nvAssertFailed: Assertion failed: (shifted & pField->maskPos) == shifted @ field_desc.h:273 +[ 3742.841281] NVRM: nvAssertFailed: Assertion failed: (shifted >> pField->shift) == value @ field_desc.h:272 +[ 3742.841292] NVRM: nvAssertFailed: Assertion failed: (shifted & pField->maskPos) == shifted @ field_desc.h:273 +[ 3742.865948] NVRM: GPU at PCI:0000:01:00: GPU-49c7a6c9-e3a8-3b48-f0ba-171520d77dd1 +[ 3742.865956] NVRM: Xid (PCI:0000:01:00): 31, pid=21804, name=simpleP2P, Ch 00000013, intr 00000000. MMU Fault: ENGINE CE3 HUBCLIENT_CE1 faulted @ 0x7f97_94000000. Fault is of type FAULT_INFO_TYPE_UNSUPPORTED_KIND ACCESS_TYPE_VIRT_WRITE +``` + +Failing with an MMU fault. So you dive into this and find that it's using `GMMU_APERTURE_PEER` as the mapping type. That doesn't seem supported in the 4090. So let's see what types are supported, `GMMU_APERTURE_VIDEO`,`GMMU_APERTURE_SYS_NONCOH`, and `GMMU_APERTURE_SYS_COH`. We don't care about being coherent with the CPU's L2 cache, but it does have to go out the PCIe bus, so we rewrite `GMMU_APERTURE_PEER` to `GMMU_APERTURE_SYS_NONCOH`. We also no longer set the peer id that was corrupting the page table. + +``` +cudaMemcpyPeer / cudaMemcpy between GPU0 and GPU1: 24.21GB/s +Preparing host buffer and memcpy to GPU0... +Run kernel on GPU1, taking source data from GPU0 and writing to GPU1... +Run kernel on GPU0, taking source data from GPU1 and writing to GPU0... +Copy data back to host from GPU0 and verify results... +Verification error @ element 1: val = 0.000000, ref = 4.000000 +Verification error @ element 2: val = 0.000000, ref = 8.000000 +``` + +Progress! `./simpleP2P` appears to work, however the copy isn't happening. The address is likely wrong. It turns out they have a separate field for the peer address called `fldAddrPeer`, we change that to `fldAddrSysmem`. We also print out the addresses and note that the physical BAR address isn't being added properly, they provide a field `fabricBaseAddress` for `GMMU_APERTURE_PEER`, we reuse it and put the `BAR1` base address in there. + +That's it. Thanks to NVIDIA for writing such a stable driver. And with this, the tinybox green is even better. + +~ the tiny corp + +## Functional + +``` +Enabling peer access between GPU0 and GPU1... +Allocating buffers (64MB on GPU0, GPU1 and CPU Host)... +Creating event handles... +cudaMemcpyPeer / cudaMemcpy between GPU0 and GPU1: 24.44GB/s +Preparing host buffer and memcpy to GPU0... +Run kernel on GPU1, taking source data from GPU0 and writing to GPU1... +Run kernel on GPU0, taking source data from GPU1 and writing to GPU0... +Copy data back to host from GPU0 and verify results... +Disabling peer access... +Shutting down... +Test passed +``` + +## Fast + +``` +Bidirectional P2P=Enabled Bandwidth Matrix (GB/s) + D\D 0 1 2 3 4 5 + 0 919.39 50.11 50.15 51.22 50.59 51.22 + 1 50.19 921.29 50.31 51.21 50.62 51.22 + 2 50.23 50.55 921.83 51.22 50.39 51.22 + 3 50.33 50.65 51.20 920.20 50.43 51.22 + 4 50.18 50.68 50.26 51.22 922.30 51.23 + 5 50.12 50.09 50.44 51.22 51.21 921.29 +``` + +## And NCCL (aka torch) compatible! + +``` +tiny@tiny14:~/build/nccl-tests/build$ ./all_reduce_perf -g 6 +# nThread 1 nGpus 6 minBytes 33554432 maxBytes 33554432 step: 1048576(bytes) warmup iters: 5 iters: 20 agg iters: 1 validation: 1 graph: 0 +# +# Using devices +# Rank 0 Group 0 Pid 26230 on tiny14 device 0 [0x01] NVIDIA GeForce RTX 4090 +# Rank 1 Group 0 Pid 26230 on tiny14 device 1 [0x42] NVIDIA GeForce RTX 4090 +# Rank 2 Group 0 Pid 26230 on tiny14 device 2 [0x81] NVIDIA GeForce RTX 4090 +# Rank 3 Group 0 Pid 26230 on tiny14 device 3 [0x82] NVIDIA GeForce RTX 4090 +# Rank 4 Group 0 Pid 26230 on tiny14 device 4 [0xc1] NVIDIA GeForce RTX 4090 +# Rank 5 Group 0 Pid 26230 on tiny14 device 5 [0xc2] NVIDIA GeForce RTX 4090 +# +# out-of-place in-place +# size count type redop root time algbw busbw #wrong time algbw busbw #wrong +# (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s) + 33554432 8388608 float sum -1 2275.1 14.75 24.58 0 2282.5 14.70 24.50 0 +# Out of bounds values : 0 OK +# Avg bus bandwidth : 24.5413 +# +``` diff --git a/install.sh b/install.sh new file mode 100755 index 00000000..d11f1edd --- /dev/null +++ b/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash +sudo rmmod nvidia_drm nvidia_modeset nvidia_uvm nvidia +set -e +make modules -j$(nproc) +sudo make modules_install -j$(nproc) +sudo depmod +nvidia-smi diff --git a/kernel-open/nvidia/nv-pci.c b/kernel-open/nvidia/nv-pci.c index 40cf248c..d456d927 100644 --- a/kernel-open/nvidia/nv-pci.c +++ b/kernel-open/nvidia/nv-pci.c @@ -177,11 +177,11 @@ static int nv_resize_pcie_bars(struct pci_dev *pci_dev) { struct pci_host_bridge *host; #endif - if (NVreg_EnableResizableBar == 0) + /*if (NVreg_EnableResizableBar == 0) { nv_printf(NV_DBG_INFO, "NVRM: resizable BAR disabled by regkey, skipping\n"); return 0; - } + }*/ // Check if BAR1 has PCIe rebar capabilities sizes = pci_rebar_get_possible_sizes(pci_dev, NV_GPU_BAR1); @@ -397,7 +397,7 @@ nv_init_coherent_link_info /* Fail for the baremetal case */ goto failed; } - + /* * For the virtualization usecase on SHH, the coherent GPU memory * PA is exposed as BAR2 to the VM and the "nvidia,gpu-mem-base-pa" @@ -540,7 +540,7 @@ nv_pci_probe goto failed; } - if (pci_dev->dev.bus->iommu_ops == NULL) + if (pci_dev->dev.bus->iommu_ops == NULL) { nv = NV_STATE_PTR(nvl); if (rm_is_iommu_needed_for_sriov(sp, nv)) @@ -1021,7 +1021,7 @@ nv_pci_remove(struct pci_dev *pci_dev) nv_printf(NV_DBG_ERRORS, "NVRM: Failed removal of device %04x:%02x:%02x.%x!\n", NV_PCI_DOMAIN_NUMBER(pci_dev), NV_PCI_BUS_NUMBER(pci_dev), - NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn)); + NV_PCI_SLOT_NUMBER(pci_dev), PCI_FUNC(pci_dev->devfn)); WARN_ON(1); goto done; } diff --git a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c index 39cc8812..efd4cda8 100644 --- a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c +++ b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c @@ -709,7 +709,7 @@ _kbifInitRegistryOverrides NvU32 data32; // P2P Override - pKernelBif->p2pOverride = BIF_P2P_NOT_OVERRIDEN; + pKernelBif->p2pOverride = 0x11; if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK) { pKernelBif->p2pOverride = data32; @@ -718,7 +718,7 @@ _kbifInitRegistryOverrides } // P2P force type override - pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT; + pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_BAR1P2P; if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FORCE_P2P_TYPE, &data32) == NV_OK && (data32 <= NV_REG_STR_RM_FORCE_P2P_TYPE_MAX)) { diff --git a/src/nvidia/src/kernel/gpu/bus/arch/hopper/kern_bus_gh100.c b/src/nvidia/src/kernel/gpu/bus/arch/hopper/kern_bus_gh100.c index dcac5b9e..6423745f 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/hopper/kern_bus_gh100.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/hopper/kern_bus_gh100.c @@ -333,7 +333,7 @@ kbusVerifyBar2_GH100 { memdescCreateExisting(&memDesc, pGpu, size, ADDR_FBMEM, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); } - memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_95, + memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_95, (&memDesc)); if (status != NV_OK) { @@ -765,7 +765,7 @@ kbusCreateP2PMapping_GH100 if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes)) { - return kbusCreateP2PMappingForBar1P2P_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes); + return kbusCreateP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes); } if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes)) @@ -816,7 +816,7 @@ kbusRemoveP2PMapping_GH100 if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes)) { - return kbusRemoveP2PMappingForBar1P2P_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes); + return kbusRemoveP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes); } if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes)) @@ -1139,7 +1139,7 @@ kbusIsBar1P2PCapable_GH100 // // GPU BAR1 supports the SYSMEM mapping for the bar1 doorbell, RM needs // to make sure that BAR1 VA has 128KB space left for such cases after all - // FB statically mapped in BAR1. Bug 3869651 #14. + // FB statically mapped in BAR1. Bug 3869651 #14. // if ((bar1VASize < (32 * RM_PAGE_SIZE)) || ((bar1VASize - (32 * RM_PAGE_SIZE)) < RM_ALIGN_UP(fbSize, RM_PAGE_SIZE_2M))) @@ -1448,7 +1448,7 @@ kbusCreateP2PMappingForBar1P2P_GH100 return NV_ERR_NOT_SUPPORTED; } - if (!kbusIsPcieBar1P2PMappingSupported_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1)) + if (!kbusIsPcieBar1P2PMappingSupported_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1)) { return NV_ERR_NOT_SUPPORTED; } @@ -1871,8 +1871,8 @@ kbusEnableStaticBar1Mapping_GH100 pKernelBus->bar1[gfid].staticBar1.pDmaMemDesc = pDmaMemDesc; pKernelBus->bar1[gfid].staticBar1.size = bar1MapSize; - NV_PRINTF(LEVEL_INFO, "Static bar1 mapped offset 0x%llx size 0x%llx\n", - bar1Offset, bar1MapSize); + NV_PRINTF(LEVEL_ERROR, "Static bar1 mapped offset 0x%llx size 0x%llx bar1BusAddr 0x%llx\n", + bar1Offset, bar1MapSize, bar1BusAddr); return NV_OK; diff --git a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c index 7565bf41..8a634b4c 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c @@ -1008,7 +1008,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) // NV_ASSERT(pKernelBus->bar1[gfid].apertureLength <= kbusGetPciBarSize(pKernelBus, 1)); - bBar1P2PCapable = kbusIsBar1P2PCapable(pGpu, pKernelBus, gfid); + bBar1P2PCapable = kbusIsBar1P2PCapable_GH100(pGpu, pKernelBus, gfid); // // If we need to preserve a console mapping at the start of BAR1, we @@ -1094,7 +1094,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid) { // Enable the static BAR1 mapping for the BAR1 P2P NV_ASSERT_OK_OR_GOTO(rmStatus, - kbusEnableStaticBar1Mapping_HAL(pGpu, pKernelBus, gfid), + kbusEnableStaticBar1Mapping_GH100(pGpu, pKernelBus, gfid), kbusInitBar1_failed); } else @@ -1605,7 +1605,7 @@ kbusSetupBar2GpuVaSpace_GM107 NV_ASSERT_OR_RETURN(status == NV_OK, status); } - memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_63, + memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_63, pKernelBus->bar2[gfid].pInstBlkMemDesc); NV_ASSERT_OR_RETURN(status == NV_OK, status); @@ -1687,7 +1687,7 @@ kbusSetupBar2GpuVaSpace_GM107 if (IS_GFID_VF(gfid)) { - memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_64, + memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_64, pPageLevelsMemDesc); NV_ASSERT_OR_GOTO(status == NV_OK, cleanup); @@ -3013,7 +3013,7 @@ _kbusMapAperture_GM107 NvU32 swizzId = KMIGMGR_SWIZZID_INVALID; NvU32 gfid; - if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) && + /*if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) && (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)) { NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); @@ -3021,7 +3021,7 @@ _kbusMapAperture_GM107 return kbusGetStaticFbAperture_HAL(pGpu, pKernelBus, pMemDesc, offset, pAperOffset, pLength, gfid); - } + }*/ // Ensure that the BAR1 VA space is the same across all subdevices if (IsSLIEnabled(pGpu) && ((mapFlags & BUS_MAP_FB_FLAGS_MAP_UNICAST) == 0)) @@ -3136,12 +3136,12 @@ _kbusUnmapAperture_GM107 NV_STATUS rmStatus = NV_OK; VirtMemAllocator *pDma = GPU_GET_DMA(pGpu); - if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) && + /*if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) && (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)) { // No op for the static bar1 mode return NV_OK; - } + }*/ rmStatus = dmaFreeMapping_HAL(pGpu, pDma, pVAS, aperOffset, pMemDesc, 0, NULL); @@ -3749,7 +3749,7 @@ kbusVerifyBar2_GM107 { memdescCreateExisting(&memDesc, pGpu, size, ADDR_FBMEM, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); } - memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_65, + memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_65, (&memDesc)); if (status != NV_OK) { @@ -4805,7 +4805,7 @@ kbusBar1InstBlkVasUpdate_GM107 NV_ASSERT_OR_RETURN(status == NV_OK, status); } - memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_66, + memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_66, pKernelBus->bar1[gfid].pInstBlkMemDesc); NV_ASSERT_OR_RETURN(status == NV_OK, status); diff --git a/src/nvidia/src/kernel/gpu/bus/arch/pascal/kern_bus_gp100.c b/src/nvidia/src/kernel/gpu/bus/arch/pascal/kern_bus_gp100.c index d9a04068..8af18d72 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/pascal/kern_bus_gp100.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/pascal/kern_bus_gp100.c @@ -70,6 +70,11 @@ kbusCreateP2PMapping_GP100 return kbusCreateP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes); } + if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes)) + { + return kbusCreateP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes); + } + NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes)); return NV_ERR_NOT_SUPPORTED; diff --git a/src/nvidia/src/kernel/gpu/bus/p2p_api.c b/src/nvidia/src/kernel/gpu/bus/p2p_api.c index 3d6e1bed..59327bba 100644 --- a/src/nvidia/src/kernel/gpu/bus/p2p_api.c +++ b/src/nvidia/src/kernel/gpu/bus/p2p_api.c @@ -575,20 +575,20 @@ p2papiConstruct_IMPL // // TODO: This function need to have a cleanup path when this function // fails after kbusCreateP2PMaping(), busBindLocalGfidForP2P() - // and busBindRemoteGfidForP2P(). The current state, the + // and busBindRemoteGfidForP2P(). The current state, the // function just returns an error. Bug 4016670 filed to track // the effort. // // setup the p2p resources NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, - kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu, + kbusCreateP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu, pRemoteKernelBus, &peer1, &peer2, pP2PApi->attributes)); if (bEgmPeer) { NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, - kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu, + kbusCreateP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu, pRemoteKernelBus, &egmPeer1, &egmPeer2, pP2PApi->attributes | DRF_DEF(_P2PAPI, _ATTRIBUTES, _REMOTE_EGM, _YES))); @@ -598,14 +598,14 @@ p2papiConstruct_IMPL (pCallContext->secInfo.privLevel >= RS_PRIV_LEVEL_KERNEL)) { NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, - kbusGetBar1P2PDmaInfo_HAL(pLocalGpu, pRemoteGpu, + kbusGetBar1P2PDmaInfo_GH100(pLocalGpu, pRemoteGpu, pRemoteKernelBus, &pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_address, &pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_size)); NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, - kbusGetBar1P2PDmaInfo_HAL(pRemoteGpu, pLocalGpu, - pLocalKernelBus, + kbusGetBar1P2PDmaInfo_GH100(pRemoteGpu, pLocalGpu, + pLocalKernelBus, &pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_address, &pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_size)); } @@ -740,7 +740,7 @@ p2papiDestruct_IMPL { // remove any resources associated with this mapping NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, - kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus, + kbusRemoveP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu, pRemoteKernelBus, pP2PApi->peerId1, pP2PApi->peerId2, pP2PApi->attributes), end); @@ -749,7 +749,7 @@ p2papiDestruct_IMPL memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pRemoteGpu)) && !GPU_IS_NVSWITCH_DETECTED(pLocalGpu)) { - status = kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus, + status = kbusRemoveP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu, pRemoteKernelBus, pP2PApi->egmPeerId1, pP2PApi->egmPeerId2, pP2PApi->attributes | diff --git a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c index a7d5c134..245ca7c5 100644 --- a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c +++ b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c @@ -3058,7 +3058,8 @@ nvGpuOpsBuildExternalAllocPtes NvBool isIndirectPeerSupported, NvBool isPeerSupported, NvU32 peerId, - gpuExternalMappingInfo *pGpuExternalMappingInfo + gpuExternalMappingInfo *pGpuExternalMappingInfo, + RmPhysAddr bar1BusAddr ) { NV_STATUS status = NV_OK; @@ -3253,7 +3254,11 @@ nvGpuOpsBuildExternalAllocPtes if (nvFieldIsValid32(&pPteFmt->fldAtomicDisable.desc)) nvFieldSetBool(&pPteFmt->fldAtomicDisable, !atomic, pte.v8); - gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); + if (aperture == GMMU_APERTURE_PEER) { + gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_NONCOH, pte.v8); + } else { + gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); + } if (!isCompressedKind) { @@ -3264,7 +3269,11 @@ nvGpuOpsBuildExternalAllocPtes } } - if (aperture == GMMU_APERTURE_PEER) + if (aperture == GMMU_APERTURE_PEER) { + fabricBaseAddress = bar1BusAddr; + } + + /*if (aperture == GMMU_APERTURE_PEER) { FlaMemory* pFlaMemory = dynamicCast(pMemory, FlaMemory); nvFieldSet32(&pPteFmt->fldPeerIndex, peerId, pte.v8); @@ -3303,7 +3312,7 @@ nvGpuOpsBuildExternalAllocPtes } } } - } + }*/ // // Both memdescGetPhysAddr() and kgmmuEncodePhysAddr() have pretty high overhead. @@ -3458,6 +3467,7 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace, Memory *pMemory = NULL; PMEMORY_DESCRIPTOR pMemDesc = NULL; OBJGPU *pMappingGpu = NULL; + RmPhysAddr bar1BusAddr = 0; NvU32 peerId = 0; NvBool isSliSupported = NV_FALSE; NvBool isPeerSupported = NV_FALSE; @@ -3603,6 +3613,8 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace, &peerId); if (status != NV_OK) goto freeGpaMemdesc; + + bar1BusAddr = gpumgrGetGpuPhysFbAddr(pAdjustedMemDesc->pGpu); } // @@ -3680,7 +3692,7 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace, status = nvGpuOpsBuildExternalAllocPtes(pVAS, pMappingGpu, pAdjustedMemDesc, pMemory, offset, size, isIndirectPeerSupported, isPeerSupported, peerId, - pGpuExternalMappingInfo); + pGpuExternalMappingInfo, bar1BusAddr); freeGpaMemdesc: if (pAdjustedMemDesc != pMemDesc) @@ -9329,7 +9341,7 @@ NV_STATUS nvGpuOpsGetChannelResourcePtes(struct gpuAddressSpace *vaSpace, status = nvGpuOpsBuildExternalAllocPtes(pVAS, pMappingGpu, pMemDesc, NULL, offset, size, NV_FALSE, NV_FALSE, - 0, pGpuExternalMappingInfo); + 0, pGpuExternalMappingInfo, 0); _nvGpuOpsLocksRelease(&acquiredLocks); threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); diff --git a/src/nvidia/src/libraries/mmu/gmmu_fmt.c b/src/nvidia/src/libraries/mmu/gmmu_fmt.c index b12e8d0b..4a9f4882 100644 --- a/src/nvidia/src/libraries/mmu/gmmu_fmt.c +++ b/src/nvidia/src/libraries/mmu/gmmu_fmt.c @@ -124,9 +124,8 @@ gmmuFmtPtePhysAddrFld { case GMMU_APERTURE_SYS_COH: case GMMU_APERTURE_SYS_NONCOH: - return &pPte->fldAddrSysmem; case GMMU_APERTURE_PEER: - return &pPte->fldAddrPeer; + return &pPte->fldAddrSysmem; case GMMU_APERTURE_VIDEO: return &pPte->fldAddrVidmem; default: