From f4348d8d175c91edda6fd0737ba0dcc0ab593699 Mon Sep 17 00:00:00 2001
From: Ryan Herbst <rherbst@slac.stanford.edu>
Date: Wed, 14 Feb 2024 11:22:17 -0800
Subject: [PATCH] Update for nvidia open driver

---
 data_gpu/driver/Makefile                    | 12 ++++------
 data_gpu/driver/README.md                   | 26 ++++++++++++++-------
 data_gpu/driver/nvidia-ko-to-module-symvers | 24 -------------------
 3 files changed, 23 insertions(+), 39 deletions(-)
 delete mode 100755 data_gpu/driver/nvidia-ko-to-module-symvers

diff --git a/data_gpu/driver/Makefile b/data_gpu/driver/Makefile
index 9601c62..34d01ef 100644
--- a/data_gpu/driver/Makefile
+++ b/data_gpu/driver/Makefile
@@ -13,16 +13,19 @@
 # contained in the LICENSE.txt file.
 # ----------------------------------------------------------------------------
 
+NVIDIA_DRIVERS := /usr/src/nvidia-535.154.05
+
 NAME := datagpu
 HOME := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 KVER := $(shell uname -r)
 ARCH := $(shell uname -m)
 CROSS_COMPILE :=
+KBUILD_MODPOST_WARN := 1
 KERNELDIR := /lib/modules/$(KVER)/build
 SRCS := $(wildcard src/*.c)
 OBJS := $(patsubst %.c,%.o,$(SRCS))
 
-#KBUILD_EXTRA_SYMBOLS := $(HOME)/nvidia.symvers
+KBUILD_EXTRA_SYMBOLS := $(NVIDIA_DRIVERS)/Module.symvers
 
 ifndef GITV
 	GITT := $(shell cd $(HOME); git describe --tags)
@@ -32,7 +35,7 @@ endif
 
 ccflags-y += -I$(HOME)/src
 ccflags-y += -DDMA_IN_KERNEL=1 -DGITV=\"$(GITV)\"
-ccflags-y += -I/usr/src/nvidia-545.29.06/nvidia
+ccflags-y += -I$(NVIDIA_DRIVERS)/nvidia
 
 $(NAME)-objs := src/dma_buffer.o src/dma_common.o
 $(NAME)-objs += src/axi_version.o src/axis_gen2.o src/gpu_async.o src/data_gpu_top.o
@@ -46,8 +49,3 @@ clean:
 	make ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) -C $(KERNELDIR) M=$(HOME) clean
 	rm -f $(OBJS)
 
-#modules: Module.symvers
-
-#Module.symvers: $(NVIDIA_KO) nvidia-ko-to-module-symvers
-#	./nvidia-ko-to-module-symvers "$<" "$@"
-
diff --git a/data_gpu/driver/README.md b/data_gpu/driver/README.md
index 8b89dd0..b4fb202 100644
--- a/data_gpu/driver/README.md
+++ b/data_gpu/driver/README.md
@@ -1,18 +1,28 @@
-# How to cross-compile the kernel driver
+# GPU Enabled Driver
 
-To cross-compile the kernel driver you need to define the `ARCH` and `CROSS_COMPILE` variables when calling `make`. Also, you need to point `KERNELDIR` to the location of the kernel sources.
+To build this driver you need to have the NVIDA Open GPU Kernel Modules installed. This driver will not compile gainst the CUDA toolkit drivers.
 
-For example, to cross-compile the driver for the SLAC buildroot `2019.08` version for the `x86_64` architecture, you should call `make` this way:
+https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
+
+See section 5.
+
+The nvidia instructions are a bit off. These are the current steps taken at the time of this readme update:
 
 ```bash
-$ make \
-ARCH=x86_64 \
-CROSS_COMPILE=/afs/slac/package/linuxRT/buildroot-2019.08/host/linux-x86_64/x86_64/usr/bin/x86_64-buildroot-linux-gnu- \
-KERNELDIR=/afs/slac/package/linuxRT/buildroot-2019.08/buildroot-2019.08-x86_64/output/build/linux-4.14.139
+$ apt-get install nvidia-kernel-source-535-open
+$ apt-get install cuda-drivers-fabricmanager-535
+$ cd /usr/src/nvidia-535.154.05
+$ make CC=gcc-12
+$ insmod nvidia.ko
 ```
 
-On the other hand, if you do not want to cross-compile the driver, and build it for the host instead, you need to call `make` without defining any variable:
+You can then build the data_gpu image.
+
+Update the makefile and edit the following line to point to the correct path to the nvida drivers:
+
+NVIDIA_DRIVERS := /usr/src/nvidia-535.154.05/nvidia
 
 ```bash
 $ make
+$ insmod data_gpu.ko
 ```
diff --git a/data_gpu/driver/nvidia-ko-to-module-symvers b/data_gpu/driver/nvidia-ko-to-module-symvers
deleted file mode 100755
index 662731b..0000000
--- a/data_gpu/driver/nvidia-ko-to-module-symvers
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-syms=()
-syms+=(nvidia_p2p_init_mapping)
-syms+=(nvidia_p2p_destroy_mapping)
-syms+=(nvidia_p2p_get_pages)
-syms+=(nvidia_p2p_put_pages)
-syms+=(nvidia_p2p_free_page_table)
-syms+=(nvidia_p2p_dma_map_pages)
-syms+=(nvidia_p2p_dma_unmap_pages)
-
-nvidia_ko_fn="$1"
-symvers_fn="$2"
-
-touch "${symvers_fn}"
-for sym in "${syms[@]}"; do
-	crc="$(objdump -t "${nvidia_ko_fn}" | grep "__crc_${sym}" | awk '{print $1}')"
-	if [ -z "${crc}" ]; then
-		echo "Warning: Can't find symbol ${sym} in ${nvidia_ko_fn}; assuming CONFIG_MODVERSIONS=n so setting CRC=0"
-		crc=00000000
-	fi
-	sed -i '/${sym}/d' "${symvers_fn}"
-	echo "0x${crc}	${sym}	${nvidia_ko_fn}	EXPORT_SYMBOL" >> ${symvers_fn}
-done