Initial dpu-virtio-fs open sourcing commit

For previous commit history, query the internal IBM repository

.gitignore

@@ -0,0 +1,34 @@
+libtool
+Makefile
+aclocal.m4
+compile
+depcomp
+install-sh
+ltmain.sh
+
+.vscode
+.DS_Store
+**/*.o
+**/*.Po
+**/*.lo
+**/*.a
+**/*.la
+**/*.lai
+**/*.so.*
+**/*.cache
+**/*.spec
+**/.deps
+**/.dirstamp
+**/.libs
+**/config.h
+**/config.h.in
+**/config.h.in~
+**/config.log
+**/config.status
+**/config.guess
+**/config.sub
+**/configure
+**/missing
+**/Makefile.in
+**/stamp-h1
+**/compile_commands.json

README.md

@@ -0,0 +1,34 @@
+# Filesystem Virtualization using DPUs
+A DPU (Data Processing Unit), for the scope and definition of this project, contains a CPU (running e.g. Linux), NIC and programmable data acceleration engines. It is also commonly referred to as SmartNIC or IPU (Infrastructure Processing Unit).
+
+DPUs are shaping up to be the center of virtualization in the Hyperconverged Infrastructure (HCI).
+[Virtio](https://developer.ibm.com/articles/l-virtio) is an abstraction layer for virtualized environments to expose virtual PCIe hardware to guest VMs.
+[Virtio-fs](https://www.kernel.org/doc/html/latest/filesystems/virtiofs.html) is one of these virtual PCIe hardware specifications. It employs the [FUSE](https://www.kernel.org/doc/html/latest/filesystems/fuse.html) protocol (only the communication protocol!) to provide a filesystem to guest VMs.
+There are now DPUs comming out on the market that have support for hardware-accelerated virtio-fs emulation. Thereby having a real hardware device implement the virtual filesystem layer of virtio.
+
+# Implementation
+![DPU virtio-fs architecture diagram](arch-diagram.png "DPU virtio-fs architecture diagram")
+## Modules
+This project uses a specific DPU library for the virtio-fs emulation layer, but is generalizable to other DPUs.
+### `virtiofs_emu_lowlevel`
+Front-end for the virtio-fs emulation layer. Only useable if steps in *Usage* are taken.
+### `virtiofs_emu_fuse_lowlevel`
+Provides a lowlevel FUSE API (close-ish compatible fork of `libfuse/fuse_lowlevel.h`) over the raw buffers that DPUlib provides the user, using `virtiofs_emu_lowlevel`.
+### `virtiofuser`
+Reflects a local filesystem via the POSIX FS API by implementing the lowlevel FUSE API in `virtiofs_emu_fuse_lowlevel`, with synchronous reads and writes.
+### `virtiofuser`
+Reflects a local filesystem via the POSIX FS API by implementing the lowlevel FUSE API in `virtiofs_emu_fuse_lowlevel`, with asynchronous reads and writes using Linux AIO systemcalls.
+### `virtionfs`
+Reflects a NFS folder with the asynchronous userspace NFS library `libnfs` by implementing the lowlevel FUSE API in `virtiofs_emu_fuse_lowlevel`.
+### `list_emulation_managers`
+Standalone program to find out which RDMA devices have emulation capabilities
+
+## TODO
+- [ ] `virtionfs` full implementation, only a proof of concept at the moment
+
+# Usage
+* Enable virtio-fs emulation in the DPU firmware with atleast one physical function (PF) for virtio-fs, and reboot the DPU
+* Determine the RDMA device that has virtio-fs emulation capabilities by running `list_emulation_managers`
+* Use `virtiofuser`, `virtiofuser_aio` or `virtionfs` by specifying the correct physical function (PF) with `-p`, virtual function (VF) zero `-v 0` and the RDMA device name that has virtio-fs emulation capabilities with `-e`
+
+## :switzerland: Cloud Data Platforms group at IBM Research Zurich

list_emulation_managers/.gitignore

@@ -0,0 +1 @@
+list_emulation_managers

list_emulation_managers/Makefile.am

@@ -0,0 +1,17 @@
+#
+# Copyright 2022- IBM Inc. All rights reserved
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+
+if HAVE_SNAP
+
+bin_PROGRAMS = list_emulation_managers
+
+list_emulation_managers_LDADD = $(srcdir)/../../src/libmlx_dev_emu_snap.a \
+                $(srcdir)/../../src/libmlx_dev_emu.a \
+                $(IBVERBS_LDFLAGS) $(SNAP_LDFLAGS) $(PYTHON3_LDFLAGS) -lpthread
+
+list_emulation_managers_CFLAGS  = $(BASE_CFLAGS) -I$(srcdir)/../../src $(SNAP_CFLAGS)
+list_emulation_managers_SOURCES = main.c
+
+endif

list_emulation_managers/main.c

@@ -0,0 +1,56 @@
+/*
+#
+# Copyright 2022- IBM Inc. All rights reserved
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+*/
+
+#include <err.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <infiniband/verbs.h>
+
+#include "nvme_emu_log.h"
+#include "mlnx_snap_pci_manager.h"
+
+int main(void) {
+    if (nvme_init_logger())
+        err(1, "Failed to open logger. Does /var/log/mlnx_snap exist?");
+
+    if (mlnx_snap_pci_manager_init())
+        err(1, "Failed to init SNAP pci manager, this can mean two things:\n"
+                "* No emulation capability is turned on in the firmware. (virtio-fs, virtio-blk, virtio-net or nvme)"
+                "* The BF needs to be reboted (full host external power cycle), because the new firmware configuration is not yet applied or the emulation firmware is in a crashed state (happens if a SNAP application was not correctly shut down)\n");
+
+    int ibv_count;
+    struct ibv_device **ibv_list = ibv_get_device_list(&ibv_count);
+    if (!ibv_list) {
+        fprintf(stderr, "Failed to open IB device list.\n");
+    goto err_pci;
+    }
+
+    for (int i = 0; i < ibv_count; i++) {
+        const char *rdma_device = ibv_get_device_name(ibv_list[i]);
+
+        struct snap_context *sctx = mlnx_snap_get_snap_context(rdma_device);
+        if (!sctx)
+            continue;
+
+        printf("Emulation manager \"%s\" supports:\n", rdma_device);
+
+        if (sctx->emulation_caps & SNAP_VIRTIO_FS)
+            printf("* virtio_fs\n");
+        if (sctx->emulation_caps & SNAP_VIRTIO_BLK)
+                printf("* virtio_blk\n");
+        if (sctx->emulation_caps & SNAP_VIRTIO_NET)
+                printf("* virtio_net\n");
+        if (sctx->emulation_caps & SNAP_NVME)
+        printf("* nvme\n");
+    } 
+
+    free(ibv_list);
+err_pci:
+    mlnx_snap_pci_manager_clear();
+
+    return 0;
+}

virtiofs_emu_fuse_lowlevel/Makefile.am

@@ -0,0 +1,13 @@
+if HAVE_SNAP
+
+noinst_LIBRARIES = libvirtiofs_emu_fuse_ll.a
+
+libvirtiofs_emu_fuse_ll_adir = $(includedir)/
+libvirtiofs_emu_fuse_ll_a_HEADERS = fuse_ll.h
+
+libvirtiofs_emu_fuse_ll_a_LIBADD = $(srcdir)/../virtiofs_emu_lowlevel/libvirtiofs_emu_ll.a
+
+libvirtiofs_emu_fuse_ll_a_CFLAGS  = $(BASE_CFLAGS) -I$(srcdir)/../../src -I$(srcdir)/../virtiofs_emu_lowlevel $(SNAP_CFLAGS)
+libvirtiofs_emu_fuse_ll_a_SOURCES = fuse_ll.c
+
+endif

virtiofs_emu_fuse_lowlevel/common.h

@@ -0,0 +1,19 @@
+/*
+#
+# Copyright 2022- IBM Inc. All rights reserved
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+*/
+
+#ifndef COMMON_H
+#define COMMON_H
+
+#ifndef O_PATH
+#define O_PATH		010000000
+#endif
+#define AT_EMPTY_PATH		0x1000	/* Allow empty relative pathname */
+
+#define MIN(x, y) x < y ? x : y
+#define MAX(x, y) x > y ? x : y
+
+#endif // COMMON_H