Mega RVFS commit

.gitignore

@@ -32,3 +32,6 @@ ltmain.sh
 **/Makefile.in
 **/stamp-h1
 **/compile_commands.json
+
+*.toml
+!/conf_example.toml

.gitmodules

@@ -1,3 +1,6 @@
-[submodule "eRPC-arm"]
-	path = eRPC-arm
+[submodule "extern/tomlcpp"]
+	path = extern/tomlcpp
+	url = [email protected]:cktan/tomlcpp.git
+[submodule "extern/eRPC-arm"]
+	path = extern/eRPC-arm
 	url = [email protected]:Peter-JanGootzen/eRPC-arm.git

conf_example.toml

@@ -0,0 +1,34 @@
+[snap_hal]
+# Time between every poll
+polling_interval_usec = 0
+# Physical function ID
+pf_id = 0
+# Virtual function ID
+vf_id = -1
+# Emulation manager, DPU specific
+emu_manager = "mlx5_0"
+# Amount of polling threads 0 for single threaded mode, >0 for multithreaded mode
+# Multithreaded not supported currently!
+nthreads = 0
+# Must be a power of 2 and >= 1! The virtio-fs specification requires this
+queue_depth = 64
+# Filesystem tag (i.e. the name of the virtiofs device to mount for the host)
+tag = "dpfs"
+
+[rvfs_hal]
+# Time between every poll
+polling_interval_usec = 0
+
+[nfs]
+# The NFS server that you want to mirror
+server = "10.100.0.1"
+# The NFS server's path that you want to mirror
+export = "/mnt/shared"
+
+# This is for dpfs_rvfs_dpu and the dpfs_hal implementation that uses RVFS
+[rvfs]
+remote_uri = "10.100.0.1:38510"
+dpu_uri = "10.100.0.115:38510"
+# Useful for the DPU and remote side to have the QD synced
+# Must be a power of 2 and >= 1
+queue_depth = 64

dpfs_fuse/Makefile.am

@@ -10,20 +10,17 @@ lib_LTLIBRARIES = libdpfs_fuse.la
 libdpfs_fuse_adir = $(includedir)/
 include_HEADERS = dpfs_fuse.h
 
-libdpfs_fuse_la_LIBADD = $(srcdir)/../dpfs_hal/libdpfs_hal.a \
-	$(srcdir)/../../src/libmlx_dev_emu.a $(srcdir)/../../src/libmlx_dev_emu_snap.a \
-	$(srcdir)/../eRPC-arm/build/liberpc.a \
-	$(IBVERBS_LDFLAGS) $(SNAP_LDFLAGS) $(PYTHON3_LDFLAGS) 
-
+libdpfs_fuse_la_LIBADD = $(srcdir)/../dpfs_hal/libdpfs_hal.la \
+	$(srcdir)/../extern/eRPC-arm/build/liberpc.a
+
 libdpfs_fuse_la_LDFLAGS = -lboost_system -lboost_thread \
-	--exclude-libs=libmlx_dev_emu.a --exclude-libs=libmlx_dev_emu_snap.a --exclude-libs=liberpc.a -fvisibility=hidden \
-	$(IBVERBS_LDFLAGS) $(SNAP_LDFLAGS) $(PYTHON3_LDFLAGS) 
+	--exclude-libs=liberpc.a -fvisibility=hidden
 
 libdpfs_fuse_la_CPPFLAGS  = $(BASE_CPPFLAGS) \
 	-I$(srcdir)/../../src $(SNAP_CFLAGS) \
-	-I$(srcdir)/../dpfs_hal \
-	-I$(srcdir)/../eRPC-arm/third_party/asio/include \
-	-I$(srcdir)/../eRPC-arm/src \
+	-I$(srcdir)/../dpfs_hal/include \
+	-I$(srcdir)/../extern/eRPC-arm/third_party/asio/include \
+	-I$(srcdir)/../extern/eRPC-arm/src \
 	-DERPC_INFINIBAND -Wno-address-of-packed-member # eRPC required flags for its headers
 
 libdpfs_fuse_la_SOURCES = dpfs_fuse.cpp 

dpfs_fuse/debug.h

@@ -7,7 +7,7 @@
 
 #include <linux/fuse.h>
 #include <stdio.h>
-#include "dpfs_hal.h"
+#include "dpfs/hal.h"
 
 #ifdef __cplusplus
 extern "C" {

dpfs_fuse/dpfs_fuse.cpp

@@ -33,10 +33,8 @@
 #include "config.h"
 #include "common.h"
 #include "debug.h"
-#include "dpfs_hal.h"
+#include "dpfs/hal.h"
 #include "dpfs_fuse.h"
-#include "rpc.h"
-using namespace erpc;
 
 struct fuse_ll;
 typedef int (*fuse_handler_t) (struct fuse_ll *,
@@ -1783,12 +1781,7 @@ static int fuse_handle_req(void *u,
     }
 }
 
-
-void erpc_handler(ReqHandle *req_handle, void *context) {
-
-}
-
-int dpfs_fuse_main(struct fuse_ll_operations *ops, struct virtiofs_emu_params *emu_params,
+int dpfs_fuse_main(struct fuse_ll_operations *ops, const char *hal_conf_path,
                    void *user_data, bool debug)
 {
 #ifdef DEBUG_ENABLED
@@ -1812,11 +1805,8 @@ int dpfs_fuse_main(struct fuse_ll_operations *ops, struct virtiofs_emu_params *e
 
     fuse_ll_map(f_ll);
 
-#define DPFS_FUSE_HAL
-#if defined(DPFS_FUSE_HAL)
     struct dpfs_hal_params hal_params;
     memset(&hal_params, 0, sizeof(hal_params));
-    memcpy(&hal_params.emu_params, emu_params, sizeof(struct virtiofs_emu_params));
     hal_params.user_data = f_ll;
     hal_params.request_handler = fuse_handle_req;
 
@@ -1827,13 +1817,6 @@ int dpfs_fuse_main(struct fuse_ll_operations *ops, struct virtiofs_emu_params *e
     }
     dpfs_hal_loop(emu);
     dpfs_hal_destroy(emu);
-#elif defined(DPFS_FUSE_ERPC)
-
-    // init eRPC
-    // allocate some eRPC buffers
-    // plug eRPC into dpfs_fuse
-    // busy while loop { eRPC event loop once }
-#endif
 
     return 0;
 }

dpfs_fuse/dpfs_fuse.h

@@ -16,7 +16,7 @@
 #include <sys/stat.h>
 #include <sys/statvfs.h>
 #include <linux/fuse.h>
-#include "dpfs_hal.h"
+#include "dpfs/hal.h"
 
 // Beginning of libfuse/include/fuse_lowlevel.h selective copy
 
@@ -266,7 +266,7 @@ struct fuse_ll_operations {
                       void *completion_context);
 };
 
-int dpfs_fuse_main(struct fuse_ll_operations *ops, struct virtiofs_emu_params *emu_params,
+int dpfs_fuse_main(struct fuse_ll_operations *ops, const char *hal_conf_path,
         void *user_data, bool debug);
 
 #ifdef __cplusplus

dpfs_hal/Makefile.am

@@ -1,17 +1,55 @@
 #
-# Copyright 2022- IBM Inc. All rights reserved
+# Copyright 2023- IBM Inc. All rights reserved
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
 
+lib_LTLIBRARIES = libdpfs_hal.la
+
+libdpfs_hal_ladir = $(includedir)/
+libdpfs_hal_la_HEADERS = include/dpfs/hal.h
+
+libdpfs_hal_la_CFLAGS = -I$(builddir)/include/dpfs \
+	-fPIC -fvisibility=hidden
+
+libdpfs_hal_la_SOURCES = src/cpu_latency.c \
+	$(builddir)/../extern/tomlcpp/toml.c
+
+libdpfs_hal_la_LDFLAGS = $(IBVERBS_LDFLAGS)
+
+if !DPFS_RVFS
 if HAVE_SNAP
 
-noinst_LIBRARIES = libdpfs_hal.a
+libdpfs_hal_la_LIBADD = $(srcdir)/../../src/libmlx_dev_emu.a \
+	$(srcdir)/../../src/libmlx_dev_emu_snap.a \
+	$(IBVERBS_LDFLAGS) $(SNAP_LDFLAGS) $(PYTHON3_LDFLAGS)
+
+libdpfs_hal_la_LDFLAGS += --exclude-libs=libmlx_dev_emu.a \
+	--exclude-libs=libmlx_dev_emu_snap.a \
+	$(SNAP_LDFLAGS) $(PYTHON3_LDFLAGS) 
+
+# Mellanox Snap BlueField virtio-fs emulation
+libdpfs_hal_la_CFLAGS += $(BASE_CFLAGS) \
+	$(SNAP_CFLAGS) -I$(builddir)/../../src \
+	-I$(builddir)/../extern/tomlcpp
+
+libdpfs_hal_la_SOURCES += src/snap.c
+
+endif HAVE_SNAP
+else DPFS_RVFS
+# RVFS using eRPC
+
+libdpfs_hal_la_LIBADD = $(srcdir)/../extern/eRPC-arm/build/liberpc.a \
+	-lboost_system -lboost_thread -lnuma $(IBVERBS_LDFLAGS) 
 
-libdpfs_hal_adir = $(includedir)/
-libdpfs_hal_a_HEADERS = dpfs_hal.h
+# eRPC required flags for its headers
+libdpfs_hal_la_CPPFLAGS = $(BASE_CPPFLAGS) $(libdpfs_hal_la_CFLAGS) \
+	-I$(builddir)/../extern/eRPC-arm/third_party/asio/include \
+	-I$(builddir)/../extern/eRPC-arm/src \
+	-DERPC_INFINIBAND -Wno-address-of-packed-member
+# toml library
+libdpfs_hal_la_CPPFLAGS += -I$(builddir)/../extern/tomlcpp
 
-libdpfs_hal_a_CFLAGS  = $(BASE_CFLAGS) -I$(srcdir)/../../src $(SNAP_CFLAGS) \
-	-fPIC
-libdpfs_hal_a_SOURCES = dpfs_hal.c cpu_latency.c
+libdpfs_hal_la_SOURCES += src/rvfs.cpp \
+	$(builddir)/../extern/tomlcpp/tomlcpp.cpp
 
-endif
+endif DPFS_RVFS

dpfs_hal/dpfs_hal.h → dpfs_hal/include/dpfs/hal.h

@@ -31,23 +31,10 @@ typedef int (*dpfs_hal_handler_t) (void *user_data,
                                    struct iovec *fuse_out_iov, int out_iovcnt,
                                    void *completion_context);
 
-struct virtiofs_emu_params {
-    useconds_t polling_interval_usec; // Time between every poll
-    int pf_id; // Physical function ID
-    int vf_id; // Virtual function ID
-    const char *emu_manager; // Emulation manager, DPU specific
-    // Amount of polling threads 0 for single threaded mode, >0 for multithreaded mode
-    // Multithreaded not supported currently!
-    uint32_t nthreads;
-    // Must be a power of 2!
-    uint32_t queue_depth;
-    const char *tag; // Filesystem tag (i.e. the name of the virtiofs device to mount for the host)
-};
-
 struct dpfs_hal_params {
     dpfs_hal_handler_t request_handler;
     void *user_data; // Pointer to user data that gets passed with every dpfs_hal_handler
-    struct virtiofs_emu_params emu_params;
+    const char *conf_path; // See the example toml files in the root of dpfs_hal
 };
 
 enum dpfs_hal_completion_status {

dpfs_hal/cpu_latency.c → dpfs_hal/src/cpu_latency.c

@@ -1,3 +1,10 @@
+/*
+#
+# Copyright 2023- IBM Inc. All rights reserved
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+*/
+
 #include <stdint.h>
 #include <stdio.h>
 #include <fcntl.h>

dpfs_hal/cpu_latency.h → dpfs_hal/src/cpu_latency.h

@@ -1,3 +1,10 @@
+/*
+#
+# Copyright 2023- IBM Inc. All rights reserved
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+*/
+
 #ifndef CPU_LATENCY_H
 #define CPU_LATENCY_H