Started work on implementing NFS 4.1 with session trunking (IBM#2)

Signed-off-by: Peter-Jan Gootzen <[email protected]>

.gitattributes

@@ -1 +0,0 @@
-*.svg filter=lfs diff=lfs merge=lfs -text

virtionfs/nfs_v4.c

@@ -92,45 +92,131 @@ int nfs4_fill_create_attrs(struct fuse_in_header *in_hdr, uint32_t mode, fattr4
     return 0;
 }
 
+bool nfs4_check_session_trunking_allowed(EXCHANGE_ID4resok *l, EXCHANGE_ID4resok *r) {
+    return l->eir_clientid == r->eir_clientid 
+                && l->eir_server_owner.so_major_id.so_major_id_len == r->eir_server_owner.so_major_id.so_major_id_len
+                && memcmp(l->eir_server_owner.so_major_id.so_major_id_val, r->eir_server_owner
+                    .so_major_id.so_major_id_val, l->eir_server_owner.so_major_id.so_major_id_len) == 0
+                && l->eir_server_owner.so_minor_id == r->eir_server_owner.so_minor_id
+                && l->eir_server_scope.eir_server_scope_len == r->eir_server_scope.eir_server_scope_len
+                && memcmp(l->eir_server_scope.eir_server_scope_val, r->eir_server_scope
+                    .eir_server_scope_val, l->eir_server_scope.eir_server_scope_len) == 0;
+}
+
+int nfs4_op_createsession(nfs_argop4 *op, clientid4 clientid)
+{
+   CREATE_SESSION4args *arg;
+   op[0].argop = OP_BIND_CONN_TO_SESSION;
+   arg = &op[0].nfs_argop4_u.opcreatesession;
+
+   arg->csa_clientid = clientid;
+   arg->csa_flags = 0;
+   arg->csa_cb_program = 0;
+   arg->csa_sec_parms.csa_sec_parms_val = NULL;
+   arg->csa_sec_parms.csa_sec_parms_len = 0;
+
+   // Currently no caching support
+   arg->csa_fore_chan_attrs.ca_maxresponsesize_cached = 0;
+   arg->csa_fore_chan_attrs.ca_maxrequests = NFS4_MAXREQUESTSIZE;
+   arg->csa_fore_chan_attrs.ca_maxresponsesize = NFS4_MAXRESPONSESIZE;
+   arg->csa_fore_chan_attrs.ca_maxrequestsize = NFS4_MAXREQUESTSIZE;
+   // We have too little control over libnfs to properly use this
+   arg->csa_fore_chan_attrs.ca_headerpadsize = 0;
+   // Magic from the Linux kernel, 8 seems about right
+   arg->csa_fore_chan_attrs.ca_maxoperations = NFS4_MAX_OPS;
+   // No RDMA here
+   arg->csa_fore_chan_attrs.ca_rdma_ird.ca_rdma_ird_val = NULL;
+   arg->csa_fore_chan_attrs.ca_rdma_ird.ca_rdma_ird_len = 0;
+
+   // Currently no caching support
+   arg->csa_back_chan_attrs.ca_maxresponsesize_cached = 0;
+   arg->csa_back_chan_attrs.ca_maxrequests = NFS4_MAXREQUESTSIZE;
+   arg->csa_back_chan_attrs.ca_maxresponsesize = NFS4_MAXRESPONSESIZE;
+   arg->csa_back_chan_attrs.ca_maxrequestsize = NFS4_MAXREQUESTSIZE;
+   // We have too little control over libnfs to properly use this
+   arg->csa_back_chan_attrs.ca_headerpadsize = 0;
+   // Magic from the Linux kernel, 8 seems about right
+   arg->csa_back_chan_attrs.ca_maxoperations = NFS4_MAX_OPS;
+   // No RDMA here
+   arg->csa_back_chan_attrs.ca_rdma_ird.ca_rdma_ird_val = NULL;
+   arg->csa_back_chan_attrs.ca_rdma_ird.ca_rdma_ird_len = 0;
+
+   return 1;
+}
+
+
+int nfs4_op_bindconntosession(nfs_argop4 *op, sessionid4 *sessionid, channel_dir_from_client4 channel, bool rdma)
+{
+   BIND_CONN_TO_SESSION4args *bindargs;
+   op[0].argop = OP_BIND_CONN_TO_SESSION;
+   bindargs = &op[0].nfs_argop4_u.opbindconntosession;
+
+   memcpy(&bindargs->bctsa_sessid, sessionid, sizeof(sessionid4));
+   bindargs->bctsa_dir = CDFC4_FORE_OR_BOTH;
+   bindargs->bctsa_use_conn_in_rdma_mode = rdma;
+
+   return 1;
+}
+
+int nfs4_op_exchangeid(nfs_argop4 *op, verifier4 verifier, const char *client_name)
+{
+   EXCHANGE_ID4args *exidargs;
+   op[0].argop = OP_EXCHANGE_ID;
+   exidargs = &op[0].nfs_argop4_u.opexchangeid;
+
+   memcpy(exidargs->eia_clientowner.co_verifier, verifier, sizeof(verifier4));
+   exidargs->eia_clientowner.co_ownerid.co_ownerid_val = (char *) client_name;
+   exidargs->eia_clientowner.co_ownerid.co_ownerid_len = strlen(client_name);
+
+   exidargs->eia_state_protect.spa_how = SP4_NONE;
+
+   exidargs->eia_flags = 0;
+
+   exidargs->eia_client_impl_id.eia_client_impl_id_val = NULL;
+   exidargs->eia_client_impl_id.eia_client_impl_id_len = 0;
+
+   return 1;
+}
+
 /* Functions taken and modified from libnfs/lib/nfs_v4.c
  * commit: 2678dfecd9c797991b7768490929b1478f339809 */
 
 int nfs4_op_setclientid(nfs_argop4 *op, verifier4 verifier, const char *client_name)
 {
-        SETCLIENTID4args *scidargs;
-
-        op[0].argop = OP_SETCLIENTID;
-        scidargs = &op[0].nfs_argop4_u.opsetclientid;
-        memcpy(scidargs->client.verifier, verifier, sizeof(verifier4));
-        scidargs->client.id.id_len = strlen(client_name);
-        scidargs->client.id.id_val = (char *) client_name;
-        /* TODO: Decide what we should do here. As long as we only
-         * expose a single FD to the application we will not be able to
-         * do NFSv4 callbacks easily.
-         * Just give it garbage for now until we figure out how we should
-         * solve this. Until then we will just have to avoid doing things
-         * that require a callback.
-         * ( Clients (i.e. Linux) ignore this anyway and just call back to
-         *   the originating address and program anyway. )
-         */
-        scidargs->callback.cb_program = 0; /* NFS4_CALLBACK */
-        scidargs->callback.cb_location.r_netid = "tcp";
-        scidargs->callback.cb_location.r_addr = "0.0.0.0.0.0";
-        scidargs->callback_ident = 0x00000001;
-
-        return 1;
+   SETCLIENTID4args *scidargs;
+   op[0].argop = OP_SETCLIENTID;
+   scidargs = &op[0].nfs_argop4_u.opsetclientid;
+
+   memcpy(scidargs->client.verifier, verifier, sizeof(verifier4));
+   scidargs->client.id.id_len = strlen(client_name);
+   scidargs->client.id.id_val = (char *) client_name;
+   /* TODO: Decide what we should do here. As long as we only
+    * expose a single FD to the application we will not be able to
+    * do NFSv4 callbacks easily.
+    * Just give it garbage for now until we figure out how we should
+    * solve this. Until then we will just have to avoid doing things
+    * that require a callback.
+    * ( Clients (i.e. Linux) ignore this anyway and just call back to
+    *   the originating address and program anyway. )
+    */
+   scidargs->callback.cb_program = 0; /* NFS4_CALLBACK */
+   scidargs->callback.cb_location.r_netid = "tcp";
+   scidargs->callback.cb_location.r_addr = "0.0.0.0.0.0";
+   scidargs->callback_ident = 0x00000001;
+
+   return 1;
 }
 
 int nfs4_op_setclientid_confirm(struct nfs_argop4 *op, uint64_t clientid, verifier4 verifier)
 {
-        SETCLIENTID_CONFIRM4args *scidcargs;
+   SETCLIENTID_CONFIRM4args *scidcargs;
 
-        op[0].argop = OP_SETCLIENTID_CONFIRM;
-        scidcargs = &op[0].nfs_argop4_u.opsetclientid_confirm;
-        scidcargs->clientid = clientid;
-        memcpy(scidcargs->setclientid_confirm, verifier, NFS4_VERIFIER_SIZE);
+   op[0].argop = OP_SETCLIENTID_CONFIRM;
+   scidcargs = &op[0].nfs_argop4_u.opsetclientid_confirm;
+   scidcargs->clientid = clientid;
+   memcpy(scidcargs->setclientid_confirm, verifier, NFS4_VERIFIER_SIZE);
 
-        return 1;
+   return 1;
 }
 
 int nfs4_find_op(COMPOUND4res *res, int op)

virtionfs/nfs_v4.h

@@ -3,11 +3,15 @@
 # Copyright 2022- IBM Inc. All rights reserved
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
+# GPL-2.0 WITH Linux-syscall-note
+# from https://elixir.bootlin.com/linux/v6.0.12/source/include/uapi/linux/nfs4.h
+#
 */
 
 #ifndef NFS_V4_H
 #define NFS_V4_H
 
+#include <stdbool.h>
 #include <linux/fuse.h>
 #include <nfsc/libnfs-raw-nfs4.h>
 
@@ -16,6 +20,38 @@
 // Empirically verified with Linux kernel 5.11
 #define NFS_ROOT_FILEID 2
 
+// 1MB is max read/write size in Linux, + some overhead
+#define NFS4_MAXRESPONSESIZE (1 << 20)
+#define NFS4_MAXREQUESTSIZE (1 << 20)
+
+#define NFS4_MAX_OUTSTANDING_REQUESTS 64
+
+/*
+ *  linux/include/linux/nfs4.h
+ *
+ *  NFSv4 protocol definitions.
+ *
+ *  Copyright (c) 2002 The Regents of the University of Michigan.
+ *  All rights reserved.
+ *
+ *  Kendrick Smith <[email protected]>
+ *  Andy Adamson   <[email protected]>
+ */
+
+/* An NFS4 sessions server must support at least NFS4_MAX_OPS operations.
+ * If a compound requires more operations, adjust NFS4_MAX_OPS accordingly.
+ */
+#define NFS4_MAX_OPS   8
+
+/* Our NFS4 client back channel server only wants the cb_sequene and the
+ * actual operation per compound
+ */
+#define NFS4_MAX_BACK_CHANNEL_OPS 2
+
+/*
+ * End of Linux header
+ */
+
 // No malloc needed
 struct vnfs_fh4 {
     u_int len;
@@ -26,6 +62,11 @@ typedef struct vnfs_fh4 vnfs_fh4;
 int nfs4_clone_fh(vnfs_fh4 *dst, nfs_fh4 *src);
 int nfs4_find_op(COMPOUND4res *res, int op);
 int nfs4_fill_create_attrs(struct fuse_in_header *in_hdr, uint32_t flags, fattr4 *attr);
+bool nfs4_check_session_trunking_allowed(EXCHANGE_ID4resok *l, EXCHANGE_ID4resok *r);
+// Supply the clientid received from EXCHANGE_ID
+int nfs4_op_createsession(nfs_argop4 *op, clientid4 clientid);
+int nfs4_op_bindconntosession(nfs_argop4 *op, sessionid4 *sessionid, channel_dir_from_client4 channel, bool rdma);
+int nfs4_op_exchangeid(nfs_argop4 *op, verifier4 verifier, const char *client_name);
 int nfs4_op_setclientid(nfs_argop4 *op, verifier4 verifier, const char *client_name);
 int nfs4_op_setclientid_confirm(struct nfs_argop4 *op, uint64_t clientid, verifier4 verifier);
 int nfs4_op_getattr(nfs_argop4 *op, uint32_t *attributes, int count);

virtionfs/virtionfs.c

@@ -1309,6 +1309,137 @@ int getattr(struct fuse_session *se, struct virtionfs *vnfs,
     return EWOULDBLOCK;
 }
 
+static void create_session_cb(struct rpc_context *rpc, int status, void *data, void *private_data)
+{
+    struct virtionfs *vnfs = private_data;
+    COMPOUND4res *res = data;
+
+    if (status != RPC_STATUS_SUCCESS) {
+        fprintf(stderr, "RPC with NFS:CREATE_SESSION unsuccessful: rpc error=%d\n", status);
+        return;
+    }
+    if (res->status != NFS4_OK) {
+        fprintf(stderr, "NFS:CREATE_SESSION unsuccessful: nfs error=%d\n", res->status);
+        return;
+    }
+}
+
+static int create_session(struct virtionfs *vnfs)
+{
+    COMPOUND4args args;
+    nfs_argop4 op[1];
+    memset(&args, 0, sizeof(args));
+    args.argarray.argarray_len = sizeof(op) / sizeof(nfs_argop4);
+    args.argarray.argarray_val = op;
+    memset(op, 0, sizeof(op));
+
+    nfs4_op_createsession(&op[0], &vnfs->sessionid, CDFC4_FORE_OR_BOTH, false);
+
+    if (rpc_nfs4_compound_async(vnfs->rpc, create_session_cb, &args, vnfs) != 0) {
+    	fprintf(stderr, "Failed to send NFS:create_session request\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+static void bind_conn_cb(struct rpc_context *rpc, int status, void *data, void *private_data)
+{
+    struct virtionfs *vnfs = private_data;
+    COMPOUND4res *res = data;
+
+    if (status != RPC_STATUS_SUCCESS) {
+        fprintf(stderr, "RPC with NFS:bind_conn_to_session unsuccessful: rpc error=%d\n", status);
+        return;
+    }
+    if (res->status != NFS4_OK) {
+        fprintf(stderr, "NFS:bind_conn_to_session unsuccessful: nfs error=%d\n", res->status);
+        return;
+    }
+
+    BIND_CONN_TO_SESSION4resok *ok = &res->resarray.resarray_val[0].nfs_resop4_u.opbindconntosession.BIND_CONN_TO_SESSION4res_u.bctsr_resok4;
+
+    if (memcmp(ok->bctsr_sessid, vnfs->sessionid, sizeof(sessionid4)) == 0) {
+        vnfs->connections++;
+        if (vnfs->connections < vnfs->nthreads) {
+            // Another thread and mount
+        }
+    }
+}
+
+static int bind_conn(struct virtionfs *vnfs)
+{
+    COMPOUND4args args;
+    nfs_argop4 op[1];
+    memset(&args, 0, sizeof(args));
+    args.argarray.argarray_len = sizeof(op) / sizeof(nfs_argop4);
+    args.argarray.argarray_val = op;
+    memset(op, 0, sizeof(op));
+
+    nfs4_op_bindconntosession(&op[0], &vnfs->sessionid, CDFC4_FORE_OR_BOTH, false);
+
+    if (rpc_nfs4_compound_async(vnfs->rpc, bind_conn_cb, &args, vnfs) != 0) {
+    	fprintf(stderr, "Failed to send NFS:setclientid request\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+static verifier4 default_verifier = {'0', '1', '2', '3', '4', '5', '6', '7'};
+
+static void exchangeid_cb(struct rpc_context *rpc, int status, void *data, void *private_data)
+{
+    struct virtionfs *vnfs = private_data;
+    COMPOUND4res *res = data;
+
+    if (status != RPC_STATUS_SUCCESS) {
+        fprintf(stderr, "RPC with NFS:exchange_id unsuccessful: rpc error=%d\n", status);
+        return;
+    }
+    if (res->status != NFS4_OK) {
+        fprintf(stderr, "NFS:exchange_id unsuccessful: nfs error=%d\n", res->status);
+        return;
+    }
+
+    EXCHANGE_ID4resok *ok = &res->resarray.resarray_val[0].nfs_resop4_u.opexchangeid.EXCHANGE_ID4res_u.eir_resok4;
+
+    if (vnfs->connections == 0) {
+        memcpy(&vnfs->first_exchangeid, ok, sizeof(*ok));
+        create_session(vnfs);
+    } else {
+        if (nfs4_check_session_trunking_allowed(&vnfs->first_exchangeid, ok)) {
+            bind_conn(vnfs);
+        } else {
+            // TODO handle can't trunking
+            // close connection
+        }
+    }
+}
+
+static int exchangeid(struct virtionfs *vnfs)
+{
+    COMPOUND4args args;
+    nfs_argop4 op[1];
+    memset(&args, 0, sizeof(args));
+    args.argarray.argarray_len = sizeof(op) / sizeof(nfs_argop4);
+    args.argarray.argarray_val = op;
+    memset(op, 0, sizeof(op));
+
+
+    verifier4 v;
+    memcpy(default_verifier, v, sizeof(v));
+    v[0] = vnfs->connections;
+    nfs4_op_exchangeid(&op[0], default_verifier, "virtionfs");
+
+    if (rpc_nfs4_compound_async(vnfs->rpc, exchangeid_cb, &args, vnfs) != 0) {
+    	fprintf(stderr, "Failed to send NFS:exchange_id request\n");
+        return -1;
+    }
+
+    return 0;
+}
+
 static void
 setclientid_cb_2(struct rpc_context *rpc, int status, void *data,
                 void *private_data)
@@ -1371,7 +1502,6 @@ setclientid_cb_1(struct rpc_context *rpc, int status, void *data,
     }
 }
 
-static verifier4 verifier = {'0', '1', '2', '3', '4', '5', '6', '7'};
 
 int setclientid(struct virtionfs *vnfs)
 {
@@ -1384,7 +1514,7 @@ int setclientid(struct virtionfs *vnfs)
     memset(op, 0, sizeof(op));
 
     // TODO make this verifier random and the client name somewhat unique too
-    nfs4_op_setclientid(&op[0], verifier, "virtionfs");
+    nfs4_op_setclientid(&op[0], default_verifier, "virtionfs");
 
     if (rpc_nfs4_compound_async(vnfs->rpc, setclientid_cb_1, &args, vnfs) != 0) {
     	fprintf(stderr, "Failed to send NFS:setclientid request\n");