Building all tests.

Makefile

@@ -5,11 +5,19 @@ include Kbuild
 else
 
 KVER	:= $(shell uname -r)
+#KVER	:=4.5.0
+
 KDIR	:= /lib/modules/$(KVER)/build
 PWD	:= $(shell pwd)
 INSTALL_MOD_DIR	:= kfabric
 DEPMOD	:= /usr/sbin/depmod
-KSYM	:= /usr/src/compat-rdma/Module.symvers
+
+ifneq ("","$(wildcard /usr/src/compat-rdma)")
+  #for OFED
+  KSYM	:= /usr/src/compat-rdma/Module.symvers
+else
+  KSYM :=
+endif
 
 default:
 	$(MAKE) -C $(KDIR) M=$(PWD) modules KBUILD_EXTRA_SYMBOLS+='$(KSYM)'

TODO

@@ -1,6 +1,7 @@
 
-1) Remove all instances of KFI_* error codes, replace with std linux/errno.h
-   codes.
+1) remove kfi_info.handle field, use kfi_info.connreq instead for IB connection
+   management.
 
-2) Set kfi_strerror() full-text output to be under KFI_DEBUG.
+2) Remove all instances of KFI_* error codes, replace with std linux/errno.h
+   codes.
 

include/kfabric.h

@@ -268,7 +268,7 @@ struct kfi_rx_attr {
 };
 
 struct kfi_ep_attr {
-	enum kfi_ep_type		type;
+	enum kfi_ep_type	type;
 	uint32_t		protocol;
 	uint32_t		protocol_version;
 	size_t			max_msg_size;
@@ -373,9 +373,21 @@ struct kfid {
 	atomic_t		ref_cnt;
 };
 
+/*
+ * Retrieve the fabric info based on given hints. Can return a chain of multiple
+ * kfi_info instances.
+ */
 uint32_t kfi_version(void);
-int kfi_getinfo(uint32_t version, struct kfi_info *hints, struct kfi_info **info);
+int kfi_getinfo(uint32_t version, struct kfi_info *hints,
+		struct kfi_info **info);
+
+/*
+ * All fabric info instances returned from kfi_getinfo() should be recycled
+ * through kfi_freeinfo(). If a chain of multiple kfi_info instances are passed,
+ * all instances in the chain will be freed.
+ */
 void kfi_freeinfo(struct kfi_info *info);
+
 struct kfi_info *kfi_dupinfo(const struct kfi_info *info);
 
 struct kfi_ops_fabric {

include/kfi_prov.h

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005, 2006 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
+ * Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2015 NetApp, Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _KFI_PROV_H_
+#define _KFI_PROV_H_
+
+#include <linux/types.h>
+#include <linux/completion.h>
+#include <kfabric.h>
+
+/*
+ * This header file declares types and routines to be used by KFI
+ * providers only. Not to be included by clients of the KFI framework.
+ */
+
+/*
+ * The provider instance. Each provider instance is uniquely identified
+ * by its name.
+ */
+struct kfi_provider {
+	uint32_t version;
+	uint32_t kfi_version;
+	const char *name;
+	int (*kgetinfo)(uint32_t version,
+	                struct kfi_info *hints,
+	                struct kfi_info **info);
+	int (*kfabric)(struct kfi_fabric_attr *attr,
+	                struct kfid_fabric **fabric,
+	                void *context);
+	void (*kfreeinfo)(struct kfi_info *info);
+	void (*cleanup)(void);
+	struct completion comp;
+	atomic_t ref_cnt;
+};
+
+/*
+ * Each provider must register at least one provider instance to KFI framework.
+ * Therefore fabric services of that provider can be found 
+ */
+int kfi_provider_register(struct kfi_provider *provider);
+
+/*
+ * Any provider instances registered to KFI framework must be deregistered
+ * before it can be freed.
+ */
+int kfi_provider_deregister(struct kfi_provider *provider);
+
+/*
+ * The provider instance should be referenced / dereferenced when any KFI
+ * objects (IDs) are created / destroyed by that provider.
+ */
+static inline void
+kfi_ref_provider(struct kfi_provider *provider)
+{
+	atomic_inc(&provider->ref_cnt);
+};
+
+static inline void
+kfi_deref_provider(struct kfi_provider *provider)
+{
+	if (atomic_dec_and_test(&provider->ref_cnt))
+		complete(&provider->comp);
+};
+
+/*
+ * Helper routine to allocate a kfi_info instance.
+ */
+struct kfi_info *kfi_allocinfo(void);
+
+/*
+ * Helper routine to duplicate a kfi_info instance.
+ */
+struct kfi_info *kfi_dupinfo(const struct kfi_info *info);
+
+/*
+ * Helper routine to recycle a kfi_info instance. Instances returned to clients
+ * in kfi_getinfo() should be recycled by clients through kfi_freeinfo().
+ */
+void kfi_deallocinfo(const struct kfi_info *info);
+
+/*
+ * Each KFI object should initialize its reference count through kfi_init_id()
+ * immediately after the ID is allocated.
+ */
+static inline void kfi_init_id(struct kfid *fid)
+{
+	init_completion(&fid->comp);
+	atomic_set(&fid->ref_cnt, 1);
+};
+
+/*
+ * Each KFI object should be referneced / dereferenced when a provider routine
+ * accesses the object, or another KFI object obatains / releases a reference
+ * to that object.
+ */
+static inline void kfi_ref_id(struct kfid *fid)
+{
+	atomic_inc(&fid->ref_cnt);
+};
+
+static inline void kfi_deref_id(struct kfid *fid)
+{
+	if (atomic_dec_and_test(&fid->ref_cnt))
+		complete(&fid->comp);
+};
+
+/*
+ * Each KFI object should wait for all pending references be released through
+ * kfi_close_id() before it can be destroyed.
+ */
+static inline void kfi_close_id(struct kfid *fid)
+{
+	kfi_deref_id(fid);
+	wait_for_completion(&fid->comp);
+};
+
+#endif /* _KFI_PROV_H_ */

tests/ibverbs/mm2/cli/README

@@ -1,7 +1,7 @@
 
 KFI/verbs provider test: client side.
 
-  Pass multiple message of specified len in bytes
+  Pass multiple message of specified byte len
 
 Adjust SVR=ipAddr in Makefile to send to correct server: default 192.168.x.y
 

tests/ibverbs/mm2/cli/test.c

@@ -104,17 +104,12 @@ simple_context_t *kfi_to_simple_context(void *ctx)
 int match_provider(struct kfi_info **prov)
 {
 	struct kfi_info		hints = { 0 };
-	struct kfi_fabric_attr	attr = { 0 };
 	struct sockaddr_in	addr = { 0 };
 	int			ret;
 
-	print_trace("in\n");
-
 	/* ibverbs provider */
-	hints.ep_type		= KFI_EP_MSG;
-	hints.caps		= KFI_MSG | FI_CANCEL;
+	hints.caps		= KFI_MSG | KFI_CANCEL | KFI_RECV;
 	hints.addr_format	= KFI_SOCKADDR_IN;
-
 	hints.src_addr		= &addr;
 	hints.src_addrlen	= sizeof(addr);
 	addr.sin_family		= AF_INET;
@@ -128,10 +123,8 @@ int match_provider(struct kfi_info **prov)
 		return -EINVAL;
 	}
 
-	hints.fabric_attr	= &attr;
-	hints.fabric_attr->prov_name = kstrdup("ibverbs", GFP_KERNEL);
-
-	ret = kfi_getinfo(KFI_VERSION(1, 0), &hints, prov);
+	ret = kfi_getinfo(KFI_VERSION(KFI_MAJOR_VERSION, KFI_MINOR_VERSION),
+			  &hints, prov);
 	if (ret) {
 		print_err("ERR: kfi_getinfo() '%s'\n", kfi_strerror(ret));
 		return ret;
@@ -147,15 +140,13 @@ int match_provider(struct kfi_info **prov)
 	return 0;
 }
 
-int client_connect(struct fi_info *prov, simple_context_t *ctx)
+int client_connect(struct kfi_info *prov, simple_context_t *ctx)
 {
 	struct kfi_eq_attr	eq_attr = { 0 };
 	struct kfi_cq_attr	cq_attr = { 0 };
 	struct sockaddr_in	addr = { 0 };
 	int			ret;
 
-	print_trace("in\n");
-
 	connected = 0;
 
 	ret = kfi_fabric(prov->fabric_attr, &ctx->fabric, NULL);
@@ -268,8 +259,6 @@ void client_disconnect(simple_context_t *ctx)
 {
 	int ret;
 
-	print_trace("in\n");
-
 	if (connected) {
 		ret = kfi_shutdown(ctx->ep, 0);
 		if (ret)
@@ -325,8 +314,6 @@ int do_test(void)
 	int			eagain_cnt = EAGAIN_TRIES;
 #endif
 
-	print_trace("in\n");
-
 	if (!ctx.buf) {
 		ctx.buf = kmalloc(len, GFP_KERNEL);
 		if (!ctx.buf) {
@@ -335,7 +322,7 @@ int do_test(void)
 		}
 
 		ret = kfi_mr_reg(ctx.domain, ctx.buf, len, 0, 0, 0, 0,
-				&ctx.mr, NULL);
+				&ctx.mr, NULL, NULL);
 		if (ret) {
 			print_err("kfi_mr_reg returned %d\n", ret);
 			kfree(ctx.buf);
@@ -505,8 +492,6 @@ int create_connection(void)
 	uint32_t		event;
 	int			ret = -1;
 
-	print_trace("in\n");
-
 	memset(&ctx, 0, sizeof(ctx));
 
 	if (match_provider(&prov))

tests/ibverbs/mm2/svr/README

@@ -1,4 +1,3 @@
-[ Thu Feb 26 15:15:24 PST 2015 ] stan
 
 KFI/verbs provider test: server side.
 	Pass multiple messages with post_depth,

tests/ibverbs/mm2/svr/test.c

@@ -115,10 +115,8 @@ int match_provider(struct kfi_info **prov)
 	int			ret;
 
 	/* ibverbs matching provider */
-//XXX	hints.ep_type		= KFI_EP_MSG;
-//XXX	hints.caps		= KFI_MSG | KFI_CANCEL | KFI_SOURCE;
-
-//XXX	hints.addr_format	= KFI_SOCKADDR_IN;
+	hints.caps		= KFI_MSG | KFI_CANCEL | KFI_SOURCE;
+	hints.addr_format	= KFI_SOCKADDR_IN;
 	hints.src_addr		= &addr;
 	hints.src_addrlen	= sizeof(addr);
 
@@ -133,10 +131,8 @@ int match_provider(struct kfi_info **prov)
 		return -EINVAL;
 	}
 
-	hints.fabric_attr		= &attr;
-	hints.fabric_attr->prov_name	= kstrdup("ibverbs", GFP_KERNEL);
-
-	ret = kfi_getinfo(KFI_VERSION(1, 0), &hints, prov);
+	ret = kfi_getinfo(KFI_VERSION(KFI_MAJOR_VERSION, KFI_MINOR_VERSION),
+			  &hints, prov);
 	if (ret) {
 		print_err("ERR: kfi_getinfo() '%s'\n", kfi_strerror(ret));
 		return ret;