From 4948417abacfa0795cea2d287fefafe3ceb07a1a Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sun, 25 Aug 2019 17:29:23 +0300
Subject: [PATCH 01/41] Removed some features that are not supported in
 rdma-core

Descprition:
	Pieces of code under HAVE_RSS_EXP, HAVE_ACCL_VERBS,
	HAVE_MASKED_ATOMICS, HAVE_PACKET_PACING_EXP,
	HAVE_PACKET_PACING_EXTENSION_EXP, HAVE_SCATTER_FCS,
	HAVE_RAW_ETH_EXP, HAVE_EXP_ODP, HAVE_VERBS_EXP,
	HAVE_DC, SNIFFER_EXP, EXP_OOO_ATTR, RAW_ETH_EXP
	were removed

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac                      |  139 +--
 src/atomic_bw.c                   |   12 -
 src/atomic_lat.c                  |   13 -
 src/perftest_communication.c      |    4 +-
 src/perftest_parameters.c         |  120 +-
 src/perftest_parameters.h         |   20 +-
 src/perftest_resources.c          | 1888 +++--------------------------
 src/perftest_resources.h          |   94 +-
 src/raw_ethernet_resources.c      |  228 +---
 src/raw_ethernet_resources.h      |   16 -
 src/raw_ethernet_send_burst_lat.c |   37 -
 src/raw_ethernet_send_bw.c        |   74 +-
 src/raw_ethernet_send_lat.c       |   38 -
 13 files changed, 184 insertions(+), 2499 deletions(-)

diff --git a/configure.ac b/configure.ac
index c9268593..17aec4c9 100755
--- a/configure.ac
+++ b/configure.ac
@@ -44,20 +44,11 @@ AC_SUBST(MINOR_VERSION)
 AC_ARG_VAR(CUDA_H_PATH, help-string)
 AC_ARG_VAR(RANLIB, ranlib tool)
 
-AC_ARG_ENABLE([verbs_exp],
-[AS_HELP_STRING([--disable-verbs_exp],
-[Disable Verbs Experimental])],
-[], [enable_verbs_exp=yes])
-
 AC_ARG_ENABLE([ibv_wr_api],
 [AS_HELP_STRING([--enable-ibv_wr_api],
 [Enable new post send API])],
 [enable_ibv_wr_api=yes], [enable_ibv_wr_api=no])
 
-AS_IF([test "x$enable_verbs_exp" = "xyes"],
-      [USE_VERBS_EXP=yes],
-        [USE_VERBS_EXP=no])
-
 AS_IF([test "x$enable_ibv_wr_api" = "xyes"],
       [USE_IBV_WR_API=yes],
         [USE_IBV_WR_API=no])
@@ -77,14 +68,6 @@ AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-
 AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found]))
 AC_CHECK_LIB([m], [log], [LIBMATH=-lm], AC_MSG_ERROR([libm not found]))
 
-AC_TRY_LINK([#include <infiniband/verbs.h>],
-	[struct ibv_exp_flow *t = ibv_exp_create_flow(NULL,NULL);],[HAVE_RAW_ETH_EXP=yes], [HAVE_RAW_ETH_EXP=no])
-
-AM_CONDITIONAL([HAVE_RAW_ETH_EXP],[test "x$HAVE_RAW_ETH_EXP" = "xyes"])
-if test $HAVE_RAW_ETH_EXP = yes; then
-	AC_DEFINE([HAVE_RAW_ETH_EXP], [1], [Enable RAW_ETH_TEST_EXP])
-fi
-
 AC_TRY_LINK([#include <infiniband/verbs.h>],
 	[struct ibv_flow *t = ibv_create_flow(NULL,NULL);],[HAVE_RAW_ETH_REG=yes], [HAVE_RAW_ETH_REG=no])
 
@@ -94,7 +77,7 @@ if test $HAVE_RAW_ETH_REG = yes; then
 fi
 
 AM_CONDITIONAL([HAVE_RAW_ETH],true)
-if [test $HAVE_RAW_ETH_REG = yes] || [test $HAVE_RAW_ETH_EXP = yes]; then
+if [test $HAVE_RAW_ETH_REG = yes]; then
 	AC_DEFINE([HAVE_RAW_ETH], [1], [Enable RAW_ETH_TEST])
 else
 	AM_CONDITIONAL([HAVE_RAW_ETH],false)
@@ -141,14 +124,6 @@ if test $HAVE_SCIF = yes; then
         AC_DEFINE([HAVE_SCIF], [1], [Enable SCIF link Layer])
 fi
 
-AC_CHECK_HEADERS([infiniband/verbs_exp.h],[HAVE_VERBS_EXP=yes],[HAVE_VERBS_EXP=no],[#include <infiniband/verbs.h>])
-AM_CONDITIONAL([HAVE_VERBS_EXP],[test "x$HAVE_VERBS_EXP" = "xyes"])
-if [test $HAVE_VERBS_EXP = yes] && [test $USE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_VERBS_EXP], [1], [Have verbs_exp header])
-else
-	HAVE_VERBS_EXP=no
-fi
-
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;],[HAVE_IBV_WR_API=yes],[HAVE_IBV_WR_API=no])
@@ -159,90 +134,27 @@ else
 	HAVE_IBV_WR_API=no
 fi
 
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS;],[HAVE_MASKED_ATOMICS=yes], [HAVE_MASKED_ATOMICS=no])
-AM_CONDITIONAL([HAVE_MASKED_ATOMICS],[test "x$HAVE_MASKED_ATOMICS" = "xyes"])
-if [test $HAVE_MASKED_ATOMICS = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_MASKED_ATOMICS], [1], [Have MASKED ATOMICS in verbs_exp header])
-fi
-
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ;],[HAVE_RSS_EXP=yes], [HAVE_RSS_EXP=no])
-AM_CONDITIONAL([HAVE_RSS_EXP],[test "x$HAVE_RSS_EXP" = "xyes"])
-if [test $HAVE_RSS_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_RSS_EXP], [1], [Have RSS in verbs_exp header])
-fi
-
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_DCT_CREATE_FLAGS_MASK;],[HAVE_DC=yes], [HAVE_DC=no])
-AM_CONDITIONAL([HAVE_DC],[test "x$HAVE_DC" = "xyes"])
-if [test $HAVE_DC = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_DC], [1], [Enable DC feature])
-fi
-
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_INTF_GLOBAL;],[HAVE_ACCL_VERBS=yes], [HAVE_ACCL_VERBS=no])
-AM_CONDITIONAL([HAVE_ACCL_VERBS],[test "x$HAVE_ACCL_VERBS" = "xyes"])
-if [test $HAVE_ACCL_VERBS = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_ACCL_VERBS], [1], [Enable Accelerated verbs feature])
-fi
-
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_FLOW_SPEC_IPV6;],[HAVE_IPV6=yes], [HAVE_IPV6=no])
 AM_CONDITIONAL([HAVE_IPV6],[test "x$HAVE_IPV6" = "xyes"])
-if [test $HAVE_IPV6 = yes] && [test $HAVE_VERBS_EXP = no]; then
+if [test $HAVE_IPV6 = yes]; then
         AC_DEFINE([HAVE_IPV6], [1], [Enable IPv6 Flow Specification])
 fi
 
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_FLOW_SPEC_IPV6_EXT;],[HAVE_IPV6=yes], [HAVE_IPV6=no])
-AM_CONDITIONAL([HAVE_IPV6],[test "x$HAVE_IPV6" = "xyes"])
-if [test $HAVE_IPV6 = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_IPV6], [1], [Enable IPv6 Flow Specification])
-fi
-
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_FLOW_SPEC_IPV4_EXT;],[HAVE_IPV4_EXT=yes], [HAVE_IPV4_EXT=no])
-AM_CONDITIONAL([HAVE_IPV4_EXT],[test "x$HAVE_IPV4_EXT" = "xyes"])
-if [test $HAVE_IPV4_EXT = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_IPV4_EXT], [1], [Enable IPv4 Extended Flow Specification])
-fi
-
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_FLOW_SPEC_IPV4_EXT;],[HAVE_IPV4_EXT=yes], [HAVE_IPV4_EXT=no])
 AM_CONDITIONAL([HAVE_IPV4_EXT],[test "x$HAVE_IPV4_EXT" = "xyes"])
-if [test $HAVE_IPV4_EXT = yes] && [test $HAVE_VERBS_EXP = no]; then
+if [test $HAVE_IPV4_EXT = yes]; then
         AC_DEFINE([HAVE_IPV4_EXT], [1], [Enable IPv4 Extended Flow Specification])
 fi
 
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_FLOW_ATTR_SNIFFER;],[HAVE_SNIFFER_EXP=yes], [HAVE_SNIFFER_EXP=no])
-AM_CONDITIONAL([HAVE_SNIFFER_EXP],[test "x$HAVE_SNIFFER_EXP" = "xyes"])
-if [test $HAVE_SNIFFER_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_SNIFFER_EXP], [1], [Enable Sniffer Flow Specification])
-fi
-
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_FLOW_ATTR_SNIFFER;],[HAVE_SNIFFER=yes], [HAVE_SNIFFER=no])
 AM_CONDITIONAL([HAVE_SNIFFER],[test "x$HAVE_SNIFFER" = "xyes"])
-if [test $HAVE_SNIFFER = yes] && [test $HAVE_VERBS_EXP = no]; then
+if [test $HAVE_SNIFFER = yes]; then
         AC_DEFINE([HAVE_SNIFFER], [1], [Enable Sniffer Flow Specification])
 fi
 
@@ -261,55 +173,12 @@ if [test $HAVE_EX_ODP = yes] && [test $HAVE_EX = yes]; then
         AC_DEFINE([HAVE_EX_ODP], [1], [Have Extended  ODP support])
 fi
 
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_ACCESS_ON_DEMAND;],[HAVE_EXP_ODP=yes], [HAVE_EXP_ODP=no])
-AM_CONDITIONAL([HAVE_EXP_ODP],[test "x$HAVE_EXP_ODP" = "xyes"])
-if [test $HAVE_EXP_ODP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-	if  [test $HAVE_EX_ODP = no] ; then
-		AC_DEFINE([HAVE_EXP_ODP], [1], [Have Experimental ODP support])
-	fi
-fi
-
 if [test "$CUDA_H_PATH" ]; then
 	AC_DEFINE([HAVE_CUDA], [1], [Enable CUDA feature])
 	AC_DEFINE_UNQUOTED([CUDA_PATH], "$CUDA_H_PATH" , [Enable CUDA feature])
 	LIBS=$LIBS" -lcuda"
 fi
 
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_DEVICE_SCATTER_FCS;],[HAVE_SCATTER_FCS=yes], [HAVE_SCATTER_FCS=no])
-AM_CONDITIONAL([HAVE_SCATTER_FCS],[test "x$HAVE_SCATTER_FCS" = "xyes"])
-if [test $HAVE_SCATTER_FCS = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_SCATTER_FCS], [1], [Have SCATTER_FCS support])
-fi
-
-AC_TRY_LINK([
-#include <infiniband/verbs.h>
-#include <infiniband/verbs_exp.h>],
-        [int x = ibv_exp_query_gid_attr(NULL, 1, 0, NULL);],[HAVE_GID_ATTR=yes], [HAVE_GID_ATTR=no])
-AM_CONDITIONAL([HAVE_GID_ATTR],[test "x$HAVE_GID_ATTR" = "xyes"])
-if [test $HAVE_GID_ATTR = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_GID_ATTR], [1], [Enable GID Attribute query feature])
-fi
-
-AC_TRY_LINK([ #include <infiniband/verbs_exp.h>],
-	[struct ibv_exp_qp_attr *attr; int x = attr->rate_limit;],[HAVE_PACKET_PACING_EXP=yes], [HAVE_PACKET_PACING_EXP=no])
-AM_CONDITIONAL([HAVE_PACKET_PACING_EXP],[test "x$HAVE_PACKET_PACING_EXP" = "xyes"])
-if [test $HAVE_PACKET_PACING_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-	AC_DEFINE([HAVE_PACKET_PACING_EXP], [1], [Have PACKET_PACING_EXP support])
-fi
-
-AC_TRY_LINK([ #include <infiniband/verbs_exp.h>],
-	[struct ibv_exp_qp_attr *attr; struct ibv_exp_burst_info x = attr->burst_info;],[HAVE_PACKET_PACING_EXTENSION_EXP=yes], [HAVE_PACKET_PACING_EXTENSION_EXP=no])
-AM_CONDITIONAL([HAVE_PACKET_PACING_EXTENSION_EXP],[test "x$HAVE_PACKET_PACING_EXTENSION_EXP" = "xyes"])
-if [test $HAVE_PACKET_PACING_EXTENSION_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-	AC_DEFINE([HAVE_PACKET_PACING_EXTENSION_EXP], [1], [Have PACKET_PACING_EXTENSION_EXP support])
-fi
-
 AC_TRY_LINK([#include <infiniband/verbs.h>],
 	[struct ibv_qp_attr *attr; int x = attr->rate_limit;],[HAVE_PACKET_PACING=yes], [HAVE_PACKET_PACING=no])
 AM_CONDITIONAL([HAVE_PACKET_PACING],[test "x$HAVE_PACKET_PACING" = "xyes"])
diff --git a/src/atomic_bw.c b/src/atomic_bw.c
index fee4b291..15f8ea29 100755
--- a/src/atomic_bw.c
+++ b/src/atomic_bw.c
@@ -87,18 +87,6 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	#ifdef HAVE_MASKED_ATOMICS
-	if (check_masked_atomics_support(&ctx)) {
-		user_param.masked_atomics = 1;
-		user_param.use_exp = 1;
-	}
-
-	if (user_param.masked_atomics && (user_param.work_rdma_cm || user_param.use_rdma_cm)) {
-		fprintf(stderr, "atomic test is not supported with -R/-z flag (rdma_cm) with this device.\n");
-		return FAILURE;
-	}
-	#endif
-
 	/* See if MTU and link type are valid and supported. */
 	if (check_link(ctx.context, &user_param)) {
 		fprintf(stderr, " Couldn't get context for the device\n");
diff --git a/src/atomic_lat.c b/src/atomic_lat.c
index cc0c8460..f038b2d4 100755
--- a/src/atomic_lat.c
+++ b/src/atomic_lat.c
@@ -103,19 +103,6 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	#ifdef HAVE_MASKED_ATOMICS
-	if (check_masked_atomics_support(&ctx)) {
-		user_param.masked_atomics = 1;
-		user_param.use_exp = 1;
-	}
-
-	if (user_param.masked_atomics && (user_param.work_rdma_cm || user_param.use_rdma_cm)) {
-		fprintf(stderr, "atomic test is not supported with -R/-z flag (rdma_cm) with this device.\n");
-		return FAILURE;
-	}
-
-	#endif
-
 	/* See if MTU and link type are valid and supported. */
 	if (check_link(ctx.context,&user_param)) {
 		fprintf(stderr, " Couldn't get context for the device\n");
diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index b26152fd..264ddfa8 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -527,7 +527,7 @@ static int rdma_read_keys(struct pingpong_dest *rem_dest,
 	return 0;
 }
 
-
+//TODO: Move to another gid
 #ifdef HAVE_GID_ATTR
 enum who_is_better {LEFT_IS_BETTER, EQUAL, RIGHT_IS_BETTER};
 
@@ -838,7 +838,6 @@ int set_up_connection(struct pingpong_context *ctx,
 	}
 	#endif
 
-	#ifdef HAVE_DC
 	if(user_param->machine == SERVER || user_param->duplex || user_param->tst == LAT) {
 		if (user_param->connection_type == DC) {
 			for (i=0; i < user_param->num_of_qps; i++) {
@@ -849,7 +848,6 @@ int set_up_connection(struct pingpong_context *ctx,
 			}
 		}
 	}
-	#endif
 	return 0;
 }
 
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index b71f5fc8..c23f24ce 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -421,7 +421,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf(" Create memory region for each qp.\n");
 	}
 
-	#if defined HAVE_EX_ODP || defined HAVE_EXP_ODP
+	#if defined HAVE_EX_ODP
 	printf("      --odp ");
 	printf(" Use On Demand Paging instead of Memory Registration.\n");
 	#endif
@@ -472,22 +472,8 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf(" Use CUDA lib for GPU-Direct testing.\n");
 		#endif
 
-		#ifdef HAVE_VERBS_EXP
-		printf("      --use_exp ");
-		printf(" Use Experimental verbs in data path. Default is OFF.\n");
-		#endif
-
 		printf("      --use_hugepages ");
 		printf(" Use Hugepages instead of contig, memalign allocations.\n");
-
-
-		#ifdef HAVE_ACCL_VERBS
-		printf("      --use_res_domain ");
-		printf(" Use shared resource domain\n");
-
-		printf("      --verb_type=<option> ");
-		printf(" Set verb type: normal, accl. Default is normal.\n");
-		#endif
 	}
 
 	if (tst == BW || tst == LAT_BY_BW) {
@@ -514,7 +500,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf("        Note: in Latency under load test SW rate limit is forced\n");
 
 	}
-	#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
+	#if defined HAVE_OOO_ATTR
 	printf("      --use_ooo ");
 	printf(" Use out of order data placement\n");
 	#endif
@@ -574,11 +560,6 @@ void usage_raw_ethernet(TestType tst)
 		printf("  -v, --mac_fwd ");
 		printf(" run mac forwarding test \n");
 
-		#ifdef HAVE_SCATTER_FCS
-		printf("      --disable_fcs ");
-		printf(" Disable Scatter FCS feature. (Scatter FCS is enabled by default when using --use_exp flag). \n");
-		#endif
-
 		printf("      --flows");
 		printf(" set number of TCP/UDP flows, starting from <src_port, dst_port>. \n");
 
@@ -591,7 +572,7 @@ void usage_raw_ethernet(TestType tst)
 		printf("      --reply_every ");
 		printf(" in latency test, receiver pong after number of received pings\n");
 
-		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+		#if defined HAVE_SNIFFER
 		printf("      --sniffer");
 		printf(" run sniffer mode.\n");
 		#endif
@@ -715,7 +696,6 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->cpu_util_data.enable	= 0;
 	user_param->retry_count			= DEF_RETRY_COUNT;
 	user_param->dont_xchg_versions		= 0;
-	user_param->use_exp			= 0;
 	user_param->ipv6			= 0;
 	user_param->report_per_port		= 0;
 	user_param->use_odp			= 0;
@@ -724,21 +704,16 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->use_sniffer			= 0;
 	user_param->check_alive_exited		= 0;
 	user_param->raw_mcast			= 0;
-	user_param->masked_atomics		= 0;
 	user_param->cache_line_size		= get_cache_line_size();
 	user_param->cycle_buffer		= sysconf(_SC_PAGESIZE);
 
 	if (user_param->cycle_buffer <= 0) {
 		user_param->cycle_buffer = DEF_PAGE_SIZE;
 	}
-	user_param->verb_type		= NORMAL_INTF;
-	user_param->is_exp_cq		= 0;
-	user_param->is_exp_qp		= 0;
 	user_param->use_res_domain	= 0;
 	user_param->mr_per_qp		= 0;
 	user_param->dlid		= 0;
 	user_param->traffic_class	= 0;
-	user_param->disable_fcs		= 0;
 	user_param->flows		= DEF_FLOWS;
 	user_param->flows_burst		= 1;
 	user_param->perform_warm_up	= 0;
@@ -806,6 +781,7 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
 		exit(1);
 		#endif
 	} else if (strcmp(connStr[5], optarg)==0) {
+		// TODO: remove when DC for rdma-core is supported
 		#ifdef HAVE_DC
 		*cptr = DC;
 		#else
@@ -1385,12 +1361,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		user_param->margin = user_param->duration / 4;
 	}
 
-	#if defined(HAVE_VERBS_EXP) && defined(HAVE_DC)
-	if (user_param->connection_type == DC) {
-		user_param->use_exp = 1;
-	}
-	#endif
-
 	#ifdef HAVE_CUDA
 	if (user_param->use_cuda) {
 		if (user_param->tst != BW) {
@@ -1437,52 +1407,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit(1);
 	}
 
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->verb_type != NORMAL_INTF || user_param->use_res_domain) {
-		user_param->is_exp_cq = 1;
-		user_param->use_exp = 1;
-	}
-
-	if (user_param->verb_type == ACCL_INTF) {
-		if (user_param->connection_type != RC &&
-			user_param->connection_type != UC && user_param->connection_type != RawEth) {
-			fprintf(stderr, "Accelerated verbs support RC/UC/RAW_ETH connections only.\n");
-			exit(1);
-		}
-
-		if (user_param->verb != SEND) {
-			fprintf(stderr, "Accelerated verbs support SEND opcode only.\n");
-			exit(1);
-		}
-
-		if (user_param->num_of_qps > 1) {
-			fprintf(stderr, "Accelerated verbs in perftest support only 1 qp for now.\n");
-			exit(1);
-		}
-
-		if (user_param->tst != BW) {
-			fprintf(stderr, "Accelerated verbs in perftest supports only BW tests for now.\n");
-			exit(1);
-		}
-
-		if (user_param->duplex) {
-			fprintf(stderr, "Accelerated verbs in perftest supports only unidir tests for now\n");
-			exit(1);
-		}
-
-		if (user_param->test_method == RUN_INFINITELY) {
-			fprintf(stderr, "Accelerated verbs in perftest does not support Run Infinitely mode for now\n");
-			exit(1);
-		}
-	}
-	if (user_param->perform_warm_up &&
-	    !(user_param->tst == BW &&
-	    (user_param->verb == ATOMIC || user_param->verb == WRITE || user_param->verb == READ))) {
-		fprintf(stderr, "Perform warm up is available in ATOMIC, READ and WRITE BW tests only");
-		exit(1);
-	}
-	#endif
-
 	return;
 }
 /******************************************************************************
@@ -1867,7 +1791,6 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int flow_label_flag = 0;
 	static int retry_count_flag = 0;
 	static int dont_xchg_versions_flag = 0;
-	static int use_exp_flag = 0;
 	static int use_cuda_flag = 0;
 	static int mmap_file_flag = 0;
 	static int mmap_offset_flag = 0;
@@ -1879,13 +1802,11 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int use_promiscuous_flag = 0;
 	static int use_sniffer_flag = 0;
 	static int raw_mcast_flag = 0;
-	static int verb_type_flag = 0;
 	static int use_res_domain_flag = 0;
 	static int mr_per_qp_flag = 0;
 	static int dlid_flag = 0;
 	static int tclass_flag = 0;
 	static int wait_destroy_flag = 0;
-	static int disable_fcs_flag = 0;
 	static int flows_flag = 0;
 	static int flows_burst_flag = 0;
 	static int force_link_flag = 0;
@@ -2002,24 +1923,15 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{ .name = "odp",		.has_arg = 0, .flag = &odp_flag, .val = 1},
 			{ .name = "use_hugepages",		.has_arg = 0, .flag = &hugepages_flag, .val = 1},
 			{ .name = "promiscuous",	.has_arg = 0, .flag = &use_promiscuous_flag, .val = 1},
-			#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+			#if defined HAVE_SNIFFER
 			{ .name = "sniffer",		.has_arg = 0, .flag = &use_sniffer_flag, .val = 1},
 			#endif
 			{ .name = "raw_mcast",		.has_arg = 0, .flag = &raw_mcast_flag, .val = 1},
-			#ifdef HAVE_VERBS_EXP
-			{ .name = "use_exp",		.has_arg = 0, .flag = &use_exp_flag, .val = 1},
-			#endif
-			#ifdef HAVE_ACCL_VERBS
-			{ .name = "verb_type",		.has_arg = 1, .flag = &verb_type_flag, .val = 1},
-			{ .name = "use_res_domain",	.has_arg = 0, .flag = &use_res_domain_flag, .val = 1},
-			#endif
+
 			{ .name = "mr_per_qp",		.has_arg = 0, .flag = &mr_per_qp_flag, .val = 1},
 			{ .name = "dlid",		.has_arg = 1, .flag = &dlid_flag, .val = 1},
 			{ .name = "tclass",		.has_arg = 1, .flag = &tclass_flag, .val = 1},
 			{ .name = "wait_destroy",	.has_arg = 1, .flag = &wait_destroy_flag, .val = 1},
-			#ifdef HAVE_SCATTER_FCS
-			{ .name = "disable_fcs",	.has_arg = 0, .flag = &disable_fcs_flag, .val = 1},
-			#endif
 			{ .name = "flows",		.has_arg = 1, .flag = &flows_flag, .val = 1},
 			{ .name = "flows_burst",	.has_arg = 1, .flag = &flows_burst_flag, .val = 1},
 			{ .name = "reply_every",	.has_arg = 1, .flag = &reply_every_flag, .val = 1},
@@ -2027,7 +1939,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{ .name = "vlan_en",            .has_arg = 0, .flag = &vlan_en, .val = 1 },
 			{ .name = "vlan_pcp",		.has_arg = 1, .flag = &vlan_pcp_flag, .val = 1 },
 
-			#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
+			#if defined HAVE_OOO_ATTR
 			{ .name = "use_ooo",		.has_arg = 0, .flag = &use_ooo_flag, .val = 1},
 			#endif
 			{ 0 }
@@ -2377,17 +2289,6 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 					}
 					retry_count_flag = 0;
 				}
-				if (verb_type_flag) {
-					if (strcmp("normal",optarg) == 0) {
-						user_param->verb_type = NORMAL_INTF;
-					} else if (strcmp("accl",optarg) == 0) {
-						user_param->verb_type = ACCL_INTF;
-					} else {
-						fprintf(stderr, " Invalid verb type. Please choose normal/accl.\n");
-						return FAILURE;
-					}
-					verb_type_flag = 0;
-				}
 				if (mmap_file_flag) {
 					user_param->mmap_file = strdup(optarg);
 					mmap_file_flag = 0;
@@ -2524,10 +2425,6 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 		user_param->dont_xchg_versions = 1;
 	}
 
-	if (use_exp_flag) {
-		user_param->use_exp = 1;
-	}
-
 	if (use_res_domain_flag) {
 		user_param->use_res_domain = 1;
 	}
@@ -2646,9 +2543,6 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 		user_param->mr_per_qp = 1;
 	}
 
-	if (disable_fcs_flag) {
-		user_param->disable_fcs = 1;
-	}
 	if (perform_warm_up_flag) {
 		user_param->perform_warm_up = 1;
 	}
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index 98210a05..0e874d6c 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -66,10 +66,6 @@
 #include <config.h>
 #endif
 
-#ifdef HAVE_VERBS_EXP
-#include <infiniband/verbs_exp.h>
-#endif
-
 #ifdef HAVE_CUDA
 #include CUDA_PATH
 #endif
@@ -312,12 +308,6 @@ enum rate_limiter_types {HW_RATE_LIMIT, SW_RATE_LIMIT, PP_RATE_LIMIT, DISABLE_RA
 /* Verbosity Levels for test report */
 enum verbosity_level {FULL_VERBOSITY=-1, OUTPUT_BW=0, OUTPUT_MR, OUTPUT_LAT };
 
-/*Accelerated verbs */
-enum verbs_intf {
-	NORMAL_INTF,
-	ACCL_INTF,
-};
-
 struct cpu_util_data {
 	int enable;
 	long long ustat[2];
@@ -487,7 +477,6 @@ struct perftest_parameters {
 	int 				flow_label;
 	int 				retry_count;
 	int 				dont_xchg_versions;
-	int 				use_exp;
 	int 				ipv6;
 	int 				raw_ipv6;
 	int 				report_per_port;
@@ -500,21 +489,17 @@ struct perftest_parameters {
 	int				masked_atomics;
 	int				cycle_buffer;
 	int				cache_line_size;
-	enum verbs_intf			verb_type;
-	int				is_exp_cq;
-	int				is_exp_qp;
 	int				use_res_domain;
 	int				mr_per_qp;
 	uint16_t			dlid;
 	uint8_t				traffic_class;
 	uint32_t			wait_destroy;
-	int				disable_fcs;
 	int				flows;
 	int				flows_burst;
 	uint32_t			reply_every;
 	int				perform_warm_up;
 	int				use_ooo;
-	int                             vlan_en;
+	int				vlan_en;
 	uint32_t			vlan_pcp;
 	void 				(*print_eth_func)(void*);
 	struct counter_context		*counter_ctx;
@@ -568,9 +553,6 @@ static const struct rate_gbps_string RATE_VALUES[RATE_VALUES_COUNT] = {
 	{IBV_RATE_MAX, "MAX"}
 };
 
-
-
-
 /* link_layer_str
  *
  * Description : Return a String representation of the link type.
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index a5aab82c..801ee9f6 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -29,10 +29,6 @@
 #include "perftest_resources.h"
 #include "raw_ethernet_resources.h"
 
-#ifdef HAVE_VERBS_EXP
-static enum ibv_exp_wr_opcode exp_opcode_verbs_array[] = {IBV_EXP_WR_SEND,IBV_EXP_WR_RDMA_WRITE,IBV_EXP_WR_RDMA_READ};
-static enum ibv_exp_wr_opcode exp_opcode_atomic_array[] = {IBV_EXP_WR_ATOMIC_CMP_AND_SWP,IBV_EXP_WR_ATOMIC_FETCH_AND_ADD};
-#endif
 static enum ibv_wr_opcode opcode_verbs_array[] = {IBV_WR_SEND,IBV_WR_RDMA_WRITE,IBV_WR_RDMA_READ};
 static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP,IBV_WR_ATOMIC_FETCH_AND_ADD};
 
@@ -172,26 +168,6 @@ static int pp_free_mmap(struct pingpong_context *ctx)
 	return 0;
 }
 
-#ifdef HAVE_VERBS_EXP
-static void get_verbs_pointers(struct pingpong_context *ctx)
-{
-	ctx->exp_post_send_func_pointer = ibv_exp_get_provider_func(ctx->context,IBV_EXP_POST_SEND_FUNC);
-	if (!ctx->exp_post_send_func_pointer) {
-		fprintf(stderr, "Couldn't get ibv_exp_post_send pointer\n");
-		ctx->exp_post_send_func_pointer = &ibv_exp_post_send;
-	}
-	ctx->post_send_func_pointer = ibv_exp_get_provider_func(ctx->context,IBV_POST_SEND_FUNC);
-	if (!ctx->post_send_func_pointer) {
-		fprintf(stderr, "Couldn't get ibv_post_send pointer\n");
-		ctx->post_send_func_pointer = &ibv_post_send;
-	}
-	ctx->poll_cq_func_pointer = ibv_exp_get_provider_func(ctx->context,IBV_POLL_CQ_FUNC);
-	if (!ctx->poll_cq_func_pointer) {
-		fprintf(stderr, "Couldn't get ibv_poll_cq pointer\n");
-	}
-}
-#endif
-
 static int next_word_string(char* input, char* output, int from_index)
 {
 	int i = from_index;
@@ -253,21 +229,6 @@ static void get_cpu_stats(struct perftest_parameters *duration_param,int stat_in
 	}
 }
 
-#ifdef HAVE_VERBS_EXP
-static int check_for_contig_pages_support(struct ibv_context *context)
-{
-	int answer;
-	struct ibv_exp_device_attr attr;
-	memset(&attr,0,sizeof attr);
-	if (ibv_exp_query_device(context,&attr)) {
-		fprintf(stderr, "Couldn't get device attributes\n");
-		return FAILURE;
-	}
-	answer = ( attr.exp_device_cap_flags &= IBV_EXP_DEVICE_MR_ALLOCATE) ? SUCCESS : FAILURE;
-	return answer;
-}
-#endif
-
 /* _new_post_send.
  *
  * Description :
@@ -588,11 +549,7 @@ static struct ibv_qp *ctx_xrc_qp_create(struct pingpong_context *ctx,
 	enum ibv_wr_opcode opcode;
 	#endif
 
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_qp_init_attr qp_init_attr;
-	#else
 	struct ibv_qp_init_attr_ex qp_init_attr;
-	#endif
 
 	memset(&qp_init_attr, 0, sizeof(qp_init_attr));
 
@@ -637,176 +594,7 @@ static struct ibv_qp *ctx_xrc_qp_create(struct pingpong_context *ctx,
 	}
 	#endif
 
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->use_res_domain) {
-		qp_init_attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_RES_DOMAIN;
-		qp_init_attr.res_domain = ctx->res_domain;
-	}
-	#endif
-
-	#ifdef HAVE_VERBS_EXP
-	qp = ibv_exp_create_qp(ctx->context, &qp_init_attr);
-	#else
 	qp = ibv_create_qp_ex(ctx->context, &qp_init_attr);
-	#endif
-
-	return qp;
-}
-#endif
-
-#ifdef HAVE_DC
-/******************************************************************************
- *
- ******************************************************************************/
-static int ctx_dc_tgt_create(struct pingpong_context *ctx,struct perftest_parameters *user_param,int dct_index)
-{
-	struct ibv_exp_device_attr dattr;
-	int err;
-	int num_of_qps = user_param->num_of_qps;
-	int num_of_qps_per_port = user_param->num_of_qps / 2;
-	int port_num;
-	int access_flags = 0;
-
-	memset(&dattr,0,sizeof(struct ibv_exp_device_attr));
-
-	/* in dc with bidirectional,
-	 * there are send qps and recv qps. the actual number of send/recv qps
-	 * is num_of_qps / 2.
-	 */
-	if (user_param->duplex || user_param->tst == LAT) {
-		num_of_qps /= 2;
-		num_of_qps_per_port = num_of_qps / 2;
-	}
-
-	/* first half of qps are for ib_port and second half are for ib_port2
-	 * in dc with bidirectional, the first half of qps are dc_ini qps and
-	 * the second half are dc_tgts . the first half of the send/recv qps
-	 * are for ib_port1 and the second half are for ib_port2
-	 */
-	if (user_param->dualport == ON && (dct_index % num_of_qps >= num_of_qps_per_port))
-		port_num = user_param->ib_port2;
-	else
-		port_num = user_param->ib_port;
-
-	dattr.comp_mask = IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
-		IBV_EXP_DEVICE_DC_RD_REQ |
-		IBV_EXP_DEVICE_DC_RD_RES;
-
-	err = ibv_exp_query_device(ctx->context, &dattr);
-	if (err) {
-		printf("couldn't query device extended attributes\n");
-		return -1;
-	} else {
-		if (!(dattr.comp_mask & IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS)) {
-			printf("no extended capability flgas\n");
-			return -1;
-		}
-		if (!(dattr.exp_device_cap_flags & IBV_EXP_DEVICE_DC_TRANSPORT)) {
-			printf("DC transport not enabled\n");
-			return -1;
-		}
-
-		if (!(dattr.comp_mask & IBV_EXP_DEVICE_DC_RD_REQ)) {
-			printf("no report on max requestor rdma/atomic resources\n");
-			return -1;
-		}
-
-		if (!(dattr.comp_mask & IBV_EXP_DEVICE_DC_RD_RES)) {
-			printf("no report on max responder rdma/atomic resources\n");
-			return -1;
-		}
-	}
-
-	if (user_param->verb == WRITE)
-		access_flags |= IBV_ACCESS_REMOTE_WRITE;
-	else if (user_param->verb == READ)
-		access_flags |= IBV_ACCESS_REMOTE_READ;
-	else if (user_param->verb == ATOMIC)
-		access_flags |= IBV_ACCESS_REMOTE_ATOMIC;
-
-	struct ibv_exp_dct_init_attr dctattr = {
-		.pd = ctx->pd,
-		.cq = (user_param->verb == SEND && (user_param->duplex || user_param->tst == LAT)) ? ctx->recv_cq : ctx->send_cq,
-		.srq = ctx->srq,
-		.dc_key = user_param->dct_key,
-		.port = port_num,
-		.access_flags = access_flags,
-		.min_rnr_timer = 2,
-		.tclass = user_param->traffic_class,
-		.flow_label = 0,
-		.mtu = user_param->curr_mtu,
-		.pkey_index = user_param->pkey_index,
-		.gid_index = user_param->gid_index,
-		.hop_limit = 1,
-		.inline_size = user_param->inline_size,
-	};
-
-	ctx->dct[dct_index] = ibv_exp_create_dct(ctx->context, &dctattr);
-	if (!ctx->dct[dct_index]) {
-		printf("create dct failed\n");
-		return FAILURE;
-	}
-
-	struct ibv_exp_dct_attr dcqattr;
-	memset(&dcqattr,0,sizeof(struct ibv_exp_dct_attr));
-
-	err = ibv_exp_query_dct(ctx->dct[dct_index], &dcqattr);
-	if (err) {
-		printf("query dct failed\n");
-		return FAILURE;
-	} else if (dcqattr.dc_key != user_param->dct_key) {
-		printf("queried dckry (0x%llx) is different then provided at create (0x%llx)\n",
-				(unsigned long long)dcqattr.dc_key,
-				(unsigned long long)user_param->dct_key);
-		return FAILURE;
-	} else if (dcqattr.state != IBV_EXP_DCT_STATE_ACTIVE) {
-		printf("state is not active %d\n", dcqattr.state);
-		return FAILURE;
-	}
-
-	return 0;
-}
-#endif
-
-#ifdef HAVE_RSS_EXP
-static struct ibv_qp *ctx_rss_eth_qp_create(struct pingpong_context *ctx,struct perftest_parameters *user_param,int qp_index)
-{
-
-	struct ibv_exp_qp_init_attr attr;
-	struct ibv_qp* qp = NULL;
-
-	memset(&attr, 0, sizeof(struct ibv_exp_qp_init_attr));
-
-	attr.send_cq = ctx->send_cq;
-	attr.recv_cq = ctx->recv_cq;
-	attr.cap.max_send_wr  = user_param->tx_depth;
-	attr.cap.max_send_sge = MAX_SEND_SGE;
-	attr.cap.max_inline_data = user_param->inline_size;
-	attr.cap.max_recv_wr  = user_param->rx_depth;
-	attr.cap.max_recv_sge = MAX_RECV_SGE;
-	attr.qp_type = IBV_QPT_RAW_PACKET;
-	attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_QPG;
-	attr.pd = ctx->pd;
-
-	if (qp_index == 0) { /* rss parent */
-		#ifdef HAVE_VERBS_EXP
-		attr.qpg.qpg_type = IBV_EXP_QPG_PARENT;
-		#else
-		attr.qpg.qpg_type = IBV_QPG_PARENT;
-		#endif
-		attr.qpg.qpg_parent = NULL;
-		attr.qpg.parent_attrib.tss_child_count = 0;
-		attr.qpg.parent_attrib.rss_child_count = user_param->num_of_qps - 1;
-	} else { /* rss childs */
-		#ifdef HAVE_VERBS_EXP
-		attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX;
-		#else
-		attr.qpg.qpg_type = IBV_QPG_CHILD_RX;
-		#endif
-		attr.qpg.qpg_parent = ctx->qp[0];
-	}
-	qp = ibv_exp_create_qp(ctx->context,&attr);
-
 
 	return qp;
 }
@@ -943,20 +731,6 @@ void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_par
 	ALLOCATE(ctx->mr, struct ibv_mr*, user_param->num_of_qps);
 	ALLOCATE(ctx->buf, void* , user_param->num_of_qps);
 
-	#ifdef HAVE_ACCL_VERBS
-	ALLOCATE(ctx->qp_burst_family, struct ibv_exp_qp_burst_family*, user_param->num_of_qps);
-	#endif
-
-	#ifdef HAVE_DC
-	if (user_param->connection_type == DC) {
-		#ifdef HAVE_VERBS_EXP
-		ALLOCATE(ctx->dct, struct ibv_exp_dct*, user_param->num_of_qps);
-		#else
-		ALLOCATE(ctx->dct, struct ibv_dct*, user_param->num_of_qps);
-		#endif
-	}
-	#endif
-
 	if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && (user_param->machine == CLIENT || user_param->duplex)) {
 
 		ALLOCATE(user_param->tcompleted,cycles_t,tarr_size);
@@ -981,9 +755,6 @@ void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_par
 	if (user_param->machine == CLIENT || user_param->tst == LAT || user_param->duplex) {
 
 		ALLOCATE(ctx->sge_list, struct ibv_sge,user_param->num_of_qps * user_param->post_list);
-		#ifdef HAVE_VERBS_EXP
-		ALLOCATE(ctx->exp_wr, struct ibv_exp_send_wr, user_param->num_of_qps * user_param->post_list);
-		#endif
 		ALLOCATE(ctx->wr, struct ibv_send_wr, user_param->num_of_qps * user_param->post_list);
 		if ((user_param->verb == SEND && user_param->connection_type == UD) ||
 				user_param->connection_type == DC || user_param->connection_type == SRD) {
@@ -1070,23 +841,12 @@ int destroy_ctx(struct pingpong_context *ctx,
 				test_result = 1;
 			}
 		}
-		#ifdef HAVE_DC
-		if (user_param->connection_type == DC && ((!(user_param->duplex || user_param->tst == LAT)
-						&& (user_param->machine == SERVER)) || ((user_param->duplex || user_param->tst == LAT) && (i >= num_of_qps)))) {
-			if (ibv_exp_destroy_dct(ctx->dct[i])) {
-				fprintf(stderr, "Failed to destroy dct\n");
+		if (user_param->work_rdma_cm == OFF) {
+			if (ibv_destroy_qp(ctx->qp[i])) {
+				fprintf(stderr, "Couldn't destroy QP - %s\n", strerror(errno));
 				test_result = 1;
 			}
-			if ( i == user_param->num_of_qps -1 )
-				return test_result;
-		} else
-		#endif
-			if (user_param->work_rdma_cm == OFF) {
-				if (ibv_destroy_qp(ctx->qp[i])) {
-					fprintf(stderr, "Couldn't destroy QP - %s\n", strerror(errno));
-					test_result = 1;
-				}
-			}
+		}
 	}
 
 	if (user_param->use_rss) {
@@ -1216,9 +976,6 @@ int destroy_ctx(struct pingpong_context *ctx,
 
 		free(ctx->sge_list);
 		free(ctx->wr);
-		#ifdef HAVE_VERBS_EXP
-		free(ctx->exp_wr);
-		#endif
 	}
 
 	if (user_param->verb == SEND && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex)) {
@@ -1242,49 +999,14 @@ int destroy_ctx(struct pingpong_context *ctx,
 /******************************************************************************
  *
  ******************************************************************************/
-#ifdef HAVE_VERBS_EXP
-static int check_inline_recv_support(struct pingpong_context *ctx,
-					struct perftest_parameters *user_param)
-{
-	struct ibv_exp_device_attr dattr;
-	int ret = 0;
-
-	memset(&dattr, 0, sizeof(dattr));
-
-	dattr.comp_mask |= IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ;
-	ret = ibv_exp_query_device(ctx->context, &dattr);
-	if (ret) {
-		printf("  Couldn't query device for inline-receive capabilities.\n");
-	} else if (!(dattr.comp_mask & IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ)) {
-		printf("  Inline-receive not supported by driver.\n");
-		ret = 1;
-	} else if (dattr.inline_recv_sz < user_param->inline_recv_size) {
-		printf("  Max inline-receive(%d) < Requested inline-receive(%d).\n",
-			dattr.inline_recv_sz, user_param->inline_recv_size);
-	}
-
-	return ret;
-
-}
-#endif
-
-/******************************************************************************
- *
- ******************************************************************************/
-#if defined HAVE_EX_ODP || defined HAVE_EXP_ODP
+#ifdef HAVE_EX_ODP
 static int check_odp_support(struct pingpong_context *ctx)
 {
-	#ifdef HAVE_EX_ODP
 	struct ibv_device_attr_ex dattr;
 	int odp_support_send = IBV_ODP_SUPPORT_SEND;
 	int odp_support_recv = IBV_ODP_SUPPORT_RECV;
 	int ret = ibv_query_device_ex(ctx->context, NULL, &dattr);
-	#elif defined  HAVE_EXP_ODP
-	struct ibv_exp_device_attr dattr;
-	int ret = ibv_exp_query_device(ctx->context, &dattr);
-	int odp_support_send = IBV_EXP_ODP_SUPPORT_SEND;
-	int odp_support_recv = IBV_EXP_ODP_SUPPORT_RECV;
-	#endif
+
 	if (ret) {
 		fprintf(stderr, " Couldn't query device for on-demand paging capabilities.\n");
 		return 0;
@@ -1325,70 +1047,6 @@ int create_reg_cqs(struct pingpong_context *ctx,
 	return SUCCESS;
 }
 
-/******************************************************************************
- *
- ******************************************************************************/
-#ifdef HAVE_VERBS_EXP
-int create_exp_cqs(struct pingpong_context *ctx,
-		   struct perftest_parameters *user_param,
-		   int tx_buffer_depth, int need_recv_cq)
-{
-	struct ibv_exp_cq_init_attr attr;
-	#ifdef HAVE_ACCL_VERBS
-	enum ibv_exp_query_intf_status intf_status;
-	struct ibv_exp_query_intf_params intf_params;
-	#endif
-
-	memset(&attr, 0, sizeof(attr));
-
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->use_res_domain) {
-		attr.res_domain = ctx->res_domain;
-	}
-
-	if (user_param->verb_type == ACCL_INTF) {
-		memset(&intf_params, 0, sizeof(intf_params));
-		intf_params.intf_scope = IBV_EXP_INTF_GLOBAL;
-		intf_params.intf = IBV_EXP_INTF_CQ;
-	}
-	#endif
-
-	ctx->send_cq = ibv_exp_create_cq(ctx->context, tx_buffer_depth *
-						user_param->num_of_qps, NULL,
-						ctx->channel, 0, &attr);
-	if (!ctx->send_cq) {
-		fprintf(stderr, "Couldn't create exp CQ\n");
-		return FAILURE;
-	}
-
-	if (need_recv_cq) {
-		ctx->recv_cq = ibv_create_cq(ctx->context,user_param->rx_depth *
-						user_param->num_of_qps,NULL,ctx->channel,0);
-		if (!ctx->recv_cq) {
-			fprintf(stderr, "Couldn't create a receiver CQ\n");
-			return FAILURE;
-		}
-	}
-
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->verb_type == ACCL_INTF) {
-		/* Check CQ family */
-		intf_params.obj = ctx->send_cq;
-		ctx->send_cq_family = ibv_exp_query_intf(ctx->context, &intf_params, &intf_status);
-		intf_params.obj = ctx->recv_cq;
-		ctx->recv_cq_family = ibv_exp_query_intf(ctx->context, &intf_params, &intf_status);
-
-		if (!ctx->send_cq_family || (!ctx->recv_cq_family && need_recv_cq)) {
-			fprintf(stderr, "Couldn't create CQ family.\n");
-			return FAILURE;
-		}
-	}
-	#endif
-
-	return SUCCESS;
-}
-#endif
-
 /******************************************************************************
  *
  ******************************************************************************/
@@ -1408,54 +1066,11 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa
 	if ((user_param->connection_type == DC && !dct_only) || (user_param->verb == SEND))
 		need_recv_cq = 1;
 
-	#ifdef HAVE_VERBS_EXP
-	if (user_param->is_exp_cq)
-		ret = create_exp_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq);
-	else
-	#endif
-		ret = create_reg_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq);
+	ret = create_reg_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq);
 
 	return ret;
 }
 
-/******************************************************************************
- *
- ******************************************************************************/
-#ifdef HAVE_ACCL_VERBS
-struct ibv_exp_res_domain* create_res_domain(struct pingpong_context *ctx, struct perftest_parameters *user_param)
-{
-	struct ibv_exp_res_domain_init_attr res_domain_attr;
-	struct ibv_exp_device_attr dattr;
-	uint32_t req_comp_mask;
-
-	/* Query device */
-	req_comp_mask = IBV_EXP_DEVICE_ATTR_CALC_CAP		|
-			IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS	|
-			IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN;
-	dattr.comp_mask = req_comp_mask;
-
-	if (ibv_exp_query_device(ctx->context, &dattr)) {
-		fprintf(stderr, "Couldn't query device capabilities.\n");
-		return NULL;
-	} else if (dattr.comp_mask !=  req_comp_mask) {
-		fprintf(stderr, "This device does not support resource domain / accelerated verbs.\n");
-		return NULL;
-	}
-
-	/* Allocate resource domain */
-	res_domain_attr.comp_mask = IBV_EXP_RES_DOMAIN_THREAD_MODEL | IBV_EXP_RES_DOMAIN_MSG_MODEL;
-	res_domain_attr.thread_model = IBV_EXP_THREAD_SINGLE;
-	if (user_param->tst == BW)
-		res_domain_attr.msg_model = IBV_EXP_MSG_HIGH_BW;
-	else if (user_param->tst == LAT)
-		res_domain_attr.msg_model = IBV_EXP_MSG_LOW_LATENCY;
-	else
-		res_domain_attr.msg_model = IBV_EXP_MSG_DEFAULT;
-
-	return ibv_exp_create_res_domain(ctx->context, &res_domain_attr);
-}
-#endif
-
 /******************************************************************************
  *
  ******************************************************************************/
@@ -1464,28 +1079,20 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 	int i;
 	int flags = IBV_ACCESS_LOCAL_WRITE;
 
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_reg_mr_in reg_mr_exp_in;
-	uint64_t exp_flags = IBV_EXP_ACCESS_LOCAL_WRITE;
-	#endif
 
 	#if defined(__FreeBSD__)
 	ctx->is_contig_supported = FAILURE;
 	#endif
 
 	/* ODP */
-	#if defined HAVE_EX_ODP || defined HAVE_EXP_ODP
+	#ifdef HAVE_EX_ODP
 	if (user_param->use_odp) {
 		if ( !check_odp_support(ctx) )
 			return FAILURE;
 
 		/* ODP does not support contig pages */
 		ctx->is_contig_supported = FAILURE;
-		#ifdef HAVE_EX_ODP
 		flags |= IBV_ACCESS_ON_DEMAND;
-		#elif defined  HAVE_EXP_ODP
-		exp_flags |= IBV_EXP_ACCESS_ON_DEMAND;
-		#endif
 	}
 	#endif
 
@@ -1537,52 +1144,22 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 			memset(ctx->buf[qp_index], 0, ctx->buff_size);
 		} else {
 			ctx->buf[qp_index] = NULL;
-			#ifdef HAVE_VERBS_EXP
-			exp_flags |= IBV_EXP_ACCESS_ALLOCATE_MR;
-			#else
 			flags |= (1 << 5);
-			#endif
 		}
 	}
 
 	if (user_param->verb == WRITE) {
 		flags |= IBV_ACCESS_REMOTE_WRITE;
-		#ifdef HAVE_VERBS_EXP
-		exp_flags |= IBV_EXP_ACCESS_REMOTE_WRITE;
-		#endif
 	} else if (user_param->verb == READ) {
 		flags |= IBV_ACCESS_REMOTE_READ;
-		#ifdef HAVE_VERBS_EXP
-		exp_flags |= IBV_EXP_ACCESS_REMOTE_READ;
-		#endif
 		if (user_param->transport_type == IBV_TRANSPORT_IWARP)
 			flags |= IBV_ACCESS_REMOTE_WRITE;
-		#ifdef HAVE_VERBS_EXP
-		exp_flags |= IBV_EXP_ACCESS_REMOTE_WRITE;
-		#endif
 	} else if (user_param->verb == ATOMIC) {
 		flags |= IBV_ACCESS_REMOTE_ATOMIC;
-		#ifdef HAVE_VERBS_EXP
-		exp_flags |= IBV_EXP_ACCESS_REMOTE_ATOMIC;
-		#endif
 	}
 
 	/* Allocating Memory region and assigning our buffer to it. */
-	#ifdef HAVE_VERBS_EXP
-	if (ctx->is_contig_supported == SUCCESS || user_param->use_odp) {
-		reg_mr_exp_in.pd = ctx->pd;
-		reg_mr_exp_in.addr = ctx->buf[qp_index];
-		reg_mr_exp_in.length = ctx->buff_size;
-		reg_mr_exp_in.exp_access = exp_flags;
-		reg_mr_exp_in.comp_mask = 0;
-
-		ctx->mr[qp_index] = ibv_exp_reg_mr(&reg_mr_exp_in);
-	}
-	else
-		ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags);
-	#else
 	ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags);
-	#endif
 
 	if (!ctx->mr[qp_index]) {
 		fprintf(stderr, "Couldn't allocate MR\n");
@@ -1693,17 +1270,13 @@ int verify_params_with_device_context(struct ibv_context *context,
 	return SUCCESS;
 }
 
-#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
+#if defined HAVE_OOO_ATTR
 static int verify_ooo_settings(struct pingpong_context *ctx,
 			       struct perftest_parameters *user_param)
 {
 	#ifdef HAVE_OOO_ATTR
 	struct ibv_device_attr_ex attr = { };
 	if (ibv_query_device_ex(ctx->context, NULL, &attr))
-	#elif HAVE_EXP_OOO_ATTR
-	struct ibv_exp_device_attr attr = { };
-	attr.comp_mask = IBV_EXP_DEVICE_ATTR_RESERVED - 1;
-	if (ibv_exp_query_device(ctx->context, &attr))
 	#endif
 		return FAILURE;
 
@@ -1737,14 +1310,6 @@ static int verify_ooo_settings(struct pingpong_context *ctx,
 		} else {
 			return SUCCESS;
 		}
-	#elif HAVE_EXP_OOO_ATTR
-	} else if (user_param->connection_type == DC) {
-		if (attr.ooo_caps.dc_caps == 0) {
-			fprintf(stderr, " OOO unsupported by HCA on DC QP\n");
-			return FAILURE;
-		} else {
-			return SUCCESS;
-		}
 	#endif
 	} else {
 		return FAILURE;
@@ -1756,18 +1321,7 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 	int i;
 	int num_of_qps = user_param->num_of_qps / 2;
 
-	#ifdef HAVE_ACCL_VERBS
-	enum ibv_exp_query_intf_status intf_status;
-	struct ibv_exp_query_intf_params intf_params;
-	#endif
-
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_device_attr dattr;
-	memset(&dattr, 0, sizeof(dattr));
-	get_verbs_pointers(ctx);
-	#endif
-
-	#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
+	#if defined HAVE_OOO_ATTR
 	if (user_param->use_ooo) {
 		if (verify_ooo_settings(ctx, user_param) != SUCCESS) {
 			fprintf(stderr, "Incompatible OOO settings\n");
@@ -1777,10 +1331,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 	#endif
 
 	ctx->is_contig_supported = FAILURE;
-	#ifdef HAVE_VERBS_EXP
-	if (!user_param->use_hugepages)
-		ctx->is_contig_supported  = check_for_contig_pages_support(ctx->context);
-	#endif
 
 	/* Allocating an event channel if requested. */
 	if (user_param->use_event) {
@@ -1798,16 +1348,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 		return FAILURE;
 	}
 
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->use_res_domain) {
-		ctx->res_domain = create_res_domain(ctx, user_param);
-		if (!ctx->res_domain) {
-			fprintf(stderr, "Couldn't create resource domain\n");
-			return FAILURE;
-		}
-	}
-	#endif
-
 	if (create_mr(ctx, user_param)) {
 		fprintf(stderr, "Failed to create MR\n");
 		return FAILURE;
@@ -1852,34 +1392,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 		}
 	}
 
-	#ifdef HAVE_RSS_EXP
-	if (user_param->use_rss) {
-		struct ibv_exp_device_attr attr;
-
-		attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
-			IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ;
-		if (ibv_exp_query_device(ctx->context, &attr)) {
-			fprintf(stderr, "Experimental ibv_exp_query_device.\n");
-			exit(1);
-		}
-
-		if (!((attr.exp_device_cap_flags & IBV_EXP_DEVICE_QPG) &&
-					(attr.exp_device_cap_flags & IBV_EXP_DEVICE_UD_RSS) &&
-					(attr.comp_mask & IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ) &&
-					(attr.max_rss_tbl_sz > 0))) {
-			fprintf(stderr, "RSS not supported .\n");
-			exit(1);
-		}
-
-		/* num of qps includes the parent */
-		if (user_param->num_of_qps > attr.max_rss_tbl_sz + 1) {
-			fprintf(stderr, "RSS limit is %d .\n",
-					attr.max_rss_tbl_sz);
-			exit(1);
-		}
-	}
-	#endif
-
 	/*
 	* QPs creation in RDMA CM flow will be done separately.
 	* Unless, the function called with RDMA CM connection contexts,
@@ -1897,19 +1409,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 
 		if (user_param->work_rdma_cm == OFF) {
 			modify_qp_to_init(ctx, user_param, i, num_of_qps);
-			#ifdef HAVE_ACCL_VERBS
-			if (user_param->verb_type == ACCL_INTF) {
-				memset(&intf_params, 0, sizeof(intf_params));
-				intf_params.intf_scope = IBV_EXP_INTF_GLOBAL;
-				intf_params.intf = IBV_EXP_INTF_QP_BURST;
-				intf_params.obj = ctx->qp[i];
-				ctx->qp_burst_family[i] = ibv_exp_query_intf(ctx->context, &intf_params, &intf_status);
-				if (!ctx->qp_burst_family[i]) {
-					fprintf(stderr, "Couldn't get QP burst family.\n");
-					return FAILURE;
-				}
-			}
-			#endif
 		}
 	}
 
@@ -1921,29 +1420,9 @@ int modify_qp_to_init(struct pingpong_context *ctx,
 {
 	uint64_t init_flag = 0;
 
-	#ifdef HAVE_RSS_EXP
-	if (qp_index == 0 && user_param->use_rss) {
-		init_flag = IBV_EXP_QP_GROUP_RSS;
-	}
-	else
-	#endif
-		init_flag = 0;
-
-	if(user_param->connection_type == DC) {
-		if ( !((!(user_param->duplex || user_param->tst == LAT) && (user_param->machine == SERVER) )
-					|| ((user_param->duplex || user_param->tst == LAT) && (qp_index >= num_of_qps)))) {
-			#ifdef HAVE_DC
-			if (ctx_modify_dc_qp_to_init(ctx->qp[qp_index],user_param)) {
-				fprintf(stderr," Unable to create DC QP.\n");
-				return FAILURE;
-			}
-			#endif
-		}
-	} else {
-		if (ctx_modify_qp_to_init(ctx->qp[qp_index],user_param,init_flag)) {
-			fprintf(stderr, "Failed to modify QP to INIT\n");
-			return FAILURE;
-		}
+	if (ctx_modify_qp_to_init(ctx->qp[qp_index],user_param,init_flag)) {
+		fprintf(stderr, "Failed to modify QP to INIT\n");
+		return FAILURE;
 	}
 
 	return SUCCESS;
@@ -1953,8 +1432,8 @@ int modify_qp_to_init(struct pingpong_context *ctx,
  *
  ******************************************************************************/
 int create_reg_qp_main(struct pingpong_context *ctx,
-		       struct perftest_parameters *user_param,
-		       int i, int num_of_qps)
+				struct perftest_parameters *user_param,
+				int i, int num_of_qps)
 {
 	if (user_param->use_xrc) {
 		#ifdef HAVE_XRCD
@@ -1975,213 +1454,18 @@ int create_reg_qp_main(struct pingpong_context *ctx,
 	return SUCCESS;
 }
 
-
-int create_exp_qp_main(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param, int i, int num_of_qps)
-{
-	int is_dc_tgt_query = 0;
-	is_dc_tgt_query |= !(user_param->duplex || user_param->tst == LAT) && (user_param->machine == SERVER);
-	is_dc_tgt_query |= (user_param->duplex || user_param->tst == LAT) && (i >= num_of_qps);
-	is_dc_tgt_query &= user_param->connection_type == DC;
-
-	if (is_dc_tgt_query) {
-		#ifdef HAVE_DC
-		if(ctx_dc_tgt_create(ctx,user_param,i)) {
-			return FAILURE;
-		}
-		/* in order to not change anything in the test */
-		ALLOCATE(ctx->qp[i], struct ibv_qp, 1);
-		ctx->qp[i]->qp_num = ctx->dct[i]->dct_num;
-		#else
-		fprintf(stderr, "DC is not supported.\n");
-		return FAILURE;
-		#endif
-	} else if (user_param->use_rss && user_param->connection_type == RawEth) {
-		#ifdef HAVE_RSS_EXP
-		ctx->qp[i] = ctx_rss_eth_qp_create(ctx, user_param, i);
-		#endif
-		if (ctx->qp[i] == NULL) {
-			fprintf(stderr," Unable to create RSS QP.\n");
-			return FAILURE;
-		}
-	} else if (user_param->use_xrc) {
-		#ifdef HAVE_XRCD
-		ctx->qp[i] = ctx_xrc_qp_create(ctx, user_param, i);
-		#endif
-		if (ctx->qp[i] == NULL) {
-			fprintf(stderr," Unable to create XRC QP.\n");
-			return FAILURE;
-		}
-	} else {
-		#ifdef HAVE_VERBS_EXP
-		ctx->qp[i] = ctx_exp_qp_create(ctx, user_param, i);
-		#endif
-		if (ctx->qp[i] == NULL) {
-			fprintf(stderr, " Unable to create exp QP.\n");
-			return FAILURE;
-		}
-	}
-	return SUCCESS;
-}
-
 int create_qp_main(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param, int i, int num_of_qps)
 {
 	int ret;
-	#ifdef HAVE_VERBS_EXP
-	int query;
-
-	/* flag that indicates that we are going to use exp QP */
-	query = (user_param->connection_type == DC);
-	query |= (user_param->use_rss && user_param->connection_type == RawEth);
-	query |= user_param->use_xrc;
-	query |= user_param->inline_recv_size != 0;
-	query |= user_param->masked_atomics;
-	query |= user_param->verb_type != NORMAL_INTF;
-	query |= user_param->use_res_domain;
-	query |= user_param->use_exp;
-
-	if (query == 1)
-		user_param->is_exp_qp = 1;
-
-	if (user_param->is_exp_qp)
-		ret = create_exp_qp_main(ctx, user_param, i, num_of_qps);
-	else
-	#endif
-		ret = create_reg_qp_main(ctx, user_param, i, num_of_qps);
-
+	ret = create_reg_qp_main(ctx, user_param, i, num_of_qps);
 	return ret;
 }
 
-#ifdef HAVE_VERBS_EXP
-#ifdef HAVE_SCATTER_FCS
-static int check_scatter_fcs_support(struct pingpong_context *ctx,
+struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param)
 {
-	struct ibv_exp_device_attr dev_attr;
-
-	memset(&dev_attr, 0, sizeof(dev_attr));
-	dev_attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS;
-	dev_attr.exp_device_cap_flags = IBV_EXP_DEVICE_SCATTER_FCS;
-	if (ibv_exp_query_device(ctx->context, &dev_attr)) {
-		fprintf(stderr, "ibv_exp_query_device failed\n");
-		return 1;
-	}
-
-	return MASK_IS_SET(IBV_EXP_DEVICE_SCATTER_FCS, dev_attr.exp_device_cap_flags);
-}
-#endif
-#endif
-
-#ifdef HAVE_VERBS_EXP
-struct ibv_qp* ctx_exp_qp_create(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param, int qp_index)
-{
-	struct ibv_exp_qp_init_attr attr;
-	struct ibv_qp* qp = NULL;
-	struct ibv_exp_device_attr dev_attr;
-
-	memset(&attr, 0, sizeof(attr));
-	memset(&dev_attr, 0, sizeof(dev_attr));
-	attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS;
-
-	if (user_param->masked_atomics) {
-		#ifdef HAVE_MASKED_ATOMICS
-		dev_attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS;
-
-		if (ibv_exp_query_device(ctx->context, &dev_attr)) {
-			fprintf(stderr, "ibv_exp_query_device failed\n");
-			return NULL;
-		}
-
-		attr.max_atomic_arg = pow(2,dev_attr.ext_atom.log_max_atomic_inline);
-		attr.exp_create_flags |= IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY;
-		attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG;
-		#else
-		fprintf(stderr, "Can't create masked atomic QP.\n");
-		return NULL;
-		#endif
-	}
-
-	if (user_param->inline_recv_size) {
-
-		if (check_inline_recv_support(ctx, user_param)) {
-			fprintf(stderr, "Failed to create QP with inline receive.\n");
-			return NULL;
-		}
-
-		attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV;
-		attr.max_inl_recv = user_param->inline_recv_size;
-		attr.sq_sig_all = (1 == user_param->cq_mod) ? 1 : 0; //inline receive on requestor must QP's sq_sig_all to be applied
-	}
-
-	#ifdef HAVE_ACCL_VERBS
-	if (user_param->use_res_domain) {
-		attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_RES_DOMAIN;
-		attr.res_domain = ctx->res_domain;
-	}
-	#endif
-
-	attr.pd = ctx->pd;
-	attr.send_cq = ctx->send_cq;
-	attr.recv_cq = (user_param->verb == SEND) ? ctx->recv_cq : ctx->send_cq;
-	attr.cap.max_send_wr  = user_param->tx_depth;
-	attr.cap.max_send_sge = MAX_SEND_SGE;
-	attr.cap.max_inline_data = user_param->inline_size;
-
-	if (user_param->use_srq && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex == ON)) {
-		attr.srq = ctx->srq;
-		attr.cap.max_recv_wr  = 0;
-		attr.cap.max_recv_sge = 0;
-	} else {
-		attr.srq = NULL;
-		attr.cap.max_recv_wr  = user_param->rx_depth;
-		attr.cap.max_recv_sge = MAX_RECV_SGE;
-	}
-
-	switch (user_param->connection_type) {
-
-		case RC : attr.qp_type = IBV_QPT_RC; break;
-		case UC : attr.qp_type = IBV_QPT_UC; break;
-		case UD : attr.qp_type = IBV_QPT_UD; break;
-		#ifdef HAVE_RAW_ETH
-		case RawEth : attr.qp_type = IBV_QPT_RAW_PACKET; break;
-		#endif
-		case DC : attr.qp_type = IBV_EXP_QPT_DC_INI; break;
-		default:  fprintf(stderr, "Unknown connection type \n");
-			  return NULL;
-	}
-
-	#ifdef HAVE_SCATTER_FCS
-	if (!user_param->disable_fcs && (user_param->connection_type == RawEth)) {
-		if(check_scatter_fcs_support(ctx, user_param)) {
-			attr.exp_create_flags |= IBV_EXP_QP_CREATE_SCATTER_FCS;
-		}
-	}
-	#endif
-
-	qp = ibv_exp_create_qp(ctx->context, &attr);
-	if (!qp)
-		return NULL;
-
-	if (user_param->inline_recv_size > attr.max_inl_recv)
-		printf("  Actual inline-receive(%d) < requested inline-receive(%d)\n",
-				attr.max_inl_recv, user_param->inline_recv_size);
-
-	if (user_param->inline_size > attr.cap.max_inline_data) {
-		user_param->inline_size = attr.cap.max_inline_data;
-		printf("  Actual inline-size(%d) > requested inline-size(%d)\n",
-			attr.cap.max_inline_data, user_param->inline_size);
-	}
-
-	return qp;
-}
-#endif
-
-struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param)
-{
-	struct ibv_qp* qp = NULL;
+	struct ibv_qp* qp = NULL;
 
 	#ifdef HAVE_IBV_WR_API
 	enum ibv_wr_opcode opcode;
@@ -2280,134 +1564,6 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	return qp;
 }
 
-#ifdef HAVE_MASKED_ATOMICS
-/******************************************************************************
- *
- ******************************************************************************/
-struct ibv_qp* ctx_atomic_qp_create(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param)
-{
-	struct ibv_exp_qp_init_attr	attr;
-	struct ibv_qp* qp = NULL;
-	struct ibv_exp_device_attr dev_attr;
-
-	memset(&dev_attr, 0, sizeof(dev_attr));
-
-	dev_attr.comp_mask |= IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS;
-
-	if (ibv_exp_query_device(ctx->context, &dev_attr)) {
-		fprintf(stderr, "ibv_exp_query_device failed\n");
-		exit(1);
-	}
-
-	memset(&attr, 0, sizeof(struct ibv_exp_qp_init_attr));
-	attr.pd = ctx->pd;
-	attr.send_cq = ctx->send_cq;
-	attr.recv_cq = (user_param->verb == SEND) ? ctx->recv_cq : ctx->send_cq;
-	attr.cap.max_send_wr  = user_param->tx_depth;
-	attr.cap.max_send_sge = MAX_SEND_SGE;
-	attr.cap.max_inline_data = user_param->inline_size;
-	attr.max_atomic_arg = pow(2,dev_attr.ext_atom.log_max_atomic_inline);
-	attr.exp_create_flags = IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY;
-	attr.comp_mask = IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS | IBV_EXP_QP_INIT_ATTR_PD;
-	attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG;
-
-	if (user_param->use_srq && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex == ON)) {
-		attr.srq = ctx->srq;
-		attr.cap.max_recv_wr  = 0;
-		attr.cap.max_recv_sge = 0;
-	} else {
-		attr.srq = NULL;
-		attr.cap.max_recv_wr  = user_param->rx_depth;
-		attr.cap.max_recv_sge = MAX_RECV_SGE;
-	}
-
-	switch (user_param->connection_type) {
-
-		case RC : attr.qp_type = IBV_QPT_RC; break;
-		case UC : attr.qp_type = IBV_QPT_UC; break;
-		case UD : attr.qp_type = IBV_QPT_UD; break;
-		#ifdef HAVE_XRCD
-		case XRC : attr.qp_type = IBV_QPT_XRC; break;
-		#endif
-		#ifdef HAVE_RAW_ETH
-		case RawEth : attr.qp_type = IBV_QPT_RAW_PACKET; break;
-		#endif
-		default:  fprintf(stderr, "Unknown connection type \n");
-			  return NULL;
-	}
-
-	qp = ibv_exp_create_qp(ctx->context, &attr);
-
-	return qp;
-}
-#endif
-
-#ifdef HAVE_DC
-/******************************************************************************
- *
- ******************************************************************************/
-int ctx_modify_dc_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param)
-{
-	int num_of_qps = user_param->num_of_qps;
-	int num_of_qps_per_port = user_param->num_of_qps / 2;
-	int err;
-	uint64_t flags;
-
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_qp_attr attr;
-	memset(&attr, 0, sizeof(struct ibv_exp_qp_attr));
-	flags = IBV_EXP_QP_STATE | IBV_EXP_QP_PKEY_INDEX | IBV_EXP_QP_PORT;
-	#else
-	struct ibv_qp_attr attr;
-	memset(&attr, 0, sizeof(struct ibv_qp_attr));
-	flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
-	#endif
-
-	static int portindex=0;  /* for dual-port support */
-
-	attr.qp_state        = IBV_QPS_INIT;
-	attr.pkey_index      = user_param->pkey_index;
-	attr.qp_access_flags = 0;
-	attr.dct_key = user_param->dct_key;
-
-	if (user_param->duplex || user_param->tst == LAT) {
-		num_of_qps /= 2;
-		num_of_qps_per_port = num_of_qps / 2;
-	}
-
-	if (user_param->dualport==ON) {
-		if (portindex % num_of_qps < num_of_qps_per_port) {
-			attr.port_num = user_param->ib_port;
-			user_param->port_by_qp[portindex] = 0;
-		} else {
-			attr.port_num = user_param->ib_port2;
-			user_param->port_by_qp[portindex] = 1;
-		}
-		portindex++;
-
-	} else {
-
-		attr.port_num = user_param->ib_port;
-	}
-
-	#ifdef HAVE_VERBS_EXP
-	flags |= IBV_EXP_QP_DC_KEY;
-	err = ibv_exp_modify_qp(qp,&attr,flags);
-	#else
-	flags |= IBV_QP_DC_KEY;
-	attr.comp_mask = IBV_QP_ATTR_DCT_KEY;
-	err = ibv_modify_qp(qp,&attr,flags);
-	#endif
-
-	if (err) {
-		fprintf(stderr, "Failed to modify QP to INIT\n");
-		return 1;
-	}
-	return 0;
-}
-#endif
-
 /******************************************************************************
  *
  ******************************************************************************/
@@ -2419,11 +1575,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 	struct ibv_qp_attr attr;
 	int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
 
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_qp_attr exp_attr;
-	uint64_t exp_flags = 0;
-	#endif
-
 	static int portindex=0;  /* for dual-port support */
 	int ret = 0;
 
@@ -2431,12 +1582,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 	attr.qp_state        = IBV_QPS_INIT;
 	attr.pkey_index      = user_param->pkey_index;
 
-	#ifdef HAVE_VERBS_EXP
-	memset(&exp_attr, 0, sizeof(struct ibv_exp_qp_attr));
-	exp_attr.qp_state        = attr.qp_state;
-	exp_attr.pkey_index      = attr.pkey_index;
-	#endif
-
 	if ( user_param->use_xrc && (user_param->duplex || user_param->tst == LAT)) {
 		num_of_qps /= 2;
 		num_of_qps_per_port = num_of_qps / 2;
@@ -2457,15 +1602,8 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 		attr.port_num = user_param->ib_port;
 	}
 
-	#ifdef HAVE_VERBS_EXP
-	exp_attr.port_num = attr.port_num;
-	#endif
-
 	if (user_param->connection_type == RawEth) {
 		flags = IBV_QP_STATE | IBV_QP_PORT;
-		#ifdef HAVE_VERBS_EXP
-		exp_flags = init_flag | IBV_EXP_QP_STATE | IBV_EXP_QP_PORT;
-		#endif
 
 	} else if (user_param->connection_type == UD || user_param->connection_type == SRD) {
 		attr.qkey = DEFF_QKEY;
@@ -2481,20 +1619,7 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 		flags |= IBV_QP_ACCESS_FLAGS;
 	}
 
-	#ifdef HAVE_MASKED_ATOMICS
-	if (user_param->masked_atomics) {
-		exp_attr.qp_access_flags = IBV_ACCESS_REMOTE_ATOMIC | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
-		exp_flags = IBV_EXP_QP_STATE | IBV_EXP_QP_PKEY_INDEX | IBV_EXP_QP_PORT | IBV_EXP_QP_ACCESS_FLAGS;
-	}
-	#endif
-
-	#ifdef HAVE_VERBS_EXP
-	if ( (init_flag != 0 && user_param->use_rss) || user_param->masked_atomics ) {
-		ret = ibv_exp_modify_qp(qp,&exp_attr,exp_flags);
-	}
-	else
-	#endif
-		ret = ibv_modify_qp(qp,&attr,flags);
+	ret = ibv_modify_qp(qp,&attr,flags);
 
 	if (ret) {
 		fprintf(stderr, "Failed to modify QP to INIT, ret=%d\n",ret);
@@ -2503,67 +1628,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 	return 0;
 }
 
-#ifdef HAVE_DC
-/******************************************************************************
- *
- ******************************************************************************/
-static int ctx_modify_dc_qp_to_rtr(struct ibv_qp *qp,
-		struct ibv_exp_qp_attr *attr,
-		struct perftest_parameters *user_param,
-		struct pingpong_dest *dest,
-		struct pingpong_dest *my_dest,
-		int qpindex)
-{
-	int num_of_qps = user_param->num_of_qps;
-	int num_of_qps_per_port = user_param->num_of_qps / 2;
-
-	int flags = IBV_EXP_QP_STATE | IBV_EXP_QP_PATH_MTU | IBV_EXP_QP_AV;
-	attr->qp_state = IBV_QPS_RTR;
-	attr->ah_attr.src_path_bits = 0;
-
-	/* in DC with bidirectional,
-	 * there are send qps and recv qps. the actual number of send/recv qps
-	 * is num_of_qps / 2.
-	 */
-	if (user_param->duplex || user_param->tst == LAT) {
-		num_of_qps /= 2;
-		num_of_qps_per_port = num_of_qps / 2;
-	}
-
-	/* first half of qps are for ib_port and second half are for ib_port2
-	 * in DC with bidirectional, the first half of qps are DC_INI qps and
-	 * the second half are DC_TGT qps. the first half of the send/recv qps
-	 * are for ib_port1 and the second half are for ib_port2
-	 */
-	if (user_param->dualport == ON && (qpindex % num_of_qps >= num_of_qps_per_port))
-		attr->ah_attr.port_num = user_param->ib_port2;
-	else
-		attr->ah_attr.port_num = user_param->ib_port;
-
-	attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
-	if (user_param->gid_index == DEF_GID_INDEX) {
-
-		attr->ah_attr.is_global = 0;
-		attr->ah_attr.sl = user_param->sl;
-
-	} else {
-		attr->ah_attr.is_global  = 1;
-		attr->ah_attr.grh.dgid = dest->gid;
-		attr->ah_attr.grh.sgid_index = user_param->gid_index;
-		attr->ah_attr.grh.hop_limit = 1;
-		attr->ah_attr.grh.traffic_class = user_param->traffic_class;
-		attr->ah_attr.sl = 0;
-	}
-
-	attr->max_dest_rd_atomic = 0;
-	attr->min_rnr_timer = 0;
-	attr->dct_key = user_param->dct_key;
-	attr->path_mtu = user_param->curr_mtu;
-
-	return ibv_exp_modify_qp(qp,attr,flags);
-}
-#endif
-
 /******************************************************************************
  *
  ******************************************************************************/
@@ -2642,8 +1706,6 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 
 	#ifdef HAVE_OOO_ATTR
 		ooo_flags |= IBV_QP_OOO_RW_DATA_PLACEMENT;
-	#elif HAVE_EXP_OOO_ATTR
-		ooo_flags |= IBV_EXP_QP_OOO_RW_DATA_PLACEMENT;
 	#endif
 
 	if (user_param->use_ooo)
@@ -2651,43 +1713,6 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 	return ibv_modify_qp(qp, attr, flags);
 }
 
-#ifdef HAVE_DC
-/******************************************************************************
- *
- ******************************************************************************/
-static int ctx_modify_dc_qp_to_rts(struct ibv_qp *qp,
-		#ifdef HAVE_VERBS_EXP
-		struct ibv_exp_qp_attr *attr,
-		#else
-		struct ibv_qp_attr_ex *attr,
-		#endif
-		struct perftest_parameters *user_param,
-		struct pingpong_dest *dest,
-		struct pingpong_dest *my_dest)
-{
-
-	#ifdef HAVE_VERBS_EXP
-	int flags = IBV_EXP_QP_STATE | IBV_EXP_QP_TIMEOUT | IBV_EXP_QP_RETRY_CNT |
-					 IBV_EXP_QP_RNR_RETRY | IBV_EXP_QP_MAX_QP_RD_ATOMIC;
-	#else
-	int flags = IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC;
-	#endif
-
-	attr->qp_state = IBV_QPS_RTS;
-
-	attr->timeout   = user_param->qp_timeout;
-	attr->retry_cnt = 7;
-	attr->rnr_retry = 7;
-	attr->max_rd_atomic  = dest->out_reads;
-
-	#ifdef HAVE_VERBS_EXP
-	return ibv_exp_modify_qp(qp,attr,flags);
-	#else
-	return ibv_modify_qp_ex(qp,attr,flags);
-	#endif
-}
-#endif
-
 /******************************************************************************
  *
  ******************************************************************************/
@@ -2697,11 +1722,7 @@ static int ctx_modify_qp_to_rts(struct ibv_qp *qp,
 		struct pingpong_dest *dest,
 		struct pingpong_dest *my_dest)
 {
-	#ifdef HAVE_PACKET_PACING_EXP
-	uint64_t flags = IBV_QP_STATE;
-	#else
 	int flags = IBV_QP_STATE;
-	#endif
 	struct ibv_qp_attr *attr = (struct ibv_qp_attr*)_attr;
 
 	attr->qp_state = IBV_QPS_RTS;
@@ -2721,21 +1742,7 @@ static int ctx_modify_qp_to_rts(struct ibv_qp *qp,
 		}
 	}
 
-	#ifdef HAVE_PACKET_PACING_EXP
-	if (user_param->rate_limit_type == PP_RATE_LIMIT) {
-		((struct ibv_exp_qp_attr*)_attr)->rate_limit = user_param->rate_limit;
-
-	#ifdef HAVE_PACKET_PACING_EXTENSION_EXP
-		((struct ibv_exp_qp_attr*)_attr)->burst_info.max_burst_sz =
-			user_param->burst_size;
-		((struct ibv_exp_qp_attr*)_attr)->burst_info.typical_pkt_sz =
-			user_param->typical_pkt_size;
-		((struct ibv_exp_qp_attr*)_attr)->comp_mask |= IBV_EXP_QP_ATTR_BURST_INFO;
-	#endif
-		flags |= IBV_EXP_QP_RATE_LIMIT;
-		return ibv_exp_modify_qp(qp, (struct ibv_exp_qp_attr*)_attr, flags);
-	}
-	#elif HAVE_PACKET_PACING
+	#ifdef HAVE_PACKET_PACING
 	if (user_param->rate_limit_type == PP_RATE_LIMIT) {
 		attr->rate_limit = user_param->rate_limit;
 		flags |= IBV_QP_RATE_LIMIT;
@@ -2754,15 +1761,6 @@ int ctx_connect(struct pingpong_context *ctx,
 		struct pingpong_dest *my_dest)
 {
 	int i;
-	#ifdef HAVE_DC
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_qp_attr attr_ex;
-	#else
-	struct ibv_qp_attr_ex attr_ex;
-	#endif
-	#elif HAVE_PACKET_PACING_EXP
-	struct ibv_exp_qp_attr attr_ex;
-	#endif
 	struct ibv_qp_attr attr;
 	int xrc_offset = 0;
 
@@ -2777,82 +1775,40 @@ int ctx_connect(struct pingpong_context *ctx,
 				continue;
 			}
 		}
-		#if defined (HAVE_DC) || defined (HAVE_PACKET_PACING_EXP)
-		memset(&attr_ex, 0, sizeof attr_ex);
-		#endif
 		memset(&attr, 0, sizeof attr);
 
 		if (user_param->rate_limit_type == HW_RATE_LIMIT)
 			attr.ah_attr.static_rate = user_param->valid_hw_rate_limit_index;
 
-		#if defined (HAVE_PACKET_PACING_EXP) || defined (HAVE_PACKET_PACING)
+		#if defined (HAVE_PACKET_PACING)
 		if (user_param->rate_limit_type == PP_RATE_LIMIT) {
 			if (check_packet_pacing_support(ctx) == FAILURE) {
 				fprintf(stderr, "Packet Pacing isn't supported.\n");
 				return FAILURE;
 			}
-			#if defined (HAVE_PACKET_PACING_EXTENSION_EXP)
-			if (check_packet_pacing_extension_support(ctx) == FAILURE &&
-			    (user_param->burst_size || user_param->typical_pkt_size)) {
-				fprintf(stderr, "Burst control isn't supported.\n");
-				return FAILURE;
-			}
-			#endif
 		}
 		#endif
 
 		if ((i >= xrc_offset) && (user_param->use_xrc || user_param->connection_type == DC) && (user_param->duplex || user_param->tst == LAT))
 			xrc_offset = -1*xrc_offset;
 
-		if(user_param->connection_type == DC) {
-			#ifdef HAVE_DC
-			if(ctx_modify_dc_qp_to_rtr(ctx->qp[i],&attr_ex,user_param,&dest[xrc_offset + i],&my_dest[i],i)) {
-				fprintf(stderr, "Failed to modify QP %d to RTR\n",ctx->qp[i]->qp_num);
-				return FAILURE;
-			}
-			#endif
-		} else {
-			if(ctx_modify_qp_to_rtr(ctx->qp[i], &attr, user_param, &dest[xrc_offset + i], &my_dest[i], i)) {
-				fprintf(stderr, "Failed to modify QP %d to RTR\n",ctx->qp[i]->qp_num);
-				return FAILURE;
-			}
+
+		if(ctx_modify_qp_to_rtr(ctx->qp[i], &attr, user_param, &dest[xrc_offset + i], &my_dest[i], i)) {
+			fprintf(stderr, "Failed to modify QP %d to RTR\n",ctx->qp[i]->qp_num);
+			return FAILURE;
 		}
+
 		if (user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex) {
-			if(user_param->connection_type == DC) {
-				#ifdef HAVE_DC
-				if(ctx_modify_dc_qp_to_rts(ctx->qp[i], &attr_ex, user_param, &dest[xrc_offset + i], &my_dest[i])) {
-					fprintf(stderr, "Failed to modify QP to RTS\n");
-					return FAILURE;
-				}
-				#endif
-			} else {
-				#ifdef HAVE_PACKET_PACING_EXP
-				if (user_param->rate_limit_type == PP_RATE_LIMIT) {
-					if(ctx_modify_qp_to_rts(ctx->qp[i], &attr_ex, user_param, &dest[xrc_offset + i], &my_dest[i])) {
-						fprintf(stderr, "Failed to modify QP %x to RTS\n", ctx->qp[i]->qp_num);
-						return FAILURE;
-					}
-				} else {
-				#endif
-					if(ctx_modify_qp_to_rts(ctx->qp[i], &attr, user_param, &dest[xrc_offset + i], &my_dest[i])) {
-						fprintf(stderr, "Failed to modify QP to RTS\n");
-						return FAILURE;
-					}
-				#ifdef HAVE_PACKET_PACING_EXP
-				}
-				#endif
+			if(ctx_modify_qp_to_rts(ctx->qp[i], &attr, user_param, &dest[xrc_offset + i], &my_dest[i])) {
+				fprintf(stderr, "Failed to modify QP to RTS\n");
+				return FAILURE;
 			}
 		}
 
 		if ((user_param->connection_type == UD || user_param->connection_type == DC || user_param->connection_type == SRD) &&
 				(user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex)) {
 
-			#ifdef HAVE_DC
-			if(user_param->connection_type == DC)
-				ctx->ah[i] = ibv_create_ah(ctx->pd,&(attr_ex.ah_attr));
-			else
-			#endif
-				ctx->ah[i] = ibv_create_ah(ctx->pd,&(attr.ah_attr));
+			ctx->ah[i] = ibv_create_ah(ctx->pd,&(attr.ah_attr));
 
 			if (!ctx->ah[i]) {
 				fprintf(stderr, "Failed to create AH\n");
@@ -2868,219 +1824,43 @@ int ctx_connect(struct pingpong_context *ctx,
 			memset(&qp_attr,0,sizeof(struct ibv_qp_attr));
 			memset(&init_attr,0,sizeof(struct ibv_qp_init_attr));
 
-			err = ibv_query_qp(ctx->qp[i], &qp_attr, IBV_QP_AV, &init_attr);
-			if (err)
-				fprintf(stderr, "ibv_query_qp failed to get ah_attr\n");
-			else
-				qp_static_rate = (int)(qp_attr.ah_attr.static_rate);
-
-			//- Fall back to SW Limit only if flag undefined
-			if(err ||
-			   qp_static_rate != user_param->valid_hw_rate_limit_index ||
-			   user_param->link_type != IBV_LINK_LAYER_INFINIBAND) {
-				if(!user_param->is_rate_limit_type) {
-					user_param->rate_limit_type = SW_RATE_LIMIT;
-					fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit, providing SW rate limit \x1b[0m\n");
-				} else {
-					fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit  \x1b[0m\n");
-					return FAILURE;
-				}
-
-			}
-		}
-
-		if((user_param->use_xrc || user_param->connection_type == DC) && (user_param->duplex || user_param->tst == LAT))
-			xrc_offset = user_param->num_of_qps / 2;
-
-	}
-	return SUCCESS;
-}
-
-/******************************************************************************
- *
- ******************************************************************************/
-void ctx_set_send_wqes(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param,
-		struct pingpong_dest *rem_dest)
-{
-
-	#ifdef HAVE_VERBS_EXP
-	if (user_param->use_exp == 1) {
-		ctx_set_send_exp_wqes(ctx,user_param,rem_dest);
-	}
-	else {
-	#endif
-		ctx_set_send_reg_wqes(ctx,user_param,rem_dest);
-	#ifdef HAVE_VERBS_EXP
-	}
-	#endif
-}
-
-#ifdef HAVE_VERBS_EXP
-/******************************************************************************
- *
- ******************************************************************************/
-void ctx_set_send_exp_wqes(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param,
-		struct pingpong_dest *rem_dest)
-{
-	int i,j;
-	int num_of_qps = user_param->num_of_qps;
-	int xrc_offset = 0;
-	uint32_t remote_qpn, remote_qkey;
-
-	if((user_param->use_xrc || user_param->connection_type == DC) && (user_param->duplex || user_param->tst == LAT)) {
-		num_of_qps /= 2;
-		xrc_offset = num_of_qps;
-	}
-
-	for (i = 0; i < num_of_qps ; i++) {
-		memset(&ctx->exp_wr[i*user_param->post_list],0,sizeof(struct ibv_exp_send_wr));
-		ctx->sge_list[i*user_param->post_list].addr = (uintptr_t)ctx->buf[i];
-
-		if (user_param->mac_fwd) {
-			if (user_param->mr_per_qp) {
-				ctx->sge_list[i*user_param->post_list].addr =
-					(uintptr_t)ctx->buf[0] + (num_of_qps + i)*BUFF_SIZE(ctx->size,ctx->cycle_buffer);
-			} else {
-				ctx->sge_list[i*user_param->post_list].addr = (uintptr_t)ctx->buf[i];
-			}
-		}
-
-		if (user_param->verb == WRITE || user_param->verb == READ)
-			ctx->exp_wr[i*user_param->post_list].wr.rdma.remote_addr   = rem_dest[xrc_offset + i].vaddr;
-
-		else if (user_param->verb == ATOMIC)
-			ctx->exp_wr[i*user_param->post_list].wr.atomic.remote_addr = rem_dest[xrc_offset + i].vaddr;
-
-		if (user_param->tst == BW || user_param->tst == LAT_BY_BW ) {
-
-			ctx->scnt[i] = 0;
-			ctx->ccnt[i] = 0;
-			ctx->my_addr[i] = (uintptr_t)ctx->buf[i];
-			if (user_param->verb != SEND)
-				ctx->rem_addr[i] = rem_dest[xrc_offset + i].vaddr;
-		}
-
-		for (j = 0; j < user_param->post_list; j++) {
-
-			ctx->sge_list[i*user_param->post_list + j].length =
-				(user_param->connection_type == RawEth) ? (user_param->size - HW_CRC_ADDITION) : user_param->size;
-
-			ctx->sge_list[i*user_param->post_list + j].lkey = ctx->mr[i]->lkey;
-
-			if (j > 0) {
-
-				ctx->sge_list[i*user_param->post_list +j].addr = ctx->sge_list[i*user_param->post_list + (j-1)].addr;
-
-				if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2))
-					increase_loc_addr(&ctx->sge_list[i*user_param->post_list +j],user_param->size,
-							j-1,ctx->my_addr[i],0,ctx->cache_line_size,ctx->cycle_buffer);
-			}
-
-			ctx->exp_wr[i*user_param->post_list + j].sg_list = &ctx->sge_list[i*user_param->post_list + j];
-			ctx->exp_wr[i*user_param->post_list + j].num_sge = MAX_SEND_SGE;
-			ctx->exp_wr[i*user_param->post_list + j].wr_id   = i;
-
-			if (j == (user_param->post_list - 1)) {
-				ctx->exp_wr[i*user_param->post_list + j].next = NULL;
-			} else {
-				ctx->exp_wr[i*user_param->post_list + j].next = &ctx->exp_wr[i*user_param->post_list+j+1];
-			}
-
-			if ((j + 1) % user_param->cq_mod == 0) {
-				#ifdef HAVE_ACCL_VERBS
-				if (user_param->verb_type == ACCL_INTF)
-					ctx->exp_wr[i*user_param->post_list + j].exp_send_flags = IBV_EXP_QP_BURST_SIGNALED;
-				else
-				#endif
-					ctx->exp_wr[i*user_param->post_list + j].exp_send_flags = IBV_EXP_SEND_SIGNALED;
-			} else {
-				ctx->exp_wr[i*user_param->post_list + j].exp_send_flags = 0;
-			}
-
-			if (user_param->verb == ATOMIC) {
-				ctx->exp_wr[i*user_param->post_list + j].exp_opcode = exp_opcode_atomic_array[user_param->atomicType];
-			}
-			else {
-				ctx->exp_wr[i*user_param->post_list + j].exp_opcode = exp_opcode_verbs_array[user_param->verb];
-			}
-			if (user_param->verb == WRITE || user_param->verb == READ) {
-
-				ctx->exp_wr[i*user_param->post_list + j].wr.rdma.rkey = rem_dest[xrc_offset + i].rkey;
-
-				if (j > 0) {
-
-					ctx->exp_wr[i*user_param->post_list + j].wr.rdma.remote_addr = ctx->exp_wr[i*user_param->post_list + (j-1)].wr.rdma.remote_addr;
-
-					if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2))
-						increase_exp_rem_addr(&ctx->exp_wr[i*user_param->post_list + j],user_param->size,
-								j-1,ctx->rem_addr[i],WRITE,ctx->cache_line_size,ctx->cycle_buffer);
-				}
-
-			} else if (user_param->verb == ATOMIC) {
-
-				ctx->exp_wr[i*user_param->post_list + j].wr.atomic.rkey = rem_dest[xrc_offset + i].rkey;
-
-				if (j > 0) {
-
-					ctx->exp_wr[i*user_param->post_list + j].wr.atomic.remote_addr = ctx->exp_wr[i*user_param->post_list + j-1].wr.atomic.remote_addr;
-					if (user_param->tst == BW || user_param->tst == LAT_BY_BW)
-						increase_exp_rem_addr(&ctx->exp_wr[i*user_param->post_list + j],user_param->size,
-								j-1,ctx->rem_addr[i],ATOMIC,ctx->cache_line_size,ctx->cycle_buffer);
-				}
-
-				if (user_param->atomicType == FETCH_AND_ADD)
-					ctx->exp_wr[i*user_param->post_list + j].wr.atomic.compare_add = ATOMIC_ADD_VALUE;
-
-				else
-					ctx->exp_wr[i*user_param->post_list + j].wr.atomic.swap = ATOMIC_SWAP_VALUE;
-
-
-			} else if (user_param->verb == SEND) {
-
-				if (user_param->connection_type == UD) {
-
-					ctx->exp_wr[i*user_param->post_list + j].wr.ud.ah = ctx->ah[i];
-					if (user_param->work_rdma_cm) {
-						remote_qpn = ctx->cma_master.nodes[i].remote_qpn;
-						remote_qkey = ctx->cma_master.nodes[i].remote_qkey;
-					} else {
-						remote_qpn = rem_dest[xrc_offset + i].qpn;
-						remote_qkey = DEF_QKEY;
-					}
-					ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qkey = remote_qkey;
-					ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qpn = remote_qpn;
-
-				#ifdef HAVE_DC
-				} else if (user_param->connection_type == DC) {
-					ctx->exp_wr[i*user_param->post_list + j].dc.ah = ctx->ah[i];
-					ctx->exp_wr[i*user_param->post_list + j].dc.dct_access_key = user_param->dct_key;
-					ctx->exp_wr[i*user_param->post_list + j].dc.dct_number = rem_dest[xrc_offset + i].qpn;
-				#endif
+			err = ibv_query_qp(ctx->qp[i], &qp_attr, IBV_QP_AV, &init_attr);
+			if (err)
+				fprintf(stderr, "ibv_query_qp failed to get ah_attr\n");
+			else
+				qp_static_rate = (int)(qp_attr.ah_attr.static_rate);
+
+			//- Fall back to SW Limit only if flag undefined
+			if(err ||
+			   qp_static_rate != user_param->valid_hw_rate_limit_index ||
+			   user_param->link_type != IBV_LINK_LAYER_INFINIBAND) {
+				if(!user_param->is_rate_limit_type) {
+					user_param->rate_limit_type = SW_RATE_LIMIT;
+					fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit, providing SW rate limit \x1b[0m\n");
+				} else {
+					fprintf(stderr, "\x1b[31mThe QP failed to accept HW rate limit  \x1b[0m\n");
+					return FAILURE;
 				}
 			}
+		}
 
-			#ifdef HAVE_DC
-			if (user_param->connection_type == DC) {
-				ctx->exp_wr[i*user_param->post_list + j].dc.ah = ctx->ah[i];
-				ctx->exp_wr[i*user_param->post_list + j].dc.dct_access_key = user_param->dct_key;
-				ctx->exp_wr[i*user_param->post_list + j].dc.dct_number = rem_dest[xrc_offset + i].qpn;
-			}
-			#endif
+		if((user_param->use_xrc || user_param->connection_type == DC) && (user_param->duplex || user_param->tst == LAT))
+			xrc_offset = user_param->num_of_qps / 2;
 
+	}
+	return SUCCESS;
+}
 
-			if ((user_param->verb == SEND || user_param->verb == WRITE) && user_param->size <= user_param->inline_size)
-				ctx->exp_wr[i*user_param->post_list + j].exp_send_flags |= IBV_EXP_SEND_INLINE;
+/******************************************************************************
+ *
+ ******************************************************************************/
+void ctx_set_send_wqes(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct pingpong_dest *rem_dest)
+{
 
-			#ifdef HAVE_XRCD
-			if (user_param->use_xrc)
-				ctx->exp_wr[i*user_param->post_list + j].qp_type.xrc.remote_srqn = rem_dest[xrc_offset + i].srqn;
-			#endif
-		}
-	}
+	ctx_set_send_reg_wqes(ctx,user_param,rem_dest);
 }
-#endif
 
 /* ctx_post_send_work_request_func_pointer.
  *
@@ -3533,9 +2313,6 @@ int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *use
 {
 	int 			ne,index,warmindex,warmupsession;
 	int 			err = 0;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr 	*bad_exp_wr = NULL;
-	#endif
 	struct ibv_send_wr 	*bad_wr = NULL;
 	struct ibv_wc 		wc;
 	struct ibv_wc 		*wc_for_cleaning = NULL;
@@ -3555,15 +2332,7 @@ int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *use
 
 		for (warmindex = 0 ;warmindex < warmupsession ;warmindex += user_param->post_list) {
 
-			#ifdef HAVE_VERBS_EXP
-			if (user_param->use_exp == 1)
-				err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],
-					&ctx->exp_wr[index*user_param->post_list], &bad_exp_wr);
-			else
-				err = (ctx->post_send_func_pointer)(ctx->qp[index],&ctx->wr[index*user_param->post_list],&bad_wr);
-			#else
 			err = ibv_post_send(ctx->qp[index],&ctx->wr[index*user_param->post_list],&bad_wr);
-			#endif
 
 			if (err) {
 				fprintf(stderr,"Couldn't post send during warm up: qp %d scnt=%d \n",index,warmindex);
@@ -3634,14 +2403,6 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 	int                	index,ne;
 	uint64_t	   	tot_iters;
 	int			err = 0;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr 	*bad_exp_wr = NULL;
-	struct ibv_send_wr 	*bad_wr = NULL;
-	#ifdef HAVE_ACCL_VERBS
-	int pl_index;
-	struct ibv_sge		*sg_l;
-	#endif
-	#endif
 	struct ibv_wc 	   	*wc = NULL;
 	int 			num_of_qps = user_param->num_of_qps;
 	/* Rate Limiter*/
@@ -3744,24 +2505,11 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
 
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1)
-							ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED;
-						else
-					#endif
-						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-						#endif
 
-					#ifdef HAVE_ACCL_VERBS
-					}
+					#ifdef HAVE_IBV_WR_API
+					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					#else
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
@@ -3771,43 +2519,12 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->test_type == DURATION && user_param->state == END_STATE)
 					break;
 
-				#ifdef HAVE_VERBS_EXP
-				#ifdef HAVE_ACCL_VERBS
-				if (user_param->verb_type == ACCL_INTF) {
-					for (pl_index = 0; pl_index < user_param->post_list; pl_index++) {
-						sg_l = ctx->exp_wr[index*user_param->post_list + pl_index].sg_list;
-						ctx->qp_burst_family[index]->send_pending(ctx->qp[index], sg_l->addr, sg_l->length, sg_l->lkey,
-											ctx->exp_wr[index*user_param->post_list + pl_index].exp_send_flags);
-					}
-					ctx->qp_burst_family[index]->send_flush(ctx->qp[index]);
-				} else {
-				#endif
-					if (user_param->use_exp == 1) {
-						err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],
-						&ctx->exp_wr[index*user_param->post_list],&bad_exp_wr);
-					}
-					else {
-						err = (ctx->post_send_func_pointer)(ctx->qp[index],
-							&ctx->wr[index*user_param->post_list],&bad_wr);
-					}
-					if (err) {
-						fprintf(stderr,"Couldn't post send: qp %d scnt=%lu \n",index,ctx->scnt[index]);
-						return_value = FAILURE;
-						goto cleaning;
-					}
-
-				#ifdef HAVE_ACCL_VERBS
-				}
-				#endif
-				#else
-
 				err = post_send_method(ctx, index, user_param);
 				if (err) {
 					fprintf(stderr,"Couldn't post send: qp %d scnt=%lu \n",index,ctx->scnt[index]);
 					return_value = FAILURE;
 					goto cleaning;
 				}
-				#endif
 
 				/* if we have more than single flow and the burst iter is the last one */
 				if (user_param->flows != DEF_FLOWS) {
@@ -3823,28 +2540,14 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 
 				/* in multiple flow scenarios we will go to next cycle buffer address in the main buffer*/
 				if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
-					#ifdef HAVE_VERBS_EXP
-					if (user_param->use_exp == 1)
-						increase_loc_addr(ctx->exp_wr[index].sg_list,user_param->size,
-								ctx->scnt[index], ctx->my_addr[index] + address_offset, 0,
-									ctx->cache_line_size, ctx->cycle_buffer);
-					else
-					#endif
 						increase_loc_addr(ctx->wr[index].sg_list,user_param->size, ctx->scnt[index],
 								ctx->my_addr[index] + address_offset , 0, ctx->cache_line_size,
 								ctx->cycle_buffer);
 
 					if (user_param->verb != SEND) {
-						#ifdef HAVE_VERBS_EXP
-						if (user_param->use_exp == 1)
-							increase_exp_rem_addr(&ctx->exp_wr[index], user_param->size,
-									ctx->scnt[index], ctx->rem_addr[index], user_param->verb,
-									ctx->cache_line_size, ctx->cycle_buffer);
-						else
-						#endif
-							increase_rem_addr(&ctx->wr[index], user_param->size,
-									ctx->scnt[index], ctx->rem_addr[index], user_param->verb,
-									ctx->cache_line_size, ctx->cycle_buffer);
+						increase_rem_addr(&ctx->wr[index], user_param->size,
+								ctx->scnt[index], ctx->rem_addr[index], user_param->verb,
+								ctx->cache_line_size, ctx->cycle_buffer);
 					}
 				}
 
@@ -3855,25 +2558,11 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->post_list == 1 &&
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags |= IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1) {
-							ctx->exp_wr[index].exp_send_flags |= IBV_EXP_SEND_SIGNALED;
-						}
-						else
-					#endif
 						#ifdef HAVE_IBV_WR_API
 						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
 						#else
 						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 						#endif
-					#ifdef HAVE_ACCL_VERBS
-					}
-					#endif
 				}
 
 				/* Check if a full burst was sent. */
@@ -3893,24 +2582,15 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 						goto cleaning;
 					}
 				}
-
-				#ifdef HAVE_ACCL_VERBS
-				if (user_param->verb_type == ACCL_INTF)
-					ne = ctx->send_cq_family->poll_cnt(ctx->send_cq, CTX_POLL_BATCH);
-				else
-				#endif
-					ne = ibv_poll_cq(ctx->send_cq, CTX_POLL_BATCH, wc);
+				ne = ibv_poll_cq(ctx->send_cq, CTX_POLL_BATCH, wc);
 				if (ne > 0) {
 					for (i = 0; i < ne; i++) {
-						wc_id = (user_param->verb_type == ACCL_INTF) ?
-							0 : (int)wc[i].wr_id;
+						wc_id = (int)wc[i].wr_id;
 
-						if (user_param->verb_type != ACCL_INTF) {
-							if (wc[i].status != IBV_WC_SUCCESS) {
-								NOTIFY_COMP_ERROR_SEND(wc[i],totscnt,totccnt);
-								return_value = FAILURE;
-								goto cleaning;
-							}
+						if (wc[i].status != IBV_WC_SUCCESS) {
+							NOTIFY_COMP_ERROR_SEND(wc[i],totscnt,totccnt);
+							return_value = FAILURE;
+							goto cleaning;
 						}
 
 						ctx->ccnt[wc_id] += user_param->cq_mod;
@@ -4028,18 +2708,10 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
 			if (user_param->test_type == DURATION && user_param->state == END_STATE)
 				break;
 
-			#ifdef HAVE_ACCL_VERBS
-			if (user_param->verb_type == ACCL_INTF)
-				ne = ctx->recv_cq_family->poll_cnt(ctx->recv_cq, CTX_POLL_BATCH);
-			else {
-			#endif
-				if (user_param->connection_type == DC)
-					ne = ibv_poll_cq(ctx->send_cq,CTX_POLL_BATCH,wc);
-				else
-					ne = ibv_poll_cq(ctx->recv_cq,CTX_POLL_BATCH,wc);
-			#ifdef HAVE_ACCL_VERBS
-			}
-			#endif
+			if (user_param->connection_type == DC)
+				ne = ibv_poll_cq(ctx->send_cq,CTX_POLL_BATCH,wc);
+			else
+				ne = ibv_poll_cq(ctx->recv_cq,CTX_POLL_BATCH,wc);
 
 			if (ne > 0) {
 				if (firstRx) {
@@ -4048,16 +2720,12 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
 				}
 
 				for (i = 0; i < ne; i++) {
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-						0 : (int)wc[i].wr_id;
-
-					if (user_param->verb_type != ACCL_INTF) {
-						if (wc[i].status != IBV_WC_SUCCESS) {
+					wc_id = (int)wc[i].wr_id;
+					if (wc[i].status != IBV_WC_SUCCESS) {
 
-							NOTIFY_COMP_ERROR_RECV(wc[i],rcnt_for_qp[wc_id]);
-							return_value = FAILURE;
-							goto cleaning;
-						}
+						NOTIFY_COMP_ERROR_RECV(wc[i],rcnt_for_qp[wc_id]);
+						return_value = FAILURE;
+						goto cleaning;
 					}
 					rcnt_for_qp[wc_id]++;
 					rcnt++;
@@ -4070,42 +2738,30 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
 						user_param->iters++;
 					}
 					if (user_param->test_type==DURATION || rcnt_for_qp[wc_id] + size_per_qp <= user_param->iters) {
-						#ifdef HAVE_ACCL_VERBS
-						if (user_param->verb_type == ACCL_INTF) {
-							if (ctx->qp_burst_family[wc_id]->recv_burst(ctx->qp[wc_id], ctx->rwr[wc_id].sg_list, 1)) {
-								fprintf(stderr, "Couldn't post recv burst (accelerated verbs).\n");
+						if (user_param->use_srq) {
+							if (ibv_post_srq_recv(ctx->srq, &ctx->rwr[wc_id],&bad_wr_recv)) {
+								fprintf(stderr, "Couldn't post recv SRQ. QP = %d: counter=%lu\n", wc_id,rcnt);
 								return_value = FAILURE;
 								goto cleaning;
 							}
-						} else {
-						#endif
-							if (user_param->use_srq) {
-								if (ibv_post_srq_recv(ctx->srq, &ctx->rwr[wc_id],&bad_wr_recv)) {
-									fprintf(stderr, "Couldn't post recv SRQ. QP = %d: counter=%lu\n", wc_id,rcnt);
-									return_value = FAILURE;
-									goto cleaning;
-								}
-
-							} else {
-								if (ibv_post_recv(ctx->qp[wc_id],&ctx->rwr[wc_id],&bad_wr_recv)) {
-									fprintf(stderr, "Couldn't post recv Qp=%d rcnt=%ld\n",wc_id,rcnt_for_qp[wc_id]);
-									return_value = 15;
-									goto cleaning;
-								}
 
+						} else {
+							if (ibv_post_recv(ctx->qp[wc_id],&ctx->rwr[wc_id],&bad_wr_recv)) {
+								fprintf(stderr, "Couldn't post recv Qp=%d rcnt=%ld\n",wc_id,rcnt_for_qp[wc_id]);
+								return_value = 15;
+								goto cleaning;
 							}
-							if (user_param->flows != DEF_FLOWS) {
-								if (++recv_flows_burst == user_param->flows_burst) {
-									recv_flows_burst = 0;
-									if (++recv_flows_index == user_param->flows)
-										recv_flows_index = 0;
-									address_flows_offset = recv_flows_index * ctx->cycle_buffer;
-									ctx->recv_sge_list[0].addr = primary_recv_addr + address_flows_offset;
-								}
+
+						}
+						if (user_param->flows != DEF_FLOWS) {
+							if (++recv_flows_burst == user_param->flows_burst) {
+								recv_flows_burst = 0;
+								if (++recv_flows_index == user_param->flows)
+									recv_flows_index = 0;
+								address_flows_offset = recv_flows_index * ctx->cycle_buffer;
+								ctx->recv_sge_list[0].addr = primary_recv_addr + address_flows_offset;
 							}
-						#ifdef HAVE_ACCL_VERBS
 						}
-						#endif
 						if (SIZE(user_param->connection_type,user_param->size,!(int)user_param->machine) <= (ctx->cycle_buffer / 2)) {
 							increase_loc_addr(ctx->rwr[wc_id].sg_list,
 									user_param->size,
@@ -4200,10 +2856,6 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 	int 			index = 0,ne;
 	int 			err = 0;
 	int			wc_id;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr 	*bad_exp_wr = NULL;
-	struct ibv_send_wr 	*bad_wr = NULL;
-	#endif
 	uint64_t		*scnt_for_qp = NULL;
 	struct ibv_wc 		*wc = NULL;
 	int 			num_of_qps = user_param->num_of_qps;
@@ -4252,35 +2904,14 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 				}
 
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)) {
-
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1)
-							ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED;
-						else
-					#endif
-						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-						#endif
-					#ifdef HAVE_ACCL_VERBS
-					}
+					#ifdef HAVE_IBV_WR_API
+					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					#else
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
-				#ifdef HAVE_VERBS_EXP
-				if (user_param->use_exp == 1)
-					err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],&ctx->exp_wr[index*user_param->post_list],&bad_exp_wr);
-				else
-					err = (ctx->post_send_func_pointer)(ctx->qp[index],&ctx->wr[index*user_param->post_list],&bad_wr);
-				#else
 				err = post_send_method(ctx, index, user_param);
-				#endif
 				if (err) {
 					fprintf(stderr,"Couldn't post send: %d scnt=%lu \n",index,ctx->scnt[index]);
 					return_value = FAILURE;
@@ -4294,23 +2925,10 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 				if (user_param->post_list == 1 &&
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags |= IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1)
-							ctx->exp_wr[index].exp_send_flags |= IBV_EXP_SEND_SIGNALED;
-						else
-					#endif
-						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-						#endif
-					#ifdef HAVE_ACCL_VERBS
-					}
+					#ifdef HAVE_IBV_WR_API
+					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
+					#else
+					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#endif
 				}
 			}
@@ -4326,8 +2944,7 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 						return_value = FAILURE;
 						goto cleaning;
 					}
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-							0 : (int)wc[i].wr_id;
+					wc_id = (int)wc[i].wr_id;
 					user_param->iters += user_param->cq_mod;
 					totccnt += user_param->cq_mod;
 					ctx->ccnt[wc_id] += user_param->cq_mod;
@@ -4480,10 +3097,6 @@ int run_iter_bi(struct pingpong_context *ctx,
 	struct ibv_wc 		*wc = NULL;
 	struct ibv_wc 		*wc_tx = NULL;
 	struct ibv_recv_wr 	*bad_wr_recv = NULL;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr 	*bad_exp_wr      = NULL;
-	struct ibv_send_wr 	*bad_wr      = NULL;
-	#endif
 	int 			num_of_qps = user_param->num_of_qps;
 	/* This is to ensure SERVER will not start to send packets before CLIENT start the test. */
 	int 			before_first_rx = ON;
@@ -4547,16 +3160,11 @@ int run_iter_bi(struct pingpong_context *ctx,
 				}
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
-					#ifdef HAVE_VERBS_EXP
-					if (user_param->use_exp ==1)
-						ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED;
-					else
+					#ifdef HAVE_IBV_WR_API
+					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					#else
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
-						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-						#endif
 				}
 				if (user_param->noPeak == OFF)
 					user_param->tposted[totscnt] = get_cycles();
@@ -4564,16 +3172,8 @@ int run_iter_bi(struct pingpong_context *ctx,
 				if (user_param->test_type == DURATION && duration_param->state == END_STATE)
 					break;
 
-				#ifdef HAVE_VERBS_EXP
-				if (user_param->use_exp == 1)
-					err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],
-						&ctx->exp_wr[index*user_param->post_list],&bad_exp_wr);
-				else
-					err = (ctx->post_send_func_pointer)(ctx->qp[index],
-						&ctx->wr[index*user_param->post_list],&bad_wr);
-				#else
 				err = post_send_method(ctx, index, user_param);
-				#endif
+
 				if (err) {
 					fprintf(stderr,"Couldn't post send: qp %d scnt=%lu \n",index,ctx->scnt[index]);
 					return_value = FAILURE;
@@ -4581,14 +3181,8 @@ int run_iter_bi(struct pingpong_context *ctx,
 				}
 
 				if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
-					#ifdef HAVE_VERBS_EXP
-					if (user_param->use_exp == 1)
-						increase_loc_addr(ctx->exp_wr[index].sg_list,user_param->size,ctx->scnt[index],
-								ctx->my_addr[index],0,ctx->cache_line_size,ctx->cycle_buffer);
-					else
-					#endif
-						increase_loc_addr(ctx->wr[index].sg_list,user_param->size,ctx->scnt[index],
-								ctx->my_addr[index],0,ctx->cache_line_size,ctx->cycle_buffer);
+					increase_loc_addr(ctx->wr[index].sg_list,user_param->size,ctx->scnt[index],
+						ctx->my_addr[index],0,ctx->cache_line_size,ctx->cycle_buffer);
 				}
 
 				ctx->scnt[index] += user_param->post_list;
@@ -4598,16 +3192,11 @@ int run_iter_bi(struct pingpong_context *ctx,
 					(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 						(user_param->test_type == ITERATIONS && ctx->scnt[index] == iters-1))) {
 
-					#ifdef HAVE_VERBS_EXP
-					if (user_param->use_exp == 1)
-						ctx->exp_wr[index].exp_send_flags |= IBV_EXP_SEND_SIGNALED;
-					else
+					#ifdef HAVE_IBV_WR_API
+					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
+					#else
+					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#endif
-						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-						#endif
 				}
 			}
 		}
@@ -4825,35 +3414,21 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 	int 			poll_buf_offset = 0;
 	volatile char           *poll_buf = NULL;
 	volatile char           *post_buf = NULL;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr  *bad_exp_wr = NULL;
-	struct ibv_send_wr      *bad_wr = NULL;
-	#endif
+
 	struct ibv_wc           wc;
 
 	int 			cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f);
 	int 			total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 		end_cycle, start_gap=0;
 
-	#ifdef HAVE_VERBS_EXP
-	if (user_param->use_exp == 1) {
-		ctx->exp_wr[0].sg_list->length = user_param->size;
-		ctx->exp_wr[0].exp_send_flags      = IBV_EXP_SEND_SIGNALED;
-		if (user_param->size <= user_param->inline_size)
-			ctx->exp_wr[0].exp_send_flags |= IBV_EXP_SEND_INLINE;
-	} else {
-	#endif
-		ctx->wr[0].sg_list->length = user_param->size;
-		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-		#else
-		ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
-		#endif
-		if (user_param->size <= user_param->inline_size)
-			ctx->wr[0].send_flags |= IBV_SEND_INLINE;
-	#ifdef HAVE_VERBS_EXP
-	}
+	ctx->wr[0].sg_list->length = user_param->size;
+	#ifdef HAVE_IBV_WR_API
+	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+	#else
+	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#endif
+	if (user_param->size <= user_param->inline_size)
+		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 
 	if((user_param->use_xrc || user_param->connection_type == DC))
 		poll_buf_offset = 1;
@@ -4896,14 +3471,9 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 				user_param->tposted[scnt] = get_cycles();
 
 			*post_buf = (char)++scnt;
-			#ifdef HAVE_VERBS_EXP
-			if (user_param->use_exp == 1)
-				err = (ctx->exp_post_send_func_pointer)(ctx->qp[0],&ctx->exp_wr[0],&bad_exp_wr);
-			else
-				err = (ctx->post_send_func_pointer)(ctx->qp[0],&ctx->wr[0],&bad_wr);
-			#else
+
 			err = post_send_method(ctx, 0, user_param);
-			#endif
+
 			if (err) {
 				fprintf(stderr,"Couldn't post send: scnt=%lu\n",scnt);
 				return 1;
@@ -4945,30 +3515,18 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	uint64_t	scnt = 0;
 	int 		ne;
 	int		err = 0;
-	#ifdef HAVE_VERBS_EXP
-	struct 		ibv_exp_send_wr *bad_exp_wr = NULL;
-	struct 		ibv_send_wr *bad_wr = NULL;
-	#endif
+
 	struct 		ibv_wc wc;
 
 	int 		cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f);
 	int 		total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 	end_cycle, start_gap=0;
 
-	#ifdef HAVE_VERBS_EXP
-	if (user_param->use_exp == 1) {
-		ctx->exp_wr[0].sg_list->length = user_param->size;
-		ctx->exp_wr[0].exp_send_flags = IBV_EXP_SEND_SIGNALED;
-	} else {
-	#endif
-		ctx->wr[0].sg_list->length = user_param->size;
-		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-		#else
-		ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
-		#endif
-	#ifdef HAVE_VERBS_EXP
-	}
+	ctx->wr[0].sg_list->length = user_param->size;
+	#ifdef HAVE_IBV_WR_API
+	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+	#else
+	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#endif
 
 	/* Duration support in latency tests. */
@@ -4993,14 +3551,8 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 		if (user_param->test_type == ITERATIONS)
 			user_param->tposted[scnt++] = get_cycles();
 
-		#ifdef HAVE_VERBS_EXP
-		if (user_param->use_exp == 1)
-			err = (ctx->exp_post_send_func_pointer)(ctx->qp[0],&ctx->exp_wr[0],&bad_exp_wr);
-		else
-			err = (ctx->post_send_func_pointer)(ctx->qp[0],&ctx->wr[0],&bad_wr);
-		#else
 		err = post_send_method(ctx, 0, user_param);
-		#endif
+
 		if (err) {
 			fprintf(stderr,"Couldn't post send: scnt=%lu\n",scnt);
 			return 1;
@@ -5049,10 +3601,7 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 	int			err = 0;
 	struct ibv_wc		wc;
 	struct ibv_recv_wr	*bad_wr_recv;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr	*bad_exp_wr;
-	struct ibv_send_wr	*bad_wr;
-	#endif
+
 	int  			firstRx = 1;
 	int 			size_per_qp = (user_param->use_srq) ?
 					user_param->rx_depth/user_param->num_of_qps : user_param->rx_depth;
@@ -5064,29 +3613,14 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 	uintptr_t		primary_send_addr = ctx->sge_list[0].addr;
 	uintptr_t		primary_recv_addr = ctx->recv_sge_list[0].addr;
 	if (user_param->connection_type != RawEth) {
-		#ifdef HAVE_VERBS_EXP
-		if (user_param->use_exp == 1) {
-			ctx->exp_wr[0].sg_list->length = user_param->size;
-			ctx->exp_wr[0].exp_send_flags = 0;
-		} else {
-		#endif
-			ctx->wr[0].sg_list->length = user_param->size;
-			ctx->wr[0].send_flags = 0;
-			#ifdef HAVE_IBV_WR_API
-			ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-			#endif
-		#ifdef HAVE_VERBS_EXP
-		}
+		ctx->wr[0].sg_list->length = user_param->size;
+		ctx->wr[0].send_flags = 0;
+		#ifdef HAVE_IBV_WR_API
+		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
 		#endif
-
 	}
 	if (user_param->size <= user_param->inline_size) {
-		#ifdef HAVE_VERBS_EXP
-		if (user_param->use_exp == 1)
-			ctx->exp_wr[0].exp_send_flags |= IBV_EXP_SEND_INLINE;
-		else
-		#endif
-			ctx->wr[0].send_flags |= IBV_SEND_INLINE;
+		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 	}
 	while (scnt < user_param->iters || rcnt < user_param->iters ||
 			( (user_param->test_type == DURATION && user_param->state != END_STATE))) {
@@ -5175,12 +3709,7 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 
 			if (scnt % user_param->cq_mod == 0 || (user_param->test_type == ITERATIONS && scnt == user_param->iters)) {
 				poll = 1;
-				#ifdef HAVE_VERBS_EXP
-				if (user_param->use_exp == 1)
-					ctx->exp_wr[0].exp_send_flags |= IBV_EXP_SEND_SIGNALED;
-				else
-				#endif
-					ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
+				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
 			}
 
 			/* if we're in duration mode and the time is over, exit from this function */
@@ -5188,14 +3717,8 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				break;
 
 			/* send the packet that's in index 0 on the buffer */
-			#ifdef HAVE_VERBS_EXP
-			if (user_param->use_exp == 1)
-				err = (ctx->exp_post_send_func_pointer)(ctx->qp[0],&ctx->exp_wr[0],&bad_exp_wr);
-			else
-				err = (ctx->post_send_func_pointer)(ctx->qp[0],&ctx->wr[0],&bad_wr);
-			#else
 			err = post_send_method(ctx, 0, user_param);
-			#endif
+
 			if (err) {
 				fprintf(stderr,"Couldn't post send: scnt=%lu \n",scnt);
 				return 1;
@@ -5236,12 +3759,7 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				}
 				poll = 0;
 
-				#ifdef HAVE_VERBS_EXP
-				if (user_param->use_exp == 1)
-					ctx->exp_wr[0].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED;
-				else
-				#endif
-					ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
+				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
 			}
 		}
 	}
@@ -5329,9 +3847,6 @@ int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters
 	int			i = 0;
 	int			wc_id;
 	struct ibv_wc		*wc;
-	#ifdef HAVE_VERBS_EXP
-	struct ibv_exp_send_wr	*bad_exp_wr;
-	#endif
 	struct ibv_send_wr	*bad_wr;
 	int			cpu_mhz;
 	int			return_value = 0;
@@ -5393,28 +3908,16 @@ int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters
 		}
 		while ((totscnt < user_param->iters)
 			&& (totscnt - totccnt) < (user_param->tx_depth) && !(is_sending_burst == 0 )) {
-			#ifdef HAVE_VERBS_EXP
-			if (user_param->use_exp == 1)
-				err = (ctx->exp_post_send_func_pointer)(ctx->qp[0],
-					&ctx->exp_wr[0], &bad_exp_wr);
-			else
-				err = (ctx->post_send_func_pointer)(ctx->qp[0],&ctx->wr[0],&bad_wr);
-			#else
+
 			err = ibv_post_send(ctx->qp[0],&ctx->wr[0],&bad_wr);
-                        #endif
+
 			if (err) {
 				fprintf(stderr, "Couldn't post send: scnt=%lu\n", totscnt);
 				return 1;
 			}
 			if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
-				#ifdef HAVE_VERBS_EXP
-				if (user_param->use_exp == 1)
-					increase_loc_addr(ctx->exp_wr[0].sg_list, user_param->size,
-								totscnt, ctx->my_addr[0], 0, ctx->cache_line_size, ctx->cycle_buffer);
-				else
-				#endif
-					increase_loc_addr(ctx->wr[0].sg_list, user_param->size, totscnt,
-							ctx->my_addr[0], 0, ctx->cache_line_size, ctx->cycle_buffer);
+				increase_loc_addr(ctx->wr[0].sg_list, user_param->size, totscnt,
+					ctx->my_addr[0], 0, ctx->cache_line_size, ctx->cycle_buffer);
 			}
 			totscnt += user_param->post_list;
 			if (totscnt % user_param->reply_every == 0 && totscnt != 0) {
@@ -5430,8 +3933,7 @@ int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters
 			ne = ibv_poll_cq(ctx->recv_cq, CTX_POLL_BATCH, wc);
 			if (ne > 0) {
 				for (i = 0; i < ne; i++) {
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-							0 : (int)wc[i].wr_id;
+					wc_id = (int)wc[i].wr_id;
 					user_param->tcompleted[totrcnt] = get_cycles();
 					totrcnt++;
 					if (wc[i].status != IBV_WC_SUCCESS) {
@@ -5452,8 +3954,7 @@ int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters
 			ns = ibv_poll_cq(ctx->send_cq, user_param->burst_size, wc);
 			if (ns > 0) {
 				for (i = 0; i < ns; i++) {
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-						0 : (int)wc[i].wr_id;
+					wc_id = (int)wc[i].wr_id;
 					if (wc[i].status != IBV_WC_SUCCESS) {
 						NOTIFY_COMP_ERROR_SEND(wc[i], totscnt, totccnt);
 						return_value = FAILURE ;
@@ -5575,69 +4076,7 @@ void *handle_signal_print_thread(void *sigmask)
 /******************************************************************************
  *
  ******************************************************************************/
-#ifdef HAVE_MASKED_ATOMICS
-int check_masked_atomics_support(struct pingpong_context *ctx)
-{
-	struct ibv_exp_device_attr attr;
-        memset(&attr,0,sizeof (struct ibv_exp_device_attr));
-
-	attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS;
-	attr.exp_atomic_cap = IBV_EXP_ATOMIC_HCA_REPLY_BE;
-
-	if (ibv_exp_query_device(ctx->context, &attr)) {
-		fprintf(stderr, "ibv_exp_query_device failed\n");
-		return -1;
-	}
-
-	return MASK_IS_SET(IBV_EXP_ATOMIC_HCA_REPLY_BE, attr.exp_atomic_cap) &&
-		MASK_IS_SET(IBV_EXP_DEVICE_EXT_ATOMICS, attr.exp_device_cap_flags);
-}
-#endif
-
-/******************************************************************************
- *
- ******************************************************************************/
-#ifdef HAVE_PACKET_PACING_EXP
-int check_packet_pacing_support(struct pingpong_context *ctx)
-{
-	struct ibv_exp_device_attr attr;
-	memset(&attr, 0, sizeof (struct ibv_exp_device_attr));
-
-	attr.comp_mask = IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS;
-
-	if (ibv_exp_query_device(ctx->context, &attr)) {
-		fprintf(stderr, "ibv_exp_query_device failed\n");
-		return FAILURE;
-	}
-
-	return MASK_IS_SET(IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS, attr.comp_mask) ?
-		SUCCESS : FAILURE;
-}
-
-#ifdef HAVE_PACKET_PACING_EXTENSION_EXP
-int check_packet_pacing_extension_support(struct pingpong_context *ctx)
-{
-	struct ibv_exp_device_attr attr;
-	memset(&attr, 0, sizeof (struct ibv_exp_device_attr));
-
-	attr.comp_mask = IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS;
-
-	if (ibv_exp_query_device(ctx->context, &attr)) {
-		fprintf(stderr, "ibv_exp_query_device failed\n");
-		return FAILURE;
-	}
-
-	if (!MASK_IS_SET(IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS, attr.comp_mask))
-		return FAILURE;
-
-	if (attr.packet_pacing_caps.cap_flags & IBV_EXP_QP_SUPPORT_BURST)
-		return SUCCESS;
-
-	return FAILURE;
-}
-#endif
-
-#elif HAVE_PACKET_PACING
+#ifdef HAVE_PACKET_PACING
 int check_packet_pacing_support(struct pingpong_context *ctx)
 {
 	struct ibv_device_attr_ex attr;
@@ -5658,13 +4097,8 @@ int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_p
 	struct raw_ethernet_info	*my_dest_info = NULL;
 	struct raw_ethernet_info	*rem_dest_info = NULL;
 
-	#ifdef HAVE_RAW_ETH_EXP
-	struct ibv_exp_flow		**flow_create_result;
-	struct ibv_exp_flow_attr	**flow_rules;
-	#else
 	struct ibv_flow			**flow_create_result;
 	struct ibv_flow_attr		**flow_rules;
-	#endif
 	int 				flow_index = 0;
 	int				qp_index = 0;
 	int				retval = SUCCESS;
@@ -5686,13 +4120,8 @@ int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_p
 	}
 
 
-	#ifdef HAVE_RAW_ETH_EXP
-        ALLOCATE(flow_create_result, struct ibv_exp_flow*, allocated_flows * user_param->num_of_qps);
-        ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, allocated_flows * user_param->num_of_qps);
-	#else
-        ALLOCATE(flow_create_result, struct ibv_flow*, allocated_flows * user_param->num_of_qps);
-        ALLOCATE(flow_rules, struct ibv_flow_attr*, allocated_flows * user_param->num_of_qps);
-	#endif
+	ALLOCATE(flow_create_result, struct ibv_flow*, allocated_flows * user_param->num_of_qps);
+	ALLOCATE(flow_rules, struct ibv_flow_attr*, allocated_flows * user_param->num_of_qps);
 
 	if(user_param->test_type == DURATION) {
 		duration_param = user_param;
@@ -5720,13 +4149,8 @@ int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_p
 					user_param->tposted[tot_fs_cnt] = get_cycles();
 				else if (user_param->test_type == DURATION && duration_param->state == END_STATE)
 					break;
-				#ifdef HAVE_RAW_ETH_EXP
-				flow_create_result[flow_index] =
-					ibv_exp_create_flow(ctx->qp[qp_index], flow_rules[(qp_index * allocated_flows) + flow_index]);
-				#else
 				flow_create_result[flow_index] =
 					ibv_create_flow(ctx->qp[qp_index], flow_rules[(qp_index * allocated_flows) + flow_index]);
-				#endif
 				if (user_param->test_type == ITERATIONS)
 					user_param->tcompleted[tot_fs_cnt] = get_cycles();
 				if (!flow_create_result[flow_index]) {
@@ -5749,11 +4173,7 @@ int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_p
 cleaning:
 	/* destroy open flows */
 	for (flow_index = 0; flow_index < tot_iters; flow_index++) {
-		#ifdef HAVE_RAW_ETH_EXP
-		if (ibv_exp_destroy_flow(flow_create_result[flow_index])) {
-		#else
 		if (ibv_destroy_flow(flow_create_result[flow_index])) {
-		#endif
 			perror("error");
 			fprintf(stderr, "Couldn't destroy flow\n");
 		}
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
index 0f68bdc1..e6233f0a 100755
--- a/src/perftest_resources.h
+++ b/src/perftest_resources.h
@@ -200,20 +200,6 @@ struct pingpong_context {
 	struct ibv_xrcd				*xrc_domain;
 	int 					fd;
 	#endif
-	#if defined(HAVE_VERBS_EXP)
-        drv_exp_post_send_func			exp_post_send_func_pointer;
-        drv_post_send_func			post_send_func_pointer;
-	drv_poll_cq_func			poll_cq_func_pointer;
-	struct ibv_exp_dct			**dct;
-	struct ibv_exp_send_wr			*exp_wr;
-	#endif
-	#ifdef HAVE_ACCL_VERBS
-	struct ibv_exp_res_domain		*res_domain;
-	struct ibv_exp_cq_family		*send_cq_family;
-	struct ibv_exp_cq_family		*recv_cq_family;
-	struct ibv_exp_qp_burst_family		**qp_burst_family;
-	#endif
-
 };
 
  struct pingpong_dest {
@@ -398,24 +384,6 @@ int ctx_connect(struct pingpong_context *ctx,
 				struct perftest_parameters *user_param,
 				struct pingpong_dest *my_dest);
 
-/* ctx_set_send_exp_wqes.
- *
- * Description :
- *
- *	Prepare the exp send work request templates for all QPs
- *
- * Parameters :
- *
- *	ctx     - Test Context.
- *	user_param  - user_parameters struct for this test.
- *  rem_dest   - pingpong_dest struct of the remote side.
- *
- */
-void ctx_set_send_exp_wqes(struct pingpong_context *ctx,
-					   struct perftest_parameters *user_param,
-					   struct pingpong_dest *rem_dest);
-
-
 /* ctx_set_send_regwqes.
  *
  * Description :
@@ -685,25 +653,6 @@ static __inline int ctx_notify_events(struct ibv_comp_channel *channel)
 	return 0;
 }
 
-#if defined(HAVE_VERBS_EXP)
-static __inline void increase_exp_rem_addr(struct ibv_exp_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
-{
-	if (verb == ATOMIC)
-		wr->wr.atomic.remote_addr += INC(size,cache_line_size);
-
-	else
-		wr->wr.rdma.remote_addr += INC(size,cache_line_size);
-
-	if ( ((scnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0) {
-
-		if (verb == ATOMIC)
-			wr->wr.atomic.remote_addr = prim_addr;
-
-		else
-			wr->wr.rdma.remote_addr = prim_addr;
-	}
-}
-#endif
 static __inline void increase_rem_addr(struct ibv_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
 {
 	if (verb == ATOMIC)
@@ -775,59 +724,18 @@ void catch_alarm_infintely();
 */
 void *handle_signal_print_thread(void *sig_mask);
 
-/* ctx_modify_dc_qp_to_init.
- *
- * Description :
- *
- *	Modifies the given QP to INIT state , according to attributes in param.
- *  The relevent attributes are ib_port, connection_type and verb.
- *
- * Parameters :
- *
- *	qp     - The QP that will be moved to INIT.
- *	param  - The parameters for the QP.
- *
- * Return Value : SUCCESS, FAILURE.
- *
- */
-int ctx_modify_dc_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param);
-
 int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *user_param);
 
-#ifdef HAVE_MASKED_ATOMICS
-struct ibv_qp* ctx_atomic_qp_create(struct pingpong_context *ctx,
-					struct perftest_parameters *user_param);
-int check_masked_atomics_support(struct pingpong_context *ctx);
-#endif
-
-#if defined (HAVE_PACKET_PACING_EXP) || defined (HAVE_PACKET_PACING)
+#if defined (HAVE_PACKET_PACING)
 int check_packet_pacing_support(struct pingpong_context *ctx);
-#if defined (HAVE_PACKET_PACING_EXTENSION_EXP)
-int check_packet_pacing_extension_support(struct pingpong_context *ctx);
-#endif
-#endif
-
-#ifdef HAVE_ACCL_VERBS
-struct ibv_exp_res_domain* create_res_domain(struct pingpong_context *ctx,
-						struct perftest_parameters *user_param);
 #endif
 
 int create_reg_qp_main(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param, int i, int num_of_qps);
 
-#ifdef HAVE_VERBS_EXP
-int create_exp_qp_main(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param, int i, int num_of_qps);
-#endif
-
 int create_qp_main(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param, int i, int num_of_qps);
 
-#ifdef HAVE_VERBS_EXP
-struct ibv_qp* ctx_exp_qp_create(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param, int qp_index);
-#endif
-
 int modify_qp_to_init(struct pingpong_context *ctx,
                 struct perftest_parameters *user_param, int qp_index, int num_of_qps);
 
diff --git a/src/raw_ethernet_resources.c b/src/raw_ethernet_resources.c
index 9e3a4556..bfc15454 100755
--- a/src/raw_ethernet_resources.c
+++ b/src/raw_ethernet_resources.c
@@ -216,46 +216,25 @@ static int get_ip_size(int is_ipv6)
 {
 #ifdef HAVE_IPV6
 	if (is_ipv6)
-#ifdef HAVE_RAW_ETH_EXP
-		return sizeof(struct ibv_exp_flow_spec_ipv6_ext);
-#else
 		return sizeof(struct ibv_flow_spec_ipv6);
 #endif
-#endif /* HAVE_IPV6 */
 
-#ifdef HAVE_RAW_ETH_EXP
-#ifdef HAVE_IPV4_EXT
-		return sizeof(struct ibv_exp_flow_spec_ipv4_ext);
-#else
-		return sizeof(struct ibv_exp_flow_spec_ipv4);
-#endif /* HAVE_IPV4_EXT */
-#else
 #ifdef HAVE_IPV4_EXT
 		return sizeof(struct ibv_flow_spec_ipv4_ext);
 #else
 		return sizeof(struct ibv_flow_spec_ipv4);
 #endif /* HAVE_IPV4_EXT */
-#endif
 }
 
 /******************************************************************************
  * print test specification
  ******************************************************************************/
-#ifdef HAVE_RAW_ETH_EXP
-void print_spec(struct ibv_exp_flow_attr* flow_rules,struct perftest_parameters* user_param)
-{
-	struct ibv_exp_flow_spec* spec_info = NULL;
-	#ifdef HAVE_IPV6
-	struct ibv_exp_flow_spec_ipv6_ext *ipv6_spec;
-	#endif
-#else
 void print_spec(struct ibv_flow_attr* flow_rules,struct perftest_parameters* user_param)
 {
 		struct ibv_flow_spec* spec_info = NULL;
 		#ifdef HAVE_IPV6
 		struct ibv_flow_spec_ipv6 *ipv6_spec;
 		#endif
-#endif
 		void* header_buff = (void*)flow_rules;
 		int ip_size = get_ip_size(user_param->raw_ipv6);
 
@@ -268,13 +247,10 @@ void print_spec(struct ibv_flow_attr* flow_rules,struct perftest_parameters* use
 			return;
 		}
 
-		#ifdef HAVE_RAW_ETH_EXP
-		header_buff = header_buff + sizeof(struct ibv_exp_flow_attr);
-		spec_info = (struct ibv_exp_flow_spec*)header_buff;
-		#else
+
 		header_buff = header_buff + sizeof(struct ibv_flow_attr);
 		spec_info = (struct ibv_flow_spec*)header_buff;
-		#endif
+
 		printf("MAC attached  : %02X:%02X:%02X:%02X:%02X:%02X\n",
 				spec_info->eth.val.dst_mac[0],
 				spec_info->eth.val.dst_mac[1],
@@ -290,19 +266,11 @@ void print_spec(struct ibv_flow_attr* flow_rules,struct perftest_parameters* use
 			char str_ip6_s[INET6_ADDRSTRLEN] = {0};
 			char str_ip6_d[INET6_ADDRSTRLEN] = {0};
 			#endif
-			#ifdef HAVE_RAW_ETH_EXP
-			header_buff = header_buff + sizeof(struct ibv_exp_flow_spec_eth);
-			spec_info = (struct ibv_exp_flow_spec*)header_buff;
-			#ifdef HAVE_IPV6
-			ipv6_spec = &spec_info->ipv6_ext;
-			#endif
-			#else
 			header_buff = header_buff + sizeof(struct ibv_flow_spec_eth);
 			spec_info = (struct ibv_flow_spec*)header_buff;
 			#ifdef HAVE_IPV6
 			ipv6_spec = &spec_info->ipv6;
 			#endif
-			#endif
 			if (!user_param->raw_ipv6) {
 				uint32_t dst_ip = spec_info->ipv4.val.dst_ip;
 				uint32_t src_ip = spec_info->ipv4.val.src_ip;
@@ -687,40 +655,16 @@ void create_raw_eth_pkt( struct perftest_parameters *user_param,
  ******************************************************************************/
 int calc_flow_rules_size(struct perftest_parameters *user_param, int is_ip_header,int is_udp_header)
 {
-	#ifdef HAVE_RAW_ETH_EXP
-	int tot_size = sizeof(struct ibv_exp_flow_attr);
-	tot_size += sizeof(struct ibv_exp_flow_spec_eth);
-	if (is_ip_header)
-		tot_size += get_ip_size(user_param->raw_ipv6);
-	if (is_udp_header)
-		tot_size += sizeof(struct ibv_exp_flow_spec_tcp_udp);
-	#else
 	int tot_size = sizeof(struct ibv_flow_attr);
 	tot_size += sizeof(struct ibv_flow_spec_eth);
 	if (is_ip_header)
 		tot_size += get_ip_size(user_param->raw_ipv6);
 	if (is_udp_header)
 		tot_size += sizeof(struct ibv_flow_spec_tcp_udp);
-	#endif
 
 	return tot_size;
 }
 
-#ifdef HAVE_RAW_ETH_EXP
-static void fill_ip_common(struct ibv_exp_flow_spec* spec_info,
-			   struct perftest_parameters *user_param)
-{
-	#ifdef HAVE_IPV6
-	struct ibv_exp_flow_spec_ipv6_ext *ipv6_spec = &spec_info->ipv6_ext;
-	static const char ipv6_mask[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-					 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	#endif
-	#ifdef HAVE_IPV4_EXT
-	struct ibv_exp_flow_spec_ipv4_ext *ipv4_spec = &spec_info->ipv4_ext;
-	#else
-	struct ibv_exp_flow_spec_ipv4 *ipv4_spec = &spec_info->ipv4;
-	#endif /* HAVE_IPV4_EXT */
-#else
 static void fill_ip_common(struct ibv_flow_spec* spec_info,
 			   struct perftest_parameters *user_param)
 {
@@ -734,7 +678,6 @@ static void fill_ip_common(struct ibv_flow_spec* spec_info,
 	#else
 	struct ibv_flow_spec_ipv4 *ipv4_spec = &spec_info->ipv4;
 	#endif
-#endif
 
 	if(user_param->machine == SERVER) {
 		if (user_param->raw_ipv6) {
@@ -773,27 +716,6 @@ static void fill_ip_common(struct ibv_flow_spec* spec_info,
 	}
 }
 
-#ifdef HAVE_RAW_ETH_EXP
-static void fill_exp_ip_spec(struct ibv_exp_flow_spec* spec_info,
-			     struct perftest_parameters *user_param)
-{
-	if (user_param->raw_ipv6) {
-		#ifdef HAVE_IPV6
-		spec_info->ipv6.type = IBV_EXP_FLOW_SPEC_IPV6_EXT;
-		spec_info->ipv6.size = sizeof(struct ibv_exp_flow_spec_ipv6_ext);
-		#endif
-	} else {
-		#ifdef HAVE_IPV4_EXT
-		spec_info->ipv4.type = IBV_EXP_FLOW_SPEC_IPV4_EXT;
-		spec_info->ipv4.size = sizeof(struct ibv_exp_flow_spec_ipv4_ext);
-		#else
-		spec_info->ipv4.type = IBV_EXP_FLOW_SPEC_IPV4;
-		spec_info->ipv4.size = sizeof(struct ibv_exp_flow_spec_ipv4);
-		#endif
-	}
-	fill_ip_common(spec_info, user_param);
-}
-#else
 static void fill_ip_spec(struct ibv_flow_spec* spec_info,
 			 struct perftest_parameters *user_param)
 {
@@ -813,26 +735,16 @@ static void fill_ip_spec(struct ibv_flow_spec* spec_info,
 	}
 	fill_ip_common(spec_info, user_param);
 }
-#endif
 
 int set_up_flow_rules(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		int local_port,
 		int remote_port)
 {
-	#ifdef HAVE_RAW_ETH_EXP
-	struct ibv_exp_flow_spec* spec_info;
-	struct ibv_exp_flow_attr* attr_info;
-	#else
 	struct ibv_flow_spec* spec_info;
 	struct ibv_flow_attr* attr_info;
-	#endif
 
 	void* header_buff;
 	int flow_rules_size;
@@ -845,13 +757,8 @@ int set_up_flow_rules(
 
 	memset(header_buff, 0, flow_rules_size);
 
-	#ifdef HAVE_RAW_ETH_EXP
-	*flow_rules = (struct ibv_exp_flow_attr*)header_buff;
-	attr_info = (struct ibv_exp_flow_attr*)header_buff;
-	#else
 	*flow_rules = (struct ibv_flow_attr*)header_buff;
 	attr_info = (struct ibv_flow_attr*)header_buff;
-	#endif
 
 	attr_info->size = flow_rules_size;
 	attr_info->priority = 0;
@@ -859,19 +766,11 @@ int set_up_flow_rules(
 	attr_info->port = user_param->ib_port;
 	attr_info->flags = 0;
 
-	#ifdef HAVE_RAW_ETH_EXP
-	attr_info->type = IBV_EXP_FLOW_ATTR_NORMAL;
-	header_buff = header_buff + sizeof(struct ibv_exp_flow_attr);
-	spec_info = (struct ibv_exp_flow_spec*)header_buff;
-	spec_info->eth.type = IBV_EXP_FLOW_SPEC_ETH;
-	spec_info->eth.size = sizeof(struct ibv_exp_flow_spec_eth);
-	#else
 	attr_info->type = IBV_FLOW_ATTR_NORMAL;
 	header_buff = header_buff + sizeof(struct ibv_flow_attr);
 	spec_info = (struct ibv_flow_spec*)header_buff;
 	spec_info->eth.type = IBV_FLOW_SPEC_ETH;
 	spec_info->eth.size = sizeof(struct ibv_flow_spec_eth);
-	#endif
 
 	spec_info->eth.val.ether_type = 0;
 
@@ -879,28 +778,16 @@ int set_up_flow_rules(
 
 	memset(spec_info->eth.mask.dst_mac, 0xFF,sizeof(spec_info->eth.mask.src_mac));
 	if(user_param->is_server_ip || user_param->is_client_ip) {
-		#ifdef HAVE_RAW_ETH_EXP
-		header_buff = header_buff + sizeof(struct ibv_exp_flow_spec_eth);
-		spec_info = (struct ibv_exp_flow_spec*)header_buff;
-		fill_exp_ip_spec(spec_info, user_param);
-		#else
 		header_buff = header_buff + sizeof(struct ibv_flow_spec_eth);
 		spec_info = (struct ibv_flow_spec*)header_buff;
 		fill_ip_spec(spec_info, user_param);
-		#endif
 	}
 
 	if(user_param->is_server_port && user_param->is_client_port) {
 		header_buff = header_buff + get_ip_size(user_param->raw_ipv6);
-		#ifdef HAVE_RAW_ETH_EXP
-		spec_info = (struct ibv_exp_flow_spec*)header_buff;
-		spec_info->tcp_udp.type = (user_param->tcp) ? IBV_EXP_FLOW_SPEC_TCP : IBV_EXP_FLOW_SPEC_UDP;
-		spec_info->tcp_udp.size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
-		#else
 		spec_info = (struct ibv_flow_spec*)header_buff;
 		spec_info->tcp_udp.type = (user_param->tcp) ? IBV_FLOW_SPEC_TCP : IBV_FLOW_SPEC_UDP;
 		spec_info->tcp_udp.size = sizeof(struct ibv_flow_spec_tcp_udp);
-		#endif
 
 		if(user_param->machine == SERVER) {
 			spec_info->tcp_udp.val.dst_port = htons(local_port);
@@ -925,11 +812,7 @@ int set_up_flow_rules(
  *set_fs_rate_rules - init flow struct for FS rate test
  ******************************************************************************/
 int set_up_fs_rules(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		uint64_t allocated_flows) {
@@ -966,11 +849,7 @@ int set_up_fs_rules(
  *send_set_up_connection - init raw_ethernet_info and ibv_flow_spec to user args
  ******************************************************************************/
 int send_set_up_connection(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		struct raw_ethernet_info *my_dest_info,
@@ -1055,15 +934,11 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 	struct ibv_wc		*wc = NULL;
 	struct ibv_wc		*wc_tx = NULL;
 	struct ibv_recv_wr	*bad_wr_recv = NULL;
-	#if defined(HAVE_VERBS_EXP)
-	struct ibv_exp_send_wr	*bad_exp_wr = NULL;
-	#endif
 	struct ibv_send_wr	*bad_wr = NULL;
 	int			firstRx = 1;
 	int 			rwqe_sent = user_param->rx_depth;
 	int			return_value = 0;
 	int			wc_id;
-
 	ALLOCATE(wc, struct ibv_wc, CTX_POLL_BATCH);
 	ALLOCATE(wc_tx, struct ibv_wc, CTX_POLL_BATCH);
 	ALLOCATE(rcnt_for_qp,uint64_t,user_param->num_of_qps);
@@ -1095,20 +970,7 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 					((ctx->scnt[index] - ctx->ccnt[index] + user_param->post_list) <= user_param->tx_depth) && (rcnt_for_qp[index] - ctx->scnt[index] > 0)) {
 
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)) {
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1)
-							ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED;
-						else
-					#endif
-							ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-					#ifdef HAVE_ACCL_VERBS
-					}
-					#endif
+						ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 				}
 
 				if (user_param->noPeak == OFF)
@@ -1118,27 +980,8 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 					break;
 				switch_smac_dmac(ctx->wr[index*user_param->post_list].sg_list);
 
-				#ifdef HAVE_VERBS_EXP
-				#ifdef HAVE_ACCL_VERBS
-				if (user_param->verb_type == ACCL_INTF) {
-					struct ibv_sge *sg_l = ctx->exp_wr[index*user_param->post_list].sg_list;
-					err = ctx->qp_burst_family[index]->send_burst(ctx->qp[index], sg_l, 1, ctx->exp_wr[index].exp_send_flags);
-				} else {
-				#endif
-					if (user_param->use_exp == 1) {
-						err = (ctx->exp_post_send_func_pointer)(ctx->qp[index],
-							&ctx->exp_wr[index*user_param->post_list], &bad_exp_wr);
-					}
-					else {
-						err = (ctx->post_send_func_pointer)(ctx->qp[index],
-							&ctx->wr[index*user_param->post_list],&bad_wr);
-					}
-				#ifdef HAVE_ACCL_VERBS
-				}
-				#endif
-				#else
 				err = ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr);
-				#endif
+
 				if(err) {
 					fprintf(stderr, "Couldn't post send: qp %d scnt=%lu \n", index, ctx->scnt[index]);
 					return_value = FAILURE;
@@ -1146,13 +989,6 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				}
 
 				if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
-					#ifdef HAVE_VERBS_EXP
-					if (user_param->use_exp == 1)
-						increase_loc_addr(ctx->exp_wr[index].sg_list, user_param->size,
-								  ctx->scnt[index], ctx->my_addr[index], 0,
-								  ctx->cache_line_size, ctx->cycle_buffer);
-					else
-					#endif
 						increase_loc_addr(ctx->wr[index].sg_list, user_param->size,
 								  ctx->scnt[index], ctx->my_addr[index], 0,
 								  ctx->cache_line_size, ctx->cycle_buffer);
@@ -1163,20 +999,7 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->post_list == 1 &&
 					(ctx->scnt[index]%user_param->cq_mod == (user_param->cq_mod - 1) ||
 						(user_param->test_type == ITERATIONS && ctx->scnt[index] == (iters - 1)))) {
-					#ifdef HAVE_VERBS_EXP
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF)
-						ctx->exp_wr[index].exp_send_flags |= IBV_EXP_QP_BURST_SIGNALED;
-					else {
-					#endif
-						if (user_param->use_exp == 1)
-							ctx->exp_wr[index].exp_send_flags |= IBV_EXP_SEND_SIGNALED;
-						else
-					#endif
 							ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-					#ifdef HAVE_ACCL_VERBS
-					}
-					#endif
 				}
 			}
 		}
@@ -1192,11 +1015,6 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 
 		if ((user_param->test_type == ITERATIONS && (totrcnt < tot_iters)) ||
 			(user_param->test_type == DURATION && user_param->state != END_STATE)) {
-			#ifdef HAVE_ACCL_VERBS
-			if (user_param->verb_type == ACCL_INTF)
-				ne = ctx->recv_cq_family->poll_cnt(ctx->recv_cq, CTX_POLL_BATCH);
-			else
-			#endif
 				ne = ibv_poll_cq(ctx->recv_cq, CTX_POLL_BATCH, wc);
 
 			if (ne > 0) {
@@ -1210,13 +1028,10 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				}
 
 				for (i = 0; i < ne; i++) {
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-						0 : (int)wc[i].wr_id;
+					wc_id = (int)wc[i].wr_id;
 
-					if (user_param->verb_type != ACCL_INTF) {
-						if (wc[i].status != IBV_WC_SUCCESS) {
-							NOTIFY_COMP_ERROR_RECV(wc[i], totrcnt);
-						}
+					if (wc[i].status != IBV_WC_SUCCESS) {
+						NOTIFY_COMP_ERROR_RECV(wc[i], totrcnt);
 					}
 
 					rcnt_for_qp[wc_id]++;
@@ -1229,22 +1044,13 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 			}
 		}
 		if ((totccnt < tot_iters) || (user_param->test_type == DURATION && user_param->state != END_STATE)) {
-			#ifdef HAVE_ACCL_VERBS
-			if (user_param->verb_type == ACCL_INTF)
-				ne = ctx->send_cq_family->poll_cnt(ctx->send_cq, CTX_POLL_BATCH);
-			else
-			#endif
-				ne = ibv_poll_cq(ctx->send_cq, CTX_POLL_BATCH, wc_tx);
-
+			ne = ibv_poll_cq(ctx->send_cq, CTX_POLL_BATCH, wc_tx);
 			if (ne > 0) {
 				for (i = 0; i < ne; i++) {
-					wc_id = (user_param->verb_type == ACCL_INTF) ?
-						0 : (int)wc[i].wr_id;
+					wc_id = (int)wc[i].wr_id;
 
-					if (user_param->verb_type != ACCL_INTF) {
-						if (wc_tx[i].status != IBV_WC_SUCCESS)
-							NOTIFY_COMP_ERROR_SEND(wc_tx[i], totscnt, totccnt);
-					}
+					if (wc_tx[i].status != IBV_WC_SUCCESS)
+						NOTIFY_COMP_ERROR_SEND(wc_tx[i], totscnt, totccnt);
 
 					totccnt += user_param->cq_mod;
 					ctx->ccnt[wc_id] += user_param->cq_mod;
@@ -1268,23 +1074,11 @@ int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 			while (rwqe_sent - totccnt < user_param->rx_depth) {    /* Post more than buffer_size */
 				if (user_param->test_type==DURATION ||
 					rcnt_for_qp[0] + user_param->rx_depth <= user_param->iters) {
-					#ifdef HAVE_ACCL_VERBS
-					if (user_param->verb_type == ACCL_INTF) {
-						if (ctx->qp_burst_family[0]->recv_burst(ctx->qp[0], ctx->rwr[0].sg_list, 1)) {
-							fprintf(stderr, "Couldn't post recv burst (accelerated verbs).\n");
-							return_value = FAILURE;
-							goto cleaning;
-						}
-					} else {
-					#endif
 						if (ibv_post_recv(ctx->qp[0], &ctx->rwr[0], &bad_wr_recv)) {
 							fprintf(stderr, "Couldn't post recv Qp=%d rcnt=%lu\n", 0, rcnt_for_qp[0]);
 							return_value = 15;
 							goto cleaning;
 						}
-					#ifdef HAVE_ACCL_VERBS
-					}
-					#endif
 
 					if (SIZE(user_param->connection_type, user_param->size, !(int)user_param->machine) <= (ctx->cycle_buffer / 2)) {
 						increase_loc_addr(ctx->rwr[0].sg_list,
diff --git a/src/raw_ethernet_resources.h b/src/raw_ethernet_resources.h
index eeb7acc4..2f285079 100755
--- a/src/raw_ethernet_resources.h
+++ b/src/raw_ethernet_resources.h
@@ -153,11 +153,7 @@ struct TCP_header {
 }__attribute__((packed));
 
 void gen_eth_header(struct ETH_header* eth_header,uint8_t* src_mac,uint8_t* dst_mac, uint16_t eth_type);
-#ifdef HAVE_RAW_ETH_EXP
-void print_spec(struct ibv_exp_flow_attr* flow_rules,struct perftest_parameters* user_param);
-#else
 void print_spec(struct ibv_flow_attr* flow_rules,struct perftest_parameters* user_param);
-#endif
 //void print_ethernet_header(struct ETH_header* p_ethernet_header);
 void print_ethernet_header(void* p_ethernet_header);
 //void print_ethernet_vlan_header(struct ETH_vlan_header* p_ethernet_header);
@@ -232,11 +228,7 @@ int calc_flow_rules_size(struct perftest_parameters *user_param, int is_ip_heade
  */
 
 int send_set_up_connection(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		struct raw_ethernet_info* my_dest_info,
@@ -324,11 +316,7 @@ static __inline void switch_smac_dmac(struct ibv_sge *sg)
  */
 
 int set_up_flow_rules(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		int local_port,
@@ -345,11 +333,7 @@ int set_up_flow_rules(
  */
 
 int set_up_fs_rules(
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr **flow_rules,
-		#else
 		struct ibv_flow_attr **flow_rules,
-		#endif
 		struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		uint64_t allocated_flows);
diff --git a/src/raw_ethernet_send_burst_lat.c b/src/raw_ethernet_send_burst_lat.c
index 63f5e5ac..dc7dac9d 100755
--- a/src/raw_ethernet_send_burst_lat.c
+++ b/src/raw_ethernet_send_burst_lat.c
@@ -64,15 +64,9 @@ int main(int argc, char *argv[])
 	struct raw_ethernet_info	my_dest_info,rem_dest_info;
 	int				ret_parser;
 	struct perftest_parameters	user_param;
-	#ifdef HAVE_RAW_ETH_EXP
-	struct ibv_exp_flow		**flow_create_result;
-	struct ibv_exp_flow_attr	**flow_rules;
-	struct ibv_exp_flow 		*flow_promisc = NULL;
-	#else
 	struct ibv_flow			**flow_create_result;
 	struct ibv_flow_attr		**flow_rules;
 	struct ibv_flow 		*flow_promisc = NULL;
-	#endif
 	struct report_options		report;
 	int				i;
 
@@ -103,13 +97,8 @@ int main(int argc, char *argv[])
 		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
 		return FAILURE;
 	}
-	#ifdef HAVE_RAW_ETH_EXP
-	ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows);
-	ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows);
-	#else
 	ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows);
 	ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows);
-	#endif
 
 
 	/*this is a bidirectional test, so we need to let the init functions
@@ -176,11 +165,7 @@ int main(int argc, char *argv[])
 
 	/* attaching the qp to the spec */
 	for (i = 0; i < user_param.flows; i++) {
-		#ifdef HAVE_RAW_ETH_EXP
-		flow_create_result[i] = ibv_exp_create_flow(ctx.qp[0], flow_rules[i]);
-		#else
 		flow_create_result[i] = ibv_create_flow(ctx.qp[0], flow_rules[i]);
-		#endif
 
 		if (!flow_create_result[i]){
 			perror("error");
@@ -190,19 +175,6 @@ int main(int argc, char *argv[])
 	}
 
 	if (user_param.use_promiscuous) {
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr attr = {
-			.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
-			.num_of_specs = 0,
-			.port = user_param.ib_port,
-			.flags = 0
-		};
-
-		if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
-			perror("error");
-			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
-		}
-		#else
 		struct ibv_flow_attr attr = {
 			.type = IBV_FLOW_ATTR_ALL_DEFAULT,
 			.num_of_specs = 0,
@@ -214,7 +186,6 @@ int main(int argc, char *argv[])
 			perror("error");
 			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
 		}
-		#endif
 	}
 
 	/* build ONE Raw Ethernet packets on ctx buffer */
@@ -258,11 +229,7 @@ int main(int argc, char *argv[])
 
 	/* destroy promisc flow */
 	if (user_param.use_promiscuous) {
-		#ifdef HAVE_RAW_ETH_EXP
-		if (ibv_exp_destroy_flow(flow_promisc)) {
-		#else
 		if (ibv_destroy_flow(flow_promisc)) {
-		#endif
 			perror("error");
 			fprintf(stderr, "Couldn't destroy promisc flow\n");
 			return FAILURE;
@@ -271,11 +238,7 @@ int main(int argc, char *argv[])
 
 	/* destroy flow */
 	for (i = 0; i < user_param.flows; i++) {
-		#ifdef HAVE_RAW_ETH_EXP
-		if (ibv_exp_destroy_flow(flow_create_result[i])) {
-		#else
 		if (ibv_destroy_flow(flow_create_result[i])) {
-		#endif
 			perror("error");
 			fprintf(stderr, "Couldn't destroy flow\n");
 			return FAILURE;
diff --git a/src/raw_ethernet_send_bw.c b/src/raw_ethernet_send_bw.c
index 30aec0a9..bdadf7c9 100755
--- a/src/raw_ethernet_send_bw.c
+++ b/src/raw_ethernet_send_bw.c
@@ -66,21 +66,12 @@ int main(int argc, char *argv[])
 	int				ret_parser;
 	struct perftest_parameters	user_param;
 
-	#ifdef HAVE_RAW_ETH_EXP
-	struct ibv_exp_flow		**flow_create_result;
-	struct ibv_exp_flow_attr	**flow_rules;
-	struct ibv_exp_flow 		**flow_promisc = NULL ;
-	#ifdef HAVE_SNIFFER_EXP
-	struct ibv_exp_flow 		**flow_sniffer = NULL;
-	#endif
-	#else
 	struct ibv_flow			**flow_create_result;
 	struct ibv_flow_attr		**flow_rules;
 	struct ibv_flow 		**flow_promisc = NULL ;
 	#ifdef HAVE_SNIFFER
 	struct ibv_flow 		**flow_sniffer = NULL;
 	#endif
-	#endif
 	int 				flow_index, qp_index;
 	union ibv_gid mgid;
 
@@ -109,21 +100,12 @@ int main(int argc, char *argv[])
 	ALLOCATE(rem_dest_info, struct raw_ethernet_info, user_param.num_of_qps);
 	memset(rem_dest_info, 0, sizeof(struct raw_ethernet_info) * user_param.num_of_qps);
 
-	#ifdef HAVE_RAW_ETH_EXP
-        ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows * user_param.num_of_qps);
-        ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows * user_param.num_of_qps);
-	#ifdef HAVE_SNIFFER_EXP
-	ALLOCATE(flow_sniffer, struct ibv_exp_flow*, user_param.num_of_qps);
-	#endif
-	ALLOCATE(flow_promisc, struct ibv_exp_flow*, user_param.num_of_qps);
-	#else
-        ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows * user_param.num_of_qps);
-        ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows * user_param.num_of_qps);
+	ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows * user_param.num_of_qps);
+	ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows * user_param.num_of_qps);
 	#ifdef HAVE_SNIFFER
 	ALLOCATE(flow_sniffer, struct ibv_flow*, user_param.num_of_qps);
 	#endif
-        ALLOCATE(flow_promisc, struct ibv_flow*, user_param.num_of_qps);
-        #endif
+	ALLOCATE(flow_promisc, struct ibv_flow*, user_param.num_of_qps);
 
 	if (user_param.raw_mcast) {
 		/* Transform IPv4 to Multicast MAC */
@@ -224,13 +206,8 @@ int main(int argc, char *argv[])
 		/* attaching the qp to the spec */
 		for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
 			for (flow_index = 0; flow_index < user_param.flows; flow_index++) {
-				#ifdef HAVE_RAW_ETH_EXP
-				flow_create_result[flow_index + qp_index * user_param.flows] =
-					ibv_exp_create_flow(ctx.qp[qp_index], flow_rules[flow_index]);
-				#else
 				flow_create_result[flow_index + qp_index * user_param.flows] =
 					ibv_create_flow(ctx.qp[qp_index], flow_rules[(qp_index * user_param.flows) + flow_index]);
-				#endif
 
 				if (!flow_create_result[flow_index + qp_index * user_param.flows]){
 					perror("error");
@@ -241,21 +218,6 @@ int main(int argc, char *argv[])
 		}
 
 		if (user_param.use_promiscuous) {
-			#ifdef HAVE_RAW_ETH_EXP
-			struct ibv_exp_flow_attr attr = {
-				.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
-				.num_of_specs = 0,
-				.port = user_param.ib_port,
-				.flags = 0
-			};
-
-			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
-				if ((flow_promisc[qp_index] = ibv_exp_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
-					perror("error");
-					fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
-				}
-			}
-			#else
 			struct ibv_flow_attr attr = {
 				.type = IBV_FLOW_ATTR_ALL_DEFAULT,
 				.num_of_specs = 0,
@@ -269,32 +231,18 @@ int main(int argc, char *argv[])
 					fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
 				}
 			}
-			#endif
 		}
-		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+		#if defined HAVE_SNIFFER
 		if (user_param.use_sniffer) {
-			#ifdef HAVE_RAW_ETH_EXP
-			struct ibv_exp_flow_attr attr = {
-				.type = IBV_EXP_FLOW_ATTR_SNIFFER,
-				.num_of_specs = 0,
-				.port = user_param.ib_port,
-				.flags = 0
-			};
-			#else
 			struct ibv_flow_attr attr = {
 				.type = IBV_FLOW_ATTR_SNIFFER,
 				.num_of_specs = 0,
 				.port = user_param.ib_port,
 				.flags = 0
 			};
-			#endif
 
 			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
-				#ifdef HAVE_RAW_ETH_EXP
-				if ((flow_sniffer[qp_index] = ibv_exp_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
-				#else
 				if ((flow_sniffer[qp_index] = ibv_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
-				#endif
 					perror("error");
 					fprintf(stderr, "Couldn't attach SNIFFER rule QP\n");
 				}
@@ -413,11 +361,7 @@ int main(int argc, char *argv[])
 		/* destroy open flows */
 		for (flow_index = 0; flow_index < user_param.flows; flow_index++) {
 			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
-				#ifdef HAVE_RAW_ETH_EXP
-				if (ibv_exp_destroy_flow(flow_create_result[flow_index + qp_index * user_param.flows])) {
-				#else
 				if (ibv_destroy_flow(flow_create_result[flow_index + qp_index * user_param.flows])) {
-				#endif
 					perror("error");
 					fprintf(stderr, "Couldn't destroy flow\n");
 					return FAILURE;
@@ -428,11 +372,7 @@ int main(int argc, char *argv[])
 
 		if (user_param.use_promiscuous) {
 			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
-				#ifdef HAVE_RAW_ETH_EXP
-				if (ibv_exp_destroy_flow(flow_promisc[qp_index])) {
-				#else
 				if (ibv_destroy_flow(flow_promisc[qp_index])) {
-				#endif
 					perror("error");
 					fprintf(stderr, "Couldn't destroy flow\n");
 					return FAILURE;
@@ -441,14 +381,10 @@ int main(int argc, char *argv[])
 			free(flow_promisc);
 		}
 
-		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+		#if defined HAVE_SNIFFER
 		if (user_param.use_sniffer) {
 			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
-				#ifdef HAVE_RAW_ETH_EXP
-				if (ibv_exp_destroy_flow(flow_sniffer[qp_index])) {
-				#else
 				if (ibv_destroy_flow(flow_sniffer[qp_index])) {
-				#endif
 					perror("error");
 					fprintf(stderr, "Couldn't destroy sniffer flow\n");
 					return FAILURE;
diff --git a/src/raw_ethernet_send_lat.c b/src/raw_ethernet_send_lat.c
index f077b463..ab587a3e 100755
--- a/src/raw_ethernet_send_lat.c
+++ b/src/raw_ethernet_send_lat.c
@@ -64,15 +64,9 @@ int main(int argc, char *argv[])
 	struct raw_ethernet_info	my_dest_info,rem_dest_info;
 	int				ret_parser;
 	struct perftest_parameters	user_param;
-	#ifdef HAVE_RAW_ETH_EXP
-	struct ibv_exp_flow		**flow_create_result;
-	struct ibv_exp_flow_attr	**flow_rules;
-	struct ibv_exp_flow 		*flow_promisc = NULL;
-	#else
 	struct ibv_flow			**flow_create_result;
 	struct ibv_flow_attr		**flow_rules;
 	struct ibv_flow 		*flow_promisc = NULL;
-	#endif
 	struct report_options		report;
 	int				i;
 
@@ -103,14 +97,8 @@ int main(int argc, char *argv[])
 		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
 		return FAILURE;
 	}
-	#ifdef HAVE_RAW_ETH_EXP
-	ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows);
-	ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows);
-	#else
 	ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows);
 	ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows);
-	#endif
-
 
 	/*this is a bidirectional test, so we need to let the init functions
 	 * think we are in duplex mode
@@ -175,11 +163,7 @@ int main(int argc, char *argv[])
 
 	/* attaching the qp to the spec */
 	for (i = 0; i < user_param.flows; i++) {
-		#ifdef HAVE_RAW_ETH_EXP
-		flow_create_result[i] = ibv_exp_create_flow(ctx.qp[0], flow_rules[i]);
-		#else
 		flow_create_result[i] = ibv_create_flow(ctx.qp[0], flow_rules[i]);
-		#endif
 
 		if (!flow_create_result[i]){
 			perror("error");
@@ -189,19 +173,6 @@ int main(int argc, char *argv[])
 	}
 
 	if (user_param.use_promiscuous) {
-		#ifdef HAVE_RAW_ETH_EXP
-		struct ibv_exp_flow_attr attr = {
-			.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
-			.num_of_specs = 0,
-			.port = user_param.ib_port,
-			.flags = 0
-		};
-
-		if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
-			perror("error");
-			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
-		}
-		#else
 		struct ibv_flow_attr attr = {
 			.type = IBV_FLOW_ATTR_ALL_DEFAULT,
 			.num_of_specs = 0,
@@ -213,7 +184,6 @@ int main(int argc, char *argv[])
 			perror("error");
 			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
 		}
-		#endif
 	}
 
 	/* build ONE Raw Ethernet packets on ctx buffer */
@@ -253,11 +223,7 @@ int main(int argc, char *argv[])
 
 	/* destroy promisc flow */
 	if (user_param.use_promiscuous) {
-		#ifdef HAVE_RAW_ETH_EXP
-		if (ibv_exp_destroy_flow(flow_promisc)) {
-		#else
 		if (ibv_destroy_flow(flow_promisc)) {
-		#endif
 			perror("error");
 			fprintf(stderr, "Couldn't destroy promisc flow\n");
 			return FAILURE;
@@ -266,11 +232,7 @@ int main(int argc, char *argv[])
 
 	/* destroy flow */
 	for (i = 0; i < user_param.flows; i++) {
-		#ifdef HAVE_RAW_ETH_EXP
-		if (ibv_exp_destroy_flow(flow_create_result[i])) {
-		#else
 		if (ibv_destroy_flow(flow_create_result[i])) {
-		#endif
 			perror("error");
 			fprintf(stderr, "Couldn't destroy flow\n");
 			return FAILURE;

From c8104c242991c8ca87aae62be0f3bcf651a3b822 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 16 Sep 2019 11:06:24 +0300
Subject: [PATCH 02/41] Added DC support for rdma-core

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 Makefile.am               |   3 +-
 src/atomic_bw.c           |  31 +++-
 src/atomic_lat.c          |  33 +++-
 src/perftest_parameters.c |  14 +-
 src/perftest_resources.c  | 321 ++++++++++++++++++++++++++++++--------
 src/perftest_resources.h  |  17 +-
 src/read_bw.c             |  29 +++-
 src/read_lat.c            |  28 +++-
 src/send_bw.c             |  28 +++-
 src/send_lat.c            |  28 +++-
 src/write_bw.c            |  31 ++--
 src/write_lat.c           |  29 +++-
 12 files changed, 452 insertions(+), 140 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index b66bca2e..0090203d 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -55,7 +55,8 @@ LIBMLX4= -lmlx4
 LIBMLX5= -lmlx5
 else
 LIBMLX4=
-LIBMLX5=
+LIBMLX5= -lmlx5
+
 endif
 
 ib_send_bw_SOURCES = src/send_bw.c src/multicast_resources.c src/multicast_resources.h
diff --git a/src/atomic_bw.c b/src/atomic_bw.c
index 15f8ea29..b60425f0 100755
--- a/src/atomic_bw.c
+++ b/src/atomic_bw.c
@@ -72,7 +72,7 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	if (user_param.use_xrc && user_param.duplex) {
+	if ((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
 		user_param.num_of_qps *= 2;
 	}
 
@@ -155,11 +155,6 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	/* Print this machine QP information */
-	for (i = 0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i], &user_comm);
-
-	user_comm.rdma_params->side = REMOTE;
 	for (i = 0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -167,7 +162,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr, "Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-		ctx_print_pingpong_data(&rem_dest[i], &user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -185,6 +179,29 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++) {
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+	}
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
 		fprintf(stderr, "Failed to exchange data between server and clients\n");
diff --git a/src/atomic_lat.c b/src/atomic_lat.c
index f038b2d4..364e122e 100755
--- a/src/atomic_lat.c
+++ b/src/atomic_lat.c
@@ -80,8 +80,9 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	if(user_param.use_xrc)
+	if (user_param.use_xrc || user_param.connection_type == DC) {
 		user_param.num_of_qps *= 2;
+	}
 
 	/* Finding the IB device selected (or defalut if no selected). */
 	ib_dev = ctx_find_dev(&user_param.ib_devname);
@@ -171,16 +172,12 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
 	/* shaking hands and gather the other side info. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
 		return FAILURE;
 	}
 
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -188,8 +185,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr,"Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -207,6 +202,30 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++) {
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+	}
+
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index c23f24ce..7923e334 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -781,11 +781,10 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
 		exit(1);
 		#endif
 	} else if (strcmp(connStr[5], optarg)==0) {
-		// TODO: remove when DC for rdma-core is supported
-		#ifdef HAVE_DC
+		#ifdef HAVE_IBV_WR_API
 		*cptr = DC;
 		#else
-		fprintf(stderr," DC not detected in libibverbs\n");
+		fprintf(stderr," DC not detected in libibverbs, IBV_WR API is needed\n");
 		exit(1);
 		#endif
 	} else if (strcmp(connStr[6], optarg) == 0) {
@@ -1098,13 +1097,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		user_param->gid_index = 0;
 	}
 
-
-	if (user_param->verb == ATOMIC && user_param->connection_type == DC) {
-		printf(RESULT_LINE);
-		fprintf(stderr, " ATOMIC tests don't support DC transport\n");
-		exit(1);
-	}
-
 	if (user_param->work_rdma_cm) {
 
 		if (user_param->connection_type == UC) {
@@ -1192,7 +1184,7 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	}
 
 	if (user_param->connection_type == DC && !user_param->use_srq)
-		user_param->use_srq = 1;
+		user_param->use_srq = ON;
 
 	/* XRC Part */
 	if (user_param->connection_type == XRC) {
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 801ee9f6..bca17d99 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -33,6 +33,7 @@ static enum ibv_wr_opcode opcode_verbs_array[] = {IBV_WR_SEND,IBV_WR_RDMA_WRITE,
 static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP,IBV_WR_ATOMIC_FETCH_AND_ADD};
 
 #define CPU_UTILITY "/proc/stat"
+#define DC_KEY 0xffeeddcc
 
 struct perftest_parameters* duration_param;
 struct check_alive_data check_alive_data;
@@ -264,7 +265,6 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 	ibv_wr_start(ctx->qpx[index]);
 
 	ctx->qpx[index]->wr_id = ctx->wr[wr_index].wr_id;
-
 	switch (op) {
 	case IBV_WR_SEND:
 		ibv_wr_send(ctx->qpx[index]);
@@ -299,8 +299,14 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 	default:
 		fprintf(stderr, "Post send failed: unknown operation code.\n");;
 	}
-
-	if (qpt == IBV_QPT_UD) {
+	if (qpt == IBV_QPT_DRIVER) {
+		mlx5dv_wr_set_dc_addr(
+			ctx->dv_qp[index],
+			ctx->ah[index],
+			ctx->r_dctn[index],
+			DC_KEY);
+	}
+	else if (qpt == IBV_QPT_UD) {
 		ibv_wr_set_ud_addr(
 			ctx->qpx[index],
 			ctx->wr[wr_index].wr.ud.ah,
@@ -350,6 +356,48 @@ static inline int _new_post_send(struct pingpong_context *ctx,
  * Return Value : int.
  *
  */
+static int new_post_write_sge_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE);
+}
+
+static int new_post_write_inl_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE);
+}
+
+static int new_post_read_sge_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_READ);
+}
+
+static int new_post_send_sge_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_SEND);
+}
+
+static int new_post_send_inl_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_SEND);
+}
+
+static int new_post_atomic_fa_sge_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_FETCH_AND_ADD);
+}
+
+static int new_post_atomic_cs_sge_dc(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_CMP_AND_SWP);
+}
+
 static int new_post_send_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
@@ -727,6 +775,8 @@ void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_par
 	ALLOCATE(ctx->qp, struct ibv_qp*, user_param->num_of_qps);
 	#ifdef HAVE_IBV_WR_API
 	ALLOCATE(ctx->qpx, struct ibv_qp_ex*, user_param->num_of_qps);
+	ALLOCATE(ctx->dv_qp, struct mlx5dv_qp_ex*, user_param->num_of_qps);
+	ALLOCATE(ctx->r_dctn, uint32_t, user_param->num_of_qps);
 	#endif
 	ALLOCATE(ctx->mr, struct ibv_mr*, user_param->num_of_qps);
 	ALLOCATE(ctx->buf, void* , user_param->num_of_qps);
@@ -955,6 +1005,7 @@ int destroy_ctx(struct pingpong_context *ctx,
 	free(ctx->qp);
 	#ifdef HAVE_IBV_WR_API
 	free(ctx->qpx);
+	free(ctx->r_dctn);
 	#endif
 
 	if ((user_param->tst == BW || user_param->tst == LAT_BY_BW ) && (user_param->machine == CLIENT || user_param->duplex)) {
@@ -1374,8 +1425,29 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 	}
 	#endif
 
-	if (user_param->use_srq && !user_param->use_xrc && (user_param->tst == LAT ||
-				user_param->machine == SERVER || user_param->duplex == ON)) {
+	if (user_param->use_srq && user_param->connection_type == DC &&
+			(user_param->tst == LAT ||
+			user_param->machine == SERVER ||
+			user_param->duplex == ON))
+	{
+		struct ibv_srq_init_attr_ex attr;
+			memset(&attr, 0, sizeof(attr));
+		attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_PD;
+		attr.attr.max_wr = user_param->rx_depth;
+		attr.attr.max_sge = 1;
+		attr.pd = ctx->pd;
+
+		attr.srq_type = IBV_SRQT_BASIC;
+		ctx->srq = ibv_create_srq_ex(ctx->context, &attr);
+		if (!ctx->srq)  {
+			fprintf(stderr, "Couldn't create SRQ\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param->use_srq && user_param->connection_type != DC &&
+			!user_param->use_xrc && (user_param->tst == LAT ||
+			user_param->machine == SERVER || user_param->duplex == ON)) {
 
 		struct ibv_srq_init_attr attr = {
 			.attr = {
@@ -1384,7 +1456,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 				.max_sge = 1
 			}
 		};
-
 		ctx->srq = ibv_create_srq(ctx->pd, &attr);
 		if (!ctx->srq)  {
 			fprintf(stderr, "Couldn't create SRQ\n");
@@ -1418,9 +1489,8 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 int modify_qp_to_init(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param, int qp_index, int num_of_qps)
 {
-	uint64_t init_flag = 0;
-
-	if (ctx_modify_qp_to_init(ctx->qp[qp_index],user_param,init_flag)) {
+	if (ctx_modify_qp_to_init(ctx->qp[qp_index], user_param,
+		qp_index)) {
 		fprintf(stderr, "Failed to modify QP to INIT\n");
 		return FAILURE;
 	}
@@ -1440,7 +1510,7 @@ int create_reg_qp_main(struct pingpong_context *ctx,
 		ctx->qp[i] = ctx_xrc_qp_create(ctx, user_param, i);
 		#endif
 	} else {
-		ctx->qp[i] = ctx_qp_create(ctx, user_param);
+		ctx->qp[i] = ctx_qp_create(ctx, user_param, i);
 	}
 
 	if (ctx->qp[i] == NULL) {
@@ -1449,6 +1519,10 @@ int create_reg_qp_main(struct pingpong_context *ctx,
 	}
 	#ifdef HAVE_IBV_WR_API
 	ctx->qpx[i] = ibv_qp_to_qp_ex(ctx->qp[i]);
+	if (user_param->connection_type == DC)
+	{
+		ctx->dv_qp[i] = mlx5dv_qp_ex_from_ibv_qp_ex(ctx->qpx[i]);
+	}
 	#endif
 
 	return SUCCESS;
@@ -1463,14 +1537,18 @@ int create_qp_main(struct pingpong_context *ctx,
 }
 
 struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
-		struct perftest_parameters *user_param)
+		struct perftest_parameters *user_param, int qp_index)
 {
 	struct ibv_qp* qp = NULL;
+	int dc_num_of_qps = user_param->num_of_qps / 2;
 
+	int is_dc_server_side = 0;
 	#ifdef HAVE_IBV_WR_API
 	enum ibv_wr_opcode opcode;
 	struct ibv_qp_init_attr_ex attr;
+	struct mlx5dv_qp_init_attr attr_dv;
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr_ex));
+	memset(&attr_dv, 0, sizeof(attr_dv));
 	#else
 	struct ibv_qp_init_attr attr;
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
@@ -1478,18 +1556,34 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 
 	attr.send_cq = ctx->send_cq;
 	attr.recv_cq = (user_param->verb == SEND) ? ctx->recv_cq : ctx->send_cq;
-	attr.cap.max_send_wr  = user_param->tx_depth;
-	attr.cap.max_send_sge = MAX_SEND_SGE;
-	attr.cap.max_inline_data = user_param->inline_size;
 
-	if (user_param->use_srq && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex == ON)) {
+	is_dc_server_side = ((!(user_param->duplex || user_param->tst == LAT) &&
+						  (user_param->machine == SERVER)) ||
+						 ((user_param->duplex || user_param->tst == LAT) &&
+						  (qp_index >= dc_num_of_qps)));
+
+	if (!(user_param->connection_type == DC &&
+			is_dc_server_side)) {
+		attr.cap.max_send_wr  = user_param->tx_depth;
+		attr.cap.max_send_sge = MAX_SEND_SGE;
+		attr.cap.max_inline_data = user_param->inline_size;
+	}
+
+	if (user_param->use_srq &&
+			(user_param->tst == LAT ||
+			 user_param->machine == SERVER ||
+			 user_param->duplex == ON)) {
 		attr.srq = ctx->srq;
-		attr.cap.max_recv_wr  = 0;
-		attr.cap.max_recv_sge = 0;
+		if (user_param->connection_type != DC) {
+			attr.cap.max_recv_wr  = 0;
+			attr.cap.max_recv_sge = 0;
+		}
 	} else {
 		attr.srq = NULL;
-		attr.cap.max_recv_wr  = user_param->rx_depth;
-		attr.cap.max_recv_sge = MAX_RECV_SGE;
+		if (user_param->connection_type != DC) {
+			attr.cap.max_recv_wr  = user_param->rx_depth;
+			attr.cap.max_recv_sge = MAX_RECV_SGE;
+		}
 	}
 
 	switch (user_param->connection_type) {
@@ -1497,6 +1591,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		case RC : attr.qp_type = IBV_QPT_RC; break;
 		case UC : attr.qp_type = IBV_QPT_UC; break;
 		case UD : attr.qp_type = IBV_QPT_UD; break;
+		case DC : attr.qp_type = IBV_QPT_DRIVER; break;
 		#ifdef HAVE_RAW_ETH
 		case RawEth : attr.qp_type = IBV_QPT_RAW_PACKET; break;
 		#endif
@@ -1544,7 +1639,26 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		#endif
 	} else {
 		#ifdef HAVE_IBV_WR_API
-		qp = ibv_create_qp_ex(ctx->context, &attr);
+		if (user_param->connection_type == DC) {
+			attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
+
+			if (is_dc_server_side) {
+				attr.srq = ctx->srq;
+				attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT;
+				attr_dv.dc_init_attr.dct_access_key = DC_KEY;
+				attr.comp_mask &= ~IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+			}
+			else {
+				attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
+				attr_dv.create_flags |= MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE;
+				attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
+				attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+			}
+
+			qp = mlx5dv_create_qp(ctx->context, &attr, &attr_dv);
+		} else {
+			qp = ibv_create_qp_ex(ctx->context, &attr);
+		}
 		#else
 		qp = ibv_create_qp(ctx->pd, &attr);
 		#endif
@@ -1567,13 +1681,14 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 /******************************************************************************
  *
  ******************************************************************************/
-int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, uint64_t init_flag)
+int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, int qp_index)
 {
 	int num_of_qps = user_param->num_of_qps;
 	int num_of_qps_per_port = user_param->num_of_qps / 2;
 
 	struct ibv_qp_attr attr;
 	int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
+	int is_dc_server_side = 0;
 
 	static int portindex=0;  /* for dual-port support */
 	int ret = 0;
@@ -1582,11 +1697,16 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 	attr.qp_state        = IBV_QPS_INIT;
 	attr.pkey_index      = user_param->pkey_index;
 
-	if ( user_param->use_xrc && (user_param->duplex || user_param->tst == LAT)) {
+	if ((user_param->connection_type == DC || user_param->use_xrc) && (user_param->duplex || user_param->tst == LAT)) {
 		num_of_qps /= 2;
 		num_of_qps_per_port = num_of_qps / 2;
 	}
 
+	is_dc_server_side = ((!(user_param->duplex || user_param->tst == LAT) &&
+						 (user_param->machine == SERVER)) ||
+						 ((user_param->duplex || user_param->tst == LAT) &&
+						 (qp_index >= num_of_qps)));
+
 	if (user_param->dualport==ON) {
 		if (portindex % num_of_qps < num_of_qps_per_port) {
 			attr.port_num = user_param->ib_port;
@@ -1598,7 +1718,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 		portindex++;
 
 	} else {
-
 		attr.port_num = user_param->ib_port;
 	}
 
@@ -1608,8 +1727,8 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 	} else if (user_param->connection_type == UD || user_param->connection_type == SRD) {
 		attr.qkey = DEFF_QKEY;
 		flags |= IBV_QP_QKEY;
-
-	} else {
+	} else if (!(user_param->connection_type == DC &&
+			!is_dc_server_side)) {
 		switch (user_param->verb) {
 			case ATOMIC: attr.qp_access_flags = IBV_ACCESS_REMOTE_ATOMIC; break;
 			case READ  : attr.qp_access_flags = IBV_ACCESS_REMOTE_READ;  break;
@@ -1618,8 +1737,7 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
 		}
 		flags |= IBV_QP_ACCESS_FLAGS;
 	}
-
-	ret = ibv_modify_qp(qp,&attr,flags);
+	ret = ibv_modify_qp(qp, &attr, flags);
 
 	if (ret) {
 		fprintf(stderr, "Failed to modify QP to INIT, ret=%d\n",ret);
@@ -1636,11 +1754,11 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 		struct perftest_parameters *user_param,
 		struct pingpong_dest *dest,
 		struct pingpong_dest *my_dest,
-		int qpindex)
+		int qp_index)
 {
 	int num_of_qps = user_param->num_of_qps;
 	int num_of_qps_per_port = user_param->num_of_qps / 2;
-
+	int is_dc_server_side = 0;
 	int flags = IBV_QP_STATE;
 	int ooo_flags = 0;
 
@@ -1651,51 +1769,85 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 	 * there are send qps and recv qps. the actual number of send/recv qps
 	 * is num_of_qps / 2.
 	 */
-	if ( user_param->use_xrc && (user_param->duplex || user_param->tst == LAT)) {
+	if ((user_param->connection_type == DC || user_param->use_xrc) && (user_param->duplex || user_param->tst == LAT)) {
 		num_of_qps /= 2;
 		num_of_qps_per_port = num_of_qps / 2;
 	}
-
+	is_dc_server_side = ((!(user_param->duplex || user_param->tst == LAT) &&
+						 (user_param->machine == SERVER)) ||
+						  ((user_param->duplex || user_param->tst == LAT) &&
+						 (qp_index >= num_of_qps)));
 	/* first half of qps are for ib_port and second half are for ib_port2
 	 * in xrc with bidirectional, the first half of qps are xrc_send qps and
 	 * the second half are xrc_recv qps. the first half of the send/recv qps
 	 * are for ib_port1 and the second half are for ib_port2
 	 */
-	if (user_param->dualport == ON && (qpindex % num_of_qps >= num_of_qps_per_port))
+	if (user_param->dualport == ON && (qp_index % num_of_qps >= num_of_qps_per_port))
 		attr->ah_attr.port_num = user_param->ib_port2;
 	else
 		attr->ah_attr.port_num = user_param->ib_port;
 
 	if (user_param->connection_type != RawEth) {
-
-		attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
-		attr->ah_attr.sl = user_param->sl;
-
-		if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
+		if (user_param->connection_type == DC)
+		{
+			if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
 				|| ((attr->ah_attr.port_num == user_param->ib_port2) && (user_param->gid_index2 == DEF_GID_INDEX) && user_param->dualport)) {
 
-			attr->ah_attr.is_global = 0;
-		} else {
+				attr->ah_attr.is_global = 0;
+				attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
+				attr->ah_attr.sl = user_param->sl;
+			} else {
+				attr->ah_attr.is_global  = 1;
+				attr->ah_attr.grh.dgid = dest->gid;
+				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
+				attr->ah_attr.grh.hop_limit = 0xFF;
+				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
+				attr->ah_attr.sl = 0;
+			}
+			if (is_dc_server_side) {
+				attr->min_rnr_timer = 12;
+				attr->path_mtu = user_param->curr_mtu;
 
-			attr->ah_attr.is_global  = 1;
-			attr->ah_attr.grh.dgid = dest->gid;
-			attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
-			attr->ah_attr.grh.hop_limit = 0xFF;
-			attr->ah_attr.grh.traffic_class = user_param->traffic_class;
+				flags |= IBV_QP_AV |
+					IBV_QP_PATH_MTU |
+					IBV_QP_MIN_RNR_TIMER;
+			} //DCT
+			else {
+				attr->path_mtu = user_param->curr_mtu;
+				flags |= IBV_QP_PATH_MTU;
+			} //DCI
 		}
-		if (user_param->connection_type != UD && user_param->connection_type != SRD) {
+		else
+		{
+			attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
+			attr->ah_attr.sl = user_param->sl;
 
-			attr->path_mtu = user_param->curr_mtu;
-			attr->dest_qp_num = dest->qpn;
-			attr->rq_psn = dest->psn;
+			if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
+					|| ((attr->ah_attr.port_num == user_param->ib_port2) && (user_param->gid_index2 == DEF_GID_INDEX) && user_param->dualport)) {
 
-			flags |= (IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN);
+				attr->ah_attr.is_global = 0;
+			} else {
 
-			if (user_param->connection_type == RC || user_param->connection_type == XRC) {
+				attr->ah_attr.is_global  = 1;
+				attr->ah_attr.grh.dgid = dest->gid;
+				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
+				attr->ah_attr.grh.hop_limit = 0xFF;
+				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
+			}
+			if (user_param->connection_type != UD && user_param->connection_type != SRD) {
 
-				attr->max_dest_rd_atomic = my_dest->out_reads;
-				attr->min_rnr_timer = 12;
-				flags |= (IBV_QP_MIN_RNR_TIMER | IBV_QP_MAX_DEST_RD_ATOMIC);
+				attr->path_mtu = user_param->curr_mtu;
+				attr->dest_qp_num = dest->qpn;
+				attr->rq_psn = dest->psn;
+
+				flags |= (IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN);
+
+				if (user_param->connection_type == RC || user_param->connection_type == XRC) {
+
+					attr->max_dest_rd_atomic = my_dest->out_reads;
+					attr->min_rnr_timer = 12;
+					flags |= (IBV_QP_MIN_RNR_TIMER | IBV_QP_MAX_DEST_RD_ATOMIC);
+				}
 			}
 		}
 	}
@@ -1732,7 +1884,9 @@ static int ctx_modify_qp_to_rts(struct ibv_qp *qp,
 		flags |= IBV_QP_SQ_PSN;
 		attr->sq_psn = my_dest->psn;
 
-		if (user_param->connection_type == RC || user_param->connection_type == XRC) {
+		if (user_param->connection_type == DC ||
+			user_param->connection_type == RC ||
+			user_param->connection_type == XRC) {
 
 			attr->timeout   = user_param->qp_timeout;
 			attr->retry_cnt = 7;
@@ -1769,12 +1923,7 @@ int ctx_connect(struct pingpong_context *ctx,
 	}
 	for (i=0; i < user_param->num_of_qps; i++) {
 
-		if (user_param->connection_type == DC) {
-			if ( ((!(user_param->duplex || user_param->tst == LAT) && (user_param->machine == SERVER) )
-						|| ((user_param->duplex || user_param->tst == LAT) && (i >= user_param->num_of_qps/2)))) {
-				continue;
-			}
-		}
+
 		memset(&attr, 0, sizeof attr);
 
 		if (user_param->rate_limit_type == HW_RATE_LIMIT)
@@ -1797,7 +1946,12 @@ int ctx_connect(struct pingpong_context *ctx,
 			fprintf(stderr, "Failed to modify QP %d to RTR\n",ctx->qp[i]->qp_num);
 			return FAILURE;
 		}
-
+		if (user_param->connection_type == DC) {
+			if ( ((!(user_param->duplex || user_param->tst == LAT) && (user_param->machine == SERVER) )
+				|| ((user_param->duplex || user_param->tst == LAT) && (i >= user_param->num_of_qps/2)))) {
+				continue;
+			}
+		}
 		if (user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex) {
 			if(ctx_modify_qp_to_rts(ctx->qp[i], &attr, user_param, &dest[xrc_offset + i], &my_dest[i])) {
 				fprintf(stderr, "Failed to modify QP to RTS\n");
@@ -1883,6 +2037,39 @@ static void ctx_post_send_work_request_func_pointer(struct pingpong_context *ctx
 {
 	int use_inl = user_param->size <= user_param->inline_size;
 	switch (user_param->connection_type) {
+	case DC:
+		switch (user_param->verb) {
+			case SEND:
+				if (use_inl) {
+					ctx->new_post_send_work_request_func_pointer = &new_post_send_inl_dc;
+				}
+				else {
+					ctx->new_post_send_work_request_func_pointer = &new_post_send_sge_dc;
+				}
+				break;
+			case WRITE:
+				if (use_inl) {
+					ctx->new_post_send_work_request_func_pointer = &new_post_write_inl_dc;
+				}
+				else {
+					ctx->new_post_send_work_request_func_pointer = &new_post_write_sge_dc;
+				}
+				break;
+			case READ:
+				ctx->new_post_send_work_request_func_pointer = &new_post_read_sge_dc;
+				break;
+			case ATOMIC:
+				if (user_param->atomicType == FETCH_AND_ADD) {
+					ctx->new_post_send_work_request_func_pointer = &new_post_atomic_fa_sge_dc;
+				}
+				else {
+					ctx->new_post_send_work_request_func_pointer = &new_post_atomic_cs_sge_dc;
+				}
+				break;
+			default:
+				fprintf(stderr, "The post send properties are not supported on DC. \n");
+		}
+		break;
 	case RC:
 		switch (user_param->verb) {
 			case SEND:
@@ -2010,7 +2197,12 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 	#ifdef HAVE_IBV_WR_API
 	ctx_post_send_work_request_func_pointer(ctx, user_param);
 	#endif
+
 	for (i = 0; i < num_of_qps ; i++) {
+		if (user_param->connection_type == DC)
+		{
+			ctx->r_dctn[i] = rem_dest[xrc_offset + i].qpn;
+		}
 		memset(&ctx->wr[i*user_param->post_list],0,sizeof(struct ibv_send_wr));
 		ctx->sge_list[i*user_param->post_list].addr = (uintptr_t)ctx->buf[i];
 
@@ -2708,10 +2900,7 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
 			if (user_param->test_type == DURATION && user_param->state == END_STATE)
 				break;
 
-			if (user_param->connection_type == DC)
-				ne = ibv_poll_cq(ctx->send_cq,CTX_POLL_BATCH,wc);
-			else
-				ne = ibv_poll_cq(ctx->recv_cq,CTX_POLL_BATCH,wc);
+			ne = ibv_poll_cq(ctx->recv_cq,CTX_POLL_BATCH,wc);
 
 			if (ne > 0) {
 				if (firstRx) {
@@ -3521,7 +3710,6 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	int 		cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f);
 	int 		total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 	end_cycle, start_gap=0;
-
 	ctx->wr[0].sg_list->length = user_param->size;
 	#ifdef HAVE_IBV_WR_API
 	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
@@ -3664,7 +3852,6 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 					 */
 					if (user_param->test_type == DURATION || (rcnt + size_per_qp <= user_param->iters)) {
 						if (user_param->use_srq) {
-
 							if (ibv_post_srq_recv(ctx->srq, &ctx->rwr[wc.wr_id], &bad_wr_recv)) {
 								fprintf(stderr, "Couldn't post recv SRQ. QP = %d: counter=%lu\n",(int)wc.wr_id, rcnt);
 								return 1;
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
index e6233f0a..81f30cc2 100755
--- a/src/perftest_resources.h
+++ b/src/perftest_resources.h
@@ -52,6 +52,7 @@
 #define PERFTEST_RESOURCES_H
 
 #include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
 #include <rdma/rdma_cma.h>
 #include <stdint.h>
 #if defined(__FreeBSD__)
@@ -167,6 +168,7 @@ struct pingpong_context {
 	struct ibv_qp				**qp;
 	#ifdef HAVE_IBV_WR_API
 	struct ibv_qp_ex			**qpx;
+	struct mlx5dv_qp_ex			**dv_qp;
 	int (*new_post_send_work_request_func_pointer) (struct pingpong_context *ctx, int index,
 		struct perftest_parameters *user_param);
 	#endif
@@ -187,6 +189,7 @@ struct pingpong_context {
 	uint64_t				*scnt;
 	uint64_t				*ccnt;
 	int					is_contig_supported;
+	uint32_t				*r_dctn;
 	uint32_t                                *ctrl_buf;
 	uint32_t                                *credit_buf;
 	struct ibv_mr                           *credit_mr;
@@ -336,15 +339,14 @@ int ctx_init(struct pingpong_context *ctx,struct perftest_parameters *user_param
  *
  * Parameters :
  *
- *	pd      - The Protection domain , each the qp will be assigned to.
- *	send_cq - The CQ that will produce send CQE.
- *	recv_qp - The CQ that will produce recv CQE.
- *	param   - The parameters for the QP.
+ *	ctx - QP context.
+ *	user_param  - The perftest parameters.
+ *	qp_index   - The qp number to create.
  *
  * Return Value : Adress of the new QP.
  */
 struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
-							 struct perftest_parameters *user_param);
+			struct perftest_parameters *user_param, int qp_index);
 
 /* ctx_modify_qp_to_init.
  *
@@ -356,12 +358,13 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
  * Parameters :
  *
  *	qp     - The QP that will be moved to INIT.
- *	param  - The parameters for the QP.
+ *	user_param  - The perftest parameters.
+ *	qp_index   - The qp number to create.
  *
  * Return Value : SUCCESS, FAILURE.
  *
  */
-int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, uint64_t init_flag);
+int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, int qp_index);
 
 /* ctx_connect.
  *
diff --git a/src/read_bw.c b/src/read_bw.c
index 4bfd5b01..3386a203 100755
--- a/src/read_bw.c
+++ b/src/read_bw.c
@@ -161,12 +161,6 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	/* Print this machine QP information */
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
-	user_comm.rdma_params->side = REMOTE;
-
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -174,7 +168,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr,"Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -193,6 +186,28 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
diff --git a/src/read_lat.c b/src/read_lat.c
index acdddc98..202f088b 100755
--- a/src/read_lat.c
+++ b/src/read_lat.c
@@ -172,16 +172,12 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
 	/*  shaking hands and gather the other side info. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
 		return FAILURE;
 	}
 
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -189,8 +185,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr,"Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -209,6 +203,28 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
diff --git a/src/send_bw.c b/src/send_bw.c
index e33e4559..6a3ea406 100755
--- a/src/send_bw.c
+++ b/src/send_bw.c
@@ -285,10 +285,6 @@ int main(int argc, char *argv[])
 	if (ctx.send_rcredit)
 		ctx_alloc_credit(&ctx,&user_param,my_dest);
 
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -296,8 +292,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr,"Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -350,6 +344,28 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	if (user_param.use_event) {
 
 		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
diff --git a/src/send_lat.c b/src/send_lat.c
index 9f7834c8..c4f49813 100755
--- a/src/send_lat.c
+++ b/src/send_lat.c
@@ -291,10 +291,6 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -302,8 +298,6 @@ int main(int argc, char *argv[])
 			fprintf(stderr,"Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -354,6 +348,28 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	if (user_param.use_event) {
 
 		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
diff --git a/src/write_bw.c b/src/write_bw.c
index 0d2eafb7..b4305640 100755
--- a/src/write_bw.c
+++ b/src/write_bw.c
@@ -158,21 +158,12 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-
-	/* Print this machine QP information */
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
-
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
 			fprintf(stderr," Failed to exchange data between server and clients\n");
 			return FAILURE;
 		}
-
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -190,6 +181,28 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
 		fprintf(stderr," Failed to exchange data between server and clients\n");
diff --git a/src/write_lat.c b/src/write_lat.c
index 5757348e..c7561ad2 100755
--- a/src/write_lat.c
+++ b/src/write_lat.c
@@ -80,7 +80,7 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	if(user_param.use_xrc || user_param.connection_type == DC) {
+	if (user_param.use_xrc || user_param.connection_type == DC) {
 		user_param.num_of_qps *= 2;
 	}
 
@@ -166,16 +166,12 @@ int main(int argc, char *argv[])
 	/* Print basic test information. */
 	ctx_print_test_info(&user_param);
 
-	for (i=0; i < user_param.num_of_qps; i++)
-		ctx_print_pingpong_data(&my_dest[i],&user_comm);
-
 	/* shaking hands and gather the other side info. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
 		return FAILURE;
 	}
 
-	user_comm.rdma_params->side = REMOTE;
 	for (i=0; i < user_param.num_of_qps; i++) {
 
 		/* shaking hands and gather the other side info. */
@@ -184,7 +180,6 @@ int main(int argc, char *argv[])
 			return FAILURE;
 		}
 
-		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	};
 
 	if (user_param.work_rdma_cm == OFF) {
@@ -202,6 +197,28 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	/* Set up connection one more time to send qpn properly for DC */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");

From 2191a8081ca2f319abce81e8e2effe6676412b00 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Wed, 18 Dec 2019 14:58:47 +0200
Subject: [PATCH 03/41] Fixing latency bugs for ibv_wr_api

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac             |  2 +-
 src/perftest_resources.c | 25 ++++++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/configure.ac b/configure.ac
index 17aec4c9..9b4af4f6 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.60],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.70],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index bca17d99..9a6bc97c 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1562,11 +1562,11 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 						 ((user_param->duplex || user_param->tst == LAT) &&
 						  (qp_index >= dc_num_of_qps)));
 
+	attr.cap.max_inline_data = user_param->inline_size;
 	if (!(user_param->connection_type == DC &&
 			is_dc_server_side)) {
 		attr.cap.max_send_wr  = user_param->tx_depth;
 		attr.cap.max_send_sge = MAX_SEND_SGE;
-		attr.cap.max_inline_data = user_param->inline_size;
 	}
 
 	if (user_param->use_srq &&
@@ -3616,8 +3616,14 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 	#else
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#endif
-	if (user_param->size <= user_param->inline_size)
+	if (user_param->size <= user_param->inline_size) {
+		#ifdef HAVE_IBV_WR_API
+		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
+		#else
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
+		#endif
+	}
+
 
 	if((user_param->use_xrc || user_param->connection_type == DC))
 		poll_buf_offset = 1;
@@ -3804,11 +3810,15 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 		ctx->wr[0].sg_list->length = user_param->size;
 		ctx->wr[0].send_flags = 0;
 		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+		ctx->qpx[0]->wr_flags = 0;
 		#endif
 	}
 	if (user_param->size <= user_param->inline_size) {
+		#ifdef HAVE_IBV_WR_API
+		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
+		#else
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
+		#endif
 	}
 	while (scnt < user_param->iters || rcnt < user_param->iters ||
 			( (user_param->test_type == DURATION && user_param->state != END_STATE))) {
@@ -3896,7 +3906,11 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 
 			if (scnt % user_param->cq_mod == 0 || (user_param->test_type == ITERATIONS && scnt == user_param->iters)) {
 				poll = 1;
+				#ifdef HAVE_IBV_WR_API
+				ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+				#else
 				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
+				#endif
 			}
 
 			/* if we're in duration mode and the time is over, exit from this function */
@@ -3946,7 +3960,12 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				}
 				poll = 0;
 
+				#ifdef HAVE_IBV_WR_API
+				ctx->qpx[0]->wr_flags &= ~IBV_SEND_SIGNALED;
+				#else
 				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
+				#endif
+
 			}
 		}
 	}

From 705ed2e7b981f3b5830efd63f9f2b5c4ea643b80 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Wed, 18 Dec 2019 15:29:04 +0200
Subject: [PATCH 04/41] Made ibv_wr_api default during the build

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/configure.ac b/configure.ac
index 9b4af4f6..4b0456e4 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.70],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.71],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
@@ -45,13 +45,13 @@ AC_ARG_VAR(CUDA_H_PATH, help-string)
 AC_ARG_VAR(RANLIB, ranlib tool)
 
 AC_ARG_ENABLE([ibv_wr_api],
-[AS_HELP_STRING([--enable-ibv_wr_api],
-[Enable new post send API])],
-[enable_ibv_wr_api=yes], [enable_ibv_wr_api=no])
+[AS_HELP_STRING([--disable-ibv_wr_api],
+[Disable new post send API])],
+[disable_ibv_wr_api=yes], [disable_ibv_wr_api=no])
 
-AS_IF([test "x$enable_ibv_wr_api" = "xyes"],
-      [USE_IBV_WR_API=yes],
-        [USE_IBV_WR_API=no])
+AS_IF([test "x$disable_ibv_wr_api" = "xyes"],
+      [USE_IBV_WR_API=no],
+        [USE_IBV_WR_API=yes])
 
 AC_PREFIX_DEFAULT("/usr")
 

From 086a856e4ef9f387310b799a9175281af8d31bca Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 24 Dec 2019 16:46:22 +0200
Subject: [PATCH 05/41] Added RDMA CM support over rdma-core

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_communication.c | 1 +
 src/perftest_resources.c     | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index 264ddfa8..ece488be 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -1207,6 +1207,7 @@ int create_comm_struct(struct perftest_comm *comm,
 		ALLOCATE(comm->rdma_ctx->mr, struct ibv_mr*, user_param->num_of_qps);
 		ALLOCATE(comm->rdma_ctx->buf, void* , user_param->num_of_qps);
 		ALLOCATE(comm->rdma_ctx->qp,struct ibv_qp*,comm->rdma_params->num_of_qps);
+		ALLOCATE(comm->rdma_ctx->qpx,struct ibv_qp_ex*,comm->rdma_params->num_of_qps);
 		comm->rdma_ctx->buff_size = user_param->cycle_buffer;
 
 		if (create_rdma_resources(comm->rdma_ctx,comm->rdma_params)) {
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 9a6bc97c..5fa3519c 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1632,6 +1632,12 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		} else {
 			qp = ctx->cm_id->qp;
 		}
+		#else
+		if (rdma_create_qp_ex(ctx->cm_id, &attr)) {
+			fprintf(stderr, "Couldn't create rdma QP - %s\n", strerror(errno));
+		} else {
+			qp = ctx->cm_id->qp;
+		}
 		#endif
 	} else if (user_param->connection_type == SRD) {
 		#ifdef HAVE_SRD

From 670ccfcef1ad30cfe9547eaadb4edadf9a93b06f Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 23 Dec 2019 02:27:07 +0200
Subject: [PATCH 06/41] Add raw_ethernet_bw support with new post send API

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_resources.c | 48 +++++++++++++++-------------------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 5fa3519c..9b2a2adc 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2201,7 +2201,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 		xrc_offset = num_of_qps;
 	}
 	#ifdef HAVE_IBV_WR_API
-	ctx_post_send_work_request_func_pointer(ctx, user_param);
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
 	#endif
 
 	for (i = 0; i < num_of_qps ; i++) {
@@ -2583,11 +2584,12 @@ static inline int post_send_method(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
 	#ifdef HAVE_IBV_WR_API
-	return (*ctx->new_post_send_work_request_func_pointer)(ctx, index, user_param);
-	#else
+	if (user_param->connection_type != RawEth)
+		return (*ctx->new_post_send_work_request_func_pointer)(ctx, index, user_param);
+	#endif
 	struct ibv_send_wr 	*bad_wr = NULL;
 	return ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr);
-	#endif
+
 }
 
 /******************************************************************************
@@ -2703,11 +2705,9 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
 
-
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
 					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#else
-					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
@@ -2756,10 +2756,9 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 				if (user_param->post_list == 1 &&
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
+						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 						#ifdef HAVE_IBV_WR_API
 						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-						#else
-						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 						#endif
 				}
 
@@ -3099,10 +3098,9 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 				}
 
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)) {
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
 					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#else
-					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
@@ -3120,10 +3118,9 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 				if (user_param->post_list == 1 &&
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
+					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
 					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-					#else
-					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#endif
 				}
 			}
@@ -3355,10 +3352,9 @@ int run_iter_bi(struct pingpong_context *ctx,
 				}
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
+					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
 					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#else
-					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 				if (user_param->noPeak == OFF)
@@ -3387,10 +3383,9 @@ int run_iter_bi(struct pingpong_context *ctx,
 					(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 						(user_param->test_type == ITERATIONS && ctx->scnt[index] == iters-1))) {
 
+					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
 					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-					#else
-					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#endif
 				}
 			}
@@ -3617,16 +3612,14 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 	cycles_t 		end_cycle, start_gap=0;
 
 	ctx->wr[0].sg_list->length = user_param->size;
+	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#ifdef HAVE_IBV_WR_API
 	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-	#else
-	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#endif
 	if (user_param->size <= user_param->inline_size) {
+		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#ifdef HAVE_IBV_WR_API
 		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
-		#else
-		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#endif
 	}
 
@@ -3723,10 +3716,9 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	int 		total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 	end_cycle, start_gap=0;
 	ctx->wr[0].sg_list->length = user_param->size;
+	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#ifdef HAVE_IBV_WR_API
 	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-	#else
-	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#endif
 
 	/* Duration support in latency tests. */
@@ -3820,10 +3812,9 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 		#endif
 	}
 	if (user_param->size <= user_param->inline_size) {
+		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#ifdef HAVE_IBV_WR_API
 		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
-		#else
-		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#endif
 	}
 	while (scnt < user_param->iters || rcnt < user_param->iters ||
@@ -3912,10 +3903,9 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 
 			if (scnt % user_param->cq_mod == 0 || (user_param->test_type == ITERATIONS && scnt == user_param->iters)) {
 				poll = 1;
+				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
 				#ifdef HAVE_IBV_WR_API
 				ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-				#else
-				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
 				#endif
 			}
 
@@ -3965,11 +3955,9 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 						return 1;
 				}
 				poll = 0;
-
+				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
 				#ifdef HAVE_IBV_WR_API
 				ctx->qpx[0]->wr_flags &= ~IBV_SEND_SIGNALED;
-				#else
-				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
 				#endif
 
 			}

From 04f3fe27a451d5afad9e80b9ee54e18aa7567d6c Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 13 Jan 2020 01:20:41 +0200
Subject: [PATCH 07/41] Removed extra setting up connection for non-DC
 connections

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/send_bw.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/send_bw.c b/src/send_bw.c
index 6a3ea406..61caf6bb 100755
--- a/src/send_bw.c
+++ b/src/send_bw.c
@@ -344,10 +344,13 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */

From 4895c1650bc207282b306d732ae53a9507d3922c Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 13 Jan 2020 15:59:56 +0200
Subject: [PATCH 08/41] Version increment to 5.72

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 4b0456e4..d03db0d6 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.71],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.72],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])

From 5300ee4d0bd464363c84eb71065ee56e9f07786b Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Thu, 23 Jan 2020 13:10:15 +0200
Subject: [PATCH 09/41] Added falling back to old post send API for ConnectX-3

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              |   2 +-
 src/atomic_bw.c           |   8 ++
 src/perftest_parameters.c | 243 +++++++++++++++++++++-----------------
 src/perftest_parameters.h |   1 +
 src/perftest_resources.c  | 170 ++++++++++++++++----------
 src/write_bw.c            |   8 ++
 src/write_lat.c           |   8 ++
 7 files changed, 271 insertions(+), 169 deletions(-)

diff --git a/configure.ac b/configure.ac
index d03db0d6..d155d301 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.72],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.74],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/atomic_bw.c b/src/atomic_bw.c
index b60425f0..e48fe27e 100755
--- a/src/atomic_bw.c
+++ b/src/atomic_bw.c
@@ -87,6 +87,14 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param))
+	{
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
 	/* See if MTU and link type are valid and supported. */
 	if (check_link(ctx.context, &user_param)) {
 		fprintf(stderr, " Couldn't get context for the device\n");
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 7923e334..46b3d098 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -430,6 +430,9 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 	printf(" Set verbosity output level: bandwidth , message_rate, latency \n");
 	printf(" Latency measurement is Average calculation \n");
 
+	printf("      --use_old_post_send");
+	printf(" Use old post send flow (ibv_post_send).\n");
+
 	if (tst != FS_RATE) {
 		printf("      --perform_warm_up");
 		printf(" Perform some iterations before start measuring in order to warming-up memory cache, valid in Atomic, Read and Write BW tests\n");
@@ -700,6 +703,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->report_per_port		= 0;
 	user_param->use_odp			= 0;
 	user_param->use_hugepages		= 0;
+	user_param->use_old_post_send		= 0;
 	user_param->use_promiscuous		= 0;
 	user_param->use_sniffer			= 0;
 	user_param->check_alive_exited		= 0;
@@ -1016,8 +1020,9 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	}
 
 	if (user_param->connection_type == RawEth) {
-
-		if (user_param->test_method == RUN_ALL) {
+		user_param->use_old_post_send = 1;
+		if (user_param->test_method == RUN_ALL)
+		{
 			fprintf(stderr, "Raw Ethernet tests do not support -a / --all flag.\n");
 			exit(1);
 		}
@@ -1197,10 +1202,27 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		user_param->use_srq = ON;
 	}
 
-	if(user_param->connection_type == DC) {
-		if (user_param->work_rdma_cm == ON) {
+	if (!user_param->use_old_post_send)
+	{
+		#ifndef HAVE_IBV_WR_API
+		printf(RESULT_LINE);
+		fprintf(stderr, " new post send flow is not supported\n");
+		exit(1);
+		#endif
+	}
+
+	if (user_param->connection_type == DC)
+	{
+		if (user_param->use_old_post_send)
+		{
+			printf(RESULT_LINE);
+			fprintf(stderr, " DC does not support old post send flow\n");
+			exit(1);
+		}
+		if (user_param->work_rdma_cm == ON)
+		{
 			printf(RESULT_LINE);
-			fprintf(stderr," DC does not support RDMA_CM\n");
+			fprintf(stderr, " DC does not support RDMA_CM\n");
 			exit(1);
 		}
 	}
@@ -1791,6 +1813,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int report_per_port_flag = 0;
 	static int odp_flag = 0;
 	static int hugepages_flag = 0;
+	static int old_post_send_flag = 0;
 	static int use_promiscuous_flag = 0;
 	static int use_sniffer_flag = 0;
 	static int raw_mcast_flag = 0;
@@ -1826,115 +1849,116 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 
 	while (1) {
 		static const struct option long_options[] = {
-			{ .name = "port",		.has_arg = 1, .val = 'p' },
-			{ .name = "ib-dev",		.has_arg = 1, .val = 'd' },
-			{ .name = "ib-port",		.has_arg = 1, .val = 'i' },
-			{ .name = "mtu",		.has_arg = 1, .val = 'm' },
-			{ .name = "size",		.has_arg = 1, .val = 's' },
-			{ .name = "iters",		.has_arg = 1, .val = 'n' },
-			{ .name = "tx-depth",		.has_arg = 1, .val = 't' },
-			{ .name = "qp-timeout",		.has_arg = 1, .val = 'u' },
-			{ .name = "sl",			.has_arg = 1, .val = 'S' },
-			{ .name = "gid-index",		.has_arg = 1, .val = 'x' },
-			{ .name = "all",		.has_arg = 0, .val = 'a' },
-			{ .name = "CPU-freq",		.has_arg = 0, .val = 'F' },
-			{ .name = "connection",		.has_arg = 1, .val = 'c' },
-			{ .name = "qp",			.has_arg = 1, .val = 'q' },
-			{ .name = "events",		.has_arg = 0, .val = 'e' },
-			{ .name = "vector",		.has_arg = 1, .val = 'X' },
-			{ .name = "inline_size",	.has_arg = 1, .val = 'I' },
-			{ .name = "outs",		.has_arg = 1, .val = 'o' },
-			{ .name = "mcg",		.has_arg = 0, .val = 'g' },
-			{ .name = "comm_rdma_cm",	.has_arg = 0, .val = 'z' },
-			{ .name = "rdma_cm",		.has_arg = 0, .val = 'R' },
-			{ .name = "tos",		.has_arg = 1, .val = 'T' },
-			{ .name = "hop_limit",		.has_arg = 1, .val = 'L' },
-			{ .name = "help",		.has_arg = 0, .val = 'h' },
-			{ .name = "MGID",		.has_arg = 1, .val = 'M' },
-			{ .name = "rx-depth",		.has_arg = 1, .val = 'r' },
-			{ .name = "bidirectional",	.has_arg = 0, .val = 'b' },
-			{ .name = "cq-mod",		.has_arg = 1, .val = 'Q' },
-			{ .name = "noPeak",		.has_arg = 0, .val = 'N' },
-			{ .name = "version",		.has_arg = 0, .val = 'V' },
-			{ .name = "report-cycles",	.has_arg = 0, .val = 'C' },
-			{ .name = "report-histogrm",	.has_arg = 0, .val = 'H' },
-			{ .name = "report-unsorted",	.has_arg = 0, .val = 'U' },
-			{ .name = "atomic_type",	.has_arg = 1, .val = 'A' },
-			{ .name = "dualport",		.has_arg = 0, .val = 'O' },
-			{ .name = "post_list",		.has_arg = 1, .val = 'l' },
-			{ .name = "duration",		.has_arg = 1, .val = 'D' },
-			{ .name = "margin",		.has_arg = 1, .val = 'f' },
-			{ .name = "source_mac",		.has_arg = 1, .val = 'B' },
-			{ .name = "dest_mac",		.has_arg = 1, .val = 'E' },
-			{ .name = "dest_ip",		.has_arg = 1, .val = 'J' },
-			{ .name = "source_ip",		.has_arg = 1, .val = 'j' },
-			{ .name = "dest_port",		.has_arg = 1, .val = 'K' },
-			{ .name = "source_port",	.has_arg = 1, .val = 'k' },
-			{ .name = "ethertype",		.has_arg = 1, .val = 'Y' },
-			{ .name = "limit_bw",		.has_arg = 1, .val = 'w' },
-			{ .name = "limit_msgrate",	.has_arg = 1, .val = 'y' },
-			{ .name = "server",		.has_arg = 0, .val = 'Z' },
-			{ .name = "client",		.has_arg = 0, .val = 'P' },
-			{ .name = "mac_fwd",		.has_arg = 0, .val = 'v' },
-			{ .name = "use_rss",		.has_arg = 0, .val = 'G' },
-			{ .name = "report-counters",	.has_arg = 1, .val = 'W' },
-			{ .name = "force-link",		.has_arg = 1, .flag = &force_link_flag, .val = 1},
-			{ .name = "remote_mac",		.has_arg = 1, .flag = &remote_mac_flag, .val = 1 },
-			{ .name = "local_mac",		.has_arg = 1, .flag = &local_mac_flag, .val = 1 },
-			{ .name = "remote_ip",		.has_arg = 1, .flag = &remote_ip_flag, .val = 1 },
-			{ .name = "local_ip",		.has_arg = 1, .flag = &local_ip_flag, .val = 1 },
-			{ .name = "remote_port",	.has_arg = 1, .flag = &remote_port_flag, .val = 1 },
-			{ .name = "local_port",		.has_arg = 1, .flag = &local_port_flag, .val = 1 },
-			{ .name = "run_infinitely",	.has_arg = 0, .flag = &run_inf_flag, .val = 1 },
-			{ .name = "report_gbits",	.has_arg = 0, .flag = &report_fmt_flag, .val = 1},
-			{ .name = "use-srq",		.has_arg = 0, .flag = &srq_flag, .val = 1},
-			{ .name = "report-both",	.has_arg = 0, .flag = &report_both_flag, .val = 1},
-			{ .name = "reversed",		.has_arg = 0, .flag = &is_reversed_flag, .val = 1},
-			{ .name = "pkey_index",		.has_arg = 1, .flag = &pkey_flag, .val = 1},
-			{ .name = "inline_recv",	.has_arg = 1, .flag = &inline_recv_flag, .val = 1},
-			{ .name = "tcp",		.has_arg = 0, .flag = &tcp_flag, .val = 1},
-			{ .name = "burst_size",		.has_arg = 1, .flag = &burst_size_flag, .val = 1},
-			{ .name = "typical_pkt_size",	.has_arg = 1, .flag = &typical_pkt_size_flag, .val = 1},
-			{ .name = "rate_limit",		.has_arg = 1, .flag = &rate_limit_flag, .val = 1},
-			{ .name = "rate_limit_type",	.has_arg = 1, .flag = &rate_limit_type_flag, .val = 1},
-			{ .name = "rate_units",		.has_arg = 1, .flag = &rate_units_flag, .val = 1},
-			{ .name = "output",		.has_arg = 1, .flag = &verbosity_output_flag, .val = 1},
-			{ .name = "cpu_util",		.has_arg = 0, .flag = &cpu_util_flag, .val = 1},
-			{ .name = "latency_gap",	.has_arg = 1, .flag = &latency_gap_flag, .val = 1},
-			{ .name = "flow_label",		.has_arg = 1, .flag = &flow_label_flag, .val = 1},
-			{ .name = "retry_count",	.has_arg = 1, .flag = &retry_count_flag, .val = 1},
-			{ .name = "dont_xchg_versions",	.has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
-			{ .name = "use_cuda",		.has_arg = 0, .flag = &use_cuda_flag, .val = 1},
-			{ .name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
-			{ .name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
-			{ .name = "ipv6",		.has_arg = 0, .flag = &ipv6_flag, .val = 1},
+			{.name = "port",		.has_arg = 1, .val = 'p' },
+			{.name = "ib-dev",		.has_arg = 1, .val = 'd' },
+			{.name = "ib-port",		.has_arg = 1, .val = 'i' },
+			{.name = "mtu",		.has_arg = 1, .val = 'm' },
+			{.name = "size",		.has_arg = 1, .val = 's' },
+			{.name = "iters",		.has_arg = 1, .val = 'n' },
+			{.name = "tx-depth",		.has_arg = 1, .val = 't' },
+			{.name = "qp-timeout",		.has_arg = 1, .val = 'u' },
+			{.name = "sl",			.has_arg = 1, .val = 'S' },
+			{.name = "gid-index",		.has_arg = 1, .val = 'x' },
+			{.name = "all",		.has_arg = 0, .val = 'a' },
+			{.name = "CPU-freq",		.has_arg = 0, .val = 'F' },
+			{.name = "connection",		.has_arg = 1, .val = 'c' },
+			{.name = "qp",			.has_arg = 1, .val = 'q' },
+			{.name = "events",		.has_arg = 0, .val = 'e' },
+			{.name = "vector",		.has_arg = 1, .val = 'X' },
+			{.name = "inline_size",	.has_arg = 1, .val = 'I' },
+			{.name = "outs",		.has_arg = 1, .val = 'o' },
+			{.name = "mcg",		.has_arg = 0, .val = 'g' },
+			{.name = "comm_rdma_cm",	.has_arg = 0, .val = 'z' },
+			{.name = "rdma_cm",		.has_arg = 0, .val = 'R' },
+			{.name = "tos",		.has_arg = 1, .val = 'T' },
+			{.name = "hop_limit",		.has_arg = 1, .val = 'L' },
+			{.name = "help",		.has_arg = 0, .val = 'h' },
+			{.name = "MGID",		.has_arg = 1, .val = 'M' },
+			{.name = "rx-depth",		.has_arg = 1, .val = 'r' },
+			{.name = "bidirectional",	.has_arg = 0, .val = 'b' },
+			{.name = "cq-mod",		.has_arg = 1, .val = 'Q' },
+			{.name = "noPeak",		.has_arg = 0, .val = 'N' },
+			{.name = "version",		.has_arg = 0, .val = 'V' },
+			{.name = "report-cycles",	.has_arg = 0, .val = 'C' },
+			{.name = "report-histogrm",	.has_arg = 0, .val = 'H' },
+			{.name = "report-unsorted",	.has_arg = 0, .val = 'U' },
+			{.name = "atomic_type",	.has_arg = 1, .val = 'A' },
+			{.name = "dualport",		.has_arg = 0, .val = 'O' },
+			{.name = "post_list",		.has_arg = 1, .val = 'l' },
+			{.name = "duration",		.has_arg = 1, .val = 'D' },
+			{.name = "margin",		.has_arg = 1, .val = 'f' },
+			{.name = "source_mac",		.has_arg = 1, .val = 'B' },
+			{.name = "dest_mac",		.has_arg = 1, .val = 'E' },
+			{.name = "dest_ip",		.has_arg = 1, .val = 'J' },
+			{.name = "source_ip",		.has_arg = 1, .val = 'j' },
+			{.name = "dest_port",		.has_arg = 1, .val = 'K' },
+			{.name = "source_port",	.has_arg = 1, .val = 'k' },
+			{.name = "ethertype",		.has_arg = 1, .val = 'Y' },
+			{.name = "limit_bw",		.has_arg = 1, .val = 'w' },
+			{.name = "limit_msgrate",	.has_arg = 1, .val = 'y' },
+			{.name = "server",		.has_arg = 0, .val = 'Z' },
+			{.name = "client",		.has_arg = 0, .val = 'P' },
+			{.name = "mac_fwd",		.has_arg = 0, .val = 'v' },
+			{.name = "use_rss",		.has_arg = 0, .val = 'G' },
+			{.name = "report-counters",	.has_arg = 1, .val = 'W' },
+			{.name = "force-link",		.has_arg = 1, .flag = &force_link_flag, .val = 1},
+			{.name = "remote_mac",		.has_arg = 1, .flag = &remote_mac_flag, .val = 1 },
+			{.name = "local_mac",		.has_arg = 1, .flag = &local_mac_flag, .val = 1 },
+			{.name = "remote_ip",		.has_arg = 1, .flag = &remote_ip_flag, .val = 1 },
+			{.name = "local_ip",		.has_arg = 1, .flag = &local_ip_flag, .val = 1 },
+			{.name = "remote_port",	.has_arg = 1, .flag = &remote_port_flag, .val = 1 },
+			{.name = "local_port",		.has_arg = 1, .flag = &local_port_flag, .val = 1 },
+			{.name = "run_infinitely",	.has_arg = 0, .flag = &run_inf_flag, .val = 1 },
+			{.name = "report_gbits",	.has_arg = 0, .flag = &report_fmt_flag, .val = 1},
+			{.name = "use-srq",		.has_arg = 0, .flag = &srq_flag, .val = 1},
+			{.name = "report-both",	.has_arg = 0, .flag = &report_both_flag, .val = 1},
+			{.name = "reversed",		.has_arg = 0, .flag = &is_reversed_flag, .val = 1},
+			{.name = "pkey_index",		.has_arg = 1, .flag = &pkey_flag, .val = 1},
+			{.name = "inline_recv",	.has_arg = 1, .flag = &inline_recv_flag, .val = 1},
+			{.name = "tcp",		.has_arg = 0, .flag = &tcp_flag, .val = 1},
+			{.name = "burst_size",		.has_arg = 1, .flag = &burst_size_flag, .val = 1},
+			{.name = "typical_pkt_size",	.has_arg = 1, .flag = &typical_pkt_size_flag, .val = 1},
+			{.name = "rate_limit",		.has_arg = 1, .flag = &rate_limit_flag, .val = 1},
+			{.name = "rate_limit_type",	.has_arg = 1, .flag = &rate_limit_type_flag, .val = 1},
+			{.name = "rate_units",		.has_arg = 1, .flag = &rate_units_flag, .val = 1},
+			{.name = "output",		.has_arg = 1, .flag = &verbosity_output_flag, .val = 1},
+			{.name = "cpu_util",		.has_arg = 0, .flag = &cpu_util_flag, .val = 1},
+			{.name = "latency_gap",	.has_arg = 1, .flag = &latency_gap_flag, .val = 1},
+			{.name = "flow_label",		.has_arg = 1, .flag = &flow_label_flag, .val = 1},
+			{.name = "retry_count",	.has_arg = 1, .flag = &retry_count_flag, .val = 1},
+			{.name = "dont_xchg_versions",	.has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
+			{.name = "use_cuda",		.has_arg = 0, .flag = &use_cuda_flag, .val = 1},
+			{.name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
+			{.name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
+			{.name = "ipv6",		.has_arg = 0, .flag = &ipv6_flag, .val = 1},
 			#ifdef HAVE_IPV6
-			{ .name = "raw_ipv6",		.has_arg = 0, .flag = &raw_ipv6_flag, .val = 1},
+			{.name = "raw_ipv6",		.has_arg = 0, .flag = &raw_ipv6_flag, .val = 1},
 			#endif
-			{ .name = "report-per-port",	.has_arg = 0, .flag = &report_per_port_flag, .val = 1},
-			{ .name = "odp",		.has_arg = 0, .flag = &odp_flag, .val = 1},
-			{ .name = "use_hugepages",		.has_arg = 0, .flag = &hugepages_flag, .val = 1},
-			{ .name = "promiscuous",	.has_arg = 0, .flag = &use_promiscuous_flag, .val = 1},
+			{.name = "report-per-port", .has_arg = 0, .flag = &report_per_port_flag, .val = 1},
+			{.name = "odp", .has_arg = 0, .flag = &odp_flag, .val = 1},
+			{.name = "use_hugepages", .has_arg = 0, .flag = &hugepages_flag, .val = 1},
+			{.name = "use_old_post_send", .has_arg = 0, .flag = &old_post_send_flag, .val = 1},
+			{.name = "promiscuous", .has_arg = 0, .flag = &use_promiscuous_flag, .val = 1},
 			#if defined HAVE_SNIFFER
-			{ .name = "sniffer",		.has_arg = 0, .flag = &use_sniffer_flag, .val = 1},
+			{.name = "sniffer", .has_arg = 0, .flag = &use_sniffer_flag, .val = 1},
 			#endif
-			{ .name = "raw_mcast",		.has_arg = 0, .flag = &raw_mcast_flag, .val = 1},
-
-			{ .name = "mr_per_qp",		.has_arg = 0, .flag = &mr_per_qp_flag, .val = 1},
-			{ .name = "dlid",		.has_arg = 1, .flag = &dlid_flag, .val = 1},
-			{ .name = "tclass",		.has_arg = 1, .flag = &tclass_flag, .val = 1},
-			{ .name = "wait_destroy",	.has_arg = 1, .flag = &wait_destroy_flag, .val = 1},
-			{ .name = "flows",		.has_arg = 1, .flag = &flows_flag, .val = 1},
-			{ .name = "flows_burst",	.has_arg = 1, .flag = &flows_burst_flag, .val = 1},
-			{ .name = "reply_every",	.has_arg = 1, .flag = &reply_every_flag, .val = 1},
-			{ .name = "perform_warm_up",	.has_arg = 0, .flag = &perform_warm_up_flag, .val = 1},
-			{ .name = "vlan_en",            .has_arg = 0, .flag = &vlan_en, .val = 1 },
-			{ .name = "vlan_pcp",		.has_arg = 1, .flag = &vlan_pcp_flag, .val = 1 },
+			{.name = "raw_mcast", .has_arg = 0, .flag = &raw_mcast_flag, .val = 1},
+
+			{.name = "mr_per_qp", .has_arg = 0, .flag = &mr_per_qp_flag, .val = 1},
+			{.name = "dlid", .has_arg = 1, .flag = &dlid_flag, .val = 1},
+			{.name = "tclass", .has_arg = 1, .flag = &tclass_flag, .val = 1},
+			{.name = "wait_destroy", .has_arg = 1, .flag = &wait_destroy_flag, .val = 1},
+			{.name = "flows", .has_arg = 1, .flag = &flows_flag, .val = 1},
+			{.name = "flows_burst", .has_arg = 1, .flag = &flows_burst_flag, .val = 1},
+			{.name = "reply_every", .has_arg = 1, .flag = &reply_every_flag, .val = 1},
+			{.name = "perform_warm_up", .has_arg = 0, .flag = &perform_warm_up_flag, .val = 1},
+			{.name = "vlan_en", .has_arg = 0, .flag = &vlan_en, .val = 1},
+			{.name = "vlan_pcp", .has_arg = 1, .flag = &vlan_pcp_flag, .val = 1},
 
 			#if defined HAVE_OOO_ATTR
-			{ .name = "use_ooo",		.has_arg = 0, .flag = &use_ooo_flag, .val = 1},
+			{.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
 			#endif
-			{ 0 }
+			{0}
 		};
 		c = getopt_long(argc,argv,"w:y:p:d:i:m:s:n:t:u:S:x:c:q:I:o:M:r:Q:A:l:D:f:B:T:L:E:J:j:K:k:X:W:aFegzRvhbNVCHUOZP",long_options,NULL);
 
@@ -2519,6 +2543,10 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 		user_param->use_hugepages = 1;
 	}
 
+	if(old_post_send_flag) {
+		user_param->use_old_post_send = 1;
+	}
+
 	if (use_promiscuous_flag) {
 		user_param->use_promiscuous = 1;
 	}
@@ -2648,7 +2676,6 @@ int check_link_and_mtu(struct ibv_context *context,struct perftest_parameters *u
 int check_link(struct ibv_context *context,struct perftest_parameters *user_param)
 {
 	user_param->transport_type = context->device->transport_type;
-
 	if (set_link_layer(context, user_param) == FAILURE){
 		fprintf(stderr, " Couldn't set the link layer\n");
 		return FAILURE;
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index 0e874d6c..67eec6ba 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -482,6 +482,7 @@ struct perftest_parameters {
 	int 				report_per_port;
 	int 				use_odp;
 	int				use_hugepages;
+	int				use_old_post_send;
 	int				use_promiscuous;
 	int				use_sniffer;
 	int				check_alive_exited;
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 9b2a2adc..cdf450c4 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -618,27 +618,33 @@ static struct ibv_qp *ctx_xrc_qp_create(struct pingpong_context *ctx,
 		qp_init_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
 		qp_init_attr.pd = ctx->pd;
 		#ifdef HAVE_IBV_WR_API
-		qp_init_attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+		if (!user_param->use_old_post_send)
+			qp_init_attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
 		#endif
 		qp_init_attr.cap.max_inline_data = user_param->inline_size;
 	}
 
 	#ifdef HAVE_IBV_WR_API
-	if (user_param->verb == ATOMIC) {
-		opcode = opcode_atomic_array[user_param->atomicType];
-		if (opcode == IBV_WR_ATOMIC_FETCH_AND_ADD)
-			qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD;
-		else if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP)
-			qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP;
-	}
-	else {
-		opcode = opcode_verbs_array[user_param->verb];
-		if (opcode == IBV_WR_SEND)
-			qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_SEND;
-		else if (opcode == IBV_WR_RDMA_WRITE)
-			qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
-		else if (opcode == IBV_WR_RDMA_READ)
-			qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
+	if (!user_param->use_old_post_send)
+	{
+		if (user_param->verb == ATOMIC)
+		{
+			opcode = opcode_atomic_array[user_param->atomicType];
+			if (opcode == IBV_WR_ATOMIC_FETCH_AND_ADD)
+				qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD;
+			else if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP)
+				qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP;
+		}
+		else
+		{
+			opcode = opcode_verbs_array[user_param->verb];
+			if (opcode == IBV_WR_SEND)
+				qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_SEND;
+			else if (opcode == IBV_WR_RDMA_WRITE)
+				qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
+			else if (opcode == IBV_WR_RDMA_READ)
+				qp_init_attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
+		}
 	}
 	#endif
 
@@ -1309,6 +1315,7 @@ int alloc_hugepage_region (struct pingpong_context *ctx, int qp_index)
 int verify_params_with_device_context(struct ibv_context *context,
 				      struct perftest_parameters *user_param)
 {
+	enum ctx_device current_dev = ib_dev_name(context);
 	if(user_param->use_event) {
 		if(user_param->eq_num > context->num_comp_vectors) {
 			fprintf(stderr, " Completion vector specified is invalid\n");
@@ -1318,6 +1325,16 @@ int verify_params_with_device_context(struct ibv_context *context,
 		}
 	}
 
+	if (current_dev == CONNECTX3_PRO || current_dev == CONNECTX3)
+	{
+		if (!user_param->use_old_post_send)
+		{
+			fprintf(stderr, " ConnectX-3 and ConnectX-3 Pro don't support new post send API!\n");
+			fprintf(stderr, " Fall back to old post send API\n");
+			user_param->use_old_post_send = 1;
+		}
+	}
+
 	return SUCCESS;
 }
 
@@ -1545,9 +1562,11 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	int is_dc_server_side = 0;
 	#ifdef HAVE_IBV_WR_API
 	enum ibv_wr_opcode opcode;
-	struct ibv_qp_init_attr_ex attr;
+	struct ibv_qp_init_attr attr;
+	struct ibv_qp_init_attr_ex attr_ex;
 	struct mlx5dv_qp_init_attr attr_dv;
-	memset(&attr, 0, sizeof(struct ibv_qp_init_attr_ex));
+	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
+	memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex));
 	memset(&attr_dv, 0, sizeof(attr_dv));
 	#else
 	struct ibv_qp_init_attr attr;
@@ -1607,22 +1626,22 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		opcode = opcode_atomic_array[user_param->atomicType];
 		if(0);
 		else if (opcode == IBV_WR_ATOMIC_FETCH_AND_ADD)
-			attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD;
+			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD;
 		else if (opcode == IBV_WR_ATOMIC_CMP_AND_SWP)
-			attr.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP;
+			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP;
 	}
 	else {
 		opcode = opcode_verbs_array[user_param->verb];
 		if(0);
 		else if (opcode == IBV_WR_SEND)
-			attr.send_ops_flags |= IBV_QP_EX_WITH_SEND;
+			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_SEND;
 		else if (opcode == IBV_WR_RDMA_WRITE)
-			attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
+			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
 		else if (opcode == IBV_WR_RDMA_READ)
-			attr.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
+			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
 	}
-	attr.pd = ctx->pd;
-	attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD;
+	attr_ex.pd = ctx->pd;
+	attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD;
 	#endif
 
 	if (user_param->work_rdma_cm) {
@@ -1633,7 +1652,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 			qp = ctx->cm_id->qp;
 		}
 		#else
-		if (rdma_create_qp_ex(ctx->cm_id, &attr)) {
+		if (rdma_create_qp_ex(ctx->cm_id, &attr_ex)) {
 			fprintf(stderr, "Couldn't create rdma QP - %s\n", strerror(errno));
 		} else {
 			qp = ctx->cm_id->qp;
@@ -1645,29 +1664,46 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		#endif
 	} else {
 		#ifdef HAVE_IBV_WR_API
-		if (user_param->connection_type == DC) {
-			attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
-
-			if (is_dc_server_side) {
-				attr.srq = ctx->srq;
-				attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT;
-				attr_dv.dc_init_attr.dct_access_key = DC_KEY;
-				attr.comp_mask &= ~IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+		if (!user_param->use_old_post_send)
+		{
+			attr_ex.send_cq = attr.send_cq;
+			attr_ex.recv_cq = attr.recv_cq;
+			attr_ex.cap.max_send_wr = attr.cap.max_send_wr;
+			attr_ex.cap.max_send_sge = attr.cap.max_send_sge;
+			attr_ex.qp_type = attr.qp_type;
+			attr_ex.srq = attr.srq;
+			attr_ex.cap.max_inline_data = attr.cap.max_inline_data;
+			attr_ex.cap.max_recv_wr  = attr.cap.max_recv_wr;
+			attr_ex.cap.max_recv_sge = attr.cap.max_recv_sge;
+
+			if (user_param->connection_type == DC)
+			{
+				attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
+
+				if (is_dc_server_side)
+				{
+					attr_ex.srq = ctx->srq;
+					attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT;
+					attr_dv.dc_init_attr.dct_access_key = DC_KEY;
+					attr_ex.comp_mask &= ~IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+				}
+				else
+				{
+					attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
+					attr_dv.create_flags |= MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE;
+					attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
+					attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+				}
+				qp = mlx5dv_create_qp(ctx->context, &attr_ex, &attr_dv);
 			}
-			else {
-				attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
-				attr_dv.create_flags |= MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE;
-				attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
-				attr.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+			else
+			{
+				qp = ibv_create_qp_ex(ctx->context, &attr_ex);
 			}
-
-			qp = mlx5dv_create_qp(ctx->context, &attr, &attr_dv);
-		} else {
-			qp = ibv_create_qp_ex(ctx->context, &attr);
 		}
-		#else
-		qp = ibv_create_qp(ctx->pd, &attr);
+		else
 		#endif
+			qp = ibv_create_qp(ctx->pd, &attr);
 	}
 
 	if (qp == NULL && errno == ENOMEM) {
@@ -2266,7 +2302,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 			if ((j + 1) % user_param->cq_mod == 0) {
 				ctx->wr[i*user_param->post_list + j].send_flags = IBV_SEND_SIGNALED;
 				#ifdef HAVE_IBV_WR_API
-				ctx->qpx[i]->wr_flags = IBV_SEND_SIGNALED;
+				if (!user_param->use_old_post_send)
+					ctx->qpx[i]->wr_flags = IBV_SEND_SIGNALED;
 				#endif
 			} else {
 				ctx->wr[i*user_param->post_list + j].send_flags = 0;
@@ -2584,7 +2621,7 @@ static inline int post_send_method(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
 	#ifdef HAVE_IBV_WR_API
-	if (user_param->connection_type != RawEth)
+	if (!user_param->use_old_post_send)
 		return (*ctx->new_post_send_work_request_func_pointer)(ctx, index, user_param);
 	#endif
 	struct ibv_send_wr 	*bad_wr = NULL;
@@ -2707,7 +2744,8 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
-					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					if (!user_param->use_old_post_send)
+						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
@@ -2758,7 +2796,8 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
 						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 						#ifdef HAVE_IBV_WR_API
-						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
+						if (!user_param->use_old_post_send)
+							ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
 						#endif
 				}
 
@@ -3100,7 +3139,8 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)) {
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
-					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					if (!user_param->use_old_post_send)
+						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 
@@ -3120,7 +3160,8 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
 					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
-					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
+					if (!user_param->use_old_post_send)
+						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
 					#endif
 				}
 			}
@@ -3354,7 +3395,8 @@ int run_iter_bi(struct pingpong_context *ctx,
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
-					ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
+					if (!user_param->use_old_post_send)
+						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
 					#endif
 				}
 				if (user_param->noPeak == OFF)
@@ -3385,7 +3427,8 @@ int run_iter_bi(struct pingpong_context *ctx,
 
 					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
 					#ifdef HAVE_IBV_WR_API
-					ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
+					if (!user_param->use_old_post_send)
+						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
 					#endif
 				}
 			}
@@ -3614,12 +3657,14 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#ifdef HAVE_IBV_WR_API
-	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+	if (!user_param->use_old_post_send)
+		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
 	#endif
 	if (user_param->size <= user_param->inline_size) {
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
+		if (!user_param->use_old_post_send)
+			ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
 		#endif
 	}
 
@@ -3718,7 +3763,8 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 	#ifdef HAVE_IBV_WR_API
-	ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+	if (!user_param->use_old_post_send)
+		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
 	#endif
 
 	/* Duration support in latency tests. */
@@ -3808,13 +3854,15 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 		ctx->wr[0].sg_list->length = user_param->size;
 		ctx->wr[0].send_flags = 0;
 		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags = 0;
+		if (!user_param->use_old_post_send)
+			ctx->qpx[0]->wr_flags = 0;
 		#endif
 	}
 	if (user_param->size <= user_param->inline_size) {
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
 		#ifdef HAVE_IBV_WR_API
-		ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
+		if (!user_param->use_old_post_send)
+			ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
 		#endif
 	}
 	while (scnt < user_param->iters || rcnt < user_param->iters ||
@@ -3905,7 +3953,8 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				poll = 1;
 				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
 				#ifdef HAVE_IBV_WR_API
-				ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
+				if (!user_param->use_old_post_send)
+					ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
 				#endif
 			}
 
@@ -3957,7 +4006,8 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				poll = 0;
 				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
 				#ifdef HAVE_IBV_WR_API
-				ctx->qpx[0]->wr_flags &= ~IBV_SEND_SIGNALED;
+				if (!user_param->use_old_post_send)
+					ctx->qpx[0]->wr_flags &= ~IBV_SEND_SIGNALED;
 				#endif
 
 			}
diff --git a/src/write_bw.c b/src/write_bw.c
index b4305640..14ed6c9f 100755
--- a/src/write_bw.c
+++ b/src/write_bw.c
@@ -90,6 +90,14 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param))
+	{
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
 	/* See if MTU and link type are valid and supported. */
 	if (check_link(ctx.context,&user_param)) {
 		fprintf(stderr, " Couldn't get context for the device\n");
diff --git a/src/write_lat.c b/src/write_lat.c
index c7561ad2..0befd50e 100755
--- a/src/write_lat.c
+++ b/src/write_lat.c
@@ -98,6 +98,14 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param))
+	{
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
 	/* See if MTU and link type are valid and supported. */
 	if (check_link(ctx.context,&user_param)) {
 		fprintf(stderr, " Couldn't get context for the device\n");

From 1f2830f393601129ff9cc777838626b80b628558 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sun, 26 Jan 2020 15:23:17 +0200
Subject: [PATCH 10/41] Fix latency multicast flow

Description:
	The handshake introduced to support DC flow was
	breaking multicast flow

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac   |  2 +-
 src/send_lat.c | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index d155d301..e13f6a72 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.74],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.76],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/send_lat.c b/src/send_lat.c
index c4f49813..ed096156 100755
--- a/src/send_lat.c
+++ b/src/send_lat.c
@@ -348,10 +348,14 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx, &user_param, my_dest))
+		{
+			fprintf(stderr, " Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */

From 7874c75da60770e5ce81150edf7b9f79680a90a1 Mon Sep 17 00:00:00 2001
From: Moosa Baransi <moosab@mellanox.com>
Date: Wed, 19 Feb 2020 11:26:23 +0200
Subject: [PATCH 11/41] Enable PCIe relaxed ordering by default

* Introduce flag of disable_pcie_relaxed in CLI
* Remove flag pcir in CLI
* Bump perftest version of 5.78
* Use PCIe relaxed ordering by default, except for ib_write_lat

Signed-off-by: Moosa Baransi <moosab@mellanox.com>
---
 configure.ac              |   2 +-
 src/perftest_parameters.c | 204 +++++++++++++++++++++-----------------
 src/perftest_parameters.h |   5 +-
 src/perftest_resources.c  |   3 +
 src/write_lat.c           |   6 ++
 5 files changed, 129 insertions(+), 91 deletions(-)

diff --git a/configure.ac b/configure.ac
index e13f6a72..5117c62a 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.76],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.78],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 46b3d098..101aa9bc 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -471,8 +471,8 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf(" Set the Traffic Class in GRH (if GRH is in use)\n");
 
 		#ifdef HAVE_CUDA
-		printf("      --use_cuda ");
-		printf(" Use CUDA lib for GPU-Direct testing.\n");
+		printf("      --use_cuda=<cuda device id>");
+		printf(" Use CUDA secific device for GPU-Direct testing\n");
 		#endif
 
 		printf("      --use_hugepages ");
@@ -483,6 +483,9 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf("      --wait_destroy=<seconds> ");
 		printf(" Wait <seconds> before destroying allocated resources (QP/CQ/PD/MR..)\n");
 
+		printf("      --disable_pcie_relaxed");
+		printf(" Disable PCIe relaxed ordering\n");
+
 		printf("\n Rate Limiter:\n");
 		printf("      --burst_size=<size>");
 		printf(" Set the amount of messages to send in a burst when using rate limiter\n");
@@ -667,7 +670,10 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->rate_limit_type	= DISABLE_RATE_LIMIT;
 	user_param->is_rate_limit_type  = 0;
 	user_param->output		= -1;
+#ifdef HAVE_CUDA
 	user_param->use_cuda		= 0;
+	user_param->cuda_device_id		= 0;
+#endif
 	user_param->mmap_file		= NULL;
 	user_param->mmap_offset		= 0;
 	user_param->iters_per_port[0]	= 0;
@@ -722,6 +728,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->flows_burst		= 1;
 	user_param->perform_warm_up	= 0;
 	user_param->use_ooo		= 0;
+	user_param->disable_pcir	= 0;
 }
 
 /******************************************************************************
@@ -1805,7 +1812,10 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int flow_label_flag = 0;
 	static int retry_count_flag = 0;
 	static int dont_xchg_versions_flag = 0;
+#ifdef HAVE_CUDA
 	static int use_cuda_flag = 0;
+#endif
+	static int disable_pcir_flag = 0;
 	static int mmap_file_flag = 0;
 	static int mmap_offset_flag = 0;
 	static int ipv6_flag = 0;
@@ -1849,90 +1859,92 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 
 	while (1) {
 		static const struct option long_options[] = {
-			{.name = "port",		.has_arg = 1, .val = 'p' },
-			{.name = "ib-dev",		.has_arg = 1, .val = 'd' },
-			{.name = "ib-port",		.has_arg = 1, .val = 'i' },
-			{.name = "mtu",		.has_arg = 1, .val = 'm' },
-			{.name = "size",		.has_arg = 1, .val = 's' },
-			{.name = "iters",		.has_arg = 1, .val = 'n' },
-			{.name = "tx-depth",		.has_arg = 1, .val = 't' },
-			{.name = "qp-timeout",		.has_arg = 1, .val = 'u' },
-			{.name = "sl",			.has_arg = 1, .val = 'S' },
-			{.name = "gid-index",		.has_arg = 1, .val = 'x' },
-			{.name = "all",		.has_arg = 0, .val = 'a' },
-			{.name = "CPU-freq",		.has_arg = 0, .val = 'F' },
-			{.name = "connection",		.has_arg = 1, .val = 'c' },
-			{.name = "qp",			.has_arg = 1, .val = 'q' },
-			{.name = "events",		.has_arg = 0, .val = 'e' },
-			{.name = "vector",		.has_arg = 1, .val = 'X' },
-			{.name = "inline_size",	.has_arg = 1, .val = 'I' },
-			{.name = "outs",		.has_arg = 1, .val = 'o' },
-			{.name = "mcg",		.has_arg = 0, .val = 'g' },
-			{.name = "comm_rdma_cm",	.has_arg = 0, .val = 'z' },
-			{.name = "rdma_cm",		.has_arg = 0, .val = 'R' },
-			{.name = "tos",		.has_arg = 1, .val = 'T' },
-			{.name = "hop_limit",		.has_arg = 1, .val = 'L' },
-			{.name = "help",		.has_arg = 0, .val = 'h' },
-			{.name = "MGID",		.has_arg = 1, .val = 'M' },
-			{.name = "rx-depth",		.has_arg = 1, .val = 'r' },
-			{.name = "bidirectional",	.has_arg = 0, .val = 'b' },
-			{.name = "cq-mod",		.has_arg = 1, .val = 'Q' },
-			{.name = "noPeak",		.has_arg = 0, .val = 'N' },
-			{.name = "version",		.has_arg = 0, .val = 'V' },
-			{.name = "report-cycles",	.has_arg = 0, .val = 'C' },
-			{.name = "report-histogrm",	.has_arg = 0, .val = 'H' },
-			{.name = "report-unsorted",	.has_arg = 0, .val = 'U' },
-			{.name = "atomic_type",	.has_arg = 1, .val = 'A' },
-			{.name = "dualport",		.has_arg = 0, .val = 'O' },
-			{.name = "post_list",		.has_arg = 1, .val = 'l' },
-			{.name = "duration",		.has_arg = 1, .val = 'D' },
-			{.name = "margin",		.has_arg = 1, .val = 'f' },
-			{.name = "source_mac",		.has_arg = 1, .val = 'B' },
-			{.name = "dest_mac",		.has_arg = 1, .val = 'E' },
-			{.name = "dest_ip",		.has_arg = 1, .val = 'J' },
-			{.name = "source_ip",		.has_arg = 1, .val = 'j' },
-			{.name = "dest_port",		.has_arg = 1, .val = 'K' },
-			{.name = "source_port",	.has_arg = 1, .val = 'k' },
-			{.name = "ethertype",		.has_arg = 1, .val = 'Y' },
-			{.name = "limit_bw",		.has_arg = 1, .val = 'w' },
-			{.name = "limit_msgrate",	.has_arg = 1, .val = 'y' },
-			{.name = "server",		.has_arg = 0, .val = 'Z' },
-			{.name = "client",		.has_arg = 0, .val = 'P' },
-			{.name = "mac_fwd",		.has_arg = 0, .val = 'v' },
-			{.name = "use_rss",		.has_arg = 0, .val = 'G' },
-			{.name = "report-counters",	.has_arg = 1, .val = 'W' },
-			{.name = "force-link",		.has_arg = 1, .flag = &force_link_flag, .val = 1},
-			{.name = "remote_mac",		.has_arg = 1, .flag = &remote_mac_flag, .val = 1 },
-			{.name = "local_mac",		.has_arg = 1, .flag = &local_mac_flag, .val = 1 },
-			{.name = "remote_ip",		.has_arg = 1, .flag = &remote_ip_flag, .val = 1 },
-			{.name = "local_ip",		.has_arg = 1, .flag = &local_ip_flag, .val = 1 },
-			{.name = "remote_port",	.has_arg = 1, .flag = &remote_port_flag, .val = 1 },
-			{.name = "local_port",		.has_arg = 1, .flag = &local_port_flag, .val = 1 },
-			{.name = "run_infinitely",	.has_arg = 0, .flag = &run_inf_flag, .val = 1 },
-			{.name = "report_gbits",	.has_arg = 0, .flag = &report_fmt_flag, .val = 1},
-			{.name = "use-srq",		.has_arg = 0, .flag = &srq_flag, .val = 1},
-			{.name = "report-both",	.has_arg = 0, .flag = &report_both_flag, .val = 1},
-			{.name = "reversed",		.has_arg = 0, .flag = &is_reversed_flag, .val = 1},
-			{.name = "pkey_index",		.has_arg = 1, .flag = &pkey_flag, .val = 1},
-			{.name = "inline_recv",	.has_arg = 1, .flag = &inline_recv_flag, .val = 1},
-			{.name = "tcp",		.has_arg = 0, .flag = &tcp_flag, .val = 1},
-			{.name = "burst_size",		.has_arg = 1, .flag = &burst_size_flag, .val = 1},
-			{.name = "typical_pkt_size",	.has_arg = 1, .flag = &typical_pkt_size_flag, .val = 1},
-			{.name = "rate_limit",		.has_arg = 1, .flag = &rate_limit_flag, .val = 1},
-			{.name = "rate_limit_type",	.has_arg = 1, .flag = &rate_limit_type_flag, .val = 1},
-			{.name = "rate_units",		.has_arg = 1, .flag = &rate_units_flag, .val = 1},
-			{.name = "output",		.has_arg = 1, .flag = &verbosity_output_flag, .val = 1},
-			{.name = "cpu_util",		.has_arg = 0, .flag = &cpu_util_flag, .val = 1},
-			{.name = "latency_gap",	.has_arg = 1, .flag = &latency_gap_flag, .val = 1},
-			{.name = "flow_label",		.has_arg = 1, .flag = &flow_label_flag, .val = 1},
-			{.name = "retry_count",	.has_arg = 1, .flag = &retry_count_flag, .val = 1},
-			{.name = "dont_xchg_versions",	.has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
-			{.name = "use_cuda",		.has_arg = 0, .flag = &use_cuda_flag, .val = 1},
-			{.name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
-			{.name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
-			{.name = "ipv6",		.has_arg = 0, .flag = &ipv6_flag, .val = 1},
+			{ .name = "port",		.has_arg = 1, .val = 'p' },
+			{ .name = "ib-dev",		.has_arg = 1, .val = 'd' },
+			{ .name = "ib-port",		.has_arg = 1, .val = 'i' },
+			{ .name = "mtu",		.has_arg = 1, .val = 'm' },
+			{ .name = "size",		.has_arg = 1, .val = 's' },
+			{ .name = "iters",		.has_arg = 1, .val = 'n' },
+			{ .name = "tx-depth",		.has_arg = 1, .val = 't' },
+			{ .name = "qp-timeout",		.has_arg = 1, .val = 'u' },
+			{ .name = "sl",			.has_arg = 1, .val = 'S' },
+			{ .name = "gid-index",		.has_arg = 1, .val = 'x' },
+			{ .name = "all",		.has_arg = 0, .val = 'a' },
+			{ .name = "CPU-freq",		.has_arg = 0, .val = 'F' },
+			{ .name = "connection",		.has_arg = 1, .val = 'c' },
+			{ .name = "qp",			.has_arg = 1, .val = 'q' },
+			{ .name = "events",		.has_arg = 0, .val = 'e' },
+			{ .name = "vector",		.has_arg = 1, .val = 'X' },
+			{ .name = "inline_size",	.has_arg = 1, .val = 'I' },
+			{ .name = "outs",		.has_arg = 1, .val = 'o' },
+			{ .name = "mcg",		.has_arg = 0, .val = 'g' },
+			{ .name = "comm_rdma_cm",	.has_arg = 0, .val = 'z' },
+			{ .name = "rdma_cm",		.has_arg = 0, .val = 'R' },
+			{ .name = "tos",		.has_arg = 1, .val = 'T' },
+			{ .name = "hop_limit",		.has_arg = 1, .val = 'L' },
+			{ .name = "help",		.has_arg = 0, .val = 'h' },
+			{ .name = "MGID",		.has_arg = 1, .val = 'M' },
+			{ .name = "rx-depth",		.has_arg = 1, .val = 'r' },
+			{ .name = "bidirectional",	.has_arg = 0, .val = 'b' },
+			{ .name = "cq-mod",		.has_arg = 1, .val = 'Q' },
+			{ .name = "noPeak",		.has_arg = 0, .val = 'N' },
+			{ .name = "version",		.has_arg = 0, .val = 'V' },
+			{ .name = "report-cycles",	.has_arg = 0, .val = 'C' },
+			{ .name = "report-histogrm",	.has_arg = 0, .val = 'H' },
+			{ .name = "report-unsorted",	.has_arg = 0, .val = 'U' },
+			{ .name = "atomic_type",	.has_arg = 1, .val = 'A' },
+			{ .name = "dualport",		.has_arg = 0, .val = 'O' },
+			{ .name = "post_list",		.has_arg = 1, .val = 'l' },
+			{ .name = "duration",		.has_arg = 1, .val = 'D' },
+			{ .name = "margin",		.has_arg = 1, .val = 'f' },
+			{ .name = "source_mac",		.has_arg = 1, .val = 'B' },
+			{ .name = "dest_mac",		.has_arg = 1, .val = 'E' },
+			{ .name = "dest_ip",		.has_arg = 1, .val = 'J' },
+			{ .name = "source_ip",		.has_arg = 1, .val = 'j' },
+			{ .name = "dest_port",		.has_arg = 1, .val = 'K' },
+			{ .name = "source_port",	.has_arg = 1, .val = 'k' },
+			{ .name = "ethertype",		.has_arg = 1, .val = 'Y' },
+			{ .name = "limit_bw",		.has_arg = 1, .val = 'w' },
+			{ .name = "limit_msgrate",	.has_arg = 1, .val = 'y' },
+			{ .name = "server",		.has_arg = 0, .val = 'Z' },
+			{ .name = "client",		.has_arg = 0, .val = 'P' },
+			{ .name = "mac_fwd",		.has_arg = 0, .val = 'v' },
+			{ .name = "use_rss",		.has_arg = 0, .val = 'G' },
+			{ .name = "report-counters",	.has_arg = 1, .val = 'W' },
+			{ .name = "force-link",		.has_arg = 1, .flag = &force_link_flag, .val = 1},
+			{ .name = "remote_mac",		.has_arg = 1, .flag = &remote_mac_flag, .val = 1 },
+			{ .name = "local_mac",		.has_arg = 1, .flag = &local_mac_flag, .val = 1 },
+			{ .name = "remote_ip",		.has_arg = 1, .flag = &remote_ip_flag, .val = 1 },
+			{ .name = "local_ip",		.has_arg = 1, .flag = &local_ip_flag, .val = 1 },
+			{ .name = "remote_port",	.has_arg = 1, .flag = &remote_port_flag, .val = 1 },
+			{ .name = "local_port",		.has_arg = 1, .flag = &local_port_flag, .val = 1 },
+			{ .name = "run_infinitely",	.has_arg = 0, .flag = &run_inf_flag, .val = 1 },
+			{ .name = "report_gbits",	.has_arg = 0, .flag = &report_fmt_flag, .val = 1},
+			{ .name = "use-srq",		.has_arg = 0, .flag = &srq_flag, .val = 1},
+			{ .name = "report-both",	.has_arg = 0, .flag = &report_both_flag, .val = 1},
+			{ .name = "reversed",		.has_arg = 0, .flag = &is_reversed_flag, .val = 1},
+			{ .name = "pkey_index",		.has_arg = 1, .flag = &pkey_flag, .val = 1},
+			{ .name = "inline_recv",	.has_arg = 1, .flag = &inline_recv_flag, .val = 1},
+			{ .name = "tcp",		.has_arg = 0, .flag = &tcp_flag, .val = 1},
+			{ .name = "burst_size",		.has_arg = 1, .flag = &burst_size_flag, .val = 1},
+			{ .name = "typical_pkt_size",	.has_arg = 1, .flag = &typical_pkt_size_flag, .val = 1},
+			{ .name = "rate_limit",		.has_arg = 1, .flag = &rate_limit_flag, .val = 1},
+			{ .name = "rate_limit_type",	.has_arg = 1, .flag = &rate_limit_type_flag, .val = 1},
+			{ .name = "rate_units",		.has_arg = 1, .flag = &rate_units_flag, .val = 1},
+			{ .name = "output",		.has_arg = 1, .flag = &verbosity_output_flag, .val = 1},
+			{ .name = "cpu_util",		.has_arg = 0, .flag = &cpu_util_flag, .val = 1},
+			{ .name = "latency_gap",	.has_arg = 1, .flag = &latency_gap_flag, .val = 1},
+			{ .name = "flow_label",		.has_arg = 1, .flag = &flow_label_flag, .val = 1},
+			{ .name = "retry_count",	.has_arg = 1, .flag = &retry_count_flag, .val = 1},
+			{ .name = "dont_xchg_versions",	.has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
+			#ifdef HAVE_CUDA
+			{ .name = "use_cuda",		.has_arg = 1, .flag = &use_cuda_flag, .val = 1},
+			#endif
+			{ .name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
+			{ .name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
+			{ .name = "ipv6",		.has_arg = 0, .flag = &ipv6_flag, .val = 1},
 			#ifdef HAVE_IPV6
-			{.name = "raw_ipv6",		.has_arg = 0, .flag = &raw_ipv6_flag, .val = 1},
+			{ .name = "raw_ipv6",		.has_arg = 0, .flag = &raw_ipv6_flag, .val = 1},
 			#endif
 			{.name = "report-per-port", .has_arg = 0, .flag = &report_per_port_flag, .val = 1},
 			{.name = "odp", .has_arg = 0, .flag = &odp_flag, .val = 1},
@@ -1954,7 +1966,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{.name = "perform_warm_up", .has_arg = 0, .flag = &perform_warm_up_flag, .val = 1},
 			{.name = "vlan_en", .has_arg = 0, .flag = &vlan_en, .val = 1},
 			{.name = "vlan_pcp", .has_arg = 1, .flag = &vlan_pcp_flag, .val = 1},
-
+			{.name = "disable_pcie_relaxed", .has_arg = 0, .flag = &disable_pcir_flag, .val = 1 },
 			#if defined HAVE_OOO_ATTR
 			{.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
 			#endif
@@ -2289,6 +2301,18 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 					}
 					latency_gap_flag = 0;
 				}
+#ifdef HAVE_CUDA
+				if (use_cuda_flag) {
+					user_param->use_cuda = 1;
+					user_param->cuda_device_id = strtol(optarg, NULL, 0);
+					if (user_param->cuda_device_id < 0)
+					{
+						fprintf(stderr, "Invalid CUDA device %d\n", user_param->cuda_device_id);
+						return FAILURE;
+					}
+					use_cuda_flag = 0;
+				}
+#endif
 				if (flow_label_flag) {
 					user_param->flow_label = strtol(optarg,NULL,0);
 					if (user_param->flow_label < 0) {
@@ -2445,9 +2469,10 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 		user_param->use_res_domain = 1;
 	}
 
-	if (use_cuda_flag) {
-		user_param->use_cuda = 1;
+	if (disable_pcir_flag) {
+		user_param->disable_pcir = 1;
 	}
+
 	if (report_both_flag) {
 		user_param->report_both = 1;
 	}
@@ -2768,6 +2793,7 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
 	printf(" Dual-port       : %s\t\tDevice         : %s\n", user_param->dualport ? "ON" : "OFF",user_param->ib_devname);
 	printf(" Number of qps   : %d\t\tTransport type : %s\n", user_param->num_of_qps, transport_str(user_param->transport_type));
 	printf(" Connection type : %s\t\tUsing SRQ      : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON"  : "OFF");
+	printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF"  : "ON");
 
 	if (user_param->machine == CLIENT || user_param->duplex) {
 		printf(" TX depth        : %d\n",user_param->tx_depth);
@@ -2853,7 +2879,7 @@ void print_report_bw (struct perftest_parameters *user_param, struct bw_report_d
 	int run_inf_bi_factor;
 	int num_of_qps = user_param->num_of_qps;
 	long format_factor;
-	long num_of_calculated_iters = user_param->iters;
+	uint64_t num_of_calculated_iters = user_param->iters;
 
 	int free_my_bw_rep = 0;
 	if (user_param->test_method == RUN_INFINITELY)
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index 67eec6ba..f8abd62d 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -448,7 +448,10 @@ struct perftest_parameters {
 	int				buff_size;
 	int             		pkey_index;
 	int				raw_qos;
+#ifdef HAVE_CUDA
 	int				use_cuda;
+	int				cuda_device_id;
+#endif
 	char				*mmap_file;
 	unsigned long			mmap_offset;
 	/* New test params format pilot. will be used in all flags soon,. */
@@ -503,8 +506,8 @@ struct perftest_parameters {
 	int				vlan_en;
 	uint32_t			vlan_pcp;
 	void 				(*print_eth_func)(void*);
+	int				disable_pcir;
 	struct counter_context		*counter_ctx;
-
 };
 
 struct report_options {
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index cdf450c4..4707c387 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1215,6 +1215,9 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 		flags |= IBV_ACCESS_REMOTE_ATOMIC;
 	}
 
+	if (user_param->disable_pcir == 0) {
+		flags |= IBV_ACCESS_RELAXED_ORDERING;
+	}
 	/* Allocating Memory region and assigning our buffer to it. */
 	ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags);
 
diff --git a/src/write_lat.c b/src/write_lat.c
index 0befd50e..0c688abd 100755
--- a/src/write_lat.c
+++ b/src/write_lat.c
@@ -80,6 +80,12 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
+	/* In case of ib_write_bw, PCI relaxed ordering should be disabled since we're polling for data change
+	 * of last packet so in case of relaxed odering we might get the last packet in wrong order thus the test
+	 * would be incorrect
+	 */
+	user_param.disable_pcir = 1;
+
 	if (user_param.use_xrc || user_param.connection_type == DC) {
 		user_param.num_of_qps *= 2;
 	}

From 79f43b00d0a25a4ee4881a877bf5dbea26b06ac5 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sun, 26 Jan 2020 15:20:21 +0200
Subject: [PATCH 12/41] Fix for QP create flow to make RDMACM support both post
 send flows

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_communication.c |  5 +++--
 src/perftest_resources.c     | 20 ++++++++++----------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index ece488be..04c537f5 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -876,10 +876,8 @@ int rdma_client_connect(struct pingpong_context *ctx,struct perftest_parameters
 	if (res->ai_family != PF_INET) {
 		return FAILURE;
 	}
-
 	memcpy(&sin, res->ai_addr, sizeof(sin));
 	sin.sin_port = htons((unsigned short)user_param->port);
-
 	while (1) {
 
 		if (num_of_retry == 0) {
@@ -1190,6 +1188,7 @@ int create_comm_struct(struct perftest_comm *comm,
 	comm->rdma_params->mr_per_qp		= user_param->mr_per_qp;
 	comm->rdma_params->dlid			= user_param->dlid;
 	comm->rdma_params->cycle_buffer         = user_param->cycle_buffer;
+	comm->rdma_params->use_old_post_send = user_param->use_old_post_send;
 
 	if (user_param->use_rdma_cm) {
 
@@ -1207,7 +1206,9 @@ int create_comm_struct(struct perftest_comm *comm,
 		ALLOCATE(comm->rdma_ctx->mr, struct ibv_mr*, user_param->num_of_qps);
 		ALLOCATE(comm->rdma_ctx->buf, void* , user_param->num_of_qps);
 		ALLOCATE(comm->rdma_ctx->qp,struct ibv_qp*,comm->rdma_params->num_of_qps);
+		#ifdef HAVE_IBV_WR_API
 		ALLOCATE(comm->rdma_ctx->qpx,struct ibv_qp_ex*,comm->rdma_params->num_of_qps);
+		#endif
 		comm->rdma_ctx->buff_size = user_param->cycle_buffer;
 
 		if (create_rdma_resources(comm->rdma_ctx,comm->rdma_params)) {
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 4707c387..64424f64 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1645,6 +1645,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	}
 	attr_ex.pd = ctx->pd;
 	attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD;
+	attr_ex.send_cq = attr.send_cq;
+	attr_ex.recv_cq = attr.recv_cq;
+	attr_ex.cap.max_send_wr = attr.cap.max_send_wr;
+	attr_ex.cap.max_send_sge = attr.cap.max_send_sge;
+	attr_ex.qp_type = attr.qp_type;
+	attr_ex.srq = attr.srq;
+	attr_ex.cap.max_inline_data = attr.cap.max_inline_data;
+	attr_ex.cap.max_recv_wr  = attr.cap.max_recv_wr;
+	attr_ex.cap.max_recv_sge = attr.cap.max_recv_sge;
 	#endif
 
 	if (user_param->work_rdma_cm) {
@@ -1655,6 +1664,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 			qp = ctx->cm_id->qp;
 		}
 		#else
+
 		if (rdma_create_qp_ex(ctx->cm_id, &attr_ex)) {
 			fprintf(stderr, "Couldn't create rdma QP - %s\n", strerror(errno));
 		} else {
@@ -1669,16 +1679,6 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		#ifdef HAVE_IBV_WR_API
 		if (!user_param->use_old_post_send)
 		{
-			attr_ex.send_cq = attr.send_cq;
-			attr_ex.recv_cq = attr.recv_cq;
-			attr_ex.cap.max_send_wr = attr.cap.max_send_wr;
-			attr_ex.cap.max_send_sge = attr.cap.max_send_sge;
-			attr_ex.qp_type = attr.qp_type;
-			attr_ex.srq = attr.srq;
-			attr_ex.cap.max_inline_data = attr.cap.max_inline_data;
-			attr_ex.cap.max_recv_wr  = attr.cap.max_recv_wr;
-			attr_ex.cap.max_recv_sge = attr.cap.max_recv_sge;
-
 			if (user_param->connection_type == DC)
 			{
 				attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;

From ea5fd7f70888524451fa5c26e9f0118bb7160fc7 Mon Sep 17 00:00:00 2001
From: Moosa Baransi <moosab@mellanox.com>
Date: Wed, 29 Jan 2020 15:00:03 +0200
Subject: [PATCH 13/41] Add support for choosing CUDA device

Signed-off-by: Moosa Baransi <moosab@mellanox.com>
---
 src/perftest_resources.c | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 64424f64..c93d174d 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -60,9 +60,13 @@ struct check_alive_data check_alive_data;
 static CUdevice cuDevice;
 static CUcontext cuContext;
 
-static int pp_init_gpu(struct pingpong_context *ctx, size_t _size)
+static int pp_init_gpu(struct pingpong_context *ctx, size_t _size, int cuda_device_id)
 {
 	const size_t gpu_page_size = 64*1024;
+ 	int cuda_pci_bus_id;
+	int cuda_pci_device_id;
+	int index;
+	CUdevice cu_device;
 	size_t size = (_size + gpu_page_size - 1) & ~(gpu_page_size - 1);
 	printf("initializing CUDA\n");
 	CUresult error = cuInit(0);
@@ -81,18 +85,26 @@ static int pp_init_gpu(struct pingpong_context *ctx, size_t _size)
 	if (deviceCount == 0) {
 		printf("There are no available device(s) that support CUDA\n");
 		return 1;
-	} else if (deviceCount == 1)
-		printf("There is 1 device supporting CUDA\n");
-	else
-		printf("There are %d devices supporting CUDA, picking first...\n", deviceCount);
+	}
+	if (cuda_device_id >= deviceCount) {
+		fprintf(stderr, "No such device ID (%d) exists in system\n", cuda_device_id);
+		return 1;
+	}
+
+	printf("Listing all CUDA devices in system:\n");
+	for (index = 0; index < deviceCount; index++) {
+		CUCHECK(cuDeviceGet(&cu_device, index));
+		cuDeviceGetAttribute(&cuda_pci_bus_id, CU_DEVICE_ATTRIBUTE_PCI_BUS_ID , cu_device);
+		cuDeviceGetAttribute(&cuda_pci_device_id, CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID , cu_device);
+		printf("CUDA device %d: PCIe address is %02X:%02X\n", index, (unsigned int)cuda_pci_bus_id, (unsigned int)cuda_pci_device_id);
+  }
 
-	int devID = 0;
+ 	printf("\nPicking device No. %d\n", cuda_device_id);
 
-	/* pick up device with zero ordinal (default, or devID) */
-	CUCHECK(cuDeviceGet(&cuDevice, devID));
+	CUCHECK(cuDeviceGet(&cuDevice, cuda_device_id));
 
 	char name[128];
-	CUCHECK(cuDeviceGetName(name, sizeof(name), devID));
+	CUCHECK(cuDeviceGetName(name, sizeof(name), cuda_device_id));
 	printf("[pid = %d, dev = %d] device name = [%s]\n", getpid(), cuDevice, name);
 	printf("creating CUDA Ctx\n");
 
@@ -1157,7 +1169,7 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 	#ifdef HAVE_CUDA
 	if (user_param->use_cuda) {
 		ctx->is_contig_supported = FAILURE;
-		if(pp_init_gpu(ctx, ctx->buff_size)) {
+		if(pp_init_gpu(ctx, ctx->buff_size, user_param->cuda_device_id)) {
 			fprintf(stderr, "Couldn't allocate work buf.\n");
 			return FAILURE;
 		}
@@ -1231,7 +1243,9 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 
 
 	/* Initialize buffer with random numbers except in WRITE_LAT test that it 0's */
+#ifdef HAVE_CUDA
 	if (!user_param->use_cuda) {
+#endif
 		srand(time(NULL));
 		if (user_param->verb == WRITE && user_param->tst == LAT) {
 			memset(ctx->buf[qp_index], 0, ctx->buff_size);
@@ -1240,8 +1254,9 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 				((char*)ctx->buf[qp_index])[i] = (char)rand();
 			}
 		}
+#ifdef HAVE_CUDA
 	}
-
+#endif
 	return SUCCESS;
 }
 

From 3e7550e9f9bb7f8de3183288d39c0844d16dbdd4 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sun, 26 Jan 2020 18:23:59 +0200
Subject: [PATCH 14/41] Added RDMA CM support for ConnectX-3

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac             |  2 +-
 src/perftest_resources.c | 39 ++++++++++++++++++++++-----------------
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/configure.ac b/configure.ac
index 5117c62a..d8dc0b00 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.78],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.77],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index c93d174d..52984777 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1415,7 +1415,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 		}
 	}
 	#endif
-
 	ctx->is_contig_supported = FAILURE;
 
 	/* Allocating an event channel if requested. */
@@ -1507,7 +1506,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 		return SUCCESS;
 
 	for (i=0; i < user_param->num_of_qps; i++) {
-
 		if (create_qp_main(ctx, user_param, i, num_of_qps)) {
 			fprintf(stderr, "Failed to create QP.\n");
 			return FAILURE;
@@ -1672,20 +1670,29 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	#endif
 
 	if (user_param->work_rdma_cm) {
-		#if !defined(HAVE_IBV_WR_API)
-		if (rdma_create_qp(ctx->cm_id, ctx->pd, &attr)) {
-			fprintf(stderr, "Couldn't create rdma QP - %s\n", strerror(errno));
-		} else {
-			qp = ctx->cm_id->qp;
-		}
-		#else
-
-		if (rdma_create_qp_ex(ctx->cm_id, &attr_ex)) {
-			fprintf(stderr, "Couldn't create rdma QP - %s\n", strerror(errno));
-		} else {
-			qp = ctx->cm_id->qp;
+		#ifdef HAVE_IBV_WR_API
+		if (!user_param->use_old_post_send)
+		{
+			if (rdma_create_qp_ex(ctx->cm_id, &attr_ex))
+			{
+				fprintf(stderr, "Couldn't create rdma new QP - %s\n", strerror(errno));
+			}
+			else
+			{
+				qp = ctx->cm_id->qp;
+			}
 		}
+		else
 		#endif
+			if (rdma_create_qp(ctx->cm_id, ctx->pd, &attr))
+			{
+				fprintf(stderr, "Couldn't create rdma old QP - %s\n", strerror(errno));
+			}
+			else
+			{
+				qp = ctx->cm_id->qp;
+			}
+
 	} else if (user_param->connection_type == SRD) {
 		#ifdef HAVE_SRD
 		qp = efadv_create_driver_qp(ctx->pd, &attr, EFADV_QP_DRIVER_TYPE_SRD);
@@ -1726,7 +1733,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 
 	if (qp == NULL && errno == ENOMEM) {
 		fprintf(stderr, "Requested QP size might be too big. Try reducing TX depth and/or inline size.\n");
-		fprintf(stderr, "Current TX depth is %d and  inline size is %d .\n", user_param->tx_depth, user_param->inline_size);
+		fprintf(stderr, "Current TX depth is %d and inline size is %d .\n", user_param->tx_depth, user_param->inline_size);
 	}
 
 	if (user_param->inline_size > attr.cap.max_inline_data) {
@@ -1895,7 +1902,6 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
 			}
 			if (user_param->connection_type != UD && user_param->connection_type != SRD) {
-
 				attr->path_mtu = user_param->curr_mtu;
 				attr->dest_qp_num = dest->qpn;
 				attr->rq_psn = dest->psn;
@@ -2072,7 +2078,6 @@ void ctx_set_send_wqes(struct pingpong_context *ctx,
 		struct perftest_parameters *user_param,
 		struct pingpong_dest *rem_dest)
 {
-
 	ctx_set_send_reg_wqes(ctx,user_param,rem_dest);
 }
 

From 940b296be723a46100eebbdaedd1991db6e0d717 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Thu, 30 Jan 2020 12:57:44 +0200
Subject: [PATCH 15/41] Changed message for ConnectX-3 to be warnings

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_resources.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 52984777..7f577de8 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1347,8 +1347,8 @@ int verify_params_with_device_context(struct ibv_context *context,
 	{
 		if (!user_param->use_old_post_send)
 		{
-			fprintf(stderr, " ConnectX-3 and ConnectX-3 Pro don't support new post send API!\n");
-			fprintf(stderr, " Fall back to old post send API\n");
+			fprintf(stderr, " Warning: ConnectX-3 and ConnectX-3 Pro don't support WR postsend API!\n");
+			fprintf(stderr, " Warning: Falling back to ibv_post_send() API\n");
 			user_param->use_old_post_send = 1;
 		}
 	}

From 1cdcd5799d524dff4937ff2fec917b58cd2a104b Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 25 Feb 2020 01:54:49 +0200
Subject: [PATCH 16/41] Added devname to mcast parameters

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/send_bw.c  | 1 +
 src/send_lat.c | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/send_bw.c b/src/send_bw.c
index 61caf6bb..0dc82ed4 100755
--- a/src/send_bw.c
+++ b/src/send_bw.c
@@ -67,6 +67,7 @@ static int set_mcast_group(struct pingpong_context *ctx,
 	mcg_params->sm_sl   = port_attr.sm_sl;
 	mcg_params->ib_port = user_param->ib_port;
 	mcg_params->ib_ctx  = ctx->context;
+	mcg_params->ib_devname = user_param->ib_devname;
 
 	if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
 		/* Request for Mcast group create registery in SM. */
diff --git a/src/send_lat.c b/src/send_lat.c
index ed096156..e7d51682 100755
--- a/src/send_lat.c
+++ b/src/send_lat.c
@@ -74,6 +74,7 @@ static int set_mcast_group(struct pingpong_context *ctx,
 	mcg_params->sm_sl   = port_attr.sm_sl;
 	mcg_params->ib_port = user_param->ib_port;
 	mcg_params->ib_ctx  = ctx->context;
+	mcg_params->ib_devname = user_param->ib_devname;
 
 	if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
 		/* Request for Mcast group create registery in SM. */

From ae2799c2620df16f4b689b4f90b551f996b91960 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 25 Feb 2020 16:28:35 +0200
Subject: [PATCH 17/41] Fix DC with --sl flag

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_resources.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 7f577de8..ca72c435 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1869,7 +1869,7 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
 				attr->ah_attr.grh.hop_limit = 0xFF;
 				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
-				attr->ah_attr.sl = 0;
+				attr->ah_attr.sl = user_param->sl;
 			}
 			if (is_dc_server_side) {
 				attr->min_rnr_timer = 12;
@@ -1881,7 +1881,7 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 			} //DCT
 			else {
 				attr->path_mtu = user_param->curr_mtu;
-				flags |= IBV_QP_PATH_MTU;
+				flags |= IBV_QP_AV | IBV_QP_PATH_MTU;
 			} //DCI
 		}
 		else

From 6f49486184dbe40a08ab8b284f0e055f62be074c Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 9 Mar 2020 13:50:38 +0200
Subject: [PATCH 18/41] Add a warning that ODP is not supported with UC

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              | 2 +-
 src/perftest_parameters.c | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index d8dc0b00..d08a6713 100755
--- a/configure.ac
+++ b/configure.ac
@@ -31,7 +31,7 @@
 #
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.77],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.80],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 101aa9bc..f1b5d450 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -1163,6 +1163,12 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit(1);
 	}
 
+	if(user_param->connection_type == UC && user_param->use_odp) {
+		printf(RESULT_LINE);
+		fprintf(stderr," ODP does not support UC\n");
+		exit(1);
+	}
+
 	if (user_param->verb == SEND && user_param->tst == BW && user_param->machine == SERVER && !user_param->duplex )
 		user_param->noPeak = ON;
 

From 583a46702c814bfcaa414c569feb97591838044c Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 9 Mar 2020 15:01:00 +0200
Subject: [PATCH 19/41] Added version check to warn user if incompatible
 versions used

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/atomic_bw.c              | 13 ++++++++-----
 src/atomic_lat.c             | 13 ++++++++-----
 src/perftest_communication.c | 12 ++++++++++++
 src/perftest_communication.h |  8 ++++++++
 src/read_bw.c                | 14 +++++++++-----
 src/read_lat.c               | 14 ++++++++------
 src/send_bw.c                |  2 +-
 src/send_lat.c               |  2 +-
 src/write_bw.c               | 14 +++++++++-----
 src/write_lat.c              | 13 ++++++++-----
 10 files changed, 72 insertions(+), 33 deletions(-)

diff --git a/src/atomic_bw.c b/src/atomic_bw.c
index e48fe27e..6cb7dd59 100755
--- a/src/atomic_bw.c
+++ b/src/atomic_bw.c
@@ -120,7 +120,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -187,10 +187,13 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */
diff --git a/src/atomic_lat.c b/src/atomic_lat.c
index 364e122e..f771e09d 100755
--- a/src/atomic_lat.c
+++ b/src/atomic_lat.c
@@ -129,7 +129,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -202,10 +202,13 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */
diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index 04c537f5..d6bd1de8 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -1666,6 +1666,18 @@ void exchange_versions(struct perftest_comm *user_comm, struct perftest_paramete
 	}
 }
 
+/******************************************************************************
+ *
+ ******************************************************************************/
+void check_version_compatibility(struct perftest_parameters *user_param)
+{
+	if ((atof(user_param->rem_version) < 5.70))
+	{
+		fprintf(stderr, "Current implementation is not compatible with versions older than 5.70\n");
+		exit(1);
+	}
+}
+
 /******************************************************************************
  *
  ******************************************************************************/
diff --git a/src/perftest_communication.h b/src/perftest_communication.h
index 0e5824dd..08d0174b 100755
--- a/src/perftest_communication.h
+++ b/src/perftest_communication.h
@@ -409,6 +409,14 @@ void xchg_bw_reports (struct perftest_comm *comm, struct bw_report_data *my_bw_r
  */
 void exchange_versions (struct perftest_comm *user_comm, struct perftest_parameters *user_param);
 
+/* check_dc_compatibility.
+ *
+ * Description :
+ * 	Check compatibility between old and new implementation.
+ *
+ */
+void check_version_compatibility(struct perftest_parameters *user_param);
+
 /* check_sys_data.
  *
  * Description :
diff --git a/src/read_bw.c b/src/read_bw.c
index 3386a203..65edce90 100755
--- a/src/read_bw.c
+++ b/src/read_bw.c
@@ -118,7 +118,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -186,10 +186,13 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */
@@ -208,6 +211,7 @@ int main(int argc, char *argv[])
 		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
 	}
 
+
 	/* An additional handshake is required after moving qp to RTR. */
 	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
 		fprintf(stderr,"Failed to exchange data between server and clients\n");
diff --git a/src/read_lat.c b/src/read_lat.c
index 202f088b..7686482b 100755
--- a/src/read_lat.c
+++ b/src/read_lat.c
@@ -129,7 +129,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -203,12 +203,14 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
-
 	/* Print this machine QP information */
 	for (i=0; i < user_param.num_of_qps; i++)
 		ctx_print_pingpong_data(&my_dest[i],&user_comm);
diff --git a/src/send_bw.c b/src/send_bw.c
index 0dc82ed4..a73a4aac 100755
--- a/src/send_bw.c
+++ b/src/send_bw.c
@@ -237,7 +237,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
diff --git a/src/send_lat.c b/src/send_lat.c
index e7d51682..614ed53d 100755
--- a/src/send_lat.c
+++ b/src/send_lat.c
@@ -249,7 +249,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
diff --git a/src/write_bw.c b/src/write_bw.c
index 14ed6c9f..b46211cc 100755
--- a/src/write_bw.c
+++ b/src/write_bw.c
@@ -123,7 +123,7 @@ int main(int argc, char *argv[])
 	}
 	sleep(1);
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -189,10 +189,14 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx, &user_param, my_dest))
+		{
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */
diff --git a/src/write_lat.c b/src/write_lat.c
index 0c688abd..d466a50e 100755
--- a/src/write_lat.c
+++ b/src/write_lat.c
@@ -137,7 +137,7 @@ int main(int argc, char *argv[])
 	}
 
 	exchange_versions(&user_comm, &user_param);
-
+	check_version_compatibility(&user_param);
 	check_sys_data(&user_comm, &user_param);
 
 	/* See if MTU and link type are valid and supported. */
@@ -211,10 +211,13 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	/* Set up connection one more time to send qpn properly for DC */
-	if (set_up_connection(&ctx,&user_param,my_dest)) {
-		fprintf(stderr," Unable to set up socket connection\n");
-		return FAILURE;
+	if (user_param.connection_type == DC)
+	{
+		/* Set up connection one more time to send qpn properly for DC */
+		if (set_up_connection(&ctx,&user_param,my_dest)) {
+			fprintf(stderr," Unable to set up socket connection\n");
+			return FAILURE;
+		}
 	}
 
 	/* Print this machine QP information */

From 549876cd3f6d089520e881a1eeafc2483fdde645 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 9 Mar 2020 16:01:06 +0200
Subject: [PATCH 20/41] Fixed a typo in comment to check_version_compatibility
 description

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_communication.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/perftest_communication.h b/src/perftest_communication.h
index 08d0174b..f68a5c65 100755
--- a/src/perftest_communication.h
+++ b/src/perftest_communication.h
@@ -409,7 +409,7 @@ void xchg_bw_reports (struct perftest_comm *comm, struct bw_report_data *my_bw_r
  */
 void exchange_versions (struct perftest_comm *user_comm, struct perftest_parameters *user_param);
 
-/* check_dc_compatibility.
+/* check_version_compatibility.
  *
  * Description :
  * 	Check compatibility between old and new implementation.

From 12dda9a216c0d6ffc1958af1a398ce37f8e47861 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sat, 11 Apr 2020 00:33:22 +0300
Subject: [PATCH 21/41] Aligned configure.ac format to internal standart

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac | 2 --
 1 file changed, 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index d08a6713..edf5f10b 100755
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,3 @@
-#
 # Copyright (C) 2001-2011 Mellanox Technologies Ltd.  All rights reserved.
 #
 # This software is available to you under a choice of one of two
@@ -28,7 +27,6 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-#
 dnl Process this file with autoconf to produce a configure script.
 
 AC_INIT([perftest],[5.80],[linux-rdma@vger.kernel.org])

From 37f9a165265d02f2187c98dfa0fb128c960506c3 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 9 Mar 2020 17:44:45 +0200
Subject: [PATCH 22/41] Fixed DC failure on infiniband if --gid-index provided

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac             | 2 +-
 src/perftest_resources.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/configure.ac b/configure.ac
index edf5f10b..1704a8fa 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.80],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.81],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index ca72c435..5740a16e 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1857,19 +1857,19 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 	if (user_param->connection_type != RawEth) {
 		if (user_param->connection_type == DC)
 		{
+			attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
+			attr->ah_attr.sl = user_param->sl;
+
 			if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
 				|| ((attr->ah_attr.port_num == user_param->ib_port2) && (user_param->gid_index2 == DEF_GID_INDEX) && user_param->dualport)) {
 
 				attr->ah_attr.is_global = 0;
-				attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
-				attr->ah_attr.sl = user_param->sl;
 			} else {
 				attr->ah_attr.is_global  = 1;
 				attr->ah_attr.grh.dgid = dest->gid;
 				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
 				attr->ah_attr.grh.hop_limit = 0xFF;
 				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
-				attr->ah_attr.sl = user_param->sl;
 			}
 			if (is_dc_server_side) {
 				attr->min_rnr_timer = 12;

From af3d86380cbf2970bd1b6353a4cfcd363d43c73b Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 10 Mar 2020 01:04:58 +0200
Subject: [PATCH 23/41] Removed disabling CQ moderation for --events flag

Description:
	Disabled CQ moderation leads to issues with events
	processing under stress.

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              |  2 +-
 src/perftest_parameters.c | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1704a8fa..e7b29eb8 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.81],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.82],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index f1b5d450..ce274ac1 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -920,9 +920,18 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		}
 	}
 
+	/* we disable cq_mod for large message size to prevent from incorrect BW calculation
+	 *    (and also because it is not needed)
+	 * we don't disable cq_mod for UD because it doesn't support large enough messages
+	 * we don't disable cq_mod for RUN_ALL mode because we cannot change it in accordance to
+	 *    message size during RUN_ALL run
+	 * we don't disable cq_mod for use_event, because having a lot of processes with use_event leads
+	 *     to bugs (probably due to issues with events processing, thus we have less events)
+	 */
 	if (user_param->size > MSG_SIZE_CQ_MOD_LIMIT &&
 		user_param->connection_type != UD &&
-		user_param->test_method != RUN_ALL)
+		user_param->test_method != RUN_ALL &&
+		!user_param->use_event)
 	{
 		if (!user_param->req_cq_mod) // user didn't request any cq_mod
 		{

From 25721333bf91789238d930d7ddef0a03db8d0230 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 10 Mar 2020 01:57:11 +0200
Subject: [PATCH 24/41] Forced depth of SRQ to be not less than number of QPs

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              | 2 +-
 src/perftest_parameters.c | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index e7b29eb8..bbe2a8d9 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.82],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.83],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index ce274ac1..34d00ef8 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -973,6 +973,12 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit (1);
 	}
 
+	if (user_param->use_srq && user_param->num_of_qps > user_param->rx_depth) {
+		printf(RESULT_LINE);
+		printf(" Using SRQ depth should be greater than number of QPs.\n");
+		exit (1);
+	}
+
 	if (user_param->dualport == ON) {
 
 		user_param->num_of_qps *= 2;
@@ -990,7 +996,7 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	}
 
 	if (user_param->post_list > 1) {
-		#ifdef HAVE_IBV_WR_API
+#ifdef HAVE_IBV_WR_API
 		printf(RESULT_LINE);
 		fprintf(stderr, " Post list greater than 1 is not supported with new WR API\n");
 		exit(1);

From 263ab72d42d1885b22f91131db1b910a7a447d7f Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 10 Mar 2020 12:27:44 +0200
Subject: [PATCH 25/41] Refactored modify_qp_to_rtr()

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_resources.c | 59 +++++++++++++---------------------------
 1 file changed, 19 insertions(+), 40 deletions(-)

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 5740a16e..463be12c 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1855,53 +1855,32 @@ static int ctx_modify_qp_to_rtr(struct ibv_qp *qp,
 		attr->ah_attr.port_num = user_param->ib_port;
 
 	if (user_param->connection_type != RawEth) {
-		if (user_param->connection_type == DC)
-		{
-			attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
-			attr->ah_attr.sl = user_param->sl;
+		attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
+		attr->ah_attr.sl = user_param->sl;
 
-			if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
+		if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
 				|| ((attr->ah_attr.port_num == user_param->ib_port2) && (user_param->gid_index2 == DEF_GID_INDEX) && user_param->dualport)) {
 
-				attr->ah_attr.is_global = 0;
-			} else {
-				attr->ah_attr.is_global  = 1;
-				attr->ah_attr.grh.dgid = dest->gid;
-				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
-				attr->ah_attr.grh.hop_limit = 0xFF;
-				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
-			}
-			if (is_dc_server_side) {
-				attr->min_rnr_timer = 12;
-				attr->path_mtu = user_param->curr_mtu;
+			attr->ah_attr.is_global = 0;
+		} else {
 
-				flags |= IBV_QP_AV |
-					IBV_QP_PATH_MTU |
-					IBV_QP_MIN_RNR_TIMER;
-			} //DCT
-			else {
+			attr->ah_attr.is_global  = 1;
+			attr->ah_attr.grh.dgid = dest->gid;
+			attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
+			attr->ah_attr.grh.hop_limit = 0xFF;
+			attr->ah_attr.grh.traffic_class = user_param->traffic_class;
+		}
+		if (user_param->connection_type != UD && user_param->connection_type != SRD) {
+			if (user_param->connection_type == DC) {
 				attr->path_mtu = user_param->curr_mtu;
 				flags |= IBV_QP_AV | IBV_QP_PATH_MTU;
-			} //DCI
-		}
-		else
-		{
-			attr->ah_attr.dlid = (user_param->dlid) ? user_param->dlid : dest->lid;
-			attr->ah_attr.sl = user_param->sl;
-
-			if (((attr->ah_attr.port_num == user_param->ib_port) && (user_param->gid_index == DEF_GID_INDEX))
-					|| ((attr->ah_attr.port_num == user_param->ib_port2) && (user_param->gid_index2 == DEF_GID_INDEX) && user_param->dualport)) {
-
-				attr->ah_attr.is_global = 0;
-			} else {
-
-				attr->ah_attr.is_global  = 1;
-				attr->ah_attr.grh.dgid = dest->gid;
-				attr->ah_attr.grh.sgid_index = (attr->ah_attr.port_num == user_param->ib_port) ? user_param->gid_index : user_param->gid_index2;
-				attr->ah_attr.grh.hop_limit = 0xFF;
-				attr->ah_attr.grh.traffic_class = user_param->traffic_class;
+				if (is_dc_server_side)
+				{
+					attr->min_rnr_timer = 12;
+					flags |= IBV_QP_MIN_RNR_TIMER;
+				} //DCT
 			}
-			if (user_param->connection_type != UD && user_param->connection_type != SRD) {
+			else {
 				attr->path_mtu = user_param->curr_mtu;
 				attr->dest_qp_num = dest->qpn;
 				attr->rq_psn = dest->psn;

From 17cd3a717cc45c96c28c122797e75e6f5af4e5df Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Wed, 11 Mar 2020 12:42:10 +0200
Subject: [PATCH 26/41] Support post list mode with new IBV send WR API

Description:
	Iterate over post list in new post send method, instead of
	posting only the first WR. Because the send flags in post
	list mode are determined per WR and not per list, the
	extended QP wr_flags field should be set according to the
	WR flags before invoking each WR builder.

Signed-off-by: Daniel Kranzdorf <dkkranzd@amazon.com>
Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_parameters.c |  14 ++-
 src/perftest_resources.c  | 210 +++++++++++++++-----------------------
 2 files changed, 91 insertions(+), 133 deletions(-)

diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 34d00ef8..cfc09746 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -996,18 +996,16 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	}
 
 	if (user_param->post_list > 1) {
-#ifdef HAVE_IBV_WR_API
-		printf(RESULT_LINE);
-		fprintf(stderr, " Post list greater than 1 is not supported with new WR API\n");
-		exit(1);
-		#endif
-		if (!user_param->req_cq_mod) {
+		if (!user_param->req_cq_mod)
+		{
 			user_param->cq_mod = user_param->post_list;
 			printf(RESULT_LINE);
 			printf("Post List requested - CQ moderation will be the size of the post list\n");
-		} else if ((user_param->post_list % user_param->cq_mod) != 0) {
+		}
+		else if ((user_param->post_list % user_param->cq_mod) != 0)
+		{
 			printf(RESULT_LINE);
-			fprintf(stderr, " Post list size must be a multiple of CQ moderation\n");
+			fprintf(stderr, "Post list size must be a multiple of CQ moderation\n");
 			exit(1);
 		}
 	}
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 463be12c..4f241fcb 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -273,77 +273,93 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 	enum ibv_qp_type qpt, enum ibv_wr_opcode op)
 {
 	int rc;
-	int wr_index = index*user_param->post_list;
+	int wr_index = index * user_param->post_list;
+	struct ibv_send_wr *wr = &ctx->wr[wr_index];
+
 	ibv_wr_start(ctx->qpx[index]);
+	while (wr)
+	{
+		ctx->qpx[index]->wr_id = wr->wr_id;
+		ctx->qpx[index]->wr_flags = wr->send_flags;
 
-	ctx->qpx[index]->wr_id = ctx->wr[wr_index].wr_id;
-	switch (op) {
-	case IBV_WR_SEND:
-		ibv_wr_send(ctx->qpx[index]);
-		break;
-	case IBV_WR_RDMA_WRITE:
-		ibv_wr_rdma_write(
-			ctx->qpx[index],
-			ctx->wr[wr_index].wr.rdma.rkey,
-			ctx->wr[wr_index].wr.rdma.remote_addr);
-		break;
-	case IBV_WR_RDMA_READ:
-		ibv_wr_rdma_read(
-			ctx->qpx[index],
-			ctx->wr[wr_index].wr.rdma.rkey,
-			ctx->wr[wr_index].wr.rdma.remote_addr);
-		break;
-	case IBV_WR_ATOMIC_FETCH_AND_ADD:
-		ibv_wr_atomic_fetch_add(
-			ctx->qpx[index],
-			ctx->wr[wr_index].wr.atomic.rkey,
-			ctx->wr[wr_index].wr.atomic.remote_addr,
-			ctx->wr[wr_index].wr.atomic.compare_add);
-		break;
-	case IBV_WR_ATOMIC_CMP_AND_SWP:
-		ibv_wr_atomic_cmp_swp(
-			ctx->qpx[index],
-			ctx->wr[wr_index].wr.atomic.rkey,
-			ctx->wr[wr_index].wr.atomic.remote_addr,
-			ctx->wr[wr_index].wr.atomic.compare_add,
-			ctx->wr[wr_index].wr.atomic.swap);
-		break;
-	default:
-		fprintf(stderr, "Post send failed: unknown operation code.\n");;
-	}
-	if (qpt == IBV_QPT_DRIVER) {
-		mlx5dv_wr_set_dc_addr(
-			ctx->dv_qp[index],
-			ctx->ah[index],
-			ctx->r_dctn[index],
-			DC_KEY);
-	}
-	else if (qpt == IBV_QPT_UD) {
-		ibv_wr_set_ud_addr(
-			ctx->qpx[index],
-			ctx->wr[wr_index].wr.ud.ah,
-			ctx->wr[wr_index].wr.ud.remote_qpn,
-			ctx->wr[wr_index].wr.ud.remote_qkey);
-	}
-	#ifdef HAVE_XRCD
-	else if (qpt == IBV_QPT_XRC_SEND) {
-		ibv_wr_set_xrc_srqn(
-			ctx->qpx[index],
-			ctx->wr[wr_index].qp_type.xrc.remote_srqn);
-	}
-	#endif
+		switch (op)
+		{
+		case IBV_WR_SEND:
+			ibv_wr_send(ctx->qpx[index]);
+			break;
+		case IBV_WR_RDMA_WRITE:
+			ibv_wr_rdma_write(
+				ctx->qpx[index],
+				wr->wr.rdma.rkey,
+				wr->wr.rdma.remote_addr);
+			break;
+		case IBV_WR_RDMA_READ:
+			ibv_wr_rdma_read(
+				ctx->qpx[index],
+				wr->wr.rdma.rkey,
+				wr->wr.rdma.remote_addr);
+			break;
+		case IBV_WR_ATOMIC_FETCH_AND_ADD:
+			ibv_wr_atomic_fetch_add(
+				ctx->qpx[index],
+				wr->wr.atomic.rkey,
+				wr->wr.atomic.remote_addr,
+				wr->wr.atomic.compare_add);
+			break;
+		case IBV_WR_ATOMIC_CMP_AND_SWP:
+			ibv_wr_atomic_cmp_swp(
+				ctx->qpx[index],
+				wr->wr.atomic.rkey,
+				wr->wr.atomic.remote_addr,
+				wr->wr.atomic.compare_add,
+				wr->wr.atomic.swap);
+			break;
+		default:
+			fprintf(stderr, "Post send failed: unknown operation code.\n");
+		}
 
-	if (inl) {
-		ibv_wr_set_inline_data(
-			ctx->qpx[index],
-			(void*)ctx->wr[wr_index].sg_list->addr,
-			user_param->size);
-	} else {
-		ibv_wr_set_sge(
-			ctx->qpx[index],
-			ctx->wr[wr_index].sg_list->lkey,
-			ctx->wr[wr_index].sg_list->addr,
-			user_param->size);
+		if (qpt == IBV_QPT_DRIVER)
+		{
+			mlx5dv_wr_set_dc_addr(
+				ctx->dv_qp[index],
+				ctx->ah[index],
+				ctx->r_dctn[index],
+				DC_KEY);
+		}
+		else if (qpt == IBV_QPT_UD)
+		{
+			ibv_wr_set_ud_addr(
+				ctx->qpx[index],
+				wr->wr.ud.ah,
+				wr->wr.ud.remote_qpn,
+				wr->wr.ud.remote_qkey);
+		}
+
+		#ifdef HAVE_XRCD
+		else if (qpt == IBV_QPT_XRC_SEND)
+		{
+			ibv_wr_set_xrc_srqn(
+				ctx->qpx[index],
+				wr->qp_type.xrc.remote_srqn);
+		}
+		#endif
+
+		if (inl)
+		{
+			ibv_wr_set_inline_data(
+				ctx->qpx[index],
+				(void*) wr->sg_list->addr,
+				user_param->size);
+		}
+		else
+		{
+			ibv_wr_set_sge(
+				ctx->qpx[index],
+				wr->sg_list->lkey,
+				wr->sg_list->addr,
+				user_param->size);
+		}
+		wr = wr->next;
 	}
 	rc = ibv_wr_complete(ctx->qpx[index]);
 
@@ -2303,10 +2319,6 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 
 			if ((j + 1) % user_param->cq_mod == 0) {
 				ctx->wr[i*user_param->post_list + j].send_flags = IBV_SEND_SIGNALED;
-				#ifdef HAVE_IBV_WR_API
-				if (!user_param->use_old_post_send)
-					ctx->qpx[i]->wr_flags = IBV_SEND_SIGNALED;
-				#endif
 			} else {
 				ctx->wr[i*user_param->post_list + j].send_flags = 0;
 			}
@@ -2745,10 +2757,6 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
 
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-					#ifdef HAVE_IBV_WR_API
-					if (!user_param->use_old_post_send)
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#endif
 				}
 
 				if (user_param->noPeak == OFF)
@@ -2797,10 +2805,6 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
 						ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-						#ifdef HAVE_IBV_WR_API
-						if (!user_param->use_old_post_send)
-							ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-						#endif
 				}
 
 				/* Check if a full burst was sent. */
@@ -3140,10 +3144,6 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)) {
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-					#ifdef HAVE_IBV_WR_API
-					if (!user_param->use_old_post_send)
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#endif
 				}
 
 				err = post_send_method(ctx, index, user_param);
@@ -3161,10 +3161,6 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 						(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
 							(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
 					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-					#ifdef HAVE_IBV_WR_API
-					if (!user_param->use_old_post_send)
-						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-					#endif
 				}
 			}
 		}
@@ -3396,10 +3392,6 @@ int run_iter_bi(struct pingpong_context *ctx,
 				if (user_param->post_list == 1 && (ctx->scnt[index] % user_param->cq_mod == 0 && user_param->cq_mod > 1)
 					&& !(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
 					ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
-					#ifdef HAVE_IBV_WR_API
-					if (!user_param->use_old_post_send)
-						ctx->qpx[index]->wr_flags &= ~IBV_SEND_SIGNALED;
-					#endif
 				}
 				if (user_param->noPeak == OFF)
 					user_param->tposted[totscnt] = get_cycles();
@@ -3428,10 +3420,6 @@ int run_iter_bi(struct pingpong_context *ctx,
 						(user_param->test_type == ITERATIONS && ctx->scnt[index] == iters-1))) {
 
 					ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
-					#ifdef HAVE_IBV_WR_API
-					if (!user_param->use_old_post_send)
-						ctx->qpx[index]->wr_flags |= IBV_SEND_SIGNALED;
-					#endif
 				}
 			}
 		}
@@ -3658,16 +3646,9 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
-	#ifdef HAVE_IBV_WR_API
-	if (!user_param->use_old_post_send)
-		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-	#endif
+
 	if (user_param->size <= user_param->inline_size) {
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
-		#ifdef HAVE_IBV_WR_API
-		if (!user_param->use_old_post_send)
-			ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
-		#endif
 	}
 
 
@@ -3764,10 +3745,6 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	cycles_t 	end_cycle, start_gap=0;
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
-	#ifdef HAVE_IBV_WR_API
-	if (!user_param->use_old_post_send)
-		ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-	#endif
 
 	/* Duration support in latency tests. */
 	if (user_param->test_type == DURATION) {
@@ -3855,17 +3832,9 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 	if (user_param->connection_type != RawEth) {
 		ctx->wr[0].sg_list->length = user_param->size;
 		ctx->wr[0].send_flags = 0;
-		#ifdef HAVE_IBV_WR_API
-		if (!user_param->use_old_post_send)
-			ctx->qpx[0]->wr_flags = 0;
-		#endif
 	}
 	if (user_param->size <= user_param->inline_size) {
 		ctx->wr[0].send_flags |= IBV_SEND_INLINE;
-		#ifdef HAVE_IBV_WR_API
-		if (!user_param->use_old_post_send)
-			ctx->qpx[0]->wr_flags |= IBV_SEND_INLINE;
-		#endif
 	}
 	while (scnt < user_param->iters || rcnt < user_param->iters ||
 			( (user_param->test_type == DURATION && user_param->state != END_STATE))) {
@@ -3954,10 +3923,6 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 			if (scnt % user_param->cq_mod == 0 || (user_param->test_type == ITERATIONS && scnt == user_param->iters)) {
 				poll = 1;
 				ctx->wr[0].send_flags |= IBV_SEND_SIGNALED;
-				#ifdef HAVE_IBV_WR_API
-				if (!user_param->use_old_post_send)
-					ctx->qpx[0]->wr_flags |= IBV_SEND_SIGNALED;
-				#endif
 			}
 
 			/* if we're in duration mode and the time is over, exit from this function */
@@ -4007,11 +3972,6 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 				}
 				poll = 0;
 				ctx->wr[0].send_flags &= ~IBV_SEND_SIGNALED;
-				#ifdef HAVE_IBV_WR_API
-				if (!user_param->use_old_post_send)
-					ctx->qpx[0]->wr_flags &= ~IBV_SEND_SIGNALED;
-				#endif
-
 			}
 		}
 	}

From a22dafce4596eb93b94f22468d2eafc155a55622 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Wed, 11 Mar 2020 12:49:45 +0200
Subject: [PATCH 27/41] Forbid post list mode for non-BW tests.

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              |  2 +-
 src/perftest_parameters.c | 21 +++++++++++++++------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/configure.ac b/configure.ac
index bbe2a8d9..02ed98fd 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.83],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.84],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index cfc09746..cba2d842 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -996,16 +996,25 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	}
 
 	if (user_param->post_list > 1) {
-		if (!user_param->req_cq_mod)
+		if (user_param->tst == BW || user_param->tst == LAT_BY_BW)
 		{
-			user_param->cq_mod = user_param->post_list;
-			printf(RESULT_LINE);
-			printf("Post List requested - CQ moderation will be the size of the post list\n");
+			if (!user_param->req_cq_mod)
+			{
+				user_param->cq_mod = user_param->post_list;
+				printf(RESULT_LINE);
+				printf("Post List requested - CQ moderation will be the size of the post list\n");
+			}
+			else if ((user_param->post_list % user_param->cq_mod) != 0)
+			{
+				printf(RESULT_LINE);
+				fprintf(stderr, "Post list size must be a multiple of CQ moderation\n");
+				exit(1);
+			}
 		}
-		else if ((user_param->post_list % user_param->cq_mod) != 0)
+		else
 		{
 			printf(RESULT_LINE);
-			fprintf(stderr, "Post list size must be a multiple of CQ moderation\n");
+			fprintf(stderr, "Post list is supported in BW tests only\n");
 			exit(1);
 		}
 	}

From 26cb9b69dda8ebf9bdf1b6440c5335cb905eaa4e Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sun, 22 Mar 2020 14:49:29 +0200
Subject: [PATCH 28/41] Hide relaxed ordering if it is not supported by verbs.h

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac              | 10 +++++++++-
 src/perftest_parameters.c | 10 ++++++++--
 src/perftest_resources.c  |  3 +++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 02ed98fd..e7f1e99a 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.84],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.85],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
@@ -156,6 +156,14 @@ if [test $HAVE_SNIFFER = yes]; then
         AC_DEFINE([HAVE_SNIFFER], [1], [Enable Sniffer Flow Specification])
 fi
 
+AC_TRY_LINK([
+#include <infiniband/verbs.h>],
+        [int x = IBV_ACCESS_RELAXED_ORDERING;],[HAVE_RO=yes], [HAVE_RO=no])
+AM_CONDITIONAL([HAVE_RO],[test "x$HAVE_RO" = "xyes"])
+if [test $HAVE_RO = yes]; then
+        AC_DEFINE([HAVE_RO], [1], [Enable Relaxed Ordering])
+fi
+
 AC_TRY_LINK([#include <infiniband/verbs.h>],
 	[struct ibv_device_attr_ex *attr_ex; int x = attr_ex->comp_mask;],[HAVE_EX=yes], [HAVE_EX=no])
 AM_CONDITIONAL([HAVE_EX],[test "x$HAVE_EX" = "xyes"])
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index cba2d842..e0910763 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -483,9 +483,10 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf("      --wait_destroy=<seconds> ");
 		printf(" Wait <seconds> before destroying allocated resources (QP/CQ/PD/MR..)\n");
 
+		#if defined HAVE_RO
 		printf("      --disable_pcie_relaxed");
 		printf(" Disable PCIe relaxed ordering\n");
-
+		#endif
 		printf("\n Rate Limiter:\n");
 		printf("      --burst_size=<size>");
 		printf(" Set the amount of messages to send in a burst when using rate limiter\n");
@@ -1994,7 +1995,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{.name = "perform_warm_up", .has_arg = 0, .flag = &perform_warm_up_flag, .val = 1},
 			{.name = "vlan_en", .has_arg = 0, .flag = &vlan_en, .val = 1},
 			{.name = "vlan_pcp", .has_arg = 1, .flag = &vlan_pcp_flag, .val = 1},
+			#if defined HAVE_RO
 			{.name = "disable_pcie_relaxed", .has_arg = 0, .flag = &disable_pcir_flag, .val = 1 },
+			#endif
 			#if defined HAVE_OOO_ATTR
 			{.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
 			#endif
@@ -2821,8 +2824,11 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
 	printf(" Dual-port       : %s\t\tDevice         : %s\n", user_param->dualport ? "ON" : "OFF",user_param->ib_devname);
 	printf(" Number of qps   : %d\t\tTransport type : %s\n", user_param->num_of_qps, transport_str(user_param->transport_type));
 	printf(" Connection type : %s\t\tUsing SRQ      : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON"  : "OFF");
+	#ifdef HAVE_RO
 	printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF"  : "ON");
-
+	#else
+	printf(" PCIe relax order: %s\n", "Unsupported");
+	#endif
 	if (user_param->machine == CLIENT || user_param->duplex) {
 		printf(" TX depth        : %d\n",user_param->tx_depth);
 	}
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 4f241fcb..0173b849 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -1243,9 +1243,12 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 		flags |= IBV_ACCESS_REMOTE_ATOMIC;
 	}
 
+#ifdef HAVE_RO
 	if (user_param->disable_pcir == 0) {
 		flags |= IBV_ACCESS_RELAXED_ORDERING;
 	}
+#endif
+
 	/* Allocating Memory region and assigning our buffer to it. */
 	ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags);
 

From f9b3b87d3ec742afe7a56cebb4bb29e7ab390986 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Tue, 24 Mar 2020 00:45:38 +0200
Subject: [PATCH 29/41] Fixed latency at low message size with --all flag

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac             |  2 +-
 src/perftest_resources.c | 58 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/configure.ac b/configure.ac
index e7f1e99a..86c8de3c 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.85],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.86],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 0173b849..c9319007 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2257,10 +2257,6 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 		num_of_qps /= 2;
 		xrc_offset = num_of_qps;
 	}
-	#ifdef HAVE_IBV_WR_API
-	if (user_param->connection_type != RawEth)
-		ctx_post_send_work_request_func_pointer(ctx, user_param);
-	#endif
 
 	for (i = 0; i < num_of_qps ; i++) {
 		if (user_param->connection_type == DC)
@@ -2675,8 +2671,12 @@ int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_pa
 	int			address_offset = 0;
 	int			flows_burst_iter = 0;
 
-	ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
 
+	ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
 	if (user_param->test_type == DURATION) {
 		duration_param=user_param;
 		duration_param->state = START_STATE;
@@ -2917,6 +2917,11 @@ int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters
 	int			recv_flows_burst = 0;
 	int			address_flows_offset =0;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
 	ALLOCATE(swc ,struct ibv_wc ,user_param->tx_depth);
 
@@ -3104,6 +3109,11 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 	int 			return_value = 0;
 	int 			single_thread_handler;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
 	ALLOCATE(scnt_for_qp,uint64_t,user_param->num_of_qps);
 	memset(scnt_for_qp,0,sizeof(uint64_t)*user_param->num_of_qps);
@@ -3211,6 +3221,11 @@ int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_
 	int                     *scredit_for_qp = NULL;
 	int 			return_value = 0;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc ,struct ibv_wc ,CTX_POLL_BATCH);
 	ALLOCATE(swc ,struct ibv_wc ,user_param->tx_depth);
 
@@ -3337,6 +3352,11 @@ int run_iter_bi(struct pingpong_context *ctx,
 	int 			size_per_qp = (user_param->use_srq) ? user_param->rx_depth/user_param->num_of_qps : user_param->rx_depth;
 	int 			return_value = 0;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc_tx,struct ibv_wc,CTX_POLL_BATCH);
 	ALLOCATE(rcnt_for_qp,uint64_t,user_param->num_of_qps);
 	ALLOCATE(scredit_for_qp,int,user_param->num_of_qps);
@@ -3647,6 +3667,11 @@ int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *
 	int 			total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 		end_cycle, start_gap=0;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 
@@ -3746,6 +3771,12 @@ int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_p
 	int 		cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f);
 	int 		total_gap_cycles = user_param->latency_gap * cpu_mhz;
 	cycles_t 	end_cycle, start_gap=0;
+
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ctx->wr[0].sg_list->length = user_param->size;
 	ctx->wr[0].send_flags = IBV_SEND_SIGNALED;
 
@@ -3832,6 +3863,12 @@ int run_iter_lat_send(struct pingpong_context *ctx,struct perftest_parameters *u
 	cycles_t 		end_cycle, start_gap=0;
 	uintptr_t		primary_send_addr = ctx->sge_list[0].addr;
 	uintptr_t		primary_recv_addr = ctx->recv_sge_list[0].addr;
+
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	if (user_param->connection_type != RawEth) {
 		ctx->wr[0].sg_list->length = user_param->size;
 		ctx->wr[0].send_flags = 0;
@@ -3997,6 +4034,11 @@ int run_iter_lat_burst_server(struct pingpong_context *ctx, struct perftest_para
 	struct ibv_recv_wr      *bad_wr_recv = NULL;
 	int wc_id;
 
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc, struct ibv_wc, user_param->burst_size);
 
 	/* main loop for polling */
@@ -4074,6 +4116,12 @@ int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters
 	int			burst_iter = 0;
 	int			is_sending_burst = 0;
 	struct ibv_recv_wr      *bad_wr_recv = NULL;
+
+	#ifdef HAVE_IBV_WR_API
+	if (user_param->connection_type != RawEth)
+		ctx_post_send_work_request_func_pointer(ctx, user_param);
+	#endif
+
 	ALLOCATE(wc, struct ibv_wc, user_param->burst_size);
 
 	tot_iters = (uint64_t)user_param->iters;

From 1ada1b963b1edeae3dc37d928244d07fac6583bb Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 30 Mar 2020 16:46:10 +0300
Subject: [PATCH 30/41] Fixed choosing gid_index (RoCE v2 is prefered now)

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac                 |  2 +-
 src/perftest_communication.c | 48 ++++++++++++++++++------------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/configure.ac b/configure.ac
index 86c8de3c..467cd20c 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.86],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.87],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index d6bd1de8..fc60bea8 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -527,38 +527,45 @@ static int rdma_read_keys(struct pingpong_dest *rem_dest,
 	return 0;
 }
 
-//TODO: Move to another gid
-#ifdef HAVE_GID_ATTR
+// declaration is copied from rdma-core, since it is made private there.
+enum ibv_gid_type
+{
+	IBV_GID_TYPE_IB_ROCE_V1,
+	IBV_GID_TYPE_ROCE_V2,
+};
+
+int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num,
+	unsigned int index, enum ibv_gid_type *type);
+
+
 enum who_is_better {LEFT_IS_BETTER, EQUAL, RIGHT_IS_BETTER};
 
 struct roce_version_sorted_enum {
-	enum ibv_exp_roce_gid_type type;
+	enum ibv_gid_type type;
 	int rate;
 };
 
 /* This struct defines which RoCE version is more important for default usage */
-struct roce_version_sorted_enum roce_versions_sorted [] = {
-	{IBV_EXP_IB_ROCE_V1_GID_TYPE, 1},
-	{IBV_EXP_ROCE_V2_GID_TYPE, 2},
-	{IBV_EXP_ROCE_V1_5_GID_TYPE, 3}
+struct roce_version_sorted_enum roce_versions_sorted[] = {
+	{IBV_GID_TYPE_IB_ROCE_V1, 1},
+	{IBV_GID_TYPE_ROCE_V2, 2},
 };
 
-int find_roce_version_rate (int roce_ver)
-{
+int find_roce_version_rate(enum ibv_gid_type roce_ver) {
 	int i;
 	int arr_len = GET_ARRAY_SIZE(roce_versions_sorted);
 
-	for (i = 0; i < arr_len; i++) {
+	for (i = 0; i < arr_len; i++)
+	{
 		if (roce_versions_sorted[i].type == roce_ver)
 			return roce_versions_sorted[i].rate;
 	}
-
 	return -1;
 }
 
-/* RoCE V1.5 > V2 > V1
+/* RoCE V2 > V1
  * other RoCE versions will be ignored until added to roce_versions_sorted array */
-static int check_better_roce_version (int roce_ver, int roce_ver_rival)
+static int check_better_roce_version(enum ibv_gid_type roce_ver, enum ibv_gid_type roce_ver_rival)
 {
 	int roce_ver_rate = find_roce_version_rate(roce_ver);
 	int roce_ver_rate_rival = find_roce_version_rate(roce_ver_rival);
@@ -570,7 +577,6 @@ static int check_better_roce_version (int roce_ver, int roce_ver_rival)
 	else
 		return EQUAL;
 }
-#endif
 
 static int get_best_gid_index (struct pingpong_context *ctx,
 		  struct perftest_parameters *user_param,
@@ -596,24 +602,18 @@ static int get_best_gid_index (struct pingpong_context *ctx,
 			gid_index = i;
 		else if (!is_ipv4_rival && is_ipv4 && user_param->ipv6)
 			gid_index = i;
-#ifdef HAVE_GID_ATTR
 		else {
-			int roce_version, roce_version_rival;
-			struct ibv_exp_gid_attr gid_attr;
+			enum ibv_gid_type roce_version, roce_version_rival;
 
-			gid_attr.comp_mask = IBV_EXP_QUERY_GID_ATTR_TYPE;
-			if (ibv_exp_query_gid_attr(ctx->context, port, gid_index, &gid_attr))
+			if (ibv_query_gid_type(ctx->context, port, gid_index, &roce_version))
 				continue;
-			roce_version = gid_attr.type;
 
-			if (ibv_exp_query_gid_attr(ctx->context, port, i, &gid_attr))
+			if (ibv_query_gid_type(ctx->context, port, i, &roce_version_rival))
 				continue;
-			roce_version_rival = gid_attr.type;
 
 			if (check_better_roce_version(roce_version, roce_version_rival) == RIGHT_IS_BETTER)
-				gid_index = i;
+			 	gid_index = i;
 		}
-#endif
 	}
 
 	return gid_index;

From 26a9d54a83b7957f15659bdb8791e0b8f0afefff Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Mon, 6 Apr 2020 17:43:19 +0300
Subject: [PATCH 31/41] Compilation fixes for legacy-libs and FreeBSD

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 Makefile.am                  |  3 ---
 configure.ac                 | 20 +++++++++++++++++++-
 src/get_clock.c              |  2 +-
 src/perftest_communication.c |  7 +++++--
 src/perftest_parameters.c    |  8 ++++----
 src/perftest_resources.c     | 22 ++++++++++++++++------
 src/perftest_resources.h     |  4 ++++
 7 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 0090203d..e6a1132f 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -52,11 +52,8 @@ endif
 
 if IS_FREEBSD
 LIBMLX4= -lmlx4
-LIBMLX5= -lmlx5
 else
 LIBMLX4=
-LIBMLX5= -lmlx5
-
 endif
 
 ib_send_bw_SOURCES = src/send_bw.c src/multicast_resources.c src/multicast_resources.h
diff --git a/configure.ac b/configure.ac
index 467cd20c..b9633b96 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.87],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.88],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
@@ -112,6 +112,7 @@ AC_TRY_LINK([],
 AM_CONDITIONAL([IS_FREEBSD],[test "x$IS_FREEBSD" = "xyes"])
 if test $IS_FREEBSD = yes; then
 	AC_DEFINE([IS_FREEBSD], [1], [OS is FreeBSD])
+	LIBMLX5=-lmlx5
 fi
 
 AC_TRY_LINK([#include <infiniband/verbs.h>],
@@ -126,12 +127,22 @@ AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;],[HAVE_IBV_WR_API=yes],[HAVE_IBV_WR_API=no])
 AM_CONDITIONAL([HAVE_IBV_WR_API],[test "x$HAVE_IBV_WR_API" = "xyes"])
+# TODO: Add appropriate check. Currently we consider that if rdma-core
+# has new post send, it also has ibv_query_gid_type
+if [test $HAVE_IBV_WR_API = yes]; then
+	AC_DEFINE([HAVE_GID_TYPE], [1], [Have a way to check gid type])
+fi
 if [test $HAVE_IBV_WR_API = yes] && [test $USE_IBV_WR_API = yes]; then
         AC_DEFINE([HAVE_IBV_WR_API], [1], [Have new post send API support])
 else
 	HAVE_IBV_WR_API=no
 fi
 
+AC_TRY_LINK([
+#include <infiniband/mlx5dv.h>],
+        [int x = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;],[HAVE_MLX5DV=yes], [HAVE_MLX5DV=no])
+AM_CONDITIONAL([HAVE_MLX5DV],[test "x$HAVE_MLX5DV" = "xyes"])
+
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_FLOW_SPEC_IPV6;],[HAVE_IPV6=yes], [HAVE_IPV6=no])
@@ -213,6 +224,13 @@ if [test $HAVE_SRD = yes] && [test $HAVE_IBV_WR_API = no]; then
 	AC_SUBST([LIBEFA])
 fi
 
+AC_CHECK_LIB([mlx5], [mlx5dv_create_qp], [HAVE_MLX5DV_LIB=yes LIBMLX5=-lmlx5], [HAVE_MLX5DV_LIB=no])
+AM_CONDITIONAL([HAVE_MLX5DV_LIB], [test "x$HAVE_MLX5DV_LIB" = "xyes"])
+if [test $HAVE_MLX5DV_LIB = yes] && [test $HAVE_MLX5DV = yes]; then
+	AC_DEFINE([HAVE_MLX5DV], [1], [Have Direct Verbs support])
+	AC_SUBST([LIBMLX5])
+fi
+
 CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS"
 LIBS=$LIBS" -lpthread"
 AC_SUBST([LIBUMAD])
diff --git a/src/get_clock.c b/src/get_clock.c
index 91aaf723..acdc6f17 100755
--- a/src/get_clock.c
+++ b/src/get_clock.c
@@ -39,7 +39,7 @@
 /* #define GET_CPU_MHZ_FROM_PROC 1 */
 
 /* For gettimeofday */
-#define _BSD_SOURCE
+#define _DEFAULT_SOURCE
 #include <sys/time.h>
 
 #include <unistd.h>
diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index fc60bea8..3ee76390 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -527,7 +527,8 @@ static int rdma_read_keys(struct pingpong_dest *rem_dest,
 	return 0;
 }
 
-// declaration is copied from rdma-core, since it is made private there.
+#ifdef HAVE_GID_TYPE
+// declaration is copied from rdma-core, since it is private there.
 enum ibv_gid_type
 {
 	IBV_GID_TYPE_IB_ROCE_V1,
@@ -577,6 +578,7 @@ static int check_better_roce_version(enum ibv_gid_type roce_ver, enum ibv_gid_ty
 	else
 		return EQUAL;
 }
+#endif
 
 static int get_best_gid_index (struct pingpong_context *ctx,
 		  struct perftest_parameters *user_param,
@@ -602,6 +604,7 @@ static int get_best_gid_index (struct pingpong_context *ctx,
 			gid_index = i;
 		else if (!is_ipv4_rival && is_ipv4 && user_param->ipv6)
 			gid_index = i;
+#ifdef HAVE_GID_TYPE
 		else {
 			enum ibv_gid_type roce_version, roce_version_rival;
 
@@ -614,8 +617,8 @@ static int get_best_gid_index (struct pingpong_context *ctx,
 			if (check_better_roce_version(roce_version, roce_version_rival) == RIGHT_IS_BETTER)
 			 	gid_index = i;
 		}
+#endif
 	}
-
 	return gid_index;
 }
 
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index e0910763..969a23ea 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -793,10 +793,10 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
 		exit(1);
 		#endif
 	} else if (strcmp(connStr[5], optarg)==0) {
-		#ifdef HAVE_IBV_WR_API
+		#ifdef HAVE_MLX5DV
 		*cptr = DC;
 		#else
-		fprintf(stderr," DC not detected in libibverbs, IBV_WR API is needed\n");
+		fprintf(stderr," DC not supported, mlx5dv.h is needed\n");
 		exit(1);
 		#endif
 	} else if (strcmp(connStr[6], optarg) == 0) {
@@ -1242,8 +1242,8 @@ static void force_dependecies(struct perftest_parameters *user_param)
 	{
 		#ifndef HAVE_IBV_WR_API
 		printf(RESULT_LINE);
-		fprintf(stderr, " new post send flow is not supported\n");
-		exit(1);
+		fprintf(stderr, " new post send flow is not supported, falling back to ibv_post_send\n");
+		user_param->use_old_post_send = 1;
 		#endif
 	}
 
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index c9319007..2c05f545 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -317,7 +317,7 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 		default:
 			fprintf(stderr, "Post send failed: unknown operation code.\n");
 		}
-
+		#ifdef HAVE_MLX5DV
 		if (qpt == IBV_QPT_DRIVER)
 		{
 			mlx5dv_wr_set_dc_addr(
@@ -326,7 +326,9 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 				ctx->r_dctn[index],
 				DC_KEY);
 		}
-		else if (qpt == IBV_QPT_UD)
+		else
+		#endif
+		if (qpt == IBV_QPT_UD)
 		{
 			ibv_wr_set_ud_addr(
 				ctx->qpx[index],
@@ -809,7 +811,9 @@ void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_par
 	ALLOCATE(ctx->qp, struct ibv_qp*, user_param->num_of_qps);
 	#ifdef HAVE_IBV_WR_API
 	ALLOCATE(ctx->qpx, struct ibv_qp_ex*, user_param->num_of_qps);
+	#ifdef HAVE_MLX5DV
 	ALLOCATE(ctx->dv_qp, struct mlx5dv_qp_ex*, user_param->num_of_qps);
+	#endif
 	ALLOCATE(ctx->r_dctn, uint32_t, user_param->num_of_qps);
 	#endif
 	ALLOCATE(ctx->mr, struct ibv_mr*, user_param->num_of_qps);
@@ -1571,11 +1575,13 @@ int create_reg_qp_main(struct pingpong_context *ctx,
 	}
 	#ifdef HAVE_IBV_WR_API
 	ctx->qpx[i] = ibv_qp_to_qp_ex(ctx->qp[i]);
+	#ifdef HAVE_MLX5DV
 	if (user_param->connection_type == DC)
 	{
 		ctx->dv_qp[i] = mlx5dv_qp_ex_from_ibv_qp_ex(ctx->qpx[i]);
 	}
 	#endif
+	#endif
 
 	return SUCCESS;
 }
@@ -1599,10 +1605,12 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	enum ibv_wr_opcode opcode;
 	struct ibv_qp_init_attr attr;
 	struct ibv_qp_init_attr_ex attr_ex;
+	#ifdef HAVE_MLX5DV
 	struct mlx5dv_qp_init_attr attr_dv;
+	memset(&attr_dv, 0, sizeof(attr_dv));
+	#endif
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
 	memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex));
-	memset(&attr_dv, 0, sizeof(attr_dv));
 	#else
 	struct ibv_qp_init_attr attr;
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
@@ -1645,7 +1653,9 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		case RC : attr.qp_type = IBV_QPT_RC; break;
 		case UC : attr.qp_type = IBV_QPT_UC; break;
 		case UD : attr.qp_type = IBV_QPT_UD; break;
+		#ifdef HAVE_IBV_WR_API
 		case DC : attr.qp_type = IBV_QPT_DRIVER; break;
+		#endif
 		#ifdef HAVE_RAW_ETH
 		case RawEth : attr.qp_type = IBV_QPT_RAW_PACKET; break;
 		#endif
@@ -1720,6 +1730,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 		#ifdef HAVE_IBV_WR_API
 		if (!user_param->use_old_post_send)
 		{
+			#ifdef HAVE_MLX5DV
 			if (user_param->connection_type == DC)
 			{
 				attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
@@ -1741,12 +1752,11 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 				qp = mlx5dv_create_qp(ctx->context, &attr_ex, &attr_dv);
 			}
 			else
-			{
+			#endif // HAVE_MLX5DV
 				qp = ibv_create_qp_ex(ctx->context, &attr_ex);
-			}
 		}
 		else
-		#endif
+		#endif // HAVE_IBV_WR_API
 			qp = ibv_create_qp(ctx->pd, &attr);
 	}
 
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
index 81f30cc2..2c8160ba 100755
--- a/src/perftest_resources.h
+++ b/src/perftest_resources.h
@@ -52,7 +52,9 @@
 #define PERFTEST_RESOURCES_H
 
 #include <infiniband/verbs.h>
+#if defined(HAVE_MLX5DV)
 #include <infiniband/mlx5dv.h>
+#endif
 #include <rdma/rdma_cma.h>
 #include <stdint.h>
 #if defined(__FreeBSD__)
@@ -168,7 +170,9 @@ struct pingpong_context {
 	struct ibv_qp				**qp;
 	#ifdef HAVE_IBV_WR_API
 	struct ibv_qp_ex			**qpx;
+	#ifdef HAVE_MLX5DV
 	struct mlx5dv_qp_ex			**dv_qp;
+	#endif
 	int (*new_post_send_work_request_func_pointer) (struct pingpong_context *ctx, int index,
 		struct perftest_parameters *user_param);
 	#endif

From dd2ca908be20331a349834f12c5c2ceb6fc6c6a5 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sat, 11 Apr 2020 00:01:56 +0300
Subject: [PATCH 32/41] Ported bugfixes from community

Description:
	Fix allocation in ctx_alloc_credit
	Fix two compliation issues for gcc 10
	Fix - Failed to deallocate PD - Device or resource busy

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac                 | 2 +-
 src/perftest_resources.c     | 2 +-
 src/raw_ethernet_resources.c | 2 +-
 src/send_bw.c                | 1 -
 src/write_lat.c              | 2 +-
 5 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index b9633b96..60c7c0c0 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.88],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.89],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 2c05f545..a80e7d21 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -2484,7 +2484,7 @@ int ctx_alloc_credit(struct pingpong_context *ctx,
 	int flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE;
 	int i;
 
-	ALLOCATE(ctx->ctrl_buf,uint32_t,user_param->num_of_qps);
+	ALLOCATE(ctx->ctrl_buf,uint32_t,2*user_param->num_of_qps);
 	memset(&ctx->ctrl_buf[0],0,buf_size);
 
 	ctx->credit_buf = (uint32_t *)ctx->ctrl_buf + user_param->num_of_qps;
diff --git a/src/raw_ethernet_resources.c b/src/raw_ethernet_resources.c
index bfc15454..8d99f9d4 100755
--- a/src/raw_ethernet_resources.c
+++ b/src/raw_ethernet_resources.c
@@ -55,7 +55,7 @@
 #include <config.h>
 #endif
 
-struct perftest_parameters* duration_param;
+extern struct perftest_parameters* duration_param;
 
 int check_flow_steering_support(char *dev_name)
 {
diff --git a/src/send_bw.c b/src/send_bw.c
index a73a4aac..d8e23891 100755
--- a/src/send_bw.c
+++ b/src/send_bw.c
@@ -560,7 +560,6 @@ int main(int argc, char *argv[])
 		fprintf(stderr," Failed to close connection between server and client\n");
 		fprintf(stderr," Trying to close this side resources\n");
 	}
-	user_param.work_rdma_cm = OFF;
 	/* Destroy all test resources, including Mcast if exists */
 	if (send_destroy_ctx(&ctx,&user_param,&mcg_params)) {
 		fprintf(stderr,"Couldn't destroy all SEND resources\n");
diff --git a/src/write_lat.c b/src/write_lat.c
index d466a50e..b2a97cf9 100755
--- a/src/write_lat.c
+++ b/src/write_lat.c
@@ -80,7 +80,7 @@ int main(int argc, char *argv[])
 		return FAILURE;
 	}
 
-	/* In case of ib_write_bw, PCI relaxed ordering should be disabled since we're polling for data change
+	/* In case of ib_write_lat, PCI relaxed ordering should be disabled since we're polling for data change
 	 * of last packet so in case of relaxed odering we might get the last packet in wrong order thus the test
 	 * would be incorrect
 	 */

From 068b205c953f633f865e145cdafc143da1ad622e Mon Sep 17 00:00:00 2001
From: sues <sues@mellanox.com>
Date: Tue, 7 Apr 2020 10:53:26 +0300
Subject: [PATCH 33/41] Fix race condition issue on iteration

Description:
With --run_infinitely specified in BW test, race condition is found across 2 threads -
  thread#1 run_iter_bw_infinitely to run traffic, it keeps "iterations" increasing while running traffic
  thread#2 print_report_bw to report BW result, called periodly to report result and clear "iterations"

Sometimes "iterations" is found to be twice as expected in BW result, caused by race condition:
 bytes     iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
-----------------------------------------------------------------------------------
 65536      882256           0.00                 92.70            0.176805
 65536      882012           0.00                 92.70            0.176804
 65536      1803183          0.00                 189.42           0.361283  <--- unexpected

The fix is to make sure "iteration" across thread#1 and thread#2 is synchronized.

Signed-off-by: Sue Sun <sues@mellanox.com>
---
 src/perftest_parameters.c | 8 +++++++-
 src/perftest_parameters.h | 3 ++-
 src/perftest_resources.c  | 3 ++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 969a23ea..7a63bbd2 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -2916,8 +2916,14 @@ void print_report_bw (struct perftest_parameters *user_param, struct bw_report_d
 	uint64_t num_of_calculated_iters = user_param->iters;
 
 	int free_my_bw_rep = 0;
-	if (user_param->test_method == RUN_INFINITELY)
+	if (user_param->test_method == RUN_INFINITELY) {
 		user_param->tcompleted[opt_posted]= get_cycles();
+		/*
+                 * cumulative iterations may reach maximum and restarts from 0
+                 * then iters < last_iters
+                 */
+		num_of_calculated_iters = (uint64_t)(user_param->iters - user_param->last_iters);
+	}
 
 	cycles_t t,opt_delta, peak_up, peak_down,tsize;
 
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index f8abd62d..eadac439 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -357,8 +357,9 @@ struct perftest_parameters {
 	uint64_t			size;
 	int				req_size;
 	uint64_t			dct_key;
-	uint64_t				iters;
+	uint64_t			iters;
 	uint64_t			iters_per_port[2];
+	uint64_t			last_iters;
 	uint64_t			*port_by_qp;
 	int				tx_depth;
 	uint8_t				qp_timeout;
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index a80e7d21..0d246500 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -3146,6 +3146,7 @@ int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_paramete
 
 	alarm(user_param->duration);
 	user_param->iters = 0;
+	user_param->last_iters = 0;
 
 	/* Will be 0, in case of Duration (look at force_dependencies or in the exp above) */
 	if (user_param->duplex && (user_param->use_xrc || user_param->connection_type == DC))
@@ -4317,7 +4318,7 @@ void check_alive(int sig)
 void catch_alarm_infintely()
 {
 	print_report_bw(duration_param,NULL);
-	duration_param->iters = 0;
+	duration_param->last_iters = duration_param->iters;
 	alarm(duration_param->duration);
 	duration_param->tposted[0] = get_cycles();
 }

From 8e00a31be106b5eb679f6da69a350feb62867872 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sat, 11 Apr 2020 02:20:57 +0300
Subject: [PATCH 34/41] Fixed wording in --use-cuda flag help

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_parameters.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 7a63bbd2..d2c70374 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -472,7 +472,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 
 		#ifdef HAVE_CUDA
 		printf("      --use_cuda=<cuda device id>");
-		printf(" Use CUDA secific device for GPU-Direct testing\n");
+		printf(" Use CUDA specific device for GPUDirect RDMA testing\n");
 		#endif
 
 		printf("      --use_hugepages ");

From 05e8b869df72da790d65f8707c646979372c8708 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Thu, 16 Apr 2020 13:36:14 +0300
Subject: [PATCH 35/41] Added support for ConnectX-6Lx and ConnectX-7

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_parameters.c | 2 ++
 src/perftest_parameters.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index d2c70374..add08aa1 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -1554,6 +1554,8 @@ enum ctx_device ib_dev_name(struct ibv_context *context)
 			case 4124  : dev_fname = CONNECTX6; break;
 			case 4125  : dev_fname = CONNECTX6DX; break;
 			case 4126  : dev_fname = MLX5GENVF; break;
+			case 4127  : dev_fname = CONNECTX6LX; break;
+			case 4129  : dev_fname = CONNECTX7; break;
 			case 41682 : dev_fname = BLUEFIELD; break;
 			case 41683 : dev_fname = BLUEFIELD; break;
 			case 41686 : dev_fname = BLUEFIELD2; break;
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
index eadac439..8b983333 100755
--- a/src/perftest_parameters.h
+++ b/src/perftest_parameters.h
@@ -297,6 +297,8 @@ enum ctx_device {
 	INTEL_ALL		= 22,
 	NETXTREME		= 23,
 	EFA			= 24,
+	CONNECTX6LX		= 25,
+	CONNECTX7		= 26,
 };
 
 /* Units for rate limiter */

From c8485776555fead98079a6cbe3726927c003f50f Mon Sep 17 00:00:00 2001
From: Gal Pressman <galpress@amazon.com>
Date: Sun, 21 Jul 2019 16:52:37 +0300
Subject: [PATCH 36/41] Fix CUDA runs with mr_per_qp flag

When using CUDA with the mr_per_qp flag, buffer[0] is always used
instead of per QP buffer. Fix that by splitting the GPU init flow and
buffer allocation flow.

Signed-off-by: Gal Pressman <galpress@amazon.com>
---
 configure.ac             |  2 +-
 src/perftest_resources.c | 63 +++++++++++++++++++++++-----------------
 2 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/configure.ac b/configure.ac
index 60c7c0c0..91fd91c4 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.89],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.90],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 0d246500..3e3b6fd7 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -60,14 +60,13 @@ struct check_alive_data check_alive_data;
 static CUdevice cuDevice;
 static CUcontext cuContext;
 
-static int pp_init_gpu(struct pingpong_context *ctx, size_t _size, int cuda_device_id)
+static int pp_init_gpu(struct pingpong_context *ctx, int cuda_device_id)
 {
-	const size_t gpu_page_size = 64*1024;
- 	int cuda_pci_bus_id;
+	int cuda_pci_bus_id;
 	int cuda_pci_device_id;
 	int index;
 	CUdevice cu_device;
-	size_t size = (_size + gpu_page_size - 1) & ~(gpu_page_size - 1);
+
 	printf("initializing CUDA\n");
 	CUresult error = cuInit(0);
 	if (error != CUDA_SUCCESS) {
@@ -122,33 +121,13 @@ static int pp_init_gpu(struct pingpong_context *ctx, size_t _size, int cuda_devi
 		return 1;
 	}
 
-	printf("cuMemAlloc() of a %zd bytes GPU buffer\n", size);
-	CUdeviceptr d_A;
-	error = cuMemAlloc(&d_A, size);
-	if (error != CUDA_SUCCESS) {
-		printf("cuMemAlloc error=%d\n", error);
-		return 1;
-	}
-	printf("allocated GPU buffer address at %016llx pointer=%p\n", d_A,
-	       (void *) d_A);
-	ctx->buf[0] = (void*)d_A;
-
 	return 0;
 }
 
-static int pp_free_gpu(struct pingpong_context *ctx)
+static void pp_free_gpu(struct pingpong_context *ctx)
 {
-	int ret = 0;
-	CUdeviceptr d_A = (CUdeviceptr) ctx->buf[0];
-
-	printf("deallocating RX GPU buffer\n");
-	cuMemFree(d_A);
-	d_A = 0;
-
 	printf("destroying current CUDA Ctx\n");
 	CUCHECK(cuCtxDestroy(cuContext));
-
-	return ret;
 }
 #endif
 
@@ -1025,6 +1004,13 @@ int destroy_ctx(struct pingpong_context *ctx,
 
 	#ifdef HAVE_CUDA
 	if (user_param->use_cuda) {
+		for (i = 0; i < dereg_counter; i++) {
+			CUdeviceptr d_A = (CUdeviceptr)ctx->buf[i];
+
+			printf("deallocating RX GPU buffer %016llx\n", d_A);
+			cuMemFree(d_A);
+			d_A = 0;
+		}
 		pp_free_gpu(ctx);
 	}
 	else
@@ -1188,11 +1174,25 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
 
 	#ifdef HAVE_CUDA
 	if (user_param->use_cuda) {
+		CUdeviceptr d_A;
+		int error;
+		const size_t gpu_page_size = 64 * 1024;
+		size_t size = (ctx->buff_size + gpu_page_size - 1) &
+			~(gpu_page_size - 1);
+
 		ctx->is_contig_supported = FAILURE;
-		if(pp_init_gpu(ctx, ctx->buff_size, user_param->cuda_device_id)) {
-			fprintf(stderr, "Couldn't allocate work buf.\n");
+
+		printf("cuMemAlloc() of a %zd bytes GPU buffer\n",
+		       ctx->buff_size);
+		error = cuMemAlloc(&d_A, size);
+		if (error != CUDA_SUCCESS) {
+			printf("cuMemAlloc error=%d\n", error);
 			return FAILURE;
 		}
+
+		printf("allocated GPU buffer address at %016llx pointer=%p\n",
+		       d_A, (void *)d_A);
+		ctx->buf[qp_index] = (void *)d_A;
 	} else
 	#endif
 
@@ -1456,6 +1456,15 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 		return FAILURE;
 	}
 
+	#ifdef HAVE_CUDA
+	if (user_param->use_cuda) {
+		if (pp_init_gpu(ctx, user_param->cuda_device_id)) {
+			fprintf(stderr, "Couldn't init GPU context\n");
+			return FAILURE;
+		}
+	}
+	#endif
+
 	if (create_mr(ctx, user_param)) {
 		fprintf(stderr, "Failed to create MR\n");
 		return FAILURE;

From 562a3a1a6da55c5f877709358dbdbb64637fa896 Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Sat, 18 Apr 2020 18:05:04 +0300
Subject: [PATCH 37/41] Disabled RO for Broadwell and Haswell

Description:
	Relaxed Ordering lead to performance degradaion
	due to https://lore.kernel.org/patchwork/patch/820922/

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 configure.ac | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 91fd91c4..1b06fe82 100755
--- a/configure.ac
+++ b/configure.ac
@@ -29,7 +29,7 @@
 # SOFTWARE.
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.90],[linux-rdma@vger.kernel.org])
+AC_INIT([perftest],[5.91],[linux-rdma@vger.kernel.org])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
@@ -167,11 +167,23 @@ if [test $HAVE_SNIFFER = yes]; then
         AC_DEFINE([HAVE_SNIFFER], [1], [Enable Sniffer Flow Specification])
 fi
 
+CPU_IS_RO_COMPLIANT=yes
+# Actually this is check for being affected by a known issue
+# with Intel CPUs:
+# https://lore.kernel.org/patchwork/patch/820922/
+intel_no_ro_devices="6f01 6f02 6f03 6f04 6f05 6f06 6f07 6f08 6f09 6f0a 6f0b 6f0c 6f0d 6f0e 2f01 2f02 2f03 2f04 2f05 2f06 2f07 2f08 2f09 2f0a 2f0b 2f0c 2f0d 2f0e"
+for cpu_num in $intel_no_ro_devices; do
+	ro_nonsupported=$(lspci -nn | grep -i 8086:$cpu_num)
+	if [ test "$ro_nonsupported" ]; then
+		CPU_IS_RO_COMPLIANT=no
+	fi
+done
+
 AC_TRY_LINK([
 #include <infiniband/verbs.h>],
         [int x = IBV_ACCESS_RELAXED_ORDERING;],[HAVE_RO=yes], [HAVE_RO=no])
 AM_CONDITIONAL([HAVE_RO],[test "x$HAVE_RO" = "xyes"])
-if [test $HAVE_RO = yes]; then
+if [test $HAVE_RO = yes] && [test "x$CPU_IS_RO_COMPLIANT" = "xyes"]; then
         AC_DEFINE([HAVE_RO], [1], [Enable Relaxed Ordering])
 fi
 

From ba02cce5282b469ca9f4e3768e554feffa51072d Mon Sep 17 00:00:00 2001
From: Daniel Kranzdorf <dkkranzd@amazon.com>
Date: Thu, 15 Aug 2019 16:48:32 +0300
Subject: [PATCH 38/41] Add SRD support for new IBV send WR API

Add support for the extended QP send WR API, using Scalable Reliable Datagram
(SRD) connection type.

Signed-off-by: Daniel Kranzdorf <dkkranzd@amazon.com>
---
 configure.ac             |  10 +++-
 src/perftest_resources.c | 115 ++++++++++++++++++++++++++-------------
 2 files changed, 83 insertions(+), 42 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1b06fe82..69079813 100755
--- a/configure.ac
+++ b/configure.ac
@@ -229,11 +229,15 @@ if [test $HAVE_EXP_OOO_ATTR = yes]; then
 	AC_DEFINE([HAVE_EXP_OOO_ATTR], [1], [Have Experimental Out of order data placement support])
 fi
 
-AC_CHECK_LIB([efa], [efadv_create_driver_qp], [HAVE_SRD=yes LIBEFA=-lefa], [HAVE_SRD=no])
+if [test $HAVE_IBV_WR_API = yes]; then
+	AC_CHECK_LIB([efa], [efadv_create_qp_ex], [HAVE_SRD=yes], [HAVE_SRD=no])
+else
+	AC_CHECK_LIB([efa], [efadv_create_driver_qp], [HAVE_SRD=yes], [HAVE_SRD=no])
+fi
 AM_CONDITIONAL([HAVE_SRD], [test "x$HAVE_SRD" = "xyes"])
-if [test $HAVE_SRD = yes] && [test $HAVE_IBV_WR_API = no]; then
+if [test $HAVE_SRD = yes]; then
 	AC_DEFINE([HAVE_SRD], [1], [Have SRD support])
-	AC_SUBST([LIBEFA])
+	AC_SUBST([LIBEFA], [-lefa])
 fi
 
 AC_CHECK_LIB([mlx5], [mlx5dv_create_qp], [HAVE_MLX5DV_LIB=yes LIBMLX5=-lmlx5], [HAVE_MLX5DV_LIB=no])
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 3e3b6fd7..1d85e2e5 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -232,12 +232,13 @@ static void get_cpu_stats(struct perftest_parameters *duration_param,int stat_in
  *
  * Parameters :
  *
- *	ctx         - Test Context.
- *	user_param  - user_parameters struct for this test.
- *	inl         - use inline or not.
- *	index       - qp index.
- *	qpt         - qp type.
- *	op          - RDMA operation code.
+ *	ctx             - Test Context.
+ *	user_param      - user_parameters struct for this test.
+ *	inl             - use inline or not.
+ *	index           - qp index.
+ *	qpt             - qp type.
+ *	op              - RDMA operation code.
+ *	connection_type - Type of the connection.
  *
  * Return Value : int.
  *
@@ -245,11 +246,11 @@ static void get_cpu_stats(struct perftest_parameters *duration_param,int stat_in
 #ifdef HAVE_IBV_WR_API
 static inline int _new_post_send(struct pingpong_context *ctx,
 	struct perftest_parameters *user_param, int inl, int index,
-	enum ibv_qp_type qpt, enum ibv_wr_opcode op)
+	enum ibv_qp_type qpt, enum ibv_wr_opcode op, int connection_type)
 	__attribute__((always_inline));
 static inline int _new_post_send(struct pingpong_context *ctx,
 	struct perftest_parameters *user_param, int inl, int index,
-	enum ibv_qp_type qpt, enum ibv_wr_opcode op)
+	enum ibv_qp_type qpt, enum ibv_wr_opcode op, int connection_type)
 {
 	int rc;
 	int wr_index = index * user_param->post_list;
@@ -297,7 +298,7 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 			fprintf(stderr, "Post send failed: unknown operation code.\n");
 		}
 		#ifdef HAVE_MLX5DV
-		if (qpt == IBV_QPT_DRIVER)
+		if (qpt == IBV_QPT_DRIVER && connection_type == DC)
 		{
 			mlx5dv_wr_set_dc_addr(
 				ctx->dv_qp[index],
@@ -307,8 +308,8 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 		}
 		else
 		#endif
-		if (qpt == IBV_QPT_UD)
-		{
+		if (qpt == IBV_QPT_UD ||
+			(qpt == IBV_QPT_DRIVER && connection_type == SRD)) {
 			ibv_wr_set_ud_addr(
 				ctx->qpx[index],
 				wr->wr.ud.ah,
@@ -368,164 +369,176 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 static int new_post_write_sge_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE, DC);
 }
 
 static int new_post_write_inl_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_RDMA_WRITE, DC);
 }
 
 static int new_post_read_sge_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_READ);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_READ, DC);
 }
 
 static int new_post_send_sge_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_SEND, DC);
 }
 
 static int new_post_send_inl_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_SEND, DC);
 }
 
 static int new_post_atomic_fa_sge_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_FETCH_AND_ADD);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_FETCH_AND_ADD, DC);
 }
 
 static int new_post_atomic_cs_sge_dc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_CMP_AND_SWP);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_ATOMIC_CMP_AND_SWP, DC);
 }
 
 static int new_post_send_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_SEND, RC);
 }
 
 static int new_post_send_inl_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_RC, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_RC, IBV_WR_SEND, RC);
 }
 
 static int new_post_write_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_RDMA_WRITE, RC);
 }
 
 static int new_post_write_inl_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_RC, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_RC, IBV_WR_RDMA_WRITE, RC);
 }
 
 static int new_post_read_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_RDMA_READ);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_RDMA_READ, RC);
 }
 
 static int new_post_atomic_fa_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_ATOMIC_FETCH_AND_ADD);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_ATOMIC_FETCH_AND_ADD, RC);
 }
 
 static int new_post_atomic_cs_sge_rc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_ATOMIC_CMP_AND_SWP);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_RC, IBV_WR_ATOMIC_CMP_AND_SWP, RC);
 }
 
 static int new_post_send_sge_ud(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UD, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UD, IBV_WR_SEND, UD);
 }
 
 static int new_post_send_inl_ud(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UD, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UD, IBV_WR_SEND, UD);
 }
 
 static int new_post_send_sge_uc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UC, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UC, IBV_WR_SEND, UC);
 }
 
 static int new_post_send_inl_uc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UC, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UC, IBV_WR_SEND, UC);
 }
 
 static int new_post_write_sge_uc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UC, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_UC, IBV_WR_RDMA_WRITE, UC);
 }
 
 static int new_post_write_inl_uc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UC, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_UC, IBV_WR_RDMA_WRITE, UC);
+}
+
+static int new_post_send_sge_srd(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_SEND, SRD);
+}
+
+static int new_post_send_inl_srd(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_SEND, SRD);
 }
 
 #ifdef HAVE_XRCD
 static int new_post_send_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_SEND, XRC);
 }
 
 static int new_post_send_inl_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_XRC_SEND, IBV_WR_SEND);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_XRC_SEND, IBV_WR_SEND, XRC);
 }
 
 static int new_post_write_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_WRITE, XRC);
 }
 
 static int new_post_write_inl_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_WRITE);
+	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_WRITE, XRC);
 }
 
 static int new_post_read_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_READ);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_RDMA_READ, XRC);
 }
 
 static int new_post_atomic_fa_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_ATOMIC_FETCH_AND_ADD);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_ATOMIC_FETCH_AND_ADD, XRC);
 }
 
 static int new_post_atomic_cs_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
 {
-	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_ATOMIC_CMP_AND_SWP);
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_XRC_SEND, IBV_WR_ATOMIC_CMP_AND_SWP, XRC);
 }
 #endif
 #endif
@@ -1620,6 +1633,9 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 	#endif
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
 	memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex));
+	#ifdef HAVE_SRD
+	struct efadv_qp_init_attr efa_attr = {};
+	#endif
 	#else
 	struct ibv_qp_init_attr attr;
 	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
@@ -1733,7 +1749,14 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
 
 	} else if (user_param->connection_type == SRD) {
 		#ifdef HAVE_SRD
-		qp = efadv_create_driver_qp(ctx->pd, &attr, EFADV_QP_DRIVER_TYPE_SRD);
+		#ifdef HAVE_IBV_WR_API
+		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
+		qp = efadv_create_qp_ex(ctx->context, &attr_ex,
+					&efa_attr, sizeof(efa_attr));
+		#else
+		qp = efadv_create_driver_qp(ctx->pd, &attr,
+					    EFADV_QP_DRIVER_TYPE_SRD);
+		#endif
 		#endif
 	} else {
 		#ifdef HAVE_IBV_WR_API
@@ -2254,6 +2277,20 @@ static void ctx_post_send_work_request_func_pointer(struct pingpong_context *ctx
 				fprintf(stderr, "The post send properties are not supported on RC. \n");
 		}
 		break;
+	case SRD:
+		switch (user_param->verb) {
+			case SEND:
+				if (use_inl) {
+					ctx->new_post_send_work_request_func_pointer = &new_post_send_inl_srd;
+				}
+				else {
+					ctx->new_post_send_work_request_func_pointer = &new_post_send_sge_srd;
+				}
+				break;
+			default:
+				fprintf(stderr, "The post send properties are not supported on SRD.\n");
+		}
+		break;
 	default:
 		fprintf(stderr, "Unsupported transport. \n");
 	}

From 4421857fc74610e6c72784fa30b89878765e9d5d Mon Sep 17 00:00:00 2001
From: Daniel Kranzdorf <dkkranzd@amazon.com>
Date: Thu, 2 Jan 2020 11:28:08 +0200
Subject: [PATCH 39/41] Forbid messages larger than SRD max msg size

Fail user requested message size if it is greater than max supported with SRD.

Signed-off-by: Daniel Kranzdorf <dkkranzd@amazon.com>
---
 src/perftest_communication.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index 3ee76390..dd19c3ac 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -1791,11 +1791,16 @@ int check_mtu(struct ibv_context *context,struct perftest_parameters *user_param
 		}
 
 		if (user_param->size > port_attr.max_msg_sz) {
-			if (user_param->test_method == RUN_ALL) {
-				fprintf(stderr, " Max msg size is %u\n", port_attr.max_msg_sz);
+			if (user_param->test_method == RUN_ALL || !user_param->req_size) {
+				fprintf(stderr, " Max msg size is %u\n",
+					port_attr.max_msg_sz);
 				fprintf(stderr, " Changing to this size\n");
+				user_param->size = port_attr.max_msg_sz;
+			} else {
+				fprintf(stderr," Max message size in SRD cannot be greater than %u \n",
+					port_attr.max_msg_sz);
+				return FAILURE;
 			}
-			user_param->size = port_attr.max_msg_sz;
 		}
 	}
 

From e9a7d96cf8214bb26d41bedf096aedd917377a63 Mon Sep 17 00:00:00 2001
From: Daniel Kranzdorf <dkkranzd@amazon.com>
Date: Sun, 28 Jul 2019 14:04:45 +0300
Subject: [PATCH 40/41] Add SRD support for RDMA read

Support posting new RDMA read Work Requests over SRD connection, using the
extended QP send WR API.  Since the WR cannot contain both rdma and ud data,
get the ah and remote qpn from the test context instead.

Signed-off-by: Daniel Kranzdorf <dkkranzd@amazon.com>
---
 configure.ac                 |  5 +++
 src/perftest_communication.c | 63 +++++++++++++++++++++++++++---------
 src/perftest_parameters.c    |  4 +--
 src/perftest_resources.c     | 42 +++++++++++++++++-------
 src/perftest_resources.h     |  1 +
 5 files changed, 87 insertions(+), 28 deletions(-)

diff --git a/configure.ac b/configure.ac
index 69079813..bdecf4e1 100755
--- a/configure.ac
+++ b/configure.ac
@@ -231,6 +231,11 @@ fi
 
 if [test $HAVE_IBV_WR_API = yes]; then
 	AC_CHECK_LIB([efa], [efadv_create_qp_ex], [HAVE_SRD=yes], [HAVE_SRD=no])
+	AC_TRY_LINK([#include <infiniband/efadv.h>],
+		    [int x = EFADV_DEVICE_ATTR_CAPS_RDMA_READ;], [HAVE_RDMA_READ_SRD=yes], [HAVE_RDMA_READ_SRD=no])
+	if [test $HAVE_RDMA_READ_SRD = yes]; then
+		AC_DEFINE([HAVE_SRD_WITH_RDMA_READ], [1], [Have SRD with RDMA read support])
+	fi
 else
 	AC_CHECK_LIB([efa], [efadv_create_driver_qp], [HAVE_SRD=yes], [HAVE_SRD=no])
 fi
diff --git a/src/perftest_communication.c b/src/perftest_communication.c
index dd19c3ac..693a1e49 100755
--- a/src/perftest_communication.c
+++ b/src/perftest_communication.c
@@ -15,6 +15,9 @@
 #include <sys/socket.h>
 #include <netdb.h>
 #include "perftest_communication.h"
+#ifdef HAVE_SRD
+#include <infiniband/efadv.h>
+#endif
 #if defined(__FreeBSD__)
 #include <ctype.h>
 #endif
@@ -1783,24 +1786,54 @@ int check_mtu(struct ibv_context *context,struct perftest_parameters *user_param
 			user_param->size = RAWETH_MIN_MSG_SIZE;
 		}
 	} else if (user_param->connection_type == SRD) {
-		struct ibv_port_attr port_attr;
+		if (user_param->verb == SEND) {
+			struct ibv_port_attr port_attr;
 
-		if (ibv_query_port(context, user_param->ib_port, &port_attr)) {
-			fprintf(stderr, " Error when trying to query port\n");
-			exit(1);
-		}
+			if (ibv_query_port(context, user_param->ib_port, &port_attr)) {
+				fprintf(stderr, " Error when trying to query port\n");
+				exit(1);
+			}
 
-		if (user_param->size > port_attr.max_msg_sz) {
-			if (user_param->test_method == RUN_ALL || !user_param->req_size) {
-				fprintf(stderr, " Max msg size is %u\n",
-					port_attr.max_msg_sz);
-				fprintf(stderr, " Changing to this size\n");
-				user_param->size = port_attr.max_msg_sz;
-			} else {
-				fprintf(stderr," Max message size in SRD cannot be greater than %u \n",
-					port_attr.max_msg_sz);
-				return FAILURE;
+			if (user_param->size > port_attr.max_msg_sz) {
+				if (user_param->test_method == RUN_ALL || !user_param->req_size) {
+					fprintf(stderr, " Max msg size is %u\n",
+						port_attr.max_msg_sz);
+					fprintf(stderr, " Changing to this size\n");
+					user_param->size = port_attr.max_msg_sz;
+				} else {
+					fprintf(stderr," Max message size in SRD cannot be greater than %u \n",
+						port_attr.max_msg_sz);
+					return FAILURE;
+				}
 			}
+		} else if (user_param->verb == READ) {
+#ifdef HAVE_SRD_WITH_RDMA_READ
+			struct efadv_device_attr efa_device_attr = {};
+
+			if (efadv_query_device(context, &efa_device_attr, sizeof(efa_device_attr))) {
+				fprintf(stderr, " Error when trying to query EFA device\n");
+				exit(1);
+			}
+			if (!(efa_device_attr.device_caps & EFADV_DEVICE_ATTR_CAPS_RDMA_READ)) {
+				fprintf(stderr, "Read verb is not supported with this EFA device\n");
+				exit(1);
+			}
+			if (user_param->size > efa_device_attr.max_rdma_size) {
+				if (user_param->test_method == RUN_ALL || !user_param->req_size) {
+					fprintf(stderr, " Max RDMA request size is %u\n",
+						efa_device_attr.max_rdma_size);
+					fprintf(stderr, " Changing to this size\n");
+					user_param->size = efa_device_attr.max_rdma_size;
+				} else {
+					fprintf(stderr," Max RDMA read size in SRD cannot be greater than %u\n",
+						efa_device_attr.max_rdma_size);
+					return FAILURE;
+				}
+			}
+#else
+			fprintf(stderr, "SRD connection not possible in READ verb\n");
+			exit(1);
+#endif
 		}
 	}
 
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index add08aa1..4b81a710 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -801,8 +801,8 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
 		#endif
 	} else if (strcmp(connStr[6], optarg) == 0) {
 		#ifdef HAVE_SRD
-		if (verb != SEND) {
-			fprintf(stderr, " SRD connection only possible in SEND verb\n");
+		if (verb != SEND && verb != READ ) {
+			fprintf(stderr, " SRD connection only possible in SEND/READ verbs\n");
 			exit(1);
 		}
 		*cptr = SRD;
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
index 1d85e2e5..5a209179 100755
--- a/src/perftest_resources.c
+++ b/src/perftest_resources.c
@@ -308,13 +308,18 @@ static inline int _new_post_send(struct pingpong_context *ctx,
 		}
 		else
 		#endif
-		if (qpt == IBV_QPT_UD ||
-			(qpt == IBV_QPT_DRIVER && connection_type == SRD)) {
+		if (qpt == IBV_QPT_UD) {
 			ibv_wr_set_ud_addr(
 				ctx->qpx[index],
 				wr->wr.ud.ah,
 				wr->wr.ud.remote_qpn,
 				wr->wr.ud.remote_qkey);
+		} else if (qpt == IBV_QPT_DRIVER && connection_type == SRD) {
+			ibv_wr_set_ud_addr(
+				ctx->qpx[index],
+				ctx->ah[index],
+				ctx->rem_qpn[index],
+				DEF_QKEY);
 		}
 
 		#ifdef HAVE_XRCD
@@ -498,6 +503,12 @@ static int new_post_send_inl_srd(struct pingpong_context *ctx, int index,
 	return _new_post_send(ctx, user_param, 1, index, IBV_QPT_DRIVER, IBV_WR_SEND, SRD);
 }
 
+static int new_post_read_sge_srd(struct pingpong_context *ctx, int index,
+	struct perftest_parameters *user_param)
+{
+	return _new_post_send(ctx, user_param, 0, index, IBV_QPT_DRIVER, IBV_WR_RDMA_READ, SRD);
+}
+
 #ifdef HAVE_XRCD
 static int new_post_send_sge_xrc(struct pingpong_context *ctx, int index,
 	struct perftest_parameters *user_param)
@@ -836,10 +847,13 @@ void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_par
 
 		ALLOCATE(ctx->sge_list, struct ibv_sge,user_param->num_of_qps * user_param->post_list);
 		ALLOCATE(ctx->wr, struct ibv_send_wr, user_param->num_of_qps * user_param->post_list);
+		ALLOCATE(ctx->rem_qpn, uint32_t, user_param->num_of_qps);
 		if ((user_param->verb == SEND && user_param->connection_type == UD) ||
 				user_param->connection_type == DC || user_param->connection_type == SRD) {
 			ALLOCATE(ctx->ah, struct ibv_ah*, user_param->num_of_qps);
 		}
+	} else if (user_param->verb == READ && user_param->connection_type == SRD) {
+		ALLOCATE(ctx->ah, struct ibv_ah*, user_param->num_of_qps);
 	}
 
 	if (user_param->verb == SEND && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex)) {
@@ -912,10 +926,11 @@ int destroy_ctx(struct pingpong_context *ctx,
 		first = 0;
 	for (i = first; i < user_param->num_of_qps; i++) {
 
-		if (((user_param->connection_type == DC && !((!(user_param->duplex || user_param->tst == LAT) && user_param->machine == SERVER)
+		if ((((user_param->connection_type == DC && !((!(user_param->duplex || user_param->tst == LAT) && user_param->machine == SERVER)
 							|| ((user_param->duplex || user_param->tst == LAT) && i >= num_of_qps))) ||
 					user_param->connection_type == UD || user_param->connection_type == SRD) &&
-				(user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex)) {
+				(user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex)) ||
+				(user_param->connection_type == SRD && user_param->verb == READ)) {
 			if (ibv_destroy_ah(ctx->ah[i])) {
 				fprintf(stderr, "Failed to destroy AH\n");
 				test_result = 1;
@@ -2065,8 +2080,9 @@ int ctx_connect(struct pingpong_context *ctx,
 			}
 		}
 
-		if ((user_param->connection_type == UD || user_param->connection_type == DC || user_param->connection_type == SRD) &&
-				(user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex)) {
+		if (((user_param->connection_type == UD || user_param->connection_type == DC || user_param->connection_type == SRD) &&
+				(user_param->tst == LAT || user_param->machine == CLIENT || user_param->duplex)) ||
+				(user_param->connection_type == SRD && user_param->verb == READ)) {
 
 			ctx->ah[i] = ibv_create_ah(ctx->pd,&(attr.ah_attr));
 
@@ -2287,6 +2303,9 @@ static void ctx_post_send_work_request_func_pointer(struct pingpong_context *ctx
 					ctx->new_post_send_work_request_func_pointer = &new_post_send_sge_srd;
 				}
 				break;
+			case READ:
+				ctx->new_post_send_work_request_func_pointer = &new_post_read_sge_srd;
+				break;
 			default:
 				fprintf(stderr, "The post send properties are not supported on SRD.\n");
 		}
@@ -2307,7 +2326,7 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 	int i,j;
 	int num_of_qps = user_param->num_of_qps;
 	int xrc_offset = 0;
-	uint32_t remote_qpn, remote_qkey;
+	uint32_t remote_qkey;
 
 	if((user_param->use_xrc || user_param->connection_type == DC) && (user_param->duplex || user_param->tst == LAT)) {
 		num_of_qps /= 2;
@@ -2387,7 +2406,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 			if (user_param->verb == WRITE || user_param->verb == READ) {
 
 				ctx->wr[i*user_param->post_list + j].wr.rdma.rkey = rem_dest[xrc_offset + i].rkey;
-
+				if (user_param->connection_type == SRD)
+					ctx->rem_qpn[xrc_offset + i] = rem_dest[xrc_offset + i].qpn;
 				if (j > 0) {
 
 					ctx->wr[i*user_param->post_list + j].wr.rdma.remote_addr =
@@ -2424,14 +2444,14 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
 
 					ctx->wr[i*user_param->post_list + j].wr.ud.ah = ctx->ah[i];
 					if (user_param->work_rdma_cm) {
-						remote_qpn = ctx->cma_master.nodes[i].remote_qpn;
+						ctx->rem_qpn[xrc_offset + i] = ctx->cma_master.nodes[i].remote_qpn;
 						remote_qkey = ctx->cma_master.nodes[i].remote_qkey;
 					} else {
-						remote_qpn = rem_dest[xrc_offset + i].qpn;
+						ctx->rem_qpn[xrc_offset + i] = rem_dest[xrc_offset + i].qpn;
 						remote_qkey = DEF_QKEY;
 					}
 					ctx->wr[i*user_param->post_list + j].wr.ud.remote_qkey = remote_qkey;
-					ctx->wr[i*user_param->post_list + j].wr.ud.remote_qpn = remote_qpn;
+					ctx->wr[i*user_param->post_list + j].wr.ud.remote_qpn = ctx->rem_qpn[xrc_offset + i];
 				}
 			}
 
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
index 2c8160ba..cf18bee7 100755
--- a/src/perftest_resources.h
+++ b/src/perftest_resources.h
@@ -185,6 +185,7 @@ struct pingpong_context {
 	uint64_t				*my_addr;
 	uint64_t				*rx_buffer_addr;
 	uint64_t				*rem_addr;
+	uint32_t 				*rem_qpn;
 	uint64_t				buff_size;
 	uint64_t				send_qp_buff_size;
 	uint64_t				flow_buff_size;

From 54b25983d433bc16a96159c76fff710ba94f67ed Mon Sep 17 00:00:00 2001
From: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
Date: Thu, 30 Apr 2020 18:18:06 +0300
Subject: [PATCH 41/41] Fix typo in the calculation of the factor between MB/s
 and Gb/s

Signed-off-by: Dmitry Akhmedzhanov <dmitryak@mellanox.com>
---
 src/perftest_parameters.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 4b81a710..57239031 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -447,7 +447,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		printf("      --report_gbits ");
 		printf(" Report Max/Average BW of test in Gbit/sec (instead of MB/sec)\n");
 		printf("        Note: MB=2^20 byte, while Gb=10^9 bits. Use these formulas for conversion:\n");
-		printf("        Factor=10^9/(20^2*8)=119.2; MB=Gb_result * factor; Gb=MB_result / factor\n");
+		printf("        Factor=10^9/(2^20*8)=119.2; MB=Gb_result * factor; Gb=MB_result / factor\n");
 
 		if (connection_type != RawEth) {
 			printf("      --report-per-port ");