enhance code for soft metering

include/dp_log.h

@@ -70,6 +70,8 @@ extern "C" {
 #define DP_LOG_MINPORT(VALUE) _DP_LOG_UINT("minport", VALUE)
 #define DP_LOG_MAXPORT(VALUE) _DP_LOG_UINT("maxport", VALUE)
 #define DP_LOG_FLOW_ERROR(VALUE) _DP_LOG_STR("flow_error", VALUE)
+#define DP_LOG_METER_TOTAL(VALUE) _DP_LOG_UINT("meter_total", VALUE)
+#define DP_LOG_METER_PUBLIC(VALUE) _DP_LOG_UINT("meter_public", VALUE)
 // gRPC worker I/O
 #define DP_LOG_GRPCRET(VALUE) _DP_LOG_INT("grpc_error", VALUE), _DP_LOG_STR("grpc_message", dp_grpc_strerror(VALUE))
 #define DP_LOG_GRPCREQUEST(VALUE) _DP_LOG_INT("grpc_request", VALUE)

include/dp_port.h

@@ -41,6 +41,8 @@ struct dp_port_iface {
 	uint32_t				nat_ip;
 	uint16_t				nat_port_range[2];
 	bool					ready;
+	uint64_t				total_flow_rate_cap;
+	uint64_t				public_flow_rate_cap;
 };
 
 struct dp_port {
@@ -61,7 +63,6 @@ struct dp_port {
 	struct rte_flow					*default_capture_flow;
 	bool							captured;
 	struct dp_port_stats			stats;
-	bool							soft_metering_enabled;
 	struct rte_meter_srtcm			port_srtcm;
 	struct rte_meter_srtcm_profile	port_srtcm_profile;
 };

proto/dpdk.proto

@@ -71,6 +71,7 @@ message Interface {
 	bytes primary_ipv6 = 4;
 	bytes underlay_route = 5;
 	string pci_name = 6;
+	MeteringParams meteringParams = 7;
 }
 
 message IpConfig {

src/dp_port.c

@@ -22,8 +22,8 @@
 #define DP_PORT_INIT_PF true
 #define DP_PORT_INIT_VF false
 
-#define DP_METER_CIR_BASE_VALUE  1024 * 1024 // 1 Mbits
-#define DP_METER_EBS_BREAK_VALUE 100     // 100 Mbits/s
+#define DP_METER_CIR_BASE_VALUE  (1024 * 1024) // 1 Mbits
+#define DP_METER_EBS_BREAK_VALUE 100     // 100 Mbits/s, it used to differentiate different ebs calculation strategy to achieve relative stable metering results. epirical value.
 #define DP_METER_MBITS_TO_BYTES  (1024 * 1024 / 8)
 
 static const struct rte_eth_conf port_conf_default = {
@@ -479,6 +479,11 @@ static int dp_port_public_flow_meter_config(struct dp_port *port, uint64_t publi
 	struct rte_meter_srtcm_params srtcm_params = dp_srtcm_params_base;
 	int ret;
 
+	if (!public_flow_rate_cap) {
+		port->port_srtcm_profile = (struct rte_meter_srtcm_profile){0}; // reset the srtcm profile to 0 to erase any value inside it
+		return DP_OK;
+	}
+
 	srtcm_params.cir = DP_METER_CIR_BASE_VALUE * (public_flow_rate_cap / 8); // Mbits/s -> bytes/s
 	if (public_flow_rate_cap < DP_METER_EBS_BREAK_VALUE)
 		srtcm_params.ebs = public_flow_rate_cap * DP_METER_MBITS_TO_BYTES;
@@ -492,6 +497,7 @@ static int dp_port_public_flow_meter_config(struct dp_port *port, uint64_t publi
 	ret = rte_meter_srtcm_config(&port->port_srtcm, &port->port_srtcm_profile);
 	if (DP_FAILED(ret)) {
 		DPS_LOG_ERR("Cannot configure meter", DP_LOG_PORT(port), DP_LOG_RET(ret));
+		port->port_srtcm_profile = (struct rte_meter_srtcm_profile){0}; // reset the srtcm profile to 0 to erase the above value assignment
 		return DP_ERROR;
 	}
 
@@ -501,30 +507,27 @@ static int dp_port_public_flow_meter_config(struct dp_port *port, uint64_t publi
 int dp_port_meter_config(struct dp_port *port, uint64_t total_flow_rate_cap, uint64_t public_flow_rate_cap)
 {
 	if (public_flow_rate_cap > total_flow_rate_cap) {
-		DPS_LOG_ERR("Public flow rate cap cannot be greater than total flow rate cap", DP_LOG_PORT(port));
+		DPS_LOG_ERR("Public flow rate cap cannot be greater than total flow rate cap",
+					DP_LOG_PORT(port), DP_LOG_METER_TOTAL(total_flow_rate_cap), DP_LOG_METER_PUBLIC(public_flow_rate_cap));
 		return DP_ERROR;
 	}
 
 	if (DP_FAILED(dp_port_total_flow_meter_config(port, total_flow_rate_cap))) {
 		DPS_LOG_ERR("Cannot set total flow meter", DP_LOG_PORT(port));
 		return DP_ERROR;
 	}
-
-	if (public_flow_rate_cap == 0) {
-		port->soft_metering_enabled = false;
-		return DP_OK;
-	}
+	port->iface.total_flow_rate_cap = total_flow_rate_cap;
 
 	if (DP_FAILED(dp_port_public_flow_meter_config(port, public_flow_rate_cap))) {
 		DPS_LOG_ERR("Cannot set public flow meter", DP_LOG_PORT(port));
 		if (DP_FAILED(dp_port_total_flow_meter_config(port, 0))) {
 			DPS_LOG_ERR("Cannot reset total flow meter", DP_LOG_PORT(port));
 			return DP_ERROR;
 		}
+		port->iface.total_flow_rate_cap = 0;
 		return DP_ERROR;
 	}
-
-	port->soft_metering_enabled = true;
+	port->iface.public_flow_rate_cap = public_flow_rate_cap;
 
 	return DP_OK;
 }

src/dp_util.c

@@ -44,15 +44,16 @@ static int get_num_of_vfs_sriov(void)
 	int vfs;
 	char filename[DP_SYSFS_MAX_PATH];
 	FILE *fp;
+	int res;
 
-	if (snprintf(filename, sizeof(filename),
+	res = snprintf(filename, sizeof(filename),
 				 "%s%s%s",
 				 DP_SYSFS_PREFIX_MLX_VF_COUNT,
 				 dp_conf_get_pf0_name(),
-				 DP_SYSFS_SUFFIX_MLX_VF_COUNT)
-			>= (int)sizeof(filename)
-	) {
-		DPS_LOG_ERR("SR-IOV sysfs path to number of VFs is too long");
+				 DP_SYSFS_SUFFIX_MLX_VF_COUNT);
+
+	if (res < 0 || (unsigned)res >= sizeof(filename)) {
+		DPS_LOG_ERR("Faled to generate SR-IOV sysfs path or SR-IOV sysfs path to number of VFs is too long");
 		return DP_ERROR;
 	}
 
@@ -175,25 +176,24 @@ int dp_set_vf_rate_limit(uint16_t port_id, uint64_t rate)
 	FILE *fp;
 	struct dp_port *port = dp_get_port_by_id(port_id);
 	uint64_t rate_in_mbits = rate;
+	int res;
 
-	if (!port) {
-		DPS_LOG_ERR("Cannot get port by id", DP_LOG_PORTID(port_id));
+	if (!port)
 		return DP_ERROR;
-	}
 
 	while (*(pattern + vf_pattern_len) != '\0')
 		vf_pattern_len++;
 
-	if (snprintf(filename, sizeof(filename),
+	res = snprintf(filename, sizeof(filename),
 				"%s%s%s%s%s",
 				DP_SYSFS_PREFIX_MLX_DEVICE,
 				dp_conf_get_pf0_name(),
 				DP_SYSFS_PREFIX_MLX_MAX_TX_RATE,
 				port->vf_name + vf_pattern_len,
-				DP_SYSFS_SUFFIX_MLX_MAX_TX_RATE)
-			>= (int)sizeof(filename)
-	) {
-		DPS_LOG_ERR("SR-IOV sysfs path to vf's max tx rate is too long");
+				DP_SYSFS_SUFFIX_MLX_MAX_TX_RATE);
+
+	if (res < 0 || (unsigned)res >= (int)sizeof(filename)) {
+		DPS_LOG_ERR("Failed to generate SR-IOV sysfs path or SR-IOV sysfs path to vf's max tx rate is too long");
 		return DP_ERROR;
 	}
 

src/grpc/dp_async_grpc.cpp

@@ -225,7 +225,9 @@ const char* CreateInterfaceCall::FillRequest(struct dpgrpc_request* request)
 					DP_LOG_VNI(request_.vni()),
 					DP_LOG_IPV4STR(request_.ipv4_config().primary_address().c_str()),
 					DP_LOG_IPV6STR(request_.ipv6_config().primary_address().c_str()),
-					DP_LOG_PCI(request_.device_name().c_str()));
+					DP_LOG_PCI(request_.device_name().c_str()),
+					DP_LOG_METER_TOTAL(request_.metering_parameters().total_rate()),
+					DP_LOG_METER_PUBLIC(request_.metering_parameters().public_rate()));
 	if (!GrpcConv::IsInterfaceIdValid(request_.interface_id()))
 		return "Invalid interface_id";
 	request->add_iface.vni = request_.vni();

src/grpc/dp_grpc_conv.cpp

@@ -181,6 +181,7 @@ bool Ipv6PrefixLenToMask(uint32_t prefix_length, uint8_t *mask) {
 void DpToGrpcInterface(const struct dpgrpc_iface *dp_iface, Interface *grpc_iface)
 {
 	char strbuf[INET6_ADDRSTRLEN];
+	MeteringParams *metering_params;
 
 	grpc_iface->set_primary_ipv4(GrpcConv::Ipv4ToStr(dp_iface->ip4_addr));
 	inet_ntop(AF_INET6, dp_iface->ip6_addr, strbuf, sizeof(strbuf));
@@ -190,6 +191,10 @@ void DpToGrpcInterface(const struct dpgrpc_iface *dp_iface, Interface *grpc_ifac
 	grpc_iface->set_pci_name(dp_iface->pci_name);
 	inet_ntop(AF_INET6, dp_iface->ul_addr6, strbuf, sizeof(strbuf));
 	grpc_iface->set_underlay_route(strbuf);
+	metering_params = new MeteringParams();
+	metering_params->set_total_rate(dp_iface->total_flow_rate_cap);
+	metering_params->set_public_rate(dp_iface->public_flow_rate_cap);
+	grpc_iface->set_allocated_meteringparams(metering_params);
 }
 
 uint32_t DpIpv6MaskToPrefixLen(const uint8_t *mask)

src/grpc/dp_grpc_impl.c

@@ -502,7 +502,7 @@ static int dp_process_create_interface(struct dp_grpc_responder *responder)
 		goto route6_err;
 	}
 
-	if (!port->is_pf)
+	if (!port->is_pf && (request->total_flow_rate_cap || request->public_flow_rate_cap))
 		if (DP_FAILED(dp_port_meter_config(port, request->total_flow_rate_cap, request->public_flow_rate_cap))) {
 			ret = DP_GRPC_ERR_PORT_METER;
 			goto err;
@@ -542,6 +542,11 @@ static int dp_process_delete_interface(struct dp_grpc_responder *responder)
 	ipv4 = port->iface.cfg.own_ip;
 	vni = port->iface.vni;
 
+	// avoid reseting global metering via sysfs for tap devices
+	if (!port->is_pf && dp_conf_get_nic_type() != DP_CONF_NIC_TYPE_TAP)
+		if (DP_FAILED(dp_port_meter_config(port, 0, 0)))
+			ret = DP_GRPC_ERR_PORT_METER;
+
 	dp_del_vnf(port->iface.ul_ipv6);
 	if (DP_FAILED(dp_stop_port(port)))
 		ret = DP_GRPC_ERR_PORT_STOP;
@@ -553,10 +558,6 @@ static int dp_process_delete_interface(struct dp_grpc_responder *responder)
 #endif
 	dp_remove_iface_flows(port->port_id, ipv4, vni);
 
-	if (!port->is_pf)
-		if (DP_FAILED(dp_port_meter_config(port, 0, 0)))
-			ret = DP_GRPC_ERR_PORT_METER;
-
 	return ret;
 }
 
@@ -579,6 +580,8 @@ static int dp_process_get_interface(struct dp_grpc_responder *responder)
 	static_assert(sizeof(reply->pci_name) == sizeof(port->dev_name), "Incompatible PCI name size");
 	rte_memcpy(reply->pci_name, port->dev_name, sizeof(reply->pci_name));
 	rte_memcpy(reply->ul_addr6, port->iface.ul_ipv6, sizeof(reply->ul_addr6));
+	reply->total_flow_rate_cap = port->iface.total_flow_rate_cap;
+	reply->public_flow_rate_cap = port->iface.public_flow_rate_cap;
 	return DP_GRPC_OK;
 }
 
@@ -796,6 +799,8 @@ static int dp_process_list_interfaces(struct dp_grpc_responder *responder)
 		static_assert(sizeof(reply->pci_name) == sizeof(port->dev_name), "Incompatible PCI name size");
 		rte_memcpy(reply->pci_name, port->dev_name, sizeof(reply->pci_name));
 		rte_memcpy(reply->ul_addr6, port->iface.ul_ipv6, sizeof(reply->ul_addr6));
+		reply->total_flow_rate_cap = port->iface.total_flow_rate_cap;
+		reply->public_flow_rate_cap = port->iface.public_flow_rate_cap;
 	}
 
 	return DP_GRPC_OK;

src/nodes/snat_node.c

@@ -153,7 +153,7 @@ static __rte_always_inline rte_edge_t get_next_index(__rte_unused struct rte_nod
 	struct dp_port *in_port = dp_get_in_port(m);
 	enum rte_color color;
 
-	if (!in_port->is_pf && in_port->soft_metering_enabled && df->flow_type == DP_FLOW_SOUTH_NORTH
+	if (!in_port->is_pf && in_port->iface.public_flow_rate_cap && df->flow_type == DP_FLOW_SOUTH_NORTH
 		&& (df->l3_type == RTE_ETHER_TYPE_IPV4 || df->l3_type == RTE_ETHER_TYPE_IPV6)) {
 		color = rte_meter_srtcm_color_blind_check(&in_port->port_srtcm, &in_port->port_srtcm_profile, rte_rdtsc(), df->l3_payload_length);
 		if (color == RTE_COLOR_RED)