Merge branch 'ntop:dev' into dev

http_src/utilities/datatable/sprymedia-datatable-utils.js

@@ -813,7 +813,6 @@ export class DataTableRenders {
             labelVlan = `${label}@${flow.vlan.label}`;
             titleVlan = `${title}@${flow.vlan.title}`;
         }
-        labelVlan = NtopUtils.shortenLabel(labelVlan, 16, ".")
         return DataTableRenders.filterize(key, valueVlan, labelVlan, labelVlan, titleVlan);
     }
 

include/InterfaceStatsHash.h

@@ -37,7 +37,7 @@ class InterfaceStatsHash {
   bool set(const sFlowInterfaceStats *const stats);
 
   void luaDeviceList(lua_State *vm);
-  void luaDeviceInfo(lua_State *vm, u_int32_t deviceIP);
+  void luaDeviceInfo(lua_State *vm, u_int32_t deviceID);
 };
 
 #endif /* _INTERFACE_STATS_HASH_H_ */

include/NetworkInterface.h

@@ -997,20 +997,29 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
   void updateBehaviorStats(const struct timeval *tv);
 
   virtual void getFlowDevices(lua_State *vm);
-  virtual void getFlowDeviceInfo(lua_State *vm, u_int32_t deviceIP, bool showAllStats = true) {
+  virtual void getFlowDeviceInfo(lua_State *vm, u_int32_t deviceID, bool showAllStats = true) {
     if (flow_interfaces_stats) {
       lua_newtable(vm);
-      flow_interfaces_stats->luaDeviceInfo(vm, deviceIP, this, showAllStats);
+      flow_interfaces_stats->luaDeviceInfo(vm, deviceID, this, showAllStats);
+      lua_pushinteger(vm, get_id());
+      lua_insert(vm, -2);
+      lua_settable(vm, -3);
+    }
+  };  
+  virtual void getFlowDeviceInfoByIP(lua_State *vm, u_int32_t deviceIP, bool showAllStats = true) {
+    if (flow_interfaces_stats) {
+      lua_newtable(vm);
+      flow_interfaces_stats->luaDeviceInfoByIP(vm, deviceIP, this, showAllStats);
       lua_pushinteger(vm, get_id());
       lua_insert(vm, -2);
       lua_settable(vm, -3);
     }
   };
 #endif
   virtual void getSFlowDevices(lua_State *vm, bool add_table);
-  virtual void getSFlowDeviceInfo(lua_State *vm, u_int32_t deviceIP) {
+  virtual void getSFlowDeviceInfo(lua_State *vm, u_int32_t deviceID) {
     if (interfaceStats)
-      interfaceStats->luaDeviceInfo(vm, deviceIP);
+      interfaceStats->luaDeviceInfo(vm, deviceID);
     else
       lua_newtable(vm);
   };

include/ViewInterface.h

@@ -94,6 +94,7 @@ class ViewInterface : public NetworkInterface {
 #ifdef NTOPNG_PRO
   virtual void getFlowDevices(lua_State *vm);
   virtual void getFlowDeviceInfo(lua_State *vm, u_int32_t deviceIP, bool showAllStats);
+  virtual void getFlowDeviceInfoByIP(lua_State *vm, u_int32_t deviceIP, bool showAllStats);
 #endif
   virtual void getSFlowDevices(lua_State *vm, bool add_table);
   virtual void getSFlowDeviceInfo(lua_State *vm, u_int32_t deviceIP);

scripts/historical/tables/top_vlans.json

@@ -0,0 +1,54 @@
+{
+	"name" : "Top VLAN",
+	"i18n_name" : "top_vlan",
+	"data_source" : "flows",
+	"show_in_page" : "overview",
+	"visualization_type" : "table",
+	"hourly": true,
+	"sql" : "SELECT VLAN_ID, SUM(SRC2DST_BYTES) AS total_src2dst_bytes, SUM(DST2SRC_BYTES) AS total_dst2src_bytes, SUM(TOTAL_BYTES) AS total_bytes $FROM$ $WHERE$ $GROUPBY$ $ORDERBY$ $LIMIT$",
+	"select" : {
+		"items" : [
+			{
+				"name" : "VLAN_ID",
+				"func" : "",
+				"value_type" : "vlan_id"
+			},
+			{
+				"name" : "total_dst2src_bytes",
+				"func" : "",
+				"value_type" : "bytes"
+			},
+			{
+				"name" : "total_src2dst_bytes",
+				"func" : "",
+				"value_type" : "bytes"
+			},
+			{
+				"name" : "total_bytes",
+				"func" : "",
+				"value_type" : "bytes"
+			}
+		]
+	},
+	"filters" : {
+		"items" : [
+			{
+			}
+		]
+	},
+	"groupby" : {
+		"items" : [
+			{
+				"name" : "VLAN_ID"
+			}
+		]
+	},
+	"sortby" : {
+		"items" : [
+			{
+				"name" : "total_bytes",
+				"order" : "DESC"
+			}
+		]
+	}
+}

scripts/locales/en.lua

@@ -707,6 +707,8 @@ local lang = {
   ["total"] = "Total",
   ["total_alerts"] = "Total Alerts",
   ["total_bytes"] = "Total Bytes",
+  ["total_src2dst_bytes"] = "Total Sent Bytes",
+  ["total_dst2src_bytes"] = "Total Rcvd Bytes",
   ["total_flow_score"] = "Total Flow Score",
   ["total_incoming_alerted_flows"] = "Total Alerted Flows As Server",
   ["total_incoming_unreachable_flows"] = "Total Incoming Unreach Flows",

scripts/lua/hosts_stats.lua

@@ -603,10 +603,6 @@ if page == 'active_hosts' and ntop.isnEdge() then
         end
 
         print('</ul></div>\'')
-
-        if ntop.isPro() then
-            printHostsDeviceFilterDropdown(base_url, page_params)
-        end
         local alignment_c_info = 'center'
         if (ntop.isnEdge()) then 
             alignment_c_info = 'nowrap'

scripts/lua/modules/check_definitions/interface/no_if_activity.lua

@@ -24,6 +24,11 @@ local script = {
 -- #################################################################
 
 local function check_interface_activity(params)
+
+   if interface.isPcapDumpInterface() or interface.isDatabaseViewInterface() then
+      return -- Not a live interface, skip this check
+   end
+
    -- Get total number of packets, flows and interface id
    local num_packets = params.entity_info.eth.packets
    local num_flows = params.entity_info.stats.new_flows -- .new_flows keep the cumulative total, .flows is just a gauge

scripts/lua/modules/lua_utils_get.lua

@@ -77,8 +77,8 @@ function getProbesName(flowdevs, show_vlan, shorten_len)
     for interface, devices in pairs(flowdevs or {}) do
         local device_list = {} 
         if table.len(devices or {}) > 0 then
-            for ip, _ in pairsByValues(devices or {}, asc) do
-                device_list[ip] = getProbeName(ip, show_vlan, shorten_len)
+            for id, device_info in pairsByValues(devices or {}, asc) do
+                device_list[device_info.exporter_ip] = getProbeName(device_info.exporter_ip, show_vlan, shorten_len)
             end
             probes_list[interface] = device_list
         end
@@ -1222,6 +1222,22 @@ function mapServiceName(port, protocol)
    return(services[key])
 end
 
+-- ##############################################
+
+function getExporterList()
+    local flowdevs = interface.getFlowDevices()
+    local unified_exporters = {}
+    for interface_id, device_list in pairs(flowdevs or {}) do
+        for device_id, exporter_info in pairs(device_list, asc) do
+            local exporter_ip = exporter_info.exporter_ip
+            if not unified_exporters[exporter_ip] then
+                unified_exporters[exporter_ip] = exporter_info
+            end
+        end
+    end
+
+    return unified_exporters
+end
 
 -- ##############################################
 

scripts/lua/modules/lua_utils_print.lua

@@ -283,57 +283,6 @@ function printTrafficTypeFilterDropdown(base_url, page_params)
       </ul>]]
 end
 
--- ##############################################
-
-function printHostsDeviceFilterDropdown(base_url, page_params)
-   -- Getting probes
-   local flowdevs = interface.getFlowDevices() or {}
-   local ordering_fun = pairsByKeys
-   local cur_dev = _GET["deviceIP"]
-   local cur_dev_filter = ''
-   -- table.clone needed to modify some parameters while keeping the original unchanged
-   local dev_params = table.clone(page_params)
-   local devips = getProbesName(flowdevs)
-   local devips_order = ntop.getPref("ntopng.prefs.flow_table_probe_order") == "1" -- Order by Probe Name
-
-   if devips_order then
-      ordering_fun = pairsByValues
-   end
-
-   if not isEmptyString(cur_dev) then
-      cur_dev_filter = '<span class="fas fa-filter"></span>'
-   end
-
-   dev_params["deviceIP"] = nil
-
-   print[[, '<div class="btn-group float-right">]]
-
-   print[[
-      <button class="btn btn-link dropdown-toggle" data-bs-toggle="dropdown">]] print(i18n("flows_page.device_ip")) print[[]] print(cur_dev_filter) print[[<span class="caret"></span></button>\
-      <ul class="dropdown-menu dropdown-menu-end scrollable-dropdown" role="menu" id="flow_dropdown">\
-      ]]print('<li><a class="dropdown-item') print(page_params.deviceIP == nil and ' active' or '') print[[" href="]] print(getPageUrl(base_url, dev_params)) print[[">]] print(i18n("flows_page.all_devices")) print[[</a></li>\]]
-   for interface, device_list in pairs(devips or {}) do
-      for dev_ip, dev_resolved_name in ordering_fun(device_list, asc) do
-         local dev_name = dev_ip
-
-         dev_params["deviceIP"] = dev_name
-
-         if not isEmptyString(dev_resolved_name) and dev_resolved_name ~= dev_name then
-            dev_name = dev_name .. " ["..shortenString(dev_resolved_name).."]"
-         end
-
-         print[[
-         <li><a class="dropdown-item ]] print(dev_ip == cur_dev and 'active' or '') print[[" href="]] print(getPageUrl(base_url, dev_params)) print[[">]] print(i18n("flows_page.device_ip").." "..dev_name) print[[</a></li>\]]
-      end
-   end
-
-   print[[
-      </ul>\
-   ]]
-
-   print[[</div>']]
-end
-
 function processColor(proc)
   if(proc == nil) then
     return("")

scripts/lua/modules/ntop_utils.lua

@@ -881,3 +881,45 @@ function starts(String, Start)
 
     return string.sub(String, 1, string.len(Start)) == Start
 end
+
+-- ##############################################
+
+function getNetFlowExportersUnifiedStats()
+    local flowdevs = interface.getFlowDevices()
+    local unified_exporters = {}
+    for interface_id, device_list in pairs(flowdevs or {}) do
+        for device_id, exporter_info in pairs(device_list, asc) do
+            local exporter_ip = exporter_info.exporter_ip
+            if not unified_exporters[exporter_ip] then
+                unified_exporters[exporter_ip] = {}
+            end
+            local ports_table = interface.getFlowDeviceInfo(exporter_ip)
+            for _, ports in pairs(ports_table) do
+                for port_idx, port_info in pairs(ports) do
+                    if not unified_exporters[exporter_ip][port_idx] then
+                        unified_exporters[exporter_ip][port_idx] = port_info
+                    else
+                        local tmp = unified_exporters[exporter_ip][port_idx]
+                        tmp["throughput"] = port_info["throughput"] + (tmp["throughput"] or 0)
+                        tmp["bytes.in_bytes"] = port_info["bytes.in_bytes"] + (tmp["bytes.in_bytes"] or 0)
+                        tmp["bytes.out_bytes"] = port_info["bytes.out_bytes"] + (tmp["bytes.out_bytes"] or 0)
+                        if not tmp.ndpi then
+                            tmp["ndpi"] = {}
+                        end
+                        for proto, proto_info in pairs(port_info.ndpi or {}) do
+                            if not tmp["ndpi"][proto] then
+                                tmp["ndpi"][proto] = {}
+                            end
+                            for field, value in pairs(proto_info or {}) do
+                                tmp["ndpi"][proto][field] = value + (tmp["ndpi"][proto][field] or 0)
+                            end
+                        end
+                        unified_exporters[exporter_ip][port_idx] = tmp
+                    end
+                end
+            end
+        end
+    end
+
+    return unified_exporters
+end

scripts/lua/modules/tag_utils.lua

@@ -1153,16 +1153,13 @@ function tag_utils.get_tag_info(id, entity, hide_exporters_name, restrict_filter
 
         -- Add both Flow devices
         if interface.getFlowDevices then -- Pro Only
-            for interface, device_list in pairs(interface.getFlowDevices() or {}) do
-                for probe, _ in pairsByValues(device_list or {}, asc) do
-                    local probe_name = getProbeName(probe)
-                    -- local label = format_name_value(probe_name, probe)
-                    full_dev_list[probe] = {
-                        value = probe,
-                        label = probe_name,
-                        display_more_filters = true
-                    }
-                end
+            for exporter_ip, _ in pairs(getExporterList()) do
+                local probe_name = getProbeName(exporter_ip)
+                full_dev_list[exporter_ip] = {
+                    value = exporter_ip,
+                    label = probe_name,
+                    display_more_filters = true
+                }                
             end
         end
         -- And sFlow devices
@@ -1250,7 +1247,12 @@ function tag_utils.get_tag_info(id, entity, hide_exporters_name, restrict_filter
                         [tmp[1]] = true
                     }}
                 else
-                    flow_devices = interface.getFlowDevices()
+                    local tmp = interface.getFlowDevices() 
+                    for _, dev_list in pairs(tmp) do
+                        for _, exporter_info in pairs(dev_list) do
+                            flow_devices[exporter_info.exporter_ip] = 1
+                        end
+                    end
                 end
                 -- SNMP devices
                 local snmp_config = require "snmp_config"

src/LuaEngineInterface.cpp

@@ -2877,6 +2877,30 @@ static int ntop_get_flow_devices(lua_State *vm) {
 /* ****************************************** */
 
 static int ntop_get_flow_device_info(lua_State *vm) {
+  NetworkInterface *curr_iface = getCurrentInterface(vm);
+  u_int32_t device_id;
+  bool showAllStats = true;
+
+  ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__);
+  lua_newtable(vm);
+
+  if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ERROR));
+  device_id = (u_int32_t )lua_tonumber(vm, 1);
+  if (lua_type(vm, 2) == LUA_TBOOLEAN)
+    showAllStats = (bool)lua_toboolean(vm, 2);
+
+  if (!curr_iface)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ERROR));
+  else {
+    curr_iface->getFlowDeviceInfo(vm, device_id, showAllStats);
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_OK));
+  }
+}
+
+/* ****************************************** */
+
+static int ntop_get_flow_device_info_by_ip(lua_State *vm) {
   NetworkInterface *curr_iface = getCurrentInterface(vm);
   char *device_ip;
   bool showAllStats = true;
@@ -2895,7 +2919,7 @@ static int ntop_get_flow_device_info(lua_State *vm) {
   else {
     in_addr_t addr = inet_addr(device_ip);
 
-    curr_iface->getFlowDeviceInfo(vm, ntohl(addr), showAllStats);
+    curr_iface->getFlowDeviceInfoByIP(vm, ntohl(addr), showAllStats);
     return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_OK));
   }
 }
@@ -5526,6 +5550,7 @@ static luaL_Reg _ntop_interface_reg[] = {
     /* Flow Devices */
     {"getFlowDevices", ntop_get_flow_devices},
     {"getFlowDeviceInfo", ntop_get_flow_device_info},
+    {"getFlowDeviceInfoByIP", ntop_get_flow_device_info_by_ip},
 #endif
 
 #ifdef HAVE_NEDGE

src/NetworkInterface.cpp

@@ -2965,11 +2965,12 @@ bool NetworkInterface::dissectPacket(int32_t if_index,
       }
 
       if (vlan_id && ntop->getPrefs()->do_ignore_vlans()) vlan_id = 0;
-      if ((vlan_id == 0) && ntop->getPrefs()->do_simulate_vlans())
+      if ((vlan_id == 0) && ntop->getPrefs()->do_simulate_vlans()) {
 	vlan_id = (ip6 ? ip6->ip6_src.u6_addr.u6_addr8[15] +
 		   ip6->ip6_dst.u6_addr.u6_addr8[15]
-		   : iph->saddr + iph->daddr) &
+		   : iph->saddr + iph->daddr) %
 	  SIMULATE_VLANS_MAX_VALUE;
+      }
 
       if (ntop->getPrefs()->do_ignore_macs())
 	ethernet = &dummy_ethernet;