winlog normalization stuff

mmguero-dev · Feb 13, 2025 · 350aa20 · 350aa20
1 parent 6d519d2
commit 350aa20
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 57 deletions.
diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf
@@ -737,6 +737,7 @@ filter {
       mutate { id => "mutate_rename_fluentbit_winlog"
                rename => { "[winlog][ActivityID]" => "[winlog][activity_id]" }
                rename => { "[winlog][Computer]" => "[winlog][computer_name]" }
+               rename => { "[winlog][Channel]" => "[winlog][channel]" }
                rename => { "[winlog][Data]" => "[winlog][data]" }
                rename => { "[winlog][ComputerName]" => "[winlog][computer_name]" }
                rename => { "[winlog][EventCategory]" => "[winlog][task]" }
@@ -790,7 +791,7 @@ filter {
           id => "fingerprint_malcolm_fluentbit_winlog"
           source => [ "[winlog][computer_name]",
                       "[agent][type]",
-                      "[event][dataset]",
+                      "[winlog][channel]",
                       "[winlog][provider_name]",
                       "[winlog][event_id]",
                       "[winlog][record_id]",

diff --git a/logstash/pipelines/beats/13_normalize.conf b/logstash/pipelines/beats/13_normalize.conf
@@ -198,34 +198,55 @@ filter {
       }
 
       # ECS -> related.user
+
       if ([winlog][event_data][NewTargetUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_newtargetusername"
+        mutate { id => "mutate_winlog_event_data_NewTargetUserName"
                  merge => { "[related][user]" => "[winlog][event_data][NewTargetUserName]" } }
       }
       if ([winlog][event_data][OldTargetUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_oldtargetusername"
+        mutate { id => "mutate_winlog_event_data_OldTargetUserName"
                  merge => { "[related][user]" => "[winlog][event_data][OldTargetUserName]" } }
       }
+      if ([winlog][event_data][RemoteUserID]) {
+        mutate { id => "mutate_winlog_event_data_RemoteUserID"
+                 merge => { "[related][user]" => "[winlog][event_data][RemoteUserID]" } }
+      }
       if ([winlog][event_data][SourceUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_sourceusername"
+        mutate { id => "mutate_winlog_event_data_SourceUserName"
                  merge => { "[related][user]" => "[winlog][event_data][SourceUserName]" } }
       }
       if ([winlog][event_data][SubjectUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_subjectusername"
+        mutate { id => "mutate_winlog_event_data_SubjectUserName"
                  merge => { "[related][user]" => "[winlog][event_data][SubjectUserName]" } }
       }
       if ([winlog][event_data][TargetOutboundUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_targetoutboundusername"
+        mutate { id => "mutate_winlog_event_data_TargetOutboundUserName"
                  merge => { "[related][user]" => "[winlog][event_data][TargetOutboundUserName]" } }
       }
       if ([winlog][event_data][TargetUserName]) {
-        mutate { id => "mutate_merge_eventdata_user_targetusername"
+        mutate { id => "mutate_winlog_event_data_TargetUserName"
                  merge => { "[related][user]" => "[winlog][event_data][TargetUserName]" } }
       }
+      if ([winlog][event_data][UserId]) {
+        mutate { id => "mutate_winlog_event_data_UserId"
+                 merge => { "[related][user]" => "[winlog][event_data][UserId]" } }
+      }
+      if ([winlog][event_data][UserName]) {
+        mutate { id => "mutate_winlog_event_data_UserName"
+                 merge => { "[related][user]" => "[winlog][event_data][UserName]" } }
+      }
+      if ([winlog][event_data][Username]) {
+        mutate { id => "mutate_winlog_event_data_Username"
+                 merge => { "[related][user]" => "[winlog][event_data][Username]" } }
+      }
       if ([winlog][event_data][username]) {
-        mutate { id => "mutate_merge_eventdata_user_username"
+        mutate { id => "mutate_winlog_event_data_username"
                  merge => { "[related][user]" => "[winlog][event_data][username]" } }
       }
+      if ([winlog][event_data][User]) {
+        mutate { id => "mutate_merge_eventdata_user_user"
+                 merge => { "[related][user]" => "[winlog][event_data][User]" } }
+      }
       if ([winlog][event_data][Detection_User]) {
         mutate { id => "mutate_merge_eventdata_user_detection_user"
                  merge => { "[related][user]" => "[winlog][event_data][Detection_User]" } }
@@ -242,10 +263,6 @@ filter {
         mutate { id => "mutate_merge_eventdata_user_targetuser"
                  merge => { "[related][user]" => "[winlog][event_data][TargetUser]" } }
       }
-      if ([winlog][event_data][User]) {
-        mutate { id => "mutate_merge_eventdata_user_user"
-                 merge => { "[related][user]" => "[winlog][event_data][User]" } }
-      }
 
       # ECS hash
       if ([winlog][event_data][Hashes]) {
@@ -371,51 +388,6 @@ filter {
                  merge => { "[process][executable]" => "[winlog][event_data][ProcessName]" } }
       }
 
-      if ([winlog][event_data][NewTargetUserName]) {
-        mutate { id => "mutate_winlog_event_data_NewTargetUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][NewTargetUserName]" } }
-      }
-      if ([winlog][event_data][OldTargetUserName]) {
-        mutate { id => "mutate_winlog_event_data_OldTargetUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][OldTargetUserName]" } }
-      }
-      if ([winlog][event_data][RemoteUserID]) {
-        mutate { id => "mutate_winlog_event_data_RemoteUserID"
-                 merge => { "[related][user]" => "[winlog][event_data][RemoteUserID]" } }
-      }
-      if ([winlog][event_data][SourceUserName]) {
-        mutate { id => "mutate_winlog_event_data_SourceUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][SourceUserName]" } }
-      }
-      if ([winlog][event_data][SubjectUserName]) {
-        mutate { id => "mutate_winlog_event_data_SubjectUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][SubjectUserName]" } }
-      }
-      if ([winlog][event_data][TargetOutboundUserName]) {
-        mutate { id => "mutate_winlog_event_data_TargetOutboundUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][TargetOutboundUserName]" } }
-      }
-      if ([winlog][event_data][TargetUserName]) {
-        mutate { id => "mutate_winlog_event_data_TargetUserName"
-                 merge => { "[related][user]" => "[winlog][event_data][TargetUserName]" } }
-      }
-      if ([winlog][event_data][UserId]) {
-        mutate { id => "mutate_winlog_event_data_UserId"
-                 merge => { "[related][user]" => "[winlog][event_data][UserId]" } }
-      }
-      if ([winlog][event_data][UserName]) {
-        mutate { id => "mutate_winlog_event_data_UserName"
-                 merge => { "[related][user]" => "[winlog][event_data][UserName]" } }
-      }
-      if ([winlog][event_data][Username]) {
-        mutate { id => "mutate_winlog_event_data_Username"
-                 merge => { "[related][user]" => "[winlog][event_data][Username]" } }
-      }
-      if ([winlog][event_data][username]) {
-        mutate { id => "mutate_winlog_event_data_username"
-                 merge => { "[related][user]" => "[winlog][event_data][username]" } }
-      }
-
     } # if winlog.event_data
 
     if ([winlog][computer_name]) {

diff --git a/logstash/pipelines/beats/96_make_unique.conf b/logstash/pipelines/beats/96_make_unique.conf
@@ -0,0 +1,89 @@
+# Copyright (c) 2025 Battelle Energy Alliance, LLC.  All rights reserved.
+
+# take array fields that are already generic (i.e., mostly ECS) and deduplicate them.
+
+filter {
+
+  if ([related][hash]) {
+    ruby {
+      id => "ruby_miscbeat_related_hash_uniq"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[related][hash]"
+      }
+    }
+  }
+
+  if ([related][user]) {
+    ruby {
+      id => "ruby_miscbeat_related_user_uniq"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[related][user]"
+      }
+    }
+  }
+
+  if ([process][executable]) {
+    ruby {
+      id => "ruby_miscbeat_process_executable"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[process][executable]"
+      }
+    }
+  }
+
+  if ([process][pid]) {
+    ruby {
+      id => "ruby_miscbeat_process_pid"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[process][pid]"
+      }
+    }
+  }
+
+  if ([process][thread][id]) {
+    ruby {
+      id => "ruby_miscbeat_process_thread_id"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[process][thread][id]"
+      }
+    }
+  }
+
+  if ([event][action]) {
+    ruby {
+      id => "ruby_miscbeat_event_action_uniq"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[event][action]"
+      }
+    }
+  }
+
+  if ([event][result]) {
+    ruby {
+      id => "ruby_miscbeat_event_result_uniq"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[event][result]"
+      }
+    }
+  }
+
+  if ([event][id]) {
+    ruby {
+      id => "ruby_miscbeat_event_id_uniq"
+      path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb"
+      script_params => {
+        "field" => "[event][id]"
+      }
+    }
+  }
+
+
+
+}