Skip to content

Commit

Permalink
Clean tests and Fallback to use latest available metadata if non found
Browse files Browse the repository at this point in the history
  • Loading branch information
@marc-gr
marc-gr committed Jan 14, 2025
1 parent 3fd5e3f commit 61cb7a0
Show file tree
Hide file tree
Showing 10 changed files with 86 additions and 131 deletions.
34 changes: 17 additions & 17 deletions winlogbeat/eventlog/wineventlog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ import (
"github.com/elastic/elastic-agent-libs/logp"
)

// winEventLogRaw implements the EventLog interface for reading from the Windows
// winEventLog implements the EventLog interface for reading from the Windows
// Event Log API.
type winEventLogRaw struct {
type winEventLog struct {
config config
query string
id string // Identifier of this event log.
Expand Down Expand Up @@ -98,7 +98,7 @@ func newWinEventLog(options *conf.C) (EventLog, error) {
log = logp.NewLogger("wineventlog").With("id", id).With("channel", c.Name)
}

l := &winEventLogRaw{
l := &winEventLog{
config: c,
query: xmlQuery,
id: id,
Expand Down Expand Up @@ -131,27 +131,27 @@ func newWinEventLog(options *conf.C) (EventLog, error) {
return l, nil
}

func (l *winEventLogRaw) isForwarded() bool {
func (l *winEventLog) isForwarded() bool {
c := l.config
return (c.Forwarded != nil && *c.Forwarded) || (c.Forwarded == nil && c.Name == "ForwardedEvents")
}

// Name returns the name of the event log (i.e. Application, Security, etc.).
func (l *winEventLogRaw) Name() string {
func (l *winEventLog) Name() string {
return l.id
}

// Channel returns the event log's channel name.
func (l *winEventLogRaw) Channel() string {
func (l *winEventLog) Channel() string {
return l.channelName
}

// IsFile returns true if the event log is an evtx file.
func (l *winEventLogRaw) IsFile() bool {
func (l *winEventLog) IsFile() bool {
return l.file
}

func (l *winEventLogRaw) Open(state checkpoint.EventLogState) error {
func (l *winEventLog) Open(state checkpoint.EventLogState) error {
l.lastRead = state
// we need to defer metrics initialization since when the event log
// is used from winlog input it would register it twice due to CheckConfig calls
Expand All @@ -168,7 +168,7 @@ func (l *winEventLogRaw) Open(state checkpoint.EventLogState) error {
return err
}

func (l *winEventLogRaw) open(state checkpoint.EventLogState) (win.EvtHandle, error) {
func (l *winEventLog) open(state checkpoint.EventLogState) (win.EvtHandle, error) {
var bookmark win.Bookmark
if len(state.Bookmark) > 0 {
var err error
Expand All @@ -185,7 +185,7 @@ func (l *winEventLogRaw) open(state checkpoint.EventLogState) (win.EvtHandle, er
return l.openChannel(bookmark)
}

func (l *winEventLogRaw) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) {
func (l *winEventLog) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) {
path := l.channelName

h, err := win.EvtQuery(0, path, l.query, win.EvtQueryFilePath|win.EvtQueryForwardDirection)
Expand Down Expand Up @@ -222,7 +222,7 @@ func (l *winEventLogRaw) openFile(state checkpoint.EventLogState, bookmark win.B
return h, err
}

func (l *winEventLogRaw) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) {
func (l *winEventLog) openChannel(bookmark win.Bookmark) (win.EvtHandle, error) {
// Using a pull subscription to receive events. See:
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa385771(v=vs.85).aspx#pull
signalEvent, err := windows.CreateEvent(nil, 0, 0, nil)
Expand Down Expand Up @@ -264,7 +264,7 @@ func (l *winEventLogRaw) openChannel(bookmark win.Bookmark) (win.EvtHandle, erro
}
}

func (l *winEventLogRaw) Read() ([]Record, error) {
func (l *winEventLog) Read() ([]Record, error) {
//nolint:prealloc // Avoid unnecessary preallocation for each reader every second when event log is inactive.
var records []Record
defer func() {
Expand Down Expand Up @@ -302,7 +302,7 @@ func (l *winEventLogRaw) Read() ([]Record, error) {
return records, nil
}

func (l *winEventLogRaw) processHandle(h win.EvtHandle) (*Record, error) {
func (l *winEventLog) processHandle(h win.EvtHandle) (*Record, error) {
defer h.Close()

// NOTE: Render can return an error and a partial event.
Expand Down Expand Up @@ -339,7 +339,7 @@ func (l *winEventLogRaw) processHandle(h win.EvtHandle) (*Record, error) {
return r, nil
}

func (l *winEventLogRaw) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) {
func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, error) {
bookmark, err := win.NewBookmarkFromEvent(evtHandle)
if err != nil {
return "", fmt.Errorf("failed to create new bookmark from event handle: %w", err)
Expand All @@ -349,18 +349,18 @@ func (l *winEventLogRaw) createBookmarkFromEvent(evtHandle win.EvtHandle) (strin
return bookmark.XML()
}

func (l *winEventLogRaw) Reset() error {
func (l *winEventLog) Reset() error {
l.log.Debug("Closing event log reader handles for reset.")
return l.close()
}

func (l *winEventLogRaw) Close() error {
func (l *winEventLog) Close() error {
l.log.Debug("Closing event log reader handles.")
l.metrics.close()
return l.close()
}

func (l *winEventLogRaw) close() error {
func (l *winEventLog) close() error {
if l.iterator == nil {
return l.renderer.Close()
}
Expand Down
4 changes: 1 addition & 3 deletions winlogbeat/eventlog/wineventlog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ func TestWinEventLogConfig_Validate(t *testing.T) {
}
}

func TestWindowsEventLogAPIRaw(t *testing.T) {
// for the raw api using include xml behave differently than not
// so we must test both settings
func TestWindowsEventLogAPI(t *testing.T) {
testWindowsEventLog(t, true)
testWindowsEventLog(t, false)
}
Expand Down
16 changes: 1 addition & 15 deletions winlogbeat/sys/wineventlog/format_message_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,21 +65,7 @@ func TestFormatMessage(t *testing.T) {
}

assert.Contains(t, msg, `{{eventParam $ 2}}`)

// NOTE: In this test case I noticed the messages contains
// "Logon ID: 0x0"
// but it should contain
// "Logon ID: {{eventParam $ 9}}"
//
// This may mean that certain windows.GUID values cannot be
// substituted with string values. So we shouldn't rely on this
// method to create text/templates. Instead we can use the
// getMessageStringFromMessageID (see test below) that works as
// expected.
//
// Note: This is not the case under 32-bit Windows 7.
// Disabling the assertion for now.
//assert.NotContains(t, msg, `{{eventParam $ 9}}`)
assert.NotContains(t, msg, `{{eventParam $ 9}}`)
})
})

Expand Down
23 changes: 17 additions & 6 deletions winlogbeat/sys/wineventlog/metadata_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,11 @@ type PublisherMetadataStore struct {
winevent.WinMeta

// Event ID to event metadata (message and event data param names).
Events map[uint32]*EventMetadata
// Keeps track of the latest metadata available for each event.
EventsNewest map[uint16]*EventMetadata
// Event ID to event metadata (message and event data param names).
// Keeps track of all available versions for each event.
EventsByVersion map[uint32]*EventMetadata
// Event ID to map of fingerprints to event metadata. The fingerprint value
// is hash of the event data parameters count and types.
EventFingerprints map[uint32]map[uint64]*EventMetadata
Expand Down Expand Up @@ -103,7 +107,8 @@ func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *Publishe
Levels: map[uint8]string{},
Tasks: map[uint16]string{},
},
Events: map[uint32]*EventMetadata{},
EventsNewest: map[uint16]*EventMetadata{},
EventsByVersion: map[uint32]*EventMetadata{},
EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
MessagesByID: map[uint32]string{},
log: log.With("publisher", provider, "empty", true),
Expand Down Expand Up @@ -183,15 +188,17 @@ func (s *PublisherMetadataStore) initEvents() error {
}
defer itr.Close()

s.Events = map[uint32]*EventMetadata{}
s.EventsNewest = map[uint16]*EventMetadata{}
s.EventsByVersion = map[uint32]*EventMetadata{}
for itr.Next() {
evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata)
if err != nil {
s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.",
"error", err)
continue
}
s.Events[getEventCombinedID(evt.EventID, evt.Version)] = evt
s.EventsNewest[evt.EventID] = evt
s.EventsByVersion[getEventCombinedID(evt.EventID, evt.Version)] = evt
}
return itr.Err()
}
Expand Down Expand Up @@ -235,8 +242,12 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, version uint8,
// metadata then we just associate the fingerprint with a pointer to the
// providers metadata for the event ID.

defaultEM := s.Events[combinedID]

defaultEM, found := s.EventsByVersion[combinedID]
if !found {
// if we do not have a specific metadata for this event version
// we fallback to get the newest available one
defaultEM = s.EventsNewest[eventID]
}
// Use XML to get the parameters names.
em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion winlogbeat/sys/wineventlog/metadata_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func TestPublisherMetadataStore(t *testing.T) {
}
defer s.Close()

assert.NotEmpty(t, s.Events)
assert.NotEmpty(t, s.EventsByVersion)
assert.Empty(t, s.EventFingerprints)

t.Run("event_metadata_from_handle", func(t *testing.T) {
Expand Down
3 changes: 0 additions & 3 deletions winlogbeat/tests/system/config/winlogbeat.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
winlogbeat.event_logs:
{% for log in event_logs -%}
- name: {{ log.name }}
{%- if log.api is defined %}
api: {{ log.api }}
{% endif %}
{%- if log.ignore_older is defined %}
ignore_older: {{ log.ignore_older }}
{% endif %}
Expand Down
18 changes: 0 additions & 18 deletions winlogbeat/tests/system/test_wineventlog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ class Test(WriteReadTest):

@classmethod
def setUpClass(self):
self.api = "wineventlog"
super(WriteReadTest, self).setUpClass()

def test_read_one_event(self):
Expand All @@ -33,7 +32,6 @@ def test_read_one_event(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], msg=msg, extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
})

def test_resume_reading_events(self):
Expand All @@ -46,7 +44,6 @@ def test_resume_reading_events(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], msg=msg, extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
})

# remove the output file, otherwise there is a race condition
Expand All @@ -60,7 +57,6 @@ def test_resume_reading_events(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], msg=msg, extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
})

def test_cleared_channel_restarts(self):
Expand Down Expand Up @@ -151,7 +147,6 @@ def test_read_unknown_event_id(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], eventID="1111", extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
})

self.assertEqual(
Expand All @@ -174,7 +169,6 @@ def test_read_unknown_sid(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], msg=msg, sid=accountIdentifier, extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
})

def test_fields_under_root(self):
Expand All @@ -190,7 +184,6 @@ def test_fields_under_root(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"tags": ["local"],
"fields_under_root": True,
"fields": {"local": "field", "env": "dev"}
Expand All @@ -200,7 +193,6 @@ def test_fields_under_root(self):
self.assertTrue(len(evts), 1)
self.assert_common_fields(evts[0], msg=msg, level="overwrite", extra={
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
"global": "field",
"env": "dev",
"local": "field",
Expand All @@ -218,7 +210,6 @@ def test_fields_not_under_root(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"fields": {"local": "field", "env": "dev", "num": 1}
}
]
Expand All @@ -227,7 +218,6 @@ def test_fields_not_under_root(self):
self.assert_common_fields(evts[0], msg=msg, extra={
"log.level": "information",
"winlog.keywords": ["Classic"],
"winlog.opcode": "Info",
"fields.global": "field",
"fields.env": "dev",
"fields.level": "overwrite",
Expand Down Expand Up @@ -273,7 +263,6 @@ def test_query_event_id(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"event_id": "50, 100-200, -150"
}
]
Expand All @@ -298,7 +287,6 @@ def test_query_level_single(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"level": "warning"
}
]
Expand All @@ -322,7 +310,6 @@ def test_query_level_multiple(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"level": "error, warning"
}
]
Expand All @@ -344,7 +331,6 @@ def test_query_ignore_older(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"ignore_older": "2s"
}
]
Expand All @@ -363,7 +349,6 @@ def test_query_provider(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"provider": [self.otherAppName]
}
]
Expand All @@ -384,7 +369,6 @@ def test_query_multi_param(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"event_id": "10-20, 30-40, -35, -18, 400-1000, -432",
"level": "warn, error",
"provider": [self.otherAppName]
Expand All @@ -407,7 +391,6 @@ def test_utf16_characters(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"include_xml": True,
}
]
Expand Down Expand Up @@ -439,7 +422,6 @@ def test_processors(self):
"event_logs": [
{
"name": self.providerName,
"api": self.api,
"extras": {
"processors": [
{
Expand Down
Loading

0 comments on commit 61cb7a0

Please sign in to comment.