From f6d5accade0a325df0325830f729295178d3d47c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 13 Jan 2025 15:12:09 +0100 Subject: [PATCH] Several improvements to winlogbeat raw api: (#42297) - Fingerprint templates by eventID+version - Properly parse opcodes - Improve handling of user data --- winlogbeat/sys/wineventlog/metadata_store.go | 63 +++++++++++++------ .../sys/wineventlog/publisher_metadata.go | 9 ++- winlogbeat/sys/wineventlog/renderer.go | 13 ++-- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go index 4373451717a..821e04d2dfc 100644 --- a/winlogbeat/sys/wineventlog/metadata_store.go +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -20,6 +20,7 @@ package wineventlog import ( + "encoding/xml" "fmt" "strconv" "strings" @@ -51,10 +52,10 @@ type PublisherMetadataStore struct { winevent.WinMeta // Event ID to event metadata (message and event data param names). - Events map[uint16]*EventMetadata + Events map[uint32]*EventMetadata // Event ID to map of fingerprints to event metadata. The fingerprint value // is hash of the event data parameters count and types. - EventFingerprints map[uint16]map[uint64]*EventMetadata + EventFingerprints map[uint32]map[uint64]*EventMetadata // Stores used messages by their ID. Message can be found in events as references // such as %%1111. This need to be formatted the first time, and they are stored // from that point after. @@ -71,7 +72,7 @@ func NewPublisherMetadataStore(session EvtHandle, provider string, locale uint32 } store := &PublisherMetadataStore{ Metadata: md, - EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, + EventFingerprints: map[uint32]map[uint64]*EventMetadata{}, MessagesByID: map[uint32]string{}, log: log.With("publisher", provider), } @@ -102,8 +103,8 @@ func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *Publishe Levels: map[uint8]string{}, Tasks: map[uint16]string{}, }, - Events: map[uint16]*EventMetadata{}, - EventFingerprints: map[uint16]map[uint64]*EventMetadata{}, + Events: map[uint32]*EventMetadata{}, + EventFingerprints: map[uint32]map[uint64]*EventMetadata{}, MessagesByID: map[uint32]string{}, log: log.With("publisher", provider, "empty", true), } @@ -137,7 +138,7 @@ func (s *PublisherMetadataStore) initOpcodes() error { if val == "" { val = opcodeMeta.Name } - s.Opcodes[uint8(opcodeMeta.Mask)] = val + s.Opcodes[uint8(opcodeMeta.Opcode)] = val } return nil } @@ -182,7 +183,7 @@ func (s *PublisherMetadataStore) initEvents() error { } defer itr.Close() - s.Events = map[uint16]*EventMetadata{} + s.Events = map[uint32]*EventMetadata{} for itr.Next() { evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata) if err != nil { @@ -190,15 +191,16 @@ func (s *PublisherMetadataStore) initEvents() error { "error", err) continue } - s.Events[evt.EventID] = evt + s.Events[getEventCombinedID(evt.EventID, evt.Version)] = evt } return itr.Err() } -func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata { +func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, version uint8, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata { // Use a read lock to get a cached value. s.mutex.RLock() - fingerprints, found := s.EventFingerprints[eventID] + combinedID := getEventCombinedID(eventID, version) + fingerprints, found := s.EventFingerprints[combinedID] if found { em, found := fingerprints[eventDataFingerprint] if found { @@ -212,10 +214,10 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge s.mutex.Lock() defer s.mutex.Unlock() - fingerprints, found = s.EventFingerprints[eventID] + fingerprints, found = s.EventFingerprints[combinedID] if !found { fingerprints = map[uint64]*EventMetadata{} - s.EventFingerprints[eventID] = fingerprints + s.EventFingerprints[combinedID] = fingerprints } em, found := fingerprints[eventDataFingerprint] @@ -233,7 +235,7 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge // metadata then we just associate the fingerprint with a pointer to the // providers metadata for the event ID. - defaultEM := s.Events[eventID] + defaultEM := s.Events[combinedID] // Use XML to get the parameters names. em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle) @@ -250,6 +252,15 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge return defaultEM } + // The first time we need to identify if the event has event data or + // user data from a handle, since this information is not available + // from the metadata or anywhere else. It is not ideal to update the defaultEM + // here but there is no way around it at the moment. + if em.EventData.IsUserData { + defaultEM.EventData.IsUserData = true + defaultEM.EventData.Name = em.EventData.Name + } + // Are the parameters the same as what the provider metadata listed? // (This ignores the message values.) if em.equal(defaultEM) { @@ -324,13 +335,18 @@ func (s *PublisherMetadataStore) Close() error { return nil } +type EventDataParams struct { + IsUserData bool + Name xml.Name + Params []EventData +} + type EventMetadata struct { EventID uint16 // Event ID. Version uint8 // Event format version. MsgStatic string // Used when the message has no parameters. MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data. - EventData []EventData // Names of parameters from XML template. - HasUserData bool // Event has a UserData section or not. + EventData EventDataParams // Names of parameters from XML template. } // newEventMetadataFromEventHandle collects metadata about an event type using @@ -354,12 +370,13 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E } if len(event.EventData.Pairs) > 0 { for _, pair := range event.EventData.Pairs { - em.EventData = append(em.EventData, EventData{Name: pair.Key}) + em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key}) } } else { - em.HasUserData = true + em.EventData.IsUserData = true + em.EventData.Name = event.UserData.Name for _, pair := range event.UserData.Pairs { - em.EventData = append(em.EventData, EventData{Name: pair.Key}) + em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key}) } } @@ -435,7 +452,7 @@ func (em *EventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error kv.Name = eventDataNameTransform.Replace(kv.Name) } - em.EventData = tmpl.Data + em.EventData.Params = tmpl.Data return nil } @@ -477,6 +494,10 @@ func (em *EventMetadata) setMessage(msg string) error { return nil } +func getEventCombinedID(eventID uint16, version uint8) uint32 { + return (uint32(eventID) << 16) | uint32(version) +} + // containsTemplatedValues traverses the template nodes to check if there are // any dynamic values. func containsTemplatedValues(tmpl *template.Template) bool { @@ -513,7 +534,9 @@ func (em *EventMetadata) equal(other *EventMetadata) bool { return em.EventID == other.EventID && em.Version == other.Version && - eventDataNamesEqual(em.EventData, other.EventData) + em.EventData.IsUserData == other.EventData.IsUserData && + em.EventData.Name == other.EventData.Name && + eventDataNamesEqual(em.EventData.Params, other.EventData.Params) } type publisherMetadataCache struct { diff --git a/winlogbeat/sys/wineventlog/publisher_metadata.go b/winlogbeat/sys/wineventlog/publisher_metadata.go index 2e5ed2411c5..453a7bb7946 100644 --- a/winlogbeat/sys/wineventlog/publisher_metadata.go +++ b/winlogbeat/sys/wineventlog/publisher_metadata.go @@ -231,7 +231,8 @@ func NewMetadataKeyword(publisherMetadataHandle EvtHandle, arrayHandle EvtObject type MetadataOpcode struct { Name string - Mask uint32 + Opcode uint16 + Task uint16 MessageID uint32 Message string } @@ -292,11 +293,15 @@ func NewMetadataOpcode(publisherMetadataHandle EvtHandle, arrayHandle EvtObjectA if err != nil { return nil, err } + // Mask high word contains the opcode value and the low word contains the task to which it belongs. + // If the low word is zero, the opcode is defined globally; otherwise, the opcode is task specific. + // Use the low word value to determine the task that defines the opcode. valueMask := v.(uint32) return &MetadataOpcode{ Name: name, - Mask: valueMask, + Opcode: uint16((valueMask >> 16) & 0xFFFF), + Task: uint16(valueMask & 0xFFFF), MessageID: messageID, Message: message, }, nil diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go index 6b7042d25dd..1353fe5423e 100644 --- a/winlogbeat/sys/wineventlog/renderer.go +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -126,7 +126,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, string, error) { } // Load cached event metadata or try to bootstrap it from the event's XML. - eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), fingerprint, handle) + eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), uint8(event.Version), fingerprint, handle) // Associate key names with the event data values. r.addEventData(eventMeta, eventData, event) @@ -304,13 +304,13 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev r.log.Warnw("Event metadata not found.", "provider", event.Provider.Name, "event_id", event.EventIdentifier.ID) - } else if len(values) != len(evtMeta.EventData) { + } else if len(values) != len(evtMeta.EventData.Params) { r.log.Warnw("The number of event data parameters doesn't match the number "+ "of parameters in the template.", "provider", event.Provider.Name, "event_id", event.EventIdentifier.ID, "event_parameter_count", len(values), - "template_parameter_count", len(evtMeta.EventData), + "template_parameter_count", len(evtMeta.EventData.Params), "template_version", evtMeta.Version, "event_version", event.Version) } @@ -322,8 +322,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev // updated). If software was updated it could also be that this cached // template is now stale. paramName := func(idx int) string { - if evtMeta != nil && idx < len(evtMeta.EventData) { - return evtMeta.EventData[idx].Name + if evtMeta != nil && idx < len(evtMeta.EventData.Params) { + return evtMeta.EventData.Params[idx].Name } return "param" + strconv.Itoa(idx) } @@ -346,7 +346,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev } } - if evtMeta != nil && evtMeta.HasUserData { + if evtMeta != nil && evtMeta.EventData.IsUserData { + event.UserData.Name = evtMeta.EventData.Name event.UserData.Pairs = pairs } else { event.EventData.Pairs = pairs