diff --git a/plaso/formatters/winevt.py b/plaso/formatters/winevt.py index ae4d619aca..af296dfc3e 100644 --- a/plaso/formatters/winevt.py +++ b/plaso/formatters/winevt.py @@ -42,12 +42,13 @@ def FormatEventValues(self, output_mediator, event_values): try: message_string = message_string_template.format(*string_values) except (IndexError, TypeError) as exception: + provider_identifier = provider_identifier or '' + strings = ', '.join(string_values) logger.error(( - 'Unable to format message: 0x{0:08x} of provider: {1:s} ' - 'template: "{2:s}" and strings: "{3:s}" with error: ' - '{4!s}').format( - message_identifier, provider_identifier or '', - message_string_template, ', '.join(string_values), exception)) + f'Unable to format message: 0x{message_identifier:08x} of ' + f'provider: {provider_identifier:s} template: ' + f'"{message_string_template:s}" and strings: "{strings:s}" ' + f'with error: {exception!s}')) # Unable to create the message string. # TODO: consider returning the unformatted message string. diff --git a/plaso/multi_process/extraction_engine.py b/plaso/multi_process/extraction_engine.py index f2ece67764..f847511b1f 100644 --- a/plaso/multi_process/extraction_engine.py +++ b/plaso/multi_process/extraction_engine.py @@ -154,8 +154,8 @@ def __init__( except NotImplementedError: logger.error(( - 'Unable to determine number of CPUs defaulting to {0:d} worker ' - 'processes.').format(self._WORKER_PROCESSES_MINIMUM)) + f'Unable to determine number of CPUs defaulting to ' + f'{self._WORKER_PROCESSES_MINIMUM:d} worker processes.')) cpu_count = self._WORKER_PROCESSES_MINIMUM number_of_worker_processes = cpu_count @@ -185,15 +185,16 @@ def __init__( self._resolver_context = context.Context() self._status = definitions.STATUS_INDICATOR_IDLE self._status_update_callback = status_update_callback + self._system_configurations = None self._task_manager = task_manager.TaskManager() self._task_merge_helper = None self._task_merge_helper_on_hold = None self._task_queue = None self._task_queue_port = None self._task_storage_format = None + self._windows_event_log_providers = None self._worker_memory_limit = worker_memory_limit self._worker_timeout = worker_timeout - self._system_configurations = None def _CacheFileSystem(self, file_system): """Caches a dfVFS file system object. @@ -262,7 +263,7 @@ def _CollectInitialEventSources(self, storage_writer, file_system_path_specs): if self._CheckExcludedPathSpec(file_system, path_spec): display_name = path_helper.PathHelper.GetDisplayNameForPathSpec( path_spec) - logger.debug('Excluded from extraction: {0:s}.'.format(display_name)) + logger.debug(f'Excluded from extraction: {display_name:s}.') continue # TODO: determine if event sources should be DataStream or FileEntry @@ -305,8 +306,7 @@ def _CreateTask(self, storage_writer, session_identifier, event_source): if self._CheckExcludedPathSpec(file_system, event_source.path_spec): display_name = path_helper.PathHelper.GetDisplayNameForPathSpec( event_source.path_spec) - logger.debug('Excluded from extraction: {0:s}.'.format( - display_name)) + logger.debug(f'Excluded from extraction: {display_name:s}.') return None task = self._task_manager.CreateTask( @@ -409,11 +409,10 @@ def _MergeAttributeContainer(self, storage_writer, merge_helper, container): # TODO: store this as a merge warning so this is preserved # in the storage file. logger.error(( - 'Unable to merge {0:s} attribute container: {1:s} since ' - 'corresponding event data stream: {2:s} could not be ' - 'found.').format( - container.CONTAINER_TYPE, identifier_string, - event_data_stream_lookup_key)) + f'Unable to merge {container.CONTAINER_TYPE:s} attribute ' + f'container: {identifier_string:s} since corresponding event ' + f'data stream: {event_data_stream_lookup_key:s} could not be ' + f'found.')) return elif container.CONTAINER_TYPE in ( @@ -438,10 +437,9 @@ def _MergeAttributeContainer(self, storage_writer, merge_helper, container): description = 'WEVT_TEMPLATE event definition' logger.error(( - 'Unable to merge {0:s} attribute container: {1:s} since ' - 'corresponding Windows EventLog message file: {2:s} could not ' - 'be found.').format( - description, identifier_string, message_file_lookup_key)) + f'Unable to merge {description:s} attribute container: ' + f'{identifier_string:s} since corresponding Windows EventLog ' + f'message file: {message_file_lookup_key:s} could not be found.')) return lookup_key = None @@ -554,9 +552,9 @@ def _MergeTaskStorage(self, storage_writer, session_identifier): self._task_manager.UpdateTaskAsPendingMerge(task) except KeyError as exception: - logger.error( - 'Unable to retrieve task: {0:s} to prepare it to be merged ' - 'with error: {1!s}.'.format(task_identifier, exception)) + logger.error(( + f'Unable to retrieve task: {task_identifier:s} to prepare it to ' + f'be merged with error: {exception!s}.')) continue if self._processing_profiler: @@ -590,8 +588,8 @@ def _MergeTaskStorage(self, storage_writer, session_identifier): except IOError as exception: logger.error(( - 'Unable to merge results of task: {0:s} ' - 'with error: {1!s}').format(task.identifier, exception)) + f'Unable to merge results of task: {task.identifier:s} with ' + f'error: {exception!s}')) self._task_merge_helper = None if self._task_merge_helper: @@ -638,9 +636,9 @@ def _MergeTaskStorage(self, storage_writer, session_identifier): self._task_manager.CompleteTask(self._merge_task) except KeyError as exception: - logger.error( - 'Unable to complete task: {0:s} with error: {1!s}'.format( - self._merge_task.identifier, exception)) + logger.error(( + f'Unable to complete task: {self._merge_task.identifier:s} with ' + f'error: {exception!s}')) if not self._task_merge_helper_on_hold: self._merge_task = None @@ -720,9 +718,10 @@ def _ProcessEventSources(self, storage_writer, session_identifier): else: path_spec_string = self._GetPathSpecificationString(task.path_spec) - logger.debug( - 'Scheduled task: {0:s} for path specification: {1:s}'.format( - task.identifier, path_spec_string.replace('\n', ' '))) + path_spec_string = path_spec_string.replace('\n', ' ') + logger.debug(( + f'Scheduled task: {task.identifier:s} for path specification: ' + f'{path_spec_string:s}')) self._task_manager.SampleTaskStatus(task, 'scheduled') @@ -753,8 +752,8 @@ def _ProcessEventSources(self, storage_writer, session_identifier): # from being killed by an uncaught exception. except Exception as exception: # pylint: disable=broad-except self._ProduceExtractionWarning(storage_writer, ( - 'unable to process path specification with error: ' - '{0!s}').format(exception), event_source.path_spec) + f'unable to process path specification with error: ' + f'{exception!s}'), event_source.path_spec) event_source = None for task in self._task_manager.GetFailedTasks(): @@ -862,9 +861,9 @@ def _StartWorkerProcess(self, process_name): MultiProcessWorkerProcess: extraction worker process or None if the process could not be started. """ - logger.debug('Starting worker process {0:s}'.format(process_name)) + logger.debug(f'Starting worker process {process_name:s}') - queue_name = '{0:s} task queue'.format(process_name) + queue_name = f'{process_name:s} task queue' task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name=queue_name, port=self._task_queue_port, @@ -872,7 +871,7 @@ def _StartWorkerProcess(self, process_name): process = extraction_process.ExtractionWorkerProcess( task_queue, self._processing_configuration, self._system_configurations, - self._registry_find_specs, + self._windows_event_log_providers, self._registry_find_specs, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) # Remove all possible log handlers to prevent a child process from logging @@ -894,9 +893,8 @@ def _StartWorkerProcess(self, process_name): except (IOError, KeyError) as exception: pid = process.pid logger.error(( - 'Unable to monitor replacement worker process: {0:s} ' - '(PID: {1:d}) with error: {2!s}').format( - process_name, pid, exception)) + f'Unable to monitor replacement worker process: {process_name:s} ' + f'(PID: {pid:d}) with error: {exception!s}')) self._TerminateProcess(process) return None @@ -1010,8 +1008,8 @@ def _UpdateProcessingStatus(self, pid, process_status, used_memory): current_timestamp = time.time() if current_timestamp > last_activity_timestamp: logger.error(( - 'Process {0:s} (PID: {1:d}) has not reported activity within ' - 'the timeout period.').format(process.name, pid)) + f'Process {process.name:s} (PID: {pid:d}) has not reported ' + f'activity within the timeout period.')) processing_status = definitions.STATUS_INDICATOR_NOT_RESPONDING self._processing_status.UpdateWorkerStatus( @@ -1029,9 +1027,9 @@ def _UpdateProcessingStatus(self, pid, process_status, used_memory): self._task_manager.UpdateTaskAsProcessingByIdentifier(task_identifier) return except KeyError: - logger.debug( - 'Worker {0:s} is processing unknown task: {1:s}.'.format( - process.name, task_identifier)) + logger.debug(( + f'Worker {process.name:s} is processing unknown task: ' + f'{task_identifier:s}.')) def _UpdateStatus(self): """Updates the status.""" @@ -1098,8 +1096,7 @@ def ProcessSourceMulti( filter_file_path=processing_configuration.filter_file) except errors.InvalidFilter as exception: raise errors.BadConfigOption( - 'Unable to build collection filters with error: {0!s}'.format( - exception)) + f'Unable to build collection filters with error: {exception!s}') self._event_data_timeliner = timeliner.EventDataTimeliner( data_location=processing_configuration.data_location, @@ -1120,6 +1117,8 @@ def ProcessSourceMulti( self._storage_file_path = storage_file_path self._storage_writer = storage_writer self._task_storage_format = processing_configuration.task_storage_format + self._windows_event_log_providers = list( + storage_writer.GetAttributeContainers('windows_eventlog_provider')) # Set up the task queue. task_outbound_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue( @@ -1138,11 +1137,10 @@ def ProcessSourceMulti( self._StartTaskStorage(self._task_storage_format) for worker_number in range(self._number_of_worker_processes): - process_name = 'Worker_{0:02d}'.format(self._last_worker_number) + process_name = f'Worker_{self._last_worker_number:02d}' worker_process = self._StartWorkerProcess(process_name) if not worker_process: - logger.error('Unable to create worker process: {0:d}'.format( - worker_number)) + logger.error(f'Unable to create worker process: {worker_number:d}') self._StartProfiling(self._processing_configuration.profiling) self._task_manager.StartProfiling( @@ -1198,8 +1196,7 @@ def ProcessSourceMulti( self._task_storage_format, session_identifier, abort=task_storage_abort) except (IOError, OSError) as exception: - logger.error('Unable to stop task storage with error: {0!s}'.format( - exception)) + logger.error(f'Unable to stop task storage with error: {exception!s}') if self._abort: logger.debug('Processing aborted.') @@ -1219,5 +1216,6 @@ def ProcessSourceMulti( self._storage_writer = None self._system_configurations = None self._task_storage_format = None + self._windows_event_log_providers = None return self._processing_status diff --git a/plaso/multi_process/extraction_process.py b/plaso/multi_process/extraction_process.py index a3163d6a8c..cc40bb3550 100644 --- a/plaso/multi_process/extraction_process.py +++ b/plaso/multi_process/extraction_process.py @@ -22,7 +22,7 @@ class ExtractionWorkerProcess(task_process.MultiProcessTaskProcess): def __init__( self, task_queue, processing_configuration, system_configurations, - registry_find_specs, **kwargs): + windows_event_log_providers, registry_find_specs, **kwargs): """Initializes an extraction worker process. Non-specified keyword arguments (kwargs) are directly passed to @@ -34,8 +34,10 @@ def __init__( configuration. system_configurations (list[SystemConfigurationArtifact]): system configurations. - registry_find_specs (list[dfwinreg.FindSpec]): Windows Registry find - specifications. + windows_event_log_providers (list[WindowsEventLogProviderArtifact]): + Windows EventLog providers. + registry_find_specs (list[dfwinreg.FindSpec]): Windows Registry find + specifications. kwargs: keyword arguments to pass to multiprocessing.Process. """ super(ExtractionWorkerProcess, self).__init__( @@ -53,6 +55,7 @@ def __init__( self._task = None self._task_queue = task_queue self._system_configurations = system_configurations + self._windows_event_log_providers = windows_event_log_providers def _CacheFileSystem(self, file_system): """Caches a dfVFS file system object. @@ -76,7 +79,8 @@ def _CacheFileSystem(self, file_system): self._file_system_cache.append(file_system) def _CreateParserMediator( - self, resolver_context, processing_configuration, system_configurations): + self, resolver_context, processing_configuration, system_configurations, + windows_event_log_providers): """Creates a parser mediator. Args: @@ -85,6 +89,8 @@ def _CreateParserMediator( configuration. system_configurations (list[SystemConfigurationArtifact]): system configurations. + windows_event_log_providers (list[WindowsEventLogProviderArtifact]): + Windows EventLog providers. Returns: ParserMediator: parser mediator. @@ -105,6 +111,8 @@ def _CreateParserMediator( parser_mediator.SetTemporaryDirectory( processing_configuration.temporary_directory) + parser_mediator.SetWindowsEventLogProviders(windows_event_log_providers) + return parser_mediator def _GetStatus(self): @@ -143,7 +151,7 @@ def _GetStatus(self): # XML RPC does not support integer values > 2 GiB so we format them # as a string. - used_memory = '{0:d}'.format(used_memory) + used_memory = f'{used_memory:d}' status = { 'display_name': self._current_display_name, @@ -177,7 +185,7 @@ def _Main(self): self._parser_mediator = self._CreateParserMediator( self._resolver_context, self._processing_configuration, - self._system_configurations) + self._system_configurations, self._windows_event_log_providers) # We need to initialize the parser and hasher objects after the process # has forked otherwise on Windows the "fork" will fail with @@ -200,21 +208,21 @@ def _Main(self): if self._processing_profiler: self._extraction_worker.SetProcessingProfiler(self._processing_profiler) - logger.debug('Worker: {0!s} (PID: {1:d}) started.'.format( - self._name, self._pid)) + logger.debug(f'Worker: {self._name!s} (PID: {self._pid:d}) started.') self._status = definitions.STATUS_INDICATOR_RUNNING try: - logger.debug('{0!s} (PID: {1:d}) started monitoring task queue.'.format( - self._name, self._pid)) + logger.debug( + f'{self._name!s} (PID: {self._pid:d}) started monitoring task queue.') while not self._abort: try: task = self._task_queue.PopItem() except (errors.QueueClose, errors.QueueEmpty) as exception: - logger.debug('ConsumeItems exiting with exception: {0!s}.'.format( - type(exception))) + exception_type = type(exception) + logger.debug( + f'ConsumeItems exiting with exception: {exception_type!s}.') break if isinstance(task, plaso_queue.QueueAbort): @@ -223,15 +231,15 @@ def _Main(self): self._ProcessTask(task) - logger.debug('{0!s} (PID: {1:d}) stopped monitoring task queue.'.format( - self._name, self._pid)) + logger.debug( + f'{self._name!s} (PID: {self._pid:d}) stopped monitoring task queue.') # All exceptions need to be caught here to prevent the process # from being killed by an uncaught exception. except Exception as exception: # pylint: disable=broad-except - logger.warning( - 'Unhandled exception in process: {0!s} (PID: {1:d}).'.format( - self._name, self._pid)) + logger.warning(( + f'Unhandled exception in process: {self._name!s} ' + f'(PID: {self._pid:d}).')) logger.exception(exception) self._abort = True @@ -255,13 +263,12 @@ def _Main(self): else: self._status = definitions.STATUS_INDICATOR_COMPLETED - logger.debug('Worker: {0!s} (PID: {1:d}) stopped.'.format( - self._name, self._pid)) + logger.debug(f'Worker: {self._name!s} (PID: {self._pid:d}) stopped.') try: self._task_queue.Close(abort=self._abort) except errors.QueueAlreadyClosed: - logger.error('Queue for {0:s} was already closed.'.format(self.name)) + logger.error(f'Queue for {self.name:s} was already closed.') def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec): """Processes a path specification. @@ -278,8 +285,8 @@ def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec): file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=parser_mediator.resolver_context) if file_entry is None: - logger.warning('Unable to open file entry: {0:s}'.format( - self._current_display_name)) + logger.warning( + f'Unable to open file entry: {self._current_display_name:s}') return if (path_spec and not path_spec.IsSystemLevel() and @@ -291,13 +298,13 @@ def _ProcessPathSpec(self, extraction_worker, parser_mediator, path_spec): except Exception as exception: # pylint: disable=broad-except parser_mediator.ProduceExtractionWarning(( - 'unable to process path specification with error: ' - '{0!s}').format(exception), path_spec=path_spec) + f'unable to process path specification with error: ' + f'{exception!s}'), path_spec=path_spec) if self._processing_configuration.debug_output: logger.warning(( - 'Unhandled exception while processing path specification: ' - '{0:s}.').format(self._current_display_name)) + f'Unhandled exception while processing path specification: ' + f'{self._current_display_name:s}.')) logger.exception(exception) def _ProcessTask(self, task): @@ -306,7 +313,7 @@ def _ProcessTask(self, task): Args: task (Task): task. """ - logger.debug('Started processing task: {0:s}.'.format(task.identifier)) + logger.debug(f'Started processing task: {task.identifier:s}.') if self._tasks_profiler: self._tasks_profiler.Sample(task, 'processing_started') @@ -359,7 +366,7 @@ def _ProcessTask(self, task): if self._tasks_profiler: self._tasks_profiler.Sample(task, 'processing_completed') - logger.debug('Completed processing task: {0:s}.'.format(task.identifier)) + logger.debug(f'Completed processing task: {task.identifier:s}.') def SignalAbort(self): """Signals the process to abort.""" diff --git a/plaso/output/winevt_rc.py b/plaso/output/winevt_rc.py index f89f1dbb23..12f0ff9feb 100644 --- a/plaso/output/winevt_rc.py +++ b/plaso/output/winevt_rc.py @@ -87,10 +87,15 @@ def GetValues(self, table_names, column_names, condition): raise RuntimeError('Cannot retrieve values database not opened.') if condition: - condition = ' WHERE {0:s}'.format(condition) + condition = f' WHERE {condition:s}' + else: + condition = '' - sql_query = 'SELECT {1:s} FROM {0:s}{2:s}'.format( - ', '.join(table_names), ', '.join(column_names), condition) + table_names_string = ', '.join(table_names) + column_names_string = ', '.join(column_names) + sql_query = ( + f'SELECT {column_names_string:s} FROM {table_names_string:s}' + f'{condition:s}') self._cursor.execute(sql_query) @@ -161,7 +166,7 @@ def _GetEventLogProviderKey(self, log_source): """ table_names = ['event_log_providers'] column_names = ['event_log_provider_key'] - condition = 'log_source == "{0:s}"'.format(log_source) + condition = f'log_source == "{log_source:s}"' values_list = list(self._database_file.GetValues( table_names, column_names, condition)) @@ -190,14 +195,14 @@ def _GetMessage(self, message_file_key, lcid, message_identifier): Raises: RuntimeError: if more than one value is found in the database. """ - table_name = 'message_table_{0:d}_0x{1:08x}'.format(message_file_key, lcid) + table_name = f'message_table_{message_file_key:d}_0x{lcid:08x}' has_table = self._database_file.HasTable(table_name) if not has_table: return None column_names = ['message_string'] - condition = 'message_identifier == "0x{0:08x}"'.format(message_identifier) + condition = f'message_identifier == "0x{message_identifier:08x}"' values = list(self._database_file.GetValues( [table_name], column_names, condition)) @@ -222,8 +227,7 @@ def _GetMessageFileKeys(self, event_log_provider_key): """ table_names = ['message_file_per_event_log_provider'] column_names = ['message_file_key'] - condition = 'event_log_provider_key == {0:d}'.format( - event_log_provider_key) + condition = f'event_log_provider_key == {event_log_provider_key:d}' generator = self._database_file.GetValues( table_names, column_names, condition) @@ -286,7 +290,7 @@ def GetMetadataAttribute(self, attribute_name): return None column_names = ['value'] - condition = 'name == "{0:s}"'.format(attribute_name) + condition = f'name == "{attribute_name:s}"' values = list(self._database_file.GetValues( [table_name], column_names, condition)) @@ -318,15 +322,14 @@ def Open(self, filename): version = self.GetMetadataAttribute('version') if not version or version != '20150315': - raise RuntimeError('Unsupported version: {0:s}'.format(version)) + raise RuntimeError(f'Unsupported version: {version:s}') string_format = self.GetMetadataAttribute('string_format') if not string_format: string_format = 'wrc' if string_format not in ('pep3101', 'wrc'): - raise RuntimeError('Unsupported string format: {0:s}'.format( - string_format)) + raise RuntimeError(f'Unsupported string format: {string_format:s}') self._string_format = string_format return True @@ -381,17 +384,16 @@ def _CacheMessageString( self._message_string_cache.popitem(last=True) if provider_identifier: - lookup_key = '{0:s}:0x{1:08x}'.format( - provider_identifier, message_identifier) + lookup_key = f'{provider_identifier:s}:0x{message_identifier:08x}' if event_version is not None: - lookup_key = '{0:s}:{1:d}'.format(lookup_key, event_version) + lookup_key = f'{lookup_key:s}:{event_version:d}' self._message_string_cache[lookup_key] = message_string self._message_string_cache.move_to_end(lookup_key, last=False) if log_source: - lookup_key = '{0:s}:0x{1:08x}'.format(log_source, message_identifier) + lookup_key = f'{log_source:s}:0x{message_identifier:08x}' if event_version is not None: - lookup_key = '{0:s}:{1:d}'.format(lookup_key, event_version) + lookup_key = f'{lookup_key:s}:{event_version:d}' self._message_string_cache[lookup_key] = message_string self._message_string_cache.move_to_end(lookup_key, last=False) @@ -412,16 +414,15 @@ def _GetCachedMessageString( message_string = None if provider_identifier: - lookup_key = '{0:s}:0x{1:08x}'.format( - provider_identifier, message_identifier) + lookup_key = f'{provider_identifier:s}:0x{message_identifier:08x}' if event_version is not None: - lookup_key = '{0:s}:{1:d}'.format(lookup_key, event_version) + lookup_key = f'{lookup_key:s}:{event_version:d}' message_string = self._message_string_cache.get(lookup_key, None) if not message_string and log_source: - lookup_key = '{0:s}:0x{1:08x}'.format(log_source, message_identifier) + lookup_key = f'{log_source:s}:0x{message_identifier:08x}' if event_version is not None: - lookup_key = '{0:s}:{1:d}'.format(lookup_key, event_version) + lookup_key = f'{lookup_key:s}:{event_version:d}' message_string = self._message_string_cache.get(lookup_key, None) if message_string: @@ -438,9 +439,9 @@ def _GetWinevtRcDatabaseReader(self): """ if not self._winevt_database_reader and self._data_location: logger.warning(( - 'Falling back to {0:s}. Please make sure the Windows EventLog ' - 'message strings in the database correspond to those in the ' - 'EventLog files.').format(self._WINEVT_RC_DATABASE)) + f'Falling back to {self._WINEVT_RC_DATABASE:s}. Please make sure ' + f'the Windows EventLog message strings in the database correspond ' + f'to those in the EventLog files.')) database_path = os.path.join( self._data_location, self._WINEVT_RC_DATABASE) @@ -547,24 +548,26 @@ def _ReadWindowsEventLogMessageString( 'windows_eventlog_message_string'): return None + original_message_identifier = message_identifier + # Map the event identifier to a message identifier as defined by the # WEVT_TEMPLATE event definition. if provider_identifier and storage_reader.HasAttributeContainers( 'windows_wevt_template_event'): # TODO: add message_file_identifiers to filter_expression filter_expression = ( - 'provider_identifier == "{0:s}" and identifier == {1:d}').format( - provider_identifier, message_identifier) + f'provider_identifier == "{provider_identifier:s}" and ' + f'identifier == {message_identifier:d}') if event_version is not None: - filter_expression = '{0:s} and version == {1:d}'.format( - filter_expression, event_version) + filter_expression = ( + f'{filter_expression:s} and version == {event_version:d}') for event_definition in storage_reader.GetAttributeContainers( 'windows_wevt_template_event', filter_expression=filter_expression): - logger.debug( - 'Message: 0x{0:08x} of provider: {1:s} maps to: 0x{2:08x}'.format( - message_identifier, provider_identifier, - event_definition.message_identifier)) + logger.debug(( + f'Message: 0x{message_identifier:08x} of provider: ' + f'{provider_identifier:s} maps to: ' + f'0x{event_definition.message_identifier:08x}')) message_identifier = event_definition.message_identifier break @@ -580,7 +583,7 @@ def _ReadWindowsEventLogMessageString( message_file_identifier = message_file_identifier.CopyToString() message_file_identifiers.append(message_file_identifier) - mui_filename = '{0:s}.mui'.format(filename) + mui_filename = f'{filename:s}.mui' lookup_path = '\\'.join([path, self._language_tag, mui_filename]).lower() message_file_identifier = self._windows_eventlog_message_files.get( lookup_path, None) @@ -589,17 +592,17 @@ def _ReadWindowsEventLogMessageString( message_file_identifiers.append(message_file_identifier) if not message_file_identifiers: - logger.warning( - 'No message file for message: 0x{0:08x} of provider: {1:s}'.format( - message_identifier, lookup_key)) + logger.warning(( + f'No message file for message: 0x{message_identifier:08x} ' + f'(original: 0x{original_message_identifier:08x}) ' + f'of provider: {lookup_key:s}')) return None message_strings = [] # TODO: add message_file_identifiers to filter_expression filter_expression = ( - 'language_identifier == {0:d} and ' - 'message_identifier == {1:d}').format( - self._lcid, message_identifier) + f'language_identifier == {self._lcid:d} and ' + f'message_identifier == {message_identifier:d}') for message_string in storage_reader.GetAttributeContainers( 'windows_eventlog_message_string', @@ -611,8 +614,9 @@ def _ReadWindowsEventLogMessageString( if not message_strings: logger.warning(( - 'No message string for message: 0x{0:08x} of provider: ' - '{1:s}').format(message_identifier, lookup_key)) + f'No message string for message: 0x{message_identifier:08x} ' + f'(original: 0x{original_message_identifier:08x}) ' + f'of provider: {lookup_key:s}')) return None return message_strings[0].string diff --git a/plaso/parsers/mediator.py b/plaso/parsers/mediator.py index 1c4d9296c9..e733671e07 100644 --- a/plaso/parsers/mediator.py +++ b/plaso/parsers/mediator.py @@ -76,6 +76,8 @@ def __init__( self._resolver_context = resolver_context self._storage_writer = None self._temporary_directory = None + self._windows_event_log_providers = None + self._windows_event_log_providers_per_filename = None self._windows_event_log_providers_per_path = None self.registry_find_specs = registry_find_specs @@ -298,7 +300,7 @@ def GetFilename(self): data_stream = getattr(self._file_entry.path_spec, 'data_stream', None) if data_stream: - return '{0:s}:{1:s}'.format(self._file_entry.name, data_stream) + return f'{self._file_entry.name:s}:{data_stream:s}' return self._file_entry.name @@ -358,15 +360,19 @@ def GetWindowsEventLogMessageFile(self): found. """ path_spec = getattr(self._file_entry, 'path_spec', None) + if not path_spec: + return None - if (self._windows_event_log_providers_per_path is None and - self._storage_writer): + if (self._windows_event_log_providers_per_filename is None and + self._windows_event_log_providers_per_path is None): environment_variables = self._GetEnvironmentVariablesByPathSpec(path_spec) + if not environment_variables: + return None + self._windows_event_log_providers_per_filename = {} self._windows_event_log_providers_per_path = {} - for provider in self._storage_writer.GetAttributeContainers( - 'windows_eventlog_provider'): + for provider in self._windows_event_log_providers: for windows_path in provider.event_message_files or []: path, filename = path_helper.PathHelper.GetWindowsSystemPath( windows_path, environment_variables) @@ -379,33 +385,44 @@ def GetWindowsEventLogMessageFile(self): self._windows_event_log_providers_per_path[path] = {} # Note that multiple providers can share EventLog message files. + self._windows_event_log_providers_per_filename[filename] = provider self._windows_event_log_providers_per_path[path][filename] = provider - message_file = None - if path_spec: - relative_path = path_helper.PathHelper.GetRelativePathForPathSpec( - path_spec) - lookup_path = relative_path.lower() + relative_path = path_helper.PathHelper.GetRelativePathForPathSpec(path_spec) + lookup_path = relative_path.lower() + + path_segment_separator = path_helper.PathHelper.GetPathSegmentSeparator( + path_spec) - path_segment_separator = path_helper.PathHelper.GetPathSegmentSeparator( - path_spec) + lookup_path, _, lookup_filename = lookup_path.rpartition( + path_segment_separator) - lookup_path, _, lookup_filename = lookup_path.rpartition( - path_segment_separator) + # Language specific EventLog message file paths contain a language tag + # such as "en-US". + base_lookup_path, _, last_path_segment = lookup_path.rpartition( + path_segment_separator) + if language_tags.LanguageTagHelper.IsLanguageTag(last_path_segment): + lookup_path = base_lookup_path - # Language specific EventLog message file paths contain a language tag - # such as "en-US". - base_lookup_path, _, last_path_segment = lookup_path.rpartition( - path_segment_separator) - if language_tags.LanguageTagHelper.IsLanguageTag(last_path_segment): - lookup_path = base_lookup_path + message_file = None + if (lookup_path == '\\windows\\systemresources' and + lookup_filename[-4:] == '.mun'): + lookup_filename = lookup_filename[:-4] + + provider = self._windows_event_log_providers_per_filename.get( + lookup_filename, None) + if provider: + windows_path = '\\'.join(['', 'Windows', 'System32', lookup_filename]) + message_file = artifacts.WindowsEventLogMessageFileArtifact( + path=relative_path, windows_path=windows_path) + + else: providers_per_filename = self._windows_event_log_providers_per_path.get( lookup_path, {}) for filename, provider in providers_per_filename.items(): - mui_filename = '{0:s}.mui'.format(filename) - if lookup_filename in (filename, mui_filename): + if lookup_filename in (filename, f'{filename:s}.mui'): windows_path = '\\'.join([lookup_path, filename]) message_file = artifacts.WindowsEventLogMessageFileArtifact( path=relative_path, windows_path=windows_path) @@ -545,6 +562,15 @@ def ProduceRecoveryWarning(self, message, path_spec=None): self.last_activity_timestamp = time.time() + def SetWindowsEventLogProviders(self, windows_event_log_providers): + """Sets the Windows EventLog providers. + + Rags: + windows_event_log_providers (list[WindowsEventLogProviderArtifact]): + Windows EventLog providers. + """ + self._windows_event_log_providers = windows_event_log_providers + def ResetFileEntry(self): """Resets the active file entry.""" self._file_entry = None @@ -649,8 +675,7 @@ def SetPreferredLanguage(self, language_tag): if language_tag: lcid = languages.WindowsLanguageHelper.GetLCIDForLanguageTag(language_tag) if not lcid: - raise ValueError('No LCID found for language tag: {0:s}.'.format( - language_tag)) + raise ValueError(f'No LCID found for language tag: {language_tag:s}.') language_tag = language_tag.lower() @@ -694,14 +719,14 @@ def StartProfiling(self, configuration, identifier, process_information): return if configuration.HaveProfileFormatChecks(): - identifier = '{0:s}-format_checks'.format(identifier) + identifier = f'{identifier:s}-format_checks' self._format_checks_cpu_time_profiler = profilers.CPUTimeProfiler( identifier, configuration) self._format_checks_cpu_time_profiler.Start() if configuration.HaveProfileParsers(): - identifier = '{0:s}-parsers'.format(identifier) + identifier = f'{identifier:s}-parsers' self._parsers_cpu_time_profiler = profilers.CPUTimeProfiler( identifier, configuration) diff --git a/plaso/single_process/extraction_engine.py b/plaso/single_process/extraction_engine.py index dac6bf460e..23f485ecec 100644 --- a/plaso/single_process/extraction_engine.py +++ b/plaso/single_process/extraction_engine.py @@ -128,7 +128,7 @@ def _CollectInitialEventSources( if self._CheckExcludedPathSpec(file_system, path_spec): display_name = parser_mediator.GetDisplayNameForPathSpec(path_spec) - logger.debug('Excluded from extraction: {0:s}.'.format(display_name)) + logger.debug(f'Excluded from extraction: {display_name:s}.') continue # TODO: determine if event sources should be DataStream or FileEntry @@ -244,8 +244,8 @@ def _ProcessPathSpec(self, parser_mediator, path_spec): file_entry = path_spec_resolver.Resolver.OpenFileEntry( path_spec, resolver_context=parser_mediator.resolver_context) if file_entry is None: - logger.warning('Unable to open file entry: {0:s}'.format( - self._current_display_name)) + logger.warning( + f'Unable to open file entry: {self._current_display_name:s}') return file_system = file_entry.GetFileSystem() @@ -255,8 +255,8 @@ def _ProcessPathSpec(self, parser_mediator, path_spec): self._CacheFileSystem(file_system) if self._CheckExcludedPathSpec(file_system, path_spec): - logger.debug('Excluded from extraction: {0:s}.'.format( - self._current_display_name)) + logger.debug( + f'Excluded from extraction: {self._current_display_name:s}.') return self._extraction_worker.ProcessFileEntry(parser_mediator, file_entry) @@ -272,15 +272,15 @@ def _ProcessPathSpec(self, parser_mediator, path_spec): # from being killed by an uncaught exception. except Exception as exception: # pylint: disable=broad-except parser_mediator.ProduceExtractionWarning(( - 'unable to process path specification with error: ' - '{0!s}').format(exception), path_spec=path_spec) + f'unable to process path specification with error: ' + f'{exception!s}'), path_spec=path_spec) if getattr(self._processing_configuration, 'debug_output', False): self._StopStatusUpdateThread() - logger.warning( - 'Unhandled exception while processing path spec: {0:s}.'.format( - self._current_display_name)) + logger.warning(( + f'Unhandled exception while processing path spec: ' + f'{self._current_display_name:s}.')) logger.exception(exception) pdb.post_mortem() @@ -362,7 +362,7 @@ def _UpdateStatus(self): def _CreateParserMediator( self, storage_writer, resolver_context, processing_configuration, - system_configurations): + system_configurations, windows_event_log_providers): """Creates a parser mediator. Args: @@ -372,6 +372,8 @@ def _CreateParserMediator( configuration. system_configurations (list[SystemConfigurationArtifact]): system configurations. + windows_event_log_providers (list[WindowsEventLogProviderArtifact]): + Windows EventLog providers. Returns: ParserMediator: parser mediator. @@ -393,8 +395,7 @@ def _CreateParserMediator( filter_file_path=processing_configuration.filter_file) except errors.InvalidFilter as exception: raise errors.BadConfigOption( - 'Unable to build collection filters with error: {0!s}'.format( - exception)) + f'Unable to build collection filters with error: {exception!s}') parser_mediator = parsers_mediator.ParserMediator( registry_find_specs=self._registry_find_specs, @@ -412,6 +413,8 @@ def _CreateParserMediator( parser_mediator.SetTemporaryDirectory( processing_configuration.temporary_directory) + parser_mediator.SetWindowsEventLogProviders(windows_event_log_providers) + return parser_mediator def ProcessSource( @@ -442,9 +445,12 @@ def ProcessSource( processing_configuration.artifact_definitions_path, processing_configuration.custom_artifacts_path) + windows_event_log_providers = list(storage_writer.GetAttributeContainers( + 'windows_eventlog_provider')) + parser_mediator = self._CreateParserMediator( storage_writer, resolver_context, processing_configuration, - system_configurations) + system_configurations, windows_event_log_providers) parser_mediator.SetStorageWriter(storage_writer) self._extraction_worker = worker.EventExtractionWorker( diff --git a/tests/multi_process/extraction_process.py b/tests/multi_process/extraction_process.py index 4b0bc79260..1cf2bc8bd3 100644 --- a/tests/multi_process/extraction_process.py +++ b/tests/multi_process/extraction_process.py @@ -52,7 +52,7 @@ def testInitialization(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') self.assertIsNotNone(test_process) def testGetStatus(self): @@ -62,7 +62,7 @@ def testGetStatus(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) @@ -94,7 +94,7 @@ def testMain(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - input_task_queue, configuration, [], None, name='TestWorker') + input_task_queue, configuration, [], [], None, name='TestWorker') test_process.start() @@ -114,7 +114,7 @@ def testProcessPathSpec(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') task_storage_writer = self._CreateStorageWriter() parser_mediator = self._CreateParserMediator(task_storage_writer) @@ -136,7 +136,7 @@ def testProcessTask(self): configuration.task_storage_format = definitions.STORAGE_FORMAT_SQLITE test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') test_process._extraction_worker = TestEventExtractionWorker() task_storage_writer = self._CreateStorageWriter() @@ -157,7 +157,7 @@ def testStartAndStopProfiling(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') test_process._extraction_worker = TestEventExtractionWorker() test_process._StartProfiling(None) @@ -172,7 +172,7 @@ def testSignalAbort(self): configuration.task_storage_path = temp_directory test_process = extraction_process.ExtractionWorkerProcess( - None, configuration, [], None, name='TestWorker') + None, configuration, [], [], None, name='TestWorker') test_process.SignalAbort()