diff --git a/plaso/data/formatters/ios.yaml b/plaso/data/formatters/ios.yaml index bd3c3b005c..f8ee6310a3 100644 --- a/plaso/data/formatters/ios.yaml +++ b/plaso/data/formatters/ios.yaml @@ -1,6 +1,35 @@ # Plaso iOS related event formatters. --- type: 'conditional' +data_type: 'apple:pstxt:entry' +message: +- 'Command: {command}' +- 'Control terminal name: {control_terminal_name}' +- '%CPU: {cpu}' +- 'Flags: {flags}' +- '%Memory: {memory}' +- 'Nice Value: {nice_value}' +- 'Persona: {persona}' +- 'Process identifier: {process_identifier}' +- 'Parent process identifier: {parent_process_identifier}' +- 'Resident set size: {resident_set_size}' +- 'Scheduling_priority: {scheduling_priority}' +- 'Symbolic_process_state: {symbolic_process_state}' +- 'CPU time: {up_time}' +- 'User: {user}' +- 'User_identifier: {user_identifier}' +- 'Virtual Size (kb): {virtual_size}' +- 'Wait Channel: {wait_channel}' +short_message: +- 'Command: {command}' +- 'Process identifier: {process_identifier}' +- 'Parent process identifier: {parent_process_identifier}' +- 'User: {user}' +- 'User_identifier: {user_identifier}' +short_source: 'Log' +source: 'Apple ps.txt' +--- +type: 'conditional' data_type: 'ios:app_privacy:access' message: - 'Accessor Identifier: {accessor_identifier}' diff --git a/plaso/data/presets.yaml b/plaso/data/presets.yaml index 19d4a6fe2b..8adc8273c9 100644 --- a/plaso/data/presets.yaml +++ b/plaso/data/presets.yaml @@ -27,6 +27,7 @@ parsers: - sqlite/ios_screentime - sqlite/kik_ios - sqlite/twitter_ios +- text/apple_ps_txt - text/ios_lockdownd - text/ios_logd - text/ios_sysdiag_log @@ -89,6 +90,7 @@ parsers: - sqlite/mackeeper_cache - sqlite/mac_knowledgec - sqlite/skype +- text/apple_ps_txt - text/bash_history - text/gdrive_synclog - text/mac_appfirewall_log diff --git a/plaso/data/timeliner.yaml b/plaso/data/timeliner.yaml index 2022e8a8c4..40c99b8bd4 100644 --- a/plaso/data/timeliner.yaml +++ b/plaso/data/timeliner.yaml @@ -95,6 +95,12 @@ attribute_mappings: description: 'Recorded Time' place_holder_event: true --- +data_type: 'apple:pstxt:entry' +attribute_mappings: +- name: 'start_time' + description: 'Start time' +place_holder_event: true +--- data_type: 'av:defender:detection_history' attribute_mappings: - name: 'recorded_time' diff --git a/plaso/lib/dateless_helper.py b/plaso/lib/dateless_helper.py index bbb2a0918a..fe55e51a03 100644 --- a/plaso/lib/dateless_helper.py +++ b/plaso/lib/dateless_helper.py @@ -10,6 +10,16 @@ class DateLessLogFormatHelper(object): """Date-less log format helper mix-in.""" + _WEEKDAY_DICT = { + 'mon': 1, + 'tue': 2, + 'wed': 3, + 'thu': 4, + 'fri': 5, + 'sat': 6, + 'sun': 7 + } + _MONTH_DICT = { 'jan': 1, 'feb': 2, @@ -85,6 +95,18 @@ def _GetMonthFromString(self, month_string): # TODO: add support for localization. return self._MONTH_DICT.get(month_string.lower(), None) + def _GetWeekDayFromString(self, weekday_string): + """Retrieves a numeric day of week value from a string. + + Args: + weekday_string (str): day of week formatted as a string. + + Returns: + int: day of week formatted as integer, where Monday is 1. + """ + # TODO: add support for localization. + return self._WEEKDAY_DICT.get(weekday_string.lower(), None) + def _GetRelativeYear(self): """Retrieves the relative year. diff --git a/plaso/parsers/text_plugins/__init__.py b/plaso/parsers/text_plugins/__init__.py index 9494e75b83..253c48cc74 100644 --- a/plaso/parsers/text_plugins/__init__.py +++ b/plaso/parsers/text_plugins/__init__.py @@ -3,6 +3,7 @@ from plaso.parsers.text_plugins import android_logcat from plaso.parsers.text_plugins import apache_access +from plaso.parsers.text_plugins import apple_pstxt from plaso.parsers.text_plugins import apt_history from plaso.parsers.text_plugins import aws_elb_access from plaso.parsers.text_plugins import bash_history diff --git a/plaso/parsers/text_plugins/apple_pstxt.py b/plaso/parsers/text_plugins/apple_pstxt.py new file mode 100644 index 0000000000..dfb2e79b90 --- /dev/null +++ b/plaso/parsers/text_plugins/apple_pstxt.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +"""Text parser plugin for Apple ps.txt files from sysdiagnose files.""" + +import datetime +import pyparsing + +from dfdatetime import time_elements as dfdatetime_time_elements + +from plaso.containers import events +from plaso.lib import dateless_helper +from plaso.lib import errors +from plaso.parsers import text_parser +from plaso.parsers.text_plugins import interface + + +class ApplePSTxtEventData(events.EventData): + """Apple ps.txt event data. + + Attributes: + command (str): Command and arguments that launched the process. + control_terminal_name (str): control terminal name (two letter + abbreviation). + cpu (str): % of cpu usage. + flags (str): Process flags, in hexadecimal. + memory (str): % of memory usage. + nice_value (str): nice_value. + persona (str): persona. + process_identifier (str): process identifier. + parent_process_identifier (str): process_parent_identifier + resident_set_size (str): Resident Set Size + scheduling_priority (str): Scheduling Priority + start_time (dfdatetime.DateTimeValues): Start time of the process. + symbolic_process_state (str): Symbolic process state. + up_time (str): Accumulated cpu time. + user (str): User name. + user_identifier (str): User identifier + virtual_size (str): Virtual size in kilobytes. + wait_channel (str): Wait channel. + """ + + DATA_TYPE = 'apple:pstxt:entry' + + def __init__(self): + """Initializes event data.""" + super(ApplePSTxtEventData, self).__init__(data_type=self.DATA_TYPE) + self.command = None + self.control_terminal_name = None + self.cpu = None + self.flags = None + self.memory = None + self.nice_value = None + self.persona = None + self.process_identifier = None + self.parent_process_identifier = None + self.resident_set_size = None + self.scheduling_priority = None + self.start_time = None + self.symbolic_process_state = None + self.up_time = None + self.user = None + self.user_identifier = None + self.virtual_size = None + self.wait_channel = None + + +class ApplePSTextPlugin( + interface.TextPlugin, dateless_helper.DateLessLogFormatHelper): + """Text parser plugin for Apple ps.txt files.""" + + NAME = 'apple_ps_txt' + DATA_FORMAT = 'Apple ps.txt files' + + _DATE = ( + pyparsing.Word(pyparsing.nums, min=1, max=2).set_results_name('day') + + pyparsing.Word(pyparsing.alphanums, exact=3).set_results_name('month') + + pyparsing.Word(pyparsing.nums, exact=2).set_results_name('year') + ) + + _TIME = ( + pyparsing.Word(pyparsing.nums, min=1, max=2).set_results_name('hours') + + pyparsing.Suppress(':') + + pyparsing.Word(pyparsing.nums, exact=2).set_results_name('minutes') + + pyparsing.one_of('AM PM').set_results_name('ampm') + ) + + _WEEKDAY = ( + pyparsing.Word(pyparsing.alphas, exact=3).set_results_name('weekday') + + pyparsing.Word(pyparsing.nums, exact=2).set_results_name('hours') + + pyparsing.one_of('AM PM').set_results_name('ampm') + ) + + _HEADER = ( + pyparsing.Literal('USER UID PRSNA PID PPID F %CPU ' + '%MEM PRI NI VSZ RSS WCHAN TT STAT STARTED' + ' TIME COMMAND') + pyparsing.LineEnd()) + + _COMMAND = pyparsing.OneOrMore( + pyparsing.Word(pyparsing.printables), + stop_on=pyparsing.LineEnd()).set_parse_action(' '.join) + + _LOG_LINE = ( + pyparsing.Word(pyparsing.alphanums + '_').set_results_name('user') + + pyparsing.Word(pyparsing.nums).set_results_name('user_identifier') + + pyparsing.Word(pyparsing.nums + '-').set_results_name('persona') + + pyparsing.Word(pyparsing.nums).set_results_name('process_identifier') + + pyparsing.Word(pyparsing.nums).set_results_name( + 'parent_process_identifier') + + pyparsing.Word(pyparsing.alphanums).set_results_name('flags') + + pyparsing.Word(pyparsing.nums + '.').set_results_name('cpu') + + pyparsing.Word(pyparsing.nums + '.').set_results_name('memory') + + pyparsing.Word(pyparsing.nums).set_results_name('scheduling_priority') + + pyparsing.Word(pyparsing.nums).set_results_name('nice_value') + + pyparsing.Word(pyparsing.nums).set_results_name('virtual_size') + + pyparsing.Word(pyparsing.nums).set_results_name('resident_set_size') + + pyparsing.Word(pyparsing.nums + '-').set_results_name('wait_channel') + + pyparsing.Word(pyparsing.alphanums + '?').set_results_name( + 'control_terminal_name') + + pyparsing.Word(pyparsing.alphanums).set_results_name( + 'symbolic_process_state') + + pyparsing.Word(pyparsing.alphanums + ':').set_results_name('start_time') + + pyparsing.Word(pyparsing.nums + '.:').set_results_name('up_time') + + _COMMAND.set_results_name('command') + ) + + _LINE_STRUCTURES = [('log_line', _LOG_LINE), ('header', _HEADER)] + + VERIFICATION_GRAMMAR = pyparsing.Suppress(_HEADER) + _LOG_LINE + + def _ParseRecord(self, parser_mediator, key, structure): + """Parses a pyparsing structure. + + Args: + parser_mediator (ParserMediator): mediates interactions between parsers + and other components, such as storage and dfVFS. + key (str): name of the parsed structure. + structure (pyparsing.ParseResults): tokens from a parsed log line. + + Raises: + ParseError: if the structure cannot be parsed. + """ + if key == 'log_line': + event_data = ApplePSTxtEventData() + event_data.user = self._GetValueFromStructure(structure, 'user') + event_data.user_identifier = self._GetValueFromStructure( + structure, 'user_identifier') + event_data.persona = self._GetValueFromStructure(structure, 'persona') + event_data.process_identifier = self._GetValueFromStructure( + structure, 'process_identifier') + event_data.parent_process_identifier = self._GetValueFromStructure( + structure, 'parent_process_identifier') + event_data.flags = self._GetValueFromStructure(structure, 'flags') + event_data.cpu = self._GetValueFromStructure(structure, 'cpu') + event_data.memory = self._GetValueFromStructure(structure, 'memory') + event_data.scheduling_priority = self._GetValueFromStructure( + structure, 'scheduling_priority') + event_data.nice_value = self._GetValueFromStructure( + structure, 'nice_value') + event_data.virtual_size = self._GetValueFromStructure( + structure, 'virtual_size') + event_data.resident_set_size = self._GetValueFromStructure( + structure, 'resident_set_size') + event_data.wait_channel = self._GetValueFromStructure( + structure, 'wait_channel') + event_data.control_terminal_name = self._GetValueFromStructure( + structure, 'control_terminal_name') + event_data.symbolic_process_state = self._GetValueFromStructure( + structure, 'symbolic_process_state') + event_data.up_time = self._GetValueFromStructure(structure, 'up_time') + event_data.command = self._GetValueFromStructure(structure, 'command') + + start_time_string = self._GetValueFromStructure( + structure, 'start_time') + event_data.start_time = self._ParseStartTime(start_time_string) + + parser_mediator.ProduceEventData(event_data) + + def _ParseStartTime(self, start_time_string): + """Parses the start time element of a log line. + + Args: + start_time_string (str): start time element of a log line. + + Returns: + dfdatetime.TimeElements: date and time value. + + Raises: + ParseError: if a valid date and time value cannot be derived from + the time elements. + """ + try: + # Case where it's just a time + if ( + start_time_string[0] in pyparsing.nums + and start_time_string[-1] == 'M'): + parsed_elements = self._TIME.parse_string(start_time_string) + + hours = int(parsed_elements.get('hours')) + minutes = int(parsed_elements.get('minutes')) + ampm = parsed_elements.get('ampm') + if ampm == 'PM': + hours += 12 + if hours == 24: + hours = 0 + + # Retrieve year, month, day from dateless helper + year = self._date[0] + month = self._date[1] + day = self._date[2] + + time_elements_tuple = (year, month, day, hours, minutes, 0) + + # Case where it is a weekday with a time + elif start_time_string[0] in pyparsing.alphas: + parsed_elements = self._WEEKDAY.parse_string(start_time_string) + + hours = int(parsed_elements.get('hours')) + weekday_string = parsed_elements.get('weekday') + weekday = self._GetWeekDayFromString(weekday_string) + ampm = parsed_elements.get('ampm') + if ampm == 'PM': + hours += 12 + if hours == 24: + hours = 0 + + # Retrieve year, month, day from dateless helper + year = self._date[0] + month = self._date[1] + day = self._date[2] + + date_from_file = datetime.datetime(year, month, day) + days_before = (date_from_file.isoweekday() - weekday + 7) % 7 + date_of_record = date_from_file - datetime.timedelta(days=days_before) + + time_elements_tuple = ( + date_of_record.year, date_of_record.month, date_of_record.day, + hours, 0, 0) + + # Case where it is a date with no time + elif start_time_string[0] in pyparsing.nums: + parsed_elements = self._DATE.parse_string(start_time_string) + + year = int(parsed_elements.get('year')) + 2000 + month_str = parsed_elements.get('month') + month = self._GetMonthFromString(month_str) + day = int(parsed_elements.get('day')) + + time_elements_tuple = (year, month, day, 0, 0, 0) + + else: + raise errors.ParseError('start time was not in an expected format.') + + return dfdatetime_time_elements.TimeElements( + time_elements_tuple=time_elements_tuple) + + except (TypeError, ValueError) as exception: + raise errors.ParseError( + f'Unable to parse time elements with error: {exception!s}') + + def CheckRequiredFormat(self, parser_mediator, text_reader): + """Check if the log record has the minimal structure required by the parser. + + Args: + parser_mediator (ParserMediator): mediates interactions between parsers + and other components, such as storage and dfVFS. + text_reader (EncodedTextReader): text reader. + + Returns: + bool: True if this is the correct parser, False otherwise. + """ + try: + structure = self._VerifyString(text_reader.lines) + except errors.ParseError: + return False + + start_time_string = self._GetValueFromStructure( + structure, 'start_time') + + try: + self._ParseStartTime(start_time_string) + except errors.ParseError: + return False + + # Retrieve the data from the file's metadata + self._SetEstimatedDate(parser_mediator) + + return True + + +text_parser.TextLogParser.RegisterPlugin(ApplePSTextPlugin) diff --git a/test_data/text_parser/ps.txt.gz b/test_data/text_parser/ps.txt.gz new file mode 100644 index 0000000000..fd86fa4b0a Binary files /dev/null and b/test_data/text_parser/ps.txt.gz differ diff --git a/tests/parsers/text_plugins/apple_pstxt.py b/tests/parsers/text_plugins/apple_pstxt.py new file mode 100644 index 0000000000..0d419dcdd8 --- /dev/null +++ b/tests/parsers/text_plugins/apple_pstxt.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Tests for the Apple ps.txt files text parser plugin.""" + +import unittest + +from plaso.parsers.text_plugins import apple_pstxt + +from tests.parsers.text_plugins import test_lib + + +class ApplePSTextPluginTest(test_lib.TextPluginTestCase): + """Tests for the ApplePSTextPlugin parser plugin.""" + + def testProcess(self): + """Tests the Process function.""" + plugin = apple_pstxt.ApplePSTextPlugin() + + storage_writer = self._ParseCompressedTextFileWithPlugin( + 'test_data/text_parser/ps.txt.gz', plugin) + + number_of_event_data = storage_writer.GetNumberOfAttributeContainers( + 'event_data') + self.assertEqual(number_of_event_data, 299) + + number_of_warnings = storage_writer.GetNumberOfAttributeContainers( + 'extraction_warning') + self.assertEqual(number_of_warnings, 0) + + expected_event_values = { + 'command': '/usr/libexec/UserEventAgent (System)', + 'control_terminal_name': '??', + 'cpu': '3.1', + 'flags': '4004004', + 'memory': '0.5', + 'nice_value': '0', + 'persona': '199', + 'process_identifier': '29', + 'parent_process_identifier': '1', + 'resident_set_size': '10208', + 'scheduling_priority': '37', + 'start_time': '2022-02-23T00:00:00+00:00', + 'symbolic_process_state': 'Ss', + 'up_time': '154:26.18', + 'user': 'root', + 'user_identifier': '0', + 'virtual_size': '407963424', + 'wait_channel': '-' + } + + event_data = storage_writer.GetAttributeContainerByIndex('event_data', 1) + self.CheckEventData(event_data, expected_event_values) + + expected_event_values = { + 'command': '/usr/libexec/findmydeviced', + 'control_terminal_name': '??', + 'cpu': '0.0', + 'flags': '4004004', + 'memory': '0.2', + 'nice_value': '0', + 'persona': '199', + 'process_identifier': '18988', + 'parent_process_identifier': '1', + 'resident_set_size': '3456', + 'scheduling_priority': '4', + 'start_time': '2022-04-08T11:00:00+00:00', + 'symbolic_process_state': 'Ss', + 'up_time': '0:11.52', + 'user': 'mobile', + 'user_identifier': '501', + 'virtual_size': '407957312', + 'wait_channel': '-' + } + + event_data = storage_writer.GetAttributeContainerByIndex('event_data', 52) + self.CheckEventData(event_data, expected_event_values) + + expected_event_values = { + 'command': ('/System/Library/PrivateFrameworks/' + 'MediaAnalysis.framework/mediaanalysisd'), + 'control_terminal_name': '??', + 'cpu': '0.0', + 'flags': '4004004', + 'memory': '0.1', + 'nice_value': '0', + 'persona': '199', + 'process_identifier': '64983', + 'parent_process_identifier': '1', + 'resident_set_size': '1680', + 'scheduling_priority': '4', + 'start_time': '2022-04-13T06:21:00+00:00', + 'symbolic_process_state': 'Ss', + 'up_time': '0:00.18', + 'user': 'mobile', + 'user_identifier': '501', + 'virtual_size': '407952752', + 'wait_channel': '-' + } + + event_data = storage_writer.GetAttributeContainerByIndex('event_data', 258) + self.CheckEventData(event_data, expected_event_values) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parsers/text_plugins/test_lib.py b/tests/parsers/text_plugins/test_lib.py index 42df6635b7..e737e50c11 100644 --- a/tests/parsers/text_plugins/test_lib.py +++ b/tests/parsers/text_plugins/test_lib.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- """Text parser plugin related functions and classes for testing.""" +from dfvfs.lib import definitions as dfvfs_definitions +from dfvfs.path import factory as path_spec_factory +from dfvfs.resolver import resolver as path_spec_resolver + from plaso.containers import events from plaso.parsers import mediator as parsers_mediator from plaso.parsers import text_parser @@ -65,3 +69,63 @@ def _ParseTextFileWithPlugin(self, path_segments, plugin): parser_mediator.AddDateLessLogHelper(date_less_log_helper) return storage_writer + + def _ParseCompressedTextFileWithPlugin(self, path_string, plugin): + """Parses a file contained in an archive as a text log file and returns an + event generator. + + This method will first test if a text log file has the required format + using plugin.CheckRequiredFormat() and then extracts events using + plugin.Process(). + + Args: + path_string (str): path segments inside the test data directory. + plugin (TextPlugin): text log file plugin. + + Returns: + FakeStorageWriter: storage writer. + + Raises: + SkipTest: if the path inside the test data directory does not exist and + the test should be skipped. + """ + parser_mediator = parsers_mediator.ParserMediator() + + storage_writer = self._CreateStorageWriter() + parser_mediator.SetStorageWriter(storage_writer) + + os_path_spec = path_spec_factory.Factory.NewPathSpec( + dfvfs_definitions.TYPE_INDICATOR_OS, location=path_string) + gzip_path_spec = path_spec_factory.Factory.NewPathSpec( + dfvfs_definitions.TYPE_INDICATOR_GZIP, parent=os_path_spec) + file_entry = path_spec_resolver.Resolver.OpenFileEntry(gzip_path_spec) + parser_mediator.SetFileEntry(file_entry) + + if file_entry: + event_data_stream = events.EventDataStream() + event_data_stream.path_spec = file_entry.path_spec + + parser_mediator.ProduceEventDataStream(event_data_stream) + + # AppendToParserChain needs to be run after SetFileEntry. + parser_mediator.AppendToParserChain('text') + + encoding = plugin.ENCODING + if not encoding: + encoding = parser_mediator.GetCodePage() + + file_object = file_entry.GetFileObject() + text_reader = text_parser.EncodedTextReader(file_object, encoding=encoding) + + text_reader.ReadLines() + + required_format = plugin.CheckRequiredFormat(parser_mediator, text_reader) + self.assertTrue(required_format) + + plugin.UpdateChainAndProcess(parser_mediator, file_object=file_object) + + if hasattr(plugin, 'GetDateLessLogHelper'): + date_less_log_helper = plugin.GetDateLessLogHelper() + parser_mediator.AddDateLessLogHelper(date_less_log_helper) + + return storage_writer