From 00df5f2fe12d7cc23fdb50bcb22e1fd66fc52342 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Sun, 21 May 2023 10:26:04 +0200 Subject: [PATCH] Worked on Apple Unified Logging format support --- ...ging and Activity Tracing formats.asciidoc | 232 ++++++++++++++--- dtformats/macos_core_location.yaml | 142 ++++++++++ dtformats/unified_logging.py | 244 +++++++++++++----- tests/unified_logging.py | 140 ++++++++-- 4 files changed, 642 insertions(+), 116 deletions(-) create mode 100644 dtformats/macos_core_location.yaml diff --git a/documentation/Apple Unified Logging and Activity Tracing formats.asciidoc b/documentation/Apple Unified Logging and Activity Tracing formats.asciidoc index 2cfeaef..9269663 100644 --- a/documentation/Apple Unified Logging and Activity Tracing formats.asciidoc +++ b/documentation/Apple Unified Logging and Activity Tracing formats.asciidoc @@ -778,8 +778,9 @@ Contains a 32-bit value | 0x02 | | [yellow-background]*Unknown (integer)* + Contains a 8-bit, 16-bit, 32-bit or 64-bit value 3+| -| 0x12 | | [yellow-background]*Unknown* + -Contains a 32-bit value +| 0x12 | | [yellow-background]*Unknown (format string precision)* + +Contains a 32-bit value + +This value has been seen to be used in combination with format strings like "%.16s" and "%.*s", where this value contains the number of characters of the string that should be printed. 3+| | 0x20 | | [yellow-background]*Unknown (string)* + Consists of a <> where the value data contains an UTF-8 encoded string with an optional end-of-string character. @@ -802,12 +803,12 @@ Consists of a <> where the value data contains an UTF-8 encoded string with an optional end-of-string character. | 0x41 | | [yellow-background]*Unknown (private string)* + -Contains a 32-bit value +Contains a 32-bit value, formatted as "" | 0x42 | | [yellow-background]*Unknown (string)* + Consists of a <> where the value data contains an UTF-8 encoded string with an optional end-of-string character. 3+| -| 0xf2 | | [yellow-background]*Unknown (UUID)* -Consists of a <> where the value data contains an UUID. +| 0xf2 | | [yellow-background]*Unknown (binary data)* + +Consists of a <> where the value data contains binary data. |=== ===== [[tracev3_firehose_tracepoint_data_time_with_value_data_range]]Firehose tracepoint data item with value data range @@ -1020,33 +1021,77 @@ The built-in value type decoders are: [cols="1,1,5",options="header"] |=== | Value | Identifier | Description -| "{bitrate}" | | Formatted as a bit-rate value, for example "123 kbps" -| "{bool}" | | Formatted as a lower-case boolean value, for example "true" or "false" -| "{BOOL}" | | Formatted as a uppoer-case boolean value, for example "YES" or "NO" -| "{bytes}" | | Formatted a bytes value, for example "4.72 kB" -| "{darwin.errno}" | | Formatted as a system error, for example "[32: Broken pipe]" -| "{darwin.mode}" | | Formatted as a file mode value, for example "drwxr-xr-x" -| "{darwin.signal}" | | Formatted as a signal, for example "[sigsegv: Segmentation Fault]" -| "{errno}" | | Formatted as a system error, for example "[32: Broken pipe]" -| "{iec-bitrate}" | | Formatted as an IEC bit-rate value, for example "118 Kibps" -| "{iec-bytes}" | | Formatted as IEC bytes value, for example "4.61 KiB" -| "{in_addr}" | | Formatted as an IPv4 address, for example "127.0.0.1" -| "{in6_addr}" | | Formatted as an IPv6 address, for example "fe80::f:86ff:fee9:5c16" -| "{private}" | | Private log argument -| "{public}" | | Public log argument -| "{sockaddr}" | | Formatted as socket address, for example "fe80::f:86ff:fee9:5c16" -| "{time_t}" | | Formatted as a seconds precision date and time value, for example "2016-01-12 19:41:37" -| "{timespec}" | | Formatted as a nanoseconds precision date and time value, for example "2016-01-12 19:41:37.2382382823" -| "{timeval}" | | Formatted as a microseconds precision date and time value, for example "2016-01-12 19:41:37.774236" -| "{uuid_t}" | | Formatted as an UUID, for example "10742E39-0657-41F8-AB99-878C5EC2DCAA" +| "bitrate" | | Formatted as a bit-rate value, for example "123 kbps" +| "bool" | | Formatted as a lower-case boolean value, for example "true" or "false" +| "BOOL" | | Formatted as a uppoer-case boolean value, for example "YES" or "NO" +| "bytes" | | Formatted a bytes value, for example "4.72 kB" +| "darwin.errno" | | Formatted as a system error, for example "[32: Broken pipe]" +| "darwin.mode" | | Formatted as a file mode value, for example "drwxr-xr-x" +| "darwin.signal" | | Formatted as a signal, for example "[sigsegv: Segmentation Fault]" +| "iec-bitrate" | | Formatted as an IEC bit-rate value, for example "118 Kibps" +| "iec-bytes" | | Formatted as IEC bytes value, for example "4.61 KiB" +| "in_addr" | | Formatted as an IPv4 address, for example "127.0.0.1" +| "in6_addr" | | Formatted as an IPv6 address, for example "fe80::f:86ff:fee9:5c16" +| "sockaddr" | | Formatted as socket address, for example "fe80::f:86ff:fee9:5c16" +| "time_t" | | Formatted as a seconds precision date and time value, for example "2016-01-12 19:41:37" +| "timespec" | | Formatted as a nanoseconds precision date and time value, for example "2016-01-12 19:41:37.2382382823" +| "timeval" | | Formatted as a microseconds precision date and time value, for example "2016-01-12 19:41:37.774236" +| "uuid_t" | | Formatted as an UUID, for example "10742E39-0657-41F8-AB99-878C5EC2DCAA" +|=== + +Other observerd value type decoders are: + +[cols="1,1,5",options="header"] +|=== +| Value | Identifier | Description +| "errno" | | Formatted as a system error, for example "[32: Broken pipe]" +| "location:_CLClientManagerStateTrackerState" | | Formatted as a <> +| "location:_CLLocationManagerStateTrackerState" | | Formatted as a <> +| "location:CLClientAuthorizationStatus" | | +| "location:CLDaemonStatus_Type::Reachability" | | +| "location:CLSubHarvesterIdentifier" | | +| "location:escape_only" | | +| "location:IOMessage" | | +| "location:SqliteResult" | | +| "mask.hash" | | Formatted as "" where "%s" contains the base64 encoded value. +| "mdns:acceptable" | | +| "mdns:addrmv" | | +| "mdns:dns.counts" | | +| "mdns:dns.idflags" | | +| "mdns:dnshdr" | | +| "mdns:gaiopts" | | +| "mdns:nreason" | | +| "mdns:protocol" | | +| "mdns:rd.svcb" | | +| "mdns:rrtype" | | +| "mdns:yesno" | | +| "mdnsresponder:domain_name" | | +| "mdnsresponder:ip_addr" | | +| "mdnsresponder:mac_addr" | | +| "network:in_addr" | | +| "network:in6_addr" | | +| "network:sockaddr" | | +| "network:tcp_flags" | | +| "network:tcp_state" | | +| "odtypes:ODError" | | +| "odtypes:mbr_details" | | +| "odtypes:mbridtype" | | +| "odtypes:nt_sid_t" | | +| "sensitive" | | +| "private" | | Private log argument +| "public" | | Public log argument |=== +[NOTE] +The multiple value type decoders can be used in combination for example +"%{public,uuid_t}.16P" or "%{private, mask.hash, mdnsresponder:ip_addr}.20P". + The flags are defined as: [cols="1,1,5",options="header"] |=== | Value | Identifier | Description -| "#" | | +| "#" | | Value should be converted to an "alternate form" | "0" | | Value should be padded with 0 | "-" | | | " " | | @@ -1069,36 +1114,45 @@ The length modifiers are defined as: | "z" | | size_t |=== +The .precision is defined as: + +[cols="1,1,5",options="header"] +|=== +| Value | Identifier | Description +| "0" | | Observed that this has no effect in "%.0s" +| "*" | | An additional integer argument supplies the field width or precision. +|=== + The types are defined as: [cols="1,1,5",options="header"] |=== | Value | Identifier | Description | "@" | | Obj-C/CF/Swift object -| "a" | | -| "A" | | +| "a" | | Floating-point value +| "A" | | Floating-point value | "c" | | Character value -| "C" | | Equivalent to "lc" +| "C" | | wide character value, equivalent to "lc" | "d" | | Signed decimal integer value -| "D" | | -| "e" | | -| "E" | | -| "f" | | -| "F" | | -| "g" | | -| "G" | | -| "i" | | +| "D" | | Long signed decimal integer value, equivalent to "ld" +| "e" | | Floating-point value +| "E" | | Floating-point value +| "f" | | Floating-point value +| "F" | | Floating-point value +| "g" | | Floating-point value +| "G" | | Floating-point value +| "i" | | Signed decimal integer value | "n" | | -| "o" | | -| "O" | | -| "p" | | Pointer value +| "o" | | Octal integer value +| "O" | | Long octal integer value, equivalent to "lo" +| "p" | | Pointer value, equivalent to "0x%x" | "P" | | Binary data | "s" | | String value -| "S" | | Equivalent to "ls" +| "S" | | Wide character string value, equivalent to "ls" | "u" | | Unsigned decimal integer value -| "U" | | -| "x" | | Lower case hexadecimal interger value -| "X" | | Upper case hexadecimal interger value +| "U" | | Long unsigned decimal integer value, equivalent to "lu" +| "x" | | Hexadecimal interger value, formatter in lower case +| "X" | | Hexadecimal interger value, formatter in upper case |=== === Oversize chunk @@ -1439,6 +1493,98 @@ The UUID text (uuidtext) entry descriptor is 8 bytes of size and consists of: Contains an UTF-8 formatted string with an end-of-string character |=== +== Value type decoders + +=== [[core_location_client_manager_state_tracker_state]]Core location client manager (CLClientManager) state tracker state + +[cols="1,1,1,5",options="header"] +|=== +| Offset | Size | Value | Description +| 0 | 4 | | Location enabled status +| 4 | 4 | | Location restricted + +Contains a boolean value where false if 0 or true otherwise +|=== + +[yellow-background]*TODO confirm location enabled status is the first value in the +structure. Only seen data where both values are 0.* + +The value is formatted as: + +.... +{"locationRestricted":false,"locationServicesEnabledStatus":0} +.... + +=== [[core_location_location_manager_state_tracker_state]]Core location location manager (CLLocationManager) state tracker state + +[cols="1,1,1,5",options="header"] +|=== +| Offset | Size | Value | Description +| 0 | 8 | | Distance filter + +Contains a floating-point value +| 8 | 8 | | Desired accuracy + +Contains a floating-point value +| 16 | 1 | | Updating location + +Contains a boolean value where false if 0 or true otherwise +| 17 | 1 | | Requestiong location + +Contains a boolean value where false if 0 or true otherwise +| 18 | 1 | | Requestiong ranging + +Contains a boolean value where false if 0 or true otherwise +| 19 | 1 | | Updating ranging + +Contains a boolean value where false if 0 or true otherwise +| 20 | 1 | | Updating heading + +Contains a boolean value where false if 0 or true otherwise +| 21 | 3 | | [yellow-background]*Unknown* +| 24 | 8 | | Heading filter + +Contains a floating-point value +| 32 | 1 | | Allows location prompts + +Contains a boolean value where false if 0 or true otherwise +| 33 | 1 | | Allows altered accessory location + +Contains a boolean value where false if 0 or true otherwise +| 34 | 1 | | Dynamic accuracy reduction enabled + +Contains a boolean value where false if 0 or true otherwise +| 35 | 1 | | Previous authorization status valid + +Contains a boolean value where false if 0 or true otherwise +| 36 | 4 | | Previous authorization status +| 40 | 1 | | Limits precision + +Contains a boolean value where false if 0 or true otherwise +| 41 | 7 | | [yellow-background]*Unknown* +| 48 | 8 | | Activity type + +Contains a signed integer +| 56 | 4 | | Pauses location updates automatically + +Contains a signed integer +| 60 | 1 | | Paused + +Contains a boolean value where false if 0 or true otherwise +| 61 | 1 | | Allows background location + +Contains a boolean value where false if 0 or true otherwise +| 62 | 1 | | Shows background location + +Contains a boolean value where false if 0 or true otherwise +| 63 | 1 | | Allows map correction + +Contains a boolean value where false if 0 or true otherwise +4+| _Additional values if size > 64_ +| 64 | 1 | | Batching location + +Contains a boolean value where false if 0 or true otherwise +| 65 | 1 | | Updating vehicle speed + +Contains a boolean value where false if 0 or true otherwise +| 66 | 1 | | Updating vehicle heading + +Contains a boolean value where false if 0 or true otherwise +| 67 | 1 | | Match information enabled + +Contains a boolean value where false if 0 or true otherwise +| 68 | 1 | | Ground altitude enabled + +Contains a boolean value where false if 0 or true otherwise +| 69 | 1 | | Fusion information enabled + +Contains a boolean value where false if 0 or true otherwise +| 70 | 1 | | Courtesy prompt needed + +Contains a boolean value where false if 0 or true otherwise +| 71 | 1 | | Is authorized for widget updates + +Contains a boolean value where false if 0 or true otherwise +|=== + +The value is formatted as: + +.... +{"previousAuthorizationStatusValid":false,"paused":false,"requestingLocation":false,"updatingVehicleSpeed:false,"desiredAccuracy":100,"allowsBackgroundLocationUpdates":false,"dynamicAccuracyReductionEnabled":false,"distancFilter":-1,"allowsLocationPrompts":true,"activityType":0,"groundAltitudeEnabled":false,"pausesLocationUpdatesAutomatially":1,"fusionInfoEnabled":false,"isAuthorizedForWidgetUpdates":false,"updatingVehicleHeading":false,"batchingLocation":false,"showsBackgroundLocationIndicator":false,"updatingLocation":false,"requestingRanging":false,"updatingHeading:false,"previousAuthorizationStatus":0,"allowsMapCorrection":true,"matchInfoEnabled":false,"allowsAlteredAccessoryLoctions":false,"updatingRanging":false,"limitsPrecision":false,"courtesyPromptNeeded":false,"headingFilter":1} +.... + == Notes .... diff --git a/dtformats/macos_core_location.yaml b/dtformats/macos_core_location.yaml new file mode 100644 index 0000000..51455da --- /dev/null +++ b/dtformats/macos_core_location.yaml @@ -0,0 +1,142 @@ +# dtFabric format specification. +--- +name: macos_core_location +type: format +description: MacOS Core Location (CL) format definitions +--- +name: byte +type: integer +attributes: + format: unsigned + size: 1 + units: bytes +--- +name: bool8 +type: boolean +attributes: + size: 1 + units: bytes + false_value: 0 +--- +name: bool32 +type: boolean +attributes: + size: 4 + units: bytes + false_value: 0 +--- +name: int32 +type: integer +attributes: + format: signed + size: 4 + units: bytes +--- +name: uint32 +type: integer +attributes: + format: unsigned + size: 4 + units: bytes +--- +name: int64 +type: integer +attributes: + format: signed + size: 8 + units: bytes +--- +name: float64 +type: floating-point +attributes: + size: 8 + units: bytes +--- +name: client_manager_state_tracker_state +type: structure +description: Client manager (CLClientManager) state tracker state +attributes: + byte_order: little-endian +members: +- name: location_restricted + data_type: bool32 +- name: location_enabled_status + data_type: uint32 +--- +name: location_manager_tracker_state +type: structure +description: Location manager (CLLocationManager) tracker state +attributes: + byte_order: little-endian +members: +- name: distance_filter + data_type: float64 +- name: desired_accuracy + data_type: float64 +- name: updating_location + data_type: bool8 +- name: requesting_location + data_type: bool8 +- name: requesting_ranging + data_type: bool8 +- name: updating_ranging + data_type: bool8 +- name: updating_heading + data_type: bool8 +- name: unknown1 + type: stream + element_data_type: byte + number_of_elements: 3 +- name: heading_filter + data_type: float64 +- name: allows_location_prompts + data_type: bool8 +- name: allows_altered_accessory_location + data_type: bool8 +- name: dynamic_accuracy_reduction_enabled + data_type: bool8 +- name: previous_authorization_status_valid + data_type: bool8 +- name: previous_authorization_status + data_type: uint32 +- name: limits_precision + data_type: bool8 +- name: unknown2 + type: stream + element_data_type: byte + number_of_elements: 7 +- name: activity_type + data_type: int64 +- name: pause_location_updates_auto + data_type: int32 +- name: paused + data_type: bool8 +- name: allows_background_location + data_type: bool8 +- name: shows_background_location + data_type: bool8 +- name: allows_map_correction + data_type: bool8 +--- +name: location_manager_tracker_state_extra +type: structure +description: Location manager (CLLocationManager) tracker state +attributes: + byte_order: little-endian +members: +- name: batching_location + data_type: bool8 +- name: updating_vehicle_speed + data_type: bool8 +- name: updating_vehicle_heading + data_type: bool8 +- name: match_info + data_type: bool8 +- name: ground_altitude + data_type: bool8 +- name: fusion_info + data_type: bool8 +- name: courtesy_prompt + data_type: bool8 +- name: is_authorized_widget + data_type: bool8 diff --git a/dtformats/unified_logging.py b/dtformats/unified_logging.py index 04beb70..fc84cf4 100644 --- a/dtformats/unified_logging.py +++ b/dtformats/unified_logging.py @@ -5,9 +5,12 @@ import collections import re import os +import uuid import lz4.block +from dfdatetime import posix_time as dfdatetime_posix_time + from dtfabric.runtime import data_maps as dtfabric_data_maps from dtformats import darwin @@ -73,8 +76,45 @@ def FormatValue(cls, value): """ -class ErrorFormatStringDecoder(BaseFormatStringDecoder): - """Error format string decoder.""" +class BooleanFormatStringDecoder(BaseFormatStringDecoder): + """Boolean value format string decoder.""" + + @classmethod + def FormatValue(cls, value): + """Formats a boolean value. + + Args: + value (int): boolean value. + + Returns: + str: formatted boolean value. + """ + if value: + return 'true' + + return 'false' + + +class DateTimeInSecondsFormatStringDecoder(BaseFormatStringDecoder): + """Date and time value in seconds format string decoder.""" + + @classmethod + def FormatValue(cls, value): + """Formats a date and time value in seconds. + + Args: + value (int): timestamp that contains the number of seconds since + 1970-01-01 00:00:00. + + Returns: + str: formatted date and time value in seconds. + """ + date_time = dfdatetime_posix_time.PosixTime(timestamp=value) + return date_time.CopyToDateTimeString() + + +class ErrorCodeFormatStringDecoder(BaseFormatStringDecoder): + """Error code format string decoder.""" @classmethod def FormatValue(cls, error_code): # pylint: disable=arguments-renamed @@ -143,6 +183,60 @@ def FormatValue(cls, mode): # pylint: disable=arguments-renamed return ''.join(string_parts) +class LocationEscapeOnlyFormatStringDecoder(BaseFormatStringDecoder): + """Location escape only format string decoder.""" + + @classmethod + def FormatValue(cls, value): + """Formats a location value. + + Args: + value (str): location value. + + Returns: + str: formatted location value. + """ + value = value or '' + value = value.replace('/', '\\/') + return ''.join(['"', value, '"']) + + +class UUIDFormatStringDecoder(BaseFormatStringDecoder): + """UUID value format string decoder.""" + + @classmethod + def FormatValue(cls, value): + """Formats an UUID value. + + Args: + value (bytes): UUID value. + + Returns: + str: formatted UUID value. + """ + uuid_value = uuid.UUID(bytes=value) + return f'{uuid_value!s}'.upper() + + +class YesNoFormatStringDecoder(BaseFormatStringDecoder): + """Yes/No value format string decoder.""" + + @classmethod + def FormatValue(cls, value): + """Formats a yes/no value. + + Args: + value (int): yes/no value. + + Returns: + str: formatted yes/no value. + """ + if value: + return 'YES' + + return 'NO' + + class DSCFile(data_format.BinaryDataFile): """Shared-Cache Strings (dsc) file. @@ -883,14 +977,27 @@ class TraceV3File(data_format.BinaryDataFile): _FORMAT_STRING_TYPE_HINTS = { 'd': 'signed', + 'i': 'signed', 'p': 'unsigned', 'u': 'unsigned', 'x': 'unsigned'} + _FORMAT_STRING_PYTHON_SPECIFIERS = { + '@': 's', + 'i': 'd', + 'p': 'x', + 'P': 's', + 'u': 'd'} + _FORMAT_STRING_DECODERS = { - 'darwin.errno': ErrorFormatStringDecoder, + 'bool': BooleanFormatStringDecoder, + 'BOOL': YesNoFormatStringDecoder, + 'darwin.errno': ErrorCodeFormatStringDecoder, 'darwin.mode': FileModeFormatStringDecoder, - 'error': ErrorFormatStringDecoder} + 'errno': ErrorCodeFormatStringDecoder, + 'location:escape_only': LocationEscapeOnlyFormatStringDecoder, + 'time_t': DateTimeInSecondsFormatStringDecoder, + 'uuid_t': UUIDFormatStringDecoder} _MAXIMUM_CACHED_FILES = 64 _MAXIMUM_CACHED_FORMAT_STRINGS = 1024 @@ -1241,8 +1348,8 @@ def _GetFirehostTracepointFormatString( if load_address_lower <= ( uuid_entry.load_address_lower + uuid_entry.size): - uuid = self._catalog.uuids[uuid_entry.uuid_index] - uuid_string = uuid.hex.upper() + uuid_value = self._catalog.uuids[uuid_entry.uuid_index] + uuid_string = uuid_value.hex.upper() break elif format_string_type == 0x000a: @@ -1586,21 +1693,13 @@ def _ReadFirehoseChunkData(self, chunk_data, chunk_data_size, data_offset): format_string, value_type_decoders = self._RewriteFormatString( format_string) - number_of_data_items = getattr( - tracepoint_data_object, 'number_of_data_items', 0) - if not number_of_data_items: + data_items = getattr(tracepoint_data_object, 'data_items', None) + if not data_items: values = [] else: - number_of_value_type_decoders = len(value_type_decoders) - if (number_of_value_type_decoders > 0 and - number_of_value_type_decoders != number_of_data_items): - raise errors.ParseError( - 'Mismatch in number of data items and value type decoders.') - values = self._ReadFirehoseTracepointDataItems( - firehose_tracepoint.data, data_offset, - tracepoint_data_object.data_items, bytes_read, - value_type_decoders) + firehose_tracepoint.data, data_offset + chunk_data_offset, + data_items, bytes_read, value_type_decoders) if format_string: if values: @@ -1701,18 +1800,20 @@ def _ReadFirehoseTracepointDataItems( Raises: ParseError: if the data items cannot be read. """ - values = [] + number_of_value_type_decoders = len(value_type_decoders) + value_type_decoder_index = 0 - decoder = None - type_hint = None + values = [] - for item_index, data_item in enumerate(data_items): + for data_item in data_items: value = None - if value_type_decoders: - decoder, type_hint = value_type_decoders[item_index] + if value_type_decoder_index < number_of_value_type_decoders: + decoder, type_hint = value_type_decoders[value_type_decoder_index] + else: + decoder, type_hint = None, None - if data_item.value_type in (0x01, 0x02): + if data_item.value_type in (0x00, 0x01, 0x02): data_type_map_name = self._DATA_ITEM_INTEGER_DATA_MAP_NAMES.get( type_hint or 'unsigned', {}).get(data_item.data_size, None) @@ -1720,15 +1821,16 @@ def _ReadFirehoseTracepointDataItems( data_type_map = self._GetDataTypeMap(data_type_map_name) # TODO: calculate data offset for debugging purposes. - data_item.integer = self._ReadStructureFromByteStream( + _ = data_offset + + value = self._ReadStructureFromByteStream( data_item.data, 0, data_type_map, data_type_map_name) - value = data_item.integer + if self._debug: + data_item.integer = value elif data_item.value_type in self._DATA_ITEM_STRING_VALUE_TYPES: - if data_item.value_data_size == 0: - data_item.string = '(null)' - else: + if data_item.value_data_size > 0: # Note that the string data does not necessarily include # an end-of-string character hence the cstring data_type_map is not # used here. @@ -1737,36 +1839,23 @@ def _ReadFirehoseTracepointDataItems( value_data_offset:value_data_offset + data_item.value_data_size] try: - data_item.string = string_data.decode('utf-8').rstrip('\x00') + value = string_data.decode('utf-8').rstrip('\x00') except UnicodeDecodeError: pass - value = data_item.string + if self._debug: + data_item.string = value - elif data_item.value_type == 0x21: + elif data_item.value_type in (0x21, 0x41): value = '' - elif data_item.value_type == 0x32: + elif data_item.value_type in (0x32, 0xf2): value_data_offset = values_data_offset + data_item.value_data_offset - data_item.value_data = tracepoint_data[ + value = tracepoint_data[ value_data_offset:value_data_offset + data_item.value_data_size] - value = data_item.value_data - - elif data_item.value_type == 0xf2: - if data_item.value_data_size != 16: - raise errors.ParseError(( - f'Unsupported data item value size: ' - f'{data_item.value_data_size:d}.')) - - data_type_map = self._GetDataTypeMap('uuid_be') - - value_data_offset = values_data_offset + data_item.value_data_offset - data_item.uuid = self._ReadStructureFromByteStream( - tracepoint_data[value_data_offset:], - data_offset + value_data_offset, data_type_map, 'UUID') - - value = tracepoint_data[value_data_offset:value_data_offset + 16] + if self._debug: + data_item.value_data = value if self._debug: self._DebugPrintStructureObject( @@ -1779,12 +1868,22 @@ def _ReadFirehoseTracepointDataItems( f'Unsupported data item value type: ' f'0x{data_item.value_type:02x}.')) + if data_item.value_type == 0x12: + # TODO: use this value to limit the size of the next string value + continue + decoder_class = self._FORMAT_STRING_DECODERS.get(decoder, None) if decoder_class: value = decoder_class.FormatValue(value) + if (value is None and + data_item.value_type in self._DATA_ITEM_STRING_VALUE_TYPES): + value = '(null)' + values.append(value) + value_type_decoder_index += 1 + return values def _ReadFirehoseTracepointLogData(self, flags, tracepoint_data, data_offset): @@ -2057,25 +2156,51 @@ def _RewriteFormatString(self, format_string): value_type_decoders = [] last_match_end = 0 - for match in self._FORMAT_STRING_OPERATOR_REGEX.finditer(format_string): + for match_index, match in enumerate( + self._FORMAT_STRING_OPERATOR_REGEX.finditer(format_string)): literal, decoder, flags, width, precision, specifier = match.groups() match_start, match_end = match.span() if match_start > last_match_end: - format_string_segments.append(format_string[last_match_end:match_start]) + string_segment = format_string[last_match_end:match_start] + string_segment = string_segment.replace('{', '{{') + string_segment = string_segment.replace('}', '}}') + format_string_segments.append(string_segment) if literal == '%%': literal = '%' elif specifier: + if specifier == 'P': + precision = '' + elif specifier == 's' and precision in ('.0', '.*'): + precision = '' + else: + precision = precision or '' + type_hint = self._FORMAT_STRING_TYPE_HINTS.get(specifier, None) - if specifier in ('p', 'u'): - specifier = 'd' + if decoder: + # Remove private and public value type decoders. + decoder_segments = [value.strip() for value in decoder.split(',')] + decoder_segments = [ + value for value in decoder_segments + if value not in ('private', 'public')] + decoder = ','.join(decoder_segments) + + if decoder in self._FORMAT_STRING_DECODERS: + specifier = 's' - precision = precision or '' width = width or '' - literal = f'{{:{flags:s}{precision:s}{width:s}{specifier:s}}}' + python_specifier = self._FORMAT_STRING_PYTHON_SPECIFIERS.get( + specifier, specifier) + + literal = (f'{{{match_index:d}:{flags:s}{precision:s}{width:s}' + f'{python_specifier:s}}}') + + if specifier == 'p': + literal = ''.join(['0x', literal]) + decoder = decoder or None value_type_decoders.append((decoder, type_hint)) format_string_segments.append(literal) @@ -2084,7 +2209,10 @@ def _RewriteFormatString(self, format_string): string_size = len(format_string) if string_size > last_match_end: - format_string_segments.append(format_string[last_match_end:string_size]) + string_segment = format_string[last_match_end:string_size] + string_segment = string_segment.replace('{', '{{') + string_segment = string_segment.replace('}', '}}') + format_string_segments.append(string_segment) return ''.join(format_string_segments), value_type_decoders diff --git a/tests/unified_logging.py b/tests/unified_logging.py index 75ecfed..36f9f5f 100644 --- a/tests/unified_logging.py +++ b/tests/unified_logging.py @@ -15,12 +15,39 @@ from tests import test_lib -class ErrorFormatStringDecoderTest(test_lib.BaseTestCase): - """Error format string decoder tests.""" +class BooleanFormatStringDecoderTest(test_lib.BaseTestCase): + """Boolean value format string decoder tests.""" def testFormatValue(self): """Tests the FormatValue function.""" - formatted_value = unified_logging.ErrorFormatStringDecoder.FormatValue(2) + decoder_class = unified_logging.BooleanFormatStringDecoder + + formatted_value = decoder_class.FormatValue(1) + self.assertEqual(formatted_value, 'true') + + formatted_value = decoder_class.FormatValue(0) + self.assertEqual(formatted_value, 'false') + + +class DateTimeInSecondsFormatStringDecoderTest(test_lib.BaseTestCase): + """Date and time value in seconds format string decoder tests.""" + + def testFormatValue(self): + """Tests the FormatValue function.""" + decoder_class = unified_logging.DateTimeInSecondsFormatStringDecoder + + formatted_value = decoder_class.FormatValue(1684642680) + self.assertEqual(formatted_value, '2023-05-21 04:18:00') + + +class ErrorCodeFormatStringDecoderTest(test_lib.BaseTestCase): + """Error code format string decoder tests.""" + + def testFormatValue(self): + """Tests the FormatValue function.""" + decoder_class = unified_logging.ErrorCodeFormatStringDecoder + + formatted_value = decoder_class.FormatValue(2) self.assertEqual(formatted_value, '[2: No such file or directory]') @@ -29,11 +56,54 @@ class FileModeFormatStringDecoderTest(test_lib.BaseTestCase): def testFormatValue(self): """Tests the FormatValue function.""" - formatted_value = unified_logging.FileModeFormatStringDecoder.FormatValue( - 0o700) + decoder_class = unified_logging.FileModeFormatStringDecoder + + formatted_value = decoder_class.FormatValue(0o700) self.assertEqual(formatted_value, '-rwx------') +class LocationEscapeOnlyFormatStringDecoderTest(test_lib.BaseTestCase): + """Location escape only format string decoder tests.""" + + def testFormatValue(self): + """Tests the FormatValue function.""" + decoder_class = unified_logging.LocationEscapeOnlyFormatStringDecoder + + formatted_value = decoder_class.FormatValue(None) + self.assertEqual(formatted_value, '""') + + formatted_value = decoder_class.FormatValue( + 'NSBundle ') + self.assertEqual(formatted_value, ( + '"NSBundle <\\/System\\/Library\\/LocationBundles\\/TimeZone.bundle>"')) + + +class UUIDFormatStringDecoderTest(test_lib.BaseTestCase): + """UUID value format string decoder tests.""" + + def testFormatValue(self): + """Tests the FormatValue function.""" + decoder_class = unified_logging.UUIDFormatStringDecoder + + formatted_value = decoder_class.FormatValue( + b'\x1d\x1f\xd3\xfb\xe9\xa6Fj\xb72\x7f\xb6\x98a\x02\xb2') + self.assertEqual(formatted_value, '1D1FD3FB-E9A6-466A-B732-7FB6986102B2') + + +class YesNoFormatStringDecoderTest(test_lib.BaseTestCase): + """Yes/No value format string decoder tests.""" + + def testFormatValue(self): + """Tests the FormatValue function.""" + decoder_class = unified_logging.YesNoFormatStringDecoder + + formatted_value = decoder_class.FormatValue(1) + self.assertEqual(formatted_value, 'YES') + + formatted_value = decoder_class.FormatValue(0) + self.assertEqual(formatted_value, 'NO') + + class DSCFileTest(test_lib.BaseTestCase): """Shared-Cache Strings (dsc) file tests.""" @@ -609,32 +679,72 @@ def testRewriteFormatString(self): self.assertEqual(decoders, []) format_string, decoders = test_file._RewriteFormatString('%d') - self.assertEqual(format_string, '{:d}') + self.assertEqual(format_string, '{0:d}') self.assertEqual(decoders, [(None, 'signed')]) - format_string, decoders = test_file._RewriteFormatString('%s') - self.assertEqual(format_string, '{:s}') - self.assertEqual(decoders, [(None, None)]) + format_string, decoders = test_file._RewriteFormatString('%i') + self.assertEqual(format_string, '{0:d}') + self.assertEqual(decoders, [(None, 'signed')]) format_string, decoders = test_file._RewriteFormatString('%p') - self.assertEqual(format_string, '{:d}') + self.assertEqual(format_string, '0x{0:x}') self.assertEqual(decoders, [(None, 'unsigned')]) format_string, decoders = test_file._RewriteFormatString('%u') - self.assertEqual(format_string, '{:d}') + self.assertEqual(format_string, '{0:d}') + self.assertEqual(decoders, [(None, 'unsigned')]) + + format_string, decoders = test_file._RewriteFormatString('%x') + self.assertEqual(format_string, '{0:x}') self.assertEqual(decoders, [(None, 'unsigned')]) format_string, decoders = test_file._RewriteFormatString('0x%lx') - self.assertEqual(format_string, '0x{:x}') + self.assertEqual(format_string, '0x{0:x}') self.assertEqual(decoders, [(None, 'unsigned')]) format_string, decoders = test_file._RewriteFormatString('0x%02x') - self.assertEqual(format_string, '0x{:02x}') + self.assertEqual(format_string, '0x{0:02x}') self.assertEqual(decoders, [(None, 'unsigned')]) + format_string, decoders = test_file._RewriteFormatString('%s') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [(None, None)]) + + format_string, decoders = test_file._RewriteFormatString('%@') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [(None, None)]) + + format_string, decoders = test_file._RewriteFormatString('%.*s') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [(None, None)]) + + format_string, decoders = test_file._RewriteFormatString('%.16P') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [(None, None)]) + format_string, decoders = test_file._RewriteFormatString('%{public}s') - self.assertEqual(format_string, '{:s}') - self.assertEqual(decoders, [('public', None)]) + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [(None, None)]) + + format_string, decoders = test_file._RewriteFormatString( + '%{public,uuid_t}.16P') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [('uuid_t', None)]) + + format_string, decoders = test_file._RewriteFormatString( + '"msg%{public}.0s"') + self.assertEqual(format_string, '"msg{0:s}"') + self.assertEqual(decoders, [(None, None)]) + + format_string, decoders = test_file._RewriteFormatString( + '%{public, location:escape_only}s') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [('location:escape_only', None)]) + + format_string, decoders = test_file._RewriteFormatString( + '%{private, mask.hash, mdnsresponder:ip_addr}.20P') + self.assertEqual(format_string, '{0:s}') + self.assertEqual(decoders, [('mask.hash,mdnsresponder:ip_addr', None)]) def testReadFileObject(self): """Tests the ReadFileObject function."""