diff --git a/airbyte_cdk/sources/declarative/extractors/dpath_enhancing_extractor.py b/airbyte_cdk/sources/declarative/extractors/dpath_enhancing_extractor.py index 8736852ee..dde35f335 100644 --- a/airbyte_cdk/sources/declarative/extractors/dpath_enhancing_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/dpath_enhancing_extractor.py @@ -6,8 +6,11 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor - -from airbyte_cdk.sources.declarative.extractors.record_extractor import add_service_key,is_service_key,SERVICE_KEY_PREFIX +from airbyte_cdk.sources.declarative.extractors.record_extractor import ( + SERVICE_KEY_PREFIX, + add_service_key, + is_service_key, +) # The name of the service key that holds a reference to the original root response SERVICE_KEY_ROOT = "root" @@ -109,7 +112,7 @@ def _set_parent(self,body: Any, parent:Any) -> Any: :return: a copy of the body enhanced in a subclass-specific way. None only when body is None. """ if isinstance(body, dict): - result = dict() + result:dict[str, Any] = dict() if parent: result = add_service_key(result, SERVICE_KEY_PARENT, parent) attributes_only = dict(result) @@ -117,11 +120,11 @@ def _set_parent(self,body: Any, parent:Any) -> Any: if v and not isinstance(v,dict) and not isinstance(v,list)}) for k,v in body.items(): result[k] = self._set_parent(v,attributes_only) + return result elif isinstance(body, list): - result = [self._set_parent(v,parent) for v in body] + return [self._set_parent(v,parent) for v in body] else: - result = body - return result + return body def update_record(self, record: Any, root: Any) -> Any: """ diff --git a/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py index b8faf62cb..073471009 100644 --- a/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -9,9 +9,10 @@ import requests from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.types import Config -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor + @dataclass class DpathExtractor(RecordExtractor): diff --git a/airbyte_cdk/sources/declarative/extractors/record_extractor.py b/airbyte_cdk/sources/declarative/extractors/record_extractor.py index c95c558d3..4e0c8dac6 100644 --- a/airbyte_cdk/sources/declarative/extractors/record_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/record_extractor.py @@ -14,7 +14,7 @@ SERVICE_KEY_PREFIX = "$" -def add_service_key(mapping: Mapping[str, Any], key:str, value:Any) -> Mapping[str, Any]: +def add_service_key(mapping: Mapping[str, Any], key:str, value:Any) -> dict[str, Any]: """ :param mapping: non-null mapping :param key: the name of the key, not including any specific prefixes @@ -32,12 +32,11 @@ def exclude_service_keys(struct: Any) -> Any: :return: a copy of struct without any service fields at any level of nesting """ if isinstance(struct, dict): - result = {k: exclude_service_keys(v) for k, v in struct.items() if not is_service_key(k)} + return {k: exclude_service_keys(v) for k, v in struct.items() if not is_service_key(k)} elif isinstance(struct, list): - result = [exclude_service_keys(v) for v in struct] + return [exclude_service_keys(v) for v in struct] else: - result = struct - return result + return struct def is_service_key(key: str) -> bool: return key.find(SERVICE_KEY_PREFIX) == 0 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 5e7872cdb..9c9d2eb75 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -99,7 +99,7 @@ the value of the **dockerImageTag** parameter. - `poetry run pytest` to run all unit tests. - `poetry run pytest -k ` to run specific unit tests. - `poetry run pytest-fast` to run the subset of PyTest tests, which are not flagged as `slow`. (It should take <5 min for fast tests only.) -- `python -m pytest -s unit_tests` if you want to pass pytest options. +- `poetry run pytest -s unit_tests` if you want to pass pytest options. ### Run Code Formatting diff --git a/unit_tests/sources/declarative/extractors/test_dpath_enhancing_extractor.py b/unit_tests/sources/declarative/extractors/test_dpath_enhancing_extractor.py index f0fdd483b..63f24f70e 100644 --- a/unit_tests/sources/declarative/extractors/test_dpath_enhancing_extractor.py +++ b/unit_tests/sources/declarative/extractors/test_dpath_enhancing_extractor.py @@ -14,8 +14,11 @@ JsonDecoder, JsonlDecoder, ) -from airbyte_cdk.sources.declarative.extractors.dpath_enhancing_extractor import DpathEnhancingExtractor, \ - SERVICE_KEY_ROOT, SERVICE_KEY_PARENT +from airbyte_cdk.sources.declarative.extractors.dpath_enhancing_extractor import ( + SERVICE_KEY_PARENT, + SERVICE_KEY_ROOT, + DpathEnhancingExtractor, +) from airbyte_cdk.sources.declarative.extractors.record_extractor import ( SERVICE_KEY_PREFIX, ) diff --git a/unit_tests/sources/declarative/extractors/test_record_selector.py b/unit_tests/sources/declarative/extractors/test_record_selector.py index bf5c591ec..d4de75f36 100644 --- a/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -9,11 +9,14 @@ import requests from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.dpath_enhancing_extractor import ( + SERVICE_KEY_PARENT, + SERVICE_KEY_ROOT, +) from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor -from airbyte_cdk.sources.declarative.extractors.dpath_enhancing_extractor import SERVICE_KEY_ROOT, SERVICE_KEY_PARENT from airbyte_cdk.sources.declarative.extractors.record_extractor import ( + SERVICE_KEY_PREFIX, exclude_service_keys, - SERVICE_KEY_PREFIX ) from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector