From ed637af2cb05beb5a9abbfaa62356e2f0643c291 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Thu, 6 Feb 2025 11:30:10 +0100 Subject: [PATCH 1/7] feat(ini): allow custom parsers/serializers for ini options Signed-off-by: Sylvain Leclerc --- antarest/study/storage/rawstudy/ini_reader.py | 71 +++++++++++++++++-- antarest/study/storage/rawstudy/ini_writer.py | 44 ++++++++++-- .../antares_io/reader/test_ini_reader.py | 34 ++++++++- .../antares_io/writer/test_ini_writer.py | 29 ++++++++ 4 files changed, 166 insertions(+), 12 deletions(-) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/study/storage/rawstudy/ini_reader.py index e003878e7c..c264c53d01 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/study/storage/rawstudy/ini_reader.py @@ -15,13 +15,25 @@ import typing as t from abc import ABC, abstractmethod from pathlib import Path +from typing import Callable, Optional from typing_extensions import override from antarest.core.model import JSON +PrimitiveType = t.Union[str, int, float, bool] +ValueParser = Callable[[str], PrimitiveType] +SelectionPredicate = Callable[[str], bool] -def convert_value(value: str) -> t.Union[str, int, float, bool]: + +def _lower_case(input: str) -> str: + return input.lower() + + +LOWER_CASE_PARSER: ValueParser = _lower_case + + +def _convert_value(value: str) -> PrimitiveType: """Convert value to the appropriate type for JSON.""" try: @@ -38,7 +50,44 @@ def convert_value(value: str) -> t.Union[str, int, float, bool]: return value -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) +class OptionMatcher: + """ + Defines a location in INI file data. + A None section means all sections. + """ + + section: Optional[str] + key: Optional[str] + + +def any_section_option_matcher(key: str) -> OptionMatcher: + """ + Return a matcher which will match the provided key in any section. + """ + return OptionMatcher(section=None, key=key) + + +class ValueParsers: + _parsers: t.Dict[OptionMatcher, ValueParser] + + def __init__(self, parsers: t.Dict[OptionMatcher, ValueParser]): + self._parsers = parsers + + def find_parser(self, section: str, key: str) -> ValueParser: + if not self._parsers: + return _convert_value + possible_keys = [ + OptionMatcher(section=section, key=key), + OptionMatcher(section=None, key=key), + ] + for k in possible_keys: + if parser := self._parsers.get(k, None): + return parser + return _convert_value + + +@dataclasses.dataclass(frozen=True) class IniFilter: """ Filter sections and options in an INI file based on regular expressions. @@ -115,8 +164,8 @@ def read(self, path: t.Any, **kwargs: t.Any) -> JSON: Parse `.ini` file to json object. Args: - path: Path to `.ini` file or file-like object. - kwargs: Additional options used for reading. + path: Path to `.ini` file or file-like object. + options: Additional options used for reading. Returns: Dictionary of parsed `.ini` file which can be converted to JSON. @@ -152,11 +201,17 @@ class IniReader(IReader): This class is not compatible with standard `.ini` readers. """ - def __init__(self, special_keys: t.Sequence[str] = (), section_name: str = "settings") -> None: + def __init__( + self, + special_keys: t.Sequence[str] = (), + section_name: str = "settings", + value_parsers: t.Dict[OptionMatcher, ValueParser] | None = None, + ) -> None: super().__init__() # Default section name to use if `.ini` file has no section. self._special_keys = set(special_keys) + self._value_parsers = ValueParsers(value_parsers or {}) # List of keys which should be parsed as list. self._section_name = section_name @@ -313,10 +368,12 @@ def _handle_option(self, ini_filter: IniFilter, section: str, key: str, value: s def _append_option(self, section: str, key: str, value: str) -> None: self._curr_sections.setdefault(section, {}) values = self._curr_sections[section] + parser = self._value_parsers.find_parser(section, key) + parsed = parser(value) if key in self._special_keys: - values.setdefault(key, []).append(convert_value(value)) + values.setdefault(key, []).append(parsed) else: - values[key] = convert_value(value) + values[key] = parsed self._curr_option = key diff --git a/antarest/study/storage/rawstudy/ini_writer.py b/antarest/study/storage/rawstudy/ini_writer.py index 9ff81c213e..363ab64d25 100644 --- a/antarest/study/storage/rawstudy/ini_writer.py +++ b/antarest/study/storage/rawstudy/ini_writer.py @@ -12,24 +12,52 @@ import ast import configparser -import typing as t from pathlib import Path +from typing import Callable, Dict, List, Optional, Union from typing_extensions import override from antarest.core.model import JSON +from antarest.study.storage.rawstudy.ini_reader import OptionMatcher + +PrimitiveType = Union[str, int, float, bool] +ValueSerializer = Callable[[str], PrimitiveType] + + +def _lower_case(input: str) -> str: + return input.lower() + + +LOWER_CASE_SERIALIZER: ValueSerializer = _lower_case class IniConfigParser(configparser.RawConfigParser): - def __init__(self, special_keys: t.Optional[t.List[str]] = None) -> None: + def __init__( + self, + special_keys: Optional[List[str]] = None, + value_serializers: Optional[Dict[OptionMatcher, ValueSerializer]] = None, + ) -> None: super().__init__() self.special_keys = special_keys + self._value_serializers = value_serializers or {} # noinspection SpellCheckingInspection @override def optionxform(self, optionstr: str) -> str: return optionstr + def _get_serializer(self, section: str, key: str) -> Optional[ValueSerializer]: + if not self._value_serializers: + return None + possible_keys = [ + OptionMatcher(section=section, key=key), + OptionMatcher(section=None, key=key), + ] + for k in possible_keys: + if parser := self._value_serializers.get(k, None): + return parser + return None + def _write_line( # type:ignore self, delimiter, @@ -41,6 +69,9 @@ def _write_line( # type:ignore value = self._interpolation.before_write( # type:ignore self, section_name, key, value ) + if self._value_serializers: + if serializer := self._get_serializer(section_name, key): + value = serializer(value) if value is not None or not self._allow_no_value: # type:ignore value = delimiter + str(value).replace("\n", "\n\t") else: @@ -70,8 +101,13 @@ class IniWriter: Standard INI writer. """ - def __init__(self, special_keys: t.Optional[t.List[str]] = None): + def __init__( + self, + special_keys: Optional[List[str]] = None, + value_serializers: Optional[Dict[OptionMatcher, ValueSerializer]] = None, + ): self.special_keys = special_keys + self._value_serializers = value_serializers or {} def write(self, data: JSON, path: Path) -> None: """ @@ -81,7 +117,7 @@ def write(self, data: JSON, path: Path) -> None: data: JSON content. path: path to `.ini` file. """ - config_parser = IniConfigParser(special_keys=self.special_keys) + config_parser = IniConfigParser(special_keys=self.special_keys, value_serializers=self._value_serializers) config_parser.read_dict(data) with path.open("w") as fp: config_parser.write(fp) diff --git a/tests/storage/repository/antares_io/reader/test_ini_reader.py b/tests/storage/repository/antares_io/reader/test_ini_reader.py index b36ee066d7..6f687c12e5 100644 --- a/tests/storage/repository/antares_io/reader/test_ini_reader.py +++ b/tests/storage/repository/antares_io/reader/test_ini_reader.py @@ -14,7 +14,12 @@ import textwrap from pathlib import Path -from antarest.study.storage.rawstudy.ini_reader import IniReader, SimpleKeyValueReader +from antarest.study.storage.rawstudy.ini_reader import ( + IniReader, + OptionMatcher, + SimpleKeyValueReader, + any_section_option_matcher, +) class TestIniReader: @@ -324,6 +329,33 @@ def test_read__filtered_option(self, tmp_path) -> None: expected = {"part1": {"bar": "hello"}, "part2": {"bar": "salut"}} assert actual == expected + def test_read__with_custom_parser(self, tmp_path): + path = Path(tmp_path) / "test.ini" + path.write_text( + textwrap.dedent( + """ + [part1] + bar = Hello + + [part2] + bar = Hello + """ + ) + ) + + def to_lower(input: str) -> str: + return input.lower() + + value_parsers = {OptionMatcher("part2", "bar"): to_lower} + actual = IniReader(value_parsers=value_parsers).read(path) + expected = {"part1": {"bar": "Hello"}, "part2": {"bar": "hello"}} + assert actual == expected + + value_parsers = {any_section_option_matcher("bar"): to_lower} + actual = IniReader(value_parsers=value_parsers).read(path) + expected = {"part1": {"bar": "hello"}, "part2": {"bar": "hello"}} + assert actual == expected + class TestSimpleKeyValueReader: def test_read(self) -> None: diff --git a/tests/storage/repository/antares_io/writer/test_ini_writer.py b/tests/storage/repository/antares_io/writer/test_ini_writer.py index edb73035f3..df005cc706 100644 --- a/tests/storage/repository/antares_io/writer/test_ini_writer.py +++ b/tests/storage/repository/antares_io/writer/test_ini_writer.py @@ -15,6 +15,7 @@ import pytest +from antarest.study.storage.rawstudy.ini_reader import OptionMatcher, any_section_option_matcher from antarest.study.storage.rawstudy.ini_writer import IniWriter @@ -59,3 +60,31 @@ def test_write(tmp_path: str, ini_cleaner: Callable) -> None: writer.write(json_data, path) assert ini_cleaner(ini_content) == ini_cleaner(path.read_text()) + + +@pytest.mark.unit_test +def test_write_with_custom_serializer(tmp_path: str, ini_cleaner: Callable) -> None: + path = Path(tmp_path) / "test.ini" + + serializers = {any_section_option_matcher("group"): lambda x: x.lower()} + writer = IniWriter(value_serializers=serializers) + + expected = """ + [part1] + group = gas + + [part2] + group = gas + + [part3] + other = Gas + """ + + json_data = { + "part1": {"group": "Gas"}, + "part2": {"group": "Gas"}, + "part3": {"other": "Gas"}, + } + writer.write(json_data, path) + + assert ini_cleaner(path.read_text()) == ini_cleaner(expected) From cf7641a14cbad8b8916fa6bf00a1d6cb814a2f43 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Thu, 6 Feb 2025 11:41:18 +0100 Subject: [PATCH 2/7] fix(tests): improve tests Signed-off-by: Sylvain Leclerc --- antarest/study/storage/rawstudy/ini_reader.py | 2 - antarest/study/storage/rawstudy/ini_writer.py | 39 +++++++++++-------- .../antares_io/reader/test_ini_reader.py | 8 ++-- .../antares_io/writer/test_ini_writer.py | 4 +- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/study/storage/rawstudy/ini_reader.py index c264c53d01..5a9db6dd3b 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/study/storage/rawstudy/ini_reader.py @@ -69,8 +69,6 @@ def any_section_option_matcher(key: str) -> OptionMatcher: class ValueParsers: - _parsers: t.Dict[OptionMatcher, ValueParser] - def __init__(self, parsers: t.Dict[OptionMatcher, ValueParser]): self._parsers = parsers diff --git a/antarest/study/storage/rawstudy/ini_writer.py b/antarest/study/storage/rawstudy/ini_writer.py index 363ab64d25..a2ff05fe95 100644 --- a/antarest/study/storage/rawstudy/ini_writer.py +++ b/antarest/study/storage/rawstudy/ini_writer.py @@ -21,6 +21,8 @@ from antarest.study.storage.rawstudy.ini_reader import OptionMatcher PrimitiveType = Union[str, int, float, bool] + +# Value serializers may be used to customize the way INI options are serialized ValueSerializer = Callable[[str], PrimitiveType] @@ -31,33 +33,38 @@ def _lower_case(input: str) -> str: LOWER_CASE_SERIALIZER: ValueSerializer = _lower_case +class ValueSerializers: + def __init__(self, serializers: Dict[OptionMatcher, ValueSerializer]): + self._serializers = serializers + + def find_serializer(self, section: str, key: str) -> Optional[ValueSerializer]: + if not self._serializers: + return None + possible_keys = [ + OptionMatcher(section=section, key=key), + OptionMatcher(section=None, key=key), + ] + for k in possible_keys: + if parser := self._serializers.get(k, None): + return parser + return None + + class IniConfigParser(configparser.RawConfigParser): def __init__( self, special_keys: Optional[List[str]] = None, - value_serializers: Optional[Dict[OptionMatcher, ValueSerializer]] = None, + value_serializers: Optional[ValueSerializers] = None, ) -> None: super().__init__() self.special_keys = special_keys - self._value_serializers = value_serializers or {} + self._value_serializers = value_serializers or ValueSerializers({}) # noinspection SpellCheckingInspection @override def optionxform(self, optionstr: str) -> str: return optionstr - def _get_serializer(self, section: str, key: str) -> Optional[ValueSerializer]: - if not self._value_serializers: - return None - possible_keys = [ - OptionMatcher(section=section, key=key), - OptionMatcher(section=None, key=key), - ] - for k in possible_keys: - if parser := self._value_serializers.get(k, None): - return parser - return None - def _write_line( # type:ignore self, delimiter, @@ -70,7 +77,7 @@ def _write_line( # type:ignore self, section_name, key, value ) if self._value_serializers: - if serializer := self._get_serializer(section_name, key): + if serializer := self._value_serializers.find_serializer(section_name, key): value = serializer(value) if value is not None or not self._allow_no_value: # type:ignore value = delimiter + str(value).replace("\n", "\n\t") @@ -107,7 +114,7 @@ def __init__( value_serializers: Optional[Dict[OptionMatcher, ValueSerializer]] = None, ): self.special_keys = special_keys - self._value_serializers = value_serializers or {} + self._value_serializers = ValueSerializers(value_serializers or {}) def write(self, data: JSON, path: Path) -> None: """ diff --git a/tests/storage/repository/antares_io/reader/test_ini_reader.py b/tests/storage/repository/antares_io/reader/test_ini_reader.py index 6f687c12e5..054bb459bc 100644 --- a/tests/storage/repository/antares_io/reader/test_ini_reader.py +++ b/tests/storage/repository/antares_io/reader/test_ini_reader.py @@ -15,6 +15,7 @@ from pathlib import Path from antarest.study.storage.rawstudy.ini_reader import ( + LOWER_CASE_PARSER, IniReader, OptionMatcher, SimpleKeyValueReader, @@ -343,15 +344,12 @@ def test_read__with_custom_parser(self, tmp_path): ) ) - def to_lower(input: str) -> str: - return input.lower() - - value_parsers = {OptionMatcher("part2", "bar"): to_lower} + value_parsers = {OptionMatcher("part2", "bar"): LOWER_CASE_PARSER} actual = IniReader(value_parsers=value_parsers).read(path) expected = {"part1": {"bar": "Hello"}, "part2": {"bar": "hello"}} assert actual == expected - value_parsers = {any_section_option_matcher("bar"): to_lower} + value_parsers = {any_section_option_matcher("bar"): LOWER_CASE_PARSER} actual = IniReader(value_parsers=value_parsers).read(path) expected = {"part1": {"bar": "hello"}, "part2": {"bar": "hello"}} assert actual == expected diff --git a/tests/storage/repository/antares_io/writer/test_ini_writer.py b/tests/storage/repository/antares_io/writer/test_ini_writer.py index df005cc706..3cc44b32af 100644 --- a/tests/storage/repository/antares_io/writer/test_ini_writer.py +++ b/tests/storage/repository/antares_io/writer/test_ini_writer.py @@ -16,7 +16,7 @@ import pytest from antarest.study.storage.rawstudy.ini_reader import OptionMatcher, any_section_option_matcher -from antarest.study.storage.rawstudy.ini_writer import IniWriter +from antarest.study.storage.rawstudy.ini_writer import LOWER_CASE_SERIALIZER, IniWriter @pytest.mark.unit_test @@ -66,7 +66,7 @@ def test_write(tmp_path: str, ini_cleaner: Callable) -> None: def test_write_with_custom_serializer(tmp_path: str, ini_cleaner: Callable) -> None: path = Path(tmp_path) / "test.ini" - serializers = {any_section_option_matcher("group"): lambda x: x.lower()} + serializers = {any_section_option_matcher("group"): LOWER_CASE_SERIALIZER} writer = IniWriter(value_serializers=serializers) expected = """ From 8dfaf0e57fbdd8d82372b148822e5333916e18a0 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Thu, 6 Feb 2025 11:43:39 +0100 Subject: [PATCH 3/7] fix(typing): stricter typing Signed-off-by: Sylvain Leclerc --- antarest/study/storage/rawstudy/ini_reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/study/storage/rawstudy/ini_reader.py index 5a9db6dd3b..19501ef151 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/study/storage/rawstudy/ini_reader.py @@ -53,12 +53,12 @@ def _convert_value(value: str) -> PrimitiveType: @dataclasses.dataclass(frozen=True) class OptionMatcher: """ - Defines a location in INI file data. - A None section means all sections. + Used to match a location in an INI file: + a None section means any section. """ section: Optional[str] - key: Optional[str] + key: str def any_section_option_matcher(key: str) -> OptionMatcher: From 10173a848440eed1f03acfe1d41638b913854d73 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Thu, 6 Feb 2025 12:53:25 +0100 Subject: [PATCH 4/7] fix(typing): remove unions Signed-off-by: Sylvain Leclerc --- antarest/study/storage/rawstudy/ini_reader.py | 5 ++--- antarest/study/storage/rawstudy/ini_writer.py | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/study/storage/rawstudy/ini_reader.py index 19501ef151..77aafb088e 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/study/storage/rawstudy/ini_reader.py @@ -21,9 +21,8 @@ from antarest.core.model import JSON -PrimitiveType = t.Union[str, int, float, bool] +PrimitiveType = str | int | float | bool ValueParser = Callable[[str], PrimitiveType] -SelectionPredicate = Callable[[str], bool] def _lower_case(input: str) -> str: @@ -39,7 +38,7 @@ def _convert_value(value: str) -> PrimitiveType: try: # Infinity values are not supported by JSON, so we use a string instead. mapping = {"true": True, "false": False, "+inf": "+Inf", "-inf": "-Inf", "inf": "+Inf"} - return t.cast(t.Union[str, int, float, bool], mapping[value.lower()]) + return t.cast(PrimitiveType, mapping[value.lower()]) except KeyError: try: return int(value) diff --git a/antarest/study/storage/rawstudy/ini_writer.py b/antarest/study/storage/rawstudy/ini_writer.py index a2ff05fe95..df59cd3a0a 100644 --- a/antarest/study/storage/rawstudy/ini_writer.py +++ b/antarest/study/storage/rawstudy/ini_writer.py @@ -13,14 +13,12 @@ import ast import configparser from pathlib import Path -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, List, Optional from typing_extensions import override from antarest.core.model import JSON -from antarest.study.storage.rawstudy.ini_reader import OptionMatcher - -PrimitiveType = Union[str, int, float, bool] +from antarest.study.storage.rawstudy.ini_reader import OptionMatcher, PrimitiveType # Value serializers may be used to customize the way INI options are serialized ValueSerializer = Callable[[str], PrimitiveType] From b434dbf2191b4e8dc05f68ef35fdd4f574d853f3 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Fri, 7 Feb 2025 10:52:10 +0100 Subject: [PATCH 5/7] refacto(ini): move ini related features to core.serde Signed-off-by: Sylvain Leclerc --- ...populate_tag_and_study_tag_tables_with_.py | 2 +- antarest/core/cache/business/local_chache.py | 2 +- antarest/core/cache/business/redis_cache.py | 3 +- antarest/core/configdata/model.py | 2 +- antarest/core/configdata/repository.py | 2 +- antarest/core/core_blueprint.py | 2 +- antarest/core/filesystem_blueprint.py | 2 +- antarest/core/filetransfer/model.py | 2 +- antarest/core/interfaces/eventbus.py | 2 +- antarest/core/jwt.py | 2 +- antarest/core/model.py | 2 +- antarest/core/serde/__init__.py | 34 ++++++++++++++ antarest/core/serde/ini_common.py | 22 +++++++++ .../rawstudy => core/serde}/ini_reader.py | 47 ++++++------------- .../rawstudy => core/serde}/ini_writer.py | 19 ++++---- .../__init__.py => serde/json.py} | 15 ------ antarest/core/tasks/model.py | 2 +- antarest/core/version_info.py | 2 +- antarest/eventbus/web.py | 3 +- antarest/front.py | 2 +- antarest/launcher/adapters/log_parser.py | 2 +- .../adapters/slurm_launcher/slurm_launcher.py | 4 +- antarest/launcher/model.py | 3 +- antarest/launcher/ssh_config.py | 2 +- antarest/login/auth.py | 3 +- antarest/login/main.py | 2 +- antarest/login/model.py | 2 +- antarest/login/web.py | 3 +- antarest/matrixstore/matrix_editor.py | 2 +- antarest/matrixstore/model.py | 2 +- antarest/matrixstore/service.py | 2 +- antarest/study/business/area_management.py | 2 +- .../business/areas/st_storage_management.py | 2 +- .../business/binding_constraint_management.py | 2 +- antarest/study/business/district_manager.py | 2 +- antarest/study/business/model/link_model.py | 2 +- .../business/timeseries_config_management.py | 2 +- antarest/study/business/utils.py | 2 +- .../study/business/xpansion_management.py | 2 +- antarest/study/model.py | 2 +- antarest/study/repository.py | 2 +- antarest/study/service.py | 2 +- .../study/storage/abstract_storage_service.py | 2 +- antarest/study/storage/patch_service.py | 2 +- .../model/filesystem/config/cluster.py | 2 +- .../rawstudy/model/filesystem/config/files.py | 12 ++--- .../model/filesystem/config/identifier.py | 2 +- .../model/filesystem/config/ini_properties.py | 3 +- .../rawstudy/model/filesystem/config/model.py | 2 +- .../model/filesystem/ini_file_node.py | 6 +-- .../model/filesystem/json_file_node.py | 6 +-- .../model/filesystem/root/input/areas/sets.py | 4 +- .../root/input/hydro/allocation/area.py | 2 +- .../filesystem/root/settings/generaldata.py | 4 +- .../root/user/expansion/settings.py | 4 +- .../study/storage/study_download_utils.py | 2 +- antarest/study/storage/utils.py | 4 +- .../command/create_binding_constraint.py | 2 +- .../model/command/create_user_resource.py | 2 +- .../variantstudy/model/command/icommand.py | 2 +- .../model/command/remove_user_resource.py | 2 +- .../variantstudy/model/command_context.py | 2 +- .../storage/variantstudy/model/dbmodel.py | 2 +- .../study/storage/variantstudy/model/model.py | 2 +- .../variantstudy/variant_study_service.py | 2 +- antarest/study/web/raw_studies_blueprint.py | 2 +- .../study/web/xpansion_studies_blueprint.py | 2 +- antarest/worker/archive_worker.py | 2 +- antarest/worker/worker.py | 2 +- tests/cache/test_redis_cache.py | 2 +- .../antares_io => core/serde}/__init__.py | 0 .../reader => core/serde}/test_ini_reader.py | 9 +--- .../writer => core/serde}/test_ini_writer.py | 4 +- .../business/test_study_version_upgrader.py | 2 +- .../areas/test_st_storage_management.py | 2 +- .../business/test_all_optional_metaclass.py | 2 +- .../model/command/test_create_area.py | 5 +- .../model/command/test_create_link.py | 5 +- .../test_manage_binding_constraints.py | 6 +-- .../model/command/test_manage_district.py | 4 +- .../model/command/test_update_config.py | 5 +- 81 files changed, 174 insertions(+), 171 deletions(-) create mode 100644 antarest/core/serde/__init__.py create mode 100644 antarest/core/serde/ini_common.py rename antarest/{study/storage/rawstudy => core/serde}/ini_reader.py (93%) rename antarest/{study/storage/rawstudy => core/serde}/ini_writer.py (90%) rename antarest/core/{serialization/__init__.py => serde/json.py} (73%) rename tests/{storage/repository/antares_io => core/serde}/__init__.py (100%) rename tests/{storage/repository/antares_io/reader => core/serde}/test_ini_reader.py (98%) rename tests/{storage/repository/antares_io/writer => core/serde}/test_ini_writer.py (92%) diff --git a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py index 7be22d1e24..b2203e171e 100644 --- a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py +++ b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py @@ -15,7 +15,7 @@ from sqlalchemy.engine import Connection # type: ignore from antarest.study.css4_colors import COLOR_NAMES -from antarest.core.serialization import from_json, to_json +from antarest.core.serde.json import from_json, to_json # revision identifiers, used by Alembic. revision = "dae93f1d9110" diff --git a/antarest/core/cache/business/local_chache.py b/antarest/core/cache/business/local_chache.py index 0defd81a4b..2571d3e025 100644 --- a/antarest/core/cache/business/local_chache.py +++ b/antarest/core/cache/business/local_chache.py @@ -20,7 +20,7 @@ from antarest.core.config import CacheConfig from antarest.core.interfaces.cache import ICache from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel logger = logging.getLogger(__name__) diff --git a/antarest/core/cache/business/redis_cache.py b/antarest/core/cache/business/redis_cache.py index 60bd3885ba..6cae07adfc 100644 --- a/antarest/core/cache/business/redis_cache.py +++ b/antarest/core/cache/business/redis_cache.py @@ -18,7 +18,8 @@ from antarest.core.interfaces.cache import ICache from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel, from_json +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import from_json logger = logging.getLogger(__name__) diff --git a/antarest/core/configdata/model.py b/antarest/core/configdata/model.py index 531a77935b..f016779e81 100644 --- a/antarest/core/configdata/model.py +++ b/antarest/core/configdata/model.py @@ -17,7 +17,7 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class ConfigDataDTO(AntaresBaseModel): diff --git a/antarest/core/configdata/repository.py b/antarest/core/configdata/repository.py index b8c458bd6b..5a28cf1af4 100644 --- a/antarest/core/configdata/repository.py +++ b/antarest/core/configdata/repository.py @@ -16,7 +16,7 @@ from antarest.core.configdata.model import ConfigData from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.model import JSON -from antarest.core.serialization import from_json, to_json_string +from antarest.core.serde.json import from_json, to_json_string from antarest.core.utils.fastapi_sqlalchemy import db diff --git a/antarest/core/core_blueprint.py b/antarest/core/core_blueprint.py index 5c50415d47..adac5095c9 100644 --- a/antarest/core/core_blueprint.py +++ b/antarest/core/core_blueprint.py @@ -15,7 +15,7 @@ from fastapi import APIRouter from antarest.core.config import Config -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.web import APITag from antarest.core.version_info import VersionInfoDTO, get_commit_id, get_dependencies diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 10145d744e..43588c7a77 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -27,7 +27,7 @@ from starlette.responses import PlainTextResponse, StreamingResponse from antarest.core.config import Config -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.web import APITag from antarest.login.auth import Auth diff --git a/antarest/core/filetransfer/model.py b/antarest/core/filetransfer/model.py index 772dabf80b..fda4d0e97e 100644 --- a/antarest/core/filetransfer/model.py +++ b/antarest/core/filetransfer/model.py @@ -19,7 +19,7 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class FileDownloadNotFound(HTTPException): diff --git a/antarest/core/interfaces/eventbus.py b/antarest/core/interfaces/eventbus.py index e42b7baaf0..3a83db3fbb 100644 --- a/antarest/core/interfaces/eventbus.py +++ b/antarest/core/interfaces/eventbus.py @@ -17,7 +17,7 @@ from typing_extensions import override from antarest.core.model import PermissionInfo -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class EventType(StrEnum): diff --git a/antarest/core/jwt.py b/antarest/core/jwt.py index 5b7806615e..647162fb00 100644 --- a/antarest/core/jwt.py +++ b/antarest/core/jwt.py @@ -13,7 +13,7 @@ from typing import List, Union from antarest.core.roles import RoleType -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.login.model import ADMIN_ID, Group, Identity diff --git a/antarest/core/model.py b/antarest/core/model.py index a0c61e9830..7f0b64f7db 100644 --- a/antarest/core/model.py +++ b/antarest/core/model.py @@ -13,7 +13,7 @@ import enum from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel if TYPE_CHECKING: # These dependencies are only used for type checking with mypy. diff --git a/antarest/core/serde/__init__.py b/antarest/core/serde/__init__.py new file mode 100644 index 0000000000..7d54736159 --- /dev/null +++ b/antarest/core/serde/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + +""" +This modules hosts technical components related to serializing and deserializing, +for various formats: INI, JSON, ... +""" +import typing as t + +import pydantic + + +class AntaresBaseModel(pydantic.BaseModel): + """ + Due to pydantic migration from v1 to v2, we can have this issue: + + class A(BaseModel): + a: str + + A(a=2) raises ValidationError as we give an int instead of a str + + To avoid this issue we created our own BaseModel class that inherits from BaseModel and allows such object creation. + """ + + model_config = pydantic.config.ConfigDict(coerce_numbers_to_str=True) diff --git a/antarest/core/serde/ini_common.py b/antarest/core/serde/ini_common.py new file mode 100644 index 0000000000..b957b277b8 --- /dev/null +++ b/antarest/core/serde/ini_common.py @@ -0,0 +1,22 @@ +import dataclasses +from typing import Optional + +PrimitiveType = str | int | float | bool + + +@dataclasses.dataclass(frozen=True) +class OptionMatcher: + """ + Used to match a location in an INI file: + a None section means any section. + """ + + section: Optional[str] + key: str + + +def any_section_option_matcher(key: str) -> OptionMatcher: + """ + Return a matcher which will match the provided key in any section. + """ + return OptionMatcher(section=None, key=key) diff --git a/antarest/study/storage/rawstudy/ini_reader.py b/antarest/core/serde/ini_reader.py similarity index 93% rename from antarest/study/storage/rawstudy/ini_reader.py rename to antarest/core/serde/ini_reader.py index 77aafb088e..cf4ad660ae 100644 --- a/antarest/study/storage/rawstudy/ini_reader.py +++ b/antarest/core/serde/ini_reader.py @@ -15,13 +15,13 @@ import typing as t from abc import ABC, abstractmethod from pathlib import Path -from typing import Callable, Optional +from typing import Callable from typing_extensions import override from antarest.core.model import JSON +from antarest.core.serde.ini_common import OptionMatcher, PrimitiveType, any_section_option_matcher -PrimitiveType = str | int | float | bool ValueParser = Callable[[str], PrimitiveType] @@ -49,39 +49,22 @@ def _convert_value(value: str) -> PrimitiveType: return value -@dataclasses.dataclass(frozen=True) -class OptionMatcher: - """ - Used to match a location in an INI file: - a None section means any section. - """ - - section: Optional[str] - key: str - - -def any_section_option_matcher(key: str) -> OptionMatcher: - """ - Return a matcher which will match the provided key in any section. - """ - return OptionMatcher(section=None, key=key) - - class ValueParsers: - def __init__(self, parsers: t.Dict[OptionMatcher, ValueParser]): + def __init__(self, default_parser: ValueParser, parsers: t.Dict[OptionMatcher, ValueParser]): + self._default_parser = default_parser self._parsers = parsers def find_parser(self, section: str, key: str) -> ValueParser: - if not self._parsers: - return _convert_value - possible_keys = [ - OptionMatcher(section=section, key=key), - OptionMatcher(section=None, key=key), - ] - for k in possible_keys: - if parser := self._parsers.get(k, None): - return parser - return _convert_value + if self._parsers: + possible_keys = [ + OptionMatcher(section=section, key=key), + any_section_option_matcher(key=key), + ] + for k in possible_keys: + if parser := self._parsers.get(k, None): + return parser + + return self._default_parser @dataclasses.dataclass(frozen=True) @@ -208,7 +191,7 @@ def __init__( # Default section name to use if `.ini` file has no section. self._special_keys = set(special_keys) - self._value_parsers = ValueParsers(value_parsers or {}) + self._value_parsers = ValueParsers(default_parser=_convert_value, parsers=value_parsers or {}) # List of keys which should be parsed as list. self._section_name = section_name diff --git a/antarest/study/storage/rawstudy/ini_writer.py b/antarest/core/serde/ini_writer.py similarity index 90% rename from antarest/study/storage/rawstudy/ini_writer.py rename to antarest/core/serde/ini_writer.py index df59cd3a0a..a887fb23e5 100644 --- a/antarest/study/storage/rawstudy/ini_writer.py +++ b/antarest/core/serde/ini_writer.py @@ -18,7 +18,7 @@ from typing_extensions import override from antarest.core.model import JSON -from antarest.study.storage.rawstudy.ini_reader import OptionMatcher, PrimitiveType +from antarest.core.serde.ini_common import OptionMatcher, PrimitiveType, any_section_option_matcher # Value serializers may be used to customize the way INI options are serialized ValueSerializer = Callable[[str], PrimitiveType] @@ -36,15 +36,14 @@ def __init__(self, serializers: Dict[OptionMatcher, ValueSerializer]): self._serializers = serializers def find_serializer(self, section: str, key: str) -> Optional[ValueSerializer]: - if not self._serializers: - return None - possible_keys = [ - OptionMatcher(section=section, key=key), - OptionMatcher(section=None, key=key), - ] - for k in possible_keys: - if parser := self._serializers.get(k, None): - return parser + if self._serializers: + possible_keys = [ + OptionMatcher(section=section, key=key), + any_section_option_matcher(key=key), + ] + for k in possible_keys: + if parser := self._serializers.get(k, None): + return parser return None diff --git a/antarest/core/serialization/__init__.py b/antarest/core/serde/json.py similarity index 73% rename from antarest/core/serialization/__init__.py rename to antarest/core/serde/json.py index 5591290ce8..5eb59b05e6 100644 --- a/antarest/core/serialization/__init__.py +++ b/antarest/core/serde/json.py @@ -32,18 +32,3 @@ def to_json(data: t.Any, indent: t.Optional[int] = None) -> bytes: def to_json_string(data: t.Any, indent: t.Optional[int] = None) -> str: return to_json(data, indent=indent).decode("utf-8") - - -class AntaresBaseModel(pydantic.BaseModel): - """ - Due to pydantic migration from v1 to v2, we can have this issue: - - class A(BaseModel): - a: str - - A(a=2) raises ValidationError as we give an int instead of a str - - To avoid this issue we created our own BaseModel class that inherits from BaseModel and allows such object creation. - """ - - model_config = pydantic.config.ConfigDict(coerce_numbers_to_str=True) diff --git a/antarest/core/tasks/model.py b/antarest/core/tasks/model.py index ce4ab5631a..cb03e1aeb7 100644 --- a/antarest/core/tasks/model.py +++ b/antarest/core/tasks/model.py @@ -21,7 +21,7 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel if t.TYPE_CHECKING: # avoid circular import diff --git a/antarest/core/version_info.py b/antarest/core/version_info.py index 0b7f312b5b..724e85f697 100644 --- a/antarest/core/version_info.py +++ b/antarest/core/version_info.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import Dict -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class VersionInfoDTO(AntaresBaseModel): diff --git a/antarest/eventbus/web.py b/antarest/eventbus/web.py index 53ea96964a..ffcee5236a 100644 --- a/antarest/eventbus/web.py +++ b/antarest/eventbus/web.py @@ -25,7 +25,8 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTUser from antarest.core.model import PermissionInfo, StudyPermissionType from antarest.core.permissions import check_permission -from antarest.core.serialization import AntaresBaseModel, to_json_string +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import to_json_string from antarest.fastapi_jwt_auth import AuthJWT from antarest.login.auth import Auth diff --git a/antarest/front.py b/antarest/front.py index 98f5ab83d8..9431180c80 100644 --- a/antarest/front.py +++ b/antarest/front.py @@ -32,7 +32,7 @@ from starlette.types import ASGIApp from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.string import to_camel_case diff --git a/antarest/launcher/adapters/log_parser.py b/antarest/launcher/adapters/log_parser.py index 0583340917..a9d6ca1375 100644 --- a/antarest/launcher/adapters/log_parser.py +++ b/antarest/launcher/adapters/log_parser.py @@ -14,7 +14,7 @@ import re import typing as t -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel _SearchFunc = t.Callable[[str], t.Optional[t.Match[str]]] diff --git a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py index 79e8b94fb9..d80103dadd 100644 --- a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py +++ b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py @@ -34,13 +34,13 @@ from antarest.core.interfaces.cache import ICache from antarest.core.interfaces.eventbus import Event, EventType, IEventBus from antarest.core.model import PermissionInfo, PublicMode +from antarest.core.serde.ini_reader import IniReader +from antarest.core.serde.ini_writer import IniWriter from antarest.core.utils.archives import unzip from antarest.core.utils.utils import assert_this from antarest.launcher.adapters.abstractlauncher import AbstractLauncher, LauncherCallbacks, LauncherInitException from antarest.launcher.adapters.log_manager import LogTailManager from antarest.launcher.model import JobStatus, LauncherParametersDTO, LogType, XpansionParametersDTO -from antarest.study.storage.rawstudy.ini_reader import IniReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter logger = logging.getLogger(__name__) logging.getLogger("paramiko").setLevel("WARN") diff --git a/antarest/launcher/model.py b/antarest/launcher/model.py index df5c9cb903..ca0b7031a3 100644 --- a/antarest/launcher/model.py +++ b/antarest/launcher/model.py @@ -20,7 +20,8 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel, from_json +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import from_json from antarest.login.model import Identity, UserInfo from antarest.study.business.all_optional_meta import camel_case_model diff --git a/antarest/launcher/ssh_config.py b/antarest/launcher/ssh_config.py index fd0bf42e9b..3cb2858b8e 100644 --- a/antarest/launcher/ssh_config.py +++ b/antarest/launcher/ssh_config.py @@ -16,7 +16,7 @@ import paramiko from pydantic import model_validator -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class SSHConfigDTO(AntaresBaseModel): diff --git a/antarest/login/auth.py b/antarest/login/auth.py index 01a38cfc09..a5d5008fe1 100644 --- a/antarest/login/auth.py +++ b/antarest/login/auth.py @@ -20,7 +20,8 @@ from antarest.core.config import Config from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTUser -from antarest.core.serialization import AntaresBaseModel, from_json +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import from_json from antarest.fastapi_jwt_auth import AuthJWT logger = logging.getLogger(__name__) diff --git a/antarest/login/main.py b/antarest/login/main.py index 34de1ffbc5..31cc6a7144 100644 --- a/antarest/login/main.py +++ b/antarest/login/main.py @@ -19,7 +19,7 @@ from antarest.core.application import AppBuildContext from antarest.core.config import Config from antarest.core.interfaces.eventbus import DummyEventBusService, IEventBus -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.core.utils.fastapi_sqlalchemy import db from antarest.fastapi_jwt_auth import AuthJWT from antarest.fastapi_jwt_auth.exceptions import AuthJWTException diff --git a/antarest/login/model.py b/antarest/login/model.py index cfa7a7a015..004c2c7420 100644 --- a/antarest/login/model.py +++ b/antarest/login/model.py @@ -24,7 +24,7 @@ from antarest.core.persistence import Base from antarest.core.roles import RoleType -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel if t.TYPE_CHECKING: # avoid circular import diff --git a/antarest/login/web.py b/antarest/login/web.py index 3c0dd9be2d..0ea784dc3e 100644 --- a/antarest/login/web.py +++ b/antarest/login/web.py @@ -21,7 +21,8 @@ from antarest.core.jwt import JWTGroup, JWTUser from antarest.core.requests import RequestParameters, UserHasNotPermissionError from antarest.core.roles import RoleType -from antarest.core.serialization import AntaresBaseModel, from_json +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import from_json from antarest.core.utils.web import APITag from antarest.fastapi_jwt_auth import AuthJWT from antarest.login.auth import Auth diff --git a/antarest/matrixstore/matrix_editor.py b/antarest/matrixstore/matrix_editor.py index a8850e1c41..5e06f71c5d 100644 --- a/antarest/matrixstore/matrix_editor.py +++ b/antarest/matrixstore/matrix_editor.py @@ -17,7 +17,7 @@ from pydantic import Field, field_validator, model_validator from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class MatrixSlice(AntaresBaseModel): diff --git a/antarest/matrixstore/model.py b/antarest/matrixstore/model.py index b117e12e1f..ae288251b0 100644 --- a/antarest/matrixstore/model.py +++ b/antarest/matrixstore/model.py @@ -19,7 +19,7 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.login.model import GroupDTO, Identity, UserInfo diff --git a/antarest/matrixstore/service.py b/antarest/matrixstore/service.py index ca69f40fa2..36c7c58b40 100644 --- a/antarest/matrixstore/service.py +++ b/antarest/matrixstore/service.py @@ -31,7 +31,7 @@ from antarest.core.filetransfer.service import FileTransferManager from antarest.core.jwt import JWTUser from antarest.core.requests import RequestParameters, UserHasNotPermissionError -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.core.tasks.model import TaskResult, TaskType from antarest.core.tasks.service import ITaskNotifier, ITaskService from antarest.core.utils.archives import ArchiveFormat, archive_dir diff --git a/antarest/study/business/area_management.py b/antarest/study/business/area_management.py index c607be980d..9b176d0c05 100644 --- a/antarest/study/business/area_management.py +++ b/antarest/study/business/area_management.py @@ -19,7 +19,7 @@ from antarest.core.exceptions import ConfigFileNotFound, DuplicateAreaName, LayerNotAllowedToBeDeleted, LayerNotFound from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model from antarest.study.business.utils import execute_or_add_commands from antarest.study.model import Patch, PatchArea, PatchCluster, RawStudy, Study diff --git a/antarest/study/business/areas/st_storage_management.py b/antarest/study/business/areas/st_storage_management.py index ef89e345d8..199785cb07 100644 --- a/antarest/study/business/areas/st_storage_management.py +++ b/antarest/study/business/areas/st_storage_management.py @@ -29,7 +29,7 @@ ) from antarest.core.model import JSON from antarest.core.requests import CaseInsensitiveDict -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model from antarest.study.business.utils import execute_or_add_commands from antarest.study.model import STUDY_VERSION_8_8, Study diff --git a/antarest/study/business/binding_constraint_management.py b/antarest/study/business/binding_constraint_management.py index fd3970b9b1..93f25f2237 100644 --- a/antarest/study/business/binding_constraint_management.py +++ b/antarest/study/business/binding_constraint_management.py @@ -32,7 +32,7 @@ ) from antarest.core.model import JSON from antarest.core.requests import CaseInsensitiveDict -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.string import to_camel_case from antarest.study.business.all_optional_meta import camel_case_model from antarest.study.business.utils import execute_or_add_commands diff --git a/antarest/study/business/district_manager.py b/antarest/study/business/district_manager.py index 2479760866..a10628ad6a 100644 --- a/antarest/study/business/district_manager.py +++ b/antarest/study/business/district_manager.py @@ -13,7 +13,7 @@ from typing import List from antarest.core.exceptions import AreaNotFound, DistrictAlreadyExist, DistrictNotFound -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.utils import execute_or_add_commands from antarest.study.model import Study from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id diff --git a/antarest/study/business/model/link_model.py b/antarest/study/business/model/link_model.py index 9d6f50e1f5..45b27eae8f 100644 --- a/antarest/study/business/model/link_model.py +++ b/antarest/study/business/model/link_model.py @@ -15,7 +15,7 @@ from pydantic import BeforeValidator, ConfigDict, Field, PlainSerializer, model_validator from antarest.core.exceptions import LinkValidationError -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.string import to_camel_case, to_kebab_case from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import STUDY_VERSION_8_2 diff --git a/antarest/study/business/timeseries_config_management.py b/antarest/study/business/timeseries_config_management.py index 5c7f5aa928..853fd832f1 100644 --- a/antarest/study/business/timeseries_config_management.py +++ b/antarest/study/business/timeseries_config_management.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.all_optional_meta import all_optional_model from antarest.study.business.utils import GENERAL_DATA_PATH, execute_or_add_commands from antarest.study.model import Study diff --git a/antarest/study/business/utils.py b/antarest/study/business/utils.py index 628dd4a0c4..3d02e40ca9 100644 --- a/antarest/study/business/utils.py +++ b/antarest/study/business/utils.py @@ -16,7 +16,7 @@ from antarest.core.exceptions import CommandApplicationError from antarest.core.requests import RequestParameters -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.login.utils import get_current_user from antarest.study.business.all_optional_meta import camel_case_model from antarest.study.model import RawStudy, Study diff --git a/antarest/study/business/xpansion_management.py b/antarest/study/business/xpansion_management.py index 7464de5ec5..34ba680049 100644 --- a/antarest/study/business/xpansion_management.py +++ b/antarest/study/business/xpansion_management.py @@ -23,7 +23,7 @@ from antarest.core.exceptions import BadZipBinary, ChildNotFoundError, LinkNotFound from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.all_optional_meta import all_optional_model from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import Study diff --git a/antarest/study/model.py b/antarest/study/model.py index 004c836844..e6e98756c8 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -37,7 +37,7 @@ from antarest.core.exceptions import ShouldNotHappenException from antarest.core.model import PublicMode from antarest.core.persistence import Base -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.login.model import Group, GroupDTO, Identity from antarest.study.css4_colors import COLOR_NAMES diff --git a/antarest/study/repository.py b/antarest/study/repository.py index a904124d4d..b3d01a1de6 100644 --- a/antarest/study/repository.py +++ b/antarest/study/repository.py @@ -22,7 +22,7 @@ from antarest.core.jwt import JWTUser from antarest.core.model import PublicMode from antarest.core.requests import RequestParameters -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.fastapi_sqlalchemy import db from antarest.login.model import Group from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, Study, StudyAdditionalData, Tag diff --git a/antarest/study/service.py b/antarest/study/service.py index 31bb80fa1f..83250c64fe 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -60,7 +60,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTGroup, JWTUser from antarest.core.model import JSON, SUB_JSON, PermissionInfo, PublicMode, StudyPermissionType from antarest.core.requests import RequestParameters, UserHasNotPermissionError -from antarest.core.serialization import to_json +from antarest.core.serde.json import to_json from antarest.core.tasks.model import TaskListFilter, TaskResult, TaskStatus, TaskType from antarest.core.tasks.service import ITaskNotifier, ITaskService, NoopNotifier from antarest.core.utils.archives import ArchiveFormat, is_archive_format diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index 69d9ba6a27..fdb97987bf 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -24,7 +24,7 @@ from antarest.core.exceptions import BadOutputError, StudyOutputNotFoundError from antarest.core.interfaces.cache import CacheConstants, ICache from antarest.core.model import JSON, PublicMode -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.core.utils.archives import ArchiveFormat, archive_dir, extract_archive, unzip from antarest.core.utils.utils import StopWatch from antarest.login.model import GroupDTO diff --git a/antarest/study/storage/patch_service.py b/antarest/study/storage/patch_service.py index f98870e240..c14c71cab7 100644 --- a/antarest/study/storage/patch_service.py +++ b/antarest/study/storage/patch_service.py @@ -13,7 +13,7 @@ import typing as t from pathlib import Path -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.study.model import Patch, PatchOutputs, RawStudy, StudyAdditionalData from antarest.study.repository import StudyMetadataRepository from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/cluster.py b/antarest/study/storage/rawstudy/model/filesystem/config/cluster.py index f26f2b45e4..c38e446ebc 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/cluster.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/cluster.py @@ -21,7 +21,7 @@ from pydantic import Field -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel @functools.total_ordering diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/files.py b/antarest/study/storage/rawstudy/model/filesystem/config/files.py index 52458c9970..e72bc44086 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/files.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/files.py @@ -20,19 +20,13 @@ from enum import Enum from pathlib import Path -import py7zr from antares.study.version import StudyVersion from antarest.core.model import JSON -from antarest.core.serialization import from_json -from antarest.core.utils.archives import ( - ArchiveFormat, - extract_lines_from_archive, - is_archive_format, - read_file_from_archive, -) +from antarest.core.serde.ini_reader import IniReader +from antarest.core.serde.json import from_json +from antarest.core.utils.archives import extract_lines_from_archive, is_archive_format, read_file_from_archive from antarest.study.model import STUDY_VERSION_8_1, STUDY_VERSION_8_6 -from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import ( DEFAULT_GROUP, DEFAULT_OPERATOR, diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/identifier.py b/antarest/study/storage/rawstudy/model/filesystem/config/identifier.py index e25c5ab9fb..1f26ddddd8 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/identifier.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/identifier.py @@ -18,7 +18,7 @@ from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel class IgnoreCaseIdentifier( diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py b/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py index 05451c6870..b5cd6ddaea 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py @@ -14,7 +14,8 @@ from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel, from_json, to_json +from antarest.core.serde import AntaresBaseModel +from antarest.core.serde.json import from_json, to_json class IniProperties( diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/model.py b/antarest/study/storage/rawstudy/model/filesystem/config/model.py index 388bfaeb5d..c07fa335b7 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/model.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/model.py @@ -18,7 +18,7 @@ from pydantic import Field, model_validator from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.utils import DTO from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.model import StudyVersionInt diff --git a/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py index 960ccc83e4..34766dbff0 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py @@ -27,9 +27,9 @@ from antarest.core.exceptions import ShouldNotHappenException from antarest.core.model import JSON, SUB_JSON -from antarest.core.serialization import from_json -from antarest.study.storage.rawstudy.ini_reader import IniReader, IReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter +from antarest.core.serde.ini_reader import IniReader, IReader +from antarest.core.serde.ini_writer import IniWriter +from antarest.core.serde.json import from_json from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.inode import INode diff --git a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py index 0ff675c650..facc7a79fb 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py @@ -17,9 +17,9 @@ from typing_extensions import override from antarest.core.model import JSON -from antarest.core.serialization import from_json, to_json -from antarest.study.storage.rawstudy.ini_reader import IReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter +from antarest.core.serde.ini_reader import IReader +from antarest.core.serde.ini_writer import IniWriter +from antarest.core.serde.json import from_json, to_json from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode diff --git a/antarest/study/storage/rawstudy/model/filesystem/root/input/areas/sets.py b/antarest/study/storage/rawstudy/model/filesystem/root/input/areas/sets.py index 0f9adda270..d0b55f3e04 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/root/input/areas/sets.py +++ b/antarest/study/storage/rawstudy/model/filesystem/root/input/areas/sets.py @@ -10,8 +10,8 @@ # # This file is part of the Antares project. -from antarest.study.storage.rawstudy.ini_reader import IniReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter +from antarest.core.serde.ini_reader import IniReader +from antarest.core.serde.ini_writer import IniWriter from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode diff --git a/antarest/study/storage/rawstudy/model/filesystem/root/input/hydro/allocation/area.py b/antarest/study/storage/rawstudy/model/filesystem/root/input/hydro/allocation/area.py index fac71885a2..f3c33ab794 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/root/input/hydro/allocation/area.py +++ b/antarest/study/storage/rawstudy/model/filesystem/root/input/hydro/allocation/area.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode diff --git a/antarest/study/storage/rawstudy/model/filesystem/root/settings/generaldata.py b/antarest/study/storage/rawstudy/model/filesystem/root/settings/generaldata.py index b3abcfba0e..9728015d40 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/root/settings/generaldata.py +++ b/antarest/study/storage/rawstudy/model/filesystem/root/settings/generaldata.py @@ -13,6 +13,8 @@ from copy import deepcopy from typing import Any, Dict +from antarest.core.serde.ini_reader import IniReader +from antarest.core.serde.ini_writer import IniWriter from antarest.study.model import ( STUDY_VERSION_6_5, STUDY_VERSION_7_0, @@ -25,8 +27,6 @@ STUDY_VERSION_8_5, STUDY_VERSION_8_6, ) -from antarest.study.storage.rawstudy.ini_reader import IniReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode diff --git a/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/settings.py b/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/settings.py index c162da1029..1516ffb626 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/settings.py +++ b/antarest/study/storage/rawstudy/model/filesystem/root/user/expansion/settings.py @@ -10,8 +10,8 @@ # # This file is part of the Antares project. -from antarest.study.storage.rawstudy.ini_reader import SimpleKeyValueReader -from antarest.study.storage.rawstudy.ini_writer import SimpleKeyValueWriter +from antarest.core.serde.ini_reader import SimpleKeyValueReader +from antarest.core.serde.ini_writer import SimpleKeyValueWriter from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode diff --git a/antarest/study/storage/study_download_utils.py b/antarest/study/storage/study_download_utils.py index 3048466a7d..bd49016fdc 100644 --- a/antarest/study/storage/study_download_utils.py +++ b/antarest/study/storage/study_download_utils.py @@ -25,7 +25,7 @@ from fastapi import HTTPException from antarest.core.exceptions import ChildNotFoundError -from antarest.core.serialization import to_json +from antarest.core.serde.json import to_json from antarest.study.model import ( STUDY_VERSION_8_1, ExportFormat, diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index e8336cc1d1..90f5cb1d7a 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -40,6 +40,8 @@ from antarest.core.model import PermissionInfo, StudyPermissionType from antarest.core.permissions import check_permission from antarest.core.requests import UserHasNotPermissionError +from antarest.core.serde.ini_reader import IniReader +from antarest.core.serde.ini_writer import IniWriter from antarest.core.utils.archives import is_archive_format from antarest.core.utils.utils import StopWatch from antarest.study.model import ( @@ -50,8 +52,6 @@ StudyDownloadLevelDTO, StudyMetadataDTO, ) -from antarest.study.storage.rawstudy.ini_reader import IniReader -from antarest.study.storage.rawstudy.ini_writer import IniWriter from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory from antarest.study.storage.rawstudy.model.filesystem.root.filestudytree import FileStudyTree from antarest.study.storage.rawstudy.model.helpers import FileStudyHelpers diff --git a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py index 1a94a256be..a84053c074 100644 --- a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py @@ -19,7 +19,7 @@ from pydantic import Field, field_validator, model_validator from typing_extensions import override -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.matrixstore.model import MatrixData from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model from antarest.study.model import STUDY_VERSION_8_3, STUDY_VERSION_8_7 diff --git a/antarest/study/storage/variantstudy/model/command/create_user_resource.py b/antarest/study/storage/variantstudy/model/command/create_user_resource.py index 7274df3e13..27e4b8d7f4 100644 --- a/antarest/study/storage/variantstudy/model/command/create_user_resource.py +++ b/antarest/study/storage/variantstudy/model/command/create_user_resource.py @@ -16,7 +16,7 @@ from antarest.core.exceptions import ChildNotFoundError from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.root.user.user import User diff --git a/antarest/study/storage/variantstudy/model/command/icommand.py b/antarest/study/storage/variantstudy/model/command/icommand.py index 018729799b..792200f09c 100644 --- a/antarest/study/storage/variantstudy/model/command/icommand.py +++ b/antarest/study/storage/variantstudy/model/command/icommand.py @@ -17,7 +17,7 @@ import typing_extensions as te -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.utils.utils import assert_this from antarest.study.model import StudyVersionStr from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig diff --git a/antarest/study/storage/variantstudy/model/command/remove_user_resource.py b/antarest/study/storage/variantstudy/model/command/remove_user_resource.py index d626c031fe..9cac0259c6 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_user_resource.py +++ b/antarest/study/storage/variantstudy/model/command/remove_user_resource.py @@ -15,7 +15,7 @@ from typing_extensions import override from antarest.core.exceptions import ChildNotFoundError -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.root.user.user import User diff --git a/antarest/study/storage/variantstudy/model/command_context.py b/antarest/study/storage/variantstudy/model/command_context.py index 9291e8bb6a..ff59a3339b 100644 --- a/antarest/study/storage/variantstudy/model/command_context.py +++ b/antarest/study/storage/variantstudy/model/command_context.py @@ -10,7 +10,7 @@ # # This file is part of the Antares project. -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.matrixstore.service import ISimpleMatrixService from antarest.study.storage.patch_service import PatchService from antarest.study.storage.variantstudy.business.matrix_constants_generator import GeneratorMatrixConstants diff --git a/antarest/study/storage/variantstudy/model/dbmodel.py b/antarest/study/storage/variantstudy/model/dbmodel.py index d0d6435732..e76d93a247 100644 --- a/antarest/study/storage/variantstudy/model/dbmodel.py +++ b/antarest/study/storage/variantstudy/model/dbmodel.py @@ -20,7 +20,7 @@ from typing_extensions import override from antarest.core.persistence import Base -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.study.model import Study from antarest.study.storage.variantstudy.model.model import CommandDTO diff --git a/antarest/study/storage/variantstudy/model/model.py b/antarest/study/storage/variantstudy/model/model.py index cb1866037f..fc168593f3 100644 --- a/antarest/study/storage/variantstudy/model/model.py +++ b/antarest/study/storage/variantstudy/model/model.py @@ -16,7 +16,7 @@ import typing_extensions as te from antarest.core.model import JSON -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.model import StudyMetadataDTO, StudyVersionStr LegacyDetailsDTO = t.Tuple[str, bool, str] diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index fc9d447ea0..77c2104f9a 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -45,7 +45,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.model import JSON, PermissionInfo, PublicMode, StudyPermissionType from antarest.core.requests import RequestParameters, UserHasNotPermissionError -from antarest.core.serialization import to_json_string +from antarest.core.serde.json import to_json_string from antarest.core.tasks.model import CustomTaskEventMessages, TaskDTO, TaskResult, TaskType from antarest.core.tasks.service import DEFAULT_AWAIT_MAX_TIMEOUT, ITaskNotifier, ITaskService, NoopNotifier from antarest.core.utils.fastapi_sqlalchemy import db diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index a06eba7515..07a3328187 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -25,7 +25,7 @@ from antarest.core.jwt import JWTUser from antarest.core.model import SUB_JSON from antarest.core.requests import RequestParameters -from antarest.core.serialization import from_json, to_json +from antarest.core.serde.json import from_json, to_json from antarest.core.swagger import get_path_examples from antarest.core.utils.utils import sanitize_string, sanitize_uuid from antarest.core.utils.web import APITag diff --git a/antarest/study/web/xpansion_studies_blueprint.py b/antarest/study/web/xpansion_studies_blueprint.py index 0871efd0f0..ce02a076cc 100644 --- a/antarest/study/web/xpansion_studies_blueprint.py +++ b/antarest/study/web/xpansion_studies_blueprint.py @@ -20,7 +20,7 @@ from antarest.core.jwt import JWTUser from antarest.core.model import JSON, StudyPermissionType from antarest.core.requests import RequestParameters -from antarest.core.serialization import to_json +from antarest.core.serde.json import to_json from antarest.core.utils.web import APITag from antarest.login.auth import Auth from antarest.study.business.xpansion_management import ( diff --git a/antarest/worker/archive_worker.py b/antarest/worker/archive_worker.py index cf62083b24..4b68abd7ac 100644 --- a/antarest/worker/archive_worker.py +++ b/antarest/worker/archive_worker.py @@ -17,7 +17,7 @@ from antarest.core.config import Config from antarest.core.interfaces.eventbus import IEventBus -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.tasks.model import TaskResult from antarest.core.utils.archives import unzip from antarest.core.utils.utils import StopWatch diff --git a/antarest/worker/worker.py b/antarest/worker/worker.py index a3525b5969..d0176dbe19 100644 --- a/antarest/worker/worker.py +++ b/antarest/worker/worker.py @@ -21,7 +21,7 @@ from antarest.core.interfaces.eventbus import Event, EventType, IEventBus from antarest.core.interfaces.service import IService from antarest.core.model import PermissionInfo, PublicMode -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.core.tasks.model import TaskResult logger = logging.getLogger(__name__) diff --git a/tests/cache/test_redis_cache.py b/tests/cache/test_redis_cache.py index 5e9aa5d7ae..6df668a41d 100644 --- a/tests/cache/test_redis_cache.py +++ b/tests/cache/test_redis_cache.py @@ -16,7 +16,7 @@ from antares.study.version import StudyVersion from antarest.core.cache.business.redis_cache import RedisCache, RedisCacheElement -from antarest.core.serialization import from_json +from antarest.core.serde.json import from_json from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, FileStudyTreeConfigDTO diff --git a/tests/storage/repository/antares_io/__init__.py b/tests/core/serde/__init__.py similarity index 100% rename from tests/storage/repository/antares_io/__init__.py rename to tests/core/serde/__init__.py diff --git a/tests/storage/repository/antares_io/reader/test_ini_reader.py b/tests/core/serde/test_ini_reader.py similarity index 98% rename from tests/storage/repository/antares_io/reader/test_ini_reader.py rename to tests/core/serde/test_ini_reader.py index 054bb459bc..916f9e1bf8 100644 --- a/tests/storage/repository/antares_io/reader/test_ini_reader.py +++ b/tests/core/serde/test_ini_reader.py @@ -14,13 +14,8 @@ import textwrap from pathlib import Path -from antarest.study.storage.rawstudy.ini_reader import ( - LOWER_CASE_PARSER, - IniReader, - OptionMatcher, - SimpleKeyValueReader, - any_section_option_matcher, -) +from antarest.core.serde.ini_common import OptionMatcher, any_section_option_matcher +from antarest.core.serde.ini_reader import LOWER_CASE_PARSER, IniReader, SimpleKeyValueReader class TestIniReader: diff --git a/tests/storage/repository/antares_io/writer/test_ini_writer.py b/tests/core/serde/test_ini_writer.py similarity index 92% rename from tests/storage/repository/antares_io/writer/test_ini_writer.py rename to tests/core/serde/test_ini_writer.py index 3cc44b32af..06587c0fe8 100644 --- a/tests/storage/repository/antares_io/writer/test_ini_writer.py +++ b/tests/core/serde/test_ini_writer.py @@ -15,8 +15,8 @@ import pytest -from antarest.study.storage.rawstudy.ini_reader import OptionMatcher, any_section_option_matcher -from antarest.study.storage.rawstudy.ini_writer import LOWER_CASE_SERIALIZER, IniWriter +from antarest.core.serde.ini_common import any_section_option_matcher +from antarest.core.serde.ini_writer import LOWER_CASE_SERIALIZER, IniWriter @pytest.mark.unit_test diff --git a/tests/storage/business/test_study_version_upgrader.py b/tests/storage/business/test_study_version_upgrader.py index c5fc6443dc..4f0e6d6e51 100644 --- a/tests/storage/business/test_study_version_upgrader.py +++ b/tests/storage/business/test_study_version_upgrader.py @@ -24,7 +24,7 @@ from pandas.errors import EmptyDataError from antarest.core.exceptions import UnsupportedStudyVersion -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.root.settings.generaldata import DUPLICATE_KEYS from antarest.study.storage.study_upgrader import ( diff --git a/tests/study/business/areas/test_st_storage_management.py b/tests/study/business/areas/test_st_storage_management.py index 8ce3b3c910..a3dfa00a05 100644 --- a/tests/study/business/areas/test_st_storage_management.py +++ b/tests/study/business/areas/test_st_storage_management.py @@ -24,10 +24,10 @@ from antarest.core.exceptions import AreaNotFound, STStorageConfigNotFound, STStorageMatrixNotFound, STStorageNotFound from antarest.core.model import PublicMode +from antarest.core.serde.ini_reader import IniReader from antarest.login.model import Group, User from antarest.study.business.areas.st_storage_management import STStorageInput, STStorageManager from antarest.study.model import RawStudy, Study, StudyContentStatus -from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import STStorageGroup from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy diff --git a/tests/study/business/test_all_optional_metaclass.py b/tests/study/business/test_all_optional_metaclass.py index 8e67794a67..e6c8fb5a96 100644 --- a/tests/study/business/test_all_optional_metaclass.py +++ b/tests/study/business/test_all_optional_metaclass.py @@ -12,7 +12,7 @@ from pydantic import Field -from antarest.core.serialization import AntaresBaseModel +from antarest.core.serde import AntaresBaseModel from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model diff --git a/tests/variantstudy/model/command/test_create_area.py b/tests/variantstudy/model/command/test_create_area.py index 0e18ae209d..3fa629f345 100644 --- a/tests/variantstudy/model/command/test_create_area.py +++ b/tests/variantstudy/model/command/test_create_area.py @@ -11,18 +11,15 @@ # This file is part of the Antares project. import configparser -from unittest.mock import Mock import pytest from antares.study.version import StudyVersion -from antarest.study.model import STUDY_VERSION_8_8 -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import EnrModelling, transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.icommand import ICommand -from antarest.study.storage.variantstudy.model.command.remove_area import RemoveArea from antarest.study.storage.variantstudy.model.command.update_config import UpdateConfig from antarest.study.storage.variantstudy.model.command_context import CommandContext diff --git a/tests/variantstudy/model/command/test_create_link.py b/tests/variantstudy/model/command/test_create_link.py index c21ee76197..ed886e8908 100644 --- a/tests/variantstudy/model/command/test_create_link.py +++ b/tests/variantstudy/model/command/test_create_link.py @@ -10,20 +10,17 @@ # # This file is part of the Antares project. -from unittest.mock import Mock - import pytest from pydantic import ValidationError from antarest.core.exceptions import LinkValidationError +from antarest.core.serde.ini_reader import IniReader from antarest.study.model import STUDY_VERSION_8_8 -from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.create_link import CreateLink from antarest.study.storage.variantstudy.model.command.icommand import ICommand -from antarest.study.storage.variantstudy.model.command.remove_link import RemoveLink from antarest.study.storage.variantstudy.model.command_context import CommandContext diff --git a/tests/variantstudy/model/command/test_manage_binding_constraints.py b/tests/variantstudy/model/command/test_manage_binding_constraints.py index aabfba93c4..e70ffc29a9 100644 --- a/tests/variantstudy/model/command/test_manage_binding_constraints.py +++ b/tests/variantstudy/model/command/test_manage_binding_constraints.py @@ -10,12 +10,9 @@ # # This file is part of the Antares project. -from unittest.mock import Mock - import pytest -from antarest.study.model import STUDY_VERSION_8_8 -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import ( BindingConstraintFrequency, BindingConstraintOperator, @@ -26,7 +23,6 @@ default_bc_weekly_daily as default_bc_weekly_daily_870, ) from antarest.study.storage.variantstudy.business.matrix_constants.binding_constraint.series_before_v87 import ( - default_bc_hourly, default_bc_weekly_daily, ) from antarest.study.storage.variantstudy.model.command.create_area import CreateArea diff --git a/tests/variantstudy/model/command/test_manage_district.py b/tests/variantstudy/model/command/test_manage_district.py index 5aa642977a..8e10c40d57 100644 --- a/tests/variantstudy/model/command/test_manage_district.py +++ b/tests/variantstudy/model/command/test_manage_district.py @@ -9,10 +9,8 @@ # SPDX-License-Identifier: MPL-2.0 # # This file is part of the Antares project. -from unittest.mock import Mock -from antarest.study.model import STUDY_VERSION_8_8 -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.files import build from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy diff --git a/tests/variantstudy/model/command/test_update_config.py b/tests/variantstudy/model/command/test_update_config.py index 6e90bcf4c1..b87583914a 100644 --- a/tests/variantstudy/model/command/test_update_config.py +++ b/tests/variantstudy/model/command/test_update_config.py @@ -11,13 +11,10 @@ # This file is part of the Antares project. import json -from unittest.mock import Mock, patch import pytest -from antarest.core.exceptions import ChildNotFoundError -from antarest.study.model import STUDY_VERSION_8_8 -from antarest.study.storage.rawstudy.ini_reader import IniReader +from antarest.core.serde.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.create_area import CreateArea From 68c6dcd7fa9992463dadb4c613cebe7eefb68c05 Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Fri, 7 Feb 2025 11:42:27 +0100 Subject: [PATCH 6/7] fix(tests): move and improve tests Signed-off-by: Sylvain Leclerc --- tests/conftest.py | 19 +++++++++++++ tests/core/serde/test_ini_reader.py | 41 +++++++++++++++++++++++++---- tests/core/serde/test_ini_writer.py | 14 +++++++--- tests/storage/conftest.py | 18 ------------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e384338bc8..b6e83bbac2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ # This file is part of the Antares project. from pathlib import Path +from typing import Callable import pytest @@ -30,3 +31,21 @@ @pytest.fixture(scope="session") def project_path() -> Path: return PROJECT_DIR + + +@pytest.fixture +def ini_cleaner() -> Callable[[str], str]: + def cleaner(txt: str) -> str: + lines = filter(None, map(str.strip, txt.splitlines(keepends=False))) + return "\n".join(lines) + + return cleaner + + +@pytest.fixture +def clean_ini_writer(ini_cleaner: Callable[[str], str]) -> Callable[[Path, str], None]: + def write_clean_ini(path: Path, txt: str) -> None: + clean_ini = ini_cleaner(txt) + path.write_text(clean_ini) + + return write_clean_ini diff --git a/tests/core/serde/test_ini_reader.py b/tests/core/serde/test_ini_reader.py index 916f9e1bf8..0c8bffdbc3 100644 --- a/tests/core/serde/test_ini_reader.py +++ b/tests/core/serde/test_ini_reader.py @@ -15,7 +15,35 @@ from pathlib import Path from antarest.core.serde.ini_common import OptionMatcher, any_section_option_matcher -from antarest.core.serde.ini_reader import LOWER_CASE_PARSER, IniReader, SimpleKeyValueReader +from antarest.core.serde.ini_reader import LOWER_CASE_PARSER, IniReader, SimpleKeyValueReader, ValueParsers + + +def test_lower_case_parser() -> None: + assert LOWER_CASE_PARSER("Hello") == "hello" + + +class TestValueParsers: + def test_find_value_parsers(self): + def default(input: str) -> str: + return "default" + + def custom_exact_match(input: str) -> str: + return "custom-exact" + + def custom_any_section_match(input: str) -> str: + return "custom-any-section" + + parsers = ValueParsers( + default_parser=default, + parsers={ + OptionMatcher("section1", "option1"): custom_exact_match, + any_section_option_matcher("option2"): custom_any_section_match, + }, + ) + + assert parsers.find_parser("section1", "option1")("test") == "custom-exact" + assert parsers.find_parser("section1", "option2")("test") == "custom-any-section" + assert parsers.find_parser("section1", "option3")("test") == "default" class TestIniReader: @@ -339,14 +367,17 @@ def test_read__with_custom_parser(self, tmp_path): ) ) - value_parsers = {OptionMatcher("part2", "bar"): LOWER_CASE_PARSER} + def double_parser(value: str) -> str: + return value + value + + value_parsers = {OptionMatcher("part2", "bar"): double_parser} actual = IniReader(value_parsers=value_parsers).read(path) - expected = {"part1": {"bar": "Hello"}, "part2": {"bar": "hello"}} + expected = {"part1": {"bar": "Hello"}, "part2": {"bar": "HelloHello"}} assert actual == expected - value_parsers = {any_section_option_matcher("bar"): LOWER_CASE_PARSER} + value_parsers = {any_section_option_matcher("bar"): double_parser} actual = IniReader(value_parsers=value_parsers).read(path) - expected = {"part1": {"bar": "hello"}, "part2": {"bar": "hello"}} + expected = {"part1": {"bar": "HelloHello"}, "part2": {"bar": "HelloHello"}} assert actual == expected diff --git a/tests/core/serde/test_ini_writer.py b/tests/core/serde/test_ini_writer.py index 06587c0fe8..e634f5ce10 100644 --- a/tests/core/serde/test_ini_writer.py +++ b/tests/core/serde/test_ini_writer.py @@ -19,6 +19,11 @@ from antarest.core.serde.ini_writer import LOWER_CASE_SERIALIZER, IniWriter +@pytest.mark.unit_test +def test_lower_case_serializer() -> None: + assert LOWER_CASE_SERIALIZER("Hello") == "hello" + + @pytest.mark.unit_test def test_write(tmp_path: str, ini_cleaner: Callable) -> None: path = Path(tmp_path) / "test.ini" @@ -66,15 +71,18 @@ def test_write(tmp_path: str, ini_cleaner: Callable) -> None: def test_write_with_custom_serializer(tmp_path: str, ini_cleaner: Callable) -> None: path = Path(tmp_path) / "test.ini" - serializers = {any_section_option_matcher("group"): LOWER_CASE_SERIALIZER} + def duplicate(value: str) -> str: + return value * 2 + + serializers = {any_section_option_matcher("group"): duplicate} writer = IniWriter(value_serializers=serializers) expected = """ [part1] - group = gas + group = GasGas [part2] - group = gas + group = GasGas [part3] other = Gas diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 297a367d08..59654ed35a 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -28,24 +28,6 @@ from antarest.core.tasks.service import ITaskService, NoopNotifier, Task -@pytest.fixture -def ini_cleaner() -> Callable[[str], str]: - def cleaner(txt: str) -> str: - lines = filter(None, map(str.strip, txt.splitlines(keepends=False))) - return "\n".join(lines) - - return cleaner - - -@pytest.fixture -def clean_ini_writer(ini_cleaner: Callable[[str], str]) -> Callable[[Path, str], None]: - def write_clean_ini(path: Path, txt: str) -> None: - clean_ini = ini_cleaner(txt) - path.write_text(clean_ini) - - return write_clean_ini - - @pytest.fixture def test_json_data() -> JSON: return { From a7e388446d02e32c696c355b29d24a4804ae2ddc Mon Sep 17 00:00:00 2001 From: Sylvain Leclerc Date: Fri, 7 Feb 2025 11:46:34 +0100 Subject: [PATCH 7/7] fix(copyright): add missing copyright Signed-off-by: Sylvain Leclerc --- antarest/core/serde/ini_common.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/antarest/core/serde/ini_common.py b/antarest/core/serde/ini_common.py index b957b277b8..4f41651434 100644 --- a/antarest/core/serde/ini_common.py +++ b/antarest/core/serde/ini_common.py @@ -1,3 +1,15 @@ +# Copyright (c) 2025, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. + import dataclasses from typing import Optional