diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d3d662b..d8b9599 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -272,7 +272,7 @@ jobs: - name: Run SonarCloud Scan uses: SonarSource/sonarcloud-github-action@master env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} e2e_test: @@ -479,4 +479,5 @@ jobs: env: EE_TOKEN: ${{ secrets.EE_TOKEN }} shell: bash + \ No newline at end of file diff --git a/tests_ce/data_mimic_test.py b/datamimic_ce/data_mimic_test.py similarity index 100% rename from tests_ce/data_mimic_test.py rename to datamimic_ce/data_mimic_test.py diff --git a/datamimic_ce/exporters/csv_exporter.py b/datamimic_ce/exporters/csv_exporter.py index 2cf3255..e412ceb 100644 --- a/datamimic_ce/exporters/csv_exporter.py +++ b/datamimic_ce/exporters/csv_exporter.py @@ -36,7 +36,6 @@ def __init__( self._task_id = setup_context.task_id # Retrieve encoding and delimiter from setup_context or use defaults - kwargs["encoding"] = encoding or setup_context.default_encoding or "utf-8" self.delimiter = delimiter or setup_context.default_separator or "," self.quotechar = quotechar or '"' self.quoting = quoting or csv.QUOTE_MINIMAL @@ -56,7 +55,7 @@ def __init__( **kwargs, ) logger.info( - f"CSVEEExporter initialized with chunk size {chunk_size}, fieldnames '{fieldnames}', " + f"CSVExporter initialized with chunk size {chunk_size}, fieldnames '{fieldnames}', " f"encoding '{self._encoding}', delimiter '{self.delimiter}'" ) @@ -102,24 +101,3 @@ def _reset_state(self): """Resets the exporter state for reuse.""" super()._reset_state() logger.debug("CSVEEExporter state has been reset.") - - # def __init__(self, setup_context: SetupContext, product_name: str): - # super().__init__("csv", setup_context, product_name) - # self._separator = setup_context.default_separator - # self._line_separator = setup_context.default_line_separator or os.linesep - - # def consume(self, product: Tuple[str, List[Dict]]) -> None: - # name = product[0] - # data = product[1] - # - # file_name = f"{name}.csv" - # with open(self._exported_data_dir / file_name, "w") as file: - # if data: - # keys = data[0].keys() - # file.write(f"{self._separator}".join(keys)) - # file.write(self._line_separator) - # for row in data: - # file.write(f"{self._separator}".join([str(row[key]) for key in keys])) - # file.write(self._line_separator) - # - # logger.debug(f"Exported data to {self._exported_data_dir / file_name}") diff --git a/datamimic_ce/exporters/exporter_util.py b/datamimic_ce/exporters/exporter_util.py index 81ded9e..d989ba2 100644 --- a/datamimic_ce/exporters/exporter_util.py +++ b/datamimic_ce/exporters/exporter_util.py @@ -71,6 +71,8 @@ def create_exporter_list( :param setup_context: :param stmt: + :param targets: + :param mp_idx: :return: """ consumers_with_operation = [] diff --git a/datamimic_ce/exporters/json_exporter.py b/datamimic_ce/exporters/json_exporter.py index 376213b..642adff 100644 --- a/datamimic_ce/exporters/json_exporter.py +++ b/datamimic_ce/exporters/json_exporter.py @@ -34,6 +34,7 @@ def __init__( mp_id: int | None, use_ndjson: bool = False, chunk_size: int = None, + encoding: str = None, ): self.use_ndjson = use_ndjson self._task_id = setup_context.task_id diff --git a/datamimic_ce/exporters/txt_exporter.py b/datamimic_ce/exporters/txt_exporter.py index 5b75927..53fa996 100644 --- a/datamimic_ce/exporters/txt_exporter.py +++ b/datamimic_ce/exporters/txt_exporter.py @@ -59,7 +59,6 @@ def __init__( ) # Pass encoding via kwargs to the base class - kwargs["encoding"] = encoding or setup_context.default_encoding or "utf-8" super().__init__( "txt", diff --git a/datamimic_ce/exporters/unified_buffered_exporter.py b/datamimic_ce/exporters/unified_buffered_exporter.py index 676cb33..309c4e2 100644 --- a/datamimic_ce/exporters/unified_buffered_exporter.py +++ b/datamimic_ce/exporters/unified_buffered_exporter.py @@ -65,11 +65,7 @@ def __init__( self.chunk_size = chunk_size # Max entities per chunk # Prepare temporary buffer directory - # / f"exporter_result_{self._task_id}_pid_{self._pid}_exporter_{self._exporter_type}_product_{self.product_name}" - self._buffer_tmp_dir = ( - self._descriptor_dir - / f"exporter_result_{self._task_id}_exporter_{self._exporter_type}_product_{self.product_name}" - ) + self._buffer_tmp_dir = self._get_buffer_tmp_dir() self._buffer_tmp_dir.mkdir(parents=True, exist_ok=True) # Initialize state variables @@ -79,14 +75,19 @@ def __init__( # Handle any additional parameters from kwargs for key, value in kwargs.items(): setattr(self, key, value) - # self._buffer_file = self._get_buffer_file() # Initialize first buffer file + + @property + def encoding(self) -> str: + return self._encoding + + def _get_buffer_tmp_dir(self) -> Path: + return self._descriptor_dir / f"exporter_result_{self._task_id}_exporter_{self._exporter_type}_product_{self.product_name}" def _get_buffer_dir_by_pid(self) -> Path: """ Returns the buffer directory for the current PID. :return: """ - # / f"exporter_result_{self._task_id}_pid_{self._pid}_exporter_{self._exporter_type}_product_{self.product_name}" return ( self._descriptor_dir / f"exporter_result_{self._task_id}_exporter_{self._exporter_type}_product_{self.product_name}" @@ -168,7 +169,6 @@ def _rotate_chunk(self) -> None: self.chunk_index += 1 self.current_counter = 0 # Reset the current count for new chunk self._save_state() - # self._buffer_file = self._get_buffer_file() # Create new buffer file for the next chunk self._is_first_write = True # Reset for the new chunk def store_data(self, data: list[dict]) -> None: @@ -282,40 +282,6 @@ def consume(self, product: tuple): logger.debug(f"Storing data for '{self.product_name}' with {len(data)} records") self.store_data(data) - # def upload_to_storage(self, bucket: str, name: str) -> None: - # """Uploads all buffered chunk files to the object storage in the specified format.""" - # buffer_dir = self._get_buffer_dir_by_pid() - # - # # logger.debug(f"Found {len(buffer_dirs)} buffer directories for export.") - # - # # Define file suffix - # suffix = self._define_suffix() - # - # buffer_files = sorted( - # buffer_dir.glob(f"*.{suffix}"), - # key=lambda x: int(re.search(r"_chunk_(\d+)", x.name).group(1)), - # ) - # logger.debug(f"Found {len(buffer_files)} buffer files in directory {buffer_dir}.") - # - # for buffer_file in buffer_files: - # # Corresponding metadata file - # metadata = self._load_metadata(buffer_file.with_suffix(".meta")) - # - # total_count, uri = self._craft_uri(metadata, suffix) - # - # with buffer_file.open("r", encoding=self._encoding) as f: - # data_content = f.read() - # data_buffer = BytesIO(data_content.encode(self._encoding)) - # content_type = self._get_content_type() - # # Perform the upload - # self._storage.write(bucket or self._default_bucket, uri, data_buffer, content_type) - # logger.debug(f"Uploaded {uri} with {total_count} records") - # - # logger.info(f"Exported buffer files successfully.") - # - # # Cleanup buffer directories - # self._reset_state() - @abstractmethod def _define_suffix(self) -> str: """Defines the file suffix based on the format.""" @@ -342,20 +308,12 @@ def _craft_uri(self, metadata, suffix): ) # Determine URI based on chunk size and multiprocessing - if self._mp: - if chunk_size is None: - uri = f"{product_name}_pid_{pid}.{suffix}" - elif chunk_size == 1: - uri = f"{product_name}_{chunk_start}_pid_{pid}.{suffix}" - else: - uri = f"{product_name}_{chunk_start}_{chunk_end}_pid_{pid}.{suffix}" + if chunk_size is None: + uri = f"{product_name}{self._pid_placeholder}.{suffix}" + elif chunk_size == 1: + uri = f"{product_name}_{chunk_start}{self._pid_placeholder}.{suffix}" else: - if chunk_size is None: - uri = f"{product_name}.{suffix}" - elif chunk_size == 1: - uri = f"{product_name}_{chunk_start}.{suffix}" - else: - uri = f"{product_name}_{chunk_start}_{chunk_end}.{suffix}" + uri = f"{product_name}_{chunk_start}_{chunk_end}{self._pid_placeholder}.{suffix}" return total_count, f"{self._task_id}/{uri}" @@ -363,23 +321,15 @@ def cleanup(self) -> None: """Cleans up the temporary buffer directories.""" # Identify all buffer directories for this task - # buffer_dir = self._get_buffer_dir_by_pid() - # - # if buffer_dir.exists(): - # # Delete all files in the buffer directory - # for file in buffer_dir.iterdir(): - # if file.is_file() and file.suffix == ".meta": - # file.unlink() - buffer_metadata_file = self._get_buffer_file().with_suffix(".meta") buffer_metadata_file.unlink() self._get_state_meta_file().unlink() - # pass - # # Remove the buffer directory - # try: - # buffer_dir.rmdir() - # except OSError as e: - # logger.error(f"Failed to remove buffer directory {buffer_dir}: {e}") + # pass + # # Remove the buffer directory + # try: + # buffer_dir.rmdir() + # except OSError as e: + # logger.error(f"Failed to remove buffer directory {buffer_dir}: {e}") logger.debug("Buffer directory cleanup complete.") def _reset_state(self): diff --git a/datamimic_ce/exporters/xml_exporter.py b/datamimic_ce/exporters/xml_exporter.py index 392cc69..feaa7aa 100644 --- a/datamimic_ce/exporters/xml_exporter.py +++ b/datamimic_ce/exporters/xml_exporter.py @@ -52,12 +52,13 @@ def __init__( self.item_element = item_element # Pass encoding via kwargs to the base class - kwargs["encoding"] = encoding or setup_context.default_encoding or "utf-8" + # kwargs["encoding"] = encoding or setup_context.default_encoding or "utf-8" super().__init__( exporter_type="xml", setup_context=setup_context, product_name=product_name, + encoding=encoding, chunk_size=chunk_size, mp_idx=mp_id, **kwargs, diff --git a/datamimic_ce/tasks/generate_task.py b/datamimic_ce/tasks/generate_task.py index 290c1b9..045f21d 100644 --- a/datamimic_ce/tasks/generate_task.py +++ b/datamimic_ce/tasks/generate_task.py @@ -959,6 +959,10 @@ def execute(self, context: SetupContext) -> dict[str, list] | None: del result # gc.collect() + # Clean temp directory on outermost gen_stmt + for temp_dir in context.descriptor_dir.glob(f"temp_result_{context.task_id}*"): + shutil.rmtree(temp_dir) + # Just return product generated in single process if gen_stmt is inner one else: # Do not apply process by page for inner gen_stmt diff --git a/tests_ce/functional_tests/csv_separatorr/test_separator.py b/tests_ce/functional_tests/csv_separatorr/test_separator.py index f5fecff..ec17e70 100644 --- a/tests_ce/functional_tests/csv_separatorr/test_separator.py +++ b/tests_ce/functional_tests/csv_separatorr/test_separator.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataIteration: diff --git a/tests_ce/functional_tests/data_source_cyclic/test_data_source_cyclic_functional.py b/tests_ce/functional_tests/data_source_cyclic/test_data_source_cyclic_functional.py index 908f671..d0c70ca 100644 --- a/tests_ce/functional_tests/data_source_cyclic/test_data_source_cyclic_functional.py +++ b/tests_ce/functional_tests/data_source_cyclic/test_data_source_cyclic_functional.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataSourceCyclic: diff --git a/tests_ce/functional_tests/test_casting_type/test_data_type_functional.py b/tests_ce/functional_tests/test_casting_type/test_data_type_functional.py index d9b5429..9549a65 100644 --- a/tests_ce/functional_tests/test_casting_type/test_data_type_functional.py +++ b/tests_ce/functional_tests/test_casting_type/test_data_type_functional.py @@ -9,7 +9,7 @@ from pathlib import Path from types import NoneType -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataType: diff --git a/tests_ce/functional_tests/test_condition/test_conditon.py b/tests_ce/functional_tests/test_condition/test_conditon.py index c420644..e203ba8 100644 --- a/tests_ce/functional_tests/test_condition/test_conditon.py +++ b/tests_ce/functional_tests/test_condition/test_conditon.py @@ -11,7 +11,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestCondition(TestCase): diff --git a/tests_ce/functional_tests/test_condition_element/test_conditon_element.py b/tests_ce/functional_tests/test_condition_element/test_conditon_element.py index d194107..5679b80 100644 --- a/tests_ce/functional_tests/test_condition_element/test_conditon_element.py +++ b/tests_ce/functional_tests/test_condition_element/test_conditon_element.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestConditionElement: diff --git a/tests_ce/functional_tests/test_datetime/test_datetime.py b/tests_ce/functional_tests/test_datetime/test_datetime.py index 20e1dea..63a4b6a 100644 --- a/tests_ce/functional_tests/test_datetime/test_datetime.py +++ b/tests_ce/functional_tests/test_datetime/test_datetime.py @@ -11,7 +11,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDateTime: diff --git a/tests_ce/functional_tests/test_functional_array/test_array_functional.py b/tests_ce/functional_tests/test_functional_array/test_array_functional.py index 24d9b73..affbd29 100644 --- a/tests_ce/functional_tests/test_functional_array/test_array_functional.py +++ b/tests_ce/functional_tests/test_functional_array/test_array_functional.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestArrayFunctional: diff --git a/tests_ce/functional_tests/test_functional_page_process/test_functional_page_process.py b/tests_ce/functional_tests/test_functional_page_process/test_functional_page_process.py index e933186..3abd705 100644 --- a/tests_ce/functional_tests/test_functional_page_process/test_functional_page_process.py +++ b/tests_ce/functional_tests/test_functional_page_process/test_functional_page_process.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestFunctionalPageProcess: diff --git a/tests_ce/functional_tests/test_functional_variable/test_variable_functional.py b/tests_ce/functional_tests/test_functional_variable/test_variable_functional.py index 6179302..90bc84b 100644 --- a/tests_ce/functional_tests/test_functional_variable/test_variable_functional.py +++ b/tests_ce/functional_tests/test_functional_variable/test_variable_functional.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariableFunctional: diff --git a/tests_ce/functional_tests/test_functional_weighted_data_source/test_functional_weighted_data_source.py b/tests_ce/functional_tests/test_functional_weighted_data_source/test_functional_weighted_data_source.py index 521df9f..1201ef2 100644 --- a/tests_ce/functional_tests/test_functional_weighted_data_source/test_functional_weighted_data_source.py +++ b/tests_ce/functional_tests/test_functional_weighted_data_source/test_functional_weighted_data_source.py @@ -9,7 +9,7 @@ import math from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestWeightedDataSourceFunctional: diff --git a/tests_ce/functional_tests/test_generate_distribution/test_generate_distribution.py b/tests_ce/functional_tests/test_generate_distribution/test_generate_distribution.py index 482c30c..5f1a4e5 100644 --- a/tests_ce/functional_tests/test_generate_distribution/test_generate_distribution.py +++ b/tests_ce/functional_tests/test_generate_distribution/test_generate_distribution.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestGenerateDistribution: diff --git a/tests_ce/functional_tests/test_generator_functional/test_generator_functional.py b/tests_ce/functional_tests/test_generator_functional/test_generator_functional.py index a51ae8b..860bb97 100644 --- a/tests_ce/functional_tests/test_generator_functional/test_generator_functional.py +++ b/tests_ce/functional_tests/test_generator_functional/test_generator_functional.py @@ -11,7 +11,7 @@ from datamimic_ce.generators.generator_util import GeneratorUtil from datamimic_ce.utils.file_util import FileUtil -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest def count_digits_after_decimal(number): diff --git a/tests_ce/functional_tests/test_include/test_include.py b/tests_ce/functional_tests/test_include/test_include.py index e51ac3c..9ff503b 100644 --- a/tests_ce/functional_tests/test_include/test_include.py +++ b/tests_ce/functional_tests/test_include/test_include.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestInclude: diff --git a/tests_ce/functional_tests/test_memstore/test_memstore.py b/tests_ce/functional_tests/test_memstore/test_memstore.py index 4f0d850..35fd578 100644 --- a/tests_ce/functional_tests/test_memstore/test_memstore.py +++ b/tests_ce/functional_tests/test_memstore/test_memstore.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestMemStore: diff --git a/tests_ce/functional_tests/test_mongodb/test_mongodb_functional.py b/tests_ce/functional_tests/test_mongodb/test_mongodb_functional.py index e24e7d2..5435f88 100644 --- a/tests_ce/functional_tests/test_mongodb/test_mongodb_functional.py +++ b/tests_ce/functional_tests/test_mongodb/test_mongodb_functional.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestMongoDbFunction: diff --git a/tests_ce/functional_tests/test_nested_generate/test_nested_generate.py b/tests_ce/functional_tests/test_nested_generate/test_nested_generate.py index a48db7f..cd8b66a 100644 --- a/tests_ce/functional_tests/test_nested_generate/test_nested_generate.py +++ b/tests_ce/functional_tests/test_nested_generate/test_nested_generate.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestNestedGenerate: diff --git a/tests_ce/functional_tests/test_nested_key_distribution/test_nested_key_distribution.py b/tests_ce/functional_tests/test_nested_key_distribution/test_nested_key_distribution.py index 9344e45..c3532e0 100644 --- a/tests_ce/functional_tests/test_nested_key_distribution/test_nested_key_distribution.py +++ b/tests_ce/functional_tests/test_nested_key_distribution/test_nested_key_distribution.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariableDistribution: diff --git a/tests_ce/functional_tests/test_nested_part/test_nested_part.py b/tests_ce/functional_tests/test_nested_part/test_nested_part.py index 631ac3c..769826c 100644 --- a/tests_ce/functional_tests/test_nested_part/test_nested_part.py +++ b/tests_ce/functional_tests/test_nested_part/test_nested_part.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestNestedPart: diff --git a/tests_ce/functional_tests/test_prefix_suffix/test_prefix_suffix.py b/tests_ce/functional_tests/test_prefix_suffix/test_prefix_suffix.py index 920071f..49b462d 100644 --- a/tests_ce/functional_tests/test_prefix_suffix/test_prefix_suffix.py +++ b/tests_ce/functional_tests/test_prefix_suffix/test_prefix_suffix.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestPrefixSuffix: diff --git a/tests_ce/functional_tests/test_rdbms_functional/test_rdbms_functional.py b/tests_ce/functional_tests/test_rdbms_functional/test_rdbms_functional.py index 58322d5..875ffb4 100644 --- a/tests_ce/functional_tests/test_rdbms_functional/test_rdbms_functional.py +++ b/tests_ce/functional_tests/test_rdbms_functional/test_rdbms_functional.py @@ -11,7 +11,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestRdbmsFunctional: diff --git a/tests_ce/functional_tests/test_selector/test_selector.py b/tests_ce/functional_tests/test_selector/test_selector.py index 41c512b..1feccdb 100644 --- a/tests_ce/functional_tests/test_selector/test_selector.py +++ b/tests_ce/functional_tests/test_selector/test_selector.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestSelector: diff --git a/tests_ce/functional_tests/test_sequence_table_generator/test_sequence_table_generator.py b/tests_ce/functional_tests/test_sequence_table_generator/test_sequence_table_generator.py index a6eebec..edc1481 100644 --- a/tests_ce/functional_tests/test_sequence_table_generator/test_sequence_table_generator.py +++ b/tests_ce/functional_tests/test_sequence_table_generator/test_sequence_table_generator.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestSequenceTableGenerator: diff --git a/tests_ce/functional_tests/test_source_script/test_source_scripted_functional.py b/tests_ce/functional_tests/test_source_script/test_source_scripted_functional.py index 14dce04..cbf4619 100644 --- a/tests_ce/functional_tests/test_source_script/test_source_scripted_functional.py +++ b/tests_ce/functional_tests/test_source_script/test_source_scripted_functional.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestSourceScripted: diff --git a/tests_ce/functional_tests/test_sqlite/test_sqlite.py b/tests_ce/functional_tests/test_sqlite/test_sqlite.py index 84c0abb..b4f4b85 100644 --- a/tests_ce/functional_tests/test_sqlite/test_sqlite.py +++ b/tests_ce/functional_tests/test_sqlite/test_sqlite.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestSqlite: diff --git a/tests_ce/functional_tests/test_string_in_key_variable_node/test_string_in_key_variable_node.py b/tests_ce/functional_tests/test_string_in_key_variable_node/test_string_in_key_variable_node.py index 0a38f1b..92c45b3 100644 --- a/tests_ce/functional_tests/test_string_in_key_variable_node/test_string_in_key_variable_node.py +++ b/tests_ce/functional_tests/test_string_in_key_variable_node/test_string_in_key_variable_node.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestStringInKeyVariableNode: diff --git a/tests_ce/functional_tests/test_variable_distribution/test_variable_distribution.py b/tests_ce/functional_tests/test_variable_distribution/test_variable_distribution.py index 4fa5440..5c89fc7 100644 --- a/tests_ce/functional_tests/test_variable_distribution/test_variable_distribution.py +++ b/tests_ce/functional_tests/test_variable_distribution/test_variable_distribution.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariableDistribution: diff --git a/tests_ce/integration_tests/consumer_csv/test_csv_consumer.py b/tests_ce/integration_tests/consumer_csv/test_csv_consumer.py index b665ffd..3e27a3a 100644 --- a/tests_ce/integration_tests/consumer_csv/test_csv_consumer.py +++ b/tests_ce/integration_tests/consumer_csv/test_csv_consumer.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestConsumerCsvMinio: diff --git a/tests_ce/integration_tests/consumer_json_minio/test_consumer_json_minio.py b/tests_ce/integration_tests/consumer_json_minio/test_consumer_json_minio.py index dac0393..fb899c3 100644 --- a/tests_ce/integration_tests/consumer_json_minio/test_consumer_json_minio.py +++ b/tests_ce/integration_tests/consumer_json_minio/test_consumer_json_minio.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestConsumerJsonMinio: diff --git a/tests_ce/integration_tests/consumer_log_consumer/test_log_consumer.py b/tests_ce/integration_tests/consumer_log_consumer/test_log_consumer.py index 0f86a1c..a47a26e 100644 --- a/tests_ce/integration_tests/consumer_log_consumer/test_log_consumer.py +++ b/tests_ce/integration_tests/consumer_log_consumer/test_log_consumer.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestLogConsumer: diff --git a/tests_ce/integration_tests/consumer_txt/test_txt_consumer.py b/tests_ce/integration_tests/consumer_txt/test_txt_consumer.py index 9b98ccc..a12c645 100644 --- a/tests_ce/integration_tests/consumer_txt/test_txt_consumer.py +++ b/tests_ce/integration_tests/consumer_txt/test_txt_consumer.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestConsumerTxtMinio: diff --git a/tests_ce/integration_tests/data_iteration/test_data_iteration.py b/tests_ce/integration_tests/data_iteration/test_data_iteration.py index ceacb5f..0d94a2b 100644 --- a/tests_ce/integration_tests/data_iteration/test_data_iteration.py +++ b/tests_ce/integration_tests/data_iteration/test_data_iteration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataIteration: diff --git a/tests_ce/integration_tests/data_migration/test_data_migration.py b/tests_ce/integration_tests/data_migration/test_data_migration.py index 66cb4c9..84fc742 100644 --- a/tests_ce/integration_tests/data_migration/test_data_migration.py +++ b/tests_ce/integration_tests/data_migration/test_data_migration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataMigration: diff --git a/tests_ce/integration_tests/data_source_cyclic/test_data_source_cyclic_intergration.py b/tests_ce/integration_tests/data_source_cyclic/test_data_source_cyclic_intergration.py index ebe6862..6e10b03 100644 --- a/tests_ce/integration_tests/data_source_cyclic/test_data_source_cyclic_intergration.py +++ b/tests_ce/integration_tests/data_source_cyclic/test_data_source_cyclic_intergration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataSourceCyclic: diff --git a/tests_ce/integration_tests/data_type/test_data_type_intergration.py b/tests_ce/integration_tests/data_type/test_data_type_intergration.py index 1a5738c..d747864 100644 --- a/tests_ce/integration_tests/data_type/test_data_type_intergration.py +++ b/tests_ce/integration_tests/data_type/test_data_type_intergration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDataType: diff --git a/tests_ce/integration_tests/datamimic_demo/test_datamimic_demo.py b/tests_ce/integration_tests/datamimic_demo/test_datamimic_demo.py index c205f1a..e7e603e 100644 --- a/tests_ce/integration_tests/datamimic_demo/test_datamimic_demo.py +++ b/tests_ce/integration_tests/datamimic_demo/test_datamimic_demo.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDatamimicDemo: diff --git a/tests_ce/integration_tests/element_properties/test_element_properties.py b/tests_ce/integration_tests/element_properties/test_element_properties.py index 7982e7f..41ccd42 100644 --- a/tests_ce/integration_tests/element_properties/test_element_properties.py +++ b/tests_ce/integration_tests/element_properties/test_element_properties.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestElementProperties: diff --git a/tests_ce/integration_tests/extensive_script/test_extensive_script.py b/tests_ce/integration_tests/extensive_script/test_extensive_script.py index 5bb4cf2..88828fe 100644 --- a/tests_ce/integration_tests/extensive_script/test_extensive_script.py +++ b/tests_ce/integration_tests/extensive_script/test_extensive_script.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestExtensiveScript: diff --git a/tests_ce/integration_tests/generator_test/test_datamimic_generator.py b/tests_ce/integration_tests/generator_test/test_datamimic_generator.py index a253207..0a29cbf 100644 --- a/tests_ce/integration_tests/generator_test/test_datamimic_generator.py +++ b/tests_ce/integration_tests/generator_test/test_datamimic_generator.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestDatamimicGenerator: diff --git a/tests_ce/integration_tests/script_source/test_script_source.py b/tests_ce/integration_tests/script_source/test_script_source.py index 420d617..43b26d6 100644 --- a/tests_ce/integration_tests/script_source/test_script_source.py +++ b/tests_ce/integration_tests/script_source/test_script_source.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestScriptSource: diff --git a/tests_ce/integration_tests/test_array/test_array.py b/tests_ce/integration_tests/test_array/test_array.py index 203b687..3b461b9 100644 --- a/tests_ce/integration_tests/test_array/test_array.py +++ b/tests_ce/integration_tests/test_array/test_array.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestArray: diff --git a/tests_ce/integration_tests/test_echo/test_echo.py b/tests_ce/integration_tests/test_echo/test_echo.py index 29707a6..bbacc7d 100644 --- a/tests_ce/integration_tests/test_echo/test_echo.py +++ b/tests_ce/integration_tests/test_echo/test_echo.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestEcho: diff --git a/tests_ce/integration_tests/test_entity/test_entity.py b/tests_ce/integration_tests/test_entity/test_entity.py index e6bf822..c224b8e 100644 --- a/tests_ce/integration_tests/test_entity/test_entity.py +++ b/tests_ce/integration_tests/test_entity/test_entity.py @@ -9,7 +9,7 @@ from pathlib import Path from datamimic_ce.utils.file_util import FileUtil -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestEntity: diff --git a/tests_ce/integration_tests/test_env/test_env.py b/tests_ce/integration_tests/test_env/test_env.py index 22157ad..1e8b68a 100644 --- a/tests_ce/integration_tests/test_env/test_env.py +++ b/tests_ce/integration_tests/test_env/test_env.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestEnv: diff --git a/tests_ce/integration_tests/test_execution/test_execution.py b/tests_ce/integration_tests/test_execution/test_execution.py index e59a51e..af09f1c 100644 --- a/tests_ce/integration_tests/test_execution/test_execution.py +++ b/tests_ce/integration_tests/test_execution/test_execution.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestExecution: diff --git a/tests_ce/integration_tests/test_exporters/test_exporters.py b/tests_ce/integration_tests/test_exporters/test_exporters.py index 201503d..c3926a0 100644 --- a/tests_ce/integration_tests/test_exporters/test_exporters.py +++ b/tests_ce/integration_tests/test_exporters/test_exporters.py @@ -7,7 +7,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestExporter: diff --git a/tests_ce/integration_tests/test_generate/test_generate.py b/tests_ce/integration_tests/test_generate/test_generate.py index 4d2e9f4..07b0e02 100644 --- a/tests_ce/integration_tests/test_generate/test_generate.py +++ b/tests_ce/integration_tests/test_generate/test_generate.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestGenerate: diff --git a/tests_ce/integration_tests/test_if/test_if.py b/tests_ce/integration_tests/test_if/test_if.py index e4d28db..377ef8b 100644 --- a/tests_ce/integration_tests/test_if/test_if.py +++ b/tests_ce/integration_tests/test_if/test_if.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestIf: diff --git a/tests_ce/integration_tests/test_iterate/test_iterate.py b/tests_ce/integration_tests/test_iterate/test_iterate.py index 7a32e2b..d8570ea 100644 --- a/tests_ce/integration_tests/test_iterate/test_iterate.py +++ b/tests_ce/integration_tests/test_iterate/test_iterate.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestIterate: diff --git a/tests_ce/integration_tests/test_large_count/test_large_count.py b/tests_ce/integration_tests/test_large_count/test_large_count.py index cb37e46..1ae3906 100644 --- a/tests_ce/integration_tests/test_large_count/test_large_count.py +++ b/tests_ce/integration_tests/test_large_count/test_large_count.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestLargeCount: diff --git a/tests_ce/integration_tests/test_lazy_parse/test_lazy_parse.py b/tests_ce/integration_tests/test_lazy_parse/test_lazy_parse.py index 22afb37..713e146 100644 --- a/tests_ce/integration_tests/test_lazy_parse/test_lazy_parse.py +++ b/tests_ce/integration_tests/test_lazy_parse/test_lazy_parse.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestLazyParse: diff --git a/tests_ce/integration_tests/test_line_separator/test_line_separator.py b/tests_ce/integration_tests/test_line_separator/test_line_separator.py index 68cd0b7..e72e275 100644 --- a/tests_ce/integration_tests/test_line_separator/test_line_separator.py +++ b/tests_ce/integration_tests/test_line_separator/test_line_separator.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestLineSeparator: diff --git a/tests_ce/integration_tests/test_list/test_list.py b/tests_ce/integration_tests/test_list/test_list.py index 0110f52..fe786a3 100644 --- a/tests_ce/integration_tests/test_list/test_list.py +++ b/tests_ce/integration_tests/test_list/test_list.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestList: diff --git a/tests_ce/integration_tests/test_memstore/test_memstore_integration.py b/tests_ce/integration_tests/test_memstore/test_memstore_integration.py index 74788b2..44cbdd9 100644 --- a/tests_ce/integration_tests/test_memstore/test_memstore_integration.py +++ b/tests_ce/integration_tests/test_memstore/test_memstore_integration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestMemStoreIntegration: diff --git a/tests_ce/integration_tests/test_mongodb/test_mongodb_intergration.py b/tests_ce/integration_tests/test_mongodb/test_mongodb_intergration.py index 1869ac4..4e6d71b 100644 --- a/tests_ce/integration_tests/test_mongodb/test_mongodb_intergration.py +++ b/tests_ce/integration_tests/test_mongodb/test_mongodb_intergration.py @@ -11,7 +11,7 @@ import pytest from datamimic_ce.config import settings -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestMongoDB: diff --git a/tests_ce/integration_tests/test_mp_consumer/test_mp_consumer.py b/tests_ce/integration_tests/test_mp_consumer/test_mp_consumer.py index e1c9f03..47dc02a 100644 --- a/tests_ce/integration_tests/test_mp_consumer/test_mp_consumer.py +++ b/tests_ce/integration_tests/test_mp_consumer/test_mp_consumer.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestMpConsumer: diff --git a/tests_ce/integration_tests/test_null_quota/test_null_quota.py b/tests_ce/integration_tests/test_null_quota/test_null_quota.py index d949bc8..96ad6d2 100644 --- a/tests_ce/integration_tests/test_null_quota/test_null_quota.py +++ b/tests_ce/integration_tests/test_null_quota/test_null_quota.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestNullQuota: diff --git a/tests_ce/integration_tests/test_page_process/test_page_process.py b/tests_ce/integration_tests/test_page_process/test_page_process.py index 8daec69..042874d 100644 --- a/tests_ce/integration_tests/test_page_process/test_page_process.py +++ b/tests_ce/integration_tests/test_page_process/test_page_process.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestPageProcess: diff --git a/tests_ce/integration_tests/test_part/test_part.py b/tests_ce/integration_tests/test_part/test_part.py index dc761eb..1ea741b 100644 --- a/tests_ce/integration_tests/test_part/test_part.py +++ b/tests_ce/integration_tests/test_part/test_part.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestPart: diff --git a/tests_ce/integration_tests/test_pattern/test_pattern.py b/tests_ce/integration_tests/test_pattern/test_pattern.py index 2234de1..e88146e 100644 --- a/tests_ce/integration_tests/test_pattern/test_pattern.py +++ b/tests_ce/integration_tests/test_pattern/test_pattern.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestPattern: diff --git a/tests_ce/integration_tests/test_primitive_generator/test_primitive_generator.py b/tests_ce/integration_tests/test_primitive_generator/test_primitive_generator.py index 085c974..ecf1bdf 100644 --- a/tests_ce/integration_tests/test_primitive_generator/test_primitive_generator.py +++ b/tests_ce/integration_tests/test_primitive_generator/test_primitive_generator.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestPrimitiveGenerator: diff --git a/tests_ce/integration_tests/test_rdbms/test_rdbms.py b/tests_ce/integration_tests/test_rdbms/test_rdbms.py index 554d04f..54b1b53 100644 --- a/tests_ce/integration_tests/test_rdbms/test_rdbms.py +++ b/tests_ce/integration_tests/test_rdbms/test_rdbms.py @@ -11,7 +11,7 @@ import pytest from datamimic_ce.config import settings -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestRdbms: diff --git a/tests_ce/integration_tests/test_source_script/test_source_scripted_intergration.py b/tests_ce/integration_tests/test_source_script/test_source_scripted_intergration.py index b5b51a7..ec3910e 100644 --- a/tests_ce/integration_tests/test_source_script/test_source_scripted_intergration.py +++ b/tests_ce/integration_tests/test_source_script/test_source_scripted_intergration.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestSourceScripted: diff --git a/tests_ce/integration_tests/test_variable/test_variable.py b/tests_ce/integration_tests/test_variable/test_variable.py index eea0f41..3b45c76 100644 --- a/tests_ce/integration_tests/test_variable/test_variable.py +++ b/tests_ce/integration_tests/test_variable/test_variable.py @@ -10,7 +10,7 @@ import pytest -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariable: diff --git a/tests_ce/integration_tests/test_xml_template/test_xml_template.py b/tests_ce/integration_tests/test_xml_template/test_xml_template.py index f2b169d..9518611 100644 --- a/tests_ce/integration_tests/test_xml_template/test_xml_template.py +++ b/tests_ce/integration_tests/test_xml_template/test_xml_template.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestXMLTemplate: diff --git a/tests_ce/integration_tests/variable_entity/test_variable_entity.py b/tests_ce/integration_tests/variable_entity/test_variable_entity.py index 152a57b..0fe78de 100644 --- a/tests_ce/integration_tests/variable_entity/test_variable_entity.py +++ b/tests_ce/integration_tests/variable_entity/test_variable_entity.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariableEntity: diff --git a/tests_ce/integration_tests/variable_script/test_variable_script.py b/tests_ce/integration_tests/variable_script/test_variable_script.py index 1d6a86c..d1cf579 100644 --- a/tests_ce/integration_tests/variable_script/test_variable_script.py +++ b/tests_ce/integration_tests/variable_script/test_variable_script.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestVariableScript: diff --git a/tests_ce/integration_tests/weighted_data_source/test_weighted_data_source.py b/tests_ce/integration_tests/weighted_data_source/test_weighted_data_source.py index d810296..ba8560a 100644 --- a/tests_ce/integration_tests/weighted_data_source/test_weighted_data_source.py +++ b/tests_ce/integration_tests/weighted_data_source/test_weighted_data_source.py @@ -8,7 +8,7 @@ from pathlib import Path -from tests_ce.data_mimic_test import DataMimicTest +from datamimic_ce.data_mimic_test import DataMimicTest class TestWeightedDataSource: