From 445cd700a55aef57d74db444990620f75f47094f Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 9 May 2022 12:27:12 +0300 Subject: [PATCH 01/17] refactor: collect step artifacts immediately after step finish --- tests/test_code_report.py | 11 +++++++-- tests/test_integration.py | 7 +++--- universum/main.py | 1 - universum/modules/artifact_collector.py | 31 +++++++++++-------------- universum/modules/launcher.py | 9 +++++-- universum/nonci.py | 1 - 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 58424b45..0f194559 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -276,8 +276,15 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, expected_log = log_success if expected_success else log_fail assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" - expected_log = r"Collecting 'source_file.html' - [^\n]*Success" if expected_artifact \ - else r"Collecting 'source_file.html' - [^\n]*Failed" + + artifacts_absence_error = "No artifacts found!" + if expected_artifact: + assert artifacts_absence_error not in log, f"'{artifacts_absence_error} found in '{log}'" + else: + assert artifacts_absence_error in log, f"'{artifacts_absence_error} not found in '{log}'" + + expected_step_result = r"Success" if expected_success and expected_artifact else "Failed" + expected_log = f"Run uncrustify - [^\n]*{expected_step_result}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" diff --git a/tests/test_integration.py b/tests/test_integration.py index 6d555275..af952b99 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -39,13 +39,14 @@ def test_artifacts(docker_main: UniversumRunner): files2 = Configuration([dict(name=" one/three/file.sh", command=["one/three/file.sh"])]) artifacts = Configuration([dict(name="Existing artifacts", artifacts="one/**/file*", report_artifacts="one/*"), - dict(name="Missing artifacts", artifacts="something", report_artifacts="something_else")]) + dict(name="Missing report artifacts", report_artifacts="non_existing_file"), + dict(name="Missing all artifacts", artifacts="something", report_artifacts="something_else")]) configs = mkdir * dirs1 + mkdir * dirs2 + mkfile * files1 + mkfile * files2 + artifacts """ log = docker_main.run(config) - assert 'Failed' in get_line_with_text("Collecting 'something' - ", log) - assert 'Success' in get_line_with_text("Collecting 'something_else' for report - ", log) + assert 'Failed' in get_line_with_text("Missing all artifacts - ", log) + assert 'Success' in get_line_with_text("Missing report artifacts - ", log) assert os.path.exists(os.path.join(docker_main.artifact_dir, "three.zip")) assert os.path.exists(os.path.join(docker_main.artifact_dir, "two2.zip")) diff --git a/universum/main.py b/universum/main.py index 453c72d4..a83f522e 100644 --- a/universum/main.py +++ b/universum/main.py @@ -87,7 +87,6 @@ def execute(self) -> None: self.launcher.launch_custom_configs(afterall_configs) self.code_report_collector.repo_diff = repo_diff self.code_report_collector.report_code_report_results() - self.artifacts.collect_artifacts() self.reporter.report_build_result() def finalize(self) -> None: diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index 5244b7dc..b8ce9cb8 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -80,9 +80,6 @@ def __init__(self, *args, **kwargs): self.reporter = self.reporter_factory() self.automation_server = self.automation_server_factory() - self.artifact_list = [] - self.report_artifact_list = [] - # Needed because of wildcards self.collected_report_artifacts = set() @@ -176,13 +173,13 @@ def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existin if artifact_list: name = "Setting and preprocessing artifacts according to configs" - self.artifact_list = self.structure.run_in_block(self.preprocess_artifact_list, - name, True, artifact_list, ignore_existing_artifacts) + self.structure.run_in_block(self.preprocess_artifact_list, + name, True, artifact_list, ignore_existing_artifacts) if report_artifact_list: name = "Setting and preprocessing artifacts to be mentioned in report" - self.report_artifact_list = self.structure.run_in_block(self.preprocess_artifact_list, - name, True, report_artifact_list, - ignore_existing_artifacts) + self.structure.run_in_block(self.preprocess_artifact_list, + name, True, report_artifact_list, + ignore_existing_artifacts) def move_artifact(self, path, is_report=False): self.out.log("Processing '" + path + "'") @@ -220,16 +217,14 @@ def move_artifact(self, path, is_report=False): artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name) self.collected_report_artifacts.add(artifact_path) - @make_block("Collecting artifacts", pass_errors=False) - def collect_artifacts(self): - self.reporter.add_block_to_report(self.structure.get_current_block()) - for path in self.report_artifact_list: - name = "Collecting '" + os.path.basename(path) + "' for report" - self.structure.run_in_block(self.move_artifact, name, False, path, is_report=True) - self.reporter.report_artifacts(list(self.collected_report_artifacts)) - for path in self.artifact_list: - name = "Collecting '" + os.path.basename(path) + "'" - self.structure.run_in_block(self.move_artifact, name, False, path) + def collect_step_artifacts(self, step): + if step.artifacts: + path = utils.parse_path(step.artifacts, self.settings.project_root) + self.move_artifact(path, is_report=False) + if step.report_artifacts: + path = utils.parse_path(step.report_artifacts, self.settings.project_root) + self.move_artifact(path, is_report=True) + def clean_artifacts_silently(self): try: diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 22acfc7d..3c5ab441 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -168,7 +168,8 @@ def __init__(self, item: configuration_support.Step, log_file: Optional[TextIO], working_directory: str, additional_environment: Dict[str, str], - background: bool) -> None: + background: bool, + artifacts: artifact_collector.ArtifactCollector) -> None: super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out @@ -187,6 +188,8 @@ def __init__(self, item: configuration_support.Step, self._postponed_out: List[Tuple[Callable[[str], None], str]] = [] self._needs_finalization: bool = True + self.artifacts = artifacts + def prepare_command(self) -> bool: # FIXME: refactor if not self.configuration.command: self.out.log("No 'command' found. Nothing to execute") @@ -257,6 +260,7 @@ def finalize(self) -> None: if self._is_background: self._is_background = False self.out.log("Nothing was executed: this background step had no command") + self.artifacts.collect_step_artifacts(self.configuration) return try: text = "" @@ -285,6 +289,7 @@ def finalize(self) -> None: if self.file: self.file.close() self._is_background = False + self.artifacts.collect_step_artifacts(self.configuration) def _handle_postponed_out(self) -> None: for item in self._postponed_out: @@ -415,7 +420,7 @@ def fail_block(line: str = "") -> None: additional_environment = self.api_support.get_environment_settings() return RunningStep(item, self.out, fail_block, self.server.add_build_tag, - log_file, working_directory, additional_environment, item.background) + log_file, working_directory, additional_environment, item.background, self.artifacts) def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None: self.structure.execute_step_structure(custom_configs, self.create_process) diff --git a/universum/nonci.py b/universum/nonci.py index 4a6a6a8d..4140d2e4 100644 --- a/universum/nonci.py +++ b/universum/nonci.py @@ -26,7 +26,6 @@ def execute(self): self.launch_project() self.reporter.report_initialized = True self.reporter.report_build_result() - self.artifacts.collect_artifacts() def finalize(self): pass From 78cb7b6fe82dc2514fa396ac33c1a80810627704 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Wed, 18 May 2022 12:03:02 +0300 Subject: [PATCH 02/17] move artifact collection to a separate step --- tests/test_code_report.py | 10 ++-------- tests/test_integration.py | 4 ++-- universum/modules/launcher.py | 29 +++++++++++++++-------------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 0f194559..c216d0ba 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -277,14 +277,8 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, expected_log = log_success if expected_success else log_fail assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" - artifacts_absence_error = "No artifacts found!" - if expected_artifact: - assert artifacts_absence_error not in log, f"'{artifacts_absence_error} found in '{log}'" - else: - assert artifacts_absence_error in log, f"'{artifacts_absence_error} not found in '{log}'" - - expected_step_result = r"Success" if expected_success and expected_artifact else "Failed" - expected_log = f"Run uncrustify - [^\n]*{expected_step_result}" + expected_artifacts_state = "Success" if expected_artifact else "Failed" + exptected_log = f"Collecting artifacts for the 'Run uncrustify' step - {expected_artifacts_state}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" diff --git a/tests/test_integration.py b/tests/test_integration.py index af952b99..6c50446d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -45,8 +45,8 @@ def test_artifacts(docker_main: UniversumRunner): configs = mkdir * dirs1 + mkdir * dirs2 + mkfile * files1 + mkfile * files2 + artifacts """ log = docker_main.run(config) - assert 'Failed' in get_line_with_text("Missing all artifacts - ", log) - assert 'Success' in get_line_with_text("Missing report artifacts - ", log) + assert 'Failed' in get_line_with_text("Collecting artifacts for the 'Missing all artifacts' step - ", log) + assert 'Success' in get_line_with_text("Collecting artifacts for the 'Missing report artifacts' step - ", log) assert os.path.exists(os.path.join(docker_main.artifact_dir, "three.zip")) assert os.path.exists(os.path.join(docker_main.artifact_dir, "two2.zip")) diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 3c5ab441..230b72b1 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -15,7 +15,7 @@ from . import automation_server, api_support, artifact_collector, reporter, code_report_collector from .output import HasOutput, Output from .project_directory import ProjectDirectory -from .structure_handler import HasStructure +from .structure_handler import HasStructure, StructureHandler __all__ = [ "Launcher", @@ -163,7 +163,7 @@ class RunningStep: # TODO: change to non-singleton module and get all dependencies by ourselves def __init__(self, item: configuration_support.Step, out: Output, - fail_block: Callable[[str], None], + structure: StructureHandler, send_tag: Callable[[str], Response], log_file: Optional[TextIO], working_directory: str, @@ -173,7 +173,8 @@ def __init__(self, item: configuration_support.Step, super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out - self.fail_block: Callable[[str], None] = fail_block + self.structure: StructureHandler = structure + self.current_block = self.structure.get_current_block() self.send_tag = send_tag self.file: Optional[TextIO] = log_file self.working_directory: str = working_directory @@ -202,7 +203,7 @@ def prepare_command(self) -> bool: # FIXME: refactor command_name = os.path.abspath(os.path.join(self.working_directory, command_name)) self.cmd = make_command(command_name) except CiException as ex: - self.fail_block(str(ex)) + self.structure.fail_block(self.current_block, str(ex)) raise StepException() from ex return True @@ -260,7 +261,7 @@ def finalize(self) -> None: if self._is_background: self._is_background = False self.out.log("Nothing was executed: this background step had no command") - self.artifacts.collect_step_artifacts(self.configuration) + self._collect_artifacts() return try: text = "" @@ -279,7 +280,7 @@ def finalize(self) -> None: text = utils.trim_and_convert_to_unicode(text) if self.file: self.file.write(text + "\n") - self.fail_block(text) + self.structure.fail_block(self.current_block, text) self.add_tag(self.configuration.fail_tag) raise StepException() @@ -289,13 +290,19 @@ def finalize(self) -> None: if self.file: self.file.close() self._is_background = False - self.artifacts.collect_step_artifacts(self.configuration) + self._collect_artifacts() def _handle_postponed_out(self) -> None: for item in self._postponed_out: item[0](item[1]) self._postponed_out = [] + def _collect_artifacts(self) -> None: + self.structure.close_block() + name = f"Collecting artifacts for the '{self.configuration.name}' step" + self.structure.open_block(name) + self.artifacts.collect_step_artifacts(self.configuration) + class Launcher(ProjectDirectory, HasOutput, HasStructure, HasErrorState): artifacts_factory = Dependency(artifact_collector.ArtifactCollector) @@ -407,19 +414,13 @@ def create_process(self, item: configuration_support.Step) -> RunningStep: working_directory = utils.parse_path(utils.strip_path_start(item.directory.rstrip("/")), self.settings.project_root) - # get_current_block() should be called while inside the required block, not afterwards - block = self.structure.get_current_block() - - def fail_block(line: str = "") -> None: - self.structure.fail_block(block, line) - log_file: Optional[TextIO] = None if self.output == "file": log_file = self.artifacts.create_text_file(item.name + "_log.txt") self.out.log("Execution log is redirected to file") additional_environment = self.api_support.get_environment_settings() - return RunningStep(item, self.out, fail_block, self.server.add_build_tag, + return RunningStep(item, self.out, self.structure, self.server.add_build_tag, log_file, working_directory, additional_environment, item.background, self.artifacts) def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None: From d1587ac537de74dfb81cd85ea015abf6f56e1058 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 23 May 2022 08:59:57 +0300 Subject: [PATCH 03/17] fix pylint issue & test --- tests/test_code_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_code_report.py b/tests/test_code_report.py index c216d0ba..1fda73ba 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -278,7 +278,7 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" expected_artifacts_state = "Success" if expected_artifact else "Failed" - exptected_log = f"Collecting artifacts for the 'Run uncrustify' step - {expected_artifacts_state}" + expected_log = f"Collecting artifacts for the 'Run uncrustify' step - [^\n]*{expected_artifacts_state}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" From ab4c80537f71efc00f42f907d2eb4d6a3372cb80 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Wed, 8 Jun 2022 12:02:03 +0300 Subject: [PATCH 04/17] move step's artifact collection logic to StructureHandler --- universum/modules/artifact_collector.py | 7 ------ universum/modules/launcher.py | 30 ++++++++++++------------- universum/modules/structure_handler.py | 22 +++++++++++++----- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index b8ce9cb8..8fc7f31f 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -123,7 +123,6 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) :param ignore_already_existing: will not check existence of artifacts when set to 'True' :return: sorted list of checked paths (including duplicates and wildcards) """ - dir_list = set() for item in artifact_list: # Check existence in place: wildcards applied matches = glob2.glob(item["path"]) @@ -153,11 +152,6 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) text += "\nPossible reason of this error: previous build results in working directory" raise CriticalCiException(text) - dir_list.add(item["path"]) - new_artifact_list = list(dir_list) - new_artifact_list.sort(key=len, reverse=True) - return new_artifact_list - @make_block("Preprocessing artifact lists") def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existing_artifacts: bool = False) -> None: self.html_output.artifact_dir_ready = True @@ -225,7 +219,6 @@ def collect_step_artifacts(self, step): path = utils.parse_path(step.report_artifacts, self.settings.project_root) self.move_artifact(path, is_report=True) - def clean_artifacts_silently(self): try: shutil.rmtree(self.artifact_dir) diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 230b72b1..0934ab14 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -15,7 +15,7 @@ from . import automation_server, api_support, artifact_collector, reporter, code_report_collector from .output import HasOutput, Output from .project_directory import ProjectDirectory -from .structure_handler import HasStructure, StructureHandler +from .structure_handler import HasStructure __all__ = [ "Launcher", @@ -163,7 +163,7 @@ class RunningStep: # TODO: change to non-singleton module and get all dependencies by ourselves def __init__(self, item: configuration_support.Step, out: Output, - structure: StructureHandler, + fail_block: Callable[[str], None], send_tag: Callable[[str], Response], log_file: Optional[TextIO], working_directory: str, @@ -173,8 +173,7 @@ def __init__(self, item: configuration_support.Step, super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out - self.structure: StructureHandler = structure - self.current_block = self.structure.get_current_block() + self.fail_block: Callable[[str], None] = fail_block self.send_tag = send_tag self.file: Optional[TextIO] = log_file self.working_directory: str = working_directory @@ -203,7 +202,7 @@ def prepare_command(self) -> bool: # FIXME: refactor command_name = os.path.abspath(os.path.join(self.working_directory, command_name)) self.cmd = make_command(command_name) except CiException as ex: - self.structure.fail_block(self.current_block, str(ex)) + self.fail_block(str(ex)) raise StepException() from ex return True @@ -261,7 +260,6 @@ def finalize(self) -> None: if self._is_background: self._is_background = False self.out.log("Nothing was executed: this background step had no command") - self._collect_artifacts() return try: text = "" @@ -280,7 +278,7 @@ def finalize(self) -> None: text = utils.trim_and_convert_to_unicode(text) if self.file: self.file.write(text + "\n") - self.structure.fail_block(self.current_block, text) + self.fail_block(text) self.add_tag(self.configuration.fail_tag) raise StepException() @@ -290,19 +288,15 @@ def finalize(self) -> None: if self.file: self.file.close() self._is_background = False - self._collect_artifacts() + + def collect_artifacts(self) -> None: + self.artifacts.collect_step_artifacts(self.configuration) def _handle_postponed_out(self) -> None: for item in self._postponed_out: item[0](item[1]) self._postponed_out = [] - def _collect_artifacts(self) -> None: - self.structure.close_block() - name = f"Collecting artifacts for the '{self.configuration.name}' step" - self.structure.open_block(name) - self.artifacts.collect_step_artifacts(self.configuration) - class Launcher(ProjectDirectory, HasOutput, HasStructure, HasErrorState): artifacts_factory = Dependency(artifact_collector.ArtifactCollector) @@ -414,13 +408,19 @@ def create_process(self, item: configuration_support.Step) -> RunningStep: working_directory = utils.parse_path(utils.strip_path_start(item.directory.rstrip("/")), self.settings.project_root) + # get_current_block() should be called while inside the required block, not afterwards + block = self.structure.get_current_block() + + def fail_block(line: str = "") -> None: + self.structure.fail_block(block, line) + log_file: Optional[TextIO] = None if self.output == "file": log_file = self.artifacts.create_text_file(item.name + "_log.txt") self.out.log("Execution log is redirected to file") additional_environment = self.api_support.get_environment_settings() - return RunningStep(item, self.out, self.structure, self.server.add_build_tag, + return RunningStep(item, self.out, fail_block, self.server.add_build_tag, log_file, working_directory, additional_environment, item.background, self.artifacts) def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None: diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 93412f6c..ef15b55d 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -68,6 +68,7 @@ class BackgroundStepInfo(TypedDict): class StructureHandler(HasOutput): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.current_block: Optional[Block] = Block("Universum") @@ -141,12 +142,14 @@ def execute_one_step(self, configuration: Step, process.start() if not configuration.background: process.finalize() - return + return process self.out.log("This step is marked to be executed in background") self.active_background_steps.append({'name': configuration.name, 'finalizer': process.finalize, + 'artifacts_collection': process.collect_artifacts, 'is_critical': is_critical}) + return None def finalize_background_step(self, background_step: BackgroundStepInfo): try: @@ -198,8 +201,14 @@ def execute_steps_recursively(self, parent: Optional[Step], cfg: Configuration, # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps - self.run_in_block(self.execute_one_step, step_name, False, - item, step_executor, obj_a.critical) + step_process = None + try: + step_process = self.run_in_block(self.execute_one_step, step_name, False, + item, step_executor, obj_a.critical) + finally: + if step_process: + artifacts_step_name = f"Collecting artifacts for the '{item.name}' step" + self.run_in_block(step_process.collect_artifacts, artifacts_step_name, False) except StepException: child_step_failed = True if obj_a.critical: @@ -211,9 +220,12 @@ def execute_steps_recursively(self, parent: Optional[Step], cfg: Configuration, def report_background_steps(self) -> bool: result = True for item in self.active_background_steps: - if not self.run_in_block(self.finalize_background_step, + finalization_result = self.run_in_block(self.finalize_background_step, "Waiting for background step '" + item['name'] + "' to finish...", - True, item): + True, item) + artifacts_step_name = f"Collecting artifacts for the '{item['name']}' step" + self.run_in_block(item["artifacts_collection"], artifacts_step_name, False) + if not finalization_result: result = False self.out.log("All ongoing background steps completed") self.active_background_steps = [] From 6e8809b29ad6f3d276fe5bd33cb58bb4e4cbda14 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Mon, 27 Jun 2022 08:08:27 +0300 Subject: [PATCH 05/17] collect artifacts after failed step too --- universum/lib/ci_exception.py | 4 ++- universum/modules/structure_handler.py | 45 ++++++++++++++++---------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/universum/lib/ci_exception.py b/universum/lib/ci_exception.py index 4046d5de..4a1168f8 100644 --- a/universum/lib/ci_exception.py +++ b/universum/lib/ci_exception.py @@ -21,4 +21,6 @@ def __init__(self, application_exit_code: int = 1) -> None: class StepException(Exception): - pass + def __init__(self, step_process=None): + super().__init__() + self.step_process = step_process diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index ef15b55d..d4ba55bf 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -64,6 +64,7 @@ def is_successful(self) -> bool: class BackgroundStepInfo(TypedDict): name: str finalizer: Callable[[], None] + artifacts_collection: Callable is_critical: bool @@ -139,17 +140,21 @@ def execute_one_step(self, configuration: Step, # step_executor is [[Step], Step], but referring to Step creates circular dependency process = step_executor(configuration) - process.start() - if not configuration.background: - process.finalize() - return process + try: + process.start() + if not configuration.background: + process.finalize() + return process + + self.out.log("This step is marked to be executed in background") + self.active_background_steps.append({'name': configuration.name, + 'finalizer': process.finalize, + 'artifacts_collection': process.collect_artifacts, + 'is_critical': is_critical}) - self.out.log("This step is marked to be executed in background") - self.active_background_steps.append({'name': configuration.name, - 'finalizer': process.finalize, - 'artifacts_collection': process.collect_artifacts, - 'is_critical': is_critical}) - return None + return process + except StepException as e: + raise StepException(process) def finalize_background_step(self, background_step: BackgroundStepInfo): try: @@ -204,11 +209,13 @@ def execute_steps_recursively(self, parent: Optional[Step], cfg: Configuration, step_process = None try: step_process = self.run_in_block(self.execute_one_step, step_name, False, - item, step_executor, obj_a.critical) - finally: - if step_process: - artifacts_step_name = f"Collecting artifacts for the '{item.name}' step" - self.run_in_block(step_process.collect_artifacts, artifacts_step_name, False) + item, step_executor, obj_a.critical) + if not item.background: + self._run_artifact_collection_step(item.name, step_process.collect_artifacts) + except StepException as e: + if not item.background: + self._run_artifact_collection_step(item.name, e.step_process.collect_artifacts) + raise e except StepException: child_step_failed = True if obj_a.critical: @@ -223,8 +230,7 @@ def report_background_steps(self) -> bool: finalization_result = self.run_in_block(self.finalize_background_step, "Waiting for background step '" + item['name'] + "' to finish...", True, item) - artifacts_step_name = f"Collecting artifacts for the '{item['name']}' step" - self.run_in_block(item["artifacts_collection"], artifacts_step_name, False) + self._run_artifact_collection_step(item['name'], item['artifacts_collection']) if not finalization_result: result = False self.out.log("All ongoing background steps completed") @@ -242,6 +248,11 @@ def execute_step_structure(self, configs: Configuration, step_executor) -> None: if self.active_background_steps: self.run_in_block(self.report_background_steps, "Reporting background steps", False) + def _run_artifact_collection_step(self, step_name, artifacts_collection_func): + artifacts_step_name = f"Collecting artifacts for the '{step_name}' step" + self.run_in_block(artifacts_collection_func, artifacts_step_name, pass_errors=False) + + class HasStructure(Module): structure_factory: ClassVar = Dependency(StructureHandler) From 40c63172b979a07b5bcb5ce30d649a10cf1b4d92 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 28 Jun 2022 11:37:56 +0300 Subject: [PATCH 06/17] review comments minor fixes --- universum/modules/artifact_collector.py | 10 +++++----- universum/modules/launcher.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index 8fc7f31f..74aba318 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -211,12 +211,12 @@ def move_artifact(self, path, is_report=False): artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name) self.collected_report_artifacts.add(artifact_path) - def collect_step_artifacts(self, step): - if step.artifacts: - path = utils.parse_path(step.artifacts, self.settings.project_root) + def collect_step_artifacts(self, step_artifacts, step_report_artifacts): + if step_artifacts: + path = utils.parse_path(step_artifacts, self.settings.project_root) self.move_artifact(path, is_report=False) - if step.report_artifacts: - path = utils.parse_path(step.report_artifacts, self.settings.project_root) + if step_report_artifacts: + path = utils.parse_path(step_report_artifacts, self.settings.project_root) self.move_artifact(path, is_report=True) def clean_artifacts_silently(self): diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 0934ab14..5ba8e936 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -169,7 +169,7 @@ def __init__(self, item: configuration_support.Step, working_directory: str, additional_environment: Dict[str, str], background: bool, - artifacts: artifact_collector.ArtifactCollector) -> None: + artifact_collector: artifact_collector.ArtifactCollector) -> None: super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out @@ -188,7 +188,7 @@ def __init__(self, item: configuration_support.Step, self._postponed_out: List[Tuple[Callable[[str], None], str]] = [] self._needs_finalization: bool = True - self.artifacts = artifacts + self.artifact_collector = artifact_collector def prepare_command(self) -> bool: # FIXME: refactor if not self.configuration.command: @@ -290,7 +290,7 @@ def finalize(self) -> None: self._is_background = False def collect_artifacts(self) -> None: - self.artifacts.collect_step_artifacts(self.configuration) + self.artifact_collector.collect_step_artifacts(self.configuration.artifacts, self.configuration.report_artifacts) def _handle_postponed_out(self) -> None: for item in self._postponed_out: @@ -348,7 +348,7 @@ def __init__(self, *args, **kwargs) -> None: if not self.config_path: self.config_path = ".universum.py" - self.artifacts = self.artifacts_factory() + self.artifact_collector = self.artifacts_factory() self.api_support = self.api_support_factory() self.reporter = self.reporter_factory() self.server = self.server_factory() @@ -367,7 +367,7 @@ def process_project_configs(self) -> configuration_support.Configuration: with open(config_path, encoding="utf-8") as config_file: exec(config_file.read(), config_globals) # pylint: disable=exec-used self.source_project_configs = config_globals["configs"] - dump_file: TextIO = self.artifacts.create_text_file("CONFIGS_DUMP.txt") + dump_file: TextIO = self.artifact_collector.create_text_file("CONFIGS_DUMP.txt") dump_file.write(self.source_project_configs.dump()) dump_file.close() config = self.source_project_configs.filter(check_if_env_set) @@ -416,12 +416,12 @@ def fail_block(line: str = "") -> None: log_file: Optional[TextIO] = None if self.output == "file": - log_file = self.artifacts.create_text_file(item.name + "_log.txt") + log_file = self.artifact_collector.create_text_file(item.name + "_log.txt") self.out.log("Execution log is redirected to file") additional_environment = self.api_support.get_environment_settings() return RunningStep(item, self.out, fail_block, self.server.add_build_tag, - log_file, working_directory, additional_environment, item.background, self.artifacts) + log_file, working_directory, additional_environment, item.background, self.artifact_collector) def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None: self.structure.execute_step_structure(custom_configs, self.create_process) From 7b2476d9217cf5cbeb97fcef1baa7be10859a1a2 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 26 Jul 2022 07:18:12 +0300 Subject: [PATCH 07/17] re-apply changes on new master --- tests/test_code_report.py | 4 +-- tests/test_integration.py | 7 +++--- universum/main.py | 1 - universum/modules/artifact_collector.py | 33 +++++++------------------ universum/modules/launcher.py | 19 +++++++++----- universum/modules/structure_handler.py | 27 +++++++++++++------- universum/nonci.py | 5 ++-- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/tests/test_code_report.py b/tests/test_code_report.py index 58424b45..9383afd9 100644 --- a/tests/test_code_report.py +++ b/tests/test_code_report.py @@ -276,8 +276,8 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner, expected_log = log_success if expected_success else log_fail assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" - expected_log = r"Collecting 'source_file.html' - [^\n]*Success" if expected_artifact \ - else r"Collecting 'source_file.html' - [^\n]*Failed" + expected_artifacts_state = "Success" if expected_artifact else "Failed" + expected_log = f"Collecting artifacts for the 'Run uncrustify' step - [^\n]*{expected_artifacts_state}" assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'" diff --git a/tests/test_integration.py b/tests/test_integration.py index fe900764..dcb9529a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -39,13 +39,14 @@ def test_artifacts(docker_main: UniversumRunner): files2 = Configuration([dict(name=" one/three/file.sh", command=["one/three/file.sh"])]) artifacts = Configuration([dict(name="Existing artifacts", artifacts="one/**/file*", report_artifacts="one/*"), - dict(name="Missing artifacts", artifacts="something", report_artifacts="something_else")]) + dict(name="Missing report artifacts", report_artifacts="non_existing_file"), + dict(name="Missing all artifacts", artifacts="something", report_artifacts="something_else")]) configs = mkdir * dirs1 + mkdir * dirs2 + mkfile * files1 + mkfile * files2 + artifacts """ log = docker_main.run(config) - assert 'Failed' in get_line_with_text("Collecting 'something' - ", log) - assert 'Success' in get_line_with_text("Collecting 'something_else' for report - ", log) + assert 'Failed' in get_line_with_text("Collecting artifacts for the 'Missing all artifacts' step - ", log) + assert 'Success' in get_line_with_text("Collecting artifacts for the 'Missing report artifacts' step - ", log) assert os.path.exists(os.path.join(docker_main.artifact_dir, "three.zip")) assert os.path.exists(os.path.join(docker_main.artifact_dir, "two2.zip")) diff --git a/universum/main.py b/universum/main.py index 453c72d4..a83f522e 100644 --- a/universum/main.py +++ b/universum/main.py @@ -87,7 +87,6 @@ def execute(self) -> None: self.launcher.launch_custom_configs(afterall_configs) self.code_report_collector.repo_diff = repo_diff self.code_report_collector.report_code_report_results() - self.artifacts.collect_artifacts() self.reporter.report_build_result() def finalize(self) -> None: diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index cd7d5478..34e1ebb4 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -80,9 +80,6 @@ def __init__(self, *args, **kwargs): self.reporter = self.reporter_factory() self.automation_server = self.automation_server_factory() - self.artifact_list = [] - self.report_artifact_list = [] - # Needed because of wildcards self.collected_report_artifacts = set() @@ -126,7 +123,6 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) :param ignore_already_existing: will not check existence of artifacts when set to 'True' :return: sorted list of checked paths (including duplicates and wildcards) """ - dir_list = set() for item in artifact_list: # Check existence in place: wildcards applied matches = glob2.glob(item["path"]) @@ -156,11 +152,6 @@ def preprocess_artifact_list(self, artifact_list, ignore_already_existing=False) text += "\nPossible reason of this error: previous build results in working directory" raise CriticalCiException(text) - dir_list.add(item["path"]) - new_artifact_list = list(dir_list) - new_artifact_list.sort(key=len, reverse=True) - return new_artifact_list - @make_block("Preprocessing artifact lists") def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existing_artifacts: bool = False) -> None: self.html_output.artifact_dir_ready = True @@ -177,13 +168,12 @@ def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existin if artifact_list: name = "Setting and preprocessing artifacts according to configs" with self.structure.block(block_name=name, pass_errors=True): - self.artifact_list = self.preprocess_artifact_list(artifact_list, ignore_existing_artifacts) + self.preprocess_artifact_list(artifact_list, ignore_existing_artifacts) if report_artifact_list: name = "Setting and preprocessing artifacts to be mentioned in report" with self.structure.block(block_name=name, pass_errors=True): - self.report_artifact_list = self.preprocess_artifact_list(report_artifact_list, - ignore_existing_artifacts) + self.preprocess_artifact_list(report_artifact_list, ignore_existing_artifacts) def move_artifact(self, path, is_report=False): self.out.log("Processing '" + path + "'") @@ -221,18 +211,13 @@ def move_artifact(self, path, is_report=False): artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name) self.collected_report_artifacts.add(artifact_path) - @make_block("Collecting artifacts", pass_errors=False) - def collect_artifacts(self): - self.reporter.add_block_to_report(self.structure.get_current_block()) - for path in self.report_artifact_list: - name = "Collecting '" + os.path.basename(path) + "' for report" - with self.structure.block(block_name=name, pass_errors=False): - self.move_artifact(path, is_report=True) - self.reporter.report_artifacts(list(self.collected_report_artifacts)) - for path in self.artifact_list: - name = "Collecting '" + os.path.basename(path) + "'" - with self.structure.block(block_name=name, pass_errors=False): - self.move_artifact(path) + def collect_step_artifacts(self, step_artifacts, step_report_artifacts): + if step_artifacts: + path = utils.parse_path(step_artifacts, self.settings.project_root) + self.move_artifact(path, is_report=False) + if step_report_artifacts: + path = utils.parse_path(step_report_artifacts, self.settings.project_root) + self.move_artifact(path, is_report=True) def clean_artifacts_silently(self): try: diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index c02a4394..7d1c8656 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -167,7 +167,8 @@ def __init__(self, item: configuration_support.Step, log_file: Optional[TextIO], working_directory: str, additional_environment: Dict[str, str], - background: bool) -> None: + background: bool, + artifact_collector: artifact_collector.ArtifactCollector) -> None: super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out @@ -185,6 +186,8 @@ def __init__(self, item: configuration_support.Step, self._postponed_out: List[Tuple[Callable[[str], None], str]] = [] self._needs_finalization: bool = True + self.artifact_collector = artifact_collector + def prepare_command(self) -> bool: # FIXME: refactor if not self.configuration.command: self.out.log("No 'command' found. Nothing to execute") @@ -288,6 +291,10 @@ def finalize(self) -> Optional[str]: self.file.close() self._is_background = False + def collect_artifacts(self) -> None: + self.artifact_collector.collect_step_artifacts(self.configuration.artifacts, + self.configuration.report_artifacts) + def _handle_postponed_out(self) -> None: for item in self._postponed_out: item[0](item[1]) @@ -344,7 +351,7 @@ def __init__(self, *args, **kwargs) -> None: if not self.config_path: self.config_path = ".universum.py" - self.artifacts = self.artifacts_factory() + self.artifact_collector = self.artifacts_factory() self.api_support = self.api_support_factory() self.reporter = self.reporter_factory() self.server = self.server_factory() @@ -364,7 +371,7 @@ def process_project_configs(self) -> configuration_support.Configuration: with open(config_path, encoding="utf-8") as config_file: exec(config_file.read(), config_globals) # pylint: disable=exec-used self.source_project_configs = config_globals["configs"] - dump_file: TextIO = self.artifacts.create_text_file("CONFIGS_DUMP.txt") + dump_file: TextIO = self.artifact_collector.create_text_file("CONFIGS_DUMP.txt") dump_file.write(self.source_project_configs.dump()) dump_file.close() config = self.source_project_configs.filter(check_if_env_set) @@ -407,12 +414,12 @@ def create_process(self, item: configuration_support.Step) -> RunningStep: log_file: Optional[TextIO] = None if self.output == "file": - log_file = self.artifacts.create_text_file(item.name + "_log.txt") + log_file = self.artifact_collector.create_text_file(item.name + "_log.txt") self.out.log("Execution log is redirected to file") additional_environment = self.api_support.get_environment_settings() - return RunningStep(item, self.out, self.server.add_build_tag, - log_file, working_directory, additional_environment, item.background) + return RunningStep(item, self.out, self.server.add_build_tag, log_file, working_directory, + additional_environment, item.background, self.artifact_collector) def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None: self.structure.execute_step_structure(custom_configs, self.create_process) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 2bce7c04..4f59c6bb 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -67,6 +67,7 @@ class BackgroundStepInfo(TypedDict): name: str block: Block finalizer: Callable[[], None] + artifats_collection: Callable is_critical: bool @@ -136,24 +137,25 @@ def block(self, *, block_name: str, pass_errors: bool) -> Generator: self.close_block() def execute_one_step(self, configuration: Step, - step_executor: Callable) -> Optional[str]: + step_executor: Callable): # step_executor is [[Step], Step], but referring to Step creates circular dependency process = step_executor(configuration) error: Optional[str] = process.start() if error is not None: - return error + return process, error if not configuration.background: error = process.finalize() - return error # could be None or error message + return process, error # could be None or error message self.out.log("This step is marked to be executed in background") self.active_background_steps.append({'name': configuration.name, 'block': self.get_current_block(), 'finalizer': process.finalize, + 'artifacts_collection': process.collect_artifacts, 'is_critical': configuration.critical}) - return None + return process, None def finalize_background_step(self, background_step: BackgroundStepInfo): error = background_step['finalizer']() @@ -175,20 +177,25 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec self.configs_current_number += 1 numbering: str = f" [ {self.configs_current_number:>{self.step_num_len}}/{self.configs_total_count} ] " step_label: str = numbering + merged_item.name + executed_successfully: bool = True if skip_execution: self.report_skipped_block(numbering + "'" + merged_item.name + "'") - return True + return executed_successfully # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps + process = None with self.block(block_name=step_label, pass_errors=False): - error: Optional[str] = self.execute_one_step(merged_item, step_executor) - if error is not None: + process, error = self.execute_one_step(merged_item, step_executor) + executed_successfully = (error is None) + if not executed_successfully: self.fail_current_block(error) - return False + if not process._is_background: + with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): + process.collect_artifacts() - return True + return executed_successfully def execute_steps_recursively(self, parent: Step, children: Configuration, @@ -244,6 +251,8 @@ def report_background_steps(self) -> bool: result = False self.out.report_skipped("The background step '" + item['name'] + "' failed, and as it is critical, " "all further steps will be skipped") + with self.block(block_name=f"Collecting artifacts for the '{item['name']}' step", pass_errors=False): + item['artifacts_collection'](); self.out.log("All ongoing background steps completed") self.active_background_steps = [] diff --git a/universum/nonci.py b/universum/nonci.py index 4a6a6a8d..4c988a1b 100644 --- a/universum/nonci.py +++ b/universum/nonci.py @@ -17,16 +17,15 @@ def __init__(self, *args, **kwargs): def execute(self): self.out.log("Cleaning artifacts...") - self.artifacts.clean_artifacts_silently() + self.artifact_collector.clean_artifacts_silently() project_configs = self.process_project_configs() self.code_report_collector.prepare_environment(project_configs) - self.artifacts.set_and_clean_artifacts(project_configs, ignore_existing_artifacts=True) + self.artifact_collector.set_and_clean_artifacts(project_configs, ignore_existing_artifacts=True) self.launch_project() self.reporter.report_initialized = True self.reporter.report_build_result() - self.artifacts.collect_artifacts() def finalize(self): pass From d043087bd02ce16c7452078e0e96b8e9f7f23485 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 26 Jul 2022 08:06:04 +0300 Subject: [PATCH 08/17] fix CI issues --- universum/modules/launcher.py | 4 ++-- universum/modules/structure_handler.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/universum/modules/launcher.py b/universum/modules/launcher.py index 7d1c8656..2bdb021c 100644 --- a/universum/modules/launcher.py +++ b/universum/modules/launcher.py @@ -168,7 +168,7 @@ def __init__(self, item: configuration_support.Step, working_directory: str, additional_environment: Dict[str, str], background: bool, - artifact_collector: artifact_collector.ArtifactCollector) -> None: + artifact_collector_obj: artifact_collector.ArtifactCollector) -> None: super().__init__() self.configuration: configuration_support.Step = item self.out: Output = out @@ -186,7 +186,7 @@ def __init__(self, item: configuration_support.Step, self._postponed_out: List[Tuple[Callable[[str], None], str]] = [] self._needs_finalization: bool = True - self.artifact_collector = artifact_collector + self.artifact_collector = artifact_collector_obj def prepare_command(self) -> bool: # FIXME: refactor if not self.configuration.command: diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 4f59c6bb..bce09dbf 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -67,7 +67,7 @@ class BackgroundStepInfo(TypedDict): name: str block: Block finalizer: Callable[[], None] - artifats_collection: Callable + artifacts_collection: Callable is_critical: bool @@ -191,7 +191,7 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec executed_successfully = (error is None) if not executed_successfully: self.fail_current_block(error) - if not process._is_background: + if not merged_item.background: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() @@ -252,7 +252,7 @@ def report_background_steps(self) -> bool: self.out.report_skipped("The background step '" + item['name'] + "' failed, and as it is critical, " "all further steps will be skipped") with self.block(block_name=f"Collecting artifacts for the '{item['name']}' step", pass_errors=False): - item['artifacts_collection'](); + item['artifacts_collection']() self.out.log("All ongoing background steps completed") self.active_background_steps = [] From 9755dc8f2c058e47ea8628603aabc61c5b030dcf Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 28 Jul 2022 08:23:23 +0300 Subject: [PATCH 09/17] review comments partial fix --- universum/modules/structure_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index bce09dbf..f02638b7 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -177,15 +177,15 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec self.configs_current_number += 1 numbering: str = f" [ {self.configs_current_number:>{self.step_num_len}}/{self.configs_total_count} ] " step_label: str = numbering + merged_item.name - executed_successfully: bool = True if skip_execution: self.report_skipped_block(numbering + "'" + merged_item.name + "'") - return executed_successfully + return True + process = None + executed_successfully: bool = True # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps - process = None with self.block(block_name=step_label, pass_errors=False): process, error = self.execute_one_step(merged_item, step_executor) executed_successfully = (error is None) From d42b89282c3e148066247bb33e8ed208bcbe9cff Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 4 Aug 2022 10:59:22 +0300 Subject: [PATCH 10/17] update Step class docstring --- universum/configuration_support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/universum/configuration_support.py b/universum/configuration_support.py index 4945624f..80bde378 100644 --- a/universum/configuration_support.py +++ b/universum/configuration_support.py @@ -61,8 +61,8 @@ class Step: one time at most. artifacts Path to the file or directory to be copied to the working directory as an execution - result. Can contain shell-style pattern matching (e.g. `"out/*.html"`), including recursive wildcards - (e.g. `"out/**/index.html"`). If not stated otherwise (see ``--no-archive`` + result immediately after step finish. Can contain shell-style pattern matching (e.g. `"out/*.html"`), + including recursive wildcards (e.g. `"out/**/index.html"`). If not stated otherwise (see ``--no-archive`` command-line parameter for details), artifact directories are copied as archives. If `artifact_prebuild_clean` key is either absent or set to `False` and stated artifacts are present in downloaded sources, it is considered a failure and configuration From 73057cc203ca441bd912fcef0c8b0e35c37d387c Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 4 Aug 2022 11:40:15 +0300 Subject: [PATCH 11/17] restore artifacts reporting --- universum/main.py | 1 + universum/modules/artifact_collector.py | 3 +++ universum/nonci.py | 1 + 3 files changed, 5 insertions(+) diff --git a/universum/main.py b/universum/main.py index a83f522e..689a5d70 100644 --- a/universum/main.py +++ b/universum/main.py @@ -87,6 +87,7 @@ def execute(self) -> None: self.launcher.launch_custom_configs(afterall_configs) self.code_report_collector.repo_diff = repo_diff self.code_report_collector.report_code_report_results() + self.artifacts.report_artifacts() self.reporter.report_build_result() def finalize(self) -> None: diff --git a/universum/modules/artifact_collector.py b/universum/modules/artifact_collector.py index 34e1ebb4..e6a5c094 100644 --- a/universum/modules/artifact_collector.py +++ b/universum/modules/artifact_collector.py @@ -219,6 +219,9 @@ def collect_step_artifacts(self, step_artifacts, step_report_artifacts): path = utils.parse_path(step_report_artifacts, self.settings.project_root) self.move_artifact(path, is_report=True) + def report_artifacts(self): + self.reporter.report_artifacts(list(self.collected_report_artifacts)) + def clean_artifacts_silently(self): try: shutil.rmtree(self.artifact_dir) diff --git a/universum/nonci.py b/universum/nonci.py index 4c988a1b..50cb22bf 100644 --- a/universum/nonci.py +++ b/universum/nonci.py @@ -25,6 +25,7 @@ def execute(self): self.launch_project() self.reporter.report_initialized = True + self.artifact_collector.report_artifacts() self.reporter.report_build_result() def finalize(self): From 4c19eb7512614a7415b27d323f4b12d072ce0e15 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Fri, 5 Aug 2022 08:47:25 +0300 Subject: [PATCH 12/17] fix static analysis errors --- universum/modules/structure_handler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 89902239..57c8dcb5 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -85,6 +85,10 @@ def finalize(self) -> None: def get_error(self) -> Optional[str]: pass + @abstractmethod + def collect_artifacts(self) -> None: + pass + class StructureHandler(HasOutput): def __init__(self, *args, **kwargs) -> None: @@ -158,7 +162,7 @@ def execute_one_step(self, configuration: Step, if process.get_error() is not None: return process if not configuration.background: - error = process.finalize() + process.finalize() return process self.out.log("This step is marked to be executed in background") self.active_background_steps.append({'name': configuration.name, @@ -199,10 +203,11 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps with self.block(block_name=step_label, pass_errors=False): - process = self.execute_one_step(merged_item, step_executor) + process= self.execute_one_step(merged_item, step_executor) error = process.get_error() executed_successfully = (error is None) if not executed_successfully: + error = error if error else "" self.fail_current_block(error) if not merged_item.background: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): From 6711770ea1b8acf153dc9eca46bcc3ed6c6c8632 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 15 Sep 2022 07:59:56 +0300 Subject: [PATCH 13/17] create artifacts colletion step only if artifacts are present --- universum/modules/structure_handler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 57c8dcb5..5e7dc488 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -69,6 +69,7 @@ class BackgroundStepInfo(TypedDict): block: Block process: RunningStepBase is_critical: bool + has_artifacts: bool class RunningStepBase(ABC): @@ -168,7 +169,8 @@ def execute_one_step(self, configuration: Step, self.active_background_steps.append({'name': configuration.name, 'block': self.get_current_block(), 'process': process, - 'is_critical': configuration.critical}) + 'is_critical': configuration.critical, + 'has_artifacts': bool(configuration.artifacts)}) return process def finalize_background_step(self, background_step: BackgroundStepInfo) -> bool: @@ -209,7 +211,7 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec if not executed_successfully: error = error if error else "" self.fail_current_block(error) - if not merged_item.background: + if not merged_item.background and merged_item.artifacts: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() @@ -269,8 +271,9 @@ def report_background_steps(self) -> bool: result = False self.out.report_skipped("The background step '" + item['name'] + "' failed, and as it is critical, " "all further steps will be skipped") - with self.block(block_name=f"Collecting artifacts for the '{item['name']}' step", pass_errors=False): - item['process'].collect_artifacts() + if item['has_artifacts']: + with self.block(block_name=f"Collecting artifacts for the '{item['name']}' step", pass_errors=False): + item['process'].collect_artifacts() self.out.log("All ongoing background steps completed") self.active_background_steps = [] From 273cd0fcfd8dc1bef83ad19de7a57d3bd33847c1 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 15 Sep 2022 08:11:14 +0300 Subject: [PATCH 14/17] add more annotations --- universum/modules/structure_handler.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 5e7dc488..a6c64e44 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -200,17 +200,16 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec self.report_skipped_block(numbering + "'" + merged_item.name + "'") return True - process = None + process: Optional[RunningStepBase] = None executed_successfully: bool = True # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps with self.block(block_name=step_label, pass_errors=False): - process= self.execute_one_step(merged_item, step_executor) - error = process.get_error() + process = self.execute_one_step(merged_item, step_executor) + error: Optional[str] = process.get_error() executed_successfully = (error is None) if not executed_successfully: - error = error if error else "" - self.fail_current_block(error) + self.fail_current_block(error) # type: ignore[arg-type] if not merged_item.background and merged_item.artifacts: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() From 24d8d6b7dcec8a63cfd38e984434d628508b4fe9 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Thu, 15 Sep 2022 08:49:47 +0300 Subject: [PATCH 15/17] count also --- universum/modules/structure_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index a6c64e44..be01a6cb 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -166,11 +166,12 @@ def execute_one_step(self, configuration: Step, process.finalize() return process self.out.log("This step is marked to be executed in background") + has_artifacts: bool = bool(configuration.artifacts) or bool(configuration.report_artifacts) self.active_background_steps.append({'name': configuration.name, 'block': self.get_current_block(), 'process': process, 'is_critical': configuration.critical, - 'has_artifacts': bool(configuration.artifacts)}) + 'has_artifacts': has_artifacts}) return process def finalize_background_step(self, background_step: BackgroundStepInfo) -> bool: @@ -210,7 +211,8 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec executed_successfully = (error is None) if not executed_successfully: self.fail_current_block(error) # type: ignore[arg-type] - if not merged_item.background and merged_item.artifacts: + has_artifacts: bool = bool(merged_item.artifacts) or bool(merged_item.report_artifacts) + if not merged_item.background and has_artifacts: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() From c680247746814636ecf9183f6f237556c47c5f46 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 20 Sep 2022 09:48:07 +0300 Subject: [PATCH 16/17] remove mypy error ignoring --- universum/modules/structure_handler.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index be01a6cb..e29ede23 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -202,21 +202,20 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec return True process: Optional[RunningStepBase] = None - executed_successfully: bool = True + error: Optional[str] = None # Here pass_errors=False, because any exception while executing build step # can be step-related and may not affect other steps with self.block(block_name=step_label, pass_errors=False): process = self.execute_one_step(merged_item, step_executor) - error: Optional[str] = process.get_error() - executed_successfully = (error is None) - if not executed_successfully: - self.fail_current_block(error) # type: ignore[arg-type] + error = process.get_error() + if error: + self.fail_current_block(error) has_artifacts: bool = bool(merged_item.artifacts) or bool(merged_item.report_artifacts) if not merged_item.background and has_artifacts: with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() - return executed_successfully + return (error is None) def execute_steps_recursively(self, parent: Step, children: Configuration, From 6cdec59c72aac128b37aaa066c1c92084c277102 Mon Sep 17 00:00:00 2001 From: miltolstoy Date: Tue, 20 Sep 2022 15:43:39 +0300 Subject: [PATCH 17/17] fix pylint issue --- universum/modules/structure_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universum/modules/structure_handler.py b/universum/modules/structure_handler.py index 0368c8ce..f4758beb 100644 --- a/universum/modules/structure_handler.py +++ b/universum/modules/structure_handler.py @@ -217,7 +217,7 @@ def process_one_step(self, merged_item: Step, step_executor: Callable, skip_exec with self.block(block_name=f"Collecting artifacts for the '{merged_item.name}' step", pass_errors=False): process.collect_artifacts() - return (error is None) + return error is None def execute_steps_recursively(self, parent: Step, children: Configuration,