refactor: collect step artifacts immediately after step finish

tests/test_code_report.py

@@ -276,8 +276,9 @@ def test_uncrustify_file_diff(runner_with_analyzers: UniversumRunner,
 
     expected_log = log_success if expected_success else log_fail
     assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'"
-    expected_log = r"Collecting 'source_file.html' - [^\n]*Success" if expected_artifact \
-        else r"Collecting 'source_file.html' - [^\n]*Failed"
+
+    expected_artifacts_state = "Success" if expected_artifact else "Failed"
+    exptected_log = f"Collecting artifacts for the 'Run uncrustify' step - {expected_artifacts_state}"
     assert re.findall(expected_log, log), f"'{expected_log}' is not found in '{log}'"
 
 

tests/test_integration.py

@@ -39,13 +39,14 @@ def test_artifacts(docker_main: UniversumRunner):
 files2 = Configuration([dict(name=" one/three/file.sh", command=["one/three/file.sh"])])
 
 artifacts = Configuration([dict(name="Existing artifacts", artifacts="one/**/file*", report_artifacts="one/*"),
-                        dict(name="Missing artifacts", artifacts="something", report_artifacts="something_else")])
+                           dict(name="Missing report artifacts", report_artifacts="non_existing_file"),
+                           dict(name="Missing all artifacts", artifacts="something", report_artifacts="something_else")])
 
 configs = mkdir * dirs1 + mkdir * dirs2 + mkfile * files1 + mkfile * files2 + artifacts
     """
     log = docker_main.run(config)
-    assert 'Failed' in get_line_with_text("Collecting 'something' - ", log)
-    assert 'Success' in get_line_with_text("Collecting 'something_else' for report - ", log)
+    assert 'Failed' in get_line_with_text("Collecting artifacts for the 'Missing all artifacts' step - ", log)
+    assert 'Success' in get_line_with_text("Collecting artifacts for the 'Missing report artifacts' step - ", log)
 
     assert os.path.exists(os.path.join(docker_main.artifact_dir, "three.zip"))
     assert os.path.exists(os.path.join(docker_main.artifact_dir, "two2.zip"))

universum/main.py

@@ -87,7 +87,6 @@ def execute(self) -> None:
                     self.launcher.launch_custom_configs(afterall_configs)
                     self.code_report_collector.repo_diff = repo_diff
             self.code_report_collector.report_code_report_results()
-        self.artifacts.collect_artifacts()
         self.reporter.report_build_result()
 
     def finalize(self) -> None:

universum/modules/artifact_collector.py

@@ -80,9 +80,6 @@ def __init__(self, *args, **kwargs):
         self.reporter = self.reporter_factory()
         self.automation_server = self.automation_server_factory()
 
-        self.artifact_list = []
-        self.report_artifact_list = []
-
         # Needed because of wildcards
         self.collected_report_artifacts = set()
 
@@ -176,13 +173,13 @@ def set_and_clean_artifacts(self, project_configs: Configuration, ignore_existin
 
         if artifact_list:
             name = "Setting and preprocessing artifacts according to configs"
-            self.artifact_list = self.structure.run_in_block(self.preprocess_artifact_list,
-                                                             name, True, artifact_list, ignore_existing_artifacts)
+            self.structure.run_in_block(self.preprocess_artifact_list,
+                                        name, True, artifact_list, ignore_existing_artifacts)
         if report_artifact_list:
             name = "Setting and preprocessing artifacts to be mentioned in report"
-            self.report_artifact_list = self.structure.run_in_block(self.preprocess_artifact_list,
-                                                                    name, True, report_artifact_list,
-                                                                    ignore_existing_artifacts)
+            self.structure.run_in_block(self.preprocess_artifact_list,
+                                        name, True, report_artifact_list,
+                                        ignore_existing_artifacts)
 
     def move_artifact(self, path, is_report=False):
         self.out.log("Processing '" + path + "'")
@@ -220,16 +217,14 @@ def move_artifact(self, path, is_report=False):
                     artifact_path = self.automation_server.artifact_path(self.artifact_dir, artifact_name)
                     self.collected_report_artifacts.add(artifact_path)
 
-    @make_block("Collecting artifacts", pass_errors=False)
-    def collect_artifacts(self):
-        self.reporter.add_block_to_report(self.structure.get_current_block())
-        for path in self.report_artifact_list:
-            name = "Collecting '" + os.path.basename(path) + "' for report"
-            self.structure.run_in_block(self.move_artifact, name, False, path, is_report=True)
-        self.reporter.report_artifacts(list(self.collected_report_artifacts))
-        for path in self.artifact_list:
-            name = "Collecting '" + os.path.basename(path) + "'"
-            self.structure.run_in_block(self.move_artifact, name, False, path)
+    def collect_step_artifacts(self, step):
+        if step.artifacts:
+            path = utils.parse_path(step.artifacts, self.settings.project_root)
+            self.move_artifact(path, is_report=False)
+        if step.report_artifacts:
+            path = utils.parse_path(step.report_artifacts, self.settings.project_root)
+            self.move_artifact(path, is_report=True)
+
 
     def clean_artifacts_silently(self):
         try:

universum/modules/launcher.py

@@ -15,7 +15,7 @@
 from . import automation_server, api_support, artifact_collector, reporter, code_report_collector
 from .output import HasOutput, Output
 from .project_directory import ProjectDirectory
-from .structure_handler import HasStructure
+from .structure_handler import HasStructure, StructureHandler
 
 __all__ = [
     "Launcher",
@@ -163,16 +163,18 @@ class RunningStep:
     # TODO: change to non-singleton module and get all dependencies by ourselves
     def __init__(self, item: configuration_support.Step,
                  out: Output,
-                 fail_block: Callable[[str], None],
+                 structure: StructureHandler,
                  send_tag: Callable[[str], Response],
                  log_file: Optional[TextIO],
                  working_directory: str,
                  additional_environment: Dict[str, str],
-                 background: bool) -> None:
+                 background: bool,
+                 artifacts: artifact_collector.ArtifactCollector) -> None:
         super().__init__()
         self.configuration: configuration_support.Step = item
         self.out: Output = out
-        self.fail_block: Callable[[str], None] = fail_block
+        self.structure: StructureHandler = structure
+        self.current_block = self.structure.get_current_block()
         self.send_tag = send_tag
         self.file: Optional[TextIO] = log_file
         self.working_directory: str = working_directory
@@ -187,6 +189,8 @@ def __init__(self, item: configuration_support.Step,
         self._postponed_out: List[Tuple[Callable[[str], None], str]] = []
         self._needs_finalization: bool = True
 
+        self.artifacts = artifacts
+
     def prepare_command(self) -> bool:  # FIXME: refactor
         if not self.configuration.command:
             self.out.log("No 'command' found. Nothing to execute")
@@ -199,7 +203,7 @@ def prepare_command(self) -> bool:  # FIXME: refactor
                 command_name = os.path.abspath(os.path.join(self.working_directory, command_name))
                 self.cmd = make_command(command_name)
         except CiException as ex:
-            self.fail_block(str(ex))
+            self.structure.fail_block(self.current_block, str(ex))
             raise StepException() from ex
         return True
 
@@ -257,6 +261,7 @@ def finalize(self) -> None:
             if self._is_background:
                 self._is_background = False
                 self.out.log("Nothing was executed: this background step had no command")
+            self._collect_artifacts()
             return
         try:
             text = ""
@@ -275,7 +280,7 @@ def finalize(self) -> None:
                 text = utils.trim_and_convert_to_unicode(text)
                 if self.file:
                     self.file.write(text + "\n")
-                self.fail_block(text)
+                self.structure.fail_block(self.current_block, text)
                 self.add_tag(self.configuration.fail_tag)
                 raise StepException()
 
@@ -285,12 +290,19 @@ def finalize(self) -> None:
             if self.file:
                 self.file.close()
             self._is_background = False
+            self._collect_artifacts()
 
     def _handle_postponed_out(self) -> None:
         for item in self._postponed_out:
             item[0](item[1])
         self._postponed_out = []
 
+    def _collect_artifacts(self) -> None:
+        self.structure.close_block()
+        name = f"Collecting artifacts for the '{self.configuration.name}' step"
+        self.structure.open_block(name)
+        self.artifacts.collect_step_artifacts(self.configuration)
+
 
 class Launcher(ProjectDirectory, HasOutput, HasStructure, HasErrorState):
     artifacts_factory = Dependency(artifact_collector.ArtifactCollector)
@@ -402,20 +414,14 @@ def create_process(self, item: configuration_support.Step) -> RunningStep:
         working_directory = utils.parse_path(utils.strip_path_start(item.directory.rstrip("/")),
                                              self.settings.project_root)
 
-        # get_current_block() should be called while inside the required block, not afterwards
-        block = self.structure.get_current_block()
-
-        def fail_block(line: str = "") -> None:
-            self.structure.fail_block(block, line)
-
         log_file: Optional[TextIO] = None
         if self.output == "file":
             log_file = self.artifacts.create_text_file(item.name + "_log.txt")
             self.out.log("Execution log is redirected to file")
 
         additional_environment = self.api_support.get_environment_settings()
-        return RunningStep(item, self.out, fail_block, self.server.add_build_tag,
-                    log_file, working_directory, additional_environment, item.background)
+        return RunningStep(item, self.out, self.structure, self.server.add_build_tag,
+                    log_file, working_directory, additional_environment, item.background, self.artifacts)
 
     def launch_custom_configs(self, custom_configs: configuration_support.Configuration) -> None:
         self.structure.execute_step_structure(custom_configs, self.create_process)

universum/nonci.py

@@ -26,7 +26,6 @@ def execute(self):
         self.launch_project()
         self.reporter.report_initialized = True
         self.reporter.report_build_result()
-        self.artifacts.collect_artifacts()
 
     def finalize(self):
         pass