Merge pull request #10 from espressif/feat/filter-by-built-app-list

Feat/filter by built app list

docs/en/guides/local_preview.rst

@@ -42,16 +42,16 @@ To preview the behavior of the test stage locally, you can run the following com
 
 .. code:: bash
 
-   idf-ci test run --target <target> --collect-only
+   idf-ci test run --target <target> --dry-run
 
 For example, to preview the behavior of the test stage for the ``esp32`` target, you can run the following command:
 
 .. code:: bash
 
-   idf-ci test run --target esp32 --collect-only
+   idf-ci test run --target esp32 --dry-run
 
 For multi-dut tests, you can pass with comma separated values:
 
 .. code:: bash
 
-   idf-ci test run --target esp32,esp32s2 --collect-only
+   idf-ci test run --target esp32,esp32s2 --dry-run

idf_ci/cli/build.py

@@ -36,11 +36,12 @@ def run(
     """
     Run build according to the given profiles
     """
-    if not isinstance(profiles, Undefined):
-        pass
-    else:
+    if isinstance(profiles, Undefined):
         profiles = CiSettings().build_profiles
 
+    if isinstance(modified_files, Undefined):
+        modified_files = None
+
     click.echo(f'Building {target} with profiles {profiles} at {paths}')
     build_cmd(
         paths,

idf_ci/cli/test.py

@@ -27,16 +27,11 @@ def test():
 @option_profiles
 @option_parallel
 @click.option(
-    '--collected-app-info-filepath',
-    type=click.Path(dir_okay=False, file_okay=True, exists=True),
-    help='File path to the recorded app info list, generated by idf-ci build',
-)
-@click.option(
-    '--collect-only',
+    '--dry-run',
     is_flag=True,
-    help='Only collect app info and show the list of test cases to be run',
+    help='Show the test cases instead of running them',
 )
-def run(*, paths, target, profiles, parallel_count, parallel_index, collected_app_info_filepath, collect_only):
+def run(*, paths, target, profiles, parallel_count, parallel_index, dry_run):
     """
     Run tests according to the given profiles
     """
@@ -52,8 +47,7 @@ def run(*, paths, target, profiles, parallel_count, parallel_index, collected_ap
         profiles=profiles,
         parallel_count=parallel_count,
         parallel_index=parallel_index,
-        collected_app_info_filepath=collected_app_info_filepath,
-        collect_only=collect_only,
+        dry_run=dry_run,
     )
 
 

idf_ci/idf_pytest/models.py

@@ -140,28 +140,27 @@ def target_selector(self) -> str:
     def all_markers(self) -> t.Set[str]:
         return {marker.name for marker in self.item.iter_markers()}
 
-    def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t.Optional[str]:
+    def get_skip_reason_if_not_built(self, app_dirs: t.Optional[t.List[str]] = None) -> t.Optional[str]:
         """
         Check if all binaries of the test case are built in the app lists.
 
-        :param app_lists: app lists to check
+        :param app_dirs: app folder paths to check
         :return: debug string if not all binaries are built in the app lists, None otherwise
         """
-        if app_lists is None:
+        if app_dirs is None:
             # ignore this feature
             return None
 
         bin_found = [0] * len(self.apps)
         for i, app in enumerate(self.apps):
-            if app.build_dir in app_lists:
+            if app.build_dir in app_dirs:
                 bin_found[i] = 1
 
         if sum(bin_found) == 0:
             msg = f'Skip test case {self.name} because all following binaries are not listed in the app lists: '
             for app in self.apps:
                 msg += f'\n - {app.build_dir}'
 
-            print(msg)
             return msg
 
         if sum(bin_found) == len(self.apps):
@@ -174,5 +173,4 @@ def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t
                 msg += f'\n - {app.build_dir}'
 
         msg += '\nMight be a issue of .build-test-rules.yml files'
-        print(msg)
         return msg

idf_ci/idf_pytest/plugin.py

@@ -15,6 +15,7 @@
 from _pytest.fixtures import FixtureRequest
 from _pytest.python import Function
 from _pytest.stash import StashKey
+from idf_build_apps import App
 from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
 
 from .models import PytestCase
@@ -40,13 +41,15 @@ def __init__(
         *,
         cli_target: str,
         sdkconfig_name: t.Optional[str] = None,
+        apps: t.Optional[t.List[App]] = None,
     ) -> None:
         """
         :param cli_target: target passed from command line, could be single target, comma separated targets, or 'all'
         :param sdkconfig_name: run only tests whose apps are built with this sdkconfig name
         """
         self.cli_target = cli_target
         self.sdkconfig_name = sdkconfig_name
+        self.apps = apps
 
         self._testing_items: t.Set[pytest.Item] = set()
 
@@ -189,6 +192,22 @@ def pytest_collection_modifyitems(self, config: Config, items: t.List[Function])
                     res.append(item)
             items[:] = res
 
+        # filter by app list
+        if self.apps is not None:
+            app_dirs = [os.path.abspath(app.build_path) for app in self.apps]
+            res = []
+            for item in items:
+                _c = self.get_case_by_item(item)
+                if _c is None:
+                    continue
+
+                if skip_reason := _c.get_skip_reason_if_not_built(app_dirs):
+                    item.add_marker(pytest.mark.skip(reason=skip_reason))
+                    deselected_items.append(item)
+                else:
+                    res.append(item)
+            items[:] = res
+
         # deselected items should be added to config.hook.pytest_deselected
         config.hook.pytest_deselected(items=deselected_items)
 

idf_ci/scripts.py

@@ -114,6 +114,8 @@ def build(
     modified_components = None
     if modified_files is not None:
         modified_components = sorted(CiSettings().get_modified_components(modified_files))
+        LOGGER.debug('Modified files: %s', modified_files)
+        LOGGER.debug('Modified components: %s', modified_components)
 
     if test_related is False and non_test_related is False:
         # call idf-build-apps build directly
@@ -132,17 +134,11 @@ def build(
             str(parallel_index),
         ]
 
-        if modified_files is not None:
+        if modified_files:
             args.extend(
                 [
                     '--modified-files',
                     ';'.join(modified_files) if modified_files else ';',
-                ]
-            )
-
-        if modified_components is not None:
-            args.extend(
-                [
                     '--modified-components',
                     ';'.join(modified_components) if modified_components else ';',
                 ]
@@ -151,10 +147,20 @@ def build(
         if dry_run:
             args.append('--dry-run')
 
-        subprocess.run(
-            args,
-            check=True,
-        )
+        LOGGER.debug('Running command: %s', args)
+        try:
+            subprocess.run(
+                args,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            LOGGER.error(
+                'Command `%s` failed with return code %s',
+                ' '.join(args),
+                e.returncode,
+            )
+            raise SystemExit(e.returncode)
+
         return
 
     # we have to call get_all_apps first, then call build_apps(apps)
@@ -190,8 +196,7 @@ def test(
     profiles: t.List[PathLike] = UNDEF,  # type: ignore
     parallel_count: int = 1,
     parallel_index: int = 1,
-    collected_app_info_filepath: t.Optional[PathLike] = None,  # noqa # FIXME
-    collect_only: bool = False,
+    dry_run: bool = False,
 ):
     test_profile = get_test_profile(profiles)
 
@@ -207,7 +212,15 @@ def test(
         str(parallel_index),
     ]
 
-    if collect_only:
+    if dry_run:
         args.append('--collect-only')
 
-    pytest.main(args, plugins=[IdfPytestPlugin(cli_target=target)])
+    pytest.main(
+        args,
+        plugins=[
+            IdfPytestPlugin(
+                cli_target=target,
+                apps=CiSettings().get_apps_list(),
+            )
+        ],
+    )

idf_ci/settings.py

@@ -1,11 +1,12 @@
 # SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
+import logging
 import os
 import re
 import typing as t
 from pathlib import Path
 
+from idf_build_apps import App, CMakeApp, json_to_app
 from pydantic_settings import (
     BaseSettings,
     PydanticBaseSettingsSource,
@@ -14,6 +15,8 @@
 
 from idf_ci._compat import PathLike
 
+LOGGER = logging.getLogger(__name__)
+
 
 # noinspection PyDataclass
 class CiSettings(BaseSettings):
@@ -37,6 +40,8 @@ class CiSettings(BaseSettings):
     build_profiles: t.List[PathLike] = ['default']
     test_profiles: t.List[PathLike] = ['default']
 
+    built_app_list_filepatterns: t.List[str] = ['app_info_*.txt']
+
     @classmethod
     def settings_customise_sources(
         cls,
@@ -76,3 +81,20 @@ def get_modified_components(self, modified_files: t.Iterable[str]) -> t.Set[str]
                     break
 
         return modified_components
+
+    def get_apps_list(self) -> t.Optional[t.List[App]]:
+        found_files = Path('.').glob('app_info_*.txt')
+        if not found_files:
+            return None
+
+        LOGGER.debug('Found built app list files: %s', found_files)
+
+        apps: t.List[App] = []
+        for filepattern in self.built_app_list_filepatterns:
+            for filepath in Path('.').glob(filepattern):
+                with open(filepath) as fr:
+                    for line in fr:
+                        if line := line.strip():
+                            apps.append(json_to_app(line, extra_classes=[CMakeApp]))
+
+        return apps

idf_ci/templates/default_ci_profile.toml

@@ -10,3 +10,7 @@ component_ignored_file_extensions = [
     ".yml",
     ".py",
 ]
+
+built_app_list_filepatterns = [
+    "app_info_*.txt",
+]