Support session dependencies

nox/_decorators.py

@@ -62,6 +62,7 @@ def __init__(
         venv_params: Any = None,
         should_warn: dict[str, Any] | None = None,
         tags: list[str] | None = None,
+        requires: list[str] | None = None,
     ):
         self.func = func
         self.python = python
@@ -70,6 +71,17 @@ def __init__(
         self.venv_params = venv_params
         self.should_warn = should_warn or dict()
         self.tags = tags or []
+        self.requires = requires or []
+
+    @property
+    def requires(self) -> list[str]:
+        # Compute dynamically on lookup since ``self.python`` can be modified after
+        # creation (e.g. on an instance from ``self.copy``).
+        return list(map(self.format_dependency, self._requires))
+
+    @requires.setter
+    def requires(self, value: list[str]) -> None:
+        self._requires = value
 
     def __call__(self, *args: Any, **kwargs: Any) -> Any:
         return self.func(*args, **kwargs)
@@ -84,8 +96,14 @@ def copy(self, name: str | None = None) -> Func:
             self.venv_params,
             self.should_warn,
             self.tags,
+            self.requires,
         )
 
+    def format_dependency(self, dependency: str) -> str:
+        if isinstance(self.python, str):
+            return dependency.format(python=self.python, py=self.python)
+        return dependency
+
 
 class Call(Func):
     def __init__(self, func: Func, param_spec: Param) -> None:
@@ -113,6 +131,7 @@ def __init__(self, func: Func, param_spec: Param) -> None:
             func.venv_params,
             func.should_warn,
             func.tags,
+            func.requires,
         )
         self.call_spec = call_spec
         self.session_signature = session_signature

nox/_resolver.py

@@ -0,0 +1,207 @@
+# Copyright 2022 Alethea Katherine Flowers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import itertools
+from collections import OrderedDict
+from typing import Hashable, Iterable, Iterator, Mapping, TypeVar
+
+Node = TypeVar("Node", bound=Hashable)
+
+
+class CycleError(ValueError):
+    """An exception indicating that a cycle was encountered in a graph."""
+
+    pass
+
+
+def lazy_stable_topo_sort(
+    dependencies: Mapping[Node, Iterable[Node]],
+    root: Node,
+    drop_root: bool = True,
+) -> Iterator[Node]:
+    """Returns the "lazy, stable" topological sort of a dependency graph.
+
+    The sort returned will be a topological sort of the subgraph containing only
+    ``root`` and its (recursive) dependencies. ``root`` will not be included in the
+    output sort if ``drop_root`` is ``True``.
+
+    The sort returned is "lazy" in the sense that a node will not appear any earlier in
+    the output sort than is necessitated by its dependents.
+
+    The sort returned is "stable" in the sense that the relative order of two nodes in
+    ``dependencies[node]`` is preserved in the output sort, except when doing so would
+    prevent the output sort from being either topological or lazy. The order of nodes in
+    ``dependencies[node]`` allows the caller to exert a preference on the order of the
+    output sort.
+
+    For example, consider:
+
+    >>> list(
+    ...     lazy_stable_topo_sort(
+    ...         dependencies = {
+    ...             "a": ["c", "b"],
+    ...             "b": [],
+    ...             "c": [],
+    ...             "d": ["e"],
+    ...             "e": ["c"],
+    ...             "root": ["a", "d"],
+    ...         },
+    ...         "root",
+    ...         drop_root=False,
+    ...     )
+    ... )
+    ["c", "b", "a", "e", "d", "root"]
+
+    Notice that:
+
+        1.  This is a topological sort of the dependency graph. That is, nodes only
+            occur in the sort after all of their dependencies occur.
+
+        2.  Had we also included a node ``"f": ["b"]`` but kept ``dependencies["root"]``
+            the same, the output would not have changed. This is because ``"f"`` was not
+            requested directly by including it in ``dependencies["root"]`` or
+            transitively as a (recursive) dependency of a node in
+            ``dependencies["root"]``.
+
+        3.  ``"e"`` occurs no earlier than was required by its dependents ``{"d"}``.
+            This is an example of the sort being "lazy". If ``"e"`` had occurred in the
+            output any earlier---for example, just before ``"a"``---the sort would not
+            have been lazy, but (in this example) the output would still have been a
+            topological sort.
+
+        4.  Because the topological order between ``"a"`` and ``"d"`` is undefined and
+            because it is possible to do so without making the output sort non-lazy,
+            ``"a"`` and ``"d"`` are kept in the relative order that they have in
+            ``dependencies["root"]``. This is an example of the sort being stable
+            between pairs in ``dependencies[node]`` whenever possible. If ``"a"``'s
+            dependency list was instead ``["d"]``, however, the relative order between
+            ``"a"`` and ``"d"`` in ``dependencies["root"]`` would have been ignored to
+            satisfy this dependency.
+
+            Similarly, ``"b"`` and ``"c"`` are kept in the relative order that they have
+            in ``dependencies["a"]``. If ``"c"``'s dependency list was instead
+            ``["b"]``, however, the relative order between ``"b"`` and ``"c"`` in
+            ``dependencies["a"]`` would have been ignored to satisfy this dependency.
+
+    This implementation of this function is recursive and thus should not be used on
+    large dependency graphs, but it is suitable for noxfile-sized dependency graphs.
+
+    Args:
+        dependencies (Mapping[~nox._resolver.Node, Iterable[~nox._resolver.Node]]):
+            A mapping from each node in the graph to the (ordered) list of nodes that it
+            depends on. Using a mapping type with O(1) lookup (e.g. `dict`) is strongly
+            suggested.
+        root (~nox._resolver.Node):
+            The root node to start the sort at. If ``drop_root`` is not ``True``,
+            ``root`` will be the last element of the output.
+        drop_root (bool):
+            If ``True``, ``root`` will be not be included in the output sort. Defaults
+            to ``True``.
+
+
+    Returns:
+        Iterator[~nox._resolver.Node]: The "lazy, stable" topological sort of the
+        subgraph containing ``root`` and its dependencies.
+
+    Raises:
+        ~nox._resolver.CycleError: If a dependency cycle is encountered.
+    """
+
+    visited = {node: False for node in dependencies}
+
+    def prepended_by_dependencies(
+        node: Node,
+        walk: OrderedDict[Node, None] | None = None,
+    ) -> Iterator[Node]:
+        """Yields a node's dependencies depth-first, followed by the node itself.
+
+        A dependency will be skipped if has already been yielded by another call of
+        ``prepended_by_dependencies``. Since ``prepended_by_dependencies`` is recursive,
+        this means that each node will only be yielded once, and only the deepest
+        occurrence of a node will be yielded.
+
+        Args:
+            node (~nox._resolver.Node):
+                A node in the dependency graph.
+            walk (OrderedDict[~nox._resolver.Node, None] | None):
+                An ``OrderedDict`` whose keys are the nodes traversed when walking a
+                path leading up to ``node`` on the reversed-edge dependency graph.
+                Defaults to ``OrderedDict()``.
+
+        Yields:
+            ~nox._resolver.Node: ``node``'s direct dependencies, each
+            prepended by their own direct dependencies, and so forth recursively,
+            depth-first, followed by ``node``.
+
+        Raises:
+            ValueError: If a dependency cycle is encountered.
+        """
+        nonlocal visited
+        # We would like for ``walk`` to be an ordered set so that we get (a) O(1) ``node
+        # in walk`` and (b) so that we can use the order to report to the user what the
+        # dependency cycle is, if one is encountered. The standard library does not have
+        # an ordered set type, so we instead use the keys of an ``OrderedDict[Node,
+        # None]`` as an ordered set.
+        walk = walk or OrderedDict()
+        walk = extend_walk(walk, node)
+        if not visited[node]:
+            visited[node] = True
+            # Recurse for each node in dependencies[node] in order so that we adhere to
+            # the ``dependencies[node]`` order preference if doing so is possible.
+            yield from itertools.chain.from_iterable(
+                prepended_by_dependencies(dependency, walk)
+                for dependency in dependencies[node]
+            )
+            yield node
+        else:
+            return
+
+    def extend_walk(
+        walk: OrderedDict[Node, None], node: Node
+    ) -> OrderedDict[Node, None]:
+        """Extend a walk by a node, checking for dependency cycles.
+
+        Args:
+            walk (OrderedDict[~nox._resolver.Node, None]):
+                See ``prepended_by_dependencies``.
+            nodes (~nox._resolver.Node):
+                A node to extend the walk with.
+
+        Returns:
+            OrderedDict[~nox._resolver.Node, None]: ``walk``, extended by
+            ``node``.
+
+        Raises:
+            ValueError: If extending ``walk`` by ``node`` introduces a cycle into the
+                represented walk on the dependency graph.
+        """
+        walk = walk.copy()
+        if node in walk:
+            # Dependency cycle found.
+            walk_list = list(walk)
+            cycle = walk_list[walk_list.index(node) :] + [node]
+            raise CycleError("Nodes are in a dependency cycle", cycle)
+        else:
+            walk[node] = None
+        return walk
+
+    sort = prepended_by_dependencies(root)
+    if drop_root:
+        return filter(
+            lambda node: not (node == root and hash(node) == hash(root)), sort
+        )
+    else:
+        return sort

nox/manifest.py

@@ -17,10 +17,12 @@
 import argparse
 import ast
 import itertools
+import operator
 from collections import OrderedDict
-from typing import Any, Iterable, Iterator, Mapping, Sequence
+from typing import Any, Iterable, Iterator, Mapping, Sequence, cast
 
 from nox._decorators import Call, Func
+from nox._resolver import CycleError, lazy_stable_topo_sort
 from nox.sessions import Session, SessionRunner
 
 WARN_PYTHONS_IGNORED = "python_ignored"
@@ -103,6 +105,32 @@ def list_all_sessions(self) -> Iterator[tuple[SessionRunner, bool]]:
         for session in self._all_sessions:
             yield session, session in self._queue
 
+    @property
+    def all_sessions_by_signature(self) -> dict[str, SessionRunner]:
+        return {
+            signature: session
+            for session in self._all_sessions
+            for signature in session.signatures
+        }
+
+    @property
+    def parametrized_sessions_by_name(self) -> dict[str, list[SessionRunner]]:
+        """Returns a mapping from names to all sessions that are parameterizations of
+        the ``@session`` with each name.
+
+        The sessions in each returned list will occur in the same order as they occur in
+        ``self._all_sessions``.
+        """
+        parametrized_sessions = filter(operator.attrgetter("multi"), self._all_sessions)
+        key = operator.attrgetter("name")
+        # Note that ``sorted`` uses a stable sorting algorithm.
+        return {
+            name: list(sessions_parametrizing_name)
+            for name, sessions_parametrizing_name in itertools.groupby(
+                sorted(parametrized_sessions, key=key), key
+            )
+        }
+
     def add_session(self, session: SessionRunner) -> None:
         """Add the given session to the manifest.
 
@@ -186,6 +214,57 @@ def filter_by_tags(self, tags: list[str]) -> None:
         """
         self._queue = [x for x in self._queue if set(x.tags).intersection(tags)]
 
+    def add_dependencies(self) -> None:
+        """Add direct and recursive dependencies to the queue.
+
+        Raises:
+            KeyError: If any depended-on sessions are not found.
+            ~nox._resolver.CycleError: If a dependency cycle is encountered.
+        """
+        sessions_by_id = self.all_sessions_by_signature
+        # For each session that was parametrized from a list of Pythons, create a fake
+        # parent session that depends on it.
+        parent_sessions: set[SessionRunner] = set()
+        for (
+            parent_name,
+            parametrized_sessions,
+        ) in self.parametrized_sessions_by_name.items():
+            parent_func = _null_session_func.copy()
+            parent_func.requires = list(
+                map(lambda session: session.signatures[0], parametrized_sessions)
+            )
+            parent_session = SessionRunner(
+                parent_name, [], parent_func, self._config, self, False
+            )
+            parent_sessions.add(parent_session)
+            sessions_by_id[parent_name] = parent_session
+
+        # Construct the dependency graph.
+        try:
+            dependency_graph = {
+                session: session.get_direct_dependencies(sessions_by_id)
+                for session in sessions_by_id.values()
+            }
+        except KeyError as exc:
+            raise KeyError(f"Session not found: {exc.args[0]}") from exc
+
+        # Resolve the dependency graph.
+        root = cast(SessionRunner, object())  # sentinel
+        try:
+            resolved_graph = list(
+                lazy_stable_topo_sort({**dependency_graph, root: self._queue}, root)
+            )
+        except CycleError as exc:
+            raise CycleError(
+                "Sessions are in a dependency cycle: "
+                + " -> ".join(session.name for session in exc.args[1])
+            ) from exc
+
+        # Remove fake parent sessions from the resolved graph.
+        self._queue = [
+            session for session in resolved_graph if session not in parent_sessions
+        ]
+
     def make_session(
         self, name: str, func: Func, multi: bool = False
     ) -> list[SessionRunner]:
@@ -250,7 +329,7 @@ def make_session(
             if func.python:
                 long_names.append(f"{name}-{func.python}")
 
-            return [SessionRunner(name, long_names, func, self._config, self)]
+            return [SessionRunner(name, long_names, func, self._config, self, multi)]
 
         # Since this function is parametrized, we need to add a distinct
         # session for each permutation.
@@ -265,13 +344,15 @@ def make_session(
                 # Ensure that specifying session-python will run all parameterizations.
                 long_names.append(f"{name}-{func.python}")
 
-            sessions.append(SessionRunner(name, long_names, call, self._config, self))
+            sessions.append(
+                SessionRunner(name, long_names, call, self._config, self, multi)
+            )
 
         # Edge case: If the parameters made it such that there were no valid
         # calls, add an empty, do-nothing session.
         if not calls:
             sessions.append(
-                SessionRunner(name, [], _null_session_func, self._config, self)
+                SessionRunner(name, [], _null_session_func, self._config, self, multi)
             )
 
         # Return the list of sessions.