Merge pull request #1083 from ClearcodeHQ/issue-1081

Run xdist tests runs with -n auto flag

.github/workflows/dockerised-postgres.yml

@@ -57,7 +57,7 @@ jobs:
         uses: fizyk/actions-reuse/.github/actions/[email protected]
         with:
           python-version: ${{ matrix.python-version }}
-          command: pytest -n 0 -k docker --postgresql-host=localhost --postgresql-port 5433 --postgresql-password=postgres --cov-report=xml:coverage-docker.xml
+          command: pytest -n 0 --max-worker-restart 0 -k docker --postgresql-host=localhost --postgresql-port 5433 --postgresql-password=postgres --cov-report=xml:coverage-docker.xml
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]
         with:

.github/workflows/single-postgres.yml

@@ -56,12 +56,12 @@ jobs:
         uses: fizyk/actions-reuse/.github/actions/[email protected]
         with:
           python-version: ${{ matrix.python-version }}
-          command: py.test -svv -n 0 --postgresql-exec="/usr/lib/postgresql/${{ inputs.postgresql }}/bin/pg_ctl" -k "not docker" --cov-report=xml
+          command: py.test -svv -p no:xdist --postgresql-exec="/usr/lib/postgresql/${{ inputs.postgresql }}/bin/pg_ctl" -k "not docker" --cov-report=xml
       - name: Run xdist test
         uses: fizyk/actions-reuse/.github/actions/[email protected]
         with:
           python-version: ${{ matrix.python-version }}
-          command: py.test -n 1 --postgresql-exec="/usr/lib/postgresql/${{ inputs.postgresql }}/bin/pg_ctl" -k "not docker" --cov-report=xml:coverage-xdist.xml
+          command: py.test -n auto --dist loadgroup --max-worker-restart 0 --postgresql-exec="/usr/lib/postgresql/${{ inputs.postgresql }}/bin/pg_ctl" -k "not docker" --cov-report=xml:coverage-xdist.xml
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

README.rst

@@ -185,6 +185,12 @@ You can pick which you prefer, but remember that these settings are handled in t
      - postgresql_port
      - yes (5432)
      - random
+   * - Port search count
+     -
+     - --postgresql-port-search-count
+     - postgresql_port_search_count
+     - -
+     - 5
    * - postgresql user
      - user
      - --postgresql-user

newsfragments/1081.misc.rst

@@ -0,0 +1 @@
+Run xdist test with -n auto, turn off xdist for xdist-less runs

newsfragments/872.feature.rst

@@ -0,0 +1,8 @@
+When running tests with xdist, pytest-postgresql now attempts to detect random ports
+selected by other nodes by writing down a .port file in session temporary directory.
+
+The number of tries it attempts to select unused port is configurable,
+and defaults to 0.
+
+In case pytest-postgresql won't be able to select unused port,
+PortForException is thrown with appropriate message.

pyproject.toml

@@ -58,7 +58,7 @@ namespaces = false
 
 [tool.pytest.ini_options]
 xfail_strict=true
-addopts = "--max-worker-restart=0 --showlocals --verbose --cov"
+addopts = "--showlocals --verbose --cov"
 testpaths = "tests"
 pytester_example_dir = "tests/examples"
 norecursedirs = "examples"

pytest_postgresql/config.py

@@ -12,6 +12,7 @@ class PostgresqlConfigDict(TypedDict):
     exec: str
     host: str
     port: Optional[str]
+    port_search_count: int
     user: str
     password: str
     options: str
@@ -35,6 +36,7 @@ def get_postgresql_option(option: str) -> Any:
         exec=get_postgresql_option("exec"),
         host=get_postgresql_option("host"),
         port=get_postgresql_option("port"),
+        port_search_count=get_postgresql_option("port_search_count"),
         user=get_postgresql_option("user"),
         password=get_postgresql_option("password"),
         options=get_postgresql_option("options"),

pytest_postgresql/factories/process.py

@@ -21,11 +21,11 @@
 import platform
 import subprocess
 from pathlib import Path
-from typing import Callable, Iterator, List, Optional, Tuple, Union
+from typing import Callable, Iterable, Iterator, List, Optional, Tuple, Union
 
 import port_for
 import pytest
-from port_for import get_port
+from port_for import PortForException, get_port
 from pytest import FixtureRequest, TempPathFactory
 
 from pytest_postgresql.config import PostgresqlConfigDict, get_config
@@ -54,9 +54,11 @@ def _pg_exe(executable: Optional[str], config: PostgresqlConfigDict) -> str:
     return postgresql_ctl
 
 
-def _pg_port(port: Optional[PortType], config: PostgresqlConfigDict) -> int:
+def _pg_port(
+    port: Optional[PortType], config: PostgresqlConfigDict, excluded_ports: Iterable[int]
+) -> int:
     """User specified port, otherwise find an unused port from config."""
-    pg_port = get_port(port) or get_port(config["port"])
+    pg_port = get_port(port, excluded_ports) or get_port(config["port"], excluded_ports)
     assert pg_port is not None
     return pg_port
 
@@ -122,7 +124,32 @@ def postgresql_proc_fixture(
         pg_dbname = dbname or config["dbname"]
         pg_load = load or config["load"]
         postgresql_ctl = _pg_exe(executable, config)
-        pg_port = _pg_port(port, config)
+        port_path = tmp_path_factory.getbasetemp()
+        if hasattr(request.config, "workerinput"):
+            port_path = tmp_path_factory.getbasetemp().parent
+
+        n = 0
+        used_ports: set[int] = set()
+        while True:
+            try:
+                pg_port = _pg_port(port, config, used_ports)
+                if pg_port in used_ports:
+                    raise PortForException(
+                        f"Port {pg_port} already in use, probably by other instances of the test."
+                    )
+                used_ports.add(pg_port)
+                with (port_path / f"postgresql-{pg_port}.port").open("x") as port_file:
+                    port_file.write(f"pg_port {pg_port}\n")
+                break
+            except FileExistsError:
+                if n >= config["port_search_count"]:
+                    raise PortForException(
+                        f"Attempted {n} times to select ports. "
+                        f"All attempted ports: {', '.join(map(str, used_ports))} are already "
+                        f"in use, probably by other instances of the test."
+                    )
+                n += 1
+
         tmpdir = tmp_path_factory.mktemp(f"pytest-postgresql-{request.fixturename}")
         datadir, logfile_path = _prepare_dir(tmpdir, str(pg_port))
 

pytest_postgresql/plugin.py

@@ -25,6 +25,7 @@
 _help_executable = "Path to PostgreSQL executable"
 _help_host = "Host at which PostgreSQL will accept connections"
 _help_port = "Port at which PostgreSQL will accept connections"
+_help_port_search_count = "Number of times, pytest-postgresql will search for free port"
 _help_user = "PostgreSQL username"
 _help_password = "PostgreSQL password"
 _help_options = "PostgreSQL connection options"
@@ -48,6 +49,7 @@ def pytest_addoption(parser: Parser) -> None:
         help=_help_port,
         default=None,
     )
+    parser.addini(name="postgresql_port_search_count", help=_help_port_search_count, default=5)
 
     parser.addini(name="postgresql_user", help=_help_user, default="postgres")
 
@@ -80,6 +82,13 @@ def pytest_addoption(parser: Parser) -> None:
     )
 
     parser.addoption("--postgresql-port", action="store", dest="postgresql_port", help=_help_port)
+    parser.addoption(
+        "--postgresql-port-search-count",
+        action="store",
+        dest="postgresql_port_search_count",
+        help=_help_port_search_count,
+        default=5,
+    )
 
     parser.addoption("--postgresql-user", action="store", dest="postgresql_user", help=_help_user)
 

tests/test_executor.py

@@ -79,7 +79,7 @@ def test_executor_init_with_password(
     config = get_config(request)
     monkeypatch.setenv("LC_ALL", locale)
     pg_exe = process._pg_exe(None, config)
-    port = process._pg_port(-1, config)
+    port = process._pg_port(-1, config, [])
     tmpdir = tmp_path_factory.mktemp(f"pytest-postgresql-{request.node.name}")
     datadir, logfile_path = process._prepare_dir(tmpdir, port)
     executor = PostgreSQLExecutor(
@@ -103,7 +103,7 @@ def test_executor_init_bad_tmp_path(
     r"""Test init with \ and space chars in the path."""
     config = get_config(request)
     pg_exe = process._pg_exe(None, config)
-    port = process._pg_port(-1, config)
+    port = process._pg_port(-1, config, [])
     tmpdir = tmp_path_factory.mktemp(f"pytest-postgresql-{request.node.name}") / r"a bad\path/"
     tmpdir.mkdir(exist_ok=True)
     datadir, logfile_path = process._prepare_dir(tmpdir, port)

tests/test_template_database.py

@@ -18,6 +18,7 @@
 )
 
 
+@pytest.mark.xdist_group(name="template_database")
 @pytest.mark.parametrize("_", range(5))
 def test_template_database(postgresql_template: Connection, _: int) -> None:
     """Check that the database structure gets recreated out of a template."""