diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..a33a6f91b9 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,9 @@ +RELEASE_TYPE: minor + +This release adds :func:`~hypothesis.register_random`, which registers +``random.Random`` instances or compatible objects to be seeded and reset +by Hypothesis to ensure that test cases are deterministic. + +We still recommend explicitly passing a ``random.Random`` instance from +:func:`~hypothesis.strategies.randoms` if possible, but registering a +framework-global state for Hypothesis to manage is better than flaky tests! diff --git a/hypothesis-python/docs/details.rst b/hypothesis-python/docs/details.rst index ff71c033ac..99f718b1aa 100644 --- a/hypothesis-python/docs/details.rst +++ b/hypothesis-python/docs/details.rst @@ -479,6 +479,22 @@ If the end user has also specified a custom executor using the be applied to the *new* inner test assigned by the test runner. +-------------------------------- +Making random code deterministic +-------------------------------- + +While Hypothesis' example generation can be used for nondeterministic tests, +debugging anything nondeterministic is usually a very frustrating excercise. +To make things worse, our example *shrinking* relies on the same input +causing the same failure each time - though we show the un-shrunk failure +and a decent error message if it doesn't. + +By default, Hypothesis will handle the global ``random`` and ``numpy.random`` +random number generators for you, and you can register others: + +.. autofunction:: hypothesis.register_random + + ------------------------------- Using Hypothesis to find values ------------------------------- diff --git a/hypothesis-python/src/hypothesis/__init__.py b/hypothesis-python/src/hypothesis/__init__.py index 65e6ad21c2..6d172ba706 100644 --- a/hypothesis-python/src/hypothesis/__init__.py +++ b/hypothesis-python/src/hypothesis/__init__.py @@ -27,6 +27,7 @@ from hypothesis.version import __version_info__, __version__ from hypothesis.control import assume, note, reject, event from hypothesis.core import given, find, example, seed, reproduce_failure, PrintSettings +from hypothesis.internal.entropy import register_random from hypothesis.utils.conventions import infer @@ -47,6 +48,7 @@ "note", "event", "infer", + "register_random", "__version__", "__version_info__", ] diff --git a/hypothesis-python/src/hypothesis/_strategies.py b/hypothesis-python/src/hypothesis/_strategies.py index 2e7ddae07a..c5f73dc305 100644 --- a/hypothesis-python/src/hypothesis/_strategies.py +++ b/hypothesis-python/src/hypothesis/_strategies.py @@ -21,7 +21,6 @@ import enum import math import operator -import random import string import sys from decimal import Context, Decimal, localcontext @@ -55,6 +54,7 @@ check_sample, integer_range, ) +from hypothesis.internal.entropy import get_seeder_and_restorer from hypothesis.internal.floats import ( count_between_floats, float_of, @@ -132,6 +132,7 @@ numpy = None if False: + import random # noqa from types import ModuleType # noqa from typing import Any, Dict, Union, Sequence, Callable, Pattern # noqa from typing import TypeVar, Tuple, List, Set, FrozenSet, overload # noqa @@ -1089,26 +1090,23 @@ def __init__(self, seed): self.seed = seed def __repr__(self): - return "random.seed(%r)" % (self.seed,) + return "RandomSeeder(%r)" % (self.seed,) class RandomModule(SearchStrategy): def do_draw(self, data): data.can_reproduce_example_from_repr = False seed = data.draw(integers(0, 2 ** 32 - 1)) - state = random.getstate() - random.seed(seed) - cleanup(lambda: random.setstate(state)) - if numpy is not None: # pragma: no cover - npstate = numpy.random.get_state() - numpy.random.seed(seed) - cleanup(lambda: numpy.random.set_state(npstate)) + seed_all, restore_all = get_seeder_and_restorer(seed) + seed_all() + cleanup(restore_all) return RandomSeeder(seed) @cacheable @defines_strategy def random_module(): + # type: () -> SearchStrategy[RandomSeeder] """The Hypothesis engine handles PRNG state for the stdlib and Numpy random modules internally, always seeding them to zero and restoring the previous state after the test. diff --git a/hypothesis-python/src/hypothesis/internal/entropy.py b/hypothesis-python/src/hypothesis/internal/entropy.py index d7762d84aa..2e1b09bf6f 100644 --- a/hypothesis-python/src/hypothesis/internal/entropy.py +++ b/hypothesis-python/src/hypothesis/internal/entropy.py @@ -20,10 +20,73 @@ import contextlib import random +from hypothesis.errors import InvalidArgument +from hypothesis.internal.compat import integer_types + +RANDOMS_TO_MANAGE = [random] # type: list + try: import numpy.random as npr except ImportError: - npr = None + pass +else: + + class NumpyRandomWrapper(object): + """A shim to remove those darn underscores.""" + + seed = npr.seed + getstate = npr.get_state + setstate = npr.set_state + + RANDOMS_TO_MANAGE.append(NumpyRandomWrapper) + + +def register_random(r): + # type: (random.Random) -> None + """Register the given Random instance for management by Hypothesis. + + You can pass ``random.Random`` instances (or other objects with seed, + getstate, and setstate methods) to ``register_random(r)`` to have their + states seeded and restored in the same way as the global PRNGs from the + ``random`` and ``numpy.random`` modules. + + All global PRNGs, from e.g. simulation or scheduling frameworks, should + be registered to prevent flaky tests. Hypothesis will ensure that the + PRNG state is consistent for all test runs, or reproducibly varied if you + choose to use the :func:`~hypothesis.strategies.random_module` strategy. + """ + if not (hasattr(r, "seed") and hasattr(r, "getstate") and hasattr(r, "setstate")): + raise InvalidArgument("r=%r does not have all the required methods" % (r,)) + if r not in RANDOMS_TO_MANAGE: + RANDOMS_TO_MANAGE.append(r) + + +def get_seeder_and_restorer(seed=0): + """Return a pair of functions which respectively seed all and restore + the state of all registered PRNGs. + + This is used by the core engine via `deterministic_PRNG`, and by users + via `register_random`. We support registration of additional random.Random + instances (or other objects with seed, getstate, and setstate methods) + to force determinism on simulation or scheduling frameworks which avoid + using the global random state. See e.g. #1709. + """ + assert isinstance(seed, integer_types) and 0 <= seed < 2 ** 32 + states = [] # type: list + + def seed_all(): + assert not states + for r in RANDOMS_TO_MANAGE: + states.append(r.getstate()) + r.seed(seed) + + def restore_all(): + assert len(states) == len(RANDOMS_TO_MANAGE) + for r, state in zip(RANDOMS_TO_MANAGE, states): + r.setstate(state) + del states[:] + + return seed_all, restore_all @contextlib.contextmanager @@ -35,15 +98,9 @@ def deterministic_PRNG(): bad idea in principle, and breaks all kinds of independence assumptions in practice. """ - _random_state = random.getstate() - random.seed(0) - # These branches are covered by tests/numpy/, not tests/cover/ - if npr is not None: # pragma: no cover - _npr_state = npr.get_state() - npr.seed(0) + seed_all, restore_all = get_seeder_and_restorer() + seed_all() try: yield finally: - random.setstate(_random_state) - if npr is not None: # pragma: no cover - npr.set_state(_npr_state) + restore_all() diff --git a/hypothesis-python/tests/cover/test_random_module.py b/hypothesis-python/tests/cover/test_random_module.py index 7e628a46c0..97b262ccf0 100644 --- a/hypothesis-python/tests/cover/test_random_module.py +++ b/hypothesis-python/tests/cover/test_random_module.py @@ -22,7 +22,9 @@ import pytest import hypothesis.strategies as st -from hypothesis import given, reporting +from hypothesis import given, register_random, reporting +from hypothesis.errors import InvalidArgument +from hypothesis.internal import entropy from tests.common.utils import capture_out @@ -36,7 +38,7 @@ def test(r): assert False test() - assert "random.seed(0)" in out.getvalue() + assert "RandomSeeder(0)" in out.getvalue() @given(st.random_module(), st.random_module()) @@ -47,3 +49,57 @@ def test_seed_random_twice(r, r2): @given(st.random_module()) def test_does_not_fail_health_check_if_randomness_is_used(r): random.getrandbits(128) + + +def test_cannot_register_non_Random(): + with pytest.raises(InvalidArgument): + register_random("not a Random instance") + + +def test_registering_a_Random_is_idempotent(): + r = random.Random() + register_random(r) + register_random(r) + assert entropy.RANDOMS_TO_MANAGE.pop() is r + assert r not in entropy.RANDOMS_TO_MANAGE + + +def test_manages_registered_Random_instance(): + r = random.Random() + register_random(r) + state = r.getstate() + result = [] + + @given(st.integers()) + def inner(x): + v = r.random() + if result: + assert v == result[0] + else: + result.append(v) + + inner() + assert state == r.getstate() + + entropy.RANDOMS_TO_MANAGE.remove(r) + assert r not in entropy.RANDOMS_TO_MANAGE + + +def test_registered_Random_is_seeded_by_random_module_strategy(): + r = random.Random() + register_random(r) + state = r.getstate() + results = set() + count = [0] + + @given(st.integers()) + def inner(x): + results.add(r.random()) + count[0] += 1 + + inner() + assert count[0] > len(results) * 0.9, "too few unique random numbers" + assert state == r.getstate() + + entropy.RANDOMS_TO_MANAGE.remove(r) + assert r not in entropy.RANDOMS_TO_MANAGE