From f46fe5f04bd0340b0c670ee00f00d2da109bbb43 Mon Sep 17 00:00:00 2001 From: tuturu-tech Date: Thu, 28 Mar 2024 14:49:15 +0100 Subject: [PATCH] simplify tests --- tests/conftest.py | 37 ++++++++- tests/test_types_echidna.py | 147 +++--------------------------------- tests/test_types_medusa.py | 147 +++--------------------------------- 3 files changed, 59 insertions(+), 272 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fd6554e..0de60cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ """ Globally available fixtures""" import os -from typing import Any +import subprocess +import re +from typing import Any, Callable import pytest from slither import Slither @@ -99,3 +101,36 @@ def value_transfer() -> TestGenerator: corpus_dir = "corpus-value" return TestGenerator(target, target_path, corpus_dir) + + +def run_generation_command_test( + generate_tests: Callable, test_name: str, fuzzer: str, pattern: str +) -> None: + """Utility function to test unit test generation from a corpus and contract""" + generate_tests() + # Ensure the file was created + path = os.path.join(os.getcwd(), "test", f"{test_name}_{fuzzer}_Test.t.sol") + assert os.path.exists(path) + + # Ensure the file can be compiled + subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) + + # Ensure the file can be tested + result = subprocess.run( + ["forge", "test", "--match-contract", f"{test_name}_{fuzzer}_Test"], + capture_output=True, + text=True, + check=False, + ) + + # Remove ansi escape sequences + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + output = ansi_escape.sub("", result.stdout) + + # Ensure all tests fail + match = re.search(pattern, output) + if match: + tests_passed = int(match.group(2)) + assert tests_passed == 0 + else: + assert False, "No tests were ran" diff --git a/tests/test_types_echidna.py b/tests/test_types_echidna.py index 1e46b35..955a13b 100644 --- a/tests/test_types_echidna.py +++ b/tests/test_types_echidna.py @@ -1,9 +1,6 @@ """ Tests for generating compilable test files from an Echidna corpus""" from pathlib import Path -import os -import re -import subprocess -from .conftest import TestGenerator +from .conftest import TestGenerator, run_generation_command_test TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" @@ -11,154 +8,34 @@ def test_echidna_basic_types(basic_types: TestGenerator) -> None: """Tests the BasicTypes contract with an Echidna corpus""" - basic_types.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "BasicTypes_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "BasicTypes_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + basic_types.echidna_generate_tests, "BasicTypes", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: """Tests the FixedArrays contract with an Echidna corpus""" - fixed_size_arrays.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "FixedArrays_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "FixedArrays_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + fixed_size_arrays.echidna_generate_tests, "FixedArrays", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: """Tests the DynamicArrays contract with an Echidna corpus""" - dynamic_arrays.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "DynamicArrays_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "DynamicArrays_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + dynamic_arrays.echidna_generate_tests, "DynamicArrays", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_structs_and_enums(structs_and_enums: TestGenerator) -> None: """Tests the TupleTypes contract with an Echidna corpus""" - structs_and_enums.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "TupleTypes_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "TupleTypes_Echidna_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + structs_and_enums.echidna_generate_tests, "TupleTypes", "Echidna", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_echidna_value_transfer(value_transfer: TestGenerator) -> None: - """Tests the BasicTypes contract with an Echidna corpus""" - value_transfer.echidna_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "ValueTransfer_Echidna_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "ValueTransfer_Echidna_Test"], - capture_output=True, - text=True, - check=False, + """Tests the ValueTransfer contract with an Echidna corpus""" + run_generation_command_test( + value_transfer.echidna_generate_tests, "ValueTransfer", "Echidna", PATTERN ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" diff --git a/tests/test_types_medusa.py b/tests/test_types_medusa.py index 35ae84c..e3e9a71 100644 --- a/tests/test_types_medusa.py +++ b/tests/test_types_medusa.py @@ -1,10 +1,7 @@ """ Tests for generating compilable test files from an Medusa corpus""" from pathlib import Path -import os -import re -import subprocess import pytest -from .conftest import TestGenerator +from .conftest import TestGenerator, run_generation_command_test TEST_DATA_DIR = Path(__file__).resolve().parent / "test_data" PATTERN = r"(\d+)\s+failing tests,\s+(\d+)\s+tests succeeded" @@ -12,155 +9,33 @@ def test_medusa_basic_types(basic_types: TestGenerator) -> None: """Tests the BasicTypes contract with a Medusa corpus""" - basic_types.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "BasicTypes_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "BasicTypes_Medusa_Test"], - capture_output=True, - text=True, - check=False, - ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" + run_generation_command_test(basic_types.medusa_generate_tests, "BasicTypes", "Medusa", PATTERN) def test_medusa_fixed_array_types(fixed_size_arrays: TestGenerator) -> None: """Tests the FixedArrays contract with a Medusa corpus""" - fixed_size_arrays.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "FixedArrays_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "FixedArrays_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + fixed_size_arrays.medusa_generate_tests, "FixedArrays", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_medusa_dynamic_array_types(dynamic_arrays: TestGenerator) -> None: """Tests the DynamicArrays contract with a Medusa corpus""" - dynamic_arrays.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "DynamicArrays_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "DynamicArrays_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + dynamic_arrays.medusa_generate_tests, "DynamicArrays", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - def test_medusa_structs_and_enums(structs_and_enums: TestGenerator) -> None: """Tests the TupleTypes contract with a Medusa corpus""" - structs_and_enums.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "TupleTypes_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "TupleTypes_Medusa_Test"], - capture_output=True, - text=True, - check=False, + run_generation_command_test( + structs_and_enums.medusa_generate_tests, "TupleTypes", "Medusa", PATTERN ) - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran" - @pytest.mark.xfail(strict=True) # type: ignore[misc] def test_medusa_value_transfer(value_transfer: TestGenerator) -> None: - """Tests the BasicTypes contract with a Medusa corpus""" - value_transfer.medusa_generate_tests() - # Ensure the file was created - path = os.path.join(os.getcwd(), "test", "ValueTransfer_Medusa_Test.t.sol") - assert os.path.exists(path) - - # Ensure the file can be compiled - subprocess.run(["forge", "build", "--build-info"], capture_output=True, text=True, check=True) - - # Ensure the file can be tested - result = subprocess.run( - ["forge", "test", "--match-contract", "ValueTransfer_Medusa_Test"], - capture_output=True, - text=True, - check=False, + """Tests the ValueTransfer contract with a Medusa corpus""" + run_generation_command_test( + value_transfer.medusa_generate_tests, "ValueTransfer", "Medusa", PATTERN ) - - # Remove ansi escape sequences - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - output = ansi_escape.sub("", result.stdout) - - # Ensure all tests fail - match = re.search(PATTERN, output) - if match: - tests_passed = int(match.group(2)) - assert tests_passed == 0 - else: - assert False, "No tests were ran"