diff --git a/bowtie/_benchmarks.py b/bowtie/_benchmarks.py index b419877bb..b23915d20 100644 --- a/bowtie/_benchmarks.py +++ b/bowtie/_benchmarks.py @@ -64,28 +64,28 @@ ) -def get_benchmark_filenames( +def get_benchmark_files( benchmark_type: str, - benchmarks: Iterable[str], dialect: Dialect, -): - bowtie_dir = Path(__file__).parent - search_dir: Path | None = None - files = [] - - if benchmark_type == "keyword": - keywords_benchmark_dir = bowtie_dir / "benchmarks/keywords" - search_dir = keywords_benchmark_dir / dialect.short_name - - elif benchmark_type == "default": - search_dir = bowtie_dir / "benchmarks" - - if search_dir and search_dir.exists(): - files = [ - str(file) - for file in search_dir.iterdir() - if file.suffix in (".json", ".py") and file.name != "__init__.py" - ] + benchmarks: Iterable[str] = [], +) -> list[Path]: + bowtie_parent_dir = Path(__file__).parent.parent + benchmarks_dir = bowtie_parent_dir / "bowtie/benchmarks" + files: list[Path] = [] + + for file in benchmarks_dir.rglob("*"): + relative_folder_path = file.parent.relative_to(bowtie_parent_dir) + module_name = str(relative_folder_path).replace(os.sep, ".") + benchmark_group = _load_benchmark_group_from_file( + file, + module_name, + ) + if ( + benchmark_group is not None + and benchmark_group.benchmark_type == benchmark_type + and dialect in benchmark_group.dialects_supported + ): + files.append(file) if benchmarks: files = [ @@ -93,15 +93,13 @@ def get_benchmark_filenames( for benchmark in benchmarks if ( matched_file := next( - (file for file in files if benchmark in file), + (file for file in files if benchmark in str(file)), None, ) ) ] - for file in files: - STDOUT.file.write(file) - STDOUT.file.write("\n") + return files def _load_benchmark_group_from_file( @@ -251,6 +249,9 @@ class BenchmarkGroup: benchmarks: Sequence[Benchmark] description: str uri: URL | None + benchmark_type: str + dialects_supported: Sequence[Dialect] = list(Dialect.known()) + varying_parameter: str | None = None @classmethod def from_folder( @@ -281,6 +282,17 @@ def from_dict( data: dict[str, Any], uri: URL | None = None, ) -> BenchmarkGroup: + varying_parameter = data.pop("varying_parameter", None) + benchmark_type = data.pop("benchmark_type", "None") + dialects_supported = [ + ( + dialect + if isinstance(dialect, Dialect) + else Dialect.from_str(dialect) + ) + for dialect in data.pop("dialects_supported", Dialect.known()) + ] + if "benchmarks" not in data: benchmark = Benchmark.from_dict( **data, @@ -290,6 +302,9 @@ def from_dict( description=benchmark.description, benchmarks=[benchmark], uri=uri, + benchmark_type=benchmark_type, + dialects_supported=dialects_supported, + varying_parameter=varying_parameter, ) benchmarks = [ @@ -303,6 +318,9 @@ def from_dict( description=data["description"], benchmarks=benchmarks, uri=uri, + benchmark_type=benchmark_type, + varying_parameter=varying_parameter, + dialects_supported=dialects_supported, ) def serializable(self) -> Message: @@ -312,14 +330,19 @@ def serializable(self) -> Message: ) if "uri" in serialized_dict: serialized_dict["uri"] = str(serialized_dict["uri"]) + serialized_dict["dialects_supported"] = [ + dialect.serializable() for dialect in self.dialects_supported + ] return serialized_dict @frozen class BenchmarkGroupResult: name: str + benchmark_type: str description: str benchmark_results: list[BenchmarkResult] + varying_parameter: str | None = None def serializable(self): return asdict(self) @@ -591,8 +614,10 @@ def benchmark_finished(): def benchmark_group_finished(): benchmark_group_result = BenchmarkGroupResult( name=benchmark_group.name, + benchmark_type=benchmark_group.benchmark_type, description=benchmark_group.description, benchmark_results=benchmark_results, + varying_parameter=benchmark_group.varying_parameter, ) self._report.results[benchmark_group.name] = benchmark_group_result if progress_bar_task is not None: @@ -620,6 +645,209 @@ def _format_value(value: float) -> str: return f"{round(value * 1000)}ms" return f"{round(value, 2)}s" + def _get_sorted_table_from_test_results( + test_results: Sequence[TestResult], + ): + results_for_connectable: dict[ + ConnectableId, + list[tuple[float, float, bool]], + ] = {} + ref_row: list[str] = [""] + + for idx, connectable_result in enumerate( + test_results[0].connectable_results, + ): + connectable_results = [ + ( + statistics.mean( + test_result.connectable_results[idx].values, + ), + statistics.stdev( + test_result.connectable_results[idx].values, + ), + test_result.connectable_results[idx].errored, + ) + for test_result in test_results + ] + results_for_connectable[connectable_result.connectable_id] = ( + connectable_results + ) + + sorted_connectables = sorted( + results_for_connectable.keys(), + key=( + lambda connectable_id: ( + geometric_mean( + [ + result_mean + for ( + result_mean, + _, + errored, + ) in results_for_connectable[connectable_id] + if not errored + ] + or [1e9], + ) + ) + ), + ) + results_for_connectable = { + connectable: results_for_connectable[connectable] + for connectable in sorted_connectables + } + columns = ["Test Name"] + rows: list[list[str]] = [] + for ( + connectable_id, + connectable_results, + ) in results_for_connectable.items(): + columns.append( + connectable_id, + ) + g_mean = geometric_mean( + [ + result_mean + for result_mean, _, errored in connectable_results + if not errored + ] + or [1e9], + ) + if g_mean == geometric_mean([1e9]): + ref_row.append("Errored") + else: + ref_row.append( + str(g_mean), + ) + + fastest_connectable_results = next( + iter( + results_for_connectable.values(), + ), + ) + + for idx, test_result in enumerate( + test_results, + ): + row_elements = [ + test_result.description, + ] + for connectable_idx, results in enumerate( + results_for_connectable.items(), + ): + _, connectable_results = results + _mean, std_dev, errored = connectable_results[idx] + + fastest_implementation_mean, _, __ = ( + fastest_connectable_results[idx] + ) + relative = _mean / fastest_implementation_mean + + if errored: + repr_string = "Errored" + else: + repr_string = ( + f"{_format_value(_mean)} +- " + f"{_format_value(std_dev)}" + ) + if connectable_idx != 0 and relative > 1: + repr_string += f": {round(relative, 2)}x slower" + elif connectable_idx: + repr_string += ( + f": {round(1 / relative, 2)}x faster" + ) + + row_elements.append(repr_string) + + rows.append(row_elements) + + for implementation_idx in range(2, len(ref_row)): + if ref_row[implementation_idx] == "Errored": + continue + rounded_off_val = round( + float(ref_row[implementation_idx]) / float(ref_row[1]), + 2, + ) + ref_row[implementation_idx] = f"{rounded_off_val}x slower" + + ref_row[1] = "Reference" + + if len(results_for_connectable) > 1: + rows.append(ref_row) + + return rows, columns + + def _table_for_benchmark_result( + benchmark_result: BenchmarkResult, + ): + markdown_content = ( + f"\n\nBenchmark: Tests with {benchmark_result.name}\n" + ) + inner_table = Table( + box=box.SIMPLE_HEAD, + title=(f"Tests with {benchmark_result.name}"), + min_width=100, + ) + rows, columns = _get_sorted_table_from_test_results( + benchmark_result.test_results, + ) + for column in columns: + inner_table.add_column(column) + for row in rows: + inner_table.add_row(*row) + markdown_content += convert_table_to_markdown(columns, rows) + markdown_content += "\n\n" + return inner_table, markdown_content + + def _table_for_common_tests( + benchmark_results: list[BenchmarkResult], + ): + markdown_content = ( + f"\nBenchmark: Tests with varying " + f"{benchmark_group_result.varying_parameter}\n" + ) + common_tests = { + test.description for test in benchmark_results[0].test_results + } + for benchmark_result in benchmark_results[1:]: + common_tests.intersection_update( + test.description for test in benchmark_result.test_results + ) + + common_test = next(iter(common_tests)) + common_test_table = Table( + box=box.SIMPLE_HEAD, + title=( + f"Tests with varying " + f"{benchmark_group_result.varying_parameter}" + ), + min_width=100, + ) + common_test_results: list[TestResult] = [ + TestResult( + description=benchmark_result.name, + connectable_results=test_result.connectable_results, + ) + for benchmark_result in benchmark_results + for test_result in benchmark_result.test_results + if test_result.description == common_test + ] + + rows, columns = _get_sorted_table_from_test_results( + common_test_results, + ) + for column in columns: + common_test_table.add_column(column) + for row in rows: + common_test_table.add_row(*row) + + markdown_content += convert_table_to_markdown( + columns, + rows, + ) + markdown_content += "\n\n" + return common_test_table, markdown_content + cpu_count = self._report.metadata.system_metadata.get( "cpu_count", "Not Available", @@ -674,157 +902,23 @@ def _format_value(value: float) -> str: box=box.SIMPLE_HEAD, caption='File "' + str(benchmark_group_path) + '"', ) - for benchmark_result in benchmark_group_result.benchmark_results: - markdown_content += f"Benchmark: {benchmark_result.name}\n" - inner_table = Table( - "Test Name", - box=box.SIMPLE_HEAD, - title=benchmark_result.name, - min_width=100, - ) - ref_row: list[str] = [""] - results_for_connectable: dict[ - ConnectableId, - list[tuple[float, float, bool]], - ] = {} - for idx, connectable_result in enumerate( - benchmark_result.test_results[0].connectable_results, - ): - connectable_results = [ - ( - statistics.mean( - test_result.connectable_results[idx].values, - ), - statistics.stdev( - test_result.connectable_results[idx].values, - ), - test_result.connectable_results[idx].errored, - ) - for test_result in benchmark_result.test_results - ] - results_for_connectable[ - connectable_result.connectable_id - ] = connectable_results - - sorted_connectables = sorted( - results_for_connectable.keys(), - key=( - lambda connectable_id: ( - geometric_mean( - [ - result_mean - for ( - result_mean, - _, - errored, - ) in results_for_connectable[ - connectable_id - ] - if not errored - ] - or [1e9], - ) - ) - ), - ) - results_for_connectable = { - connectable: results_for_connectable[connectable] - for connectable in sorted_connectables - } - columns = ["Test Name"] - rows: list[list[str]] = [] - for ( - connectable_id, - connectable_results, - ) in results_for_connectable.items(): - columns.append( - connectable_id, + if benchmark_group_result.varying_parameter: + common_tests_table, common_tests_table_markdown = ( + _table_for_common_tests( + benchmark_group_result.benchmark_results, ) - inner_table.add_column( - connectable_id, - ) - g_mean = geometric_mean( - [ - result_mean - for result_mean, _, errored in connectable_results - if not errored - ] - or [1e9], - ) - if g_mean == geometric_mean([1e9]): - ref_row.append("Errored") - else: - ref_row.append( - str(g_mean), - ) - - fastest_connectable_results = next( - iter( - results_for_connectable.values(), - ), ) - for idx, test_result in enumerate( - benchmark_result.test_results, - ): - row_elements = [ - test_result.description, - ] - for connectable_idx, results in enumerate( - results_for_connectable.items(), - ): - _, connectable_results = results - _mean, std_dev, errored = connectable_results[idx] - - fastest_implementation_mean, _, __ = ( - fastest_connectable_results[idx] - ) - relative = _mean / fastest_implementation_mean - - if errored: - repr_string = "Errored" - else: - repr_string = ( - f"{_format_value(_mean)} +- " - f"{_format_value(std_dev)}" - ) - if connectable_idx != 0 and relative > 1: - repr_string += ( - f": {round(relative, 2)}x slower" - ) - elif connectable_idx: - repr_string += ( - f": {round(1 / relative, 2)}x faster" - ) - - row_elements.append(repr_string) - - rows.append(row_elements) - inner_table.add_row(*row_elements) - - for implementation_idx in range(2, len(ref_row)): - if ref_row[implementation_idx] == "Errored": - continue - rounded_off_val = { - round( - float(ref_row[implementation_idx]) - / float(ref_row[1]), - 2, - ), - } - ref_row[implementation_idx] = f"{rounded_off_val}x slower" + outer_table.add_row(common_tests_table) + markdown_content += common_tests_table_markdown - ref_row[1] = "Reference" - inner_table.add_section() - - if len(results_for_connectable) > 1: - rows.append(ref_row) - inner_table.add_row(*ref_row) + inner_table, inner_table_markdown = _table_for_benchmark_result( + benchmark_group_result.benchmark_results[-1], + ) - markdown_content += convert_table_to_markdown(columns, rows) - markdown_content += "\n\n" - outer_table.add_row(inner_table) + outer_table.add_row(inner_table) + markdown_content += inner_table_markdown if len(benchmark_group_result.benchmark_results) > 0: table.add_row( @@ -895,6 +989,7 @@ def from_test_cases( ), ], uri=None, + benchmark_type="test_case", ) for case in cases ] @@ -906,30 +1001,39 @@ def from_test_cases( @classmethod def for_keywords(cls, dialect: Dialect, **kwargs: Any): - bowtie_dir = Path(__file__).parent - keywords_benchmark_dir = bowtie_dir / "benchmarks/keywords" - - dialect_keyword_benchmarks_dir = ( - keywords_benchmark_dir / dialect.short_name + bowtie_parent_dir = Path(__file__).parent.parent + keywords_benchmark_dir = ( + bowtie_parent_dir / "bowtie/benchmarks/keywords" ) if not keywords_benchmark_dir.exists(): raise BenchmarkLoadError("Keyword Benchmarks Folder not found.") - if not dialect_keyword_benchmarks_dir.exists(): - raise BenchmarkLoadError( - message=( - "Keyword Specific Benchmarks not present " - f"for {dialect.serializable()}" - ), - ) - module_name = f"bowtie.benchmarks.keywords.{dialect.short_name}" + keyword_benchmarks = get_benchmark_files( + benchmark_type="keyword", + dialect=dialect, + ) + + benchmark_groups: list[BenchmarkGroup] = [] + for keyword_benchmark in keyword_benchmarks: + relative_folder_path = keyword_benchmark.parent.relative_to( + bowtie_parent_dir, + ) + module_name = str(relative_folder_path).replace(os.sep, ".") + benchmark_group = BenchmarkGroup.from_file( + keyword_benchmark, + module_name, + ) + if ( + benchmark_group is not None + and dialect in benchmark_group.dialects_supported + ): + benchmark_groups.append( + benchmark_group, + ) return cls._from_dict( - benchmark_groups=BenchmarkGroup.from_folder( - dialect_keyword_benchmarks_dir, - module=module_name, - ), + benchmark_groups=benchmark_groups, **kwargs, ) @@ -970,6 +1074,7 @@ def from_input( benchmark: dict[str, Any], **kwargs: Any, ): + benchmark["benchmark_type"] = "from_input" benchmark_validated(benchmark) benchmark_group = BenchmarkGroup.from_dict( benchmark, @@ -1052,6 +1157,14 @@ async def start( some_benchmark_ran = False some_benchmark_ran_successfully = False for benchmark_group in self._benchmark_groups: + if dialect not in benchmark_group.dialects_supported: + if not quiet: + STDOUT.log( + f"Skipping Benchmark Group: {benchmark_group.name}" + f" as it does not support dialect" + f" {dialect.serializable()}", + ) + continue ( benchmark_started, benchmark_group_finished, @@ -1165,6 +1278,7 @@ async def _run_benchmark( benchmark_name = f"{benchmark.name}::{benchmark.tests[0].description}" benchmark_dict = benchmark.serializable() + benchmark_dict["schema"]["$schema"] = str(dialect.uri) if "name" in benchmark_dict: benchmark_dict.pop("name") diff --git a/bowtie/_cli.py b/bowtie/_cli.py index 96e667f10..2f266b7f4 100644 --- a/bowtie/_cli.py +++ b/bowtie/_cli.py @@ -1442,11 +1442,13 @@ def filter_benchmarks( """ Output benchmarks matching the specified criteria. """ - _benchmarks.get_benchmark_filenames( + files: list[Path] = _benchmarks.get_benchmark_files( benchmark_type, benchmarks=benchmark_names, dialect=dialect, ) + for file in files: + console.Console().file.write(f"{file}\n") LANGUAGE_ALIASES = { diff --git a/bowtie/benchmarks/contains.py b/bowtie/benchmarks/contains.py index cbf2868c3..d43c89586 100644 --- a/bowtie/benchmarks/contains.py +++ b/bowtie/benchmarks/contains.py @@ -1,4 +1,9 @@ -from bowtie._benchmarks import Benchmark +from pathlib import Path + +from url.url import URL + +from bowtie._benchmarks import BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): @@ -8,18 +13,32 @@ def get_benchmark(): end = [0] * (array_size - 1) + [37] invalid = [0] * array_size - return Benchmark.from_dict( - name="contains", - description="A benchmark for validation of the `contains` keyword.", - schema={ - "type": "array", - "contains": {"const": 37}, - }, - tests=[ - dict(description="Empty array", instance=[]), - dict(description="Beginning of array", instance=beginning), - dict(description="Middle of array", instance=middle), - dict(description="End of array", instance=end), - dict(description="Invalid array", instance=invalid), - ], + return BenchmarkGroup.from_dict( + data=dict( + name="contains", + benchmark_type="default", + dialects_supported=[ + Dialect.from_str( + "https://json-schema.org/draft/2020-12/schema", + ), + Dialect.from_str( + "https://json-schema.org/draft/2019-09/schema", + ), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + ], + description="A benchmark for validation of the `contains` keyword.", + schema={ + "type": "array", + "contains": {"const": 37}, + }, + tests=[ + dict(description="Empty array", instance=[]), + dict(description="Beginning of array", instance=beginning), + dict(description="Middle of array", instance=middle), + dict(description="End of array", instance=end), + dict(description="Invalid array", instance=invalid), + ], + ), + uri=URL.parse(Path(__file__).absolute().as_uri()), ) diff --git a/bowtie/benchmarks/draft2020_metaschema.py b/bowtie/benchmarks/draft2020_metaschema.py index cc6beb989..c92889fa9 100644 --- a/bowtie/benchmarks/draft2020_metaschema.py +++ b/bowtie/benchmarks/draft2020_metaschema.py @@ -1,23 +1,36 @@ +from pathlib import Path + from jsonschema_specifications import REGISTRY as SPECIFICATIONS +from url.url import URL -from bowtie._benchmarks import Benchmark +from bowtie._benchmarks import BenchmarkGroup +from bowtie._core import Dialect DRAFT202012_DIALECT_URI = "https://json-schema.org/draft/2020-12/schema" def get_benchmark(): - return Benchmark.from_dict( - name="Draft2020-12_MetaSchema", - description=( - "A benchmark for validation of the Draft2020-12 MetaSchema." - ), - schema=SPECIFICATIONS.contents(DRAFT202012_DIALECT_URI), - tests=[ - dict( - description="Validating metaschema against metaschema", - instance=SPECIFICATIONS.contents( - DRAFT202012_DIALECT_URI, + return BenchmarkGroup.from_dict( + data=dict( + name="Draft2020-12_MetaSchema", + benchmark_type="default", + dialects_supported=[ + Dialect.from_str( + "https://json-schema.org/draft/2020-12/schema", ), + ], + description=( + "A benchmark for validation of the Draft2020-12 MetaSchema." ), - ], + schema=SPECIFICATIONS.contents(DRAFT202012_DIALECT_URI), + tests=[ + dict( + description="Validating metaschema against metaschema", + instance=SPECIFICATIONS.contents( + DRAFT202012_DIALECT_URI, + ), + ), + ], + ), + uri=URL.parse(Path(__file__).absolute().as_uri()), ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/additionalProperties.py b/bowtie/benchmarks/keywords/additionalProperties.py similarity index 71% rename from bowtie/benchmarks/keywords/draft2020-12/additionalProperties.py rename to bowtie/benchmarks/keywords/additionalProperties.py index 28d403300..6112e0599 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/additionalProperties.py +++ b/bowtie/benchmarks/keywords/additionalProperties.py @@ -4,22 +4,24 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "additionalProperties" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the additionalProperties keyword." ) - max_array_size = 100000 - benchmarks: list[Benchmark] = [] - - array_size = 1000 + max_array_length = 100000 + varying_parameter = "Array length" - while array_size <= max_array_size: + array_length = 1000 + benchmarks: list[Benchmark] = [] + while array_length <= max_array_length: allowed_properties = [ - uuid.uuid4().hex for _ in range(max_array_size - 1) + uuid.uuid4().hex for _ in range(max_array_length - 1) ] middle_index = len(allowed_properties) // 2 @@ -57,7 +59,7 @@ def get_benchmark(): instance=_format_properties_as_instance(valid), ), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict( description="Valid", @@ -68,9 +70,9 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating additionalProperties keyword over array of size {array_size}" + f"Validating additionalProperties keyword over array of length {array_length}" ), schema=dict( properties={key: {} for key in allowed_properties}, @@ -80,12 +82,22 @@ def get_benchmark(): ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], description=description, benchmarks=benchmarks, + varying_parameter=varying_parameter, uri=URL.parse(Path(__file__).absolute().as_uri()), ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/contains.py b/bowtie/benchmarks/keywords/contains.py similarity index 55% rename from bowtie/benchmarks/keywords/draft2020-12/contains.py rename to bowtie/benchmarks/keywords/contains.py index c72d00f39..ce7ac56a9 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/contains.py +++ b/bowtie/benchmarks/keywords/contains.py @@ -3,23 +3,24 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): - name = "contains" + benchmark_type = "keyword" description = "A benchmark for validation of the `contains` keyword." + max_array_length = 100000 + varying_parameter = "Array length" - max_array_size = 100000 - array_size = 1000 - + array_length = 1000 benchmarks = [] - while array_size <= max_array_size: + while array_length <= max_array_length: - start = [37] + [0] * (array_size - 1) - middle = [0] * (array_size // 2) + [37] + [0] * (array_size // 2) - end = [0] * (array_size - 1) + [37] - invalid = [0] * array_size + start = [37] + [0] * (array_length - 1) + middle = [0] * (array_length // 2) + [37] + [0] * (array_length // 2) + end = [0] * (array_length - 1) + [37] + invalid = [0] * array_length tests = ( [ @@ -29,7 +30,7 @@ def get_benchmark(): dict(description="End of array", instance=end), dict(description="Invalid array", instance=invalid), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict(description="Middle of array", instance=middle), ] @@ -37,10 +38,10 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( "Validating contains keyword over an array " - f"of size {array_size}" + f"of length {array_length}" ), schema={ "type": "array", @@ -49,11 +50,19 @@ def get_benchmark(): tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/maxItems.py b/bowtie/benchmarks/keywords/draft2020-12/maxItems.py deleted file mode 100644 index 68e868c0d..000000000 --- a/bowtie/benchmarks/keywords/draft2020-12/maxItems.py +++ /dev/null @@ -1,50 +0,0 @@ -from pathlib import Path - -from url.url import URL - -from bowtie._benchmarks import Benchmark, BenchmarkGroup - - -def get_benchmark(): - name = "maxItems" - description = ( - "A benchmark for measuring performance of the " - "implementation for the maxItems keyword." - ) - - max_array_size = 100000 - array_size = 1000 - benchmarks = [] - tests = [] - testing_array_size = 10000 - - while testing_array_size <= max_array_size: - tests.append( - dict( - description=f"Testing Array Size - {testing_array_size}", - instance=["random" for _ in range(testing_array_size)], - ), - ) - testing_array_size *= 10 - - while array_size <= max_array_size: - benchmarks.append( - Benchmark.from_dict( - name=f"Array Size - {array_size}", - description=( - f"Validating the `maxItems` keyword over array of size {array_size}." - ), - schema={ - "maxItems": array_size, - }, - tests=tests, - ), - ) - array_size *= 10 - - return BenchmarkGroup( - name=name, - description=description, - benchmarks=benchmarks, - uri=URL.parse(Path(__file__).absolute().as_uri()), - ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/maxLength.py b/bowtie/benchmarks/keywords/draft2020-12/maxLength.py deleted file mode 100644 index 42b0d8584..000000000 --- a/bowtie/benchmarks/keywords/draft2020-12/maxLength.py +++ /dev/null @@ -1,56 +0,0 @@ -from pathlib import Path -import random -import string - -from url.url import URL - -from bowtie._benchmarks import Benchmark, BenchmarkGroup - - -def get_benchmark(): - name = "maxLength" - description = ( - "A benchmark for measuring performance of the " - "implementation for the maxLength keyword." - ) - - max_string_size = 100000 - string_size = 1000 - benchmarks = [] - - tests = [] - testing_string_size = 10000 - while testing_string_size <= max_string_size: - tests.append( - dict( - description=f"Testing String Size - {testing_string_size}", - instance="".join( - random.choice(string.ascii_letters) - for _ in range(testing_string_size) - ), - ), - ) - testing_string_size *= 10 - - while string_size <= max_string_size: - benchmarks.append( - Benchmark.from_dict( - name=f"String Size - {string_size}", - description=( - f"Validating the `maxLength` keyword over string of size {string_size}." - ), - schema={ - "type": "string", - "maxLength": string_size, - }, - tests=tests, - ), - ) - string_size *= 10 - - return BenchmarkGroup( - name=name, - description=description, - benchmarks=benchmarks, - uri=URL.parse(Path(__file__).absolute().as_uri()), - ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/minItems.py b/bowtie/benchmarks/keywords/draft2020-12/minItems.py deleted file mode 100644 index e63dc9aea..000000000 --- a/bowtie/benchmarks/keywords/draft2020-12/minItems.py +++ /dev/null @@ -1,50 +0,0 @@ -from pathlib import Path - -from url.url import URL - -from bowtie._benchmarks import Benchmark, BenchmarkGroup - - -def get_benchmark(): - name = "minItems" - description = ( - "A benchmark for measuring performance of the " - "implementation for the minItems keyword." - ) - - max_array_size = 100000 - array_size = 1000 - benchmarks = [] - - tests = [] - testing_array_size = 10000 - while testing_array_size <= max_array_size: - tests.append( - dict( - description=f"Testing Array Size - {testing_array_size}", - instance=["random" for _ in range(testing_array_size)], - ), - ) - testing_array_size *= 10 - - while array_size <= max_array_size: - benchmarks.append( - Benchmark.from_dict( - name=f"Array Size - {array_size}", - description=( - f"Validating the `minItems` keyword over array of size {array_size}." - ), - schema={ - "minItems": array_size, - }, - tests=tests, - ), - ) - array_size *= 10 - - return BenchmarkGroup( - name=name, - description=description, - benchmarks=benchmarks, - uri=URL.parse(Path(__file__).absolute().as_uri()), - ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/minLength.py b/bowtie/benchmarks/keywords/draft2020-12/minLength.py deleted file mode 100644 index 8d3d7f21f..000000000 --- a/bowtie/benchmarks/keywords/draft2020-12/minLength.py +++ /dev/null @@ -1,56 +0,0 @@ -from pathlib import Path -import random -import string - -from url.url import URL - -from bowtie._benchmarks import Benchmark, BenchmarkGroup - - -def get_benchmark(): - name = "minLength" - description = ( - "A benchmark for measuring performance of the " - "implementation for the minLength keyword." - ) - - max_string_size = 100000 - string_size = 1000 - benchmarks = [] - - tests = [] - testing_string_size = 10000 - while testing_string_size <= max_string_size: - tests.append( - dict( - description=f"Testing String Size - {testing_string_size}", - instance="".join( - random.choice(string.ascii_letters) - for _ in range(testing_string_size) - ), - ), - ) - testing_string_size *= 10 - - while string_size <= max_string_size: - benchmarks.append( - Benchmark.from_dict( - name=f"String Size - {string_size}", - description=( - f"Validating the `minLength` keyword over string of size {string_size}." - ), - schema={ - "type": "string", - "minLength": string_size, - }, - tests=tests, - ), - ) - string_size *= 10 - - return BenchmarkGroup( - name=name, - description=description, - benchmarks=benchmarks, - uri=URL.parse(Path(__file__).absolute().as_uri()), - ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/type.py b/bowtie/benchmarks/keywords/draft2020-12/type.py deleted file mode 100644 index 285e2b1df..000000000 --- a/bowtie/benchmarks/keywords/draft2020-12/type.py +++ /dev/null @@ -1,42 +0,0 @@ -from pathlib import Path -import uuid - -from url.url import URL - -from bowtie._benchmarks import Benchmark, BenchmarkGroup - - -def get_benchmark(): - name = "type" - description = ( - "A benchmark for measuring performance of the " - "implementation for the type keyword." - ) - max_array_size = 100000 - array_size = 1000 - - benchmarks = [] - while array_size <= max_array_size: - array = [uuid.uuid4().hex for _ in range(array_size)] - benchmarks.append( - Benchmark.from_dict( - name=f"Array Size - {array_size}", - description=( - f"Validating the `type` keyword over array of size {array_size}." - ), - schema={ - "type": "array", - }, - tests=[ - dict(description="Valid Array", instance=array), - ], - ), - ) - array_size *= 10 - - return BenchmarkGroup( - name=name, - description=description, - benchmarks=benchmarks, - uri=URL.parse(Path(__file__).absolute().as_uri()), - ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/enum.py b/bowtie/benchmarks/keywords/enum.py similarity index 53% rename from bowtie/benchmarks/keywords/draft2020-12/enum.py rename to bowtie/benchmarks/keywords/enum.py index 51071799e..9ad102eaf 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/enum.py +++ b/bowtie/benchmarks/keywords/enum.py @@ -4,33 +4,35 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "enum" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the implementation " "for the enum keyword." ) + max_array_length = 100000 + varying_parameter = "Array length" - max_array_size = 100000 - array_size = 1000 - + array_length = 1000 benchmarks = [] - while array_size <= max_array_size: - array = [uuid.uuid4().hex for _ in range(array_size)] + while array_length <= max_array_length: + array = [uuid.uuid4().hex for _ in range(array_length)] tests = ( [ dict(description="Valid First", instance=array[0]), dict( description="Valid Middle", - instance=array[array_size // 2], + instance=array[array_length // 2], ), dict(description="Valid Last", instance=array[-1]), dict(description="Invalid", instance=uuid.uuid4().hex), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict(description="Invalid", instance=uuid.uuid4().hex), ] @@ -38,19 +40,29 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating the `enum` keyword over array of size {array_size}." + f"Validating the `enum` keyword over array of length {array_length}." ), schema=dict(enum=array), tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/items.py b/bowtie/benchmarks/keywords/items.py similarity index 60% rename from bowtie/benchmarks/keywords/draft2020-12/items.py rename to bowtie/benchmarks/keywords/items.py index f0ab6e6f4..38fe7ecca 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/items.py +++ b/bowtie/benchmarks/keywords/items.py @@ -4,25 +4,27 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "items" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the items keyword." ) + max_array_length = 100000 + varying_parameter = "Array length" - max_array_size = 100000 - array_size = 1000 - + array_length = 1000 benchmarks = [] - while array_size <= max_array_size: - array = [uuid.uuid4().hex for _ in range(array_size)] + while array_length <= max_array_length: + array = [uuid.uuid4().hex for _ in range(array_length)] invalid_at_first = [1] + array[:-1] invalid_at_middle = ( - array[: array_size // 2] + [1] + array[array_size // 2 : -1] + array[: array_length // 2] + [1] + array[array_length // 2 : -1] ) invalid_at_last = array[:-1] + [1] valid = array @@ -43,7 +45,7 @@ def get_benchmark(): ), dict(description="Valid", instance=valid), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict(description="Valid", instance=valid), ] @@ -51,9 +53,9 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating the `items` keyword over array of size {array_size}." + f"Validating the `items` keyword over array of length {array_length}." ), schema={ "type": "array", @@ -62,11 +64,21 @@ def get_benchmark(): tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/maxContains.py b/bowtie/benchmarks/keywords/maxContains.py similarity index 66% rename from bowtie/benchmarks/keywords/draft2020-12/maxContains.py rename to bowtie/benchmarks/keywords/maxContains.py index 5f2c6e512..2f7141213 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/maxContains.py +++ b/bowtie/benchmarks/keywords/maxContains.py @@ -4,27 +4,29 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "maxContains" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the implementation " "for the maxContains keyword." ) + max_array_length = 100000 + varying_parameter = "Array length" - max_array_size = 100000 - array_size = 1000 - + array_length = 1000 benchmarks = [] - while array_size <= max_array_size: - array = [uuid.uuid4().hex for _ in range(array_size)] + while array_length <= max_array_length: + array = [uuid.uuid4().hex for _ in range(array_length)] all_at_first = [1, 1, 1] + array[:-3] all_at_middle = ( - array[2 : array_size // 2] + array[2 : array_length // 2] + [1, 1, 1] - + array[array_size // 2 : -1] + + array[array_length // 2 : -1] ) all_at_last = array[:-3] + [1, 1, 1] valid = array[:-1] + [1] @@ -36,7 +38,7 @@ def get_benchmark(): dict(description="All at Last", instance=all_at_last), dict(description="Valid", instance=valid), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict(description="Valid", instance=valid), ] @@ -44,9 +46,9 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating the `maxContains` keyword over array of size {array_size}." + f"Validating the `maxContains` keyword over array of length {array_length}." ), schema={ "type": "array", @@ -58,11 +60,17 @@ def get_benchmark(): tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/maxItems.py b/bowtie/benchmarks/keywords/maxItems.py new file mode 100644 index 000000000..7f3cb7b9e --- /dev/null +++ b/bowtie/benchmarks/keywords/maxItems.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + name = "maxItems" + benchmark_type = "keyword" + description = ( + "A benchmark for measuring performance of the " + "implementation for the maxItems keyword." + ) + max_array_length = 100000 + varying_parameter = "Array length" + + array_length = 1000 + benchmarks = [] + tests = [] + testing_array_length = 10000 + + while testing_array_length <= max_array_length: + tests.append( + dict( + description=f"Testing Array length - {testing_array_length}", + instance=["random" for _ in range(testing_array_length)], + ), + ) + testing_array_length *= 10 + + while array_length <= max_array_length: + benchmarks.append( + Benchmark.from_dict( + name=f"Array length - {array_length}", + description=( + f"Validating the `maxItems` keyword over array of length {array_length}." + ), + schema={ + "maxItems": array_length, + }, + tests=tests, + ), + ) + array_length *= 10 + + return BenchmarkGroup( + name=name, + benchmark_type=benchmark_type, + description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], + benchmarks=benchmarks, + uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, + ) diff --git a/bowtie/benchmarks/keywords/maxLength.py b/bowtie/benchmarks/keywords/maxLength.py new file mode 100644 index 000000000..43d2cdeb0 --- /dev/null +++ b/bowtie/benchmarks/keywords/maxLength.py @@ -0,0 +1,69 @@ +from pathlib import Path +import random +import string + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + name = "maxLength" + benchmark_type = "keyword" + description = ( + "A benchmark for measuring performance of the " + "implementation for the maxLength keyword." + ) + max_string_length = 100000 + varying_parameter = "String length" + + string_length = 1000 + benchmarks = [] + tests = [] + testing_string_length = 10000 + + while testing_string_length <= max_string_length: + tests.append( + dict( + description=f"Testing String length - {testing_string_length}", + instance="".join( + random.choice(string.ascii_letters) + for _ in range(testing_string_length) + ), + ), + ) + testing_string_length *= 10 + + while string_length <= max_string_length: + benchmarks.append( + Benchmark.from_dict( + name=f"String length - {string_length}", + description=( + f"Validating the `maxLength` keyword over string of length {string_length}." + ), + schema={ + "type": "string", + "maxLength": string_length, + }, + tests=tests, + ), + ) + string_length *= 10 + + return BenchmarkGroup( + name=name, + benchmark_type=benchmark_type, + description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], + benchmarks=benchmarks, + uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, + ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/maxProperties.py b/bowtie/benchmarks/keywords/maxProperties.py similarity index 77% rename from bowtie/benchmarks/keywords/draft2020-12/maxProperties.py rename to bowtie/benchmarks/keywords/maxProperties.py index 0f3f69f1f..4bec3351e 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/maxProperties.py +++ b/bowtie/benchmarks/keywords/maxProperties.py @@ -4,18 +4,20 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "maxProperties" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the implementation " "for the maxProperties keyword." ) - max_num_properties = 100000 - num_properties = 1000 + varying_parameter = "No. of maxProperties Allowed" + num_properties = 1000 benchmarks = [] while num_properties <= max_num_properties: invalid_object = _create_object_with_num_properties( @@ -64,9 +66,18 @@ def get_benchmark(): return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/minContains.py b/bowtie/benchmarks/keywords/minContains.py similarity index 67% rename from bowtie/benchmarks/keywords/draft2020-12/minContains.py rename to bowtie/benchmarks/keywords/minContains.py index ed8e56b55..d510be0c1 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/minContains.py +++ b/bowtie/benchmarks/keywords/minContains.py @@ -4,26 +4,30 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "minContains" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the implementation " "for the minContains keyword." ) + max_array_length = 100000 + varying_parameter = "Array length" - max_array_size = 100000 - array_size = 1000 - + array_length = 1000 benchmarks = [] - while array_size <= max_array_size: - array = [uuid.uuid4().hex for _ in range(array_size)] + while array_length <= max_array_length: + array = [uuid.uuid4().hex for _ in range(array_length)] both_at_first = [1, 1] + array[:-2] both_at_middle = ( - array[1 : array_size // 2] + [1, 1] + array[array_size // 2 : -1] + array[1 : array_length // 2] + + [1, 1] + + array[array_length // 2 : -1] ) both_at_last = array[:-2] + [1, 1] invalid = array @@ -38,7 +42,7 @@ def get_benchmark(): dict(description="Both at Last", instance=both_at_last), dict(description="Invalid", instance=invalid), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict( description="Both at Middle", @@ -49,7 +53,7 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"minContains_{array_size}", + name=f"minContains_{array_length}", description=( "A benchmark for validation of the `minContains` keyword." ), @@ -61,11 +65,17 @@ def get_benchmark(): tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/minItems.py b/bowtie/benchmarks/keywords/minItems.py new file mode 100644 index 000000000..93ca64b19 --- /dev/null +++ b/bowtie/benchmarks/keywords/minItems.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + name = "minItems" + benchmark_type = "keyword" + description = ( + "A benchmark for measuring performance of the " + "implementation for the minItems keyword." + ) + max_array_length = 100000 + varying_parameter = "Array length" + + array_length = 1000 + benchmarks = [] + tests = [] + testing_array_length = 10000 + + while testing_array_length <= max_array_length: + tests.append( + dict( + description=f"Testing Array length - {testing_array_length}", + instance=["random" for _ in range(testing_array_length)], + ), + ) + testing_array_length *= 10 + + while array_length <= max_array_length: + benchmarks.append( + Benchmark.from_dict( + name=f"Array length - {array_length}", + description=( + f"Validating the `minItems` keyword over array of length {array_length}." + ), + schema={ + "minItems": array_length, + }, + tests=tests, + ), + ) + array_length *= 10 + + return BenchmarkGroup( + name=name, + benchmark_type=benchmark_type, + description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], + benchmarks=benchmarks, + uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, + ) diff --git a/bowtie/benchmarks/keywords/minLength.py b/bowtie/benchmarks/keywords/minLength.py new file mode 100644 index 000000000..73aaf7cd5 --- /dev/null +++ b/bowtie/benchmarks/keywords/minLength.py @@ -0,0 +1,69 @@ +from pathlib import Path +import random +import string + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + name = "minLength" + benchmark_type = "keyword" + description = ( + "A benchmark for measuring performance of the " + "implementation for the minLength keyword." + ) + max_string_length = 100000 + varying_parameter = "String length" + + string_length = 1000 + benchmarks = [] + tests = [] + testing_string_length = 10000 + + while testing_string_length <= max_string_length: + tests.append( + dict( + description=f"Testing String length - {testing_string_length}", + instance="".join( + random.choice(string.ascii_letters) + for _ in range(testing_string_length) + ), + ), + ) + testing_string_length *= 10 + + while string_length <= max_string_length: + benchmarks.append( + Benchmark.from_dict( + name=f"String length - {string_length}", + description=( + f"Validating the `minLength` keyword over string of length {string_length}." + ), + schema={ + "type": "string", + "minLength": string_length, + }, + tests=tests, + ), + ) + string_length *= 10 + + return BenchmarkGroup( + name=name, + benchmark_type=benchmark_type, + description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], + benchmarks=benchmarks, + uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, + ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/minProperties.py b/bowtie/benchmarks/keywords/minProperties.py similarity index 77% rename from bowtie/benchmarks/keywords/draft2020-12/minProperties.py rename to bowtie/benchmarks/keywords/minProperties.py index b9694eae9..a8e34de5d 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/minProperties.py +++ b/bowtie/benchmarks/keywords/minProperties.py @@ -4,18 +4,20 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "minProperties" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the implementation " "for the minProperties keyword." ) - max_num_properties = 100000 - num_properties = 1000 + varying_parameter = "No. of minProperties Needed" + num_properties = 1000 benchmarks = [] while num_properties <= max_num_properties: valid_object = _create_object_with_num_properties(num_properties) @@ -58,9 +60,18 @@ def get_benchmark(): return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/pattern.py b/bowtie/benchmarks/keywords/pattern.py similarity index 61% rename from bowtie/benchmarks/keywords/draft2020-12/pattern.py rename to bowtie/benchmarks/keywords/pattern.py index 21440b20d..8a4c0088a 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/pattern.py +++ b/bowtie/benchmarks/keywords/pattern.py @@ -5,23 +5,25 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "pattern" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the pattern keyword." ) + max_string_length = 100000 + varying_parameter = "String length" - max_string_size = 100000 - string_size = 1000 - + string_length = 1000 benchmarks = [] - while string_size <= max_string_size: + while string_length <= max_string_length: letters = string.ascii_letters random_letter_string = "".join( - random.choice(letters) for _ in range(string_size) + random.choice(letters) for _ in range(string_length) ) tests = ( @@ -34,9 +36,9 @@ def get_benchmark(): dict( description="Invalid Char at Middle", instance=( - random_letter_string[: string_size // 2] + random_letter_string[: string_length // 2] + "1" - + random_letter_string[string_size // 2 :] + + random_letter_string[string_length // 2 :] ), ), dict( @@ -48,7 +50,7 @@ def get_benchmark(): instance=random_letter_string, ), ] - if string_size == max_string_size + if string_length == max_string_length else [ dict( description="Valid String", @@ -59,19 +61,29 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"String Size - {string_size}", + name=f"String length - {string_length}", description=( - f"Validating the `pattern` keyword over string of size {string_size}." + f"Validating the `pattern` keyword over string of length {string_length}." ), schema=dict(type="string", pattern="^[a-zA-Z]+$"), tests=tests, ), ) - string_size *= 10 + string_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/patternProperties.py b/bowtie/benchmarks/keywords/patternProperties.py similarity index 80% rename from bowtie/benchmarks/keywords/draft2020-12/patternProperties.py rename to bowtie/benchmarks/keywords/patternProperties.py index 5c4f9d861..945df888d 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/patternProperties.py +++ b/bowtie/benchmarks/keywords/patternProperties.py @@ -5,21 +5,23 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect max_num = 100000 def get_benchmark(): name = "patternProperties" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the patternProperties keyword." ) max_num_properties = 100000 - benchmarks = [] + varying_parameter = "Object length" + benchmarks = [] num_properties = 1000 - while num_properties <= max_num_properties: property_value_pairs = [ (_random_string(), random.randint(0, max_num)) @@ -88,9 +90,19 @@ def get_benchmark(): return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) @@ -99,7 +111,7 @@ def _get_instance_object(property_value_pairs): def _random_string(): - string_size = 100 + string_length = 100 return "".join( - random.choice(string.ascii_lowercase) for _ in range(string_size) + random.choice(string.ascii_lowercase) for _ in range(string_length) ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/propertyNames.py b/bowtie/benchmarks/keywords/propertyNames.py similarity index 65% rename from bowtie/benchmarks/keywords/draft2020-12/propertyNames.py rename to bowtie/benchmarks/keywords/propertyNames.py index e13cde7ad..627e0e264 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/propertyNames.py +++ b/bowtie/benchmarks/keywords/propertyNames.py @@ -6,21 +6,23 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "propertyNames" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the propertyNames keyword." ) + max_object_length = 100000 + varying_parameter = "Object length" - max_object_size = 100000 - object_size = 1000 - + object_length = 1000 benchmarks = [] - while object_size <= max_object_size: - object = {uuid.uuid4().hex: 10 for _ in range(object_size)} + while object_length <= max_object_length: + object = {uuid.uuid4().hex: 10 for _ in range(object_length)} invalid_property = "".join( random.choice(string.ascii_letters) for _ in range(7) ) @@ -29,11 +31,11 @@ def get_benchmark(): invalid_at_first.update(object) invalid_at_middle = { - uuid.uuid4().hex: 10 for _ in range(object_size // 2) + uuid.uuid4().hex: 10 for _ in range(object_length // 2) } invalid_at_middle.update({invalid_property: 10}) invalid_at_middle.update( - {uuid.uuid4().hex: 10 for _ in range(object_size // 2)}, + {uuid.uuid4().hex: 10 for _ in range(object_length // 2)}, ) invalid_at_last = object.copy() @@ -41,7 +43,7 @@ def get_benchmark(): valid = { "".join(random.choice(string.ascii_letters) for _ in range(1)): 10 - for _ in range(object_size) + for _ in range(object_length) } tests = ( @@ -60,7 +62,7 @@ def get_benchmark(): ), dict(description="Valid", instance=valid), ] - if object_size == max_object_size + if object_length == max_object_length else [ dict(description="Valid", instance=valid), ] @@ -68,19 +70,27 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Num of Properties - {object_size}", + name=f"Num of Properties - {object_length}", description=( - f"Validating the `propertyNames` keyword over object of size {object_size}." + f"Validating the `propertyNames` keyword over object of length {object_length}." ), schema={"propertyNames": {"maxLength": 5}}, tests=tests, ), ) - object_size *= 10 + object_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/required.py b/bowtie/benchmarks/keywords/required.py similarity index 66% rename from bowtie/benchmarks/keywords/draft2020-12/required.py rename to bowtie/benchmarks/keywords/required.py index a5463fc88..7395053dd 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/required.py +++ b/bowtie/benchmarks/keywords/required.py @@ -4,31 +4,33 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "required" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the required keyword." ) - max_array_size = 100000 + max_array_length = 100000 + varying_parameter = "Array length" + array_length = 1000 benchmarks = [] - array_size = 1000 - - while array_size * 2 <= max_array_size: - num_required_properties = array_size // 2 + while array_length * 2 <= max_array_length: + num_required_properties = array_length // 2 required_properties = [ uuid.uuid4().hex for _ in range(num_required_properties) ] - object_keys = [uuid.uuid4().hex for _ in range(array_size)] + object_keys = [uuid.uuid4().hex for _ in range(array_length)] all_at_beginning = required_properties + object_keys all_at_middle = ( - object_keys[: array_size // 2] + object_keys[: array_length // 2] + required_properties - + object_keys[array_size // 2 :] + + object_keys[array_length // 2 :] ) all_at_last = object_keys + required_properties none_present = object_keys + object_keys @@ -52,7 +54,7 @@ def get_benchmark(): instance=_generate_object_with_keys(none_present), ), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict( description="None of the required properties present", @@ -63,9 +65,9 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating the `required` keyword over array of size {array_size}." + f"Validating the `required` keyword over array of length {array_length}." ), schema={ "type": "object", @@ -74,13 +76,23 @@ def get_benchmark(): tests=tests, ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/keywords/type.py b/bowtie/benchmarks/keywords/type.py new file mode 100644 index 000000000..e6a5fb74f --- /dev/null +++ b/bowtie/benchmarks/keywords/type.py @@ -0,0 +1,55 @@ +from pathlib import Path +import uuid + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + name = "type" + benchmark_type = "keyword" + description = ( + "A benchmark for measuring performance of the " + "implementation for the type keyword." + ) + max_array_length = 100000 + varying_parameter = "Array length" + + array_length = 1000 + benchmarks = [] + while array_length <= max_array_length: + array = [uuid.uuid4().hex for _ in range(array_length)] + benchmarks.append( + Benchmark.from_dict( + name=f"Array length - {array_length}", + description=( + f"Validating the `type` keyword over array of length {array_length}." + ), + schema={ + "type": "array", + }, + tests=[ + dict(description="Valid Array", instance=array), + ], + ), + ) + array_length *= 10 + + return BenchmarkGroup( + name=name, + benchmark_type=benchmark_type, + description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], + benchmarks=benchmarks, + uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, + ) diff --git a/bowtie/benchmarks/keywords/draft2020-12/uniqueItems.py b/bowtie/benchmarks/keywords/uniqueItems.py similarity index 55% rename from bowtie/benchmarks/keywords/draft2020-12/uniqueItems.py rename to bowtie/benchmarks/keywords/uniqueItems.py index 711b20c51..d183f0588 100644 --- a/bowtie/benchmarks/keywords/draft2020-12/uniqueItems.py +++ b/bowtie/benchmarks/keywords/uniqueItems.py @@ -3,28 +3,31 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): name = "uniqueItems" + benchmark_type = "keyword" description = ( "A benchmark for measuring performance of the " "implementation for the uniqueItems keyword." ) - max_array_size = 200000 - benchmarks = [] + max_array_length = 200000 + varying_parameter = "Array length" - array_size = 2000 - while array_size <= max_array_size: - first_two_duplicate = [1, 1, *list(range(2, array_size - 2))] + array_length = 2000 + benchmarks = [] + while array_length <= max_array_length: + first_two_duplicate = [1, 1, *list(range(2, array_length - 2))] middle_two_duplicate = [ - *list(range(array_size // 2)), + *list(range(array_length // 2)), -1, -1, - *list(range(array_size // 2, array_size)), + *list(range(array_length // 2, array_length)), ] - last_two_duplicate = [*list(range(2, array_size - 2)), 1, 1] - valid = list(range(array_size)) + last_two_duplicate = [*list(range(2, array_length - 2)), 1, 1] + valid = list(range(array_length)) tests = ( [ @@ -42,7 +45,7 @@ def get_benchmark(): ), dict(description="Valid", instance=valid), ] - if array_size == max_array_size + if array_length == max_array_length else [ dict(description="Valid", instance=valid), ] @@ -50,9 +53,9 @@ def get_benchmark(): benchmarks.append( Benchmark.from_dict( - name=f"Array Size - {array_size}", + name=f"Array length - {array_length}", description=( - f"Validating the `uniqueItems` keyword over array of size {array_size}." + f"Validating the `uniqueItems` keyword over array of length {array_length}." ), schema=dict( uniqueItems=True, @@ -61,11 +64,21 @@ def get_benchmark(): ), ) - array_size *= 10 + array_length *= 10 return BenchmarkGroup( name=name, + benchmark_type=benchmark_type, description=description, + dialects_supported=[ + Dialect.from_str("https://json-schema.org/draft/2020-12/schema"), + Dialect.from_str("https://json-schema.org/draft/2019-09/schema"), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + Dialect.from_str("http://json-schema.org/draft-04/schema#"), + Dialect.from_str("http://json-schema.org/draft-03/schema#"), + ], benchmarks=benchmarks, uri=URL.parse(Path(__file__).absolute().as_uri()), + varying_parameter=varying_parameter, ) diff --git a/bowtie/benchmarks/nested_schemas.py b/bowtie/benchmarks/nested_schemas.py index 31c074c16..b4c295311 100644 --- a/bowtie/benchmarks/nested_schemas.py +++ b/bowtie/benchmarks/nested_schemas.py @@ -1,38 +1,51 @@ from itertools import cycle +from pathlib import Path -from bowtie._benchmarks import Benchmark +from url.url import URL + +from bowtie._benchmarks import BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): - return Benchmark.from_dict( - name="nested_schemas", - description=( - "Validating highly nested schemas shouldn't " - "cause exponential time blowups." - ), - schema={ - "$id": "https://example.com/draft/2020-12/schema/strict", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$vocabulary": { - "https://json-schema.org/draft/2020-12/vocab/core": True, - "https://json-schema.org/draft/2020-12/vocab/applicator": True, - "https://json-schema.org/draft/2020-12/vocab/unevaluated": True, - "https://json-schema.org/draft/2020-12/vocab/validation": True, - "https://json-schema.org/draft/2020-12/vocab/meta-data": True, - "https://json-schema.org/draft/2020-12/vocab/format-annotation": True, - "https://json-schema.org/draft/2020-12/vocab/content": True, + return BenchmarkGroup.from_dict( + data=dict( + name="nested_schemas", + benchmark_type="default", + dialects_supported=[ + Dialect.from_str( + "https://json-schema.org/draft/2020-12/schema", + ), + ], + description=( + "Validating highly nested schemas shouldn't " + "cause exponential time blowups." + ), + schema={ + "$id": "https://example.com/draft/2020-12/schema/strict", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": True, + "https://json-schema.org/draft/2020-12/vocab/applicator": True, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": True, + "https://json-schema.org/draft/2020-12/vocab/validation": True, + "https://json-schema.org/draft/2020-12/vocab/meta-data": True, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": True, + "https://json-schema.org/draft/2020-12/vocab/content": True, + }, + "$dynamicAnchor": "meta", + "$ref": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": False, }, - "$dynamicAnchor": "meta", - "$ref": "https://json-schema.org/draft/2020-12/schema", - "unevaluatedProperties": False, - }, - tests=[ - dict( - description=f"No of Levels {levels}", - instance=nested_object(levels=levels), - ) - for levels in range(1, 11, 3) - ], + tests=[ + dict( + description=f"No of Levels {levels}", + instance=nested_object(levels=levels), + ) + for levels in range(1, 11, 3) + ], + ), + uri=URL.parse(Path(__file__).absolute().as_uri()), ) diff --git a/bowtie/benchmarks/openapi_spec_schema.json b/bowtie/benchmarks/openapi_spec_schema.json index 4ba0e055f..e1ba86031 100644 --- a/bowtie/benchmarks/openapi_spec_schema.json +++ b/bowtie/benchmarks/openapi_spec_schema.json @@ -1,5 +1,7 @@ { "name": "OpenAPI_Spec_Schema", + "dialects_supported": ["https://json-schema.org/draft/2020-12/schema"], + "benchmark_type": "default", "description": "A benchmark for validation of OpenAPI Spec Schema.", "schema": { "$id": "https://spec.openapis.org/oas/3.1/schema/2022-10-07", diff --git a/bowtie/benchmarks/useless_keywords.py b/bowtie/benchmarks/useless_keywords.py index dcbf4258a..c66e26602 100644 --- a/bowtie/benchmarks/useless_keywords.py +++ b/bowtie/benchmarks/useless_keywords.py @@ -1,35 +1,54 @@ -from bowtie._benchmarks import Benchmark +from pathlib import Path + +from url.url import URL + +from bowtie._benchmarks import BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): num_useless_keywords = 300000 - return Benchmark.from_dict( - name="useless_keywords", - description=( - "A benchmark for validation of schemas containing " - "lots of useless keywords. " - "Checks we filter them out once, ahead of time." - ), - schema=dict( - [ - ("not", {"const": 42}), - *((str(i), i) for i in range(num_useless_keywords)), - ("type", "integer"), - *( - (str(i), i) - for i in range( - num_useless_keywords, - num_useless_keywords, - ) + return BenchmarkGroup.from_dict( + data=dict( + name="useless_keywords", + benchmark_type="default", + dialects_supported=[ + Dialect.from_str( + "https://json-schema.org/draft/2020-12/schema", ), - ("minimum", 37), + Dialect.from_str( + "https://json-schema.org/draft/2019-09/schema", + ), + Dialect.from_str("http://json-schema.org/draft-07/schema#"), + Dialect.from_str("http://json-schema.org/draft-06/schema#"), + ], + description=( + "A benchmark for validation of schemas containing " + "lots of useless keywords. " + "Checks we filter them out once, ahead of time." + ), + schema=dict( + [ + ("not", {"const": 42}), + *((str(i), i) for i in range(num_useless_keywords)), + ("type", "integer"), + *( + (str(i), i) + for i in range( + num_useless_keywords, + num_useless_keywords, + ) + ), + ("minimum", 37), + ], + ), + tests=[ + dict(description="Beginning of schema", instance=42), + dict(description="Middle of schema", instance="foo"), + dict(description="End of schema", instance=12), + dict(description="Valid", instance=3737), ], ), - tests=[ - dict(description="Beginning of schema", instance=42), - dict(description="Middle of schema", instance="foo"), - dict(description="End of schema", instance=12), - dict(description="Valid", instance=3737), - ], + uri=URL.parse(Path(__file__).absolute().as_uri()), ) diff --git a/bowtie/schemas/benchmark_report.json b/bowtie/schemas/benchmark_report.json index a321976a7..600ca1c24 100644 --- a/bowtie/schemas/benchmark_report.json +++ b/bowtie/schemas/benchmark_report.json @@ -4,6 +4,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "tag:bowtie.report,2024:benchmark_report", "type": "object", + "additionalProperties": false, "required": ["metadata", "results"], "properties": { "metadata": { @@ -120,16 +121,31 @@ "type": "object", "description": "Contains the Benchmark results for the Benchmark Group.", "title": "Benchmark Group Result", - "required": ["name", "description", "benchmark_results"], + "additionalProperties": false, + "required": [ + "name", + "benchmark_type", + "description", + "varying_parameter", + "benchmark_results" + ], "properties": { "name": { "description": "Benchmark Group's Name.", "type": "string" }, + "benchmark_type": { + "description": "The type of the Benchmark Group (Eg. keyword, default, etc.)", + "type": "string" + }, "description": { "description": "Benchmark Group's Description.", "type": "string" }, + "varying_parameter": { + "description": "The parameter which is changing in the various benchmarks present in the Benchmark Group (Eg. array_size, object_size, etc.)", + "type": ["string", "null"] + }, "benchmark_results": { "description": "Results of all Benchmarks within the Benchmark Group.", "type": "array", diff --git a/bowtie/schemas/benchmarks.json b/bowtie/schemas/benchmarks.json index 750954c19..caea95e23 100644 --- a/bowtie/schemas/benchmarks.json +++ b/bowtie/schemas/benchmarks.json @@ -10,16 +10,33 @@ }, { "type": "object", - "required": ["name", "description", "benchmarks"], + "required": [ + "name", + "benchmark_type", + "dialects_supported", + "description", + "benchmarks" + ], "properties": { "name": { "type": "string", "description": "The name of the benchmark." }, + "benchmark_type": { + "type": "string", + "description": "The type of the benchmark." + }, "description": { "type": "string", "description": "The description of what the benchmark does." }, + "dialects_supported": { + "type": "array", + "description": "An array of supported dialects by the benchmark.", + "items": { + "$ref": "tag:bowtie.report,2024:models:dialect:uri" + } + }, "benchmarks": { "type": "array", "description": "A list of benchmarks.", diff --git a/bowtie/tests/benchmarks/benchmark_with_varying_parameter.py b/bowtie/tests/benchmarks/benchmark_with_varying_parameter.py new file mode 100644 index 000000000..7b874651d --- /dev/null +++ b/bowtie/tests/benchmarks/benchmark_with_varying_parameter.py @@ -0,0 +1,48 @@ +from pathlib import Path + +from url.url import URL + +from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect + + +def get_benchmark(): + return BenchmarkGroup( + name="benchmark", + description=( + "This test is to test that when varying parameter is " + "set then the benchmark output is as expected" + ), + benchmarks=[ + Benchmark.from_dict( + name="benchmark 1", + schema={ + "type": "object", + }, + description="benchmark 1", + tests=[ + { + "description": "test", + "instance": {}, + }, + ], + ), + Benchmark.from_dict( + name="benchmark 2", + schema={ + "type": "object", + }, + description="benchmark 2", + tests=[ + { + "description": "test", + "instance": {}, + }, + ], + ), + ], + uri=URL.parse(Path(__file__).absolute().as_uri()), + benchmark_type="test", + dialects_supported=list(Dialect.known()), + varying_parameter="Array Size", + ) diff --git a/bowtie/tests/benchmarks/valid_benchmark_group.py b/bowtie/tests/benchmarks/valid_benchmark_group.py index 8a6390c3f..68b73230b 100644 --- a/bowtie/tests/benchmarks/valid_benchmark_group.py +++ b/bowtie/tests/benchmarks/valid_benchmark_group.py @@ -3,6 +3,7 @@ from url.url import URL from bowtie._benchmarks import Benchmark, BenchmarkGroup +from bowtie._core import Dialect def get_benchmark(): @@ -29,4 +30,6 @@ def get_benchmark(): ), ], uri=URL.parse(Path(__file__).absolute().as_uri()), + benchmark_type="test", + dialects_supported=list(Dialect.known()), ) diff --git a/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmark_report.json b/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmark_report.json index a321976a7..600ca1c24 100644 --- a/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmark_report.json +++ b/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmark_report.json @@ -4,6 +4,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "tag:bowtie.report,2024:benchmark_report", "type": "object", + "additionalProperties": false, "required": ["metadata", "results"], "properties": { "metadata": { @@ -120,16 +121,31 @@ "type": "object", "description": "Contains the Benchmark results for the Benchmark Group.", "title": "Benchmark Group Result", - "required": ["name", "description", "benchmark_results"], + "additionalProperties": false, + "required": [ + "name", + "benchmark_type", + "description", + "varying_parameter", + "benchmark_results" + ], "properties": { "name": { "description": "Benchmark Group's Name.", "type": "string" }, + "benchmark_type": { + "description": "The type of the Benchmark Group (Eg. keyword, default, etc.)", + "type": "string" + }, "description": { "description": "Benchmark Group's Description.", "type": "string" }, + "varying_parameter": { + "description": "The parameter which is changing in the various benchmarks present in the Benchmark Group (Eg. array_size, object_size, etc.)", + "type": ["string", "null"] + }, "benchmark_results": { "description": "Results of all Benchmarks within the Benchmark Group.", "type": "array", diff --git a/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmarks.json b/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmarks.json index 750954c19..caea95e23 100644 --- a/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmarks.json +++ b/bowtie/tests/fauxmplementations/lintsonschema/schemas/benchmarks.json @@ -10,16 +10,33 @@ }, { "type": "object", - "required": ["name", "description", "benchmarks"], + "required": [ + "name", + "benchmark_type", + "dialects_supported", + "description", + "benchmarks" + ], "properties": { "name": { "type": "string", "description": "The name of the benchmark." }, + "benchmark_type": { + "type": "string", + "description": "The type of the benchmark." + }, "description": { "type": "string", "description": "The description of what the benchmark does." }, + "dialects_supported": { + "type": "array", + "description": "An array of supported dialects by the benchmark.", + "items": { + "$ref": "tag:bowtie.report,2024:models:dialect:uri" + } + }, "benchmarks": { "type": "array", "description": "A list of benchmarks.", diff --git a/bowtie/tests/test_benchmarks.py b/bowtie/tests/test_benchmarks.py index 20f19a77d..f853b9f7f 100644 --- a/bowtie/tests/test_benchmarks.py +++ b/bowtie/tests/test_benchmarks.py @@ -84,18 +84,6 @@ def _validate_benchmark_file(file, module): benchmark_validated(data) -keyword_benchmark_files = ( - ( - dialect.short_name, - benchmark_file, - ) - for dialect in Dialect.known() - for benchmark_file in _iterate_over_benchmark_dir( - bowtie_dir / "benchmarks/keywords" / dialect.short_name, - ) -) - - class TestBenchmarkFormat: def test_validate_single_benchmark(self, valid_single_benchmark): @@ -114,16 +102,12 @@ def test_validate_default_benchmark_format(self, benchmark_file): _validate_benchmark_file(benchmark_file, benchmark_module) @pytest.mark.parametrize( - "dialect,benchmark_file", - keyword_benchmark_files, - ids=lambda param: str(param), + "benchmark_file", + _iterate_over_benchmark_dir(default_benchmarks_dir / "keywords"), + ids=lambda f: str(f), ) - def test_validate_keyword_benchmark_format( - self, - dialect, - benchmark_file, - ): - benchmark_module = f"bowtie.benchmarks.keywords.{dialect}" + def test_validate_keyword_benchmark_format(self, benchmark_file): + benchmark_module = "bowtie.benchmarks.keywords" _validate_benchmark_file(benchmark_file, benchmark_module) @@ -148,7 +132,10 @@ def test_load_single_benchmark_group_from_dict( valid_single_benchmark, ): benchmark = valid_single_benchmark.serializable() - benchmark_group = BenchmarkGroup.from_dict(benchmark) + benchmark["benchmark_type"] = "test" + benchmark_group = BenchmarkGroup.from_dict( + benchmark, + ) assert benchmark_validated(benchmark_group.serializable()) @@ -161,8 +148,8 @@ def test_load_benchmark_group_from_dict(self, valid_benchmark_group): serializable = benchmark_group.serializable() - assert benchmark_json == serializable assert benchmark_validated(serializable) + assert valid_benchmark_group.serializable() == serializable def test_load_single_benchmark_group_from_json( self, @@ -170,7 +157,9 @@ def test_load_single_benchmark_group_from_json( valid_single_benchmark, ): tmp_path = tmp_path / "test_file.json" - tmp_path.write_text(json.dumps(valid_single_benchmark.serializable())) + single_benchmark_json = valid_single_benchmark.serializable() + single_benchmark_json["benchmark_type"] = "test" + tmp_path.write_text(json.dumps(single_benchmark_json)) benchmark_group = BenchmarkGroup.from_file(tmp_path) assert benchmark_validated(benchmark_group.serializable()) @@ -195,7 +184,7 @@ def test_load_benchmark_groups_from_folder(self): Path(__file__).parent / "benchmarks", module="bowtie.tests.benchmarks", ) - valid_benchmarks_for_test = 2 + valid_benchmarks_for_test = 3 valid_count = 0 for benchmark_group in benchmark_groups: @@ -341,7 +330,9 @@ async def test_benchmark_run_markdown_output( # Benchmark Summary ## Benchmark Group: benchmark Benchmark File: None -Benchmark: benchmark + + +Benchmark: Tests with benchmark | Test Name | python-jsonschema | """.strip() @@ -359,6 +350,59 @@ async def test_benchmark_run_markdown_output( assert expected_data1 in stdout assert expected_data2 in stdout + @pytest.mark.asyncio + async def test_benchmark_run_varying_param_markdown( + self, + tmp_path, + ): + from bowtie.tests.benchmarks import benchmark_with_varying_parameter + + tmp_path.joinpath("benchmark.json").write_text( + json.dumps( + benchmark_with_varying_parameter.get_benchmark().serializable(), + ), + ) + stdout, stderr = await bowtie( + "perf", + "-i", + DIRECT_CONNECTABLE, + "-q", + "--format", + "markdown", + tmp_path / "benchmark.json", + exit_code=0, + ) + + expected_data1 = """ +# Benchmark Summary +## Benchmark Group: benchmark +Benchmark File: None + +Benchmark: Tests with varying Array Size + +| Test Name | python-jsonschema | + """.strip() + + expected_data2 = """ +Benchmark: Tests with benchmark 2 + +| Test Name | python-jsonschema | + """.strip() + + expected_data3 = """ + ## Benchmark Metadata + +Runs: 3 +Values: 2 +Warmups: 1 + """.strip() + + # Cant verify the whole output as it would be dynamic + # with differing values + assert expected_data1 in stdout + assert expected_data2 in stdout + assert expected_data3 in stdout + @pytest.mark.asyncio async def test_invalid_benchmark_run(self, invalid_benchmark, tmp_path): tmp_path.joinpath("benchmark.json").write_text(