From ea33d6f90877858ad79fb8681e0cda9028eb8526 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Thu, 30 Jan 2025 15:16:30 +0200 Subject: [PATCH] chore: move rest proxy to own module The REST Proxy requires still split from karapace.core modules, e.g. configuration class is there. --- container/compose.yml | 2 +- mypy.ini | 6 +-- performance-test/README.rst | 2 +- pyproject.toml | 2 +- src/karapace/__main__.py | 2 +- src/karapace/api/container.py | 4 +- src/karapace/api/controller.py | 39 +++++++++---------- src/karapace/api/factory.py | 12 +++--- src/karapace/api/routers/compatibility.py | 4 +- src/karapace/api/routers/config.py | 9 ++--- src/karapace/api/routers/health.py | 6 +-- .../api/routers/master_availability.py | 6 +-- src/karapace/api/routers/metrics.py | 2 +- src/karapace/api/routers/mode.py | 4 +- src/karapace/api/routers/requests.py | 2 +- src/karapace/api/routers/schemas.py | 2 +- src/karapace/api/routers/subjects.py | 8 ++-- src/karapace/api/telemetry/container.py | 6 +-- src/karapace/api/telemetry/meter.py | 4 +- src/karapace/api/telemetry/metrics.py | 6 +-- src/karapace/api/telemetry/middleware.py | 2 +- src/karapace/api/telemetry/setup.py | 6 +-- src/karapace/api/user.py | 2 +- src/karapace/backup/api.py | 2 +- src/karapace/backup/backends/v2.py | 2 +- src/karapace/core/client.py | 4 +- src/karapace/core/container.py | 2 +- .../core/coordinator/master_coordinator.py | 2 +- .../core/instrumentation/prometheus.py | 2 +- src/karapace/core/kafka/admin.py | 2 +- src/karapace/core/protobuf/syntax_reader.py | 5 ++- src/karapace/core/protobuf/type_tree.py | 6 +-- src/karapace/core/schema_reader.py | 2 +- src/karapace/core/schema_registry.py | 6 +-- src/karapace/core/utils.py | 16 -------- .../{core => }/kafka_rest_apis/__init__.py | 29 +++++++------- .../__main__.py} | 2 +- .../kafka_rest_apis/authentication.py | 2 +- .../kafka_rest_apis/consumer_manager.py | 9 +++-- .../kafka_rest_apis/convert_to_int.py | 23 +++++++++++ .../{core => }/kafka_rest_apis/error_codes.py | 0 .../{core => kafka_rest_apis}/karapace.py | 4 +- .../kafka_rest_apis/schema_cache.py | 0 src/karapace/{core => }/rapu.py | 0 stubs/confluent_kafka/__init__.pyi | 6 +-- stubs/confluent_kafka/admin/__init__.pyi | 8 ++-- stubs/confluent_kafka/admin/_config.pyi | 3 +- stubs/confluent_kafka/cimpl.pyi | 3 +- stubs/isodate/duration.pyi | 3 +- stubs/isodate/isoduration.pyi | 4 +- tests/conftest.py | 22 +++++------ tests/e2e/conftest.py | 11 +++--- tests/e2e/instrumentation/test_prometheus.py | 4 +- tests/e2e/kafka/test_admin.py | 4 +- tests/e2e/kafka/test_consumer.py | 7 ++-- tests/e2e/kafka/test_producer.py | 7 ++-- tests/e2e/schema_registry/test_jsonschema.py | 9 +++-- tests/e2e/test_karapace.py | 16 ++++---- tests/integration/backup/test_avro_export.py | 12 +++--- .../backup/test_get_topic_configurations.py | 6 +-- .../integration/backup/test_legacy_backup.py | 19 ++++----- .../backup/test_session_timeout.py | 7 ++-- tests/integration/backup/test_v3_backup.py | 35 +++++++++-------- tests/integration/conftest.py | 35 +++++++++-------- tests/integration/kafka_rest_apis/__init__.py | 0 .../{ => kafka_rest_apis}/test_rest.py | 19 ++++----- .../test_rest_consumer.py | 31 ++++++++------- .../test_rest_consumer_protobuf.py | 8 ++-- tests/integration/test_health_check.py | 7 ++-- tests/integration/test_master_coordinator.py | 7 ++-- tests/integration/test_request_forwarding.py | 13 ++++--- tests/integration/test_schema.py | 21 +++++----- .../integration/test_schema_compatibility.py | 11 +++--- tests/integration/test_schema_coordinator.py | 35 ++++++++++------- tests/integration/test_schema_protobuf.py | 11 +++--- tests/integration/test_schema_reader.py | 9 +++-- .../integration/test_schema_registry_auth.py | 9 +++-- .../integration/test_schema_registry_mode.py | 4 +- tests/integration/utils/cluster.py | 7 ++-- tests/integration/utils/kafka_server.py | 19 ++++----- tests/integration/utils/network.py | 3 +- tests/integration/utils/process.py | 8 ++-- tests/integration/utils/rest_client.py | 8 ++-- tests/integration/utils/zookeeper.py | 1 + .../anonymize_schemas/test_anonymize_avro.py | 5 ++- tests/unit/api/telemetry/test_meter.py | 8 ++-- tests/unit/api/telemetry/test_metrics.py | 10 ++--- tests/unit/api/telemetry/test_middleware.py | 11 +++--- tests/unit/api/telemetry/test_setup.py | 7 ++-- tests/unit/api/telemetry/test_tracer.py | 8 ++-- tests/unit/api/test_controller.py | 15 +++---- tests/unit/api/test_forwarding.py | 14 +++---- .../unit/avro_dataclasses/test_introspect.py | 9 +++-- tests/unit/avro_dataclasses/test_models.py | 9 +++-- tests/unit/backup/backends/test_v1.py | 8 ++-- tests/unit/backup/backends/test_v2.py | 10 ++--- tests/unit/backup/backends/v3/conftest.py | 4 +- tests/unit/backup/backends/v3/test_backend.py | 17 ++++---- .../backup/backends/v3/test_serialisation.py | 17 ++++---- tests/unit/backup/test_api.py | 17 ++++---- tests/unit/backup/test_encoders.py | 5 ++- tests/unit/backup/test_poll_timeout.py | 3 +- tests/unit/backup/test_safe_writer.py | 15 +++---- .../test_jsonschema_compatibility.py | 1 + .../unit/compatibility/test_compatibility.py | 5 ++- tests/unit/instrumentation/test_prometheus.py | 13 ++++--- .../test_authentication.py | 19 ++++----- .../test_rest_proxy_cluster_metadata_cache.py | 32 +++++++-------- tests/unit/protobuf/test_io.py | 4 +- .../unit/protobuf/test_proto_file_element.py | 6 +-- tests/unit/protobuf/test_proto_parser.py | 13 +++---- .../protobuf/test_protobuf_normalization.py | 33 ++++++++-------- .../protobuf/test_protobuf_schema_name.py | 4 +- tests/unit/protobuf/test_protoc.py | 8 ++-- tests/unit/test_auth.py | 4 +- tests/unit/test_avro_compatibility.py | 7 ++-- tests/unit/test_forwarding_client.py | 11 +++--- tests/unit/test_in_memory_database.py | 11 +++--- tests/unit/test_kafka_error_handler.py | 9 +++-- tests/unit/test_key_format.py | 7 ++-- tests/unit/test_logging_setup.py | 9 +++-- .../test_protobuf_binary_serialization.py | 4 +- tests/unit/test_protobuf_serialization.py | 15 +++---- tests/unit/test_rapu.py | 13 ++++--- tests/unit/test_rest_auth.py | 8 ++-- tests/unit/test_schema_models.py | 13 ++++--- tests/unit/test_schema_reader.py | 31 ++++++++------- tests/unit/test_serialization.py | 27 ++++++------- tests/unit/test_utils.py | 7 ++-- tests/utils.py | 23 +++++------ 130 files changed, 657 insertions(+), 582 deletions(-) rename src/karapace/{core => }/kafka_rest_apis/__init__.py (98%) rename src/karapace/{core/karapace_all.py => kafka_rest_apis/__main__.py} (96%) rename src/karapace/{core => }/kafka_rest_apis/authentication.py (98%) rename src/karapace/{core => }/kafka_rest_apis/consumer_manager.py (98%) create mode 100644 src/karapace/kafka_rest_apis/convert_to_int.py rename src/karapace/{core => }/kafka_rest_apis/error_codes.py (100%) rename src/karapace/{core => kafka_rest_apis}/karapace.py (97%) rename src/karapace/{core => }/kafka_rest_apis/schema_cache.py (100%) rename src/karapace/{core => }/rapu.py (100%) create mode 100644 tests/integration/kafka_rest_apis/__init__.py rename tests/integration/{ => kafka_rest_apis}/test_rest.py (99%) rename tests/integration/{ => kafka_rest_apis}/test_rest_consumer.py (98%) rename tests/integration/{ => kafka_rest_apis}/test_rest_consumer_protobuf.py (99%) rename tests/unit/{ => kafka_rest_apis}/test_authentication.py (97%) diff --git a/container/compose.yml b/container/compose.yml index e1a8283dc..59fb7a511 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -105,7 +105,7 @@ services: entrypoint: - python3 - -m - - karapace.core.karapace_all + - karapace.kafka_rest_apis depends_on: - kafka - karapace-schema-registry diff --git a/mypy.ini b/mypy.ini index b7c44fe87..db8294644 100644 --- a/mypy.ini +++ b/mypy.ini @@ -59,16 +59,16 @@ ignore_errors = True [mypy-karapace.core.utils] ignore_errors = True -[mypy-karapace.core.rapu] +[mypy-karapace.rapu] ignore_errors = True [mypy-karapace.core.serialization] ignore_errors = True -[mypy-karapace.core.kafka_rest_apis.consumer_manager] +[mypy-karapace.kafka_rest_apis.consumer_manager] ignore_errors = True -[mypy-karapace.core.kafka_rest_apis] +[mypy-karapace.kafka_rest_apis] ignore_errors = True # Third-party libraries with no stubs available. Before adding libraries here, diff --git a/performance-test/README.rst b/performance-test/README.rst index fbb020083..f1e5043c1 100644 --- a/performance-test/README.rst +++ b/performance-test/README.rst @@ -13,7 +13,7 @@ Create if necessary the `_schemas` topic to Kafka:: Run Karapace from repository root:: cd .. - python -m karapace.core.karapace_all karapace.config.json + python -m karapace.karapace_rest_apis karapace.config.json Performance test is run from the shell script:: ./run-locust-test.sh diff --git a/pyproject.toml b/pyproject.toml index aee3ddb94..ec7113883 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ classifiers=[ ] [project.scripts] -karapace = "karapace.core.karapace_all:main" +karapace = "karapace.kafka_rest_apis:main" karapace_schema_backup = "karapace.backup.cli:main" karapace_mkpasswd = "karapace.core.auth:main" diff --git a/src/karapace/__main__.py b/src/karapace/__main__.py index e9be07fea..da954488b 100644 --- a/src/karapace/__main__.py +++ b/src/karapace/__main__.py @@ -3,10 +3,10 @@ See LICENSE for details """ -from karapace.core.container import KarapaceContainer from karapace.api.container import SchemaRegistryContainer from karapace.api.factory import create_karapace_application, karapace_schema_registry_lifespan from karapace.api.telemetry.container import TelemetryContainer +from karapace.core.container import KarapaceContainer import karapace.api.controller import karapace.api.factory diff --git a/src/karapace/api/container.py b/src/karapace/api/container.py index 2c1903a63..e70d4509a 100644 --- a/src/karapace/api/container.py +++ b/src/karapace/api/container.py @@ -4,10 +4,10 @@ """ from dependency_injector import containers, providers -from karapace.core.container import KarapaceContainer from karapace.api.controller import KarapaceSchemaRegistryController -from karapace.core.schema_registry import KarapaceSchemaRegistry from karapace.api.telemetry.container import TelemetryContainer +from karapace.core.container import KarapaceContainer +from karapace.core.schema_registry import KarapaceSchemaRegistry class SchemaRegistryContainer(containers.DeclarativeContainer): diff --git a/src/karapace/api/controller.py b/src/karapace/api/controller.py index 9257541de..7095ba30c 100644 --- a/src/karapace/api/controller.py +++ b/src/karapace/api/controller.py @@ -8,6 +8,22 @@ from avro.errors import SchemaParseException from dependency_injector.wiring import inject, Provide from fastapi import Depends, HTTPException, Request, status +from karapace.api.forward_client import ForwardClient +from karapace.api.routers.errors import no_primary_url_error, SchemaErrorCodes, SchemaErrorMessages +from karapace.api.routers.requests import ( + CompatibilityCheckResponse, + CompatibilityLevelResponse, + CompatibilityRequest, + CompatibilityResponse, + ModeResponse, + SchemaIdResponse, + SchemaListingItem, + SchemaRequest, + SchemaResponse, + SchemasResponse, + SubjectSchemaVersionResponse, + SubjectVersion, +) from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User from karapace.core.compatibility import CompatibilityModes from karapace.core.compatibility.jsonschema.checks import is_incompatible @@ -30,7 +46,6 @@ SubjectSoftDeletedException, VersionNotFoundException, ) -from karapace.api.forward_client import ForwardClient from karapace.core.protobuf.exception import ProtobufUnresolvedDependencyException from karapace.core.schema_models import ( ParsedTypedSchema, @@ -41,25 +56,10 @@ Versioner, ) from karapace.core.schema_references import LatestVersionReference, Reference +from karapace.core.schema_registry import KarapaceSchemaRegistry from karapace.core.statsd import StatsClient from karapace.core.typing import JsonData, JsonObject, SchemaId, Subject, Version from karapace.core.utils import JSONDecodeError -from karapace.core.schema_registry import KarapaceSchemaRegistry -from karapace.api.routers.errors import no_primary_url_error, SchemaErrorCodes, SchemaErrorMessages -from karapace.api.routers.requests import ( - CompatibilityCheckResponse, - CompatibilityLevelResponse, - CompatibilityRequest, - CompatibilityResponse, - ModeResponse, - SchemaIdResponse, - SchemaListingItem, - SchemaRequest, - SchemaResponse, - SchemasResponse, - SubjectSchemaVersionResponse, - SubjectVersion, -) from typing import Any, cast import json @@ -517,8 +517,7 @@ async def subject_version_delete( detail={ "error_code": SchemaErrorCodes.SCHEMAVERSION_SOFT_DELETED.value, "message": ( - f"Subject '{subject}' Version {version} was soft deleted. " - "Set permanent=true to delete permanently" + f"Subject '{subject}' Version {version} was soft deleted. Set permanent=true to delete permanently" ), }, ) from exc @@ -528,7 +527,7 @@ async def subject_version_delete( detail={ "error_code": SchemaErrorCodes.SCHEMAVERSION_NOT_SOFT_DELETED.value, "message": ( - f"Subject '{subject}' Version {version} was not deleted " "first before being permanently deleted" + f"Subject '{subject}' Version {version} was not deleted first before being permanently deleted" ), }, ) from exc diff --git a/src/karapace/api/factory.py b/src/karapace/api/factory.py index 3e28ca1ee..cf60a2409 100644 --- a/src/karapace/api/factory.py +++ b/src/karapace/api/factory.py @@ -8,17 +8,17 @@ from dependency_injector.wiring import inject, Provide from fastapi import Depends, FastAPI from karapace import version as karapace_version -from karapace.core.auth import AuthenticatorAndAuthorizer -from karapace.core.config import Config -from karapace.api.forward_client import ForwardClient -from karapace.core.logging_setup import configure_logging, log_config_without_secrets -from karapace.core.statsd import StatsClient from karapace.api.container import SchemaRegistryContainer +from karapace.api.forward_client import ForwardClient from karapace.api.http_handlers import setup_exception_handlers from karapace.api.middlewares import setup_middlewares -from karapace.core.schema_registry import KarapaceSchemaRegistry from karapace.api.routers.setup import setup_routers from karapace.api.telemetry.setup import setup_metering, setup_tracing +from karapace.core.auth import AuthenticatorAndAuthorizer +from karapace.core.config import Config +from karapace.core.logging_setup import configure_logging, log_config_without_secrets +from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.core.statsd import StatsClient from typing import AsyncContextManager import logging diff --git a/src/karapace/api/routers/compatibility.py b/src/karapace/api/routers/compatibility.py index 38a7ec51f..c0079159b 100644 --- a/src/karapace/api/routers/compatibility.py +++ b/src/karapace/api/routers/compatibility.py @@ -5,14 +5,14 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends -from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User -from karapace.core.typing import Subject from karapace.api.container import SchemaRegistryContainer from karapace.api.controller import KarapaceSchemaRegistryController from karapace.api.routers.errors import unauthorized from karapace.api.routers.raw_path_router import RawPathRoute from karapace.api.routers.requests import CompatibilityCheckResponse, SchemaRequest from karapace.api.user import get_current_user +from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User +from karapace.core.typing import Subject from typing import Annotated from urllib.parse import unquote_plus diff --git a/src/karapace/api/routers/config.py b/src/karapace/api/routers/config.py index 6a31373f0..5eaa1d365 100644 --- a/src/karapace/api/routers/config.py +++ b/src/karapace/api/routers/config.py @@ -5,20 +5,19 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Request -from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User -from karapace.api.forward_client import ForwardClient -from karapace.core.typing import Subject from karapace.api.container import SchemaRegistryContainer from karapace.api.controller import KarapaceSchemaRegistryController -from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.api.forward_client import ForwardClient from karapace.api.routers.errors import no_primary_url_error, unauthorized from karapace.api.routers.raw_path_router import RawPathRoute from karapace.api.routers.requests import CompatibilityLevelResponse, CompatibilityRequest, CompatibilityResponse from karapace.api.user import get_current_user +from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User +from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.core.typing import Subject from typing import Annotated from urllib.parse import unquote_plus - config_router = APIRouter( prefix="/config", tags=["config"], diff --git a/src/karapace/api/routers/health.py b/src/karapace/api/routers/health.py index 33f8aee23..5a086fd8d 100644 --- a/src/karapace/api/routers/health.py +++ b/src/karapace/api/routers/health.py @@ -5,12 +5,12 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, HTTPException, status +from karapace.api.container import SchemaRegistryContainer +from karapace.api.telemetry.tracer import Tracer +from karapace.core.schema_registry import KarapaceSchemaRegistry from opentelemetry.trace import Span from opentelemetry.trace.status import StatusCode from pydantic import BaseModel -from karapace.api.container import SchemaRegistryContainer -from karapace.core.schema_registry import KarapaceSchemaRegistry -from karapace.api.telemetry.tracer import Tracer class HealthStatus(BaseModel): diff --git a/src/karapace/api/routers/master_availability.py b/src/karapace/api/routers/master_availability.py index d961b417b..aeef28df6 100644 --- a/src/karapace/api/routers/master_availability.py +++ b/src/karapace/api/routers/master_availability.py @@ -5,11 +5,11 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Request, Response -from karapace.core.config import Config -from karapace.api.forward_client import ForwardClient -from pydantic import BaseModel from karapace.api.container import SchemaRegistryContainer +from karapace.api.forward_client import ForwardClient +from karapace.core.config import Config from karapace.core.schema_registry import KarapaceSchemaRegistry +from pydantic import BaseModel from typing import Final import logging diff --git a/src/karapace/api/routers/metrics.py b/src/karapace/api/routers/metrics.py index 586b894df..313e4c3e2 100644 --- a/src/karapace/api/routers/metrics.py +++ b/src/karapace/api/routers/metrics.py @@ -5,8 +5,8 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Response -from karapace.core.instrumentation.prometheus import PrometheusInstrumentation from karapace.api.container import SchemaRegistryContainer +from karapace.core.instrumentation.prometheus import PrometheusInstrumentation metrics_router = APIRouter( prefix=PrometheusInstrumentation.METRICS_ENDPOINT_PATH, diff --git a/src/karapace/api/routers/mode.py b/src/karapace/api/routers/mode.py index bfe7dacc4..a2e78a1a6 100644 --- a/src/karapace/api/routers/mode.py +++ b/src/karapace/api/routers/mode.py @@ -5,14 +5,14 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends -from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User -from karapace.core.typing import Subject from karapace.api.container import SchemaRegistryContainer from karapace.api.controller import KarapaceSchemaRegistryController from karapace.api.routers.errors import unauthorized from karapace.api.routers.raw_path_router import RawPathRoute from karapace.api.routers.requests import ModeResponse from karapace.api.user import get_current_user +from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User +from karapace.core.typing import Subject from typing import Annotated from urllib.parse import unquote_plus diff --git a/src/karapace/api/routers/requests.py b/src/karapace/api/routers/requests.py index 8a8731fe3..d9bde265a 100644 --- a/src/karapace/api/routers/requests.py +++ b/src/karapace/api/routers/requests.py @@ -3,10 +3,10 @@ See LICENSE for details """ +from karapace.api.routers.errors import KarapaceValidationError from karapace.core.schema_type import SchemaType from karapace.core.typing import Subject from pydantic import BaseModel, ConfigDict, Field, field_validator -from karapace.api.routers.errors import KarapaceValidationError from typing import Any diff --git a/src/karapace/api/routers/schemas.py b/src/karapace/api/routers/schemas.py index 94283190b..7c8c1c944 100644 --- a/src/karapace/api/routers/schemas.py +++ b/src/karapace/api/routers/schemas.py @@ -5,11 +5,11 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Query -from karapace.core.auth import AuthenticatorAndAuthorizer, User from karapace.api.container import SchemaRegistryContainer from karapace.api.controller import KarapaceSchemaRegistryController from karapace.api.routers.requests import SchemaListingItem, SchemasResponse, SubjectVersion from karapace.api.user import get_current_user +from karapace.core.auth import AuthenticatorAndAuthorizer, User from typing import Annotated schemas_router = APIRouter( diff --git a/src/karapace/api/routers/subjects.py b/src/karapace/api/routers/subjects.py index 300d42657..73650d1eb 100644 --- a/src/karapace/api/routers/subjects.py +++ b/src/karapace/api/routers/subjects.py @@ -5,16 +5,16 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Request -from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User -from karapace.api.forward_client import ForwardClient -from karapace.core.typing import Subject from karapace.api.container import SchemaRegistryContainer from karapace.api.controller import KarapaceSchemaRegistryController -from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.api.forward_client import ForwardClient from karapace.api.routers.errors import no_primary_url_error, unauthorized from karapace.api.routers.raw_path_router import RawPathRoute from karapace.api.routers.requests import SchemaIdResponse, SchemaRequest, SchemaResponse, SubjectSchemaVersionResponse from karapace.api.user import get_current_user +from karapace.core.auth import AuthenticatorAndAuthorizer, Operation, User +from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.core.typing import Subject from typing import Annotated from urllib.parse import unquote_plus diff --git a/src/karapace/api/telemetry/container.py b/src/karapace/api/telemetry/container.py index 4dd2d3d73..6c7d1e397 100644 --- a/src/karapace/api/telemetry/container.py +++ b/src/karapace/api/telemetry/container.py @@ -4,14 +4,14 @@ """ from dependency_injector import containers, providers +from karapace.api.telemetry.meter import Meter +from karapace.api.telemetry.metrics import HTTPRequestMetrics +from karapace.api.telemetry.tracer import Tracer from karapace.core.config import Config from karapace.core.container import KarapaceContainer from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.semconv.attributes import telemetry_attributes as T -from karapace.api.telemetry.meter import Meter -from karapace.api.telemetry.metrics import HTTPRequestMetrics -from karapace.api.telemetry.tracer import Tracer def create_telemetry_resource(config: Config) -> Resource: diff --git a/src/karapace/api/telemetry/meter.py b/src/karapace/api/telemetry/meter.py index c90c1c69b..f9f3f5c4a 100644 --- a/src/karapace/api/telemetry/meter.py +++ b/src/karapace/api/telemetry/meter.py @@ -11,10 +11,10 @@ from opentelemetry.sdk.metrics.export import ( ConsoleMetricExporter, MetricExporter, + MetricExportResult, MetricReader, - PeriodicExportingMetricReader, MetricsData, - MetricExportResult, + PeriodicExportingMetricReader, ) from typing import Any diff --git a/src/karapace/api/telemetry/metrics.py b/src/karapace/api/telemetry/metrics.py index 08a44a742..3db7232eb 100644 --- a/src/karapace/api/telemetry/metrics.py +++ b/src/karapace/api/telemetry/metrics.py @@ -3,11 +3,11 @@ See LICENSE for details """ -from fastapi import Request, Response, HTTPException -from opentelemetry.metrics import Counter, Histogram, UpDownCounter +from collections.abc import Mapping +from fastapi import HTTPException, Request, Response from karapace.api.telemetry.meter import Meter +from opentelemetry.metrics import Counter, Histogram, UpDownCounter from typing import Final -from collections.abc import Mapping import time diff --git a/src/karapace/api/telemetry/middleware.py b/src/karapace/api/telemetry/middleware.py index 0853b28ff..34eebccb4 100644 --- a/src/karapace/api/telemetry/middleware.py +++ b/src/karapace/api/telemetry/middleware.py @@ -6,10 +6,10 @@ from collections.abc import Awaitable, Callable from dependency_injector.wiring import inject, Provide from fastapi import FastAPI, Request, Response -from opentelemetry.trace import SpanKind, Status, StatusCode from karapace.api.telemetry.container import TelemetryContainer from karapace.api.telemetry.metrics import HTTPRequestMetrics from karapace.api.telemetry.tracer import Tracer +from opentelemetry.trace import SpanKind, Status, StatusCode import logging diff --git a/src/karapace/api/telemetry/setup.py b/src/karapace/api/telemetry/setup.py index dcf262673..61aef45f2 100644 --- a/src/karapace/api/telemetry/setup.py +++ b/src/karapace/api/telemetry/setup.py @@ -4,13 +4,13 @@ """ from dependency_injector.wiring import inject, Provide +from karapace.api.telemetry.container import TelemetryContainer +from karapace.api.telemetry.meter import Meter +from karapace.api.telemetry.tracer import Tracer from opentelemetry import metrics, trace from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider -from karapace.api.telemetry.container import TelemetryContainer -from karapace.api.telemetry.meter import Meter -from karapace.api.telemetry.tracer import Tracer import logging diff --git a/src/karapace/api/user.py b/src/karapace/api/user.py index 712634810..5d6911517 100644 --- a/src/karapace/api/user.py +++ b/src/karapace/api/user.py @@ -6,8 +6,8 @@ from dependency_injector.wiring import inject, Provide from fastapi import Depends, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials -from karapace.core.auth import AuthenticationError, AuthenticatorAndAuthorizer, User from karapace.api.container import SchemaRegistryContainer +from karapace.core.auth import AuthenticationError, AuthenticatorAndAuthorizer, User from typing import Annotated diff --git a/src/karapace/backup/api.py b/src/karapace/backup/api.py index b2fb32c6d..88f89fc78 100644 --- a/src/karapace/backup/api.py +++ b/src/karapace/backup/api.py @@ -28,10 +28,10 @@ from confluent_kafka import Message, TopicPartition from enum import Enum from functools import partial -from karapace.core import constants from karapace.backup.backends.v1 import SchemaBackupV1Reader from karapace.backup.backends.v2 import AnonymizeAvroWriter, SchemaBackupV2Reader, SchemaBackupV2Writer, V2_MARKER from karapace.backup.backends.v3.backend import SchemaBackupV3Reader, SchemaBackupV3Writer, VerifyFailure, VerifySuccess +from karapace.core import constants from karapace.core.config import Config from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.common import translate_from_kafkaerror diff --git a/src/karapace/backup/backends/v2.py b/src/karapace/backup/backends/v2.py index ce47b864b..e4e1cc366 100644 --- a/src/karapace/backup/backends/v2.py +++ b/src/karapace/backup/backends/v2.py @@ -6,9 +6,9 @@ from __future__ import annotations from collections.abc import Generator, Sequence -from karapace.core.anonymize_schemas import anonymize_avro from karapace.backup.backends.reader import BaseItemsBackupReader from karapace.backup.backends.writer import BaseKVBackupWriter, StdOut +from karapace.core.anonymize_schemas import anonymize_avro from karapace.core.utils import json_decode, json_encode from pathlib import Path from typing import Any, ClassVar, Final, IO diff --git a/src/karapace/core/client.py b/src/karapace/core/client.py index 543b5810c..0d2b2ed9b 100644 --- a/src/karapace/core/client.py +++ b/src/karapace/core/client.py @@ -5,11 +5,11 @@ See LICENSE for details """ -from typing import Literal from aiohttp import BasicAuth, ClientSession from collections.abc import Awaitable, Callable, Mapping from karapace.core.typing import JsonData -from urllib.parse import urljoin, quote_plus +from typing import Literal +from urllib.parse import quote_plus, urljoin import logging import ssl diff --git a/src/karapace/core/container.py b/src/karapace/core/container.py index 144cdffde..4d9085b60 100644 --- a/src/karapace/core/container.py +++ b/src/karapace/core/container.py @@ -4,9 +4,9 @@ """ from dependency_injector import containers, providers +from karapace.api.forward_client import ForwardClient from karapace.core.auth import get_authorizer, HTTPAuthorizer, NoAuthAndAuthz from karapace.core.config import Config -from karapace.api.forward_client import ForwardClient from karapace.core.instrumentation.prometheus import PrometheusInstrumentation from karapace.core.statsd import StatsClient diff --git a/src/karapace/core/coordinator/master_coordinator.py b/src/karapace/core/coordinator/master_coordinator.py index 1ea2be246..649d4dca9 100644 --- a/src/karapace/core/coordinator/master_coordinator.py +++ b/src/karapace/core/coordinator/master_coordinator.py @@ -11,11 +11,11 @@ from aiokafka.errors import KafkaConnectionError from aiokafka.helpers import create_ssl_context from aiokafka.protocol.commit import OffsetCommitRequest_v2 as OffsetCommitRequest +from karapace.api.telemetry.tracer import Tracer from karapace.core.config import Config from karapace.core.coordinator.schema_coordinator import SchemaCoordinator, SchemaCoordinatorStatus from karapace.core.kafka.types import DEFAULT_REQUEST_TIMEOUT_MS from karapace.core.typing import PrimaryInfo, SchemaReaderStoppper -from karapace.api.telemetry.tracer import Tracer from threading import Thread from typing import Final diff --git a/src/karapace/core/instrumentation/prometheus.py b/src/karapace/core/instrumentation/prometheus.py index 6619cc830..2e6acca30 100644 --- a/src/karapace/core/instrumentation/prometheus.py +++ b/src/karapace/core/instrumentation/prometheus.py @@ -10,7 +10,7 @@ from aiohttp.web import middleware, Request, Response from collections.abc import Awaitable, Callable -from karapace.core.rapu import RestApp +from karapace.rapu import RestApp from prometheus_client import CollectorRegistry, Counter, Gauge, generate_latest, Histogram from typing import Final diff --git a/src/karapace/core/kafka/admin.py b/src/karapace/core/kafka/admin.py index 0ebe759ad..abcbb5209 100644 --- a/src/karapace/core/kafka/admin.py +++ b/src/karapace/core/kafka/admin.py @@ -20,6 +20,7 @@ TopicMetadata, ) from confluent_kafka.error import KafkaException +from karapace.api.telemetry.tracer import Tracer from karapace.core.constants import TOPIC_CREATION_TIMEOUT_S from karapace.core.kafka.common import ( _KafkaConfigMixin, @@ -28,7 +29,6 @@ single_futmap_result, UnknownTopicOrPartitionError, ) -from karapace.api.telemetry.tracer import Tracer from typing_extensions import Unpack diff --git a/src/karapace/core/protobuf/syntax_reader.py b/src/karapace/core/protobuf/syntax_reader.py index a886945f1..394955e55 100644 --- a/src/karapace/core/protobuf/syntax_reader.py +++ b/src/karapace/core/protobuf/syntax_reader.py @@ -3,10 +3,11 @@ See LICENSE for details """ -# Ported from square/wire: -# wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/SyntaxReader.kt from karapace.core.protobuf.exception import IllegalStateException from karapace.core.protobuf.location import Location + +# Ported from square/wire: +# wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/SyntaxReader.kt from typing import NoReturn diff --git a/src/karapace/core/protobuf/type_tree.py b/src/karapace/core/protobuf/type_tree.py index bb58c0c2d..b103bca99 100644 --- a/src/karapace/core/protobuf/type_tree.py +++ b/src/karapace/core/protobuf/type_tree.py @@ -89,9 +89,9 @@ def type_in_tree(self, queried_type: str) -> TypeTree | None: def expand_missing_absolute_path(self) -> Sequence[str]: oldest_import = self.oldest_matching_import() expanded_missing_path = self.expand_missing_absolute_path_recursive(oldest_import) - assert ( - expanded_missing_path is not None - ), "each node should have, by construction, at least a leaf that is a fully specified path" + assert expanded_missing_path is not None, ( + "each node should have, by construction, at least a leaf that is a fully specified path" + ) return expanded_missing_path[:-1] # skipping myself since I was matched @property diff --git a/src/karapace/core/schema_reader.py b/src/karapace/core/schema_reader.py index e84e81d1e..d90a7cd9d 100644 --- a/src/karapace/core/schema_reader.py +++ b/src/karapace/core/schema_reader.py @@ -27,6 +27,7 @@ from contextlib import closing, ExitStack from enum import Enum from jsonschema.validators import Draft7Validator +from karapace.api.telemetry.tracer import Tracer from karapace.core import constants from karapace.core.config import Config from karapace.core.coordinator.master_coordinator import MasterCoordinator @@ -46,7 +47,6 @@ from karapace.core.statsd import StatsClient from karapace.core.typing import JsonObject, SchemaReaderStoppper, Subject, Version from karapace.core.utils import json_decode, JSONDecodeError, shutdown -from karapace.api.telemetry.tracer import Tracer from threading import Event, Lock, Thread from typing import Final diff --git a/src/karapace/core/schema_registry.py b/src/karapace/core/schema_registry.py index fc925b811..a2bbdce74 100644 --- a/src/karapace/core/schema_registry.py +++ b/src/karapace/core/schema_registry.py @@ -8,6 +8,7 @@ from avro.compatibility import SchemaCompatibilityResult, SchemaCompatibilityType from collections.abc import Sequence from contextlib import AsyncExitStack, closing +from karapace.api.telemetry.tracer import Tracer from karapace.core.compatibility import CompatibilityModes from karapace.core.compatibility.jsonschema.checks import is_incompatible from karapace.core.compatibility.schema_compatibility import SchemaCompatibility @@ -27,6 +28,7 @@ ) from karapace.core.in_memory_database import InMemoryDatabase from karapace.core.key_format import KeyFormatter +from karapace.core.messaging import KarapaceProducer from karapace.core.offset_watcher import OffsetWatcher from karapace.core.schema_models import ( ParsedTypedSchema, @@ -36,11 +38,9 @@ ValidatedTypedSchema, Versioner, ) +from karapace.core.schema_reader import KafkaSchemaReader from karapace.core.schema_references import LatestVersionReference, Reference from karapace.core.typing import JsonObject, Mode, PrimaryInfo, SchemaId, Subject, Version -from karapace.core.messaging import KarapaceProducer -from karapace.core.schema_reader import KafkaSchemaReader -from karapace.api.telemetry.tracer import Tracer import asyncio import logging diff --git a/src/karapace/core/utils.py b/src/karapace/core/utils.py index 2190459ca..6008e8a2f 100644 --- a/src/karapace/core/utils.py +++ b/src/karapace/core/utils.py @@ -14,7 +14,6 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from decimal import Decimal -from http import HTTPStatus from pathlib import Path from types import MappingProxyType from typing import AnyStr, cast, IO, Literal, NoReturn, overload, TypeVar @@ -191,21 +190,6 @@ def raise_timeout_if_expired(self, msg_format: str, *args: object, **kwargs: obj raise Timeout(msg_format.format(*args, **kwargs)) -def convert_to_int(object_: dict, key: str, content_type: str) -> None: - if object_.get(key) is None: - return - try: - object_[key] = int(object_[key]) - except ValueError: - from karapace.core.rapu import http_error - - http_error( - message=f"{key} is not a valid int: {object_[key]}", - content_type=content_type, - code=HTTPStatus.INTERNAL_SERVER_ERROR, - ) - - class DebugAccessLogger(AccessLogger): """ Logs access logs as DEBUG instead of INFO. diff --git a/src/karapace/core/kafka_rest_apis/__init__.py b/src/karapace/kafka_rest_apis/__init__.py similarity index 98% rename from src/karapace/core/kafka_rest_apis/__init__.py rename to src/karapace/kafka_rest_apis/__init__.py index 61f3847b6..ac3ca0c8e 100644 --- a/src/karapace/core/kafka_rest_apis/__init__.py +++ b/src/karapace/kafka_rest_apis/__init__.py @@ -25,16 +25,6 @@ from karapace.core.errors import InvalidSchema from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.producer import AsyncKafkaProducer -from karapace.core.kafka_rest_apis.authentication import ( - get_auth_config_from_header, - get_expiration_time_from_header, - get_kafka_client_auth_parameters_from_config, -) -from karapace.core.kafka_rest_apis.consumer_manager import ConsumerManager -from karapace.core.kafka_rest_apis.error_codes import RESTErrorCodes -from karapace.core.kafka_rest_apis.schema_cache import TopicSchemaCache -from karapace.core.karapace import KarapaceBase -from karapace.core.rapu import HTTPRequest, JSON_CONTENT_TYPE from karapace.core.schema_models import TypedSchema, ValidatedTypedSchema from karapace.core.schema_type import SchemaType from karapace.core.serialization import ( @@ -45,7 +35,18 @@ SchemaRetrievalError, ) from karapace.core.typing import NameStrategy, SchemaId, Subject, SubjectType -from karapace.core.utils import convert_to_int, json_encode +from karapace.core.utils import json_encode +from karapace.kafka_rest_apis.authentication import ( + get_auth_config_from_header, + get_expiration_time_from_header, + get_kafka_client_auth_parameters_from_config, +) +from karapace.kafka_rest_apis.consumer_manager import ConsumerManager +from karapace.kafka_rest_apis.convert_to_int import convert_to_int +from karapace.kafka_rest_apis.error_codes import RESTErrorCodes +from karapace.kafka_rest_apis.karapace import KarapaceBase +from karapace.kafka_rest_apis.schema_cache import TopicSchemaCache +from karapace.rapu import HTTPRequest, JSON_CONTENT_TYPE from typing import TypedDict import asyncio @@ -746,7 +747,7 @@ async def cluster_metadata(self, topics: list[str] | None = None) -> _ClusterMet ) return metadata - def init_admin_client(self, verify_connection: bool = True) -> KafkaAdminClient: + def init_admin_client(self, verify_connection: bool = True) -> None: for retry in [True, True, False]: try: self.admin_client = KafkaAdminClient( @@ -1013,9 +1014,9 @@ async def validate_schema_info( ) except InvalidSchema: if f"{subject_type}_schema" in data: - err = f'schema = {data[f"{subject_type}_schema"]}' + err = f"schema = {data[f'{subject_type}_schema']}" else: - err = f'schema_id = {data[f"{subject_type}_schema_id"]}' + err = f"schema_id = {data[f'{subject_type}_schema_id']}" KafkaRest.r( body={ "error_code": RESTErrorCodes.INVALID_DATA.value, diff --git a/src/karapace/core/karapace_all.py b/src/karapace/kafka_rest_apis/__main__.py similarity index 96% rename from src/karapace/core/karapace_all.py rename to src/karapace/kafka_rest_apis/__main__.py index 1083b45eb..5ab1a3a2f 100644 --- a/src/karapace/core/karapace_all.py +++ b/src/karapace/kafka_rest_apis/__main__.py @@ -10,8 +10,8 @@ from karapace.core.config import Config from karapace.core.container import KarapaceContainer from karapace.core.instrumentation.prometheus import PrometheusInstrumentation -from karapace.core.kafka_rest_apis import KafkaRest from karapace.core.logging_setup import configure_logging, log_config_without_secrets +from karapace.kafka_rest_apis import KafkaRest import argparse import logging diff --git a/src/karapace/core/kafka_rest_apis/authentication.py b/src/karapace/kafka_rest_apis/authentication.py similarity index 98% rename from src/karapace/core/kafka_rest_apis/authentication.py rename to src/karapace/kafka_rest_apis/authentication.py index f9b92ba9d..af05016b0 100644 --- a/src/karapace/core/kafka_rest_apis/authentication.py +++ b/src/karapace/kafka_rest_apis/authentication.py @@ -7,7 +7,7 @@ from http import HTTPStatus from karapace.core.config import Config -from karapace.core.rapu import HTTPResponse, JSON_CONTENT_TYPE +from karapace.rapu import HTTPResponse, JSON_CONTENT_TYPE from typing import NoReturn, TypedDict import aiohttp diff --git a/src/karapace/core/kafka_rest_apis/consumer_manager.py b/src/karapace/kafka_rest_apis/consumer_manager.py similarity index 98% rename from src/karapace/core/kafka_rest_apis/consumer_manager.py rename to src/karapace/kafka_rest_apis/consumer_manager.py index 887f8da20..62a7cceb1 100644 --- a/src/karapace/core/kafka_rest_apis/consumer_manager.py +++ b/src/karapace/kafka_rest_apis/consumer_manager.py @@ -20,11 +20,12 @@ from karapace.core.kafka.common import translate_from_kafkaerror from karapace.core.kafka.consumer import AsyncKafkaConsumer from karapace.core.kafka.types import DEFAULT_REQUEST_TIMEOUT_MS, Timestamp -from karapace.core.kafka_rest_apis.authentication import get_kafka_client_auth_parameters_from_config -from karapace.core.kafka_rest_apis.error_codes import RESTErrorCodes -from karapace.core.karapace import empty_response, KarapaceBase from karapace.core.serialization import DeserializationError, InvalidMessageHeader, InvalidPayload, SchemaRegistrySerializer -from karapace.core.utils import convert_to_int, json_decode, JSONDecodeError +from karapace.core.utils import json_decode, JSONDecodeError +from karapace.kafka_rest_apis.convert_to_int import convert_to_int +from karapace.kafka_rest_apis.authentication import get_kafka_client_auth_parameters_from_config +from karapace.kafka_rest_apis.error_codes import RESTErrorCodes +from karapace.kafka_rest_apis.karapace import empty_response, KarapaceBase from struct import error as UnpackError from urllib.parse import urljoin diff --git a/src/karapace/kafka_rest_apis/convert_to_int.py b/src/karapace/kafka_rest_apis/convert_to_int.py new file mode 100644 index 000000000..02aaae87a --- /dev/null +++ b/src/karapace/kafka_rest_apis/convert_to_int.py @@ -0,0 +1,23 @@ +""" +karapace - convert_to_int + +Copyright (c) 2025 Aiven Ltd +See LICENSE for details +""" + +from http import HTTPStatus + + +def convert_to_int(object_: dict, key: str, content_type: str) -> None: + if object_.get(key) is None: + return + try: + object_[key] = int(object_[key]) + except ValueError: + from karapace.rapu import http_error + + http_error( + message=f"{key} is not a valid int: {object_[key]}", + content_type=content_type, + code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) diff --git a/src/karapace/core/kafka_rest_apis/error_codes.py b/src/karapace/kafka_rest_apis/error_codes.py similarity index 100% rename from src/karapace/core/kafka_rest_apis/error_codes.py rename to src/karapace/kafka_rest_apis/error_codes.py diff --git a/src/karapace/core/karapace.py b/src/karapace/kafka_rest_apis/karapace.py similarity index 97% rename from src/karapace/core/karapace.py rename to src/karapace/kafka_rest_apis/karapace.py index e1bf736c8..9cab3a107 100644 --- a/src/karapace/core/karapace.py +++ b/src/karapace/kafka_rest_apis/karapace.py @@ -1,5 +1,5 @@ """ -karapace - main +karapace - karapace base Copyright (c) 2023 Aiven Ltd See LICENSE for details @@ -13,9 +13,9 @@ from http import HTTPStatus from karapace.core.config import Config from karapace.core.dataclasses import default_dataclass -from karapace.core.rapu import HTTPRequest, HTTPResponse, RestApp from karapace.core.typing import JsonObject from karapace.core.utils import json_encode +from karapace.rapu import HTTPRequest, HTTPResponse, RestApp from karapace.version import __version__ from typing import NoReturn, TypeAlias diff --git a/src/karapace/core/kafka_rest_apis/schema_cache.py b/src/karapace/kafka_rest_apis/schema_cache.py similarity index 100% rename from src/karapace/core/kafka_rest_apis/schema_cache.py rename to src/karapace/kafka_rest_apis/schema_cache.py diff --git a/src/karapace/core/rapu.py b/src/karapace/rapu.py similarity index 100% rename from src/karapace/core/rapu.py rename to src/karapace/rapu.py diff --git a/stubs/confluent_kafka/__init__.pyi b/stubs/confluent_kafka/__init__.pyi index e27cf4880..07ffdc871 100644 --- a/stubs/confluent_kafka/__init__.pyi +++ b/stubs/confluent_kafka/__init__.pyi @@ -1,13 +1,13 @@ from ._model import IsolationLevel from .cimpl import ( - Consumer, - Message, OFFSET_BEGINNING, OFFSET_END, - Producer, TIMESTAMP_CREATE_TIME, TIMESTAMP_LOG_APPEND_TIME, TIMESTAMP_NOT_AVAILABLE, + Consumer, + Message, + Producer, TopicCollection, TopicPartition, ) diff --git a/stubs/confluent_kafka/admin/__init__.pyi b/stubs/confluent_kafka/admin/__init__.pyi index 1dafa51b8..655e1257e 100644 --- a/stubs/confluent_kafka/admin/__init__.pyi +++ b/stubs/confluent_kafka/admin/__init__.pyi @@ -1,11 +1,13 @@ +from concurrent.futures import Future +from typing import Callable + +from confluent_kafka import IsolationLevel, TopicCollection, TopicPartition + from ..cimpl import NewTopic from ._config import ConfigEntry, ConfigResource, ConfigSource from ._listoffsets import ListOffsetsResultInfo, OffsetSpec from ._metadata import BrokerMetadata, ClusterMetadata, PartitionMetadata, TopicMetadata from ._resource import ResourceType -from concurrent.futures import Future -from confluent_kafka import IsolationLevel, TopicCollection, TopicPartition -from typing import Callable __all__ = ( "AdminClient", diff --git a/stubs/confluent_kafka/admin/_config.pyi b/stubs/confluent_kafka/admin/_config.pyi index d811a4bfa..0d018b7b0 100644 --- a/stubs/confluent_kafka/admin/_config.pyi +++ b/stubs/confluent_kafka/admin/_config.pyi @@ -1,7 +1,8 @@ -from ._resource import ResourceType from enum import Enum from typing import cast +from ._resource import ResourceType + class ConfigResource: Type = ResourceType diff --git a/stubs/confluent_kafka/cimpl.pyi b/stubs/confluent_kafka/cimpl.pyi index ed163e5fc..a22425491 100644 --- a/stubs/confluent_kafka/cimpl.pyi +++ b/stubs/confluent_kafka/cimpl.pyi @@ -1,6 +1,7 @@ -from confluent_kafka.admin._metadata import ClusterMetadata from typing import Any, Callable, Final, overload +from confluent_kafka.admin._metadata import ClusterMetadata + OFFSET_BEGINNING: Final = ... OFFSET_END: Final = ... diff --git a/stubs/isodate/duration.pyi b/stubs/isodate/duration.pyi index 46f93d801..3f128441f 100644 --- a/stubs/isodate/duration.pyi +++ b/stubs/isodate/duration.pyi @@ -1,6 +1,5 @@ -from decimal import Decimal - import datetime +from decimal import Decimal class Duration: months: Decimal diff --git a/stubs/isodate/isoduration.pyi b/stubs/isodate/isoduration.pyi index 1652fdd80..17e3cb5c6 100644 --- a/stubs/isodate/isoduration.pyi +++ b/stubs/isodate/isoduration.pyi @@ -1,7 +1,7 @@ -from .duration import Duration +import datetime from typing import Literal, overload -import datetime +from .duration import Duration @overload def parse_duration( diff --git a/tests/conftest.py b/tests/conftest.py index 725f7b6b7..3a84aeb2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,22 +3,23 @@ See LICENSE for details """ -from avro.compatibility import SchemaCompatibilityResult -from karapace.core.container import KarapaceContainer +import json +import os +import re from pathlib import Path -from karapace.api.container import SchemaRegistryContainer -from karapace.api.telemetry.container import TelemetryContainer from tempfile import mkstemp -import json -import os import pytest -import re +from avro.compatibility import SchemaCompatibilityResult + import karapace.api.controller -import karapace.api.telemetry.tracer import karapace.api.telemetry.meter -import karapace.api.telemetry.setup import karapace.api.telemetry.middleware +import karapace.api.telemetry.setup +import karapace.api.telemetry.tracer +from karapace.api.container import SchemaRegistryContainer +from karapace.api.telemetry.container import TelemetryContainer +from karapace.core.container import KarapaceContainer pytest_plugins = "aiohttp.pytest_plugin" KAFKA_BOOTSTRAP_SERVERS_OPT = "--kafka-bootstrap-servers" @@ -92,8 +93,7 @@ def pytest_addoption(parser, pluginmanager) -> None: parser.addoption( "--registry-url", help=( - "URL of a running Schema Registry instance. If provided the test " - "suite will not start a Schema Registry instance" + "URL of a running Schema Registry instance. If provided the test suite will not start a Schema Registry instance" ), ) parser.addoption( diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a71a58911..c25bfa473 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -7,10 +7,15 @@ from __future__ import annotations +import asyncio +import secrets +from collections.abc import AsyncGenerator, Iterator + +import pytest from _pytest.fixtures import SubRequest from aiohttp import BasicAuth -from collections.abc import AsyncGenerator, Iterator from confluent_kafka.admin import NewTopic + from karapace.core.client import Client from karapace.core.container import KarapaceContainer from karapace.core.kafka.admin import KafkaAdminClient @@ -19,10 +24,6 @@ from tests.integration.utils.cluster import RegistryDescription, RegistryEndpoint from tests.integration.utils.kafka_server import KafkaServers -import asyncio -import pytest -import secrets - @pytest.fixture(scope="session", name="basic_auth") def fixture_basic_auth() -> BasicAuth: diff --git a/tests/e2e/instrumentation/test_prometheus.py b/tests/e2e/instrumentation/test_prometheus.py index add1c0dbd..bba3b715e 100644 --- a/tests/e2e/instrumentation/test_prometheus.py +++ b/tests/e2e/instrumentation/test_prometheus.py @@ -6,9 +6,11 @@ """ from http import HTTPStatus + +from prometheus_client.parser import text_string_to_metric_families + from karapace.core.client import Client, Result from karapace.core.instrumentation.prometheus import PrometheusInstrumentation -from prometheus_client.parser import text_string_to_metric_families async def test_metrics_endpoint(registry_async_client: Client) -> None: diff --git a/tests/e2e/kafka/test_admin.py b/tests/e2e/kafka/test_admin.py index 39a2e7d2d..5ba5adb65 100644 --- a/tests/e2e/kafka/test_admin.py +++ b/tests/e2e/kafka/test_admin.py @@ -5,14 +5,14 @@ from __future__ import annotations +import pytest from aiokafka.errors import InvalidReplicationFactorError, TopicAlreadyExistsError, UnknownTopicOrPartitionError from confluent_kafka.admin import ConfigSource, NewTopic + from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.producer import KafkaProducer from tests.utils import new_topic as create_new_topic -import pytest - class TestNewTopic: def test_new_topic_raises_for_duplicate(self, admin_client: KafkaAdminClient, new_topic: NewTopic) -> None: diff --git a/tests/e2e/kafka/test_consumer.py b/tests/e2e/kafka/test_consumer.py index 3eeba6949..99cb970be 100644 --- a/tests/e2e/kafka/test_consumer.py +++ b/tests/e2e/kafka/test_consumer.py @@ -5,19 +5,20 @@ from __future__ import annotations +from typing import Final + +import pytest from aiokafka.errors import IllegalStateError, UnknownTopicOrPartitionError from confluent_kafka import OFFSET_BEGINNING, OFFSET_END, TopicPartition from confluent_kafka.admin import NewTopic from confluent_kafka.error import KafkaError + from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer from karapace.core.kafka.producer import AsyncKafkaProducer, KafkaProducer from karapace.core.utils import Expiration from tests.integration.utils.kafka_server import KafkaServers from tests.utils import new_topic as create_new_topic -from typing import Final - -import pytest POLL_TIMEOUT_S: Final = 10 diff --git a/tests/e2e/kafka/test_producer.py b/tests/e2e/kafka/test_producer.py index 2b644381b..f3f61771d 100644 --- a/tests/e2e/kafka/test_producer.py +++ b/tests/e2e/kafka/test_producer.py @@ -5,14 +5,15 @@ from __future__ import annotations +import time + +import pytest from aiokafka.errors import MessageSizeTooLargeError, UnknownTopicOrPartitionError from confluent_kafka.admin import NewTopic + from karapace.core.kafka.producer import AsyncKafkaProducer, KafkaProducer from karapace.core.kafka.types import Timestamp -import pytest -import time - class TestSend: def test_send(self, producer: KafkaProducer, new_topic: NewTopic) -> None: diff --git a/tests/e2e/schema_registry/test_jsonschema.py b/tests/e2e/schema_registry/test_jsonschema.py index 8ef3cdfc4..94189ecbb 100644 --- a/tests/e2e/schema_registry/test_jsonschema.py +++ b/tests/e2e/schema_registry/test_jsonschema.py @@ -3,11 +3,15 @@ See LICENSE for details """ +import json + +import pytest from jsonschema import Draft7Validator + from karapace.core.client import Client from karapace.core.compatibility import CompatibilityModes -from karapace.core.typing import SchemaMetadata, SchemaRuleSet from karapace.core.schema_reader import SchemaType +from karapace.core.typing import SchemaMetadata, SchemaRuleSet from tests.schemas.json_schemas import ( A_DINT_B_DINT_OBJECT_SCHEMA, A_DINT_B_INT_OBJECT_SCHEMA, @@ -100,9 +104,6 @@ ) from tests.utils import new_random_name -import json -import pytest - async def debugging_details( newer: Draft7Validator, diff --git a/tests/e2e/test_karapace.py b/tests/e2e/test_karapace.py index 76e1b3d4b..dbb784658 100644 --- a/tests/e2e/test_karapace.py +++ b/tests/e2e/test_karapace.py @@ -3,15 +3,15 @@ See LICENSE for details """ +import socket from collections.abc import Iterator -from contextlib import closing, contextmanager, ExitStack +from contextlib import ExitStack, closing, contextmanager from pathlib import Path + from tests.integration.utils.kafka_server import KafkaServers from tests.integration.utils.process import stop_process from tests.utils import popen_karapace_all -import socket - @contextmanager def allocate_port_no_reuse() -> Iterator[int]: @@ -46,7 +46,7 @@ def test_regression_server_must_exit_on_exception( "KARAPACE_KARAPACE_REST": "true", } karapace_rest_proxy = popen_karapace_all( - module="karapace.core.karapace_all", env=karapace_rest_proxy_env, stdout=logfile, stderr=errfile + module="karapace.kafka_rest_apis", env=karapace_rest_proxy_env, stdout=logfile, stderr=errfile ) stack.callback(stop_process, karapace_rest_proxy) # make sure to stop the process if the test fails assert karapace_rest_proxy.wait(timeout=10) != 0, "Process should have exited with an error, port is already is use" @@ -57,9 +57,9 @@ def test_regression_server_must_exit_on_exception( "KARAPACE_KARAPACE_REGISTRY": "true", } karapace_schema_registry = popen_karapace_all( - module="karapace.api", env=karapace_schema_registry_env, stdout=logfile, stderr=errfile + module="karapace", env=karapace_schema_registry_env, stdout=logfile, stderr=errfile ) stack.callback(stop_process, karapace_schema_registry) # make sure to stop the process if the test fails - assert ( - karapace_schema_registry.wait(timeout=10) != 0 - ), "Process should have exited with an error, port is already is use" + assert karapace_schema_registry.wait(timeout=10) != 0, ( + "Process should have exited with an error, port is already is use" + ) diff --git a/tests/integration/backup/test_avro_export.py b/tests/integration/backup/test_avro_export.py index 828cf3e09..5e301564b 100644 --- a/tests/integration/backup/test_avro_export.py +++ b/tests/integration/backup/test_avro_export.py @@ -5,19 +5,19 @@ See LICENSE for details """ +import base64 +import json +import os +from pathlib import Path +from typing import Any + from karapace.backup import api from karapace.backup.api import BackupVersion from karapace.core.client import Client from karapace.core.config import Config from karapace.core.utils import json_encode -from pathlib import Path from tests.integration.utils.cluster import RegistryDescription from tests.integration.utils.kafka_server import KafkaServers -from typing import Any - -import base64 -import json -import os baseurl = "http://localhost:8081" diff --git a/tests/integration/backup/test_get_topic_configurations.py b/tests/integration/backup/test_get_topic_configurations.py index c80ddfe60..da15c7153 100644 --- a/tests/integration/backup/test_get_topic_configurations.py +++ b/tests/integration/backup/test_get_topic_configurations.py @@ -5,17 +5,17 @@ from __future__ import annotations +import pytest from confluent_kafka.admin import NewTopic + from karapace.backup.topic_configurations import ( ALL_CONFIG_SOURCES, - ConfigSource, DEFAULT_CONFIGS, + ConfigSource, get_topic_configurations, ) from karapace.core.kafka.admin import KafkaAdminClient -import pytest - class TestTopicConfiguration: @pytest.mark.parametrize("custom_topic_configs", [{}, {"max.message.bytes": "1234"}]) diff --git a/tests/integration/backup/test_legacy_backup.py b/tests/integration/backup/test_legacy_backup.py index 32b44c5e3..a54dd364f 100644 --- a/tests/integration/backup/test_legacy_backup.py +++ b/tests/integration/backup/test_legacy_backup.py @@ -5,7 +5,16 @@ See LICENSE for details """ +import asyncio +import json +import os +import time +from pathlib import Path +from unittest import mock + +import pytest from aiokafka.errors import InvalidTopicError + from karapace.backup import api from karapace.backup.api import BackupVersion from karapace.backup.errors import StaleConsumerError @@ -17,17 +26,9 @@ from karapace.core.kafka.consumer import KafkaConsumer from karapace.core.key_format import is_key_in_canonical_format from karapace.core.utils import Expiration -from pathlib import Path from tests.integration.utils.cluster import RegistryDescription from tests.integration.utils.kafka_server import KafkaServers -from tests.utils import new_random_name, StubMessage -from unittest import mock - -import asyncio -import json -import os -import pytest -import time +from tests.utils import StubMessage, new_random_name baseurl = "http://localhost:8081" diff --git a/tests/integration/backup/test_session_timeout.py b/tests/integration/backup/test_session_timeout.py index 3a8f20145..50758f076 100644 --- a/tests/integration/backup/test_session_timeout.py +++ b/tests/integration/backup/test_session_timeout.py @@ -3,19 +3,20 @@ See LICENSE for details """ +from pathlib import Path + +import pytest from aiokafka.errors import NoBrokersAvailable from confluent_kafka.admin import NewTopic + from karapace.backup.api import BackupVersion, create_backup from karapace.core.config import Config from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka_utils import kafka_producer_from_config -from pathlib import Path from tests.integration.conftest import create_kafka_server from tests.integration.utils.config import KafkaDescription from tests.integration.utils.kafka_server import KafkaServers -import pytest - SESSION_TIMEOUT_MS = 65000 GROUP_MIN_SESSION_TIMEOUT_MS = 60000 GROUP_MAX_SESSION_TIMEOUT_MS = 70000 diff --git a/tests/integration/backup/test_v3_backup.py b/tests/integration/backup/test_v3_backup.py index 7bfa51fce..4561de29b 100644 --- a/tests/integration/backup/test_v3_backup.py +++ b/tests/integration/backup/test_v3_backup.py @@ -5,13 +5,28 @@ from __future__ import annotations -from aiokafka.errors import InvalidReplicationFactorError, UnknownTopicOrPartitionError +import datetime +import json +import logging +import os +import shutil +import subprocess +import textwrap +import time from collections.abc import Iterator +from dataclasses import fields +from pathlib import Path +from tempfile import mkdtemp +from typing import NoReturn +from unittest.mock import patch + +import pytest +from aiokafka.errors import InvalidReplicationFactorError, UnknownTopicOrPartitionError from confluent_kafka import Message, TopicPartition from confluent_kafka.admin import NewTopic -from dataclasses import fields + from karapace.backup import api -from karapace.backup.api import _consume_records, BackupVersion, TopicName +from karapace.backup.api import BackupVersion, TopicName, _consume_records from karapace.backup.backends.v3.errors import InconsistentOffset from karapace.backup.backends.v3.readers import read_metadata from karapace.backup.backends.v3.schema import Metadata @@ -25,22 +40,8 @@ from karapace.core.kafka.types import Timestamp from karapace.core.kafka_utils import kafka_consumer_from_config, kafka_producer_from_config from karapace.version import __version__ -from pathlib import Path -from tempfile import mkdtemp from tests.integration.utils.cluster import RegistryDescription from tests.integration.utils.kafka_server import KafkaServers -from typing import NoReturn -from unittest.mock import patch - -import datetime -import json -import logging -import os -import pytest -import shutil -import subprocess -import textwrap -import time logger = logging.getLogger(__name__) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3edfe19c6..05a38b3b9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,32 +7,43 @@ from __future__ import annotations +import asyncio +import json +import os +import pathlib +import re +import secrets +import time +from collections.abc import AsyncGenerator, AsyncIterator, Iterator +from contextlib import ExitStack +from dataclasses import asdict +from pathlib import Path +from urllib.parse import urlparse + +import pytest from _pytest.fixtures import SubRequest from aiohttp.pytest_plugin import AiohttpClient from aiohttp.test_utils import TestClient -from collections.abc import AsyncGenerator, AsyncIterator, Iterator from confluent_kafka.admin import NewTopic -from contextlib import ExitStack -from dataclasses import asdict from filelock import FileLock + from karapace.core.client import Client from karapace.core.config import Config from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.consumer import AsyncKafkaConsumer, KafkaConsumer from karapace.core.kafka.producer import AsyncKafkaProducer, KafkaProducer -from karapace.core.kafka_rest_apis import KafkaRest -from pathlib import Path +from karapace.kafka_rest_apis import KafkaRest from tests.conftest import KAFKA_VERSION from tests.integration.utils.cluster import ( - after_master_is_available, RegistryDescription, RegistryEndpoint, + after_master_is_available, start_schema_registry_cluster, ) from tests.integration.utils.config import KafkaConfig, KafkaDescription, ZKConfig from tests.integration.utils.kafka_server import ( - configure_and_start_kafka, KafkaServers, + configure_and_start_kafka, maybe_download_kafka, wait_for_kafka, ) @@ -42,16 +53,6 @@ from tests.integration.utils.synchronization import lock_path_for from tests.integration.utils.zookeeper import configure_and_start_zk from tests.utils import repeat_until_master_is_available, repeat_until_successful_request -from urllib.parse import urlparse - -import asyncio -import json -import os -import pathlib -import pytest -import re -import secrets -import time REPOSITORY_DIR = pathlib.Path(__file__).parent.parent.parent.absolute() RUNTIME_DIR = REPOSITORY_DIR / "runtime" diff --git a/tests/integration/kafka_rest_apis/__init__.py b/tests/integration/kafka_rest_apis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_rest.py b/tests/integration/kafka_rest_apis/test_rest.py similarity index 99% rename from tests/integration/test_rest.py rename to tests/integration/kafka_rest_apis/test_rest.py index bc7988fd2..d44530e64 100644 --- a/tests/integration/test_rest.py +++ b/tests/integration/kafka_rest_apis/test_rest.py @@ -5,19 +5,27 @@ from __future__ import annotations +import asyncio +import base64 +import json +import time from collections.abc import Mapping from dataclasses import dataclass +from typing import Any + +import pytest + from karapace.core.client import Client from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.kafka.producer import KafkaProducer -from karapace.core.kafka_rest_apis import KafkaRest, SUBJECT_VALID_POSTFIX +from karapace.kafka_rest_apis import SUBJECT_VALID_POSTFIX, KafkaRest from karapace.core.schema_type import SchemaType from karapace.version import __version__ from tests.integration.conftest import REST_PRODUCER_MAX_REQUEST_BYTES from tests.utils import ( + REST_HEADERS, new_random_name, new_topic, - REST_HEADERS, schema_avro_json, schema_avro_json_evolution, second_obj, @@ -25,13 +33,6 @@ test_objects_avro_evolution, wait_for_topics, ) -from typing import Any - -import asyncio -import base64 -import json -import pytest -import time NEW_TOPIC_TIMEOUT = 10 diff --git a/tests/integration/test_rest_consumer.py b/tests/integration/kafka_rest_apis/test_rest_consumer.py similarity index 98% rename from tests/integration/test_rest_consumer.py rename to tests/integration/kafka_rest_apis/test_rest_consumer.py index ab6336289..db7503d1a 100644 --- a/tests/integration/test_rest_consumer.py +++ b/tests/integration/kafka_rest_apis/test_rest_consumer.py @@ -3,25 +3,26 @@ See LICENSE for details """ -from karapace.core.kafka_rest_apis.consumer_manager import KNOWN_FORMATS +import base64 +import copy +import json +import random +import time + +import pytest + +from karapace.kafka_rest_apis.consumer_manager import KNOWN_FORMATS from tests.utils import ( + REST_HEADERS, consumer_valid_payload, new_consumer, new_random_name, new_topic, repeat_until_successful_request, - REST_HEADERS, schema_data, wait_for_topics, ) -import base64 -import copy -import json -import pytest -import random -import time - @pytest.mark.parametrize("trail", ["", "/"]) async def test_create_and_delete(rest_async_client, trail): @@ -38,7 +39,7 @@ async def test_create_and_delete(rest_async_client, trail): resp = await rest_async_client.post(f"/consumers/{group_name}{trail}", json=with_name, headers=header) assert not resp.ok assert resp.status_code == 409, ( - f"Expected conflict for instance {instance_id} and group {group_name} " f"but got a different error: {resp.body}" + f"Expected conflict for instance {instance_id} and group {group_name} but got a different error: {resp.body}" ) invalid_fetch = copy.copy(consumer_valid_payload) # add with faulty params fails @@ -100,9 +101,9 @@ async def test_subscription(rest_async_client, admin_client, producer, trail): res = await rest_async_client.get(sub_path, headers=header) assert res.ok data = res.json() - assert ( - "topics" in data and len(data["topics"]) == 1 and data["topics"][0] == topic_name - ), f"expecting {topic_name} in {data}" + assert "topics" in data and len(data["topics"]) == 1 and data["topics"][0] == topic_name, ( + f"expecting {topic_name} in {data}" + ) for _ in range(3): producer.send(topic_name, value=b"foo") producer.flush() @@ -374,7 +375,7 @@ async def test_consume(rest_async_client, admin_client, producer, trail): assert data[i]["offset"] >= 0 assert data[i]["timestamp"] > 0 assert deserializers[fmt](data[i]["value"]) == values[fmt][i], ( - f"Extracted data {deserializers[fmt](data[i]['value'])}" f" does not match {values[fmt][i]} for format {fmt}" + f"Extracted data {deserializers[fmt](data[i]['value'])} does not match {values[fmt][i]} for format {fmt}" ) @@ -408,7 +409,7 @@ async def test_consume_timeout(rest_async_client, admin_client, producer): assert len(data) == len(values[fmt]), f"Expected {len(values[fmt])} element in response: {resp}" for i in range(len(values[fmt])): assert deserializers[fmt](data[i]["value"]) == values[fmt][i], ( - f"Extracted data {deserializers[fmt](data[i]['value'])}" f" does not match {values[fmt][i]} for format {fmt}" + f"Extracted data {deserializers[fmt](data[i]['value'])} does not match {values[fmt][i]} for format {fmt}" ) # Now read more using explicit 5s timeout diff --git a/tests/integration/test_rest_consumer_protobuf.py b/tests/integration/kafka_rest_apis/test_rest_consumer_protobuf.py similarity index 99% rename from tests/integration/test_rest_consumer_protobuf.py rename to tests/integration/kafka_rest_apis/test_rest_consumer_protobuf.py index 47eb2496e..f7ec54c4e 100644 --- a/tests/integration/test_rest_consumer_protobuf.py +++ b/tests/integration/kafka_rest_apis/test_rest_consumer_protobuf.py @@ -3,23 +3,23 @@ See LICENSE for details """ +import pytest + from karapace.core.client import Client from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.protobuf.kotlin_wrapper import trim_margin -from tests.integration.test_rest import NEW_TOPIC_TIMEOUT +from tests.integration.kafka_rest_apis.test_rest import NEW_TOPIC_TIMEOUT from tests.utils import ( + REST_HEADERS, new_consumer, new_random_name, new_topic, repeat_until_successful_request, - REST_HEADERS, schema_data, schema_data_second, wait_for_topics, ) -import pytest - @pytest.mark.parametrize("schema_type", ["protobuf"]) @pytest.mark.parametrize("trail", ["", "/"]) diff --git a/tests/integration/test_health_check.py b/tests/integration/test_health_check.py index 99440514b..30e64f717 100644 --- a/tests/integration/test_health_check.py +++ b/tests/integration/test_health_check.py @@ -3,13 +3,14 @@ See LICENSE for details """ +import http + +from tenacity import retry, stop_after_delay, wait_fixed + from karapace.core.client import Client from karapace.core.kafka.admin import KafkaAdminClient -from tenacity import retry, stop_after_delay, wait_fixed from tests.integration.utils.cluster import RegistryDescription -import http - async def test_health_check( registry_cluster: RegistryDescription, registry_async_client: Client, admin_client: KafkaAdminClient diff --git a/tests/integration/test_master_coordinator.py b/tests/integration/test_master_coordinator.py index 2e1025d59..b90c35c4c 100644 --- a/tests/integration/test_master_coordinator.py +++ b/tests/integration/test_master_coordinator.py @@ -5,6 +5,10 @@ See LICENSE for details """ +import asyncio + +import pytest + from karapace.core.config import Config from karapace.core.coordinator.master_coordinator import MasterCoordinator from karapace.core.typing import SchemaReaderStoppper @@ -12,9 +16,6 @@ from tests.integration.utils.network import allocate_port from tests.utils import new_random_name -import asyncio -import pytest - class AlwaysAvailableSchemaReaderStoppper(SchemaReaderStoppper): def ready(self) -> bool: diff --git a/tests/integration/test_request_forwarding.py b/tests/integration/test_request_forwarding.py index 4ce7bf40a..d14f45fbb 100644 --- a/tests/integration/test_request_forwarding.py +++ b/tests/integration/test_request_forwarding.py @@ -5,8 +5,14 @@ See LICENSE for details """ -from _pytest.fixtures import SubRequest +import asyncio +import json from typing import AsyncGenerator +from urllib.parse import quote_plus + +import pytest +from _pytest.fixtures import SubRequest + from karapace.core.client import Client from tests.integration.utils.rest_client import RetryRestClient from tests.utils import ( @@ -14,11 +20,6 @@ repeat_until_master_is_available, repeat_until_successful_request, ) -from urllib.parse import quote_plus - -import asyncio -import json -import pytest @pytest.fixture(scope="function", name="request_forwarding_retry_client") diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index dcd19e0c2..4c3e03438 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -5,15 +5,23 @@ See LICENSE for details """ -from attr import dataclass +import asyncio +import json +import os +import time from http import HTTPStatus + +import pytest +import requests +from attr import dataclass + +from karapace.api.controller import SchemaErrorMessages from karapace.core.client import Client from karapace.core.kafka.producer import KafkaProducer -from karapace.core.rapu import is_success from karapace.core.schema_type import SchemaType from karapace.core.typing import JsonData from karapace.core.utils import json_encode -from karapace.api.controller import SchemaErrorMessages +from karapace.rapu import is_success from tests.base_testcase import BaseTestCase from tests.integration.utils.cluster import RegistryDescription from tests.integration.utils.kafka_server import KafkaServers @@ -24,13 +32,6 @@ repeat_until_successful_request, ) -import asyncio -import json -import os -import pytest -import requests -import time - baseurl = "http://localhost:8081" diff --git a/tests/integration/test_schema_compatibility.py b/tests/integration/test_schema_compatibility.py index eb2d57fe6..e1e8965c5 100644 --- a/tests/integration/test_schema_compatibility.py +++ b/tests/integration/test_schema_compatibility.py @@ -5,17 +5,18 @@ from __future__ import annotations +import json +import logging from collections.abc import Callable, Coroutine from dataclasses import dataclass -from karapace.core.client import Client -from karapace.core.typing import JsonObject, Subject -from tests.base_testcase import BaseTestCase from typing import Any, Final -import json -import logging import pytest +from karapace.core.client import Client +from karapace.core.typing import JsonObject, Subject +from tests.base_testcase import BaseTestCase + SchemaRegitrationFunc = Callable[[Client, Subject], Coroutine[Any, Any, None]] LOG = logging.getLogger(__name__) diff --git a/tests/integration/test_schema_coordinator.py b/tests/integration/test_schema_coordinator.py index 99f3a3d98..533ede2db 100644 --- a/tests/integration/test_schema_coordinator.py +++ b/tests/integration/test_schema_coordinator.py @@ -9,33 +9,40 @@ from __future__ import annotations +import asyncio +import contextlib +import logging +import time +from collections.abc import AsyncGenerator, Iterator +from typing import Final +from unittest import mock + +import aiokafka.errors as Errors +import pytest from aiokafka.client import AIOKafkaClient, ConnectionGroup, CoordinationType from aiokafka.cluster import ClusterMetadata from aiokafka.protocol.api import Response from aiokafka.protocol.group import ( HeartbeatRequest_v0 as HeartbeatRequest, +) +from aiokafka.protocol.group import ( JoinGroupRequest_v0 as JoinGroupRequest, +) +from aiokafka.protocol.group import ( LeaveGroupRequest_v0 as LeaveGroupRequest, +) +from aiokafka.protocol.group import ( SyncGroupResponse_v0 as SyncGroupResponse, ) from aiokafka.util import create_future, create_task -from collections.abc import AsyncGenerator, Iterator +from tenacity import TryAgain, retry, stop_after_delay, wait_fixed + from karapace.core.coordinator.schema_coordinator import Assignment, SchemaCoordinator, SchemaCoordinatorGroupRebalance from karapace.core.utils import json_encode from karapace.version import __version__ -from tenacity import retry, stop_after_delay, TryAgain, wait_fixed from tests.integration.test_master_coordinator import AlwaysAvailableSchemaReaderStoppper from tests.integration.utils.kafka_server import KafkaServers from tests.utils import new_random_name -from typing import Final -from unittest import mock - -import aiokafka.errors as Errors -import asyncio -import contextlib -import logging -import pytest -import time UNKNOWN_MEMBER_ID = JoinGroupRequest.UNKNOWN_MEMBER_ID @@ -182,9 +189,9 @@ async def test_coordinator_workflow( # give time to the election to be forwarded to all the coordinators. await asyncio.sleep(3) - assert ( - not primary.are_we_master() - ), "after a change in the coordinator we can act as a master until we wait for the required time" + assert not primary.are_we_master(), ( + "after a change in the coordinator we can act as a master until we wait for the required time" + ) assert not secondary.are_we_master(), "also the second cannot be immediately a master" # after that time the primary can act as a master await asyncio.sleep(waiting_time_before_acting_as_master_sec) diff --git a/tests/integration/test_schema_protobuf.py b/tests/integration/test_schema_protobuf.py index a4e7e305d..003eed401 100644 --- a/tests/integration/test_schema_protobuf.py +++ b/tests/integration/test_schema_protobuf.py @@ -7,23 +7,24 @@ from __future__ import annotations +import asyncio +import logging from dataclasses import dataclass +from pathlib import Path + +import pytest + from karapace.core.client import Client from karapace.core.config import Config from karapace.core.errors import InvalidTest from karapace.core.protobuf.kotlin_wrapper import trim_margin from karapace.core.schema_type import SchemaType from karapace.core.typing import JsonData, SchemaMetadata, SchemaRuleSet -from pathlib import Path from tests.base_testcase import BaseTestCase from tests.integration.utils.cluster import after_master_is_available, start_schema_registry_cluster from tests.integration.utils.kafka_server import KafkaServers from tests.utils import create_subject_name_factory -import asyncio -import logging -import pytest - baseurl = "http://localhost:8081" diff --git a/tests/integration/test_schema_reader.py b/tests/integration/test_schema_reader.py index 97d12f6ed..a50479a29 100644 --- a/tests/integration/test_schema_reader.py +++ b/tests/integration/test_schema_reader.py @@ -3,8 +3,12 @@ See LICENSE for details """ +import asyncio from contextlib import closing from dataclasses import dataclass + +import pytest + from karapace.core.config import Config from karapace.core.constants import DEFAULT_SCHEMA_TOPIC from karapace.core.coordinator.master_coordinator import MasterCoordinator @@ -13,18 +17,15 @@ from karapace.core.kafka.producer import KafkaProducer from karapace.core.key_format import KeyFormatter, KeyMode from karapace.core.offset_watcher import OffsetWatcher +from karapace.core.schema_reader import KafkaSchemaReader from karapace.core.typing import PrimaryInfo from karapace.core.utils import json_encode -from karapace.core.schema_reader import KafkaSchemaReader from tests.base_testcase import BaseTestCase from tests.integration.test_master_coordinator import AlwaysAvailableSchemaReaderStoppper from tests.integration.utils.kafka_server import KafkaServers from tests.schemas.json_schemas import FALSE_SCHEMA, TRUE_SCHEMA from tests.utils import create_group_name_factory, create_subject_name_factory, new_random_name, new_topic -import asyncio -import pytest - async def _wait_until_reader_is_ready_and_master( master_coordinator: MasterCoordinator, diff --git a/tests/integration/test_schema_registry_auth.py b/tests/integration/test_schema_registry_auth.py index 85f4b4ea7..75567b6c5 100644 --- a/tests/integration/test_schema_registry_auth.py +++ b/tests/integration/test_schema_registry_auth.py @@ -5,6 +5,11 @@ See LICENSE for details """ +import asyncio +from urllib.parse import quote + +import aiohttp + from karapace.core.kafka.admin import KafkaAdminClient from karapace.core.schema_models import SchemaType, ValidatedTypedSchema from tests.integration.utils.rest_client import RetryRestClient @@ -16,10 +21,6 @@ test_objects_avro, wait_for_topics, ) -from urllib.parse import quote - -import aiohttp -import asyncio NEW_TOPIC_TIMEOUT = 10 diff --git a/tests/integration/test_schema_registry_mode.py b/tests/integration/test_schema_registry_mode.py index d91f49d28..9c1168b2d 100644 --- a/tests/integration/test_schema_registry_mode.py +++ b/tests/integration/test_schema_registry_mode.py @@ -3,12 +3,12 @@ See LICENSE for details """ +import json + from karapace.core.client import Client from karapace.core.typing import Mode from tests.utils import create_schema_name_factory, create_subject_name_factory -import json - async def test_global_mode(registry_async_client: Client) -> None: res = await registry_async_client.get_mode() diff --git a/tests/integration/utils/cluster.py b/tests/integration/utils/cluster.py index ca0e3b716..8a7953cbe 100644 --- a/tests/integration/utils/cluster.py +++ b/tests/integration/utils/cluster.py @@ -6,11 +6,12 @@ from __future__ import annotations from collections.abc import AsyncIterator -from contextlib import asynccontextmanager, ExitStack +from contextlib import ExitStack, asynccontextmanager from dataclasses import dataclass +from pathlib import Path + from karapace.core.client import Client from karapace.core.config import Config -from pathlib import Path from tests.integration.utils.network import allocate_port from tests.integration.utils.process import stop_process, wait_for_port_subprocess from tests.utils import new_random_name, popen_karapace_all, repeat_until_master_is_available @@ -82,7 +83,7 @@ async def start_schema_registry_cluster( "KARAPACE_WAITING_TIME_BEFORE_ACTING_AS_MASTER_MS": str(config.waiting_time_before_acting_as_master_ms), "KARAPACE_MASTER_ELIGIBILITY": str(config.master_eligibility), } - process = popen_karapace_all(module="karapace.api", env=env, stdout=logfile, stderr=errfile) + process = popen_karapace_all(module="karapace", env=env, stdout=logfile, stderr=errfile) stack.callback(stop_process, process) all_processes.append((process, port, config.host)) diff --git a/tests/integration/utils/kafka_server.py b/tests/integration/utils/kafka_server.py index 6c8466eca..d181d82de 100644 --- a/tests/integration/utils/kafka_server.py +++ b/tests/integration/utils/kafka_server.py @@ -5,22 +5,23 @@ from __future__ import annotations -from aiokafka.errors import AuthenticationFailedError, NoBrokersAvailable +import logging +import os +import tarfile +import time from dataclasses import dataclass -from karapace.core.kafka.admin import KafkaAdminClient -from karapace.core.utils import Expiration from pathlib import Path from subprocess import Popen + +import requests +from aiokafka.errors import AuthenticationFailedError, NoBrokersAvailable + +from karapace.core.kafka.admin import KafkaAdminClient +from karapace.core.utils import Expiration from tests.integration.utils.config import KafkaConfig, KafkaDescription, ZKConfig from tests.integration.utils.process import get_java_process_configuration from tests.utils import write_ini -import logging -import os -import requests -import tarfile -import time - log = logging.getLogger(__name__) diff --git a/tests/integration/utils/network.py b/tests/integration/utils/network.py index cb564efde..55a759b57 100644 --- a/tests/integration/utils/network.py +++ b/tests/integration/utils/network.py @@ -3,11 +3,10 @@ See LICENSE for details """ +import socket from collections.abc import Iterator from contextlib import closing, contextmanager -import socket - @contextmanager def allocate_port() -> Iterator[int]: diff --git a/tests/integration/utils/process.py b/tests/integration/utils/process.py index 1f67fc732..7c4b4d4a1 100644 --- a/tests/integration/utils/process.py +++ b/tests/integration/utils/process.py @@ -3,13 +3,13 @@ See LICENSE for details """ -from karapace.core.utils import Expiration -from subprocess import Popen -from tests.integration.utils.network import port_is_listening - import os import signal import time +from subprocess import Popen + +from karapace.core.utils import Expiration +from tests.integration.utils.network import port_is_listening def wait_for_port_subprocess( diff --git a/tests/integration/utils/rest_client.py b/tests/integration/utils/rest_client.py index da01c2547..e43b7618c 100644 --- a/tests/integration/utils/rest_client.py +++ b/tests/integration/utils/rest_client.py @@ -7,12 +7,14 @@ from __future__ import annotations -from aiohttp import BasicAuth from collections.abc import Mapping +from typing import Final + +from aiohttp import BasicAuth +from tenacity import retry, stop_after_attempt, wait_fixed + from karapace.core.client import Client, Headers, Path, Result from karapace.core.typing import JsonData -from tenacity import retry, stop_after_attempt, wait_fixed -from typing import Final RETRY_WAIT_SECONDS: Final = 0.5 diff --git a/tests/integration/utils/zookeeper.py b/tests/integration/utils/zookeeper.py index 39c0c2302..06b19d6cc 100644 --- a/tests/integration/utils/zookeeper.py +++ b/tests/integration/utils/zookeeper.py @@ -5,6 +5,7 @@ from pathlib import Path from subprocess import Popen + from tests.integration.utils.config import KafkaDescription, ZKConfig from tests.integration.utils.process import get_java_process_configuration from tests.utils import write_ini diff --git a/tests/unit/anonymize_schemas/test_anonymize_avro.py b/tests/unit/anonymize_schemas/test_anonymize_avro.py index 44c7c8b6a..4d5efa3a6 100644 --- a/tests/unit/anonymize_schemas/test_anonymize_avro.py +++ b/tests/unit/anonymize_schemas/test_anonymize_avro.py @@ -5,11 +5,12 @@ See LICENSE for details """ -from karapace.core.anonymize_schemas.anonymize_avro import anonymize - import json + import pytest +from karapace.core.anonymize_schemas.anonymize_avro import anonymize + PRIMITIVE_TYPE_SCHEMA = json.loads('"int"') EXPECTED_PRIMITIVE_TYPE_SCHEMA = "int" diff --git a/tests/unit/api/telemetry/test_meter.py b/tests/unit/api/telemetry/test_meter.py index 0c105dad8..af7b6f24f 100644 --- a/tests/unit/api/telemetry/test_meter.py +++ b/tests/unit/api/telemetry/test_meter.py @@ -5,12 +5,14 @@ See LICENSE for details """ -from karapace.core.config import KarapaceTelemetry -from karapace.core.container import KarapaceContainer -from karapace.api.telemetry.meter import Meter, NOOPMetricExporter from unittest.mock import patch + from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, MetricExporter +from karapace.api.telemetry.meter import Meter, NOOPMetricExporter +from karapace.core.config import KarapaceTelemetry +from karapace.core.container import KarapaceContainer + def test_meter(karapace_container: KarapaceContainer): with patch("karapace.api.telemetry.meter.metrics") as mock_metrics: diff --git a/tests/unit/api/telemetry/test_metrics.py b/tests/unit/api/telemetry/test_metrics.py index 43adc3880..db98e71f4 100644 --- a/tests/unit/api/telemetry/test_metrics.py +++ b/tests/unit/api/telemetry/test_metrics.py @@ -5,13 +5,13 @@ See LICENSE for details """ -from fastapi import Request, Response, HTTPException -from karapace.api.telemetry.metrics import HTTPRequestMetrics -from karapace.api.telemetry.meter import Meter -from unittest.mock import call, MagicMock -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, call, patch import pytest +from fastapi import HTTPException, Request, Response + +from karapace.api.telemetry.meter import Meter +from karapace.api.telemetry.metrics import HTTPRequestMetrics @pytest.fixture diff --git a/tests/unit/api/telemetry/test_middleware.py b/tests/unit/api/telemetry/test_middleware.py index 3dc2e0ed4..687fdcaea 100644 --- a/tests/unit/api/telemetry/test_middleware.py +++ b/tests/unit/api/telemetry/test_middleware.py @@ -5,17 +5,18 @@ See LICENSE for details """ +import logging +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest from _pytest.logging import LogCaptureFixture from fastapi import FastAPI, Request, Response from opentelemetry.trace import SpanKind, Status, StatusCode -from karapace.api.telemetry.metrics import HTTPRequestMetrics + from karapace.api.telemetry.meter import Meter +from karapace.api.telemetry.metrics import HTTPRequestMetrics from karapace.api.telemetry.middleware import setup_telemetry_middleware, telemetry_middleware from karapace.api.telemetry.tracer import Tracer -from unittest.mock import AsyncMock, call, MagicMock, patch - -import logging -import pytest @pytest.fixture diff --git a/tests/unit/api/telemetry/test_setup.py b/tests/unit/api/telemetry/test_setup.py index ac410955e..2a22cf9cf 100644 --- a/tests/unit/api/telemetry/test_setup.py +++ b/tests/unit/api/telemetry/test_setup.py @@ -5,13 +5,14 @@ See LICENSE for details """ +import logging +from unittest.mock import MagicMock, patch + from _pytest.logging import LogCaptureFixture from opentelemetry.sdk.trace import TracerProvider + from karapace.api.telemetry.setup import setup_tracing from karapace.api.telemetry.tracer import Tracer -from unittest.mock import MagicMock, patch - -import logging def test_setup_telemetry(caplog: LogCaptureFixture): diff --git a/tests/unit/api/telemetry/test_tracer.py b/tests/unit/api/telemetry/test_tracer.py index a4fe8fa46..53f3bd934 100644 --- a/tests/unit/api/telemetry/test_tracer.py +++ b/tests/unit/api/telemetry/test_tracer.py @@ -5,13 +5,15 @@ See LICENSE for details """ +from unittest.mock import MagicMock, call, patch + from fastapi import Request, Response -from karapace.core.config import KarapaceTelemetry -from karapace.core.container import KarapaceContainer from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SpanExporter, SpanProcessor from opentelemetry.trace.span import Span + from karapace.api.telemetry.tracer import NOOPSpanExporter, Tracer -from unittest.mock import call, MagicMock, patch +from karapace.core.config import KarapaceTelemetry +from karapace.core.container import KarapaceContainer def test_tracer(karapace_container: KarapaceContainer): diff --git a/tests/unit/api/test_controller.py b/tests/unit/api/test_controller.py index 343c2629e..0b241ae08 100644 --- a/tests/unit/api/test_controller.py +++ b/tests/unit/api/test_controller.py @@ -3,16 +3,17 @@ See LICENSE for details """ -from fastapi.exceptions import HTTPException -from karapace.core.rapu import HTTPResponse -from karapace.core.schema_models import SchemaType, ValidatedTypedSchema -from karapace.api.container import SchemaRegistryContainer -from karapace.core.schema_reader import KafkaSchemaReader -from unittest.mock import Mock, patch, PropertyMock - import asyncio import json +from unittest.mock import Mock, PropertyMock, patch + import pytest +from fastapi.exceptions import HTTPException + +from karapace.api.container import SchemaRegistryContainer +from karapace.rapu import HTTPResponse +from karapace.core.schema_models import SchemaType, ValidatedTypedSchema +from karapace.core.schema_reader import KafkaSchemaReader TYPED_AVRO_SCHEMA = ValidatedTypedSchema.parse( SchemaType.AVRO, diff --git a/tests/unit/api/test_forwarding.py b/tests/unit/api/test_forwarding.py index 4d5ea1550..7ce4dba52 100644 --- a/tests/unit/api/test_forwarding.py +++ b/tests/unit/api/test_forwarding.py @@ -5,18 +5,18 @@ See LICENSE for details """ +from unittest.mock import AsyncMock, Mock + +import pytest from fastapi import Request from fastapi.exceptions import HTTPException -from karapace.api.forward_client import ForwardClient -from karapace.core.typing import PrimaryInfo + from karapace.api.controller import KarapaceSchemaRegistryController -from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.api.forward_client import ForwardClient from karapace.api.routers.config import config_put -from unittest.mock import AsyncMock, Mock - from karapace.api.routers.requests import CompatibilityRequest - -import pytest +from karapace.core.schema_registry import KarapaceSchemaRegistry +from karapace.core.typing import PrimaryInfo async def test_forwarding_not_a_primary_and_own_primary_url() -> None: diff --git a/tests/unit/avro_dataclasses/test_introspect.py b/tests/unit/avro_dataclasses/test_introspect.py index b9a14645d..d6c61b149 100644 --- a/tests/unit/avro_dataclasses/test_introspect.py +++ b/tests/unit/avro_dataclasses/test_introspect.py @@ -3,16 +3,17 @@ See LICENSE for details """ +import datetime from collections.abc import Mapping, Sequence -from dataclasses import dataclass, Field, field, fields +from dataclasses import Field, dataclass, field, fields from enum import Enum -from karapace.core.avro_dataclasses.introspect import field_schema, record_schema, UnsupportedAnnotation -from karapace.core.avro_dataclasses.schema import FieldSchema from typing import Final -import datetime import pytest +from karapace.core.avro_dataclasses.introspect import UnsupportedAnnotation, field_schema, record_schema +from karapace.core.avro_dataclasses.schema import FieldSchema + class Symbols(Enum): a = "a" diff --git a/tests/unit/avro_dataclasses/test_models.py b/tests/unit/avro_dataclasses/test_models.py index 43f369ab0..bda1ee385 100644 --- a/tests/unit/avro_dataclasses/test_models.py +++ b/tests/unit/avro_dataclasses/test_models.py @@ -3,14 +3,15 @@ See LICENSE for details """ -from dataclasses import dataclass, field -from karapace.core.avro_dataclasses.models import AvroModel - import datetime import enum import io -import pytest import uuid +from dataclasses import dataclass, field + +import pytest + +from karapace.core.avro_dataclasses.models import AvroModel class Symbol(enum.Enum): diff --git a/tests/unit/backup/backends/test_v1.py b/tests/unit/backup/backends/test_v1.py index 5101298c2..36768d36d 100644 --- a/tests/unit/backup/backends/test_v1.py +++ b/tests/unit/backup/backends/test_v1.py @@ -3,15 +3,15 @@ See LICENSE for details """ +import json +import textwrap from functools import partial +from pathlib import Path + from karapace.backup.backends.reader import ProducerSend, RestoreTopicLegacy from karapace.backup.backends.v1 import SchemaBackupV1Reader from karapace.backup.encoders import encode_key, encode_value from karapace.core.key_format import KeyFormatter -from pathlib import Path - -import json -import textwrap def get_reader() -> SchemaBackupV1Reader: diff --git a/tests/unit/backup/backends/test_v2.py b/tests/unit/backup/backends/test_v2.py index de8a92be7..010585307 100644 --- a/tests/unit/backup/backends/test_v2.py +++ b/tests/unit/backup/backends/test_v2.py @@ -5,19 +5,19 @@ from __future__ import annotations +import datetime +import json +import time from functools import partial +from pathlib import Path + from karapace.backup.backends.reader import ProducerSend, RestoreTopicLegacy from karapace.backup.backends.v2 import AnonymizeAvroWriter, SchemaBackupV2Reader, SchemaBackupV2Writer from karapace.backup.encoders import encode_key, encode_value from karapace.core.kafka.types import Timestamp from karapace.core.key_format import KeyFormatter -from pathlib import Path from tests.utils import StubMessage -import datetime -import json -import time - def get_reader() -> SchemaBackupV2Reader: return SchemaBackupV2Reader( diff --git a/tests/unit/backup/backends/v3/conftest.py b/tests/unit/backup/backends/v3/conftest.py index 62219a5b1..d181da7e4 100644 --- a/tests/unit/backup/backends/v3/conftest.py +++ b/tests/unit/backup/backends/v3/conftest.py @@ -3,11 +3,11 @@ See LICENSE for details """ +import contextlib +import io from collections.abc import Iterator from contextlib import closing -import contextlib -import io import pytest diff --git a/tests/unit/backup/backends/v3/test_backend.py b/tests/unit/backup/backends/v3/test_backend.py index 518cb09ed..33e9e418c 100644 --- a/tests/unit/backup/backends/v3/test_backend.py +++ b/tests/unit/backup/backends/v3/test_backend.py @@ -3,9 +3,17 @@ See LICENSE for details """ +import datetime +import time from dataclasses import replace +from pathlib import Path +from unittest import mock + +import pytest +import xxhash + from karapace.backup.backends.reader import ProducerSend, RestoreTopic -from karapace.backup.backends.v3.backend import _PartitionStats, SchemaBackupV3Reader, SchemaBackupV3Writer +from karapace.backup.backends.v3.backend import SchemaBackupV3Reader, SchemaBackupV3Writer, _PartitionStats from karapace.backup.backends.v3.errors import ( InconsistentOffset, InvalidChecksum, @@ -17,14 +25,7 @@ from karapace.backup.backends.v3.readers import read_records from karapace.backup.backends.v3.schema import ChecksumAlgorithm, DataFile from karapace.core.kafka.types import Timestamp -from pathlib import Path from tests.utils import StubMessage -from unittest import mock - -import datetime -import pytest -import time -import xxhash def test_writer_reader_roundtrip(tmp_path: Path) -> None: diff --git a/tests/unit/backup/backends/v3/test_serialisation.py b/tests/unit/backup/backends/v3/test_serialisation.py index 9bddee6b1..ef7b01a0f 100644 --- a/tests/unit/backup/backends/v3/test_serialisation.py +++ b/tests/unit/backup/backends/v3/test_serialisation.py @@ -3,8 +3,17 @@ See LICENSE for details """ +import datetime +import io +import time +import uuid +from typing import IO + +import pytest from hypothesis import given from hypothesis.strategies import integers +from xxhash import xxh64 + from karapace.backup.backends.v3.errors import ( IntegerAboveBound, IntegerBelowBound, @@ -24,14 +33,6 @@ write_uint64, ) from tests.unit.backup.backends.v3.conftest import setup_buffer -from typing import IO -from xxhash import xxh64 - -import datetime -import io -import pytest -import time -import uuid @pytest.mark.parametrize( diff --git a/tests/unit/backup/test_api.py b/tests/unit/backup/test_api.py index 76198cb7e..6880106c4 100644 --- a/tests/unit/backup/test_api.py +++ b/tests/unit/backup/test_api.py @@ -5,8 +5,16 @@ from __future__ import annotations -from aiokafka.errors import KafkaError, TopicAlreadyExistsError from collections.abc import Callable +from pathlib import Path +from types import FunctionType +from typing import ContextManager, cast +from unittest import mock +from unittest.mock import MagicMock + +import pytest +from aiokafka.errors import KafkaError, TopicAlreadyExistsError + from karapace.backup.api import ( _admin, _consumer, @@ -26,13 +34,6 @@ from karapace.core.container import KarapaceContainer from karapace.core.kafka.consumer import KafkaConsumer, PartitionMetadata from karapace.core.kafka.producer import KafkaProducer -from pathlib import Path -from types import FunctionType -from typing import cast, ContextManager -from unittest import mock -from unittest.mock import MagicMock - -import pytest patch_admin_new = mock.patch( "karapace.backup.api.KafkaAdminClient.__new__", diff --git a/tests/unit/backup/test_encoders.py b/tests/unit/backup/test_encoders.py index edcb1b3c9..83e52e4f8 100644 --- a/tests/unit/backup/test_encoders.py +++ b/tests/unit/backup/test_encoders.py @@ -5,12 +5,13 @@ from __future__ import annotations -from karapace.backup.encoders import encode_key, encode_value -from karapace.core.key_format import KeyFormatter from unittest import mock import pytest +from karapace.backup.encoders import encode_key, encode_value +from karapace.core.key_format import KeyFormatter + class TestEncodeKey: @pytest.mark.parametrize("formatter", (KeyFormatter(), None)) diff --git a/tests/unit/backup/test_poll_timeout.py b/tests/unit/backup/test_poll_timeout.py index 3ce63d009..b9cec1e2f 100644 --- a/tests/unit/backup/test_poll_timeout.py +++ b/tests/unit/backup/test_poll_timeout.py @@ -4,10 +4,11 @@ """ from datetime import timedelta -from karapace.backup.poll_timeout import PollTimeout import pytest +from karapace.backup.poll_timeout import PollTimeout + class TestPollTimeout: @pytest.mark.parametrize("it", ("PT0.999S", timedelta(milliseconds=999))) diff --git a/tests/unit/backup/test_safe_writer.py b/tests/unit/backup/test_safe_writer.py index 3e4e29203..c162e299c 100644 --- a/tests/unit/backup/test_safe_writer.py +++ b/tests/unit/backup/test_safe_writer.py @@ -3,19 +3,20 @@ See LICENSE for details """ +import secrets +import sys +from pathlib import Path + +import pytest + from karapace.backup.safe_writer import ( + OverwriteRefused, + StdOut, _check_destination_file, bytes_writer, - OverwriteRefused, staging_directory, - StdOut, str_writer, ) -from pathlib import Path - -import pytest -import secrets -import sys class TestCheckDestinationFile: diff --git a/tests/unit/compatibility/jsonschema/test_jsonschema_compatibility.py b/tests/unit/compatibility/jsonschema/test_jsonschema_compatibility.py index 50bb92959..cc5b695ca 100644 --- a/tests/unit/compatibility/jsonschema/test_jsonschema_compatibility.py +++ b/tests/unit/compatibility/jsonschema/test_jsonschema_compatibility.py @@ -5,6 +5,7 @@ from avro.compatibility import SchemaCompatibilityResult, SchemaCompatibilityType from jsonschema import Draft7Validator + from karapace.core.compatibility.jsonschema.checks import compatibility from tests.schemas.json_schemas import ( A_DINT_B_DINT_OBJECT_SCHEMA, diff --git a/tests/unit/compatibility/test_compatibility.py b/tests/unit/compatibility/test_compatibility.py index d94a41614..e71b0848e 100644 --- a/tests/unit/compatibility/test_compatibility.py +++ b/tests/unit/compatibility/test_compatibility.py @@ -3,13 +3,14 @@ See LICENSE for details """ +import json + from avro.compatibility import SchemaCompatibilityType + from karapace.core.compatibility import CompatibilityModes from karapace.core.compatibility.schema_compatibility import SchemaCompatibility from karapace.core.schema_models import SchemaType, ValidatedTypedSchema -import json - def test_schema_type_can_change_when_mode_none() -> None: avro_str = json.dumps({"type": "record", "name": "Record1", "fields": [{"name": "field1", "type": "int"}]}) diff --git a/tests/unit/instrumentation/test_prometheus.py b/tests/unit/instrumentation/test_prometheus.py index f74c417a0..b27feae6a 100644 --- a/tests/unit/instrumentation/test_prometheus.py +++ b/tests/unit/instrumentation/test_prometheus.py @@ -5,15 +5,16 @@ See LICENSE for details """ -from _pytest.logging import LogCaptureFixture -from karapace.core.instrumentation.prometheus import PrometheusInstrumentation -from karapace.core.rapu import RestApp -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram -from unittest.mock import AsyncMock, call, MagicMock, patch +import logging +from unittest.mock import AsyncMock, MagicMock, call, patch import aiohttp.web -import logging import pytest +from _pytest.logging import LogCaptureFixture +from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram + +from karapace.core.instrumentation.prometheus import PrometheusInstrumentation +from karapace.rapu import RestApp class TestPrometheusInstrumentation: diff --git a/tests/unit/test_authentication.py b/tests/unit/kafka_rest_apis/test_authentication.py similarity index 97% rename from tests/unit/test_authentication.py rename to tests/unit/kafka_rest_apis/test_authentication.py index 794b6fc49..49e3f9e7c 100644 --- a/tests/unit/test_authentication.py +++ b/tests/unit/kafka_rest_apis/test_authentication.py @@ -5,22 +5,23 @@ from __future__ import annotations +import base64 +import datetime from collections.abc import Mapping from http import HTTPStatus +from typing import Any + +import jwt +import pytest + from karapace.core.container import KarapaceContainer -from karapace.core.kafka_rest_apis.authentication import ( +from karapace.kafka_rest_apis.authentication import ( + SimpleOauthTokenProvider, get_auth_config_from_header, get_expiration_time_from_header, get_kafka_client_auth_parameters_from_config, - SimpleOauthTokenProvider, ) -from karapace.core.rapu import HTTPResponse, JSON_CONTENT_TYPE -from typing import Any - -import base64 -import datetime -import jwt -import pytest +from karapace.rapu import JSON_CONTENT_TYPE, HTTPResponse def _assert_unauthorized_http_response(http_response: HTTPResponse) -> None: diff --git a/tests/unit/kafka_rest_apis/test_rest_proxy_cluster_metadata_cache.py b/tests/unit/kafka_rest_apis/test_rest_proxy_cluster_metadata_cache.py index ef972e073..e3b92ae29 100644 --- a/tests/unit/kafka_rest_apis/test_rest_proxy_cluster_metadata_cache.py +++ b/tests/unit/kafka_rest_apis/test_rest_proxy_cluster_metadata_cache.py @@ -3,12 +3,12 @@ See LICENSE for details """ -from karapace.core.container import KarapaceContainer -from karapace.core.kafka_rest_apis import UserRestProxy -from karapace.core.serialization import SchemaRegistrySerializer +import copy from unittest.mock import patch -import copy +from karapace.core.container import KarapaceContainer +from karapace.kafka_rest_apis import UserRestProxy +from karapace.core.serialization import SchemaRegistrySerializer def user_rest_proxy(karapace_container: KarapaceContainer, max_age_metadata: int = 5) -> UserRestProxy: @@ -209,9 +209,9 @@ async def test_global_cache_is_used_for_single_topic(karapace_container: Karapac with patch("time.monotonic", return_value=14): await proxy.cluster_metadata(["topic_a", "topic_b"]) - assert ( - mocked_cluster_metadata.call_count == 0 - ), "the result should still be cached since we marked it as ready at time 11 and we are at 14" + assert mocked_cluster_metadata.call_count == 0, ( + "the result should still be cached since we marked it as ready at time 11 and we are at 14" + ) async def test_cache_is_evicted_if_one_topic_is_expired(karapace_container: KarapaceContainer) -> None: @@ -312,9 +312,9 @@ async def test_update_topic_cache_do_not_evict_all_the_global_cache(karapace_con expected_metadata["topics"]["topic_a"] = TOPIC_REQUEST["topics"]["topic_a"] assert proxy._cluster_metadata == expected_metadata - assert ( - mocked_cluster_metadata.call_count == 1 - ), "we should call the server since the previous time of caching for the topic_a was 0" + assert mocked_cluster_metadata.call_count == 1, ( + "we should call the server since the previous time of caching for the topic_a was 0" + ) async def test_update_local_cache_does_not_evict_all_the_global_cache_if_no_new_data( @@ -339,13 +339,13 @@ async def test_update_local_cache_does_not_evict_all_the_global_cache_if_no_new_ expected_metadata = copy.deepcopy(ALL_TOPIC_REQUEST) expected_metadata["topics"]["topic_a"] = TOPIC_REQUEST["topics"]["topic_a"] assert proxy._cluster_metadata == expected_metadata - assert ( - proxy._cluster_metadata_complete - ), "since wasn't containing new brokers and no new topics the metadata its completed" + assert proxy._cluster_metadata_complete, ( + "since wasn't containing new brokers and no new topics the metadata its completed" + ) - assert ( - mocked_cluster_metadata.call_count == 1 - ), "we should call the server since the previous time of caching for the topic_a was 0" + assert mocked_cluster_metadata.call_count == 1, ( + "we should call the server since the previous time of caching for the topic_a was 0" + ) async def test_update_local_cache_not_evict_all_the_global_cache_if_changed_replica_data( diff --git a/tests/unit/protobuf/test_io.py b/tests/unit/protobuf/test_io.py index 63e5704c6..75f164057 100644 --- a/tests/unit/protobuf/test_io.py +++ b/tests/unit/protobuf/test_io.py @@ -3,6 +3,8 @@ See LICENSE for details """ +import textwrap + from karapace.core.dependency import Dependency from karapace.core.protobuf.io import crawl_dependencies from karapace.core.protobuf.schema import ProtobufSchema @@ -10,8 +12,6 @@ from karapace.core.schema_type import SchemaType from karapace.core.typing import Subject -import textwrap - def test_crawl_dependencies() -> None: schema_a = ValidatedTypedSchema.parse( diff --git a/tests/unit/protobuf/test_proto_file_element.py b/tests/unit/protobuf/test_proto_file_element.py index abea22fc9..f89c3878a 100644 --- a/tests/unit/protobuf/test_proto_file_element.py +++ b/tests/unit/protobuf/test_proto_file_element.py @@ -5,20 +5,20 @@ # Ported from square/wire: # wire-library/wire-schema/src/jvmTest/kotlin/com/squareup/wire/schema/internal/parser/ProtoFileElementTest.kt +import copy + from karapace.core.protobuf.extend_element import ExtendElement from karapace.core.protobuf.field import Field from karapace.core.protobuf.field_element import FieldElement from karapace.core.protobuf.kotlin_wrapper import trim_margin from karapace.core.protobuf.location import Location from karapace.core.protobuf.message_element import MessageElement -from karapace.core.protobuf.option_element import OptionElement, PACKED_OPTION_ELEMENT +from karapace.core.protobuf.option_element import PACKED_OPTION_ELEMENT, OptionElement from karapace.core.protobuf.proto_file_element import PackageName, ProtoFileElement, TypeName from karapace.core.protobuf.proto_parser import ProtoParser from karapace.core.protobuf.service_element import ServiceElement from karapace.core.protobuf.syntax import Syntax -import copy - location: Location = Location("some/folder", "file.proto") diff --git a/tests/unit/protobuf/test_proto_parser.py b/tests/unit/protobuf/test_proto_parser.py index 1c0839410..3f0f81908 100644 --- a/tests/unit/protobuf/test_proto_parser.py +++ b/tests/unit/protobuf/test_proto_parser.py @@ -5,6 +5,8 @@ # Ported from square/wire: # wire-library/wire-schema/src/jvmTest/kotlin/com/squareup/wire/schema/internal/parser/ProtoParserTest.kt +import pytest + from karapace.core.protobuf.enum_constant_element import EnumConstantElement from karapace.core.protobuf.enum_element import EnumElement from karapace.core.protobuf.exception import IllegalStateException @@ -26,8 +28,6 @@ from karapace.core.protobuf.syntax import Syntax from karapace.core.protobuf.utils import MAX_TAG_VALUE -import pytest - location: Location = Location("", "file.proto") @@ -519,7 +519,7 @@ def test_syntax_not_first_declaration_throws(): proto = trim_margin(proto) with pytest.raises( IllegalStateException, - match="Syntax error in file.proto:2:1: 'syntax' element must be the first declaration " "in a file", + match="Syntax error in file.proto:2:1: 'syntax' element must be the first declaration in a file", ): ProtoParser.parse(location, proto) pytest.fail("") @@ -632,7 +632,7 @@ def test_proto3_message_fields_forbid_required(): proto = trim_margin(proto) with pytest.raises( IllegalStateException, - match="Syntax error in file.proto:3:3: 'required' label forbidden in proto3 field " "declarations", + match="Syntax error in file.proto:3:3: 'required' label forbidden in proto3 field declarations", ): ProtoParser.parse(location, proto) pytest.fail("") @@ -679,7 +679,7 @@ def test_proto3_extension_fields_forbids_required(): proto = trim_margin(proto) with pytest.raises( IllegalStateException, - match="Syntax error in file.proto:5:3: 'required' label forbidden in proto3 field " "declarations", + match="Syntax error in file.proto:5:3: 'required' label forbidden in proto3 field declarations", ): ProtoParser.parse(location, proto) pytest.fail("") @@ -1031,8 +1031,7 @@ def test_package_declaration(): MessageElement( location=location.at(6, 1), name="FileDescriptorSet", - documentation="The protocol compiler can output a FileDescriptorSet containing the .proto\nfiles " - "it parses.", + documentation="The protocol compiler can output a FileDescriptorSet containing the .proto\nfiles it parses.", ) ], options=[OptionElement("java_package", OptionElement.Kind.STRING, "com.google.protobuf")], diff --git a/tests/unit/protobuf/test_protobuf_normalization.py b/tests/unit/protobuf/test_protobuf_normalization.py index ec3d0b4c1..f7e82296f 100644 --- a/tests/unit/protobuf/test_protobuf_normalization.py +++ b/tests/unit/protobuf/test_protobuf_normalization.py @@ -3,16 +3,17 @@ See LICENSE for details """ +from typing import Final + +import pytest + from karapace.core.dependency import Dependency from karapace.core.protobuf.compare_result import CompareResult from karapace.core.protobuf.location import Location from karapace.core.protobuf.proto_normalizations import normalize -from karapace.core.schema_models import parse_protobuf_schema_definition, ValidatedTypedSchema +from karapace.core.schema_models import ValidatedTypedSchema, parse_protobuf_schema_definition from karapace.core.schema_type import SchemaType from karapace.core.typing import Subject, Version -from typing import Final - -import pytest LOCATION: Final[Location] = Location("somefolder", "file.proto") @@ -933,15 +934,15 @@ def test_full_path_and_simple_names_are_equal() -> None: result = CompareResult() fully_qualitifed_simple_name_notation.compare(normalize(fully_qualitifed_dot_notation), result) assert result.is_compatible(), "normalized itn't equal to simple name" - assert ( - normalize(fully_qualitifed_dot_notation).to_schema() == fully_qualitifed_simple_name_notation.to_schema() - ), "normalization should transform it into an equivalent simple name" + assert normalize(fully_qualitifed_dot_notation).to_schema() == fully_qualitifed_simple_name_notation.to_schema(), ( + "normalization should transform it into an equivalent simple name" + ) fully_qualitifed_simple_name_notation.compare(normalize(fully_qualitifed_simple_name_notation), result) assert result.is_compatible(), "normalization shouldn't change a simple name notation protofile" - assert ( - fully_qualitifed_simple_name_notation.to_schema() == normalize(fully_qualitifed_dot_notation).to_schema() - ), "also the string rendering shouldn't change a simple name notation protofile" + assert fully_qualitifed_simple_name_notation.to_schema() == normalize(fully_qualitifed_dot_notation).to_schema(), ( + "also the string rendering shouldn't change a simple name notation protofile" + ) def test_full_path_and_simple_names_are_equal_with_formatter() -> None: @@ -967,9 +968,9 @@ def test_full_path_and_simple_names_are_equal_with_formatter() -> None: result = CompareResult() fully_qualitifed_simple_name_notation.compare(fully_qualitifed_dot_notation, result) assert result.is_compatible(), "normalized schemas are not compatible" - assert ( - fully_qualitifed_dot_notation.schema == fully_qualitifed_simple_name_notation.schema - ), "normalized schemas should match" + assert fully_qualitifed_dot_notation.schema == fully_qualitifed_simple_name_notation.schema, ( + "normalized schemas should match" + ) TRICKY_DEPENDENCY = """\ @@ -1060,6 +1061,6 @@ def test_full_path_and_simple_names_are_not_equal_if_simple_name_is_not_unique_w use_protobuf_formatter=True, ) - assert ( - normalized_schema.schema == schema.schema - ), "Since the simple name is not unique identifying the type isn't replacing the source" + assert normalized_schema.schema == schema.schema, ( + "Since the simple name is not unique identifying the type isn't replacing the source" + ) diff --git a/tests/unit/protobuf/test_protobuf_schema_name.py b/tests/unit/protobuf/test_protobuf_schema_name.py index e5215be81..1af99d5f8 100644 --- a/tests/unit/protobuf/test_protobuf_schema_name.py +++ b/tests/unit/protobuf/test_protobuf_schema_name.py @@ -3,13 +3,13 @@ See LICENSE for details """ +import pytest + from karapace.core.protobuf.schema import ProtobufSchema from karapace.core.schema_models import ValidatedTypedSchema from karapace.core.schema_type import SchemaType from tests.utils import schema_protobuf_second -import pytest - MESSAGE_WITH_ENUM = """\ syntax = "proto3"; diff --git a/tests/unit/protobuf/test_protoc.py b/tests/unit/protobuf/test_protoc.py index 67d22faa9..3d5d1973b 100644 --- a/tests/unit/protobuf/test_protoc.py +++ b/tests/unit/protobuf/test_protoc.py @@ -3,15 +3,15 @@ See LICENSE for details """ -from karapace.core.container import KarapaceContainer -from karapace.core.protobuf.io import calculate_class_name -from karapace.core.protobuf.kotlin_wrapper import trim_margin - import importlib import importlib.util import logging import subprocess +from karapace.core.container import KarapaceContainer +from karapace.core.protobuf.io import calculate_class_name +from karapace.core.protobuf.kotlin_wrapper import trim_margin + log = logging.getLogger(__name__) diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index 1e1be7a5d..6da0e5148 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -3,10 +3,10 @@ See LICENSE for details """ -from karapace.core.auth import ACLAuthorizer, ACLEntry, hash_password, HashAlgorithm, Operation, User - import re +from karapace.core.auth import ACLAuthorizer, ACLEntry, HashAlgorithm, Operation, User, hash_password + def test_empty_acl_authorizer() -> None: authorizer = ACLAuthorizer() diff --git a/tests/unit/test_avro_compatibility.py b/tests/unit/test_avro_compatibility.py index 081d8f538..78b086af2 100644 --- a/tests/unit/test_avro_compatibility.py +++ b/tests/unit/test_avro_compatibility.py @@ -6,13 +6,14 @@ See LICENSE for details """ +import json + +import pytest from avro.compatibility import ReaderWriterCompatibilityChecker, SchemaCompatibilityResult, SchemaCompatibilityType from avro.name import Names from avro.schema import ArraySchema, Field, MapSchema, Schema, UnionSchema -from karapace.core.schema_models import parse_avro_schema_definition -import json -import pytest +from karapace.core.schema_models import parse_avro_schema_definition # Schemas defined in AvroCompatibilityTest.java. Used here to ensure compatibility with the schema-registry schema1 = parse_avro_schema_definition('{"type":"record","name":"myrecord","fields":[{"type":"string","name":"f1"}]}') diff --git a/tests/unit/test_forwarding_client.py b/tests/unit/test_forwarding_client.py index 2df140b25..5cc7128b2 100644 --- a/tests/unit/test_forwarding_client.py +++ b/tests/unit/test_forwarding_client.py @@ -8,16 +8,17 @@ from __future__ import annotations from dataclasses import dataclass +from unittest.mock import AsyncMock, Mock, patch + +import aiohttp +import pytest from fastapi import Request from fastapi.datastructures import Headers -from karapace.api.forward_client import ForwardClient from pydantic import BaseModel from starlette.datastructures import MutableHeaders -from tests.base_testcase import BaseTestCase -from unittest.mock import AsyncMock, Mock, patch -import aiohttp -import pytest +from karapace.api.forward_client import ForwardClient +from tests.base_testcase import BaseTestCase class TestResponse(BaseModel): diff --git a/tests/unit/test_in_memory_database.py b/tests/unit/test_in_memory_database.py index 65976dbb7..18a14f9e0 100644 --- a/tests/unit/test_in_memory_database.py +++ b/tests/unit/test_in_memory_database.py @@ -7,7 +7,12 @@ from collections import defaultdict from collections.abc import Sequence +from pathlib import Path +from typing import Final + +import pytest from confluent_kafka.cimpl import KafkaError + from karapace.core.constants import DEFAULT_SCHEMA_TOPIC from karapace.core.container import KarapaceContainer from karapace.core.in_memory_database import InMemoryDatabase, KarapaceDatabase, Subject, SubjectData @@ -16,14 +21,10 @@ from karapace.core.offset_watcher import OffsetWatcher from karapace.core.protobuf.schema import ProtobufSchema from karapace.core.schema_models import SchemaVersion, TypedSchema +from karapace.core.schema_reader import KafkaSchemaReader from karapace.core.schema_references import Reference, Referents from karapace.core.schema_type import SchemaType from karapace.core.typing import SchemaId, Version -from pathlib import Path -from karapace.core.schema_reader import KafkaSchemaReader -from typing import Final - -import pytest TEST_DATA_FOLDER: Final = Path("tests/unit/test_data/") diff --git a/tests/unit/test_kafka_error_handler.py b/tests/unit/test_kafka_error_handler.py index a5c5c6212..27758f2d2 100644 --- a/tests/unit/test_kafka_error_handler.py +++ b/tests/unit/test_kafka_error_handler.py @@ -3,15 +3,16 @@ See LICENSE for details """ +import logging + +import aiokafka.errors as Errors +import pytest from _pytest.logging import LogCaptureFixture + from karapace.core.container import KarapaceContainer from karapace.core.errors import CorruptKafkaRecordException from karapace.core.kafka_error_handler import KafkaErrorHandler, KafkaErrorLocation -import aiokafka.errors as Errors -import logging -import pytest - @pytest.fixture(name="kafka_error_handler") def fixture_kafka_error_handler(karapace_container: KarapaceContainer) -> KafkaErrorHandler: diff --git a/tests/unit/test_key_format.py b/tests/unit/test_key_format.py index 5c3002d19..ffa63b510 100644 --- a/tests/unit/test_key_format.py +++ b/tests/unit/test_key_format.py @@ -6,12 +6,13 @@ """ from dataclasses import dataclass -from karapace.core.key_format import is_key_in_canonical_format, KeyFormatter, KeyMode -from karapace.core.typing import JsonData -from tests.base_testcase import BaseTestCase import pytest +from karapace.core.key_format import KeyFormatter, KeyMode, is_key_in_canonical_format +from karapace.core.typing import JsonData +from tests.base_testcase import BaseTestCase + @dataclass class KeyFormatCase(BaseTestCase): diff --git a/tests/unit/test_logging_setup.py b/tests/unit/test_logging_setup.py index ed06d0e1b..120975da7 100644 --- a/tests/unit/test_logging_setup.py +++ b/tests/unit/test_logging_setup.py @@ -3,13 +3,14 @@ See LICENSE for details """ +import logging +from unittest.mock import call, patch + +import pytest from _pytest.logging import LogCaptureFixture + from karapace.core.container import KarapaceContainer from karapace.core.logging_setup import configure_logging -from unittest.mock import patch, call - -import logging -import pytest def test_configure_logging_stdout_handler(caplog: LogCaptureFixture, karapace_container: KarapaceContainer) -> None: diff --git a/tests/unit/test_protobuf_binary_serialization.py b/tests/unit/test_protobuf_binary_serialization.py index 1f11d1e84..e5d1df519 100644 --- a/tests/unit/test_protobuf_binary_serialization.py +++ b/tests/unit/test_protobuf_binary_serialization.py @@ -3,6 +3,8 @@ See LICENSE for details """ +import pytest + from karapace.core.protobuf.schema import ProtobufSchema from karapace.core.protobuf.serialization import deserialize, serialize from tests.schemas.protobuf import ( @@ -30,8 +32,6 @@ schema_protobuf_references_bin, ) -import pytest - schema_serialized1 = ( "Cg5tZXNzYWdlcy5wcm90byIRCgNLZXkSCgoCaWQYASABKAUiMQoDRG9nEgwKBG5hbW" + "UYASABKAkSDgoGd2VpZ2h0GAIgASgFEgwKBHRveXMYBCADKAliBnByb3RvMw==" diff --git a/tests/unit/test_protobuf_serialization.py b/tests/unit/test_protobuf_serialization.py index cb4738661..e0d6a1eaf 100644 --- a/tests/unit/test_protobuf_serialization.py +++ b/tests/unit/test_protobuf_serialization.py @@ -3,27 +3,28 @@ See LICENSE for details """ +import asyncio +import logging +import struct +from unittest.mock import Mock, call + +import pytest + from karapace.core.container import KarapaceContainer from karapace.core.dependency import Dependency from karapace.core.protobuf.kotlin_wrapper import trim_margin from karapace.core.schema_models import ParsedTypedSchema, SchemaType, Versioner from karapace.core.schema_references import Reference from karapace.core.serialization import ( + START_BYTE, InvalidMessageHeader, InvalidMessageSchema, InvalidPayload, SchemaRegistryClient, SchemaRegistrySerializer, - START_BYTE, ) from karapace.core.typing import Subject from tests.utils import schema_protobuf, test_fail_objects_protobuf, test_objects_protobuf -from unittest.mock import call, Mock - -import asyncio -import logging -import pytest -import struct log = logging.getLogger(__name__) diff --git a/tests/unit/test_rapu.py b/tests/unit/test_rapu.py index 24a3afba5..5a57277ae 100644 --- a/tests/unit/test_rapu.py +++ b/tests/unit/test_rapu.py @@ -3,17 +3,18 @@ See LICENSE for details """ +import logging +from unittest.mock import Mock + +import pytest from _pytest.logging import LogCaptureFixture from aiohttp.client_exceptions import ClientConnectionError from aiohttp.web import Request + from karapace.core.container import KarapaceContainer -from karapace.core.karapace import KarapaceBase -from karapace.core.rapu import HTTPRequest, REST_ACCEPT_RE, REST_CONTENT_TYPE_RE +from karapace.kafka_rest_apis.karapace import KarapaceBase from karapace.core.statsd import StatsClient -from unittest.mock import Mock - -import logging -import pytest +from karapace.rapu import REST_ACCEPT_RE, REST_CONTENT_TYPE_RE, HTTPRequest async def test_header_get(): diff --git a/tests/unit/test_rest_auth.py b/tests/unit/test_rest_auth.py index 7d7f2c467..59869b0cc 100644 --- a/tests/unit/test_rest_auth.py +++ b/tests/unit/test_rest_auth.py @@ -5,13 +5,13 @@ from __future__ import annotations -from karapace.core.container import KarapaceContainer -from karapace.core.kafka_rest_apis import AUTH_EXPIRY_TOLERANCE, KafkaRest, UserRestProxy -from unittest.mock import call, Mock - import asyncio import datetime import time +from unittest.mock import Mock, call + +from karapace.core.container import KarapaceContainer +from karapace.kafka_rest_apis import AUTH_EXPIRY_TOLERANCE, KafkaRest, UserRestProxy def _create_mock_proxy( diff --git a/tests/unit/test_schema_models.py b/tests/unit/test_schema_models.py index 76cc0aebf..951c6d4ca 100644 --- a/tests/unit/test_schema_models.py +++ b/tests/unit/test_schema_models.py @@ -5,16 +5,17 @@ See LICENSE for details """ -from avro.schema import Schema as AvroSchema +import operator from collections.abc import Callable -from karapace.core.errors import InvalidVersion, VersionNotFoundException -from karapace.core.schema_models import parse_avro_schema_definition, SchemaVersion, TypedSchema, Versioner -from karapace.core.schema_type import SchemaType -from karapace.core.typing import Version, VersionTag from typing import Any -import operator import pytest +from avro.schema import Schema as AvroSchema + +from karapace.core.errors import InvalidVersion, VersionNotFoundException +from karapace.core.schema_models import SchemaVersion, TypedSchema, Versioner, parse_avro_schema_definition +from karapace.core.schema_type import SchemaType +from karapace.core.typing import Version, VersionTag # Schema versions factory fixture type SVFCallable = Callable[[None], Callable[[int, dict[str, Any]], dict[int, SchemaVersion]]] diff --git a/tests/unit/test_schema_reader.py b/tests/unit/test_schema_reader.py index e52fe4309..fc737973f 100644 --- a/tests/unit/test_schema_reader.py +++ b/tests/unit/test_schema_reader.py @@ -5,38 +5,39 @@ See LICENSE for details """ -from _pytest.logging import LogCaptureFixture +import json +import logging +import random +import time from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor -from confluent_kafka import Message from dataclasses import dataclass +from unittest.mock import Mock + +import confluent_kafka +import pytest +from _pytest.logging import LogCaptureFixture +from confluent_kafka import Message +from pytest import MonkeyPatch + from karapace.core.container import KarapaceContainer from karapace.core.errors import CorruptKafkaRecordException, ShutdownException from karapace.core.in_memory_database import InMemoryDatabase from karapace.core.kafka.consumer import KafkaConsumer from karapace.core.key_format import KeyFormatter from karapace.core.offset_watcher import OffsetWatcher -from karapace.core.schema_type import SchemaType -from karapace.core.typing import SchemaId, Version -from pytest import MonkeyPatch from karapace.core.schema_reader import ( - KafkaSchemaReader, MAX_MESSAGES_TO_CONSUME_AFTER_STARTUP, MAX_MESSAGES_TO_CONSUME_ON_STARTUP, - MessageType, OFFSET_EMPTY, OFFSET_UNINITIALIZED, + KafkaSchemaReader, + MessageType, ) +from karapace.core.schema_type import SchemaType +from karapace.core.typing import SchemaId, Version from tests.base_testcase import BaseTestCase from tests.utils import schema_protobuf_invalid_because_corrupted, schema_protobuf_with_invalid_ref -from unittest.mock import Mock - -import confluent_kafka -import json -import logging -import pytest -import random -import time def test_offset_watcher() -> None: diff --git a/tests/unit/test_serialization.py b/tests/unit/test_serialization.py index eb6cba43c..4999073b2 100644 --- a/tests/unit/test_serialization.py +++ b/tests/unit/test_serialization.py @@ -3,32 +3,33 @@ See LICENSE for details """ +import asyncio +import copy +import io +import json +import logging +import struct +from unittest.mock import Mock, call + +import avro +import pytest + from karapace.core.container import KarapaceContainer from karapace.core.schema_models import SchemaType, ValidatedTypedSchema, Versioner from karapace.core.serialization import ( - flatten_unions, - get_subject_name, HEADER_FORMAT, + START_BYTE, InvalidMessageHeader, InvalidMessageSchema, InvalidPayload, SchemaRegistryClient, SchemaRegistrySerializer, - START_BYTE, + flatten_unions, + get_subject_name, write_value, ) from karapace.core.typing import NameStrategy, Subject, SubjectType from tests.utils import schema_avro_json, test_objects_avro -from unittest.mock import call, Mock - -import asyncio -import avro -import copy -import io -import json -import logging -import pytest -import struct log = logging.getLogger(__name__) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 40f985510..2e749ec29 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -3,11 +3,12 @@ See LICENSE for details """ -from _pytest.logging import LogCaptureFixture -from karapace.core.utils import shutdown +import logging from unittest.mock import patch -import logging +from _pytest.logging import LogCaptureFixture + +from karapace.core.utils import shutdown def test_shutdown(caplog: LogCaptureFixture) -> None: diff --git a/tests/utils.py b/tests/utils.py index 9dedb34ec..25d70473c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,17 +3,6 @@ See LICENSE for details """ -from aiohttp.client_exceptions import ClientOSError, ServerDisconnectedError -from aiokafka.errors import TopicAlreadyExistsError -from collections.abc import Callable -from karapace.core.client import Client -from karapace.core.kafka.admin import KafkaAdminClient -from karapace.core.protobuf.kotlin_wrapper import trim_margin -from karapace.core.utils import Expiration -from pathlib import Path -from subprocess import Popen -from typing import Any, IO - import asyncio import copy import json @@ -21,6 +10,18 @@ import ssl import sys import uuid +from collections.abc import Callable +from pathlib import Path +from subprocess import Popen +from typing import IO, Any + +from aiohttp.client_exceptions import ClientOSError, ServerDisconnectedError +from aiokafka.errors import TopicAlreadyExistsError + +from karapace.core.client import Client +from karapace.core.kafka.admin import KafkaAdminClient +from karapace.core.protobuf.kotlin_wrapper import trim_margin +from karapace.core.utils import Expiration consumer_valid_payload = { "format": "avro",