Skip to content

Commit

Permalink
WIP: Add tests and fixes for protobuf references support
Browse files Browse the repository at this point in the history
  • Loading branch information
@jjaakola-aiven
jjaakola-aiven committed Oct 20, 2022
1 parent 7c36fe0 commit 9dd1c60
Show file tree
Hide file tree
Showing 13 changed files with 882 additions and 291 deletions.
40 changes: 33 additions & 7 deletions karapace/dependency.py
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,47 @@
from typing import Optional, TYPE_CHECKING
from karapace.schema_references import Reference
from karapace.typing import JsonData, Subject, Version
from typing import Any, Optional, TYPE_CHECKING

if TYPE_CHECKING:
from karapace.schema_models import ValidatedTypedSchema


class DependencyVerifierResult:
def __init__(self, result: bool, message: Optional[str] = "") -> None:
self.result = result
self.message = message


class Dependency:
def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None:
def __init__(self, name: str, subject: Subject, version: Version, target_schema: "ValidatedTypedSchema") -> None:
self.name = name
self.subject = subject
self.version = version
self.schema = schema
self.schema = target_schema

@staticmethod
def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency":
return Dependency(reference.name, reference.subject, reference.version, target_schema)

def to_dict(self) -> JsonData:
return {
"name": self.name,
"subject": self.subject,
"version": self.version,
}

def identifier(self) -> str:
return self.name + "_" + self.subject + "_" + str(self.version)

def __hash__(self) -> int:
return hash((self.name, self.subject, self.version, self.schema))

class DependencyVerifierResult:
def __init__(self, result: bool, message: Optional[str] = ""):
self.result = result
self.message = message
def __eq__(self, other: Any) -> bool:
if other is None or not isinstance(other, Dependency):
return False
return (
self.name == other.name
and self.subject == other.subject
and self.version == other.version
and self.schema == other.schema
)
9 changes: 3 additions & 6 deletions karapace/errors.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from typing import List, Union
from karapace.typing import Version
from typing import List


class VersionNotFoundException(Exception):
Expand All @@ -25,10 +26,6 @@ class InvalidReferences(Exception):
pass


class ReferencesNotSupportedException(Exception):
pass


class SchemasNotFoundException(Exception):
pass

Expand All @@ -50,7 +47,7 @@ class SubjectNotSoftDeletedException(Exception):


class ReferenceExistsException(Exception):
def __init__(self, referenced_by: List, version: Union[int, str]):
def __init__(self, referenced_by: List, version: Version):
super().__init__()
self.version = version
self.referenced_by = referenced_by
Expand Down
7 changes: 7 additions & 0 deletions karapace/protobuf/field_element.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,17 @@ def compare_message(

self_type_record = types.get_self_type(self_type)
other_type_record = types.get_other_type(other_type)

self_type_element: MessageElement = self_type_record.type_element
other_type_element: MessageElement = other_type_record.type_element

if types.self_type_short_name(self_type) != types.other_type_short_name(other_type):
result.add_modification(Modification.FIELD_NAME_ALTER)
else:
self_type_element.compare(other_type_element, result, types)

def __repr__(self):
return f"{self.element_type} {self.name} = {self.tag}"

def __str__(self):
return f"{self.element_type} {self.name} = {self.tag}"
126 changes: 63 additions & 63 deletions karapace/protobuf/known_dependency.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,77 +45,77 @@ class KnownDependency:
index: Dict = dict()
index_simple: Dict = dict()
map: Dict = {
"google/protobuf/any.proto": [".google.protobuf.Any"],
"google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"],
"google/protobuf/any.proto": ["google.protobuf.Any"],
"google/protobuf/api.proto": ["google.protobuf.Api", "google.protobuf.Method", "google.protobuf.Mixin"],
"google/protobuf/descriptor.proto": [
".google.protobuf.FileDescriptorSet",
".google.protobuf.FileDescriptorProto",
".google.protobuf.DescriptorProto",
".google.protobuf.ExtensionRangeOptions",
".google.protobuf.FieldDescriptorProto",
".google.protobuf.OneofDescriptorProto",
".google.protobuf.EnumDescriptorProto",
".google.protobuf.EnumValueDescriptorProto",
".google.protobuf.ServiceDescriptorProto",
".google.protobuf.MethodDescriptorProto",
".google.protobuf.FileOptions",
".google.protobuf.MessageOptions",
".google.protobuf.FieldOptions",
".google.protobuf.OneofOptions",
".google.protobuf.EnumOptions",
".google.protobuf.EnumValueOptions",
".google.protobuf.ServiceOptions",
".google.protobuf.MethodOptions",
".google.protobuf.UninterpretedOption",
".google.protobuf.SourceCodeInfo",
".google.protobuf.GeneratedCodeInfo",
"google.protobuf.FileDescriptorSet",
"google.protobuf.FileDescriptorProto",
"google.protobuf.DescriptorProto",
"google.protobuf.ExtensionRangeOptions",
"google.protobuf.FieldDescriptorProto",
"google.protobuf.OneofDescriptorProto",
"google.protobuf.EnumDescriptorProto",
"google.protobuf.EnumValueDescriptorProto",
"google.protobuf.ServiceDescriptorProto",
"google.protobuf.MethodDescriptorProto",
"google.protobuf.FileOptions",
"google.protobuf.MessageOptions",
"google.protobuf.FieldOptions",
"google.protobuf.OneofOptions",
"google.protobuf.EnumOptions",
"google.protobuf.EnumValueOptions",
"google.protobuf.ServiceOptions",
"google.protobuf.MethodOptions",
"google.protobuf.UninterpretedOption",
"google.protobuf.SourceCodeInfo",
"google.protobuf.GeneratedCodeInfo",
],
"google/protobuf/duration.proto": [".google.protobuf.Duration"],
"google/protobuf/empty.proto": [".google.protobuf.Empty"],
"google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"],
"google/protobuf/source_context.proto": [".google.protobuf.SourceContext"],
"google/protobuf/duration.proto": ["google.protobuf.Duration"],
"google/protobuf/empty.proto": ["google.protobuf.Empty"],
"google/protobuf/field_mask.proto": ["google.protobuf.FieldMask"],
"google/protobuf/source_context.proto": ["google.protobuf.SourceContext"],
"google/protobuf/struct.proto": [
".google.protobuf.Struct",
".google.protobuf.Value",
".google.protobuf.NullValue",
".google.protobuf.ListValue",
"google.protobuf.Struct",
"google.protobuf.Value",
"google.protobuf.NullValue",
"google.protobuf.ListValue",
],
"google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"],
"google/protobuf/timestamp.proto": ["google.protobuf.Timestamp"],
"google/protobuf/type.proto": [
".google.protobuf.Type",
".google.protobuf.Field",
".google.protobuf.Enum",
".google.protobuf.EnumValue",
".google.protobuf.Option",
".google.protobuf.Syntax",
"google.protobuf.Type",
"google.protobuf.Field",
"google.protobuf.Enum",
"google.protobuf.EnumValue",
"google.protobuf.Option",
"google.protobuf.Syntax",
],
"google/protobuf/wrappers.proto": [
".google.protobuf.DoubleValue",
".google.protobuf.FloatValue",
".google.protobuf.Int64Value",
".google.protobuf.UInt64Value",
".google.protobuf.Int32Value",
".google.protobuf.UInt32Value",
".google.protobuf.BoolValue",
".google.protobuf.StringValue",
".google.protobuf.BytesValue",
"google.protobuf.DoubleValue",
"google.protobuf.FloatValue",
"google.protobuf.Int64Value",
"google.protobuf.UInt64Value",
"google.protobuf.Int32Value",
"google.protobuf.UInt32Value",
"google.protobuf.BoolValue",
"google.protobuf.StringValue",
"google.protobuf.BytesValue",
],
"google/type/calendar_period.proto": [".google.type.CalendarPeriod"],
"google/type/color.proto": [".google.type.Color"],
"google/type/date.proto": [".google.type.Date"],
"google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"],
"google/type/dayofweek.proto": [".google.type.DayOfWeek"],
"google/type/decimal.proto": [".google.type.Decimal"],
"google/type/expr.proto": [".google.type.Expr"],
"google/type/fraction.proto": [".google.type.Fraction"],
"google/type/interval.proto": [".google.type.Interval"],
"google/type/latlng.proto": [".google.type.LatLng"],
"google/type/money.proto": [".google.type.Money"],
"google/type/month.proto": [".google.type.Month"],
"google/type/phone_number.proto": [".google.type.PhoneNumber"],
"google/type/postal_address.proto": [".google.type.PostalAddress"],
"google/type/quaternion.proto": [".google.type.Quaternion"],
"google/type/timeofday.proto": [".google.type.TimeOfDay"],
"google/type/calendar_period.proto": ["google.type.CalendarPeriod"],
"google/type/color.proto": ["google.type.Color"],
"google/type/date.proto": ["google.type.Date"],
"google/type/datetime.proto": ["google.type.DateTime", "google.type.TimeZone"],
"google/type/dayofweek.proto": ["google.type.DayOfWeek"],
"google/type/decimal.proto": ["google.type.Decimal"],
"google/type/expr.proto": ["google.type.Expr"],
"google/type/fraction.proto": ["google.type.Fraction"],
"google/type/interval.proto": ["google.type.Interval"],
"google/type/latlng.proto": ["google.type.LatLng"],
"google/type/money.proto": ["google.type.Money"],
"google/type/month.proto": ["google.type.Month"],
"google/type/phone_number.proto": ["google.type.PhoneNumber"],
"google/type/postal_address.proto": ["google.type.PostalAddress"],
"google/type/quaternion.proto": ["google.type.Quaternion"],
"google/type/timeofday.proto": ["google.type.TimeOfDay"],
"confluent/meta.proto": [".confluent.Meta"],
"confluent/type/decimal.proto": [".confluent.type.Decimal"],
}
Expand Down
25 changes: 23 additions & 2 deletions karapace/protobuf/proto_file_element.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,13 @@ def __eq__(self, other: "ProtoFileElement") -> bool: # type: ignore
def __repr__(self) -> str:
return self.to_schema()

def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareResult:

def compare(
self,
other: "ProtoFileElement",
result: CompareResult,
self_dependency_types: Optional[List[TypeElement]] = None,
other_dependency_types: Optional[List[TypeElement]] = None,
) -> CompareResult:
if self.package_name != other.package_name:
result.add_modification(Modification.PACKAGE_ALTER)
# TODO: do we need syntax check?
Expand All @@ -125,6 +130,22 @@ def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareRe
package_name = other.package_name or ""
compare_types.add_other_type(package_name, type_)

# If there are dependencies declared, add the types for both.
if self_dependency_types:
for i, type_ in enumerate(self_dependency_types):
package_name = ""

self_types[type_.name] = type_
self_indexes[type_.name] = i
compare_types.add_self_type(package_name, type_)

if other_dependency_types:
for i, type_ in enumerate(other_dependency_types):
package_name = ""
other_types[type_.name] = type_
other_indexes[type_.name] = i
compare_types.add_other_type(package_name, type_)

for name in chain(self_types.keys(), other_types.keys() - self_types.keys()):

result.push_path(str(name), True)
Expand Down
32 changes: 22 additions & 10 deletions karapace/protobuf/schema.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@
from karapace.protobuf.proto_parser import ProtoParser
from karapace.protobuf.type_element import TypeElement
from karapace.protobuf.utils import append_documentation, append_indented
from karapace.schema_references import References
from typing import Dict, Optional
from typing import List, Optional


def add_slashes(text: str) -> str:
Expand Down Expand Up @@ -106,15 +105,12 @@ def option_element_string(option: OptionElement) -> str:
class ProtobufSchema:
DEFAULT_LOCATION = Location.get("")

def __init__(
self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None
) -> None:
def __init__(self, schema: str, dependencies: Optional[List[Dependency]] = None) -> None:
if type(schema).__name__ != "str":
raise IllegalArgumentException("Non str type of schema string")
self.dirty = schema
self.cache_string = ""
self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema)
self.references = references
self.dependencies = dependencies

def gather_deps(self) -> ProtobufDependencyVerifier:
Expand All @@ -128,10 +124,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult:
return verifier.verify()

def collect_dependencies(self, verifier: ProtobufDependencyVerifier):

if self.dependencies:
for key in self.dependencies:
self.dependencies[key].schema.schema.collect_dependencies(verifier)
for dependency in self.dependencies:
dependency.schema.schema.collect_dependencies(verifier)
# verifier.add_import?? we have no access to own Kafka structure from this class...
# but we need data to analyse imports to avoid ciclyc dependencies...

Expand Down Expand Up @@ -223,4 +218,21 @@ def to_schema(self) -> str:
return "".join(strings)

def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult:
self.proto_file_element.compare(other.proto_file_element, result)
self_dependency_types: List[TypeElement] = []
other_dependency_types: List[TypeElement] = []
if self.dependencies:
for dependency in self.dependencies:
schema = dependency.schema.schema
if schema.proto_file_element.types:
self_dependency_types += schema.proto_file_element.types
if other.dependencies:
for dependency in other.dependencies:
schema = dependency.schema.schema
if schema.proto_file_element.types:
other_dependency_types += schema.proto_file_element.types
self.proto_file_element.compare(
other.proto_file_element,
result,
self_dependency_types=self_dependency_types,
other_dependency_types=other_dependency_types,
)
Loading

0 comments on commit 9dd1c60

Please sign in to comment.