Protobuf schema references support

karapace/dependency.py

@@ -1,21 +1,47 @@
-from typing import Optional, TYPE_CHECKING
+from karapace.schema_references import Reference
+from karapace.typing import JsonData, Subject, Version
+from typing import Any, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from karapace.schema_models import ValidatedTypedSchema
 
 
+class DependencyVerifierResult:
+    def __init__(self, result: bool, message: Optional[str] = "") -> None:
+        self.result = result
+        self.message = message
+
+
 class Dependency:
-    def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None:
+    def __init__(self, name: str, subject: Subject, version: Version, target_schema: "ValidatedTypedSchema") -> None:
         self.name = name
         self.subject = subject
         self.version = version
-        self.schema = schema
+        self.schema = target_schema
+
+    @staticmethod
+    def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency":
+        return Dependency(reference.name, reference.subject, reference.version, target_schema)
+
+    def to_dict(self) -> JsonData:
+        return {
+            "name": self.name,
+            "subject": self.subject,
+            "version": self.version,
+        }
 
     def identifier(self) -> str:
         return self.name + "_" + self.subject + "_" + str(self.version)
 
+    def __hash__(self) -> int:
+        return hash((self.name, self.subject, self.version, self.schema))
 
-class DependencyVerifierResult:
-    def __init__(self, result: bool, message: Optional[str] = ""):
-        self.result = result
-        self.message = message
+    def __eq__(self, other: Any) -> bool:
+        if other is None or not isinstance(other, Dependency):
+            return False
+        return (
+            self.name == other.name
+            and self.subject == other.subject
+            and self.version == other.version
+            and self.schema == other.schema
+        )

karapace/errors.py

@@ -1,4 +1,5 @@
-from typing import List, Union
+from karapace.typing import Version
+from typing import List
 
 
 class VersionNotFoundException(Exception):
@@ -25,10 +26,6 @@ class InvalidReferences(Exception):
     pass
 
 
-class ReferencesNotSupportedException(Exception):
-    pass
-
-
 class SchemasNotFoundException(Exception):
     pass
 
@@ -50,7 +47,7 @@ class SubjectNotSoftDeletedException(Exception):
 
 
 class ReferenceExistsException(Exception):
-    def __init__(self, referenced_by: List, version: Union[int, str]):
+    def __init__(self, referenced_by: List, version: Version):
         super().__init__()
         self.version = version
         self.referenced_by = referenced_by

karapace/protobuf/field_element.py

@@ -143,10 +143,17 @@ def compare_message(
 
         self_type_record = types.get_self_type(self_type)
         other_type_record = types.get_other_type(other_type)
+
         self_type_element: MessageElement = self_type_record.type_element
         other_type_element: MessageElement = other_type_record.type_element
 
         if types.self_type_short_name(self_type) != types.other_type_short_name(other_type):
             result.add_modification(Modification.FIELD_NAME_ALTER)
         else:
             self_type_element.compare(other_type_element, result, types)
+
+    def __repr__(self):
+        return f"{self.element_type} {self.name} = {self.tag}"
+
+    def __str__(self):
+        return f"{self.element_type} {self.name} = {self.tag}"

karapace/protobuf/known_dependency.py

@@ -45,77 +45,77 @@ class KnownDependency:
     index: Dict = dict()
     index_simple: Dict = dict()
     map: Dict = {
-        "google/protobuf/any.proto": [".google.protobuf.Any"],
-        "google/protobuf/api.proto": [".google.protobuf.Api", ".google.protobuf.Method", ".google.protobuf.Mixin"],
+        "google/protobuf/any.proto": ["google.protobuf.Any"],
+        "google/protobuf/api.proto": ["google.protobuf.Api", "google.protobuf.Method", "google.protobuf.Mixin"],
         "google/protobuf/descriptor.proto": [
-            ".google.protobuf.FileDescriptorSet",
-            ".google.protobuf.FileDescriptorProto",
-            ".google.protobuf.DescriptorProto",
-            ".google.protobuf.ExtensionRangeOptions",
-            ".google.protobuf.FieldDescriptorProto",
-            ".google.protobuf.OneofDescriptorProto",
-            ".google.protobuf.EnumDescriptorProto",
-            ".google.protobuf.EnumValueDescriptorProto",
-            ".google.protobuf.ServiceDescriptorProto",
-            ".google.protobuf.MethodDescriptorProto",
-            ".google.protobuf.FileOptions",
-            ".google.protobuf.MessageOptions",
-            ".google.protobuf.FieldOptions",
-            ".google.protobuf.OneofOptions",
-            ".google.protobuf.EnumOptions",
-            ".google.protobuf.EnumValueOptions",
-            ".google.protobuf.ServiceOptions",
-            ".google.protobuf.MethodOptions",
-            ".google.protobuf.UninterpretedOption",
-            ".google.protobuf.SourceCodeInfo",
-            ".google.protobuf.GeneratedCodeInfo",
+            "google.protobuf.FileDescriptorSet",
+            "google.protobuf.FileDescriptorProto",
+            "google.protobuf.DescriptorProto",
+            "google.protobuf.ExtensionRangeOptions",
+            "google.protobuf.FieldDescriptorProto",
+            "google.protobuf.OneofDescriptorProto",
+            "google.protobuf.EnumDescriptorProto",
+            "google.protobuf.EnumValueDescriptorProto",
+            "google.protobuf.ServiceDescriptorProto",
+            "google.protobuf.MethodDescriptorProto",
+            "google.protobuf.FileOptions",
+            "google.protobuf.MessageOptions",
+            "google.protobuf.FieldOptions",
+            "google.protobuf.OneofOptions",
+            "google.protobuf.EnumOptions",
+            "google.protobuf.EnumValueOptions",
+            "google.protobuf.ServiceOptions",
+            "google.protobuf.MethodOptions",
+            "google.protobuf.UninterpretedOption",
+            "google.protobuf.SourceCodeInfo",
+            "google.protobuf.GeneratedCodeInfo",
         ],
-        "google/protobuf/duration.proto": [".google.protobuf.Duration"],
-        "google/protobuf/empty.proto": [".google.protobuf.Empty"],
-        "google/protobuf/field_mask.proto": [".google.protobuf.FieldMask"],
-        "google/protobuf/source_context.proto": [".google.protobuf.SourceContext"],
+        "google/protobuf/duration.proto": ["google.protobuf.Duration"],
+        "google/protobuf/empty.proto": ["google.protobuf.Empty"],
+        "google/protobuf/field_mask.proto": ["google.protobuf.FieldMask"],
+        "google/protobuf/source_context.proto": ["google.protobuf.SourceContext"],
         "google/protobuf/struct.proto": [
-            ".google.protobuf.Struct",
-            ".google.protobuf.Value",
-            ".google.protobuf.NullValue",
-            ".google.protobuf.ListValue",
+            "google.protobuf.Struct",
+            "google.protobuf.Value",
+            "google.protobuf.NullValue",
+            "google.protobuf.ListValue",
         ],
-        "google/protobuf/timestamp.proto": [".google.protobuf.Timestamp"],
+        "google/protobuf/timestamp.proto": ["google.protobuf.Timestamp"],
         "google/protobuf/type.proto": [
-            ".google.protobuf.Type",
-            ".google.protobuf.Field",
-            ".google.protobuf.Enum",
-            ".google.protobuf.EnumValue",
-            ".google.protobuf.Option",
-            ".google.protobuf.Syntax",
+            "google.protobuf.Type",
+            "google.protobuf.Field",
+            "google.protobuf.Enum",
+            "google.protobuf.EnumValue",
+            "google.protobuf.Option",
+            "google.protobuf.Syntax",
         ],
         "google/protobuf/wrappers.proto": [
-            ".google.protobuf.DoubleValue",
-            ".google.protobuf.FloatValue",
-            ".google.protobuf.Int64Value",
-            ".google.protobuf.UInt64Value",
-            ".google.protobuf.Int32Value",
-            ".google.protobuf.UInt32Value",
-            ".google.protobuf.BoolValue",
-            ".google.protobuf.StringValue",
-            ".google.protobuf.BytesValue",
+            "google.protobuf.DoubleValue",
+            "google.protobuf.FloatValue",
+            "google.protobuf.Int64Value",
+            "google.protobuf.UInt64Value",
+            "google.protobuf.Int32Value",
+            "google.protobuf.UInt32Value",
+            "google.protobuf.BoolValue",
+            "google.protobuf.StringValue",
+            "google.protobuf.BytesValue",
         ],
-        "google/type/calendar_period.proto": [".google.type.CalendarPeriod"],
-        "google/type/color.proto": [".google.type.Color"],
-        "google/type/date.proto": [".google.type.Date"],
-        "google/type/datetime.proto": [".google.type.DateTime", ".google.type.TimeZone"],
-        "google/type/dayofweek.proto": [".google.type.DayOfWeek"],
-        "google/type/decimal.proto": [".google.type.Decimal"],
-        "google/type/expr.proto": [".google.type.Expr"],
-        "google/type/fraction.proto": [".google.type.Fraction"],
-        "google/type/interval.proto": [".google.type.Interval"],
-        "google/type/latlng.proto": [".google.type.LatLng"],
-        "google/type/money.proto": [".google.type.Money"],
-        "google/type/month.proto": [".google.type.Month"],
-        "google/type/phone_number.proto": [".google.type.PhoneNumber"],
-        "google/type/postal_address.proto": [".google.type.PostalAddress"],
-        "google/type/quaternion.proto": [".google.type.Quaternion"],
-        "google/type/timeofday.proto": [".google.type.TimeOfDay"],
+        "google/type/calendar_period.proto": ["google.type.CalendarPeriod"],
+        "google/type/color.proto": ["google.type.Color"],
+        "google/type/date.proto": ["google.type.Date"],
+        "google/type/datetime.proto": ["google.type.DateTime", "google.type.TimeZone"],
+        "google/type/dayofweek.proto": ["google.type.DayOfWeek"],
+        "google/type/decimal.proto": ["google.type.Decimal"],
+        "google/type/expr.proto": ["google.type.Expr"],
+        "google/type/fraction.proto": ["google.type.Fraction"],
+        "google/type/interval.proto": ["google.type.Interval"],
+        "google/type/latlng.proto": ["google.type.LatLng"],
+        "google/type/money.proto": ["google.type.Money"],
+        "google/type/month.proto": ["google.type.Month"],
+        "google/type/phone_number.proto": ["google.type.PhoneNumber"],
+        "google/type/postal_address.proto": ["google.type.PostalAddress"],
+        "google/type/quaternion.proto": ["google.type.Quaternion"],
+        "google/type/timeofday.proto": ["google.type.TimeOfDay"],
         "confluent/meta.proto": [".confluent.Meta"],
         "confluent/type/decimal.proto": [".confluent.type.Decimal"],
     }

karapace/protobuf/proto_file_element.py

@@ -99,8 +99,13 @@ def __eq__(self, other: "ProtoFileElement") -> bool:  # type: ignore
     def __repr__(self) -> str:
         return self.to_schema()
 
-    def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareResult:
-
+    def compare(
+        self,
+        other: "ProtoFileElement",
+        result: CompareResult,
+        self_dependency_types: Optional[List[TypeElement]] = None,
+        other_dependency_types: Optional[List[TypeElement]] = None,
+    ) -> CompareResult:
         if self.package_name != other.package_name:
             result.add_modification(Modification.PACKAGE_ALTER)
         # TODO: do we need syntax check?
@@ -125,6 +130,22 @@ def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareRe
             package_name = other.package_name or ""
             compare_types.add_other_type(package_name, type_)
 
+        # If there are dependencies declared, add the types for both.
+        if self_dependency_types:
+            for i, type_ in enumerate(self_dependency_types):
+                package_name = ""
+
+                self_types[type_.name] = type_
+                self_indexes[type_.name] = i
+                compare_types.add_self_type(package_name, type_)
+
+        if other_dependency_types:
+            for i, type_ in enumerate(other_dependency_types):
+                package_name = ""
+                other_types[type_.name] = type_
+                other_indexes[type_.name] = i
+                compare_types.add_other_type(package_name, type_)
+
         for name in chain(self_types.keys(), other_types.keys() - self_types.keys()):
 
             result.push_path(str(name), True)

karapace/protobuf/schema.py

@@ -14,8 +14,7 @@
 from karapace.protobuf.proto_parser import ProtoParser
 from karapace.protobuf.type_element import TypeElement
 from karapace.protobuf.utils import append_documentation, append_indented
-from karapace.schema_references import References
-from typing import Dict, Optional
+from typing import List, Optional
 
 
 def add_slashes(text: str) -> str:
@@ -106,15 +105,12 @@ def option_element_string(option: OptionElement) -> str:
 class ProtobufSchema:
     DEFAULT_LOCATION = Location.get("")
 
-    def __init__(
-        self, schema: str, references: Optional[References] = None, dependencies: Optional[Dict[str, Dependency]] = None
-    ) -> None:
+    def __init__(self, schema: str, dependencies: Optional[List[Dependency]] = None) -> None:
         if type(schema).__name__ != "str":
             raise IllegalArgumentException("Non str type of schema string")
         self.dirty = schema
         self.cache_string = ""
         self.proto_file_element = ProtoParser.parse(self.DEFAULT_LOCATION, schema)
-        self.references = references
         self.dependencies = dependencies
 
     def gather_deps(self) -> ProtobufDependencyVerifier:
@@ -128,10 +124,9 @@ def verify_schema_dependencies(self) -> DependencyVerifierResult:
         return verifier.verify()
 
     def collect_dependencies(self, verifier: ProtobufDependencyVerifier):
-
         if self.dependencies:
-            for key in self.dependencies:
-                self.dependencies[key].schema.schema.collect_dependencies(verifier)
+            for dependency in self.dependencies:
+                dependency.schema.schema.collect_dependencies(verifier)
         # verifier.add_import?? we have no access to own Kafka structure from this class...
         # but we need data to analyse imports to avoid ciclyc dependencies...
 
@@ -223,4 +218,21 @@ def to_schema(self) -> str:
         return "".join(strings)
 
     def compare(self, other: "ProtobufSchema", result: CompareResult) -> CompareResult:
-        self.proto_file_element.compare(other.proto_file_element, result)
+        self_dependency_types: List[TypeElement] = []
+        other_dependency_types: List[TypeElement] = []
+        if self.dependencies:
+            for dependency in self.dependencies:
+                schema = dependency.schema.schema
+                if schema.proto_file_element.types:
+                    self_dependency_types += schema.proto_file_element.types
+        if other.dependencies:
+            for dependency in other.dependencies:
+                schema = dependency.schema.schema
+                if schema.proto_file_element.types:
+                    other_dependency_types += schema.proto_file_element.types
+        self.proto_file_element.compare(
+            other.proto_file_element,
+            result,
+            self_dependency_types=self_dependency_types,
+            other_dependency_types=other_dependency_types,
+        )