Merge branch 'instaclustr-pr2_clear' into protobuf-schema-references-…

…support

.gitignore

@@ -16,3 +16,4 @@ __pycache__/
 /kafka_*/
 venv
 /karapace/version.py
+.run

karapace/dependency.py

@@ -0,0 +1,21 @@
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from karapace.schema_models import ValidatedTypedSchema
+
+
+class Dependency:
+    def __init__(self, name: str, subject: str, version: int, schema: "ValidatedTypedSchema") -> None:
+        self.name = name
+        self.subject = subject
+        self.version = version
+        self.schema = schema
+
+    def identifier(self) -> str:
+        return self.name + "_" + self.subject + "_" + str(self.version)
+
+
+class DependencyVerifierResult:
+    def __init__(self, result: bool, message: Optional[str] = ""):
+        self.result = result
+        self.message = message

karapace/errors.py

@@ -1,3 +1,6 @@
+from typing import List, Union
+
+
 class VersionNotFoundException(Exception):
     pass
 
@@ -18,6 +21,14 @@ class InvalidSchemaType(Exception):
     pass
 
 
+class InvalidReferences(Exception):
+    pass
+
+
+class ReferencesNotSupportedException(Exception):
+    pass
+
+
 class SchemasNotFoundException(Exception):
     pass
 
@@ -38,6 +49,13 @@ class SubjectNotSoftDeletedException(Exception):
     pass
 
 
+class ReferenceExistsException(Exception):
+    def __init__(self, referenced_by: List, version: Union[int, str]):
+        super().__init__()
+        self.version = version
+        self.referenced_by = referenced_by
+
+
 class SubjectSoftDeletedException(Exception):
     pass
 

karapace/protobuf/dependency.py

@@ -0,0 +1,56 @@
+from karapace.dependency import DependencyVerifierResult
+from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency
+from karapace.protobuf.one_of_element import OneOfElement
+from typing import List
+
+
+class ProtobufDependencyVerifier:
+    def __init__(self) -> None:
+        self.declared_types: List[str] = []
+        self.used_types: List[str] = []
+        self.import_path: List[str] = []
+
+    def add_declared_type(self, full_name: str) -> None:
+        self.declared_types.append(full_name)
+
+    def add_used_type(self, parent: str, element_type: str) -> None:
+        if element_type.find("map<") == 0:
+            end = element_type.find(">")
+            virgule = element_type.find(",")
+            key = element_type[4:virgule]
+            value = element_type[virgule + 1 : end]
+            value = value.strip()
+            self.used_types.append(parent + ";" + key)
+            self.used_types.append(parent + ";" + value)
+        else:
+            self.used_types.append(parent + ";" + element_type)
+
+    def add_import(self, import_name: str) -> None:
+        self.import_path.append(import_name)
+
+    def verify(self) -> DependencyVerifierResult:
+        declared_index = set(self.declared_types)
+        for used_type in self.used_types:
+            delimiter = used_type.rfind(";")
+            used_type_with_scope = ""
+            if delimiter != -1:
+                used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :]
+                used_type = used_type[delimiter + 1 :]
+
+            if not (
+                used_type in DependenciesHardcoded.index
+                or KnownDependency.index_simple.get(used_type) is not None
+                or KnownDependency.index.get(used_type) is not None
+                or used_type in declared_index
+                or (delimiter != -1 and used_type_with_scope in declared_index)
+                or "." + used_type in declared_index
+            ):
+                return DependencyVerifierResult(False, f"type {used_type} is not defined")
+
+        return DependencyVerifierResult(True)
+
+
+def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None:
+    parent = package_name + "." + parent_name
+    for field in one_of.fields:
+        verifier.add_used_type(parent, field.element_type)

karapace/protobuf/exception.py

@@ -25,6 +25,10 @@ class ProtobufTypeException(Error):
     """Generic Protobuf type error."""
 
 
+class ProtobufUnresolvedDependencyException(ProtobufException):
+    """a Protobuf schema has unresolved dependency"""
+
+
 class SchemaParseException(ProtobufException):
     """Error while parsing a Protobuf schema descriptor."""
 

karapace/protobuf/google/protobuf/any.proto

@@ -0,0 +1,158 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package google.protobuf;
+
+option csharp_namespace = "Google.Protobuf.WellKnownTypes";
+option go_package = "google.golang.org/protobuf/types/known/anypb";
+option java_package = "com.google.protobuf";
+option java_outer_classname = "AnyProto";
+option java_multiple_files = true;
+option objc_class_prefix = "GPB";
+
+// `Any` contains an arbitrary serialized protocol buffer message along with a
+// URL that describes the type of the serialized message.
+//
+// Protobuf library provides support to pack/unpack Any values in the form
+// of utility functions or additional generated methods of the Any type.
+//
+// Example 1: Pack and unpack a message in C++.
+//
+//     Foo foo = ...;
+//     Any any;
+//     any.PackFrom(foo);
+//     ...
+//     if (any.UnpackTo(&foo)) {
+//       ...
+//     }
+//
+// Example 2: Pack and unpack a message in Java.
+//
+//     Foo foo = ...;
+//     Any any = Any.pack(foo);
+//     ...
+//     if (any.is(Foo.class)) {
+//       foo = any.unpack(Foo.class);
+//     }
+//
+// Example 3: Pack and unpack a message in Python.
+//
+//     foo = Foo(...)
+//     any = Any()
+//     any.Pack(foo)
+//     ...
+//     if any.Is(Foo.DESCRIPTOR):
+//       any.Unpack(foo)
+//       ...
+//
+// Example 4: Pack and unpack a message in Go
+//
+//      foo := &pb.Foo{...}
+//      any, err := anypb.New(foo)
+//      if err != nil {
+//        ...
+//      }
+//      ...
+//      foo := &pb.Foo{}
+//      if err := any.UnmarshalTo(foo); err != nil {
+//        ...
+//      }
+//
+// The pack methods provided by protobuf library will by default use
+// 'type.googleapis.com/full.type.name' as the type URL and the unpack
+// methods only use the fully qualified type name after the last '/'
+// in the type URL, for example "foo.bar.com/x/y.z" will yield type
+// name "y.z".
+//
+//
+// JSON
+//
+// The JSON representation of an `Any` value uses the regular
+// representation of the deserialized, embedded message, with an
+// additional field `@type` which contains the type URL. Example:
+//
+//     package google.profile;
+//     message Person {
+//       string first_name = 1;
+//       string last_name = 2;
+//     }
+//
+//     {
+//       "@type": "type.googleapis.com/google.profile.Person",
+//       "firstName": <string>,
+//       "lastName": <string>
+//     }
+//
+// If the embedded message type is well-known and has a custom JSON
+// representation, that representation will be embedded adding a field
+// `value` which holds the custom JSON in addition to the `@type`
+// field. Example (for message [google.protobuf.Duration][]):
+//
+//     {
+//       "@type": "type.googleapis.com/google.protobuf.Duration",
+//       "value": "1.212s"
+//     }
+//
+message Any {
+  // A URL/resource name that uniquely identifies the type of the serialized
+  // protocol buffer message. This string must contain at least
+  // one "/" character. The last segment of the URL's path must represent
+  // the fully qualified name of the type (as in
+  // `path/google.protobuf.Duration`). The name should be in a canonical form
+  // (e.g., leading "." is not accepted).
+  //
+  // In practice, teams usually precompile into the binary all types that they
+  // expect it to use in the context of Any. However, for URLs which use the
+  // scheme `http`, `https`, or no scheme, one can optionally set up a type
+  // server that maps type URLs to message definitions as follows:
+  //
+  // * If no scheme is provided, `https` is assumed.
+  // * An HTTP GET on the URL must yield a [google.protobuf.Type][]
+  //   value in binary format, or produce an error.
+  // * Applications are allowed to cache lookup results based on the
+  //   URL, or have them precompiled into a binary to avoid any
+  //   lookup. Therefore, binary compatibility needs to be preserved
+  //   on changes to types. (Use versioned type names to manage
+  //   breaking changes.)
+  //
+  // Note: this functionality is not currently available in the official
+  // protobuf release, and it is not used for type URLs beginning with
+  // type.googleapis.com.
+  //
+  // Schemes other than `http`, `https` (or the empty scheme) might be
+  // used with implementation specific semantics.
+  //
+  string type_url = 1;
+
+  // Must be a valid serialized protocol buffer of the above specified type.
+  bytes value = 2;
+}

karapace/protobuf/google/protobuf/any_test.proto

@@ -0,0 +1,44 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package protobuf_unittest;
+
+import "google/protobuf/any.proto";
+
+option java_outer_classname = "TestAnyProto";
+
+message TestAny {
+  int32 int32_value = 1;
+  google.protobuf.Any any_value = 2;
+  repeated google.protobuf.Any repeated_any_value = 3;
+  string text = 4;
+}