codegen: enable CRD schema generation using protoc-gen-openapi

Allows generating CRDs schema using protoc-gen-openapi instead
of Cue.

This is a part of a larger effort to customize schema generation
that is harder to accomplish using Cue:
solo-io/gloo-mesh-enterprise#16049

Signed-off-by: Shashank Ram <[email protected]>

changelog/v0.36.6/protoc-gen.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink:  https://github.com/solo-io/gloo-mesh-enterprise/issues/16049
+    resolvesIssue: false
+    description: >
+      "Enable CRD schema generation using protoc-gen-openapi"
+    skipCI: "false"

ci/tools.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
 	_ "github.com/solo-io/protoc-gen-ext"
+	_ "github.com/solo-io/protoc-gen-openapi"
 	_ "golang.org/x/tools/cmd/goimports"
 	_ "k8s.io/code-generator"
 )

codegen/collector/executor.go

@@ -0,0 +1,126 @@
+package collector
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/rotisserie/eris"
+)
+
+type ProtocExecutor interface {
+	Execute(protoFile string, toFile string, imports []string) error
+}
+
+type DefaultProtocExecutor struct {
+	// The output directory
+	OutputDir string
+	// whether or not to do a regular go-proto generate while collecting descriptors
+	ShouldCompileFile func(string) bool
+	// arguments for go_out=
+	CustomGoArgs []string
+	// custom plugins
+	// each will append a <plugin>_out= directive to protoc command
+	CustomPlugins []string
+}
+
+var defaultGoArgs = []string{
+	"plugins=grpc",
+}
+
+func (d *DefaultProtocExecutor) Execute(protoFile string, toFile string, imports []string) error {
+	cmd := exec.Command("protoc")
+
+	for _, i := range imports {
+		cmd.Args = append(cmd.Args, fmt.Sprintf("-I%s", i))
+	}
+
+	if d.ShouldCompileFile(protoFile) {
+		goArgs := append(defaultGoArgs, d.CustomGoArgs...)
+		goArgsJoined := strings.Join(goArgs, ",")
+		cmd.Args = append(cmd.Args,
+			fmt.Sprintf("--go_out=%s:%s", goArgsJoined, d.OutputDir),
+			fmt.Sprintf("--ext_out=%s:%s", goArgsJoined, d.OutputDir),
+		)
+
+		for _, pluginName := range d.CustomPlugins {
+			cmd.Args = append(cmd.Args,
+				fmt.Sprintf("--%s_out=%s:%s", pluginName, goArgsJoined, d.OutputDir),
+			)
+		}
+	}
+
+	cmd.Args = append(cmd.Args,
+		"-o",
+		toFile,
+		"--include_imports",
+		"--include_source_info",
+		protoFile)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return eris.Wrapf(err, "%v failed: %s", cmd.Args, out)
+	}
+	return nil
+}
+
+type OpenApiProtocExecutor struct {
+	OutputDir string
+
+	// Whether to include descriptions in validation schemas
+	IncludeDescriptionsInSchema bool
+
+	// Whether to assign Enum fields the `x-kubernetes-int-or-string` property
+	// which allows the value to either be an integer or a string
+	EnumAsIntOrString bool
+
+	// A list of messages (core.solo.io.Status) whose validation schema should
+	// not be generated
+	MessagesWithEmptySchema []string
+}
+
+func (o *OpenApiProtocExecutor) Execute(protoFile string, toFile string, imports []string) error {
+	cmd := exec.Command("protoc")
+
+	for _, i := range imports {
+		cmd.Args = append(cmd.Args, fmt.Sprintf("-I%s", i))
+	}
+
+	// The way that --openapi_out works, is that it produces a file in an output directory,
+	// with the name of the file matching the proto package (ie gloo.solo.io).
+	// Therefore, if you have multiple protos in a single package, they will all be output
+	// to the same file, and overwrite one another.
+	// To avoid this, we generate a directory with the name of the proto file.
+	// For example my_resource.proto in the gloo.solo.io package will produce the following file:
+	//  my_resource/gloo.solo.io.yaml
+
+	// The directoryName is created by taking the name of the file and removing the extension
+	_, fileName := filepath.Split(protoFile)
+	directoryName := fileName[0 : len(fileName)-len(filepath.Ext(fileName))]
+
+	// Create the directory
+	directoryPath := filepath.Join(o.OutputDir, directoryName)
+	_ = os.Mkdir(directoryPath, os.ModePerm)
+
+	cmd.Args = append(cmd.Args,
+		fmt.Sprintf("--openapi_out=yaml=true,single_file=false,include_description=true,multiline_description=true,enum_as_int_or_string=%v,proto_oneof=true,int_native=true,additional_empty_schema=%v:%s",
+			o.EnumAsIntOrString,
+			strings.Join(o.MessagesWithEmptySchema, "+"),
+			directoryPath),
+	)
+
+	cmd.Args = append(cmd.Args,
+		"-o",
+		toFile,
+		"--include_imports",
+		"--include_source_info",
+		protoFile)
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return eris.Wrapf(err, "%v failed: %s", cmd.Args, out)
+	}
+	return nil
+}

codegen/model/resource.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/golang/protobuf/proto"
 	"github.com/solo-io/skv2/codegen/collector"
+	"github.com/solo-io/skv2/codegen/proto/schemagen"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -130,6 +131,11 @@ type Group struct {
 type GroupOptions struct {
 	// Required when using crds in the templates directory
 	EscapeGoTemplateOperators bool
+
+	// Options for generating validation schemas
+	SchemaValidationOpts schemagen.ValidationSchemaOptions
+
+	SchemaGenerator schemagen.GeneratorKind
 }
 
 func (g Group) HasProtos() bool {

codegen/proto/schemagen/generator.go

@@ -0,0 +1,44 @@
+package schemagen
+
+type GeneratorKind string
+
+const (
+	Cue              GeneratorKind = "cue"
+	ProtocGenOpenAPI GeneratorKind = "protoc-gen-openapi"
+)
+
+type ValidationSchemaOptions struct {
+	// Whether to assign Enum fields the `x-kubernetes-int-or-string` property
+	// which allows the value to either be an integer or a string
+	// If this is false, only string values are allowed
+	// Default: false
+	EnumAsIntOrString bool
+
+	// A list of messages (e.g. ratelimit.api.solo.io.Descriptor) whose validation schema should
+	// not be generated
+	MessagesWithEmptySchema []string
+}
+
+// prevent k8s from validating proto.Any fields (since it's unstructured)
+func removeProtoAnyValidation(d map[string]interface{}, propertyField string) {
+	for _, v := range d {
+		values, ok := v.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		desc, ok := values["properties"]
+		properties, isObj := desc.(map[string]interface{})
+		// detect proto.Any field from presence of [propertyField] as field under "properties"
+		if !ok || !isObj || properties[propertyField] == nil {
+			removeProtoAnyValidation(values, propertyField)
+			continue
+		}
+		// remove "properties" value
+		delete(values, "properties")
+		// remove "required" value
+		delete(values, "required")
+		// x-kubernetes-preserve-unknown-fields allows for unknown fields from a particular node
+		// see https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#specifying-a-structural-schema
+		values["x-kubernetes-preserve-unknown-fields"] = true
+	}
+}

codegen/proto/schemagen/protoc.go

@@ -0,0 +1,178 @@
+package schemagen
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+	"github.com/rotisserie/eris"
+	"github.com/solo-io/go-utils/log"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/solo-io/skv2/codegen/collector"
+)
+
+// Implementation of JsonSchemaGenerator that uses a plugin for the protocol buffer compiler
+type protocGenerator struct {
+	validationSchemaOptions *ValidationSchemaOptions
+}
+
+func NewProtocGenerator(validationSchemaOptions *ValidationSchemaOptions) *protocGenerator {
+	return &protocGenerator{
+		validationSchemaOptions: validationSchemaOptions,
+	}
+}
+
+func (p *protocGenerator) GetJSONSchemas(protoFiles []string, imports []string, gv schema.GroupVersion) (map[schema.GroupVersionKind]*apiextv1.JSONSchemaProps, error) {
+	// Use a tmp directory as the output of schemas
+	// The schemas will then be matched with the appropriate CRD
+	tmpOutputDir, err := os.MkdirTemp("", "skv2-schema-gen-")
+	if err != nil {
+		return nil, err
+	}
+	_ = os.MkdirAll(tmpOutputDir, os.ModePerm)
+	defer os.Remove(tmpOutputDir)
+
+	// The Executor used to compile protos
+	protocExecutor := &collector.OpenApiProtocExecutor{
+		OutputDir:               tmpOutputDir,
+		EnumAsIntOrString:       p.validationSchemaOptions.EnumAsIntOrString,
+		MessagesWithEmptySchema: p.validationSchemaOptions.MessagesWithEmptySchema,
+	}
+
+	// 1. Generate the openApiSchemas for the project, writing them to a temp directory (schemaOutputDir)
+	for _, f := range protoFiles {
+		if err := p.generateSchemasForProjectProto(protocExecutor, f, imports); err != nil {
+			return nil, err
+		}
+	}
+
+	// 2. Walk the schemaOutputDir and convert the open api schemas into JSONSchemaProps
+	return p.processGeneratedSchemas(gv, tmpOutputDir)
+}
+
+func (p *protocGenerator) generateSchemasForProjectProto(
+	protocExecutor collector.ProtocExecutor,
+	projectProtoFile string,
+	imports []string,
+) error {
+	log.Printf("Generating schema for proto file: %s", projectProtoFile)
+
+	// we don't use the output of protoc so use a temp file
+	tmpFile, err := os.CreateTemp("", "sv2-schema-gen-")
+	if err != nil {
+		return err
+	}
+	if err := tmpFile.Close(); err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name())
+
+	return protocExecutor.Execute(projectProtoFile, tmpFile.Name(), imports)
+}
+
+func (p *protocGenerator) processGeneratedSchemas(gv schema.GroupVersion, schemaOutputDir string) (map[schema.GroupVersionKind]*apiextv1.JSONSchemaProps, error) {
+	jsonSchemasByGVK := make(map[schema.GroupVersionKind]*apiextv1.JSONSchemaProps)
+	err := filepath.Walk(schemaOutputDir, func(schemaFile string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		if !strings.HasSuffix(schemaFile, ".yaml") {
+			return nil
+		}
+
+		log.Debugf("Generated Schema File: %s", schemaFile)
+		doc, err := p.readOpenApiDocumentFromFile(schemaFile)
+		if err != nil {
+			// Stop traversing the output directory
+			return err
+		}
+
+		schemas := doc.Components.Schemas
+		if schemas == nil {
+			// Continue traversing the output directory
+			return nil
+		}
+
+		for schemaKey, schemaValue := range schemas {
+			schemaGVK := p.getGVKForSchemaKey(gv, schemaKey)
+
+			// Spec validation schema
+			specJsonSchema, err := p.getJsonSchema(schemaKey, schemaValue)
+			if err != nil {
+				return err
+			}
+
+			jsonSchemasByGVK[schemaGVK] = specJsonSchema
+		}
+		// Continue traversing the output directory
+		return nil
+	})
+
+	return jsonSchemasByGVK, err
+}
+
+func (p *protocGenerator) readOpenApiDocumentFromFile(file string) (*openapi3.T, error) {
+	var openApiDocument *openapi3.T
+	bytes, err := os.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrapf(err, "reading file")
+	}
+	if err := yaml.Unmarshal(bytes, &openApiDocument); err != nil {
+		return nil, errors.Wrapf(err, "unmarshalling tmp file as schemas")
+	}
+	return openApiDocument, nil
+}
+
+func (p *protocGenerator) getGVKForSchemaKey(gv schema.GroupVersion, schemaKey string) schema.GroupVersionKind {
+	// The generated keys look like testing.solo.io.MockResource
+	// The kind is the `MockResource` portion
+	ss := strings.Split(schemaKey, ".")
+	kind := ss[len(ss)-1]
+
+	return schema.GroupVersionKind{
+		Group:   gv.Group,
+		Version: gv.Version,
+		Kind:    kind,
+	}
+}
+
+func (p *protocGenerator) getJsonSchema(schemaKey string, schema *openapi3.SchemaRef) (*apiextv1.JSONSchemaProps, error) {
+	if schema == nil {
+		return nil, eris.Errorf("no open api schema for %s", schemaKey)
+	}
+
+	oApiJson, err := schema.MarshalJSON()
+	if err != nil {
+		return nil, eris.Errorf("Cannot marshal OpenAPI schema for %v: %v", schemaKey, err)
+	}
+
+	var obj map[string]interface{}
+	if err = json.Unmarshal(oApiJson, &obj); err != nil {
+		return nil, err
+	}
+
+	// detect proto.Any field from presence of "typeUrl" as field under "properties"
+	removeProtoAnyValidation(obj, "typeUrl")
+
+	bytes, err := json.Marshal(obj)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonSchema := &apiextv1.JSONSchemaProps{}
+	if err = json.Unmarshal(bytes, jsonSchema); err != nil {
+		return nil, eris.Errorf("Cannot unmarshal raw OpenAPI schema to JSONSchemaProps for %v: %v", schemaKey, err)
+	}
+
+	return jsonSchema, nil
+}