Merge branch 'main' into ed/pluginPushCmd

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 - Add `buf registry plugin {create,delete,info,update}` commands to manage BSR plugins.
 - Breaking analysis support for `buf beta lsp`.
+- Fix bug when using the `--type` flag filter for `buf build` where import ordering is not
+  determinisitic.
 - Add `buf plugin push` command to push a plugin to the Buf Schema Registry. 
   Only WebAssembly check plugins are supported at this time.
 

go.mod

@@ -7,8 +7,8 @@ toolchain go1.23.3
 require (
 	buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.35.2-20241031151143-70f632351282.1
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.2-20240920164238-5a7b106cbb87.1
-	buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241025140216-aa40f2c93090.1
-	buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241025140216-aa40f2c93090.1
+	buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241125212318-4a305dc3b757.1
+	buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241125212318-4a305dc3b757.1
 	buf.build/go/bufplugin v0.6.0
 	buf.build/go/protoyaml v0.2.0
 	buf.build/go/spdx v0.2.0
@@ -30,12 +30,12 @@ require (
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
 	github.com/pkg/profile v1.7.0
-	github.com/quic-go/quic-go v0.48.1
+	github.com/quic-go/quic-go v0.48.2
 	github.com/rs/cors v1.11.1
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.10.0
-	github.com/tetratelabs/wazero v1.8.1
+	github.com/tetratelabs/wazero v1.8.2
 	go.lsp.dev/jsonrpc2 v0.10.0
 	go.lsp.dev/protocol v0.12.0
 	go.uber.org/zap v1.27.0

go.sum

@@ -2,10 +2,10 @@ buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.35.2-20241031151143-70
 buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.35.2-20241031151143-70f632351282.1/go.mod h1:vKDy7lD1bsN2UjeLhqklPEjIsHfHAPgMb/PbRx2EFDc=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.2-20240920164238-5a7b106cbb87.1 h1:7QIeAuTdLp173vC/9JojRMDFcpmqtoYrxPmvdHAOynw=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.35.2-20240920164238-5a7b106cbb87.1/go.mod h1:mnHCFccv4HwuIAOHNGdiIc5ZYbBCvbTWZcodLN5wITI=
-buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241025140216-aa40f2c93090.1 h1:FHQXg3T7S2jp8yc7/bQJgqEH1yza/rrDHXITUK2Tm0g=
-buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241025140216-aa40f2c93090.1/go.mod h1:5iwF5l+9lKCnvr1zLvDgUHrv6X+vU5nNPjvig1sbnao=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241025140216-aa40f2c93090.1 h1:a1OY15kRBrD1AF0SYIPYFLE8GCC1d85zqENnzUC3lNM=
-buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241025140216-aa40f2c93090.1/go.mod h1:EQCcR04Wp6ffVPfxNb4ZXAVJXrZJopDNKQWp37BDCgU=
+buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241125212318-4a305dc3b757.1 h1:q3PKZJfOuSZR8KC0asyo7EFID/3613pDFzFYAAO+Gd4=
+buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20241125212318-4a305dc3b757.1/go.mod h1:w+XYJEgdkHK5XJ+44Eq2YD02DZVLFNFjfwFKPz0nZcg=
+buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241125212318-4a305dc3b757.1 h1:m2QVURqXvfedovCeWV5CsWm3VIrD87RHL+9P9o7HB84=
+buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.35.2-20241125212318-4a305dc3b757.1/go.mod h1:EQCcR04Wp6ffVPfxNb4ZXAVJXrZJopDNKQWp37BDCgU=
 buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.35.2-20241007202033-cf42259fcbfc.1 h1:FcoYwX9eJhc73MdVlqyJjMOQ863akpHK0VEQ/+Zkt9U=
 buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.35.2-20241007202033-cf42259fcbfc.1/go.mod h1:uTCf/J5B6H9XCTgHuI91LC9qaNqxJxQFh0kDY/GLn2k=
 buf.build/go/bufplugin v0.6.0 h1:3lhoh+0z+IUPS3ZajTPn/27LaLIkero2BDVnV7yXD1s=
@@ -229,8 +229,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
 github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA=
-github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
+github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE=
+github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
@@ -258,8 +258,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
-github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
+github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
+github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
 github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs=
 github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

private/bufpkg/bufimage/bufimageutil/bufimageutil.go

@@ -266,16 +266,20 @@ func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts
 		// the file's WeakDependency field.
 		indexFromTo := make(map[int32]int32)
 		indexTo := 0
+		// Only handle imports and dependencies if there are any.
 		for indexFrom, importPath := range imageFileDescriptor.GetDependency() {
 			path := append(basePath, int32(indexFrom))
-			if _, ok := importsRequired[importPath]; ok {
+			// We check if the import path exists among required imports. If yes, we
+			// move and then delete from required imports as we go.
+			if importsRequired != nil && importsRequired.index(importPath) != -1 {
 				sourcePathRemapper.markMoved(path, int32(indexTo))
 				indexFromTo[int32(indexFrom)] = int32(indexTo)
 				imageFileDescriptor.Dependency[indexTo] = importPath
 				indexTo++
-				// markDeleted them as we go, so we know which ones weren't in the list
-				delete(importsRequired, importPath)
+				// delete them as we go, so we know which ones weren't in the list
+				importsRequired.delete(importPath)
 			} else {
+				// Path did not exist in required imports, we mark as deleted.
 				sourcePathRemapper.markDeleted(path)
 			}
 		}
@@ -284,9 +288,12 @@ func ImageFilteredByTypesWithOptions(image bufimage.Image, types []string, opts
 		// Add any other imports (which may not have been in the list because
 		// they were picked up via a public import). The filtered files will not
 		// use public imports.
-		for importPath := range importsRequired {
-			imageFileDescriptor.Dependency = append(imageFileDescriptor.Dependency, importPath)
+		// The imports are added in the order they are encountered when importing
+		// to maintain a deterministic ordering.
+		if importsRequired != nil {
+			imageFileDescriptor.Dependency = append(imageFileDescriptor.Dependency, importsRequired.keys()...)
 		}
+
 		imageFileDescriptor.PublicDependency = nil
 		sourcePathRemapper.markDeleted([]int32{filePublicDependencyTag})
 
@@ -458,9 +465,9 @@ type transitiveClosure struct {
 	// entire package being included). The above fields are used to filter the
 	// contents of files. But files named in this set will not be filtered.
 	completeFiles map[string]struct{}
-	// The set of imports for each file. This allows for re-writing imports
+	// The ordered set of imports for each file. This allows for re-writing imports
 	// for files whose contents have been pruned.
-	imports map[string]map[string]struct{}
+	imports map[string]*orderedImports
 }
 
 type closureInclusionMode int
@@ -485,7 +492,7 @@ func newTransitiveClosure() *transitiveClosure {
 		elements:      map[namedDescriptor]closureInclusionMode{},
 		files:         map[string]struct{}{},
 		completeFiles: map[string]struct{}{},
-		imports:       map[string]map[string]struct{}{},
+		imports:       map[string]*orderedImports{},
 	}
 }
 
@@ -495,10 +502,10 @@ func (t *transitiveClosure) addImport(fromPath, toPath string) {
 	}
 	imps := t.imports[fromPath]
 	if imps == nil {
-		imps = map[string]struct{}{}
+		imps = newOrderedImports()
 		t.imports[fromPath] = imps
 	}
-	imps[toPath] = struct{}{}
+	imps.add(toPath)
 }
 
 func (t *transitiveClosure) addFile(file string, imageIndex *imageIndex, opts *imageFilterOptions) error {
@@ -1000,3 +1007,55 @@ func stripSourceRetentionOptionsFromFile(imageFile bufimage.ImageFile) (bufimage
 		imageFile.UnusedDependencyIndexes(),
 	)
 }
+
+// orderedImports is a structure to maintain an ordered set of imports. This is needed
+// because we want to be able to iterate through imports in a deterministic way when filtering
+// the image.
+type orderedImports struct {
+	pathToIndex map[string]int
+	paths       []string
+}
+
+// newOrderedImports creates a new orderedImports structure.
+func newOrderedImports() *orderedImports {
+	return &orderedImports{
+		pathToIndex: map[string]int{},
+	}
+}
+
+// index returns the index for a given path. If the path does not exist in index map, -1
+// is returned and should be considered deleted.
+func (o *orderedImports) index(path string) int {
+	if index, ok := o.pathToIndex[path]; ok {
+		return index
+	}
+	return -1
+}
+
+// add appends a path to the paths list and the index in the map. If a key already exists,
+// then this is a no-op.
+func (o *orderedImports) add(path string) {
+	if _, ok := o.pathToIndex[path]; !ok {
+		o.pathToIndex[path] = len(o.paths)
+		o.paths = append(o.paths, path)
+	}
+}
+
+// delete removes a key from the index map of ordered imports. If a non-existent path is
+// set for deletion, then this is a no-op.
+// Note that the path is not removed from the paths list. If you want to iterate through
+// the paths, use keys() to get all non-deleted keys.
+func (o *orderedImports) delete(path string) {
+	delete(o.pathToIndex, path)
+}
+
+// keys provides all non-deleted keys from the ordered imports.
+func (o *orderedImports) keys() []string {
+	keys := make([]string, 0, len(o.pathToIndex))
+	for _, path := range o.paths {
+		if _, ok := o.pathToIndex[path]; ok {
+			keys = append(keys, path)
+		}
+	}
+	return keys
+}

private/bufpkg/bufregistryapi/bufregistryapiplugin/bufregistryapiplugin.go

@@ -30,6 +30,8 @@ var (
 	NopV1Beta1LabelServiceClientProvider V1Beta1LabelServiceClientProvider = nopClientProvider{}
 	// NopV1Beta1PluginServiceClientProvider is a V1Beta1PluginServiceClientProvider that provides unimplemented services for testing.
 	NopV1Beta1PluginServiceClientProvider V1Beta1PluginServiceClientProvider = nopClientProvider{}
+	// NopV1Beta1ResourceServiceClientProvider is a V1Beta1ResourceServiceClientProvider that provides unimplemented services for testing.
+	NopV1Beta1ResourceServiceClientProvider V1Beta1ResourceServiceClientProvider = nopClientProvider{}
 	// NopV1Beta1UploadServiceClientProvider is a V1Beta1UploadServiceClientProvider that provides unimplemented services for testing.
 	NopV1Beta1UploadServiceClientProvider V1Beta1UploadServiceClientProvider = nopClientProvider{}
 	// NopClientProvider is a ClientProvider that provides unimplemented services for testing.
@@ -56,6 +58,11 @@ type V1Beta1LabelServiceClientProvider interface {
 	V1Beta1LabelServiceClient(registry string) pluginv1beta1connect.LabelServiceClient
 }
 
+// V1Beta1ResourceServiceClientProvider provides ResourceServiceClients.
+type V1Beta1ResourceServiceClientProvider interface {
+	V1Beta1ResourceServiceClient(registry string) pluginv1beta1connect.ResourceServiceClient
+}
+
 // V1Beta1PluginServiceClientProvider provides PluginServiceClients.
 type V1Beta1PluginServiceClientProvider interface {
 	V1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient
@@ -72,6 +79,7 @@ type ClientProvider interface {
 	V1Beta1CommitServiceClientProvider
 	V1Beta1DownloadServiceClientProvider
 	V1Beta1LabelServiceClientProvider
+	V1Beta1ResourceServiceClientProvider
 	V1Beta1PluginServiceClientProvider
 	V1Beta1UploadServiceClientProvider
 }
@@ -125,6 +133,14 @@ func (c *clientProvider) V1Beta1LabelServiceClient(registry string) pluginv1beta
 	)
 }
 
+func (c *clientProvider) V1Beta1ResourceServiceClient(registry string) pluginv1beta1connect.ResourceServiceClient {
+	return connectclient.Make(
+		c.clientConfig,
+		registry,
+		pluginv1beta1connect.NewResourceServiceClient,
+	)
+}
+
 func (c *clientProvider) V1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient {
 	return connectclient.Make(
 		c.clientConfig,
@@ -159,6 +175,10 @@ func (nopClientProvider) V1Beta1LabelServiceClient(registry string) pluginv1beta
 	return pluginv1beta1connect.UnimplementedLabelServiceHandler{}
 }
 
+func (nopClientProvider) V1Beta1ResourceServiceClient(registry string) pluginv1beta1connect.ResourceServiceClient {
+	return pluginv1beta1connect.UnimplementedResourceServiceHandler{}
+}
+
 func (nopClientProvider) V1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient {
 	return pluginv1beta1connect.UnimplementedPluginServiceHandler{}
 }