feat: Add gomod git auth

docs/spec.schema.json

@@ -430,6 +430,13 @@
 					},
 					"type": "array",
 					"description": "Paths is the list of paths to run the generator on. Used to generate multi-module in a single source."
+				},
+				"auth": {
+					"additionalProperties": {
+						"$ref": "#/$defs/GomodGitAuth"
+					},
+					"type": "object",
+					"description": "Auth is the git authorization to use for gomods. The keys are the hosts, and the values are the auth to use for that host."
 				}
 			},
 			"additionalProperties": false,
@@ -448,7 +455,39 @@
 				},
 				"ssh": {
 					"type": "string",
-					"description": "SSH is the name of the secret which contains the ssh auth into when using\nssh based auth.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+					"description": "SSH is the name of the secret which contains the ssh auth info when using\nssh based auth.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				}
+			},
+			"additionalProperties": false,
+			"type": "object"
+		},
+		"GomodGitAuth": {
+			"properties": {
+				"header": {
+					"type": "string",
+					"description": "Header is the name of the secret which contains the git auth header.\nwhen using git auth header based authentication.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				},
+				"token": {
+					"type": "string",
+					"description": "Token is the name of the secret which contains a git auth token when using\ntoken based authentication.\nNote: This should not have the *actual* secret value, just the name of\nthe secret which was specified as a build secret."
+				},
+				"ssh": {
+					"$ref": "#/$defs/GomodGitAuthSSH",
+					"description": "SSH is a struct container the name of the ssh ID which contains the\naddress of the ssh auth socket, plus the username to use for the git\nremote.\nNote: This should not have the *actual* socket address, just the name of\nthe ssh ID which was specified as a build secret."
+				}
+			},
+			"additionalProperties": false,
+			"type": "object"
+		},
+		"GomodGitAuthSSH": {
+			"properties": {
+				"id": {
+					"type": "string",
+					"description": "ID is the name of the ssh socket to mount, as provided via the `--ssh`\nflag to `docker build`."
+				},
+				"username": {
+					"type": "string",
+					"description": "Username is the username to use with this particular git remote. If none\nis provided, `git` will be inserted."
 				}
 			},
 			"additionalProperties": false,

frontend/debug/handle_gomod.go

@@ -31,7 +31,7 @@ func Gomods(ctx context.Context, client gwclient.Client) (*client.Result, error)
 		worker, ok := inputs[keyGomodWorker]
 		if !ok {
 			worker = llb.Image("alpine:latest", llb.WithMetaResolver(client)).
-				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch")).Root()
+				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch openssh")).Root()
 		}
 
 		st, err := spec.GomodDeps(sOpt, worker)

generator_gomod.go

@@ -1,14 +1,20 @@
 package dalec
 
 import (
+	"bytes"
+	_ "embed"
+	"fmt"
 	"path/filepath"
+	"sort"
 
 	"github.com/moby/buildkit/client/llb"
 	"github.com/pkg/errors"
 )
 
 const (
-	gomodCacheDir = "/go/pkg/mod"
+	gomodCacheDir       = "/go/pkg/mod"
+	gitConfigMountpoint = "/dev/shm/git"
+	scriptRelativePath  = "/go_mod_download.sh"
 )
 
 func (s *Source) isGomod() bool {
@@ -32,7 +38,12 @@ func (s *Spec) HasGomods() bool {
 
 func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.ConstraintsOpt) func(llb.State) llb.State {
 	return func(in llb.State) llb.State {
-		workDir := "/work/src"
+		const (
+			fourKB           = 4096
+			workDir          = "/work/src"
+			scriptMountPoint = "/tmp/mnt"
+		)
+
 		joinedWorkDir := filepath.Join(workDir, g.Subpath)
 		srcMount := llb.AddMount(workDir, srcSt)
 
@@ -41,10 +52,17 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.Constrai
 			paths = []string{"."}
 		}
 
+		sort.Strings(paths)
+		script := g.gitconfigGeneratorScript()
+		scriptPath := filepath.Join(scriptMountPoint, scriptRelativePath)
+
 		for _, path := range paths {
 			in = worker.Run(
-				ShArgs("go mod download"),
+				ShArgs(scriptPath),
 				llb.AddEnv("GOPATH", "/go"),
+				g.withGomodSecretsAndSockets(),
+				llb.AddMount(scriptMountPoint, script),
+				llb.AddMount(gitConfigMountpoint, llb.Scratch(), llb.Tmpfs(llb.TmpfsSize(fourKB))), // to house the gitconfig, which has secrets
 				llb.Dir(filepath.Join(joinedWorkDir, path)),
 				srcMount,
 				WithConstraints(opts...),
@@ -54,6 +72,101 @@ func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.Constrai
 	}
 }
 
+func (g *SourceGenerator) gitconfigGeneratorScript() llb.State {
+	var (
+		script bytes.Buffer
+		noop   = func() {}
+
+		createPreamble = func() {
+			fmt.Fprintln(&script, `set -eu`)
+			fmt.Fprintf(&script, `ln -sf %s/.gitconfig "${HOME}/.gitconfig"`, gitConfigMountpoint)
+			script.WriteRune('\n')
+		}
+	)
+
+	fmt.Fprintln(&script, `#!/usr/bin/env sh`)
+
+	for host, auth := range g.Gomod.Auth {
+		// Only do this the first time through the loop
+		createPreamble()
+		createPreamble = noop
+
+		var headerArg string
+		if auth.Header != "" {
+			headerArg = fmt.Sprintf(`Authorization: ${%s}`, auth.Header)
+		}
+
+		if auth.Token != "" && headerArg == "" {
+			line := fmt.Sprintf(`tkn="$(echo -n "x-access-token:${%s}" | base64)"`, auth.Token)
+			fmt.Fprintln(&script, line)
+
+			headerArg = `Authorization: basic ${tkn}`
+		}
+
+		if headerArg != "" {
+			fmt.Fprintf(&script, `git config --global http."https://%s".extraheader "%s"`, host, headerArg)
+			script.WriteRune('\n')
+			continue
+		}
+
+		username := "git"
+		if auth.SSH != nil {
+			if auth.SSH.Username != "" {
+				username = auth.SSH.Username
+			}
+
+			fmt.Fprintf(&script, `git config --global "url.ssh://%[1]s@%[2]s/.insteadOf" https://%[2]s/`, username, host)
+			script.WriteRune('\n')
+		}
+	}
+
+	fmt.Fprintln(&script, "go mod download")
+	return llb.Scratch().File(llb.Mkfile(scriptRelativePath, 0o755, script.Bytes()))
+}
+
+func (g *SourceGenerator) withGomodSecretsAndSockets() llb.RunOption {
+	return runOptionFunc(func(ei *llb.ExecInfo) {
+		if g.Gomod == nil {
+			return
+		}
+
+		var (
+			setenv = func() {
+				llb.AddEnv(
+					"GIT_SSH_COMMAND",
+					`ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no`,
+				).SetRunOption(ei)
+			}
+
+			noop = func() {}
+		)
+
+		secrets := make(map[string]struct{}, len(g.Gomod.Auth))
+		for _, auth := range g.Gomod.Auth {
+			if auth.Token != "" {
+				secrets[auth.Token] = struct{}{}
+				continue
+			}
+
+			if auth.Header != "" {
+				secrets[auth.Header] = struct{}{}
+				continue
+			}
+
+			if auth.SSH != nil && auth.SSH.ID != "" {
+				llb.AddSSHSocket(llb.SSHID(auth.SSH.ID)).SetRunOption(ei)
+
+				setenv()
+				setenv = noop
+			}
+		}
+
+		for secret := range secrets {
+			secretToEnv(secret).SetRunOption(ei)
+		}
+	})
+}
+
 func (s *Spec) gomodSources() map[string]Source {
 	sources := map[string]Source{}
 	for name, src := range s.Sources {

load_test.go

@@ -385,6 +385,86 @@ func TestSourceFillDefaults(t *testing.T) {
 				Path: ".",
 			},
 		},
+		{
+			title: "auto-propagate git auth to gomod auth",
+			before: Source{
+				Git: &SourceGit{
+					URL:    "https://github.com/auth/test.git",
+					Commit: t.Name(),
+					Auth: GitAuth{
+						Token: "DALEC_GIT_AUTH_TOKEN_GITHUB",
+					},
+				},
+				Generate: []*SourceGenerator{
+					{
+						Gomod: &GeneratorGomod{},
+					},
+				},
+			},
+			after: Source{
+				Git: &SourceGit{
+					URL:    "https://github.com/auth/test.git",
+					Commit: t.Name(),
+					Auth: GitAuth{
+						Token: "DALEC_GIT_AUTH_TOKEN_GITHUB",
+					},
+				},
+				Generate: []*SourceGenerator{
+					{
+						Gomod: &GeneratorGomod{
+							Auth: map[string]GomodGitAuth{
+								"github.com": {
+									Token: "DALEC_GIT_AUTH_TOKEN_GITHUB",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			title: "don't overwrite existing git auth when auto-propagating",
+			before: Source{
+				Git: &SourceGit{
+					URL:    "https://github.com/auth/test.git",
+					Commit: t.Name(),
+					Auth: GitAuth{
+						Token: "DALEC_GIT_AUTH_TOKEN_GITHUB",
+					},
+				},
+				Generate: []*SourceGenerator{
+					{
+						Gomod: &GeneratorGomod{
+							Auth: map[string]GomodGitAuth{
+								"github.com": {
+									Token: "ANOTHER_DALEC_GIT_AUTH_TOKEN_GITHUB",
+								},
+							},
+						},
+					},
+				},
+			},
+			after: Source{
+				Git: &SourceGit{
+					URL:    "https://github.com/auth/test.git",
+					Commit: t.Name(),
+					Auth: GitAuth{
+						Token: "DALEC_GIT_AUTH_TOKEN_GITHUB",
+					},
+				},
+				Generate: []*SourceGenerator{
+					{
+						Gomod: &GeneratorGomod{
+							Auth: map[string]GomodGitAuth{
+								"github.com": {
+									Token: "ANOTHER_DALEC_GIT_AUTH_TOKEN_GITHUB",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range cases {