diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..2c57cdac --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +dotenv_if_exists diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..b1683d12 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,16 @@ +run: + tests: false + +linters: + enable: + - bodyclose + - durationcheck + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - unparam + - unused + - usestdlibvars + - usetesting diff --git a/.tool-versions b/.tool-versions index 3090ad85..71cd3e32 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,6 +1,6 @@ earthly 0.8.15 golang 1.22.7 -golangci-lint 1.62.2 +golangci-lint 1.63.4 helm 3.16.3 helm-cr 1.6.1 helm-ct 3.11.0 diff --git a/pkg/appdir/vcstoargomap.go b/pkg/appdir/vcstoargomap.go index 574cc9f8..dd50bf40 100644 --- a/pkg/appdir/vcstoargomap.go +++ b/pkg/appdir/vcstoargomap.go @@ -6,6 +6,7 @@ import ( "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/rs/zerolog/log" "github.com/zapier/kubechecks/pkg" + "github.com/zapier/kubechecks/pkg/kustomize" ) type VcsToArgoMap struct { @@ -60,7 +61,7 @@ func (v2a VcsToArgoMap) GetAppSetsInRepo(repoCloneUrl string) *AppSetDirectory { return appSetDir } -func (v2a VcsToArgoMap) WalkKustomizeApps(cloneURL string, fs fs.FS) *AppDirectory { +func (v2a VcsToArgoMap) WalkKustomizeApps(cloneURL string, rootFS fs.FS) *AppDirectory { var ( err error @@ -71,7 +72,13 @@ func (v2a VcsToArgoMap) WalkKustomizeApps(cloneURL string, fs fs.FS) *AppDirecto for _, app := range apps { appPath := app.Spec.GetSource().Path - if err = walkKustomizeFiles(result, fs, app.Name, appPath); err != nil { + + p := processor{ + appName: app.Name, + result: result, + } + + if err = kustomize.ProcessKustomizationFile(rootFS, appPath, &p); err != nil { log.Error().Err(err).Msgf("failed to parse kustomize.yaml in %s", appPath) } } diff --git a/pkg/appdir/walk_kustomize_files.go b/pkg/appdir/walk_kustomize_files.go index b285a2bd..5edbdabe 100644 --- a/pkg/appdir/walk_kustomize_files.go +++ b/pkg/appdir/walk_kustomize_files.go @@ -1,124 +1,22 @@ package appdir import ( - "io" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "k8s.io/apimachinery/pkg/util/yaml" + "github.com/zapier/kubechecks/pkg/kustomize" ) -type patchJson6902 struct { - Path string `yaml:"path"` +type processor struct { + appName string + result *AppDirectory } -func isGoGetterIsh(s string) bool { - if strings.HasPrefix(s, "github.com/") { - return true - } - - if strings.HasPrefix(s, "https://") { - return true - } - - if strings.HasPrefix(s, "http://") { - return true - } - - return false +func (p *processor) AddDir(dir string) error { + p.result.addDir(p.appName, dir) + return nil } -func walkKustomizeFiles(result *AppDirectory, fs fs.FS, appName, dirpath string) error { - kustomizeFile := filepath.Join(dirpath, "kustomization.yaml") - - var ( - err error - - kustomize struct { - Bases []string `yaml:"bases"` - Resources []string `yaml:"resources"` - PatchesJson6902 []patchJson6902 `yaml:"patchesJson6902"` - PatchesStrategicMerge []string `yaml:"patchesStrategicMerge"` - } - ) - - reader, err := fs.Open(kustomizeFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - - return errors.Wrap(err, "failed to open file") - } - - bytes, err := io.ReadAll(reader) - if err != nil { - return errors.Wrap(err, "failed to read file") - } - - if err = yaml.Unmarshal(bytes, &kustomize); err != nil { - return errors.Wrap(err, "failed to unmarshal file") - } - - for _, resource := range kustomize.Resources { - if isGoGetterIsh(resource) { - // no reason to walk remote files, since they can't be changed - continue - } - - var relPath string - if len(resource) >= 1 && resource[0] == '/' { - relPath = resource[1:] - } else { - relPath = filepath.Join(dirpath, resource) - } - - file, err := fs.Open(relPath) - if err != nil { - return errors.Wrapf(err, "failed to read %s", relPath) - } - stat, err := file.Stat() - if err != nil { - log.Warn().Err(err).Msgf("failed to stat %s", relPath) - } - - if !stat.IsDir() { - result.addFile(appName, relPath) - continue - } - - result.addDir(appName, relPath) - if err = walkKustomizeFiles(result, fs, appName, relPath); err != nil { - log.Warn().Err(err).Msgf("failed to read kustomize.yaml from resources in %s", relPath) - } - } - - for _, basePath := range kustomize.Bases { - if isGoGetterIsh(basePath) { - // no reason to walk remote files, since they can't be changed - continue - } - - relPath := filepath.Join(dirpath, basePath) - result.addDir(appName, relPath) - if err = walkKustomizeFiles(result, fs, appName, relPath); err != nil { - log.Warn().Err(err).Msgf("failed to read kustomize.yaml from bases in %s", relPath) - } - } - - for _, patchFile := range kustomize.PatchesStrategicMerge { - relPath := filepath.Join(dirpath, patchFile) - result.addFile(appName, relPath) - } - - for _, patch := range kustomize.PatchesJson6902 { - relPath := filepath.Join(dirpath, patch.Path) - result.addFile(appName, relPath) - } - +func (p *processor) AddFile(file string) error { + p.result.addFile(p.appName, file) return nil } + +var _ kustomize.Processor = new(processor) diff --git a/pkg/appdir/walk_kustomize_files_test.go b/pkg/appdir/walk_kustomize_files_test.go index c5b77fa3..5e83d30a 100644 --- a/pkg/appdir/walk_kustomize_files_test.go +++ b/pkg/appdir/walk_kustomize_files_test.go @@ -1,12 +1,14 @@ package appdir import ( + "path/filepath" "testing" "testing/fstest" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/zapier/kubechecks/pkg/kustomize" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -28,6 +30,11 @@ func TestKustomizeWalking(t *testing.T) { kustomizeApp2Path = "test/app2" fs = fstest.MapFS{ + "test/base/kustomization.yaml": { + Data: toBytes(` +resources: +- base_resource.yaml`), + }, "test/app/kustomization.yaml": { Data: toBytes(` bases: @@ -73,6 +80,7 @@ resources: "common/overlays/prod/kustomization.yaml": {Data: toBytes("hello: world")}, "test/overlays/base/some-file1.yaml": {Data: toBytes("hello: world")}, "test/overlays/base/some-file2.yaml": {Data: toBytes("hello: world")}, + "test/base/base_resource.yaml": {Data: toBytes(`hello: world`)}, } ) @@ -103,10 +111,20 @@ resources: appdir.AddApp(newApp(kustomizeApp2Name, kustomizeApp2Path, "HEAD", false, true)) appdir.AddApp(newApp(kustomizeBaseName, kustomizeBasePath, "HEAD", false, true)) - err = walkKustomizeFiles(appdir, fs, kustomizeApp1Name, kustomizeApp1Path) + testProc1 := &processor{ + appName: kustomizeApp1Name, + result: appdir, + } + + err = kustomize.ProcessKustomizationFile(fs, filepath.Join(kustomizeApp1Path, "kustomization.yaml"), testProc1) require.NoError(t, err) - err = walkKustomizeFiles(appdir, fs, kustomizeApp2Name, kustomizeApp2Path) + testProc2 := &processor{ + appName: kustomizeApp2Name, + result: appdir, + } + + err = kustomize.ProcessKustomizationFile(fs, filepath.Join(kustomizeApp2Path, "kustomization.yaml"), testProc2) require.NoError(t, err) assert.Equal(t, map[string][]string{ @@ -138,6 +156,19 @@ resources: }, appdir.appDirs) assert.Equal(t, map[string][]string{ + "common/overlays/prod/kustomization.yaml": { + kustomizeApp1Name, + kustomizeApp2Name, + }, + "test/app/kustomization.yaml": { + kustomizeApp1Name, + }, + "test/app/overlays/dev/kustomization.yaml": { + kustomizeApp1Name, + }, + "test/app2/kustomization.yaml": { + kustomizeApp2Name, + }, "test/app/file1.yaml": { kustomizeApp1Name, }, @@ -164,5 +195,19 @@ resources: "test/app2/file1.yaml": { kustomizeApp2Name, }, + "test/base/base_resource.yaml": { + kustomizeApp1Name, + }, + "test/base/kustomization.yaml": { + kustomizeApp1Name, + }, + "test/overlays/base/kustomization.yaml": { + kustomizeApp1Name, + kustomizeApp2Name, + }, + "test/overlays/common/kustomization.yaml": { + kustomizeApp1Name, + kustomizeApp2Name, + }, }, appdir.appFiles) } diff --git a/pkg/argo_client/kustomize.go b/pkg/argo_client/kustomize.go new file mode 100644 index 00000000..668a5383 --- /dev/null +++ b/pkg/argo_client/kustomize.go @@ -0,0 +1,67 @@ +package argo_client + +import ( + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/zapier/kubechecks/pkg/kustomize" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +type processor struct { + repoRoot string + tempDir string + repoFS filesys.FileSystem +} + +func (p processor) AddDir(s string) error { + return nil +} + +func (p processor) AddFile(relPath string) error { + absDepPath := filepath.Clean(filepath.Join(p.repoRoot, relPath)) + + // Get relative path from repo root + relPath, err := filepath.Rel(p.repoRoot, absDepPath) + if err != nil { + return errors.Wrapf(err, "failed to get relative path for %s", absDepPath) + } + + // check if the file exists in the temp directory + // skip copying if it exists + tempPath := filepath.Join(p.tempDir, relPath) + if _, err := os.Stat(tempPath); err == nil { + return nil + } + + dstdir := filepath.Dir(tempPath) + if err := os.MkdirAll(dstdir, 0o777); err != nil { + return errors.Wrap(err, "failed to make directories") + } + + r, err := os.Open(absDepPath) + if err != nil { + return err + } + defer r.Close() // ignore error: file was opened read-only. + + w, err := os.Create(tempPath) + if err != nil { + return err + } + + defer func() { + // Report the error, if any, from Close, but do so + // only if there isn't already an outgoing error. + if c := w.Close(); err == nil { + err = c + } + }() + + _, err = io.Copy(w, r) + return errors.Wrap(err, "failed to copy file") +} + +var _ kustomize.Processor = new(processor) diff --git a/pkg/argo_client/manifests.go b/pkg/argo_client/manifests.go index d495e5f2..16134282 100644 --- a/pkg/argo_client/manifests.go +++ b/pkg/argo_client/manifests.go @@ -24,10 +24,9 @@ import ( "github.com/rs/zerolog/log" "github.com/zapier/kubechecks/pkg" "github.com/zapier/kubechecks/pkg/git" + "github.com/zapier/kubechecks/pkg/kustomize" "github.com/zapier/kubechecks/pkg/vcs" - "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" - "sigs.k8s.io/kustomize/kyaml/yaml" ) type getRepo func(ctx context.Context, cloneURL string, branchName string) (*git.Repo, error) @@ -353,29 +352,44 @@ func copyFile(srcpath, dstpath string) error { return err } -func packageApp(ctx context.Context, source v1alpha1.ApplicationSource, refs []v1alpha1.ApplicationSource, repo *git.Repo, getRepo getRepo) (string, error) { - tempDir, err := os.MkdirTemp("", "package-*") +func packageApp( + ctx context.Context, + source v1alpha1.ApplicationSource, + refs []v1alpha1.ApplicationSource, + repo *git.Repo, + getRepo getRepo, +) (string, error) { + destDir, err := os.MkdirTemp("", "package-*") if err != nil { return "", errors.Wrap(err, "failed to make temp dir") } - repoFs := filesys.MakeFsOnDisk() - tempAppDir := filepath.Join(tempDir, source.Path) - appPath := filepath.Join(repo.Directory, source.Path) + fsIface := filesys.MakeFsOnDisk() + + destAppDir := filepath.Join(destDir, source.Path) + srcAppPath := filepath.Join(repo.Directory, source.Path) + sourceFS := os.DirFS(repo.Directory) // First copy the entire source directory - if err := copyDir(repoFs, appPath, filepath.Join(tempDir, source.Path)); err != nil { + if err := copyDir(fsIface, srcAppPath, destAppDir); err != nil { return "", errors.Wrap(err, "failed to copy base directory") } // Process kustomization dependencies - kustPath := filepath.Join(appPath, "kustomization.yaml") - if repoFs.Exists(kustPath) { - // Process kustomization dependencies with repo root - if err := processKustomizationDeps(repoFs, repo.Directory, appPath, tempDir); err != nil { + relKustPath := filepath.Join(source.Path, "kustomization.yaml") + absKustPath := filepath.Join(destDir, relKustPath) + if fsIface.Exists(absKustPath) { + p := processor{ + repoRoot: repo.Directory, + tempDir: destDir, + repoFS: fsIface, + } + + if err := kustomize.ProcessKustomizationFile(sourceFS, relKustPath, &p); err != nil { return "", errors.Wrap(err, "failed to process kustomization dependencies") } } + if source.Helm != nil { refsByName := make(map[string]v1alpha1.ApplicationSource) for _, ref := range refs { @@ -384,7 +398,7 @@ func packageApp(ctx context.Context, source v1alpha1.ApplicationSource, refs []v for index, valueFile := range source.Helm.ValueFiles { if strings.HasPrefix(valueFile, "$") { - relPath, err := processValueReference(ctx, source, valueFile, refsByName, repo, getRepo, tempDir, tempAppDir) + relPath, err := processValueReference(ctx, source, valueFile, refsByName, repo, getRepo, destDir, destAppDir) if err != nil { return "", err } @@ -406,8 +420,8 @@ func packageApp(ctx context.Context, source v1alpha1.ApplicationSource, refs []v continue // this values file is already copied } - src := filepath.Join(appPath, valueFile) - dst := filepath.Join(tempAppDir, valueFile) + src := filepath.Join(srcAppPath, valueFile) + dst := filepath.Join(destAppDir, valueFile) if err = copyFile(src, dst); err != nil { if !ignoreValuesFileCopyError(source, valueFile, err) { return "", errors.Wrapf(err, "failed to copy file: %q", valueFile) @@ -416,7 +430,7 @@ func packageApp(ctx context.Context, source v1alpha1.ApplicationSource, refs []v } } - return tempDir, nil + return destDir, nil } func processValueReference( @@ -521,107 +535,3 @@ func sendFile(ctx context.Context, sender sender, file *os.File) error { func areSameTargetRef(ref1, ref2 string) bool { return ref1 == ref2 } - -func processKustomizationDeps(fs filesys.FileSystem, repoRoot string, basePath string, tempDir string) error { - visited := make(map[string]bool) - return walkKustomizationDeps(fs, repoRoot, basePath, tempDir, visited) -} - -// walkKustomizationDeps recursively processes kustomization dependencies and copies them to the temp directory -func walkKustomizationDeps(fs filesys.FileSystem, repoRoot string, currentPath string, tempDir string, visited map[string]bool) error { - kustPath := filepath.Join(currentPath, "kustomization.yaml") - if !fs.Exists(kustPath) { - return nil // No kustomization.yaml in this directory - } - - if visited[currentPath] { - return nil // Already processed - } - visited[currentPath] = true - - // Parse using official Kustomization type - content, err := fs.ReadFile(kustPath) - if err != nil { - return errors.Wrapf(err, "failed to read kustomization.yaml at %s", currentPath) - } - - kust := &types.Kustomization{} - if err := yaml.Unmarshal(content, kust); err != nil { - return errors.Wrapf(err, "failed to parse kustomization.yaml at %s", currentPath) - } - - // Collect all dependencies from various fields - var allDeps []string - allDeps = append(allDeps, kust.Resources...) - allDeps = append(allDeps, kust.Components...) - allDeps = append(allDeps, kust.Configurations...) - allDeps = append(allDeps, kust.Crds...) - - // Handle replacements - for _, r := range kust.Replacements { - allDeps = append(allDeps, r.Path) - } - - // Process all dependencies - for _, dep := range allDeps { - absDepPath := filepath.Clean(filepath.Join(currentPath, dep)) - - // Skip remote resources - if isRemoteResource(dep) { - continue - } - - // Get relative path from repo root - relPath, err := filepath.Rel(repoRoot, absDepPath) - if err != nil { - return errors.Wrapf(err, "failed to get relative path for %s", absDepPath) - } - - // check if the file exists in the temp directory - // skip copying if it exists - tempPath := filepath.Join(tempDir, relPath) - if _, err := os.Stat(tempPath); !os.IsNotExist(err) { - continue - } - - // Copy the dependency - if err := copyDir(fs, absDepPath, tempPath); err != nil { - return errors.Wrapf(err, "failed to copy dependency %s", dep) - } - - // Recursively process nested kustomizations (e.g. base/kustomization.yaml imports other resources) - if fs.IsDir(absDepPath) { - if err := walkKustomizationDeps(fs, repoRoot, absDepPath, tempDir, visited); err != nil { - return err - } - } - } - - return nil -} - -func isRemoteResource(resource string) bool { - // Check for URL schemes - if strings.Contains(resource, "://") { - return true - } - - // Check for common Git SSH patterns - if strings.HasPrefix(resource, "git@") { - return true - } - - // Check for Kustomize's special GitHub/Bitbucket shorthand - if strings.HasPrefix(resource, "github.com/") || - strings.HasPrefix(resource, "bitbucket.org/") || - strings.HasPrefix(resource, "gitlab.com/") { - return true - } - - // Check for HTTP(S) URLs without explicit scheme (kustomize allows this) - if strings.HasPrefix(resource, "//") { - return true - } - - return false -} diff --git a/pkg/argo_client/manifests_test.go b/pkg/argo_client/manifests_test.go index 8f8a1373..1b58ac70 100644 --- a/pkg/argo_client/manifests_test.go +++ b/pkg/argo_client/manifests_test.go @@ -331,15 +331,14 @@ func TestPackageApp(t *testing.T) { filesByRepo: map[repoTarget]set[string]{ repoTarget{"git@github.com:testuser/testrepo.git", "main"}: newSet[string]( "app1/resource1.yaml", - "app1/component1.yaml", "app1/crds/crd1.yaml", "base/resource2.yaml", - "base/component2.yaml", "base/crds/crd2.yaml", + "component1/resource3.yaml", ), }, filesByRepoWithContent: map[repoTarget]map[string]string{ - repoTarget{"git@github.com:testuser/testrepo.git", "main"}: map[string]string{ + repoTarget{"git@github.com:testuser/testrepo.git", "main"}: { "app1/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -347,7 +346,7 @@ resources: - ../base - resource1.yaml components: -- component1.yaml +- ../component1 crds: - crds/crd1.yaml`, "base/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 @@ -355,20 +354,24 @@ kind: Kustomization resources: - resource2.yaml components: -- component2.yaml +- ../component1 crds: - crds/crd2.yaml`, + "component1/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- resource3.yaml`, }, }, expectedFiles: map[string]repoTargetPath{ - "app1/kustomization.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/kustomization.yaml"}, - "app1/resource1.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/resource1.yaml"}, - "app1/crds/crd1.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/crds/crd1.yaml"}, - "app1/component1.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/component1.yaml"}, - "base/kustomization.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/kustomization.yaml"}, - "base/resource2.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/resource2.yaml"}, - "base/crds/crd2.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/crds/crd2.yaml"}, - "base/component2.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/component2.yaml"}, + "app1/kustomization.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/kustomization.yaml"}, + "app1/resource1.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/resource1.yaml"}, + "app1/crds/crd1.yaml": {"git@github.com:testuser/testrepo.git", "main", "app1/crds/crd1.yaml"}, + "base/kustomization.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/kustomization.yaml"}, + "base/resource2.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/resource2.yaml"}, + "base/crds/crd2.yaml": {"git@github.com:testuser/testrepo.git", "main", "base/crds/crd2.yaml"}, + "component1/kustomization.yaml": {"git@github.com:testuser/testrepo.git", "main", "component1/kustomization.yaml"}, + "component1/resource3.yaml": {"git@github.com:testuser/testrepo.git", "main", "component1/resource3.yaml"}, }, }, } diff --git a/pkg/events/check.go b/pkg/events/check.go index 6a6cf433..cabd61bc 100644 --- a/pkg/events/check.go +++ b/pkg/events/check.go @@ -405,19 +405,3 @@ func (ce *CheckEvent) createNote(ctx context.Context) (*msg.Message, error) { return ce.ctr.VcsClient.PostMessage(ctx, ce.pullRequest, ":hourglass: kubechecks running ... ") } - -// cleanupGetManifestsError takes an error as input and returns a simplified and more user-friendly error message. -// It reformats Helm error messages by removing excess information, and makes file paths relative to the git repo root. -func cleanupGetManifestsError(err error, repoDirectory string) string { - // cleanup the chonky helm error message for a better DX - errStr := err.Error() - if strings.Contains(errStr, "helm template") && strings.Contains(errStr, "failed exit status") { - errMsgIdx := strings.Index(errStr, "Error:") - errStr = fmt.Sprintf("Helm %s", errStr[errMsgIdx:]) - } - - // strip the temp directory from any files mentioned to make file paths relative to git repo root - errStr = strings.ReplaceAll(errStr, repoDirectory+"/", "") - - return errStr -} diff --git a/pkg/events/check_test.go b/pkg/events/check_test.go index 0e6ffeeb..d6e1eb3a 100644 --- a/pkg/events/check_test.go +++ b/pkg/events/check_test.go @@ -2,7 +2,6 @@ package events import ( "context" - "errors" "fmt" "testing" @@ -25,47 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// TestCleanupGetManifestsError tests the cleanupGetManifestsError function. -func TestCleanupGetManifestsError(t *testing.T) { - repoDirectory := "/some-dir" - - tests := []struct { - name string - inputErr error - expectedError string - }{ - { - name: "helm error", - inputErr: errors.New("`helm template . --name-template kubechecks --namespace kubechecks --kube-version 1.22 --values /tmp/kubechecks-mr-clone2267947074/manifests/tooling-eks-01/values.yaml --values /tmp/kubechecks-mr-clone2267947074/manifests/tooling-eks-01/current-tag.yaml --api-versions storage.k8s.io/v1 --api-versions storage.k8s.io/v1beta1 --api-versions v1 --api-versions vault.banzaicloud.com/v1alpha1 --api-versions velero.io/v1 --api-versions vpcresources.k8s.aws/v1beta1 --include-crds` failed exit status 1: Error: execution error at (kubechecks/charts/web/charts/ingress/templates/ingress.yaml:7:20): ingressClass value is required\\n\\nUse --debug flag to render out invalid YAML"), - expectedError: "Helm Error: execution error at (kubechecks/charts/web/charts/ingress/templates/ingress.yaml:7:20): ingressClass value is required\\n\\nUse --debug flag to render out invalid YAML", - }, - { - name: "strip temp directory", - inputErr: fmt.Errorf("error: %s/tmpfile.yaml not found", repoDirectory), - expectedError: "error: tmpfile.yaml not found", - }, - { - name: "strip temp directory and helm error", - inputErr: fmt.Errorf("`helm template . --name-template in-cluster-echo-server --namespace echo-server --kube-version 1.25 --values %s/apps/echo-server/in-cluster/values.yaml --values %s/apps/echo-server/in-cluster/notexist.yaml --api-versions admissionregistration.k8s.io/v1 --api-versions admissionregistration.k8s.io/v1/MutatingWebhookConfiguration --api-versions v1/Secret --api-versions v1/Service --api-versions v1/ServiceAccount --include-crds` failed exit status 1: Error: open %s/apps/echo-server/in-cluster/notexist.yaml: no such file or directory", repoDirectory, repoDirectory, repoDirectory), - expectedError: "Helm Error: open apps/echo-server/in-cluster/notexist.yaml: no such file or directory", - }, - { - name: "other error", - inputErr: errors.New("error: unknown error"), - expectedError: "error: unknown error", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cleanedError := cleanupGetManifestsError(tt.inputErr, repoDirectory) - if cleanedError != tt.expectedError { - t.Errorf("Expected error: %s, \n Received: %s", tt.expectedError, cleanedError) - } - }) - } -} - func TestCheckEventGetRepo(t *testing.T) { cloneURL := "https://github.com/zapier/kubechecks.git" canonical, err := canonicalize(cloneURL) diff --git a/pkg/git/repo.go b/pkg/git/repo.go index ae9a40ca..59098700 100644 --- a/pkg/git/repo.go +++ b/pkg/git/repo.go @@ -68,7 +68,7 @@ func (r *Repo) Clone(ctx context.Context) error { args = append(args, "--branch", r.BranchName) } - cmd := r.execCommand("git", args...) + cmd := r.execGitCommand(args...) out, err := cmd.CombinedOutput() if err != nil { log.Error().Err(err).Msgf("unable to clone repository, %s", out) @@ -96,7 +96,7 @@ func printFile(s string, d fs.DirEntry, err error) error { } func (r *Repo) GetRemoteHead() (string, error) { - cmd := r.execCommand("git", "symbolic-ref", "refs/remotes/origin/HEAD", "--short") + cmd := r.execGitCommand("symbolic-ref", "refs/remotes/origin/HEAD", "--short") out, err := cmd.CombinedOutput() if err != nil { return "", errors.Wrap(err, "failed to determine which branch HEAD points to") @@ -119,7 +119,7 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error { )) defer span.End() - cmd := r.execCommand("git", "merge", ref) + cmd := r.execGitCommand("merge", ref) out, err := cmd.CombinedOutput() if err != nil { telemetry.SetError(span, err, "merge commit into branch") @@ -131,17 +131,17 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error { } func (r *Repo) Update(ctx context.Context) error { - cmd := r.execCommand("git", "pull") + cmd := r.execGitCommand("pull") cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout return cmd.Run() } -func (r *Repo) execCommand(name string, args ...string) *exec.Cmd { +func (r *Repo) execGitCommand(args ...string) *exec.Cmd { argsToLog := r.censorVcsToken(args) log.Debug().Strs("args", argsToLog).Msg("building command") - cmd := exec.Command(name, args...) + cmd := exec.Command("git", args...) if r.Directory != "" { cmd.Dir = r.Directory } @@ -258,7 +258,7 @@ func (r *Repo) GetListOfChangedFiles(ctx context.Context) ([]string, error) { var fileList []string - cmd := r.execCommand("git", "diff", "--name-only", fmt.Sprintf("%s/%s", "origin", r.BranchName)) + cmd := r.execGitCommand("diff", "--name-only", fmt.Sprintf("%s/%s", "origin", r.BranchName)) pipe, _ := cmd.StdoutPipe() var wg sync.WaitGroup scanner := bufio.NewScanner(pipe) diff --git a/pkg/kustomize/process.go b/pkg/kustomize/process.go new file mode 100644 index 00000000..3395cfae --- /dev/null +++ b/pkg/kustomize/process.go @@ -0,0 +1,163 @@ +package kustomize + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type Processor interface { + AddDir(string) error + AddFile(string) error +} + +func ProcessKustomizationFile(sourceFS fs.FS, relKustomizationPath string, processor Processor) error { + dirName := filepath.Dir(relKustomizationPath) + return processDir(sourceFS, dirName, processor) +} + +func processDir(sourceFS fs.FS, relBase string, processor Processor) error { + absKustPath := filepath.Join(relBase, "kustomization.yaml") + + // Parse using official Kustomization type + file, err := sourceFS.Open(absKustPath) + if err != nil { + if os.IsNotExist(err) { + return nil // No kustomization.yaml in this directory + } + return errors.Wrap(err, "failed to open file") + } + + content, err := io.ReadAll(file) + if err != nil { + return errors.Wrap(err, "failed to read file") + } + + var kust types.Kustomization + if err := yaml.Unmarshal(content, &kust); err != nil { + return errors.Wrap(err, "failed to parse kustomization.yaml") + } + + // collect all the possible files and directories that kustomize can contain + var filesOrDirectories []string + filesOrDirectories = append(filesOrDirectories, kust.Bases...) // nolint:staticcheck // deprecated doesn't mean unused + filesOrDirectories = append(filesOrDirectories, kust.Resources...) + + var directories []string + directories = append(directories, kust.Components...) + + files := []string{"kustomization.yaml"} + files = append(files, kust.Configurations...) + files = append(files, kust.Crds...) + files = append(files, kust.Transformers...) + + for _, patch := range kust.Patches { + if patch.Path != "" { + files = append(files, patch.Path) + } + } + + for _, patch := range kust.PatchesJson6902 { // nolint:staticcheck // deprecated doesn't mean unused + if patch.Path != "" { + files = append(files, patch.Path) + } + } + + for _, patch := range kust.PatchesStrategicMerge { // nolint:staticcheck // deprecated doesn't mean unused + s := string(patch) + if !strings.Contains(s, "\n") { + files = append(files, s) + } + } + + // clean up the directories and files + filesOrDirectories = cleanPaths(relBase, filesOrDirectories) + directories = cleanPaths(relBase, directories) + files = cleanPaths(relBase, files) + + // figure out if these are files or directories + for _, fileOrDirectory := range filesOrDirectories { + file, err := sourceFS.Open(fileOrDirectory) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", fileOrDirectory) + } + + if stat, err := file.Stat(); err != nil { + return errors.Wrapf(err, "failed to stat %s", fileOrDirectory) + } else if stat.IsDir() { + directories = append(directories, fileOrDirectory) + } else { + files = append(files, fileOrDirectory) + } + } + + // add files + for _, relResource := range files { + if err := processor.AddFile(relResource); err != nil { + return errors.Wrapf(err, "failed to add resource %q", relResource) + } + } + + // process directories and add them + for _, relResource := range directories { + if err = processor.AddDir(relResource); err != nil { + return errors.Wrapf(err, "failed to add directory %q", relResource) + } + if err = processDir(sourceFS, relResource, processor); err != nil { + return errors.Wrapf(err, "failed to process %q", relResource) + } + } + + return nil +} + +func cleanPaths(relativeBase string, paths []string) []string { + var result []string + + for _, path := range paths { + if isRemoteResource(path) { + continue + } + + if strings.HasPrefix(path, "/") { + path = strings.TrimPrefix(path, "/") + } else { + path = filepath.Join(relativeBase, path) + } + result = append(result, path) + } + + return result +} + +func isRemoteResource(resource string) bool { + // Check for URL schemes + if strings.Contains(resource, "://") { + return true + } + + // Check for common Git SSH patterns + if strings.HasPrefix(resource, "git@") { + return true + } + + // Check for Kustomize's special GitHub/Bitbucket shorthand + if strings.HasPrefix(resource, "github.com/") || + strings.HasPrefix(resource, "bitbucket.org/") || + strings.HasPrefix(resource, "gitlab.com/") { + return true + } + + // Check for HTTP(S) URLs without explicit scheme (kustomize allows this) + if strings.HasPrefix(resource, "//") { + return true + } + + return false +} diff --git a/pkg/vcs/gitlab_client/backoff.go b/pkg/vcs/gitlab_client/backoff.go index c1a86b22..1e029cf7 100644 --- a/pkg/vcs/gitlab_client/backoff.go +++ b/pkg/vcs/gitlab_client/backoff.go @@ -2,6 +2,7 @@ package gitlab_client import ( "fmt" + "net/http" "time" "github.com/cenkalti/backoff/v4" @@ -25,7 +26,7 @@ func getBackOff() *backoff.ExponentialBackOff { func checkReturnForBackoff(resp *gitlab.Response, err error) error { // if the error is nil lets check it out if resp != nil { - if resp.StatusCode == 429 { + if resp.StatusCode == http.StatusTooManyRequests { log.Warn().Msg("being rate limited doing backoff") return fmt.Errorf("%s", "Rate Limited") } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 531e801e..8353c54d 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -80,7 +80,7 @@ func Init(ctx context.Context, serviceName, gitTag, gitCommit string, otelEnable return bt, nil } -func createGRPCConn(ctx context.Context, enabled bool, otelHost string, otelPort string) (*grpc.ClientConn, error) { +func createGRPCConn(enabled bool, otelHost string, otelPort string) (*grpc.ClientConn, error) { if !enabled { log.Info().Msg("otel disabled") return nil, nil @@ -103,7 +103,7 @@ func createGRPCConn(ctx context.Context, enabled bool, otelHost string, otelPort } func (bt *BaseTelemetry) initProviders(res *resource.Resource, enabled bool, otelHost string, otelPort string) error { - conn, err := createGRPCConn(bt.c, enabled, otelHost, otelPort) + conn, err := createGRPCConn(enabled, otelHost, otelPort) if err != nil { return err }