From c68fe5bcaeba277a163a871042f3524d744c4c19 Mon Sep 17 00:00:00 2001 From: Fabian Holler Date: Fri, 2 Nov 2018 17:49:44 +0100 Subject: [PATCH] inputs: refactor build input resolver - move glob path, gitfile, gosource file resolvers to an own resolve/ package, the resolvers return absolute paths - the buildinput interface is removed, it is not needed, we can use the File struct instead, all resolved build inputs are files - buildinputs are not converted from the config struct and intermediate struct anymore, the App struct references the cfg.BuildInput struct directly --- app.go | 207 ++++++++++++------ build.go | 2 +- buildinput.go | 12 - buildinputpathresolver.go | 10 - command/build.go | 2 +- command/ls_inputs.go | 4 +- command/show.go | 35 ++- file.go | 5 - fileglobpath.go | 62 ------ fs/fs.go | 133 ----------- fs/fs_test.go | 45 ---- git/git.go | 7 +- gitpaths.go | 62 ------ gosrcdir.go | 79 ------- resolve/gitpath/gitpaths.go | 56 +++++ resolve/glob/glob.go | 165 ++++++++++++++ .../glob/glob_test.go | 56 ++++- .../golang.go => resolve/gosource/gosource.go | 39 +++- resolve/resolver.go | 7 + 19 files changed, 485 insertions(+), 503 deletions(-) delete mode 100644 buildinput.go delete mode 100644 buildinputpathresolver.go delete mode 100644 fileglobpath.go delete mode 100644 fs/fs_test.go delete mode 100644 gitpaths.go delete mode 100644 gosrcdir.go create mode 100644 resolve/gitpath/gitpaths.go create mode 100644 resolve/glob/glob.go rename fileglobpath_test.go => resolve/glob/glob_test.go (72%) rename golang/golang.go => resolve/gosource/gosource.go (77%) create mode 100644 resolve/resolver.go diff --git a/app.go b/app.go index f18ca023e..9253f5efe 100644 --- a/app.go +++ b/app.go @@ -1,6 +1,7 @@ package baur import ( + "fmt" "path" "path/filepath" "sort" @@ -12,6 +13,10 @@ import ( "github.com/simplesurance/baur/cfg" "github.com/simplesurance/baur/digest" "github.com/simplesurance/baur/digest/sha384" + "github.com/simplesurance/baur/log" + "github.com/simplesurance/baur/resolve/gitpath" + "github.com/simplesurance/baur/resolve/glob" + "github.com/simplesurance/baur/resolve/gosource" ) // App represents an application @@ -22,9 +27,10 @@ type App struct { BuildCmd string Repository *Repository Outputs []BuildOutput - BuildInputPaths []BuildInputPathResolver - buildInputs []BuildInput totalInputDigest *digest.Digest + + UnresolvedInputs cfg.BuildInput + buildInputs []*File } func replaceUUIDvar(in string) string { @@ -44,42 +50,6 @@ func replaceGitCommitVar(in string, r *Repository) (string, error) { return strings.Replace(in, "$GITCOMMIT", commitID, -1), nil } -func (a *App) setInputsFromCfg(r *Repository, cfg *cfg.App) error { - sliceLen := len(cfg.Build.Input.Files.Paths) - - if len(cfg.Build.Input.GitFiles.Paths) > 0 { - sliceLen++ - } - - if len(cfg.Build.Input.GolangSources.Paths) > 0 { - sliceLen++ - } - - a.BuildInputPaths = make([]BuildInputPathResolver, 0, sliceLen) - - for _, p := range cfg.Build.Input.Files.Paths { - a.BuildInputPaths = append(a.BuildInputPaths, NewFileGlobPath(r.Path, a.RelPath, p)) - } - - if len(cfg.Build.Input.GitFiles.Paths) > 0 { - a.BuildInputPaths = append(a.BuildInputPaths, - NewGitPaths(r.Path, a.RelPath, cfg.Build.Input.GitFiles.Paths)) - } - - if len(cfg.Build.Input.GolangSources.Paths) > 0 { - var gopath string - - if len(cfg.Build.Input.GolangSources.GoPath) > 0 { - gopath = filepath.Join(a.Path, cfg.Build.Input.GolangSources.GoPath) - } - - a.BuildInputPaths = append(a.BuildInputPaths, - NewGoSrcDirs(r.Path, a.RelPath, gopath, cfg.Build.Input.GolangSources.Paths)) - } - - return nil -} - func (a *App) setDockerOutputsFromCfg(cfg *cfg.App) error { for _, di := range cfg.Build.Output.DockerImage { tag, err := replaceGitCommitVar(di.RegistryUpload.Tag, a.Repository) @@ -157,9 +127,7 @@ func NewApp(repository *Repository, cfgPath string) (*App, error) { return nil, errors.Wrap(err, "processing S3 output declarations failed") } - if err := app.setInputsFromCfg(repository, cfg); err != nil { - return nil, errors.Wrap(err, "processing input declarations failed") - } + app.UnresolvedInputs = cfg.Build.Input return &app, nil } @@ -169,38 +137,151 @@ func (a *App) String() string { return a.Name } -// BuildInputs returns all deduplicated BuildInputs. -// If the function is called the first time, the BuildInputPaths are resolved -// and stored. On following calls the stored BuildInputs are returned. -func (a *App) BuildInputs() ([]BuildInput, error) { - if a.buildInputs != nil { - return a.buildInputs, nil - } +func (a *App) pathsToUniqFiles(paths []string) ([]*File, error) { + dedupMap := map[string]struct{}{} + res := make([]*File, 0, len(paths)) - if len(a.BuildInputPaths) == 0 { - a.buildInputs = []BuildInput{} - return a.buildInputs, nil + for _, path := range paths { + if _, exist := dedupMap[path]; exist { + log.Debugf("%s: removed duplicate Build Input '%s'", a.Name, path) + continue + } + dedupMap[path] = struct{}{} + + relPath, err := filepath.Rel(a.Repository.Path, path) + if err != nil { + return nil, errors.Wrapf(err, "resolving relative path to '%s' from '%s' failed", path, a.Repository.Path) + } + + // TODO: should resolving the relative path be done in + // Newfile() instead? + res = append(res, NewFile(a.Repository.Path, relPath)) } - dedupBuildInputs := map[string]BuildInput{} + return res, nil +} + +func (a *App) resolveGlobFileInputs() ([]string, error) { + var res []string - for _, inputPath := range a.BuildInputPaths { - buildInputs, err := inputPath.Resolve() + for _, globPath := range a.UnresolvedInputs.Files.Paths { + resolver := glob.NewResolver(a.Path, globPath) + paths, err := resolver.Resolve() if err != nil { - return nil, errors.Wrapf(err, "resolving %q failed", inputPath) + return nil, errors.Wrap(err, globPath) } - for _, bi := range buildInputs { - if _, exist := dedupBuildInputs[bi.URI()]; exist { - continue - } - - dedupBuildInputs[bi.URI()] = bi + if len(paths) == 0 { + return nil, fmt.Errorf("'%s' matched 0 files", globPath) } + + res = append(res, paths...) + } + + return res, nil +} + +func (a *App) resolveGitFileInputs() ([]string, error) { + if len(a.UnresolvedInputs.GitFiles.Paths) == 0 { + return []string{}, nil + } + + resolver := gitpath.NewResolver(a.Path, a.UnresolvedInputs.GitFiles.Paths...) + paths, err := resolver.Resolve() + if err != nil { + return nil, err + } + + if len(paths) == 0 { + return nil, fmt.Errorf("'%s' matched 0 files", strings.Join(a.UnresolvedInputs.GitFiles.Paths, ", ")) } - for _, bi := range dedupBuildInputs { - a.buildInputs = append(a.buildInputs, bi) + return paths, nil +} + +func (a *App) resolveGoSrcInputs() ([]string, error) { + if len(a.UnresolvedInputs.GolangSources.Paths) == 0 { + return []string{}, nil + } + + var gopath string + if a.UnresolvedInputs.GolangSources.GoPath != "" { + gopath = filepath.Join(a.Path, a.UnresolvedInputs.GolangSources.GoPath) + } + + resolver := gosource.NewResolver(a.Path, gopath, a.UnresolvedInputs.GolangSources.Paths...) + paths, err := resolver.Resolve() + if err != nil { + return nil, err + } + + if len(paths) == 0 { + return nil, fmt.Errorf("'%s' matched 0 files", strings.Join(a.UnresolvedInputs.GitFiles.Paths, ", ")) + } + + return paths, nil +} + +func (a *App) resolveBuildInputPaths() ([]string, error) { + globPaths, err := a.resolveGlobFileInputs() + if err != nil { + return nil, errors.Wrapf(err, "resolving File BuildInputs failed") + } + + gitPaths, err := a.resolveGitFileInputs() + if err != nil { + return nil, errors.Wrapf(err, "resolving GitFile BuildInputs failed") + } + + goSrcPaths, err := a.resolveGoSrcInputs() + if err != nil { + return nil, errors.Wrapf(err, "resolving GoLangSources BuildInputs failed") + } + + paths := make([]string, 0, len(globPaths)+len(gitPaths)+len(goSrcPaths)) + paths = append(paths, globPaths...) + paths = append(paths, gitPaths...) + paths = append(paths, goSrcPaths...) + + return paths, nil +} + +// HasBuildInputs returns true if BuildInputs are defined for the app +func (a *App) HasBuildInputs() bool { + if len(a.UnresolvedInputs.Files.Paths) != 0 { + return true + } + + if len(a.UnresolvedInputs.GitFiles.Paths) != 0 { + return true + } + + if len(a.UnresolvedInputs.GolangSources.Paths) != 0 { + return true + } + + return false +} + +// BuildInputs resolves all build inputs of the app. +// The BuildInputs are deduplicates before they are returned. +// If one more resolved path does not match a file an error is generated. +// If not build inputs are defined, an empty slice and no error is returned. +// If the function is called the first time, the BuildInputPaths are resolved +// and stored. On following calls the stored BuildInputs are returned. +func (a *App) BuildInputs() ([]*File, error) { + if a.buildInputs != nil { + return a.buildInputs, nil + } + + paths, err := a.resolveBuildInputPaths() + if err != nil { + return nil, err + } + + a.buildInputs, err = a.pathsToUniqFiles(paths) + if err != nil { + return nil, err } return a.buildInputs, nil diff --git a/build.go b/build.go index e46ef83d6..46095ca67 100644 --- a/build.go +++ b/build.go @@ -39,7 +39,7 @@ func (b BuildStatus) String() string { // If the function returns BuildStatusExist the returned build pointer is valid // otherwise it is nil. func GetBuildStatus(storer storage.Storer, app *App) (BuildStatus, *storage.BuildWithDuration, error) { - if len(app.BuildInputPaths) == 0 { + if !app.HasBuildInputs() { return BuildStatusInputsUndefined, nil, nil } diff --git a/buildinput.go b/buildinput.go deleted file mode 100644 index f7410bec1..000000000 --- a/buildinput.go +++ /dev/null @@ -1,12 +0,0 @@ -package baur - -import "github.com/simplesurance/baur/digest" - -// BuildInput represents an input object of an application build, can be source -// files, compiler binaries etc, everything that can influence the produced -// build output -type BuildInput interface { - Digest() (digest.Digest, error) - String() string - URI() string -} diff --git a/buildinputpathresolver.go b/buildinputpathresolver.go deleted file mode 100644 index ed32550ba..000000000 --- a/buildinputpathresolver.go +++ /dev/null @@ -1,10 +0,0 @@ -package baur - -// BuildInputPathResolver is an interface to resolve abstract paths like file glob paths to -// concrete values (files) -type BuildInputPathResolver interface { - Resolve() ([]BuildInput, error) - // Type returns the type of resolver - Type() string - String() string -} diff --git a/command/build.go b/command/build.go index f3a043c42..d1c8a94bc 100644 --- a/command/build.go +++ b/command/build.go @@ -225,7 +225,7 @@ func calcDigests(app *baur.App) ([]*storage.Input, string) { storageInputs = append(storageInputs, &storage.Input{ Digest: d.String(), - URI: s.URI(), + URI: s.RepoRelPath(), }) inputDigests = append(inputDigests, &d) diff --git a/command/ls_inputs.go b/command/ls_inputs.go index b0255560c..ecbdc328b 100644 --- a/command/ls_inputs.go +++ b/command/ls_inputs.go @@ -49,7 +49,7 @@ func lsInputs(cmd *cobra.Command, args []string) { app := mustArgToApp(rep, args[0]) writeHeaders := !lsInputsConfig.quiet && !lsInputsConfig.csv - if len(app.BuildInputPaths) == 0 { + if !app.HasBuildInputs() { log.Fatalf("No build inputs are configured in %s of %s", baur.AppCfgFile, app.Name) } @@ -73,7 +73,7 @@ func lsInputs(cmd *cobra.Command, args []string) { } sort.Slice(inputs, func(i, j int) bool { - return inputs[i].URI() < inputs[j].URI() + return inputs[i].RepoRelPath() < inputs[j].RepoRelPath() }) for _, input := range inputs { diff --git a/command/show.go b/command/show.go index b74bcffbb..94a365088 100644 --- a/command/show.go +++ b/command/show.go @@ -78,19 +78,44 @@ func showApp(arg string) { } } - if len(app.BuildInputPaths) != 0 { + if app.HasBuildInputs() { + var printNewLine bool + mustWriteRow(formatter, []interface{}{}) mustWriteRow(formatter, []interface{}{underline("Inputs:")}) - for i, bi := range app.BuildInputPaths { - mustWriteRow(formatter, []interface{}{"", "Type:", highlight(bi.Type())}) - mustWriteRow(formatter, []interface{}{"", "Configuration:", highlight(bi.String())}) + if len(app.UnresolvedInputs.Files.Paths) > 0 { + mustWriteRow(formatter, []interface{}{"", "Type:", highlight("File")}) + mustWriteRow(formatter, []interface{}{"", + "Paths:", highlight(strings.Join(app.UnresolvedInputs.Files.Paths, ", ")), + }) + + printNewLine = true - if i+1 < len(app.BuildInputPaths) { + } + + if len(app.UnresolvedInputs.GitFiles.Paths) > 0 { + if printNewLine { mustWriteRow(formatter, []interface{}{}) } + + mustWriteRow(formatter, []interface{}{"", "Type:", highlight("GitFile")}) + mustWriteRow(formatter, []interface{}{"", + "Paths:", highlight(strings.Join(app.UnresolvedInputs.GitFiles.Paths, ", "))}) + + printNewLine = true } + if len(app.UnresolvedInputs.GolangSources.Paths) > 0 { + if printNewLine { + mustWriteRow(formatter, []interface{}{}) + } + + mustWriteRow(formatter, []interface{}{"", "Type:", highlight("GolangSources")}) + mustWriteRow(formatter, []interface{}{"", + "Paths:", highlight(strings.Join(app.UnresolvedInputs.GolangSources.Paths, ", "))}) + mustWriteRow(formatter, []interface{}{"", "GoPath:", highlight(app.UnresolvedInputs.GolangSources.GoPath)}) + } } if err := formatter.Flush(); err != nil { diff --git a/file.go b/file.go index 0cb71a7de..aca63ba1c 100644 --- a/file.go +++ b/file.go @@ -57,11 +57,6 @@ func (f *File) RepoRelPath() string { return f.relPath } -// URI calls RepoRelPath() -func (f *File) URI() string { - return f.RepoRelPath() -} - // String returns it's string representation func (f *File) String() string { return f.RepoRelPath() diff --git a/fileglobpath.go b/fileglobpath.go deleted file mode 100644 index 6a77e8f52..000000000 --- a/fileglobpath.go +++ /dev/null @@ -1,62 +0,0 @@ -package baur - -import ( - "path/filepath" - - "github.com/pkg/errors" - - "github.com/simplesurance/baur/fs" -) - -//FileGlobPath is Source file of an application represented by a glob path -type FileGlobPath struct { - repositoryRootPath string - relAppPath string - globPath string -} - -// NewFileGlobPath returns a new FileGlobPath object -func NewFileGlobPath(repositoryRootPath, relAppPath, glob string) *FileGlobPath { - return &FileGlobPath{ - repositoryRootPath: repositoryRootPath, - relAppPath: relAppPath, - globPath: glob, - } -} - -// Resolve returns a list of files that are matching the glob path of the -// FileGlobPath -func (f *FileGlobPath) Resolve() ([]BuildInput, error) { - absGlobPath := filepath.Join(f.repositoryRootPath, f.relAppPath, f.globPath) - - paths, err := fs.Glob(absGlobPath) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, errors.New("glob matched 0 files") - } - - res := make([]BuildInput, 0, len(paths)) - for _, p := range paths { - relPath, err := filepath.Rel(f.repositoryRootPath, p) - if err != nil { - return nil, errors.Wrapf(err, "converting %q to relpath with basedir %q failed", p, f.repositoryRootPath) - } - - res = append(res, NewFile(f.repositoryRootPath, relPath)) - } - - return res, nil -} - -// Type returns the type of resolver -func (f *FileGlobPath) Type() string { - return "Files" -} - -// String returns the path that is resolved -func (f *FileGlobPath) String() string { - return f.globPath -} diff --git a/fs/fs.go b/fs/fs.go index 581549d56..a0d7f6370 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -7,7 +7,6 @@ import ( "os" "path" "path/filepath" - "strings" "github.com/pkg/errors" ) @@ -122,60 +121,6 @@ func FindFilesInSubDir(searchDir, filename string, maxdepth int) ([]string, erro return result, nil } -// FindAllDirs returns recursively all diretories in path, including the -// passed path dir -func FindAllDirs(path string) ([]string, error) { - resultMap := map[string]struct{}{} - - err := findAllDirsNoDups(resultMap, path) - if err != nil { - return nil, err - } - - res := make([]string, 0, len(resultMap)) - for p := range resultMap { - res = append(res, p) - } - - return res, nil -} - -func findAllDirsNoDups(result map[string]struct{}, path string) error { - isDir, err := IsDir(path) - if err != nil { - return errors.Wrapf(err, "IsDir(%s) failed", path) - } - - if !isDir { - return nil - } - - path, err = filepath.EvalSymlinks(path) - if err != nil { - return errors.Wrapf(err, "resolving symlinks in %q failed", path) - } - - if _, exist := result[path]; exist { - return nil - } - result[path] = struct{}{} - - globPath := filepath.Join(path, "*") - rootGlob, err := filepath.Glob(globPath) // is filepath.Walk() faster? - if err != nil { - return errors.Wrapf(err, "glob of %q failed", globPath) - } - - for _, path := range rootGlob { - err = findAllDirsNoDups(result, path) - if err != nil { - return err - } - } - - return nil -} - // PathsJoin returns a list where all paths in relPaths are prefixed with // rootPath func PathsJoin(rootPath string, relPaths []string) []string { @@ -216,84 +161,6 @@ func FileSize(path string) (int64, error) { return stat.Size(), nil } -// expandDoubleStarGlob takes a glob path containing '**' and returns a list of -// paths were ** is expanded recursively to all matching directories. If '**' -// is the last part in the path, the returned paths will end in '/*' to glob -// match all files in those directories -func expandDoubleStarGlob(absGlobPath string) ([]string, error) { - spl := strings.Split(absGlobPath, "**") - if len(spl) < 2 { - return nil, fmt.Errorf("%q does not contain '**'", absGlobPath) - } - - basePath := spl[0] - glob := spl[1] - - if len(glob) == 0 { - glob = "*" - } - - dirs, err := FindAllDirs(basePath) - if err != nil { - return nil, err - } - - for i := range dirs { - dirs[i] = filepath.Join(dirs[i], glob) - } - - return dirs, nil -} - -// Glob is similar then filepath.Glob() but also support '**' to match files -// and directories recursively and only returns paths to Files. -// If a Glob doesn't match any files an empty []string is returned and error is -// nil -func Glob(path string) ([]string, error) { - var globPaths []string - - if strings.Contains(path, "**") { - expandedPaths, err := expandDoubleStarGlob(path) - if err != nil { - return nil, errors.Wrap(err, "expanding '**' failed") - } - - globPaths = expandedPaths - } else { - globPaths = []string{path} - } - - // capacity could be bigger to have some memory available for the - // elements returned by filepath.Glob() that will be added - paths := make([]string, 0, len(globPaths)) - for _, globPath := range globPaths { - path, err := filepath.Glob(globPath) - if err != nil { - return nil, err - } - - if path == nil { - continue - } - - paths = append(paths, path...) - } - - res := make([]string, 0, len(paths)) - for _, p := range paths { - isFile, err := IsFile(p) - if err != nil { - return nil, errors.Wrapf(err, "resolved path %q does not exist", p) - } - - if isFile { - res = append(res, p) - } - } - - return res, nil -} - // Mkdir creates recursively directories func Mkdir(path string) error { return os.MkdirAll(path, os.FileMode(0755)) diff --git a/fs/fs_test.go b/fs/fs_test.go deleted file mode 100644 index cfe90de0e..000000000 --- a/fs/fs_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package fs - -import ( - "os" - "path/filepath" - "testing" - - "github.com/simplesurance/baur/testutils/fstest" - "github.com/simplesurance/baur/testutils/strtest" -) - -func Test_FindAllSubDirs(t *testing.T) { - tempdir, cleanupFunc := fstest.CreateTempDir(t) - defer cleanupFunc() - - expectedResults := []string{ - tempdir, - filepath.Join(tempdir, "1"), - filepath.Join(tempdir, "1/2"), - filepath.Join(tempdir, "1/2/3/"), - } - - err := os.MkdirAll(filepath.Join(tempdir, "1/2/3"), os.ModePerm) - if err != nil { - t.Fatal("creating subdirectories failed:", err) - } - - res, err := FindAllDirs(tempdir) - if err != nil { - t.Fatal(err) - } - - if len(res) != len(expectedResults) { - t.Errorf("unexpected number of elements returned, expected: %q, got: %q", - expectedResults, res) - } - - for _, er := range expectedResults { - if !strtest.InSlice(res, er) { - t.Errorf("%q is missing in result %q", er, res) - } - } - - return -} diff --git a/git/git.go b/git/git.go index 9f46e479f..d88755f35 100644 --- a/git/git.go +++ b/git/git.go @@ -9,6 +9,9 @@ import ( "github.com/simplesurance/baur/exec" ) +// ErrNotExist means that one or more files do not exist +var ErrNotExist = errors.New("file(s) do not exist") + // CommitID return the commit id of HEAD by running git rev-parse in the passed // directory func CommitID(dir string) (string, error) { @@ -31,6 +34,7 @@ func CommitID(dir string) (string, error) { // LsFiles runs git ls-files in dir, passes args as argument and returns the // output +// If no files match, ErrNotExist is returned func LsFiles(dir, args string) (string, error) { cmd := "git ls-files --error-unmatch " + args @@ -41,8 +45,7 @@ func LsFiles(dir, args string) (string, error) { if exitCode != 0 { if strings.Contains(out, "did not match any file(s)") { - splt := strings.Split(out, "Did you forget to 'git add'") - return "", errors.New(strings.Replace(splt[0], "\n", " ", -1)) + return "", ErrNotExist } return "", fmt.Errorf("%q exited with code %d, output: %q", cmd, exitCode, out) diff --git a/gitpaths.go b/gitpaths.go deleted file mode 100644 index 301a330a9..000000000 --- a/gitpaths.go +++ /dev/null @@ -1,62 +0,0 @@ -package baur - -import ( - "path/filepath" - "strings" - - "github.com/simplesurance/baur/fs" - "github.com/simplesurance/baur/git" -) - -//GitPaths resolves multiple git filepath patterns to paths in the filesystem. -type GitPaths struct { - repositoryRootPath string - relAppPath string - paths []string -} - -// NewGitPaths returns a new GitPaths -func NewGitPaths(repositoryRootPath, relAppPath string, gitPaths []string) *GitPaths { - return &GitPaths{ - repositoryRootPath: repositoryRootPath, - relAppPath: relAppPath, - paths: gitPaths, - } -} - -// Resolve returns a list of files that are matching it's path -func (g *GitPaths) Resolve() ([]BuildInput, error) { - baseDir := filepath.Join(g.repositoryRootPath, g.relAppPath) - - arg := strings.Join(g.paths, " ") - out, err := git.LsFiles(baseDir, arg) - if err != nil { - return nil, err - } - - paths := strings.Split(out, "\n") - res := make([]BuildInput, 0, len(paths)) - - for _, p := range paths { - isFile, err := fs.IsFile(filepath.Join(baseDir, p)) - if err != nil { - return nil, err - } - - if isFile { - res = append(res, NewFile(g.repositoryRootPath, filepath.Join(g.relAppPath, p))) - } - } - - return res, nil -} - -// Type returns the type of resolver -func (g *GitPaths) Type() string { - return "GitFiles" -} - -// String returns the path to resolve -func (g *GitPaths) String() string { - return strings.Join(g.paths, ", ") -} diff --git a/gosrcdir.go b/gosrcdir.go deleted file mode 100644 index 609464f7d..000000000 --- a/gosrcdir.go +++ /dev/null @@ -1,79 +0,0 @@ -package baur - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/simplesurance/baur/fs" - "github.com/simplesurance/baur/golang" -) - -// GoSrcDirs resolves Golang source files in directories to files including -// resolving all imports to files -type GoSrcDirs struct { - repositoryRootPath string - relAppPath string - paths []string - gopath string -} - -// NewGoSrcDirs returns a GoSrcDirs -func NewGoSrcDirs(repositoryRootPath, relAppPath, gopath string, paths []string) *GoSrcDirs { - return &GoSrcDirs{ - repositoryRootPath: repositoryRootPath, - relAppPath: relAppPath, - paths: paths, - gopath: gopath, - } -} - -// Resolve returns list of Go src files -func (g *GoSrcDirs) Resolve() ([]BuildInput, error) { - baseDir := filepath.Join(g.repositoryRootPath, g.relAppPath) - fullpaths := make([]string, 0, len(g.paths)) - - for _, p := range g.paths { - absPath := filepath.Join(baseDir, p) - - isDir, err := fs.IsDir(absPath) - if err != nil { - return nil, err - } - - if !isDir { - return nil, fmt.Errorf("%q is not a directory", p) - } - - fullpaths = append(fullpaths, absPath) - } - - absSrcPaths, err := golang.SrcFiles(g.gopath, fullpaths...) - if err != nil { - return nil, err - } - - res := make([]BuildInput, 0, len(absSrcPaths)) - for _, p := range absSrcPaths { - relPath, err := filepath.Rel(g.repositoryRootPath, p) - if err != nil { - return nil, errors.Wrapf(err, "converting %q to relpath with basedir %q failed", p, g.repositoryRootPath) - } - - res = append(res, NewFile(g.repositoryRootPath, relPath)) - } - - return res, nil -} - -// Type returns the type of resolver -func (g *GoSrcDirs) Type() string { - return "GolangSources" -} - -// String returns the GoPath and Paths to resolve -func (g *GoSrcDirs) String() string { - return fmt.Sprintf("GOPATH: \"%s\", Paths: \"%s\"", g.gopath, strings.Join(g.paths, ", ")) -} diff --git a/resolve/gitpath/gitpaths.go b/resolve/gitpath/gitpaths.go new file mode 100644 index 000000000..d18e9a573 --- /dev/null +++ b/resolve/gitpath/gitpaths.go @@ -0,0 +1,56 @@ +package gitpath + +import ( + "path/filepath" + "strings" + + "github.com/simplesurance/baur/fs" + "github.com/simplesurance/baur/git" +) + +// Resolver resolves one or more git glob paths in a git repository by running +// git ls-files. +// Glob path only resolve to files that are tracked in the repository. +type Resolver struct { + path string + globs []string +} + +// NewResolver returns a resolver that resolves the passed git glob paths to absolute +// paths +func NewResolver(path string, globs ...string) *Resolver { + return &Resolver{ + path: path, + globs: globs, + } +} + +// Resolve the glob paths to absolute file paths by calling +// git ls-files +func (r *Resolver) Resolve() ([]string, error) { + arg := strings.Join(r.globs, " ") + out, err := git.LsFiles(r.path, arg) + if err != nil { + return nil, err + } + + relPaths := strings.Split(out, "\n") + res := make([]string, 0, len(relPaths)) + + for _, relPath := range relPaths { + absPath := filepath.Join(r.path, relPath) + + isFile, err := fs.IsFile(absPath) + if err != nil { + return nil, err + } + + if !isFile { + continue + } + + res = append(res, absPath) + } + + return res, nil +} diff --git a/resolve/glob/glob.go b/resolve/glob/glob.go new file mode 100644 index 000000000..c6bca657e --- /dev/null +++ b/resolve/glob/glob.go @@ -0,0 +1,165 @@ +package glob + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/simplesurance/baur/fs" +) + +// Resolver resolves a glob path to files. The functionality is the same then +// filepath.Glob() with the addition that '**' is supported to match files +// directories recursively. +type Resolver struct { + path string + glob string +} + +// NewResolver returns a resolver that resolves glob relative to path +func NewResolver(path, glob string) *Resolver { + return &Resolver{ + path: path, + glob: glob, + } +} + +// Resolve returns absolute paths to files that specify the glob path +func (r *Resolver) Resolve() ([]string, error) { + absGlobPath := filepath.Join(r.path, r.glob) + + return glob(absGlobPath) +} + +func findAllDirsNoDups(result map[string]struct{}, path string) error { + isDir, err := fs.IsDir(path) + if err != nil { + return errors.Wrapf(err, "IsDir(%s) failed", path) + } + + if !isDir { + return nil + } + + path, err = filepath.EvalSymlinks(path) + if err != nil { + return errors.Wrapf(err, "resolving symlinks in %q failed", path) + } + + if _, exist := result[path]; exist { + return nil + } + result[path] = struct{}{} + + globPath := filepath.Join(path, "*") + rootGlob, err := filepath.Glob(globPath) // is filepath.Walk() faster? + if err != nil { + return errors.Wrapf(err, "glob of %q failed", globPath) + } + + for _, path := range rootGlob { + err = findAllDirsNoDups(result, path) + if err != nil { + return err + } + } + + return nil +} + +// findAllDirs returns recursively all diretories in path, including the +// passed path dir +func findAllDirs(path string) ([]string, error) { + resultMap := map[string]struct{}{} + + err := findAllDirsNoDups(resultMap, path) + if err != nil { + return nil, err + } + + res := make([]string, 0, len(resultMap)) + for p := range resultMap { + res = append(res, p) + } + + return res, nil +} + +// expandDoubleStarGlob takes a glob path containing '**' and returns a list of +// paths were ** is expanded recursively to all matching directories. If '**' +// is the last part in the path, the returned paths will end in '/*' to glob +// match all files in those directories +func expandDoubleStarGlob(absGlobPath string) ([]string, error) { + spl := strings.Split(absGlobPath, "**") + if len(spl) < 2 { + return nil, fmt.Errorf("%q does not contain '**'", absGlobPath) + } + + basePath := spl[0] + glob := spl[1] + + if len(glob) == 0 { + glob = "*" + } + + dirs, err := findAllDirs(basePath) + if err != nil { + return nil, err + } + + for i := range dirs { + dirs[i] = filepath.Join(dirs[i], glob) + } + + return dirs, nil +} + +// glob is similar then filepath.Glob() with 2 Exceptions: +//- it also supports '**' to match files and directories recursively +//- it and only returns paths to files, no directory paths +// If a Glob doesn't match any files an empty []string is returned and error is +// nil +func glob(path string) ([]string, error) { + var globPaths []string + + if strings.Contains(path, "**") { + expandedPaths, err := expandDoubleStarGlob(path) + if err != nil { + return nil, errors.Wrap(err, "expanding '**' failed") + } + + globPaths = expandedPaths + } else { + globPaths = []string{path} + } + + paths := make([]string, 0, len(globPaths)) + for _, globPath := range globPaths { + path, err := filepath.Glob(globPath) + if err != nil { + return nil, err + } + + if path == nil { + continue + } + + paths = append(paths, path...) + } + + res := make([]string, 0, len(paths)) + for _, p := range paths { + isFile, err := fs.IsFile(p) + if err != nil { + return nil, errors.Wrapf(err, "resolved path %q does not exist", p) + } + + if isFile { + res = append(res, p) + } + } + + return res, nil +} diff --git a/fileglobpath_test.go b/resolve/glob/glob_test.go similarity index 72% rename from fileglobpath_test.go rename to resolve/glob/glob_test.go index ae0592f73..24a311e7c 100644 --- a/fileglobpath_test.go +++ b/resolve/glob/glob_test.go @@ -1,4 +1,4 @@ -package baur +package glob import ( "os" @@ -9,6 +9,41 @@ import ( "github.com/simplesurance/baur/testutils/strtest" ) +func Test_FindAllSubDirs(t *testing.T) { + tempdir, cleanupFunc := fstest.CreateTempDir(t) + defer cleanupFunc() + + expectedResults := []string{ + tempdir, + filepath.Join(tempdir, "1"), + filepath.Join(tempdir, "1/2"), + filepath.Join(tempdir, "1/2/3/"), + } + + err := os.MkdirAll(filepath.Join(tempdir, "1/2/3"), os.ModePerm) + if err != nil { + t.Fatal("creating subdirectories failed:", err) + } + + res, err := findAllDirs(tempdir) + if err != nil { + t.Fatal(err) + } + + if len(res) != len(expectedResults) { + t.Errorf("unexpected number of elements returned, expected: %q, got: %q", + expectedResults, res) + } + + for _, er := range expectedResults { + if !strtest.InSlice(res, er) { + t.Errorf("%q is missing in result %q", er, res) + } + } + + return +} + func createFiles(t *testing.T, basedir string, paths []string) { for _, p := range paths { fullpath := filepath.Join(basedir, p) @@ -21,12 +56,7 @@ func createFiles(t *testing.T, basedir string, paths []string) { } } -func checkFilesInResolvedFiles(t *testing.T, tempdir string, resolvedBuildInput []BuildInput, tc *testcase) { - resolvedFiles := []*File{} - for _, i := range resolvedBuildInput { - resolvedFiles = append(resolvedFiles, i.(*File)) - } - +func checkFilesInResolvedFiles(t *testing.T, tempdir string, resolvedFiles []string, tc *testcase) { if len(resolvedFiles) != len(tc.expectedMatches) { t.Errorf("resolved to %d files (%v), expected %d (%+v)", len(resolvedFiles), resolvedFiles, @@ -34,9 +64,14 @@ func checkFilesInResolvedFiles(t *testing.T, tempdir string, resolvedBuildInput } for _, e := range resolvedFiles { - if !strtest.InSlice(tc.expectedMatches, e.RepoRelPath()) { + relPath, err := filepath.Rel(tempdir, e) + if err != nil { + t.Errorf("getting Relpath of %q to %q failed", e, tempdir) + } + + if !strtest.InSlice(tc.expectedMatches, relPath) { t.Errorf("%q (%q) was returned but is not in expected return slice (%+v), testcase: %+v", - e, e.RepoRelPath(), tc.expectedMatches, tc) + e, relPath, tc.expectedMatches, tc) } } } @@ -49,7 +84,6 @@ type testcase struct { } func Test_Resolve(t *testing.T) { - testcases := []*testcase{ &testcase{ files: []string{ @@ -152,7 +186,7 @@ func Test_Resolve(t *testing.T) { createFiles(t, tempdir, tc.files) - fs := NewFileGlobPath(tempdir, ".", tc.fileSrcGlobPath) + fs := NewResolver(tempdir, tc.fileSrcGlobPath) resolvedFiles, err := fs.Resolve() if err != nil { t.Fatal("resolving glob path:", err) diff --git a/golang/golang.go b/resolve/gosource/gosource.go similarity index 77% rename from golang/golang.go rename to resolve/gosource/gosource.go index dcbc41262..fa3081da5 100644 --- a/golang/golang.go +++ b/resolve/gosource/gosource.go @@ -1,7 +1,4 @@ -// Package golang determines all Go Source files that are imported by a Go-Files -// in a directory. -// Most of the code is based on a slightly modified version of https://github.com/rogpeppe/showdeps -package golang +package gosource import ( "os" @@ -13,19 +10,41 @@ import ( "github.com/rogpeppe/godeps/build" ) -// SrcFiles returns the Go source files in the passed directories plus all +// Resolver determines all Go Source files that are imported by Go-Files +// in a directory. +// The code is based on https://github.com/rogpeppe/showdeps +type Resolver struct { + rootPath string + goPath string + goDirs []string +} + +// NewResolver returns a resolver that resolves all go source files in the +// GoDirs and it's imports to filepaths. +// If gopath is an empty string, gopath is determined automatically. +func NewResolver(path, gopath string, goDirs ...string) *Resolver { + return &Resolver{ + rootPath: path, + goPath: gopath, + goDirs: goDirs, + } +} + +// Resolve returns the Go source files in the passed directories plus all // source files of the imported packages. // Testfiles and stdlib dependencies are ignored. -func SrcFiles(gopath string, dirs ...string) ([]string, error) { +func (r *Resolver) Resolve() ([]string, error) { var allFiles []string ctx := build.Default - if len(gopath) > 0 { - ctx.GOPATH = gopath + if len(r.goPath) > 0 { + ctx.GOPATH = r.goPath } - for _, d := range dirs { - files, err := resolve(ctx, d) + for _, dir := range r.goDirs { + absPath := filepath.Join(r.rootPath, dir) + + files, err := resolve(ctx, absPath) if err != nil { return nil, err } diff --git a/resolve/resolver.go b/resolve/resolver.go new file mode 100644 index 000000000..897807499 --- /dev/null +++ b/resolve/resolver.go @@ -0,0 +1,7 @@ +package resolve + +// Resolver specifies an interface to resolve abstract file specifications like +// glob paths or go packages to file paths +type Resolver interface { + Resolve() ([]string, error) +}