lakectl local: implement list

cmd/lakectl/cmd/local_init.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/treeverse/lakefs/pkg/git"
+	"github.com/treeverse/lakefs/pkg/local"
 )
 
 const (
@@ -36,18 +37,22 @@ var localInitCmd = &cobra.Command{
 		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
 			DieErr(err)
 		}
-		if IndexExists(localPath) && !force {
+		exists, err := local.IndexExists(localPath)
+		if err != nil {
+			DieErr(err)
+		}
+		if exists && !force {
 			DieFmt("directory '%s' already linked to a lakefs path, run command with --force to overwrite", localPath)
 		}
 
 		// dereference
 		head := resolveCommitOrDie(cmd.Context(), getClient(), remote.Repository, remote.Ref)
-		err = WriteIndex(localPath, remote, head)
+		err = local.WriteIndex(localPath, remote, head)
 		if err != nil {
 			DieErr(err)
 		}
 
-		ignoreFile, err := git.Ignore(localPath, []string{localPath, IndexFileName}, []string{IndexFileName}, IgnoreMarker)
+		ignoreFile, err := git.Ignore(localPath, []string{localPath, local.IndexFileName}, []string{local.IndexFileName}, local.IgnoreMarker)
 		if err == nil {
 			fmt.Println("location added to", ignoreFile)
 		} else if !errors.Is(err, git.ErrNotARepository) {

cmd/lakectl/cmd/local_list.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"path/filepath"
+
+	"github.com/cockroachdb/errors"
+	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/spf13/cobra"
+	"github.com/treeverse/lakefs/pkg/git"
+	"github.com/treeverse/lakefs/pkg/local"
+)
+
+const (
+	localListMinArgs = 0
+	localListMaxArgs = 1
+
+	indicesListTemplate = `{{.IndicesListTable | table -}}`
+)
+
+var localListCmd = &cobra.Command{
+	Use:   "list [directory]",
+	Short: "find and list directories that are synced with lakeFS",
+	Args:  cobra.RangeArgs(localListMinArgs, localListMaxArgs),
+	Run: func(cmd *cobra.Command, args []string) {
+		dir := "."
+		if len(args) > 0 {
+			dir = args[0]
+		}
+		abs, err := filepath.Abs(dir)
+		if err != nil {
+			DieErr(err)
+		}
+		gitRoot, err := git.GetRepositoryPath(abs)
+		if err == nil {
+			abs = gitRoot
+		} else if !(errors.Is(err, git.ErrNotARepository) || errors.Is(err, git.ErrNoGit)) { // allow support in environments with no git
+			DieErr(err)
+		}
+
+		dirs, err := local.FindIndices(abs)
+		if err != nil {
+			DieErr(err)
+		}
+
+		var rows [][]interface{}
+		for _, d := range dirs {
+			idx, err := local.ReadIndex(d)
+			if err != nil {
+				DieErr(err)
+			}
+			remote, err := idx.GetCurrentURI()
+			if err != nil {
+				DieErr(err)
+			}
+			rows = append(rows, table.Row{d, remote, idx.AtHead})
+		}
+		data := struct {
+			IndicesListTable *Table
+		}{
+			IndicesListTable: &Table{
+				Headers: []interface{}{
+					"Directory",
+					"Remote URI",
+					"Last synced HEAD",
+				},
+				Rows: rows,
+			},
+		}
+		Write(indicesListTemplate, data)
+	},
+}
+
+//nolint:gochecknoinits
+func init() {
+	localCmd.AddCommand(localListCmd)
+}

docs/reference/cli.md

@@ -2616,6 +2616,23 @@ lakectl local init <path uri> [directory] [flags]
 
 
 
+### lakectl local list
+
+find and list directories that are synced with lakeFS
+
+```
+lakectl local list [directory] [flags]
+```
+
+#### Options
+{:.no_toc}
+
+```
+  -h, --help   help for list
+```
+
+
+
 ### lakectl log
 
 Show log of commits

pkg/fileutil/io.go

@@ -1,6 +1,16 @@
 package fileutil
 
-import "os"
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+const (
+	DefaultDirectoryMask = 0o755
+)
 
 // IsDir Returns true if p is a directory, otherwise false
 func IsDir(p string) (bool, error) {
@@ -10,3 +20,34 @@ func IsDir(p string) (bool, error) {
 	}
 	return stat.IsDir(), nil
 }
+
+func FindInParents(path, filename string) (string, error) {
+	var lookup string
+	fullPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+	for fullPath != string(filepath.Separator) && fullPath != filepath.VolumeName(fullPath) {
+		info, err := os.Stat(fullPath)
+		if errors.Is(err, fs.ErrNotExist) {
+			return "", fmt.Errorf("%s: %w", fullPath, fs.ErrNotExist)
+		} else if err != nil {
+			return "", err
+		}
+		if !info.IsDir() {
+			// find filename here
+			lookup = filepath.Join(filepath.Dir(fullPath), filename)
+		} else {
+			lookup = filepath.Join(fullPath, filename)
+		}
+		_, err = os.Stat(lookup)
+		if os.IsNotExist(err) {
+			fullPath = filepath.Dir(fullPath)
+			continue
+		} else if err != nil {
+			return "", err
+		}
+		return lookup, nil
+	}
+	return "", nil
+}

pkg/fileutil/io_test.go

@@ -0,0 +1,85 @@
+package fileutil_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/treeverse/lakefs/pkg/fileutil"
+)
+
+func TestFindInParents(t *testing.T) {
+	root := t.TempDir()
+	dirTree := filepath.Join(root, "foo", "bar", "baz", "taz")
+	require.NoError(t, os.MkdirAll(dirTree, fileutil.DefaultDirectoryMask))
+	t.Run("does not exist", func(t *testing.T) {
+		found, err := fileutil.FindInParents(root, ".doesnotexist21348329043289")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if found != "" {
+			t.Errorf("expected found to be empty, got %v", found)
+		}
+	})
+
+	tests := []struct {
+		name     string
+		deep     string
+		filename string
+		filepath string
+		find     bool
+	}{
+		{
+			name:     "find_at_leaf",
+			deep:     filepath.Join(root, "foo", "bar", "baz"),
+			filename: "some_file0",
+			filepath: filepath.Join(root, "foo", "bar", "baz", "some_file0"),
+			find:     true,
+		},
+		{
+			name:     "find_at_root",
+			deep:     filepath.Join(root, "foo", "bar", "baz"),
+			filename: "some_file1",
+			filepath: filepath.Join(root, "some_file1"),
+			find:     true,
+		},
+		{
+			name:     "find_at_subpath",
+			deep:     filepath.Join(root, "foo", "bar", "baz", "taz"),
+			filename: "some_file2",
+			filepath: filepath.Join(root, "foo", "some_file2"),
+			find:     true,
+		},
+		{
+			name:     "not_found_above",
+			deep:     filepath.Join(root, "foo", "bar", "baz"),
+			filename: "some_file3",
+			filepath: filepath.Join(root, "foo", "bar", "baz", "taz", "some_file3"),
+			find:     false,
+		},
+		{
+			name:     "doesnt_exist",
+			deep:     filepath.Join(root, "foo", "bar", "baz"),
+			filename: ".doesnotexist21348329043289",
+			filepath: filepath.Join(root, "foo", "bar", "some_file4"),
+			find:     false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f, err := os.Create(tt.filepath)
+			require.NoError(t, err)
+			require.NoError(t, f.Close())
+
+			found, err := fileutil.FindInParents(tt.deep, tt.filename)
+			require.NoError(t, err)
+			if tt.find {
+				require.Equal(t, tt.filepath, found)
+			} else {
+				require.Equal(t, "", found)
+			}
+		})
+	}
+}

pkg/git/errors.go

@@ -7,4 +7,5 @@ import (
 var (
 	ErrGitError       = errors.New("git error")
 	ErrNotARepository = errors.New("not a git repository")
+	ErrNoGit          = errors.New("no git support")
 )

pkg/git/git.go

@@ -21,6 +21,10 @@ const (
 )
 
 func git(dir string, args ...string) (string, error) {
+	_, err := exec.LookPath("git") // assume git is in path, otherwise consider as not having git support
+	if err != nil {
+		return "", ErrNoGit
+	}
 	cmd := exec.Command("git", args...)
 	cmd.Dir = dir
 	out, err := cmd.CombinedOutput()