Skip to content

Commit

Permalink
updated
Browse files Browse the repository at this point in the history
  • Loading branch information
JiamingMai committed Oct 20, 2023
1 parent 3536ae5 commit 332ad1a
Showing 1 changed file with 57 additions and 57 deletions.
114 changes: 57 additions & 57 deletions cli/src/alluxio.org/cli/cmd/job/load.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,46 +12,46 @@
package job

import (
"github.com/palantir/stacktrace"
"github.com/spf13/cobra"
"github.com/palantir/stacktrace"
"github.com/spf13/cobra"

"alluxio.org/cli/cmd/names"
"alluxio.org/cli/env"
"alluxio.org/cli/cmd/names"
"alluxio.org/cli/env"
)

var Load = &LoadCommand{
BaseJobCommand: &BaseJobCommand{
BaseJavaCommand: &env.BaseJavaCommand{
CommandName: "load",
JavaClassName: names.FileSystemShellJavaClass,
},
},
BaseJobCommand: &BaseJobCommand{
BaseJavaCommand: &env.BaseJavaCommand{
CommandName: "load",
JavaClassName: names.FileSystemShellJavaClass,
},
},
}

type LoadCommand struct {
*BaseJobCommand
path string
*BaseJobCommand
path string

bandwidth string
verify bool
partialListing bool
metadataOnly bool
skipIfExists bool
bandwidth string
verify bool
partialListing bool
metadataOnly bool
skipIfExists bool
fileFilterRegx string
}

func (c *LoadCommand) Base() *env.BaseJavaCommand {
return c.BaseJavaCommand
return c.BaseJavaCommand
}

func (c *LoadCommand) ToCommand() *cobra.Command {
cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
Use: Load.CommandName,
Short: "Submit or manage load jobs",
Long: `The load command moves data from the under storage system into Alluxio storage.
cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
Use: Load.CommandName,
Short: "Submit or manage load jobs",
Long: `The load command moves data from the under storage system into Alluxio storage.
For example, load can be used to prefetch data for analytics jobs.
If load is run on a directory, files in the directory will be recursively loaded.`,
Example: `# Submit a load job
Example: `# Submit a load job
$ ./bin/alluxio job load --path /path --submit
# View the progress of a submitted job
Expand All @@ -68,46 +68,46 @@ Progress for loading path '/path':
# Stop a submitted job
$ ./bin/alluxio job load --path /path --stop`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(args)
},
})
const path = "path"
cmd.Flags().StringVar(&c.path, path, "", "[all] Source path of load operation")
cmd.MarkFlagRequired(path)
c.AttachOperationFlags(cmd)
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
return c.Run(args)
},
})
const path = "path"
cmd.Flags().StringVar(&c.path, path, "", "[all] Source path of load operation")
cmd.MarkFlagRequired(path)
c.AttachOperationFlags(cmd)

cmd.Flags().StringVar(&c.bandwidth, "bandwidth", "", "[submit] Single worker read bandwidth limit")
cmd.Flags().BoolVar(&c.verify, "verify", false, "[submit] Run verification when load finishes and load new files if any")
cmd.Flags().BoolVar(&c.partialListing, "partial-listing", false, "[submit] Use partial directory listing, initializing load before reading the entire directory but cannot report on certain progress details")
cmd.Flags().BoolVar(&c.metadataOnly, "metadata-only", false, "[submit] Only load file metadata")
cmd.Flags().BoolVar(&c.skipIfExists, "skip-if-exists", false, "[submit] Skip existing fullly cached files")
cmd.Flags().StringVar(&c.bandwidth, "bandwidth", "", "[submit] Single worker read bandwidth limit")
cmd.Flags().BoolVar(&c.verify, "verify", false, "[submit] Run verification when load finishes and load new files if any")
cmd.Flags().BoolVar(&c.partialListing, "partial-listing", false, "[submit] Use partial directory listing, initializing load before reading the entire directory but cannot report on certain progress details")
cmd.Flags().BoolVar(&c.metadataOnly, "metadata-only", false, "[submit] Only load file metadata")
cmd.Flags().BoolVar(&c.skipIfExists, "skip-if-exists", false, "[submit] Skip existing fullly cached files")
cmd.Flags().StringVar(&c.fileFilterRegx, "file-filter-regx", "", "[submit] Skip files that match the regx pattern")
return cmd
return cmd
}

func (c *LoadCommand) Run(_ []string) error {
opWithArgs, err := c.OperationWithArgs()
if err != nil {
return stacktrace.Propagate(err, "error parsing operation")
}
javaArgs := []string{"load", c.path}
javaArgs = append(javaArgs, opWithArgs...)
if c.bandwidth != "" {
javaArgs = append(javaArgs, "--bandwidth", c.bandwidth)
}
if c.partialListing {
javaArgs = append(javaArgs, "--partial-listing")
}
if c.metadataOnly {
javaArgs = append(javaArgs, "--metadata-only")
}
if c.skipIfExists {
javaArgs = append(javaArgs, "--skip-if-exists")
}
opWithArgs, err := c.OperationWithArgs()
if err != nil {
return stacktrace.Propagate(err, "error parsing operation")
}
javaArgs := []string{"load", c.path}
javaArgs = append(javaArgs, opWithArgs...)
if c.bandwidth != "" {
javaArgs = append(javaArgs, "--bandwidth", c.bandwidth)
}
if c.partialListing {
javaArgs = append(javaArgs, "--partial-listing")
}
if c.metadataOnly {
javaArgs = append(javaArgs, "--metadata-only")
}
if c.skipIfExists {
javaArgs = append(javaArgs, "--skip-if-exists")
}
if c.fileFilterRegx != "" {
javaArgs = append(javaArgs, "--file-filter-regx")
}
return c.Base().Run(javaArgs)
return c.Base().Run(javaArgs)
}

0 comments on commit 332ad1a

Please sign in to comment.