Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup some methods on rawCopy/SyncCmdArgs #2929

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+29 −45
Open

Cleanup some methods on rawCopy/SyncCmdArgs #2929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions cmd/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ type rawCopyCmdArgs struct {
deleteDestinationFileIfNecessary bool
}

func (raw *rawCopyCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
func parsePatterns(pattern string) (cookedPatterns []string) {
cookedPatterns = make([]string, 0)
rawPatterns := strings.Split(pattern, ";")
for _, pattern := range rawPatterns {
Expand Down Expand Up @@ -228,8 +228,8 @@ func blockSizeInBytes(rawBlockSizeInMiB float64) (int64, error) {

// returns result of stripping and if striptopdir is enabled
// if nothing happens, the original source is returned
func (raw rawCopyCmdArgs) stripTrailingWildcardOnRemoteSource(location common.Location) (result string, stripTopDir bool, err error) {
result = raw.src
func stripTrailingWildcardOnRemoteSource(source string, location common.Location) (result string, stripTopDir bool, err error) {
result = source
resourceURL, err := url.Parse(result)
gURLParts := common.NewGenericResourceURLParts(*resourceURL, location)

Expand Down Expand Up @@ -298,7 +298,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {

// Check if source has a trailing wildcard on a URL
if fromTo.From().IsRemote() {
tempSrc, cooked.StripTopDir, err = raw.stripTrailingWildcardOnRemoteSource(fromTo.From())
tempSrc, cooked.StripTopDir, err = stripTrailingWildcardOnRemoteSource(raw.src, fromTo.From())

if err != nil {
return cooked, err
Expand Down Expand Up @@ -410,7 +410,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
}

// warn on exclude unsupported wildcards here. Include have to be later, to cover list-of-files
raw.warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)

// unbuffered so this reads as we need it to rather than all at once in bulk
listChan := make(chan string)
Expand All @@ -433,7 +433,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
addToChannel := func(v string, paramName string) {
// empty strings should be ignored, otherwise the source root itself is selected
if len(v) > 0 {
raw.warnIfHasWildcard(includeWarningOncer, paramName, v)
warnIfHasWildcard(includeWarningOncer, paramName, v)
listChan <- v
}
}
Expand Down Expand Up @@ -477,7 +477,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
}

// This occurs much earlier than the other include or exclude filters. It would be preferable to move them closer later on in the refactor.
includePathList := raw.parsePatterns(raw.includePath)
includePathList := parsePatterns(raw.includePath)

for _, v := range includePathList {
addToChannel(v, "include-path")
Expand Down Expand Up @@ -868,19 +868,19 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
}

// parse the filter patterns
cooked.IncludePatterns = raw.parsePatterns(raw.include)
cooked.ExcludePatterns = raw.parsePatterns(raw.exclude)
cooked.ExcludePathPatterns = raw.parsePatterns(raw.excludePath)
cooked.excludeContainer = raw.parsePatterns(raw.excludeContainer)
cooked.IncludePatterns = parsePatterns(raw.include)
cooked.ExcludePatterns = parsePatterns(raw.exclude)
cooked.ExcludePathPatterns = parsePatterns(raw.excludePath)
cooked.excludeContainer = parsePatterns(raw.excludeContainer)

if (raw.includeFileAttributes != "" || raw.excludeFileAttributes != "") && fromTo.From() != common.ELocation.Local() {
return cooked, errors.New("cannot check file attributes on remote objects")
}
cooked.IncludeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
cooked.ExcludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
cooked.IncludeFileAttributes = parsePatterns(raw.includeFileAttributes)
cooked.ExcludeFileAttributes = parsePatterns(raw.excludeFileAttributes)

cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
cooked.includeRegex = parsePatterns(raw.includeRegex)
cooked.excludeRegex = parsePatterns(raw.excludeRegex)

cooked.dryrunMode = raw.dryrun

Expand All @@ -903,7 +903,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
var excludeWarningOncer = &sync.Once{}
var includeWarningOncer = &sync.Once{}

func (raw *rawCopyCmdArgs) warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
func warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
if strings.Contains(value, "*") || strings.Contains(value, "?") {
oncer.Do(func() {
glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
Expand Down
34 changes: 10 additions & 24 deletions cmd/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,22 +102,8 @@ type rawSyncCmdArgs struct {
deleteDestinationFileIfNecessary bool
}

func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
cookedPatterns = make([]string, 0)
rawPatterns := strings.Split(pattern, ";")
for _, pattern := range rawPatterns {

// skip the empty patterns
if len(pattern) != 0 {
cookedPatterns = append(cookedPatterns, pattern)
}
}

return
}

// it is assume that the given url has the SAS stripped, and safe to print
func (raw *rawSyncCmdArgs) validateURLIsNotServiceLevel(url string, location common.Location) error {
func validateURLIsNotServiceLevel(url string, location common.Location) error {
srcLevel, err := DetermineLocationLevel(url, location, true)
if err != nil {
return err
Expand Down Expand Up @@ -213,15 +199,15 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {

// we do not support service level sync yet
if cooked.fromTo.From().IsRemote() {
err = raw.validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
err = validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
if err != nil {
return cooked, err
}
}

// we do not support service level sync yet
if cooked.fromTo.To().IsRemote() {
err = raw.validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
err = validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
if err != nil {
return cooked, err
}
Expand Down Expand Up @@ -265,13 +251,13 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
}

// parse the filter patterns
cooked.includePatterns = raw.parsePatterns(raw.include)
cooked.excludePatterns = raw.parsePatterns(raw.exclude)
cooked.excludePaths = raw.parsePatterns(raw.excludePath)
cooked.includePatterns = parsePatterns(raw.include)
cooked.excludePatterns = parsePatterns(raw.exclude)
cooked.excludePaths = parsePatterns(raw.excludePath)

// parse the attribute filter patterns
cooked.includeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
cooked.excludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
cooked.includeFileAttributes = parsePatterns(raw.includeFileAttributes)
cooked.excludeFileAttributes = parsePatterns(raw.excludeFileAttributes)

cooked.preserveSMBInfo = raw.preserveSMBInfo && areBothLocationsSMBAware(cooked.fromTo)

Expand Down Expand Up @@ -367,8 +353,8 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {

cooked.mirrorMode = raw.mirrorMode

cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
cooked.includeRegex = parsePatterns(raw.includeRegex)
cooked.excludeRegex = parsePatterns(raw.excludeRegex)

cooked.dryrunMode = raw.dryrun

Expand Down
8 changes: 3 additions & 5 deletions cmd/zt_generic_filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ var _ = chk.Suite(&genericFilterSuite{})
func TestIncludeFilter(t *testing.T) {
a := assert.New(t)
// set up the filters
raw := rawSyncCmdArgs{}
includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
includePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
includeFilter := buildIncludeFilters(includePatternList)[0]

// test the positive cases
Expand All @@ -60,8 +59,7 @@ func TestIncludeFilter(t *testing.T) {
func TestExcludeFilter(t *testing.T) {
a := assert.New(t)
// set up the filters
raw := rawSyncCmdArgs{}
excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
excludePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
excludeFilterList := buildExcludeFilters(excludePatternList, false)

// test the positive cases
Expand Down Expand Up @@ -187,4 +185,4 @@ func findAmbiguousTime() (string, time.Time, time.Time, error) {
}

return "", time.Time{}, time.Time{}, noAmbiguousHourError
}
}
Loading