Merge pull request #200 from tonycai653/master

修复一些bug, 给listbucket2加无限重试的功能，用于列举大量删除文件的情况

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 2.3.2
+1. 修复fetch incorrect region
+2. 修复docs中文档格式显示问题
+3. 给listbucket2添加不限制重试次数的功能
+
 # 2.3.1
 1. batchdelete, batchchgm, batchchtype, batchmove, batchrename命令运行可导出失败，成功文件列表
 2. rput, fput, qupload上传支持设置回调用 

README.md

@@ -49,24 +49,30 @@ export PATH=$PATH:/home/jemy/tools
 
 ![windows-qshell-path-settings.png](http://devtools.qiniu.com/windows-qshell-path-settings.png)
 
-## 密钥设置
+## qshell使用
+
+1. 添加密钥和账户名称
 
 该工具有两类命令，一类需要鉴权，另一类不需要。
 
-需要鉴权的命令都需要依赖七牛账号下的 `AccessKey` 和 `SecretKey`。所以这类命令运行之前，需要使用 `account` 命令来设置下 `AccessKey` ，`SecretKey` 。
+需要鉴权的命令都需要依赖七牛账号下的 `AccessKey`, `SecretKey`和 `Name`。所以这类命令运行之前，需要使用 `account` 命令来添加 `AccessKey` ，`SecretKey`和`Name` 。
+`Name`是用户可以自定义的字符串，用来唯一表示AccessKey/SecretKey账户，qshell会对添加的每一个账户信息加密保存，可以使用自命令`user`进行切换，切换账户的时候，需要使用账户唯一标识
+`Name`。 
 
 ```
-$ qshell account ak sk name
+$ qshell account <Your AccessKey> <Your SecretKey> <Your Name>
 ```
 
 其中name表示该账号的名称, 如果ak, sk, name首字母是"-", 需要使用如下的方式添加账号, 这样避免把该项识别成命令行选项:
 
 ```
-$ qshell account -- ak sk name
+$ qshell account -- <Your AccessKey> <Your SecretKey> <Your Name>
 ```
 
 可以连续使用qshell account 添加账号ak, sk, name信息，qshell会保存这些账号的信息， 可以使用qshell user命令列举账号信息，在各个账号之间切换, 删除账号等
 
+2. 添加完账户后，就可以使用qshell上传，下载文件了
+
 ## 账户管理
 
 使用qshell user子命令可以用来管理记录的多账户信息。

cmd/root.go

@@ -86,5 +86,9 @@ func initConfig() {
 		viper.AddConfigPath(curUser.HomeDir)
 		viper.SetConfigName(".qshell")
 	}
-	viper.ReadInConfig()
+	if rErr := viper.ReadInConfig(); rErr != nil {
+		if _, ok := rErr.(viper.ConfigFileNotFoundError); !ok {
+			fmt.Fprintf(os.Stderr, "read config file: %v\n", rErr)
+		}
+	}
 }

cmd/rs.go

@@ -144,7 +144,7 @@ func init() {
 	lsBucketCmd2.Flags().StringVarP(&listMarker, "marker", "m", "", "list marker")
 	lsBucketCmd2.Flags().StringVarP(&prefix, "prefix", "p", "", "list by prefix")
 	lsBucketCmd2.Flags().StringVarP(&suffixes, "suffixes", "q", "", "list by key suffixes, separated by comma")
-	lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", 5, "max retries when error occurred")
+	lsBucketCmd2.Flags().IntVarP(&maxRetry, "max-retry", "x", 20, "max retries when error occurred")
 	lsBucketCmd2.Flags().StringVarP(&outFile, "out", "o", "", "output file")
 	lsBucketCmd2.Flags().StringVarP(&startDate, "start", "s", "", "start date with format yyyy-mm-dd-hh-MM-ss")
 	lsBucketCmd2.Flags().StringVarP(&endDate, "end", "e", "", "end date with format yyyy-mm-dd-hh-MM-ss")

cmd/rsbatch.go

@@ -109,7 +109,10 @@ var (
 
 func init() {
 	batchFetchCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "urls list file")
-	batchFetchCmd.Flags().IntVarP(&worker, "worker", "c", 5, "worker count")
+	batchFetchCmd.Flags().IntVarP(&worker, "worker", "c", 1, "worker count")
+	batchFetchCmd.Flags().StringVarP(&bsuccessFname, "success-list", "s", "", "file to save batch fetch success list")
+	batchFetchCmd.Flags().StringVarP(&bfailureFname, "failure-list", "e", "", "file to save batch fetch failure list")
+
 	batchStatCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
 	batchCopyCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
 	batchMoveCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", "input file")
@@ -195,11 +198,16 @@ func BatchFetch(cmd *cobra.Command, params []string) {
 	)
 	defer close(fItemChan)
 
-	go batchFetch(fItemChan)
+	fileExporter, fErr := iqshell.NewFileExporter(bsuccessFname, bfailureFname, "")
+	if fErr != nil {
+		fmt.Fprintf(os.Stderr, "create file exporter: %v\n", fErr)
+		os.Exit(1)
+	}
+	go batchFetch(fItemChan, fileExporter)
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		items := strings.Fields(line)
+		items := strings.Split(line, "\t")
 		if len(items) <= 0 {
 			continue
 		}
@@ -225,13 +233,24 @@ func BatchFetch(cmd *cobra.Command, params []string) {
 	}
 }
 
-func batchFetch(fItemChan chan *iqshell.FetchItem) {
+func batchFetch(fItemChan chan *iqshell.FetchItem, fileExporter *iqshell.FileExporter) {
 	for i := 0; i < worker; i++ {
 		go func() {
 			bm := iqshell.GetBucketManager()
 			for fetchItem := range fItemChan {
-				fetchResult, fErr := bm.Fetch(fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
-				fmt.Println(fetchResult, fErr)
+				_, fErr := bm.Fetch(fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+				if fErr != nil {
+					fmt.Fprintf(os.Stderr, "fetch %s => %s:%s failed\n", fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+					if fileExporter != nil {
+						fileExporter.WriteToFailedWriter(fmt.Sprintf("%s\t%s\t%v\n", fetchItem.RemoteUrl, fetchItem.Key, fErr))
+					}
+				} else {
+					fmt.Printf("fetch %s => %s:%s success\n", fetchItem.RemoteUrl, fetchItem.Bucket, fetchItem.Key)
+					if fileExporter != nil {
+						fileExporter.WriteToSuccessWriter(fmt.Sprintf("%s\t%s\n", fetchItem.RemoteUrl, fetchItem.Key))
+					}
+
+				}
 			}
 		}()
 	}

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 )
 
-var version = "v2.3.1"
+var version = "v2.3.2"
 
 var versionCmd = &cobra.Command{
 	Use:   "version",

docs/batchcopy.md

@@ -14,7 +14,7 @@ qshell batchcopy [--force] [--overwrite] [--success-list <SuccessFileName>] [--f
 # 帮助
 ```
 qshell batchcopy -h
-``**
+```
 
 # 鉴权
 

docs/batchdelete.md

@@ -11,7 +11,7 @@ qshell batchdelete [--force] [--success-list <SuccessFileName>] [--failure-list
 # 帮助
 ```
 qshell batchdelete -h
-``**
+```
 
 # 鉴权
 

docs/batchexpire.md

@@ -12,7 +12,7 @@ qshell batchexpire [--force] <Bucket> <-i KeyDeleteAfterDaysMapFile>
 # 帮助 
 ```
 qshell batchexpire -h
-``**
+```
 
 # 鉴权
 

docs/batchrename.md

@@ -9,7 +9,7 @@
 
 ```
 qshell batchrename [--force] [--overwrite] [--success-list <SuccessFileName>] [--failure-list <failureFileName>] <Bucket> [-i <OldNewKeyMapFile>]
-`****
+```
 
 # 鉴权
 

docs/batchsign.md

@@ -6,7 +6,7 @@
 
 ```
 qshell batchsign [<-i UrlListFile>] [-e <Deadline>]
-``**
+```
 
 # 鉴权
 

docs/batchstat.md

@@ -6,7 +6,7 @@
 
 ```
 qshell batchstat <Bucket> <-i KeyListFile>
-``**
+```
 
 # 鉴权
 

docs/listbucket2.md

@@ -15,9 +15,11 @@ Key\tSize\tHash\tPutTime\tMimeType\tFileType\tEndUser
 # 格式
 
 ```
-qshell listbucket2 [--prefix <Prefix> | --suffixes <suffixes1,suffixes2>] [--start <StartDate>] [--end <EndDate>] <Bucket> [-o <ListBucketResultFile>]
+qshell listbucket2 [--prefix <Prefix> | --suffixes <suffixes1,suffixes2>] [--start <StartDate>] [--max-retry <RetryCount>][--end <EndDate>] <Bucket> [-o <ListBucketResultFile>]
 ```
 
+选项max-retry， 默认列举出错的重试次数为20次，如果希望可以列举完文件列表，不限错误次数，可以设置max-retry为负数。
+
 （1）获取空间中所有的文件列表，这种情况下，可以直接指定 `Bucket` 参数和结果保存文件参数 `ListBucketResultFile` 即可。
 
 ```

iqshell/bucket.go

@@ -313,7 +313,6 @@ func GetBucketManager() *BucketManager {
 	}
 	mac := qbox.NewMac(account.AccessKey, account.SecretKey)
 	cfg := storage.Config{
-		IoHost:  viper.GetString("hosts.io_host"),
 		RsHost:  viper.GetString("hosts.rs_host"),
 		ApiHost: viper.GetString("hosts.api_host"),
 		RsfHost: viper.GetString("hosts.rsf_host"),

iqshell/list_bucket.go

@@ -56,7 +56,7 @@ func errorWarning(marker string, err error) {
 *@return listError
  */
 func (m *BucketManager) ListFiles(bucket, prefix, marker, listResultFile string) (retErr error) {
-	return m.ListBucket2(bucket, prefix, marker, listResultFile, "", time.Time{}, time.Time{}, nil, 5)
+	return m.ListBucket2(bucket, prefix, marker, listResultFile, "", time.Time{}, time.Time{}, nil, 20)
 }
 
 func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, delimiter string, startDate, endDate time.Time, suffixes []string, maxRetry int) (retErr error) {
@@ -82,14 +82,15 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli
 	notfilterTime := startDate.IsZero() && endDate.IsZero()
 	notfilterSuffix := len(suffixes) == 0
 
-	for i := 0; i < maxRetry; i++ {
+	c := 0
+	for {
 		entries, lErr := m.ListBucket(bucket, prefix, delimiter, marker)
 
 		if entries == nil && lErr == nil {
 			// no data
 			return
 		}
-		if retErr != nil {
+		if lErr != nil {
 			errorWarning(lastMarker, retErr)
 		}
 
@@ -145,6 +146,16 @@ func (m *BucketManager) ListBucket2(bucket, prefix, marker, listResultFile, deli
 		} else {
 			marker = lastMarker
 		}
+
+		if maxRetry >= 0 {
+			c++
+			if c > maxRetry {
+				break
+			}
+		}
+	}
+	if lastMarker != "" {
+		fmt.Println("Marker: ", lastMarker)
 	}
 	return
 }

main.go

@@ -11,4 +11,5 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
+
 }