Merge pull request #608 from YangSen-qn/master

pfop: new type
qiniu · Sep 13, 2024 · 1b7ecc5 · 1b7ecc5
2 parents 109f868 + 1ddc3c4
commit 1b7ecc5
Show file tree

Hide file tree

Showing 10 changed files with 173 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,8 @@
 # Changelog
+## 7.16.0（2024-09-12)
+* 支持闲时任务
+* 移除已下线区域相关域名
+
 ## 7.15.1（2024-05-29)
 * 处理在构造 Download URL 时 Key 前缀为 / 的情况
 

diff --git a/README.md b/README.md
@@ -13,12 +13,12 @@
 <dependency>
   <groupId>com.qiniu</groupId>
   <artifactId>qiniu-java-sdk</artifactId>
-  <version>[7.15.0, 7.15.99]</version>
+  <version>[7.16.0, 7.16.99]</version>
 </dependency>
 ```
 或者 Gradle:
 ```groovy
-implementation 'com.qiniu:qiniu-java-sdk:7.15.+'
+implementation 'com.qiniu:qiniu-java-sdk:7.16.+'
 ```
 
 ## 运行环境

diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java
@@ -10,7 +10,7 @@ public final class Constants {
     /**
      * 版本号
      */
-    public static final String VERSION = "7.15.1";
+    public static final String VERSION = "7.16.0";
     /**
      * 块大小，不能改变
      */

diff --git a/src/main/java/com/qiniu/processing/OperationManager.java b/src/main/java/com/qiniu/processing/OperationManager.java
@@ -80,7 +80,7 @@ public String pfop(String bucket, String key, String fops) throws QiniuException
      * @param bucket 空间名
      * @param key    文件名
      * @param fops   fops指令，如果有多个指令，需要使用分号(;)进行拼接，例如 avthumb/mp4/xxx|saveas/xxx;vframe/jpg/xxx|saveas/xxx
-     * @param params notifyURL、force、pipeline 等参数
+     * @param params notifyURL、force、pipeline、type等参数
      * @return persistentId 请求返回的任务ID，可以根据该ID查询任务状态
      * @throws QiniuException 触发失败异常，包含错误响应等信息
      *                        <a href="http://developer.qiniu.com/dora/api/persistent-data-processing-pfop"> 相关链接 </a>
@@ -154,8 +154,34 @@ public String pfop(String bucket, String key, String fops, String pipeline, bool
      */
     public String pfop(String bucket, String key, String fops, String pipeline, String notifyURL, boolean force)
             throws QiniuException {
-        StringMap params = new StringMap().putNotEmpty("pipeline", pipeline).
-                putNotEmpty("notifyURL", notifyURL).putWhen("force", 1, force);
+        StringMap params = new StringMap()
+                .putNotEmpty("pipeline", pipeline)
+                .putNotEmpty("notifyURL", notifyURL)
+                .putWhen("force", 1, force);
+        return pfop(bucket, key, fops, params);
+    }
+
+    /**
+     * 发送请求对空间中的文件进行持久化处理
+     *
+     * @param bucket    空间名
+     * @param key       文件名
+     * @param fops      fop指令
+     * @param pipeline  持久化数据处理队列名称
+     * @param notifyURL 处理结果通知地址，任务完成后自动以POST方式将处理结果提交到指定的地址
+     * @param type      任务类型，0：非闲时任务，1：闲时任务
+     * @param force     用于对同一个指令进行强制处理时指定，一般用于覆盖空间已有文件或者重试失败的指令
+     * @return persistentId 请求返回的任务ID，可以根据该ID查询任务状态
+     * @throws QiniuException 触发失败异常，包含错误响应等信息
+     *                        <a href="http://developer.qiniu.com/dora/api/persistent-data-processing-pfop"> 相关链接 </a>
+     */
+    public String pfop(String bucket, String key, String fops, String pipeline, String notifyURL, Integer type, boolean force)
+            throws QiniuException {
+        StringMap params = new StringMap()
+                .putNotNull("type", type)
+                .putNotEmpty("pipeline", pipeline)
+                .putNotEmpty("notifyURL", notifyURL)
+                .putWhen("force", 1, force);
         return pfop(bucket, key, fops, params);
     }
 

diff --git a/src/main/java/com/qiniu/processing/OperationStatus.java b/src/main/java/com/qiniu/processing/OperationStatus.java
@@ -9,30 +9,49 @@ public class OperationStatus {
      * 持久化处理的进程ID，即 persistentId
      */
     public String id;
+
     /**
      * 状态码 0 成功，1 等待处理，2 正在处理，3 处理失败，4 通知提交失败
      */
     public int code;
+
     /**
      * 与状态码相对应的详细描述
      */
     public String desc;
+
     /**
      * 处理源文件的文件名
      */
     public String inputKey;
+
     /**
      * 处理源文件所在的空间名
      */
     public String inputBucket;
+
     /**
      * 云处理操作的处理队列
      */
     public String pipeline;
+
     /**
      * 云处理请求的请求id，主要用于七牛技术人员的问题排查
      */
     public String reqid;
+
+    /**
+     * 是否是闲时任务
+     * null 或 0：非闲时任务
+     * 1：闲时任务
+     */
+    public Integer type;
+
+    /**
+     * 任务创建时间
+     */
+    public String creationDate;
+
     /**
      * 云处理操作列表，包含每个云处理操作的状态信息
      */

diff --git a/src/main/java/com/qiniu/storage/Api.java b/src/main/java/com/qiniu/storage/Api.java
@@ -395,6 +395,10 @@ public static class Request implements Cloneable {
          * @param urlPrefix 请求的 urlPrefix， scheme + host
          */
         protected Request(String urlPrefix) {
+            if (StringUtils.isNullOrEmpty(urlPrefix)) {
+                return;
+            }
+
             try {
                 URL url = new URL(urlPrefix);
                 this.scheme = url.getProtocol();
@@ -634,6 +638,15 @@ public StringMap getHeader() throws QiniuException {
             return header;
         }
 
+        /**
+         * 构造 header 信息，如果需要设置请求体，子类需要重写
+         *
+         * @throws QiniuException 组装 header 时的异常，一般为缺失必要参数的异常
+         */
+        protected void buildHeader() throws QiniuException {
+
+        }
+
         /**
          * 获取 URL
          *
@@ -804,6 +817,7 @@ boolean canRetry() {
         protected void prepareToRequest() throws QiniuException {
             buildPath();
             buildQuery();
+            buildHeader();
             buildBodyInfo();
         }
 
@@ -1052,14 +1066,18 @@ protected RequestBody get() {
                     }
 
                     final MultipartBody.Builder b = new MultipartBody.Builder();
-                    b.addFormDataPart(name, fileName, body);
-
-                    fields.forEach(new StringMap.Consumer() {
-                        @Override
-                        public void accept(String key, Object value) {
-                            b.addFormDataPart(key, value.toString());
-                        }
-                    });
+                    if (!StringUtils.isNullOrEmpty(name)) {
+                        b.addFormDataPart(name, fileName, body);
+                    }
+
+                    if (fields != null) {
+                        fields.forEach(new StringMap.Consumer() {
+                            @Override
+                            public void accept(String key, Object value) {
+                                b.addFormDataPart(key, value.toString());
+                            }
+                        });
+                    }
 
                     b.setType(MediaType.parse("multipart/form-data"));
 

diff --git a/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java b/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java
@@ -37,9 +37,14 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc
         }
 
         String reqHost = request.getHost();
-        String firstHost = hostProvider.provider();
-        if (!hostProvider.isHostValid(reqHost) && !StringUtils.isNullOrEmpty(firstHost)) {
-            request.setHost(firstHost);
+        if (!hostProvider.isHostValid(reqHost)) {
+            // 支持不配置默认的 host，未配置则从 provider 中获取
+            String firstHost = hostProvider.provider();
+            if (!StringUtils.isNullOrEmpty(firstHost)) {
+                request.setHost(firstHost);
+            } else {
+                throw QiniuException.unrecoverable("no host provide");
+            }
         }
 
         if (retryMax == 0) {

diff --git a/src/main/java/com/qiniu/util/Auth.java b/src/main/java/com/qiniu/util/Auth.java
@@ -48,6 +48,7 @@ public final class Auth {
             "fsizeMin",
             "trafficLimit",
 
+            "persistentType",
             "persistentOps",
             "persistentNotifyUrl",
             "persistentPipeline",

diff --git a/src/test/java/test/com/qiniu/processing/PfopTest.java b/src/test/java/test/com/qiniu/processing/PfopTest.java
@@ -7,6 +7,7 @@
 import com.qiniu.processing.OperationStatus;
 import com.qiniu.storage.Configuration;
 import com.qiniu.storage.Region;
+import com.qiniu.util.Auth;
 import com.qiniu.util.StringUtils;
 import com.qiniu.util.UrlSafeBase64;
 import org.junit.jupiter.api.Tag;
@@ -138,4 +139,34 @@ private void testPfopIsSuccess(String jobid) {
         assertEquals(0, status.code);
     }
 
+    @Test
+    @Tag("IntegrationTest")
+    void testPfopWithType() {
+        try {
+            Auth auth = TestConfig.testAuth;
+            Map<String, Region> bucketKeyMap = new HashMap<String, Region>();
+            TestConfig.TestFile[] files = TestConfig.getTestFileArray();
+            for (TestConfig.TestFile testFile : files) {
+                bucketKeyMap.put(testFile.getBucketName(), testFile.getRegion());
+            }
+            for (Map.Entry<String, Region> entry : bucketKeyMap.entrySet()) {
+                String bucket = entry.getKey();
+                Region region = entry.getValue();
+
+                Configuration cfg = new Configuration(region);
+                OperationManager operationManager = new OperationManager(auth, cfg);
+                String jobID = operationManager.pfop(bucket, TestConfig.testMp4FileKey, "avinfo", "", "", 1, true);
+
+                OperationStatus status = operationManager.prefop(bucket, jobID);
+                assertNotNull(status, "1. prefop type error");
+                assertNotNull(status.creationDate, "1. prefop type error");
+                assertTrue(status.code == 0 || status.code == 1 || status.code == 3, "2. prefop type error");
+                assertEquals(1, (int) status.type, "3. prefop type error");
+            }
+
+        } catch (QiniuException ex) {
+            ex.printStackTrace();
+            assertTrue(ResCode.find(ex.code(), ResCode.getPossibleResCode(612)));
+        }
+    }
 }
diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java
@@ -2,6 +2,8 @@
 
 import com.qiniu.common.QiniuException;
 import com.qiniu.http.Response;
+import com.qiniu.processing.OperationManager;
+import com.qiniu.processing.OperationStatus;
 import com.qiniu.storage.Configuration;
 import com.qiniu.storage.UpCompletionHandler;
 import com.qiniu.storage.UploadManager;
@@ -27,6 +29,56 @@ public class FormUploadTest {
 
     UploadManager uploadManager = new UploadManager(new Configuration());
 
+    @Test
+    @Tag("IntegrationTest")
+    void testUploadWithFop() {
+        TestConfig.TestFile file = TestConfig.getTestFileArray()[0];
+        final String expectKey = "test-fop";
+        final String bucket = file.getBucketName();
+
+        String persistentOpfs = String.format("%s:vframe_test_target.jpg", bucket);
+        StringMap policy = new StringMap();
+        policy.put("persistentOps", persistentOpfs);
+        policy.put("persistentType", 1);
+
+        Configuration config = new Configuration();
+        config.useHttpsDomains = true;
+
+        Response r = null;
+        try {
+            String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, policy);
+            UploadManager uploadManager = new UploadManager(config);
+            StringMap params = new StringMap().put("x:foo", "foo_val");
+            r = uploadManager.put("hello".getBytes(), expectKey, token, params, null, false);
+        } catch (QiniuException e) {
+            fail(e.toString());
+        }
+        assertEquals(200, r.statusCode);
+
+        StringMap map = null;
+        try {
+            map = r.jsonToMap();
+        } catch (QiniuException e) {
+            fail(e.toString());
+        }
+
+        assertNotNull(map, "1. testUploadWithFop error");
+
+        String persistentId = (String) map.get("persistentId");
+        assertNotNull(persistentId, "2. testUploadWithFop error");
+
+        try {
+            OperationManager operationManager = new OperationManager(TestConfig.testAuth, config);
+            OperationStatus status = operationManager.prefop(bucket, persistentId);
+            assertNotNull(status, "3. prefop type error");
+            assertNotNull(status.creationDate, "4. prefop type error");
+            assertTrue(status.code == 0 || status.code == 1 || status.code == 3, "5. prefop type error");
+            assertEquals(1, (int) status.type, "6. prefop type error");
+        } catch (QiniuException e) {
+            fail(e.toString());
+        }
+    }
+
     /**
      * hello上传测试
      */