spi-obj: support outside custom s3 client (#852)

backend/basic/src/spi/spi_constants.rs

@@ -1,6 +1,6 @@
 pub const SPI_CERT_KIND: &str = "spi";
 pub const SPI_IDENT_REL_TAG: &str = "spi_ident";
-pub(crate) const SPI_KIND_CODE_FLAG: &str = "__SPI_KIND_CODE__";
+pub const SPI_KIND_CODE_FLAG: &str = "__SPI_KIND_CODE__";
 pub(crate) const SPI_ISOLATION_FLAG: &str = "__isolation__";
 pub const SPI_PG_KIND_CODE: &str = "spi-bs-pg";
 pub const SPI_ES_KIND_CODE: &str = "spi-bs-es";

backend/spi/spi-object/src/api/ci/object_ci_obj_api.rs

@@ -1,5 +1,10 @@
 use std::collections::HashMap;
 
+use bios_basic::rbum::serv::rbum_item_serv::RbumItemCrudOperation;
+use bios_basic::rbum::serv::rbum_kind_serv::RbumKindServ;
+use bios_basic::spi::dto::spi_bs_dto::SpiBsAddReq;
+use bios_basic::spi::serv::spi_bs_serv::SpiBsServ;
+use tardis::basic::field::TrimString;
 use tardis::web::context_extractor::TardisContextExtractor;
 
 use tardis::web::poem::web::Json;
@@ -8,9 +13,9 @@ use tardis::web::poem_openapi::param::Query;
 use tardis::web::web_resp::{TardisApiResult, TardisResp, Void};
 
 use crate::dto::object_dto::{
-    ObjectBatchBuildCreatePresignUrlReq, ObjectBatchDeleteReq, ObjectCompleteMultipartUploadReq, ObjectCopyReq, ObjectInitiateMultipartUploadReq, ObjectObjPresignKind,
-    ObjectPresignBatchViewReq,
+    ClientCreateReq, ObjectBatchBuildCreatePresignUrlReq, ObjectBatchDeleteReq, ObjectCompleteMultipartUploadReq, ObjectCopyReq, ObjectInitiateMultipartUploadReq, ObjectObjPresignKind, ObjectPresignBatchViewReq
 };
+use crate::object_constants;
 use crate::serv::object_obj_serv;
 #[derive(Clone)]
 pub struct ObjectCiObjApi;
@@ -53,6 +58,12 @@ impl ObjectCiObjApi {
         // Whether or not it is temporary, the number indicates the length of time the file will be in effect.
         // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket.
         obj_exp: Query<Option<u32>>,
+        // 服务ID，使用外部自定义服务时，传入该值。
+        // Service ID, pass this value when using an external custom service.
+        bs_id: Query<Option<String>>,
+        // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+        // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+        bucket: Query<Option<String>>,
         ctx: TardisContextExtractor,
     ) -> TardisApiResult<String> {
         let funs = crate::get_tardis_inst();
@@ -65,6 +76,8 @@ impl ObjectCiObjApi {
             private.0,
             special.0,
             obj_exp.0,
+            bucket.0,
+            bs_id.0,
             &funs,
             &ctx.0,
         )
@@ -95,6 +108,12 @@ impl ObjectCiObjApi {
         // Whether or not it is temporary, the number indicates the length of time the file will be in effect.
         // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket.
         obj_exp: Query<Option<u32>>,
+        // 服务ID，使用外部自定义服务时，传入该值。
+        // Service ID, pass this value when using an external custom service.
+        bs_id: Query<Option<String>>,
+        // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+        // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+        bucket: Query<Option<String>>,
         ctx: TardisContextExtractor,
     ) -> TardisApiResult<String> {
         let funs = crate::get_tardis_inst();
@@ -107,6 +126,8 @@ impl ObjectCiObjApi {
             private.0,
             special.0,
             obj_exp.0,
+            bucket.0,
+            bs_id.0,
             &funs,
             &ctx.0,
         )
@@ -137,6 +158,12 @@ impl ObjectCiObjApi {
         // Whether or not it is temporary, the number indicates the length of time the file will be in effect.
         // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket.
         obj_exp: Query<Option<u32>>,
+        // 服务ID，使用外部自定义服务时，传入该值。
+        // Service ID, pass this value when using an external custom service.
+        bs_id: Query<Option<String>>,
+        // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+        // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+        bucket: Query<Option<String>>,
         ctx: TardisContextExtractor,
     ) -> TardisApiResult<String> {
         let funs = crate::get_tardis_inst();
@@ -149,6 +176,8 @@ impl ObjectCiObjApi {
             private.0,
             special.0,
             obj_exp.0,
+            bucket.0,
+            bs_id.0,
             &funs,
             &ctx.0,
         )
@@ -162,7 +191,7 @@ impl ObjectCiObjApi {
     #[oai(path = "/presign/batch_view", method = "post")]
     async fn batch_presign_view_obj_url(&self, req: Json<ObjectPresignBatchViewReq>, ctx: TardisContextExtractor) -> TardisApiResult<HashMap<String, String>> {
         let funs = crate::get_tardis_inst();
-        let url = object_obj_serv::batch_get_presign_obj_url(req.0.object_path, req.0.expire_sec, req.0.private, req.0.special, req.0.obj_exp, &funs, &ctx.0).await?;
+        let url = object_obj_serv::batch_get_presign_obj_url(req.0.object_path, req.0.expire_sec, req.0.private, req.0.special, req.0.obj_exp, req.0.bucket, req.0.bs_id, &funs, &ctx.0).await?;
         TardisResp::ok(url)
     }
 
@@ -202,7 +231,7 @@ impl ObjectCiObjApi {
     #[oai(path = "/object/copy", method = "post")]
     async fn object_copy(&self, req: Json<ObjectCopyReq>, ctx: TardisContextExtractor) -> TardisApiResult<Void> {
         let funs = crate::get_tardis_inst();
-        object_obj_serv::object_copy(req.0.from, req.0.to, req.0.private, req.0.special, &funs, &ctx.0).await?;
+        object_obj_serv::object_copy(req.0.from, req.0.to, req.0.private, req.0.special, req.0.bucket, req.0.bs_id, &funs, &ctx.0).await?;
         TardisResp::ok(Void)
     }
 
@@ -226,10 +255,16 @@ impl ObjectCiObjApi {
         // Whether or not it is temporary, the number indicates the length of time the file will be in effect.
         // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket.
         obj_exp: Query<Option<u32>>,
+        // 服务ID，使用外部自定义服务时，传入该值。
+        // Service ID, pass this value when using an external custom service.
+        bs_id: Query<Option<String>>,
+        // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+        // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+        bucket: Query<Option<String>>,
         ctx: TardisContextExtractor,
     ) -> TardisApiResult<Void> {
         let funs = crate::get_tardis_inst();
-        object_obj_serv::object_delete(object_path.0, private.0, special.0, obj_exp.0, &funs, &ctx.0).await?;
+        object_obj_serv::object_delete(object_path.0, private.0, special.0, obj_exp.0, bucket.0, bs_id.0, &funs, &ctx.0).await?;
         TardisResp::ok(Void)
     }
 
@@ -239,7 +274,7 @@ impl ObjectCiObjApi {
     #[oai(path = "/object/batch_delete", method = "delete")]
     async fn batch_object_delete(&self, req: Json<ObjectBatchDeleteReq>, ctx: TardisContextExtractor) -> TardisApiResult<Vec<String>> {
         let funs = crate::get_tardis_inst();
-        TardisResp::ok(object_obj_serv::batch_object_delete(req.0.object_path, req.0.private, req.0.special, req.0.obj_exp, &funs, &ctx.0).await?)
+        TardisResp::ok(object_obj_serv::batch_object_delete(req.0.object_path, req.0.private, req.0.special, req.0.obj_exp, req.0.bucket, req.0.bs_id, &funs, &ctx.0).await?)
     }
 
     /// Check object is exist
@@ -262,10 +297,38 @@ impl ObjectCiObjApi {
         // Whether or not it is temporary, the number indicates the length of time the file will be in effect.
         // When using obs, passing in a value does not take effect, it only indicates the use of the tamp bucket.
         obj_exp: Query<Option<u32>>,
+        // 服务ID，使用外部自定义服务时，传入该值。
+        // Service ID, pass this value when using an external custom service.
+        bs_id: Query<Option<String>>,
+        // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+        // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+        bucket: Query<Option<String>>,
         ctx: TardisContextExtractor,
     ) -> TardisApiResult<bool> {
         let funs = crate::get_tardis_inst();
-        TardisResp::ok(object_obj_serv::object_exist(object_path.0, private.0, special.0, obj_exp.0, &funs, &ctx.0).await?)
+        TardisResp::ok(object_obj_serv::object_exist(object_path.0, private.0, special.0, obj_exp.0, bucket.0, bs_id.0, &funs, &ctx.0).await?)
+    }
+
+    /// Check object is exist
+    ///
+    /// 添加自定义服务实例
+    #[oai(path = "/bs/add", method = "post")]
+    async fn bs_add(&self, add_req: Json<ClientCreateReq>, ctx: TardisContextExtractor,) -> TardisApiResult<String> {
+        let mut funs = crate::get_tardis_inst();
+        funs.begin().await?;
+        let kind_id = RbumKindServ::get_rbum_kind_id_by_code(object_constants::SPI_S3_KIND_CODE, &funs).await?.expect("missing event kind");
+        let result = SpiBsServ::add_item(&mut SpiBsAddReq {
+            name: add_req.0.name,
+            kind_id: TrimString::from(kind_id),
+            conn_uri: add_req.0.conn_uri,
+            ak: add_req.0.ak,
+            sk: add_req.0.sk,
+            ext: add_req.0.ext,
+            private: true,
+            disabled: None,
+        }, &funs, &ctx.0).await?;
+        funs.commit().await?;
+        TardisResp::ok(result)
     }
 
     // /// Fetch URL for temporary authorization of thumbnail

backend/spi/spi-object/src/dto/object_dto.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-use tardis::web::poem_openapi;
+use tardis::{basic::field::TrimString, web::poem_openapi};
 
 #[derive(Serialize, Deserialize, Debug)]
 pub enum ObjectObjPresignKind {
@@ -15,6 +15,12 @@ pub struct ObjectInitiateMultipartUploadReq {
     pub content_type: Option<String>,
     pub private: Option<bool>,
     pub special: Option<bool>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
 
 #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
@@ -25,6 +31,12 @@ pub struct ObjectBatchBuildCreatePresignUrlReq {
     pub expire_sec: u32,
     pub private: Option<bool>,
     pub special: Option<bool>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
 
 #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
@@ -34,6 +46,12 @@ pub struct ObjectCompleteMultipartUploadReq {
     pub parts: Vec<String>,
     pub private: Option<bool>,
     pub special: Option<bool>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
 
 #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
@@ -42,6 +60,12 @@ pub struct ObjectCopyReq {
     pub to: String,
     pub private: Option<bool>,
     pub special: Option<bool>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
 
 #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
@@ -50,6 +74,12 @@ pub struct ObjectBatchDeleteReq {
     pub private: Option<bool>,
     pub special: Option<bool>,
     pub obj_exp: Option<u32>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
 
 #[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
@@ -59,4 +89,19 @@ pub struct ObjectPresignBatchViewReq {
     pub private: Option<bool>,
     pub special: Option<bool>,
     pub obj_exp: Option<u32>,
+    // 服务ID，使用外部自定义服务时，传入该值。
+    // Service ID, pass this value when using an external custom service.
+    pub bs_id: Option<String>,
+    // 指定桶，当且仅当使用自定义服务ID时该参数有效。
+    // Specifies the bucket. This parameter is valid when and only when a custom service ID is used.
+    pub bucket: Option<String>,
 }
+
+#[derive(poem_openapi::Object, Serialize, Deserialize, Debug)]
+pub struct ClientCreateReq {
+    pub name: TrimString,
+    pub conn_uri: String,
+    pub ak: TrimString,
+    pub sk: TrimString,
+    pub ext: String,
+}

backend/spi/spi-object/src/serv.rs

@@ -1,3 +1,4 @@
 pub mod object_obj_serv;
 pub mod obs;
 pub mod s3;
+pub mod custom_s3;

backend/spi/spi-object/src/serv/custom_s3.rs

@@ -0,0 +1,2 @@
+pub mod object_custom_s3_initializer;
+pub mod object_custom_s3_obj_serv;

backend/spi/spi-object/src/serv/custom_s3/object_custom_s3_initializer.rs

@@ -0,0 +1,24 @@
+use std::collections::HashMap;
+
+use bios_basic::spi::{dto::spi_bs_dto::SpiBsCertResp, spi_funs::SpiBsInst};
+use tardis::{basic::{dto::TardisContext, error::TardisError, result::TardisResult}, config::config_dto::OSModuleConfig, os::os_client::TardisOSClient, TardisFuns};
+
+use tardis::serde_json::Value as JsonValue;
+
+/// 自定义外部obs服务初始化
+/// 外部服务由API申请资源时初始化，不需要随系统spi初始化
+pub async fn init(bs_cert: &SpiBsCertResp, _ctx: &TardisContext, _mgr: bool) -> TardisResult<SpiBsInst> {
+    let ext = TardisFuns::json.str_to_json(&bs_cert.ext)?;
+    let region = ext
+        .get("region")
+        .and_then(JsonValue::as_str)
+        .ok_or_else(|| TardisError::bad_request("Tardis context ext should have a `region` field with type string", "400-spi-invalid-tardis-ctx"))?;
+    let default_bucket = ext
+        .get("default_bucket")
+        .and_then(JsonValue::as_str)
+        .ok_or_else(|| TardisError::bad_request("Tardis context ext should have a `region` field with type string", "400-spi-invalid-tardis-ctx"))?;
+    let tardis_os_config = OSModuleConfig::builder().kind("s3").endpoint(&bs_cert.conn_uri).ak(&bs_cert.ak).sk(&bs_cert.sk).region(region).default_bucket(default_bucket).build();
+    let client = TardisOSClient::init(&tardis_os_config)?;
+    Ok(SpiBsInst { client: Box::new(client), ext: HashMap::new() })
+}
+

backend/spi/spi-object/src/serv/custom_s3/object_custom_s3_obj_serv.rs

@@ -0,0 +1,52 @@
+use bios_basic::spi::dto::spi_bs_dto::SpiBsCertResp;
+use bios_basic::spi::dto::spi_bs_dto::SpiBsDetailResp;
+use bios_basic::spi::spi_constants;
+use bios_basic::spi::spi_funs::SpiBsInst;
+use std::collections::HashMap;
+use std::sync::OnceLock;
+use std::sync::Arc;
+
+use tardis::{
+    basic::{dto::TardisContext, result::TardisResult},
+    os::os_client::TardisOSClient,
+    tokio::sync::RwLock,
+};
+
+use crate::serv::s3::S3;
+
+/// 自定义外部s3服务
+/// 文件服务支持绕过spi配置而直接根据外部传入的配置创建客户端连接。
+pub(crate) struct CustomS3Service;
+impl S3 for CustomS3Service {
+    async fn rebuild_path(_bucket_name: Option<&str>, origin_path: &str, _obj_exp: Option<u32>, _client: &TardisOSClient) -> TardisResult<String> {
+        Ok(origin_path.to_string())
+    }
+}
+
+impl CustomS3Service {
+    pub async fn get_bs(bs_cert: &SpiBsDetailResp, ctx: &TardisContext) -> TardisResult<Arc<SpiBsInst>> {
+        {
+            let read = Self::get_custom_bs_caches().read().await;
+            if let Some(inst) = read.get(&bs_cert.id).cloned() {
+                return Ok(inst);
+            }
+        }
+        let mut spi_bs_inst = crate::serv::custom_s3::object_custom_s3_initializer::init(&SpiBsCertResp {
+            kind_code: bs_cert.kind_code.clone(),
+            conn_uri: bs_cert.conn_uri.clone(),
+            ak: bs_cert.ak.clone(),
+            sk: bs_cert.sk.clone(),
+            ext: bs_cert.ext.clone(),
+            private: bs_cert.private,
+        }, ctx, true).await?;
+        {
+            let mut write = Self::get_custom_bs_caches().write().await;
+            spi_bs_inst.ext.insert(spi_constants::SPI_KIND_CODE_FLAG.to_string(), bs_cert.kind_code.clone());
+            return Ok(write.entry(bs_cert.id.clone()).or_insert(Arc::new(spi_bs_inst)).clone());
+        }
+    }
+    fn get_custom_bs_caches() -> &'static RwLock<HashMap<String, Arc<SpiBsInst>>> {
+        static CUSTOM_BS_CACHES: OnceLock<RwLock<HashMap<String, Arc<SpiBsInst>>>> = OnceLock::new();
+        CUSTOM_BS_CACHES.get_or_init(Default::default)
+    }
+}