Add asset streaming via range requests

.gitignore

@@ -6,3 +6,4 @@ pocket-ic
 .env
 
 .DS_Store
+.idea

Cargo.toml

@@ -33,6 +33,7 @@ http-body-util = "0.1"
 bytes = "1"
 base64 = "0.22"
 lazy_static = "1"
+regex = "1.10"
 serde = "1"
 serde_cbor = "0.11"
 tokio = { version = "1", features = ["full"] }
@@ -45,6 +46,7 @@ ic-agent = "0.37"
 ic-utils = "0.37"
 candid = "0.10"
 pocket-ic = "=3.0.0"
+assert_matches = "1.5"
 
 ic-certification = { git = "https://github.com/dfinity/response-verification" }
 ic-http-certification = { git = "https://github.com/dfinity/response-verification" }

packages/ic-http-gateway/Cargo.toml

@@ -26,6 +26,7 @@ http.workspace = true
 http-body.workspace = true
 http-body-util.workspace = true
 bytes.workspace = true
+regex.workspace = true
 
 ic-agent.workspace = true
 ic-utils.workspace = true
@@ -35,5 +36,6 @@ ic-http-certification.workspace = true
 ic-response-verification.workspace = true
 
 [dev-dependencies]
+assert_matches.workspace = true
 pocket-ic.workspace = true
 tokio.workspace = true

packages/ic-http-gateway/src/consts.rs

@@ -1,4 +1,5 @@
 pub(crate) static CACHE_HEADER_NAME: &str = "cache-control";
 pub(crate) static ACCEPT_ENCODING_HEADER_NAME: &str = "accept-encoding";
+pub(crate) static CONTENT_RANGE_HEADER_NAME: &str = "content-range";
 
 pub(crate) static DEFAULT_BOUNDARY_NODE_ENDPOINT: &str = "https://icp-api.io";

packages/ic-http-gateway/src/protocol/handler.rs

@@ -1,8 +1,8 @@
 use super::validate;
 use crate::{
-    get_body_and_streaming_body, CanisterRequest, CanisterResponse, HttpGatewayError,
-    HttpGatewayResponse, HttpGatewayResponseBody, HttpGatewayResponseMetadata, HttpGatewayResult,
-    ACCEPT_ENCODING_HEADER_NAME, CACHE_HEADER_NAME,
+    get_206_stream_response_body, get_body_and_streaming_body, CanisterRequest, CanisterResponse,
+    HttpGatewayError, HttpGatewayResponse, HttpGatewayResponseBody, HttpGatewayResponseMetadata,
+    HttpGatewayResult, ACCEPT_ENCODING_HEADER_NAME, CACHE_HEADER_NAME,
 };
 use candid::Principal;
 use http::{Response, StatusCode};
@@ -186,7 +186,7 @@ pub async fn process_request(
                 let validation_result = validate(
                     agent,
                     &canister_id,
-                    http_request,
+                    http_request.clone(),
                     HttpResponse {
                         status_code: agent_response.status_code,
                         headers: agent_response
@@ -299,6 +299,36 @@ pub async fn process_request(
         }
     }
 
+    let response_body: HttpGatewayResponseBody = if status_code == 206 {
+        // We got only the first chunk, turn the response into a streaming response
+        match get_206_stream_response_body(
+            agent,
+            &http_request,
+            canister_id,
+            &agent_response.headers,
+            response_body,
+        )
+        .await
+        {
+            Ok(stream_response_body) => stream_response_body,
+            Err(e) => {
+                return HttpGatewayResponse {
+                    canister_response: create_err_response(
+                        StatusCode::INTERNAL_SERVER_ERROR,
+                        &format!("Failed to create streaming responsey: {}", e),
+                    ),
+                    metadata: HttpGatewayResponseMetadata {
+                        upgraded_to_update_call: is_update_call,
+                        response_verification_version: None,
+                        internal_error: Some(e.into()),
+                    },
+                }
+            }
+        }
+    } else {
+        response_body
+    };
+
     let response = match response_builder.body(response_body) {
         Ok(response) => response,
         Err(e) => {
@@ -326,7 +356,7 @@ pub async fn process_request(
     }
 }
 
-fn handle_agent_error(error: &AgentError) -> CanisterResponse {
+pub(crate) fn handle_agent_error(error: &AgentError) -> CanisterResponse {
     match error {
         // Turn all `DestinationInvalid`s into 404
         AgentError::CertifiedReject(RejectResponse {

packages/ic-http-gateway/src/protocol/mod.rs

@@ -1,4 +1,4 @@
-mod handler;
+pub(crate) mod handler;
 pub(crate) use handler::*;
 
 mod validate;