From f55996c12a42cba13305ff707dacc9861c5150eb Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 19:10:51 -0500 Subject: [PATCH 1/8] fix block size in health check --- router/src/health.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/router/src/health.rs b/router/src/health.rs index 5ca8e8de8..1fe8c29e4 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -37,6 +37,7 @@ impl Health { } pub(crate) async fn check_generation(&mut self) -> bool { + let block_size = self.shard_info().block_size; let generation_liveness_request = Request { id: LIVENESS_ID, inputs: "liveness".to_string(), @@ -66,7 +67,7 @@ impl Health { adapter_index: 0, // Block 0 is reserved for health checks blocks: vec![0], - slots: (0..16).collect(), + slots: (0..block_size).collect(), cache_len: 0, chunk_len: None, }; From 641fa6719d48cd7d11dc7d97d78338b873d1e972 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 19:12:39 -0500 Subject: [PATCH 2/8] fix --- router/src/health.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/router/src/health.rs b/router/src/health.rs index 1fe8c29e4..b751e63e1 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -37,7 +37,6 @@ impl Health { } pub(crate) async fn check_generation(&mut self) -> bool { - let block_size = self.shard_info().block_size; let generation_liveness_request = Request { id: LIVENESS_ID, inputs: "liveness".to_string(), @@ -67,7 +66,7 @@ impl Health { adapter_index: 0, // Block 0 is reserved for health checks blocks: vec![0], - slots: (0..block_size).collect(), + slots: (0..self.shard_info().block_size).collect(), cache_len: 0, chunk_len: None, }; @@ -93,7 +92,7 @@ impl Health { stopping_parameters: None, adapter_index: 0, blocks: vec![0], - slots: (0..16).collect(), + slots: (0..self.shard_info().block_size).collect(), cache_len: 0, chunk_len: None, }; @@ -118,7 +117,7 @@ impl Health { stopping_parameters: None, adapter_index: 0, blocks: vec![0], - slots: (0..16).collect(), + slots: (0..self.shard_info().block_size).collect(), cache_len: 0, chunk_len: None, }; From 9b0e6e28bfc6020103b5f5c87ebd979d4e093b35 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 19:15:03 -0500 Subject: [PATCH 3/8] bump rust --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eccefae58..0988daf58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.79 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.83 AS chef WORKDIR /usr/src ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse From 363db67b8d4726f9000de6f567ade58972bccc62 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 20:00:31 -0500 Subject: [PATCH 4/8] router test --- .github/workflows/router_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/router_tests.yaml b/.github/workflows/router_tests.yaml index 6eb95f130..404b481d6 100644 --- a/.github/workflows/router_tests.yaml +++ b/.github/workflows/router_tests.yaml @@ -29,7 +29,7 @@ jobs: - name: Install Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.79.0 + toolchain: 1.83.0 override: true components: rustfmt, clippy - name: Install Protoc From 02e51eb87c52cceca0ea0b16abd5babefcf1734e Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 20:01:23 -0500 Subject: [PATCH 5/8] use the property directly --- router/src/health.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/router/src/health.rs b/router/src/health.rs index b751e63e1..e8eab08ce 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -66,7 +66,7 @@ impl Health { adapter_index: 0, // Block 0 is reserved for health checks blocks: vec![0], - slots: (0..self.shard_info().block_size).collect(), + slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, }; @@ -92,7 +92,7 @@ impl Health { stopping_parameters: None, adapter_index: 0, blocks: vec![0], - slots: (0..self.shard_info().block_size).collect(), + slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, }; @@ -117,7 +117,7 @@ impl Health { stopping_parameters: None, adapter_index: 0, blocks: vec![0], - slots: (0..self.shard_info().block_size).collect(), + slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, }; From c499282d396d72ae62d3736eea6bbe81ccf25990 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Thu, 16 Jan 2025 20:05:56 -0500 Subject: [PATCH 6/8] fix toolchain and bump rust --- rust-toolchain.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b6ffc9d2c..80afd2d32 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.79.0" -components = ["rustfmt", "clippy"] \ No newline at end of file +channel = "1.83.0" +components = ["rustfmt", "clippy"] From d6bfc26042af3cf062eb6223e85fbfdf33f12cce Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Fri, 17 Jan 2025 17:08:47 -0500 Subject: [PATCH 7/8] fix --- router/src/health.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/router/src/health.rs b/router/src/health.rs index e8eab08ce..486b61894 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -65,7 +65,7 @@ impl Health { }), adapter_index: 0, // Block 0 is reserved for health checks - blocks: vec![0], + blocks: vec![0, 1], slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, @@ -91,8 +91,8 @@ impl Health { parameters: None, stopping_parameters: None, adapter_index: 0, - blocks: vec![0], - slots: (0..self.shard_info.block_size).collect(), + blocks: vec![0, 1, 2], + slots: (0..self.shard_info.block_size + 1).collect(), cache_len: 0, chunk_len: None, }; @@ -116,8 +116,8 @@ impl Health { parameters: None, stopping_parameters: None, adapter_index: 0, - blocks: vec![0], - slots: (0..self.shard_info.block_size).collect(), + blocks: vec![0, 1, 2], + slots: (0..self.shard_info.block_size + 1).collect(), cache_len: 0, chunk_len: None, }; From 97ce9453c0c48f38562971b920ac8e5a3d478ba0 Mon Sep 17 00:00:00 2001 From: Magdy Saleh Date: Fri, 17 Jan 2025 17:18:45 -0500 Subject: [PATCH 8/8] tokenised inputs --- router/src/health.rs | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/router/src/health.rs b/router/src/health.rs index 486b61894..cfda0b85c 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -1,6 +1,6 @@ use lorax_client::{ - Batch, NextTokenChooserParameters, Request, ShardInfo, ShardedClient, - StoppingCriteriaParameters, + input_chunk, Batch, InputChunk, NextTokenChooserParameters, Request, ShardInfo, ShardedClient, + StoppingCriteriaParameters, TokenizedInputs, }; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -40,7 +40,12 @@ impl Health { let generation_liveness_request = Request { id: LIVENESS_ID, inputs: "liveness".to_string(), - tokenized_inputs: None, + tokenized_inputs: Some(TokenizedInputs { + ids: vec![75], + input_chunks: vec![InputChunk { + chunk: Some(input_chunk::Chunk::Text("liveness".to_string())), + }], + }), truncate: 10, prefill_logprobs: false, parameters: Some(NextTokenChooserParameters { @@ -65,7 +70,7 @@ impl Health { }), adapter_index: 0, // Block 0 is reserved for health checks - blocks: vec![0, 1], + blocks: vec![0], slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, @@ -84,15 +89,20 @@ impl Health { pub(crate) async fn check_classification(&mut self) -> bool { let classify_request = Request { id: LIVENESS_ID, - inputs: "San Francisco".to_string(), - tokenized_inputs: None, + inputs: "liveness".to_string(), + tokenized_inputs: Some(TokenizedInputs { + ids: vec![75], + input_chunks: vec![InputChunk { + chunk: Some(input_chunk::Chunk::Text("liveness".to_string())), + }], + }), truncate: 10, prefill_logprobs: false, parameters: None, stopping_parameters: None, adapter_index: 0, - blocks: vec![0, 1, 2], - slots: (0..self.shard_info.block_size + 1).collect(), + blocks: vec![0], + slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, }; @@ -109,15 +119,20 @@ impl Health { pub(crate) async fn check_embeddings(&mut self) -> bool { let embed_request = Request { id: LIVENESS_ID, - inputs: "San Francisco".to_string(), - tokenized_inputs: None, + inputs: "liveness".to_string(), + tokenized_inputs: Some(TokenizedInputs { + ids: vec![75], + input_chunks: vec![InputChunk { + chunk: Some(input_chunk::Chunk::Text("liveness".to_string())), + }], + }), truncate: 10, prefill_logprobs: false, parameters: None, stopping_parameters: None, adapter_index: 0, - blocks: vec![0, 1, 2], - slots: (0..self.shard_info.block_size + 1).collect(), + blocks: vec![0], + slots: (0..self.shard_info.block_size).collect(), cache_len: 0, chunk_len: None, };