Add storage server RPC client (#1430)

* Add storage server RPC client

* Address review comments

- Use `#[tracing::instrument]` wherever appropriate
- Remove `Mutex` for `client` and `clone` it wherever required
- Avoid reading env variable inside the lib, rather take a config object
- Make the private methods of `DurableWal` async
- Don't try to create a runtime, instead assume it always exists. Let the caller create it
- Update proto:
	- remove `max_frame_no` from `InsertFramesRequest`
	- make `page_no` to `u32` in the proto
- Update storage server to have `page_no` as `u32`

libsql-storage-server/src/memory_store.rs

@@ -4,7 +4,6 @@ use std::sync::Mutex;
 use crate::store::FrameData;
 use crate::store::FrameStore;
 use async_trait::async_trait;
-use bytes::Bytes;
 
 #[derive(Default)]
 pub(crate) struct InMemFrameStore {
@@ -16,7 +15,7 @@ struct InMemInternal {
     // contains a frame data, key is the frame number
     frames: BTreeMap<u64, FrameData>,
     // pages map contains the page number as a key and the list of frames for the page as a value
-    pages: BTreeMap<u64, Vec<u64>>,
+    pages: BTreeMap<u32, Vec<u64>>,
     max_frame_no: u64,
 }
 
@@ -29,7 +28,7 @@ impl InMemFrameStore {
 #[async_trait]
 impl FrameStore for InMemFrameStore {
     // inserts a new frame for the page number and returns the new frame value
-    async fn insert_frame(&self, _namespace: &str, page_no: u64, frame: Bytes) -> u64 {
+    async fn insert_frame(&self, _namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64 {
         let mut inner = self.inner.lock().unwrap();
         let frame_no = inner.max_frame_no + 1;
         inner.max_frame_no = frame_no;
@@ -62,7 +61,7 @@ impl FrameStore for InMemFrameStore {
     }
 
     // given a page number, return the maximum frame for the page
-    async fn find_frame(&self, _namespace: &str, page_no: u64) -> Option<u64> {
+    async fn find_frame(&self, _namespace: &str, page_no: u32) -> Option<u64> {
         self.inner
             .lock()
             .unwrap()
@@ -72,7 +71,7 @@ impl FrameStore for InMemFrameStore {
     }
 
     // given a frame num, return the page number
-    async fn frame_page_no(&self, _namespace: &str, frame_no: u64) -> Option<u64> {
+    async fn frame_page_no(&self, _namespace: &str, frame_no: u64) -> Option<u32> {
         self.inner
             .lock()
             .unwrap()

libsql-storage-server/src/store.rs

@@ -3,18 +3,18 @@ use bytes::Bytes;
 
 #[async_trait]
 pub trait FrameStore: Send + Sync {
-    async fn insert_frame(&self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64;
+    async fn insert_frame(&self, namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64;
     #[allow(dead_code)]
     async fn insert_frames(&self, namespace: &str, frames: Vec<FrameData>) -> u64;
     async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option<bytes::Bytes>;
-    async fn find_frame(&self, namespace: &str, page_no: u64) -> Option<u64>;
-    async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option<u64>;
+    async fn find_frame(&self, namespace: &str, page_no: u32) -> Option<u64>;
+    async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option<u32>;
     async fn frames_in_wal(&self, namespace: &str) -> u64;
     async fn destroy(&self, namespace: &str);
 }
 
 #[derive(Default)]
 pub struct FrameData {
-    pub(crate) page_no: u64,
+    pub(crate) page_no: u32,
     pub(crate) data: Bytes,
 }

libsql-storage/Cargo.toml

@@ -7,6 +7,13 @@ repository = "https://github.com/tursodatabase/libsql"
 license = "MIT"
 
 [dependencies]
+libsql-sys = { path = "../libsql-sys", features = ["rusqlite"] }
+sieve-cache = "0.1.4"
+tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs"] }
+tracing = { version = "0.1.37", default-features = false }
+uuid = { version = "1.7.0", features = ["v4"] }
+log = "0.4.20"
+parking_lot = "0.12.1"
 prost = "0.12"
 tonic = { version = "0.10", features = ["tls"] }
 

libsql-storage/proto/storage.proto

@@ -3,14 +3,13 @@ syntax = "proto3";
 package storage;
 
 message Frame {
-  uint64 page_no = 1;
+  uint32 page_no = 1;
   bytes data = 2;
 }
 
 message InsertFramesRequest {
   string namespace = 1;
   repeated Frame frames = 2;
-  uint64 max_frame_no = 3;
 }
 
 message InsertFramesResponse {
@@ -19,7 +18,7 @@ message InsertFramesResponse {
 
 message FindFrameRequest {
   string namespace = 1;
-  uint64 page_no = 2;
+  uint32 page_no = 2;
   uint64 max_frame_no = 3;
 }
 
@@ -58,7 +57,7 @@ message FramePageNumRequest {
 }
 
 message FramePageNumResponse {
-  uint64 page_no = 1;
+  uint32 page_no = 1;
 }
 
 message DestroyRequest {

libsql-storage/src/generated/storage.rs

@@ -2,8 +2,8 @@
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Frame {
-    #[prost(uint64, tag = "1")]
-    pub page_no: u64,
+    #[prost(uint32, tag = "1")]
+    pub page_no: u32,
     #[prost(bytes = "vec", tag = "2")]
     pub data: ::prost::alloc::vec::Vec<u8>,
 }
@@ -14,8 +14,6 @@ pub struct InsertFramesRequest {
     pub namespace: ::prost::alloc::string::String,
     #[prost(message, repeated, tag = "2")]
     pub frames: ::prost::alloc::vec::Vec<Frame>,
-    #[prost(uint64, tag = "3")]
-    pub max_frame_no: u64,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -28,8 +26,8 @@ pub struct InsertFramesResponse {
 pub struct FindFrameRequest {
     #[prost(string, tag = "1")]
     pub namespace: ::prost::alloc::string::String,
-    #[prost(uint64, tag = "2")]
-    pub page_no: u64,
+    #[prost(uint32, tag = "2")]
+    pub page_no: u32,
     #[prost(uint64, tag = "3")]
     pub max_frame_no: u64,
 }
@@ -88,8 +86,8 @@ pub struct FramePageNumRequest {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct FramePageNumResponse {
-    #[prost(uint64, tag = "1")]
-    pub page_no: u64,
+    #[prost(uint32, tag = "1")]
+    pub page_no: u32,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]