feat(chore) | add litellm cache (#9)

* add: litellm cache

* fix: api key.

.gitignore

@@ -1,4 +1,5 @@
 .env*
+!.env.sample
 .DS_Store
 
 .idea/
@@ -7,4 +8,6 @@
 .idea/**/tasks.xml
 .idea/**/usage.statistics.xml
 .idea/**/dictionaries
-.idea/**/shelf
+.idea/**/shelf
+
+litellm/redis-data

litellm/.env.sample

@@ -0,0 +1,4 @@
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_SSL=False
+OPENAI_API_KEY="YOUR_API_KEY"

litellm/README.md

@@ -0,0 +1,34 @@
+## LiteLLM Cache
+
+LiteLLM Cache is a proxy server designed to cache your LLM requests, helping to reduce costs and improve efficiency.
+
+### Requirements
+- Docker Compose
+- Docker
+
+### Setup Instructions
+
+1. **Configure Settings:**
+   - Navigate to `./config.yaml` and update the configuration as per your requirements. For more information, visit [LiteLLM Documentation](https://litellm.vercel.app/).
+
+2. **Prepare Environment Variables:**
+   - Create a `.env` file from the `.env.sample` file. Adjust the details in `.env` to match your `config.yaml` settings.
+
+3. **Start the Docker Container:**
+   ```bash
+   docker-compose up -d
+   ```
+
+4. **Update Your LLM Server URL:**
+   - Change the LLM calling server URL in your application to `http://0.0.0.0:4000`.
+
+   For example, using the OpenAI Python SDK:
+   ```python
+   from openai import OpenAI
+
+   llm = OpenAI(
+       base_url='http://0.0.0.0:4000'
+   )
+   ```
+
+With these steps, your LLM requests will be routed through the LiteLLM Cache proxy server, optimizing performance and reducing costs.

litellm/config.yaml

@@ -0,0 +1,9 @@
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo                          # The `openai/` prefix will call openai.chat.completions.create
+      api_key: os.environ/OPENAI_API_KEY                    # The `os.environ/` prefix will call os.environ.get
+
+router_settings:
+  redis_host: redis
+  redis_port: 6379

litellm/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+
+services:
+  litellm:
+    image: ghcr.io/berriai/litellm:main-latest
+    ports:
+      - "4000:4000"
+    volumes:
+      - ./config.yaml:/app/config.yaml # Mount the local configuration file
+    command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8", "--detailed_debug"]
+    env_file:
+      - .env
+
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - ./redis-data:/data
+    command: redis-server --appendonly yes
+    restart: always

py/.env.sample

@@ -0,0 +1,10 @@
+LANGCHAIN_API_KEY="YOUR-LANGCHAIN-API-KEY"
+LANGCHAIN_TRACING_V2=true
+LANGFUSE_HOST="https://us.cloud.langfuse.com"
+LANGFUSE_PUBLIC_KEY="YOUR-LANGFUSE-PUBLIC-KEY"
+LANGFUSE_SECRET_KEY="YOUR-LANGFUSE-SECRET-KEY"
+LANGSMITH_TEST_TRACKING=tests/cache/
+LUNARY_PUBLIC_KEY="YOUR-LUNARY-PUBLIC-KEY"
+OPENAI_API_KEY="YOUR-OPENAI-API"
+PAREA_API_KEY="YOUR-PAREA-API-KEY"
+BRAINTRUST_API_KEY="YOUR-BRAINTRUST-API-KEY"