add max gib 16GB (#1362)

basetenlabs · Feb 3, 2025 · 0f15e3c · 0f15e3c
1 parent 9eafca8
commit 0f15e3c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/truss/base/constants.py b/truss/base/constants.py
@@ -33,7 +33,7 @@
 BEI_MAX_CONCURRENCY_TARGET_REQUESTS = 2048
 BEI_REQUIRED_MAX_NUM_TOKENS = 16384
 
-TRTLLM_MIN_MEMORY_REQUEST_GI = 24
+TRTLLM_MIN_MEMORY_REQUEST_GI = 16
 HF_MODELS_API_URL = "https://huggingface.co/api/models"
 HF_ACCESS_TOKEN_KEY = "hf_access_token"
 TRUSSLESS_MAX_PAYLOAD_SIZE = "64M"