Bug fix

runpod-workers · Dec 19, 2023 · 8202d4c · 8202d4c
1 parent f324bef
commit 8202d4c
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 5 deletions.
diff --git a/src/constants.py b/src/constants.py
@@ -1,6 +1,5 @@
 DEFAULT_BATCH_SIZE = 10
-
-MAX_CONCURRENCY = 100
+DEFAULT_MAX_CONCURRENCY = 100
 
 sampling_param_types = {
     'n': int,

diff --git a/src/utils.py b/src/utils.py
@@ -1,15 +1,15 @@
 import os
 from typing import Any, Dict, Optional, Union
 from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams
-from constants import sampling_param_types, DEFAULT_BATCH_SIZE, MAX_CONCURRENCY
+from constants import sampling_param_types, DEFAULT_BATCH_SIZE, DEFAULT_MAX_CONCURRENCY
 import logging
 
 logging.basicConfig(level=logging.INFO)
 
 class ServerlessConfig:
     def __init__(self):
-        self._max_concurrency = int(os.environ.get('MAX_CONCURRENCY', DEFAULT_BATCH_SIZE))
-        self._default_batch_size = int(os.environ.get('DEFAULT_BATCH_SIZE', MAX_CONCURRENCY))
+        self._max_concurrency = int(os.environ.get('MAX_CONCURRENCY', DEFAULT_MAX_CONCURRENCY))
+        self._default_batch_size = int(os.environ.get('DEFAULT_BATCH_SIZE', DEFAULT_BATCH_SIZE))
 
     @property
     def max_concurrency(self):