Moved cache location and increased size.

huggingface · Dec 31, 2024 · b79d686 · b79d686
1 parent cb16cc5
commit b79d686
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/lighteval/models/transformers/base_model.py b/src/lighteval/models/transformers/base_model.py
@@ -273,7 +273,10 @@ def __init__(
 
         self.pairwise_tokenization = config.pairwise_tokenization
 
-        self._generation_cache = Cache(directory=os.path.join(env_config.cache_dir, "generation_cache"))
+        self._generation_cache = Cache(
+            directory=os.path.join(env_config.cache_dir, ".generation_cache"),
+            size_limit=10 * 1024**3,  # 10GB
+        )
 
     @classmethod
     def from_model(
@@ -860,6 +863,7 @@ def _generate_cached(
         do_sample: Optional[bool] = False,
     ) -> list[GenerativeResponse]:
         # Create a cache key from the inputs
+        # TODO: add model name to the cache key
         cache_key = (
             hashlib.sha256(str(batch.input_ids.tolist()).encode()).hexdigest(),
             max_new_tokens,