diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py
index 3398f7218..d9da629e5 100644
--- a/src/lighteval/models/vllm/vllm_model.py
+++ b/src/lighteval/models/vllm/vllm_model.py
@@ -37,7 +37,7 @@
     GenerativeResponse,
     LoglikelihoodResponse,
 )
-from lighteval.models.utils import _get_dtype, _simplify_name
+from lighteval.models.utils import _get_dtype, _get_model_sha, _simplify_name
 from lighteval.tasks.requests import (
     GreedyUntilRequest,
     LoglikelihoodRequest,
@@ -100,6 +100,9 @@ def __post_init__(self):
         if not self.generation_parameters:
             self.generation_parameters = GenerationParameters()
 
+    def get_model_sha(self):
+        return _get_model_sha(repo_id=self.pretrained, revision=self.revision)
+
 
 class VLLMModel(LightevalModel):
     def __init__(
@@ -124,10 +127,10 @@ def __init__(
         self.multichoice_continuations_start_space = config.multichoice_continuations_start_space
 
         self.model_name = _simplify_name(config.pretrained)
-        self.model_sha = ""  # config.get_model_sha()
+        self.model_sha = config.get_model_sha()
         self.precision = _get_dtype(config.dtype, config=self._config)
 
-        self.model_info = ModelInfo(model_name=self.model_name, model_sha=self.model_sha)
+        self.model_info = ModelInfo(model_name=self.model_name, model_sha=self.model_sha, model_dtype=config.dtype)
         self.sampling_params = SamplingParams(**config.generation_parameters.to_vllm_openai_dict())
         self.pairwise_tokenization = config.pairwise_tokenization
 
@@ -203,7 +206,7 @@ def _create_auto_tokenizer(self, config: VLLMModelConfig, env_config: EnvConfig)
             config.pretrained,
             tokenizer_mode="auto",
             trust_remote_code=config.trust_remote_code,
-            tokenizer_revision=config.revision,
+            revision=config.revision + (f"/{config.subfolder}" if config.subfolder is not None else ""),
         )
         tokenizer.pad_token = tokenizer.eos_token
         return tokenizer