diff --git a/src/lighteval/models/litellm_model.py b/src/lighteval/models/litellm_model.py
index 0d6954e3b..a1655e56b 100644
--- a/src/lighteval/models/litellm_model.py
+++ b/src/lighteval/models/litellm_model.py
@@ -121,6 +121,9 @@ def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_se
                 stop_sequence = self._prepare_stop_sequence(stop_sequence)
                 max_new_tokens = self._prepare_max_new_tokens(max_new_tokens)
 
+                if return_logits and not self.provider == "openai":
+                    logger.warning("Returning logits is not supported for this provider, ignoring.")
+
                 response = litellm.completion(
                     model=self.model,
                     messages=prompt,