headers corrections

src/cpp/include/openvino/genai/tokenizer.hpp

@@ -127,7 +127,7 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
     * @param prompt std::string with input prompt
     * @param add_special_tokens whether to add special tokens
     * @param max_length maximum length to which output will be padded or truncated
-    * @param padding_mode whether to pad result, allowed values are ["truncate", "longest", "max_length", "do_not_pad"]
+    * @param padding_mode either PaddingMode::TRUNCATE, PaddingMode::LONGEST, or PaddingMode::MAX_LENGTH.
     * @return pair of [input_ids, attention_mask]
     */
     template <typename... Properties>
@@ -140,7 +140,7 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
     * @param prompts vector storing batch of prompts
     * @param add_special_tokens whether to add special tokens
     * @param max_length maximum length to which output will be padded or truncated
-    * @param padding_mode whether to pad result, allowed values are ["truncate", "pad"]
+    * @param padding_mode either PaddingMode::TRUNCATE, PaddingMode::LONGEST, or PaddingMode::MAX_LENGTH.
     * @return pair of [input_ids, attention_mask]
     */
     template <typename... Properties>
@@ -178,7 +178,7 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
     /**
     * @brief decode sequence of tokens
     * @param tokens ov::Tensor with tokens with shape [batch_size, seq_len]
-    * @param detokenization_params detokenization parameters,  e.g. ov::genai::skip_special_tokens(true)
+    * @param detokenization_params detokenization parameters,  e.g. ov::genai::skip_special_tokens(true)s
     * @return vector of std::string, with size = batch_size
     */
     template <typename... Properties>
@@ -250,8 +250,10 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
  * @brief Enum class representing different padding modes for tokenization.
  *
  * This enum class defines modes that can be used to pad tokenized sequences. 
- * IMPORTANT NOTICE, even in truncation mode the padding is applied to the longest sequence in the batch
+ * IMPORTANT NOTICE: even in TRUNCATE mode the padding is applied to the longest sequence in the batch
  * since resulting tokenization is stored as a signe ov::Tensor which cannot store irregular/ragged array.
+ * For the same reason also in MAX_LENGTH mode truncation is also applied so that we have the same lengths
+ * for each sequence in the batch.
  *
  * @var PaddingMode::TRUNCATE
  * Truncate the sequence to the maximum length specified. (But also LONGEST mode is implicitly applied.)
@@ -260,7 +262,8 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
  * Pad the sequence to the length of the longest sequence in the batch. In this mode truncation is switched off.
  *
  * @var PaddingMode::MAX_LENGTH
- * Pad the sequence to a specified maximum length. In this mode truncation is switched off.
+ * Pad the sequence to a specified maximum length. In this mode truncation is switched on as well 
+ * so that the resulting sequences are of the same length.
  */
 enum class PaddingMode { TRUNCATE, LONGEST, MAX_LENGTH };
 

src/cpp/src/tokenizer.cpp

@@ -116,12 +116,12 @@ class Tokenizer::TokenizerImpl {
             && max_pad_length_val == m_max_pad_length) {
             return;
         }
-        // if (m_older_than_24_5) {
-        //     // Changing add_special_tokens at runtime was introduced in
-        //     // 24.5. Older tokenizers still allow manipulating their
-        //     // state but the effect is incorrect.
-        //     return;
-        // }
+        if (m_older_than_24_5) {
+            // Changing add_special_tokens at runtime was introduced in
+            // 24.5. Older tokenizers still allow manipulating their
+            // state but the effect is incorrect.
+            return;
+        }
 
         // add_special_tokens is managed by Select op with a bool input.
         ov::Tensor add_special_tensor = ov::Tensor(ov::element::boolean, {});
@@ -137,9 +137,9 @@ class Tokenizer::TokenizerImpl {
         *max_pad_length_tensor.data<int>() = max_pad_length_val;
 
         bool set_padding = max_length_val.has_value();
-        // Even if max_length is not set in order to disable truncation
+        // Even if max_length is not set, in order to disable truncation in LONGEST mode
         // MAX_TRUNCATION_LENGTH_VAR_ID should be updated to max numeric limit.
-        bool set_truncation = padding_mode_val != PaddingMode::TRUNCATE || max_length_val.has_value();
+        bool set_truncation = padding_mode_val == PaddingMode::LONGEST || max_length_val.has_value();
 
         for (auto& state: infer_request_guard.get().query_state()) {
             if (state.get_name().find(add_special_tokens.name()) != std::string::npos) {

src/python/openvino_genai/__init__.py

@@ -47,7 +47,8 @@
 # Tokenizers
 from .py_openvino_genai import (
     TokenizedInputs,
-    Tokenizer
+    Tokenizer,
+    PaddingMode
 )
 
 # Whisper