Add tests for humongous encodings

openai · Feb 11, 2024 · 814a09d · 814a09d
1 parent 1b9faf2
commit 814a09d
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
@@ -11,6 +11,12 @@
 from .test_helpers import ENCODING_FACTORIES, MAX_EXAMPLES
 
 
+def test_extremely_big_encoding():
+    big_value = "^" * 1000000
+    enc = tiktoken.get_encoding("r50k_base")
+    assert big_value == enc.decode(enc.encode(big_value))
+
+
 def test_simple():
     enc = tiktoken.get_encoding("gpt2")
     assert enc.encode("hello world") == [31373, 995]