From ee70d8ac8c50d3abf852f3820edc00f4c2df7514 Mon Sep 17 00:00:00 2001
From: Eternal Reclaimer <98760976+kyegomez@users.noreply.github.com>
Date: Wed, 20 Dec 2023 23:27:42 -0500
Subject: [PATCH] Delete tokenizer.py

---
 tokenizer.py | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 tokenizer.py

diff --git a/tokenizer.py b/tokenizer.py
deleted file mode 100644
index 9d04293..0000000
--- a/tokenizer.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from gemini_torch.tokenizer import MultimodalSentencePieceTokenizer
-
-# Example usage
-tokenizer_name = "hf-internal-testing/llama-tokenizer"
-tokenizer = MultimodalSentencePieceTokenizer(tokenizer_name=tokenizer_name)
-
-# Encoding and decoding examples
-encoded_audio = tokenizer.encode("Audio description", modality="audio")
-decoded_audio = tokenizer.decode(encoded_audio)
-
-print("Encoded audio:", encoded_audio)
-print("Decoded audio:", decoded_audio)