llm-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "47387681-beb4-4d52-8cf8-6a6ae3318a0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello,', ' ', 'world.', ' ', 'This,', ' ', 'is', ' ', 'a', ' ', 'test.']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "text = \"Hello, world. This, is a test.\"\n",
    "result = re.split(r'(\\s)', text)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8d4eb227-fab1-4955-925d-c2009b56cbd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', '', ' ', 'world', '.', '', ' ', 'This', ',', '', ' ', 'is', ' ', 'a', ' ', 'test', '.', '']\n"
     ]
    }
   ],
   "source": [
    "result = re.split(r'([,.]|\\s)', text)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "14fb7256-3332-4064-aeac-a443cbdc35dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', 'world', '.', 'This', ',', 'is', 'a', 'test', '.']\n"
     ]
    }
   ],
   "source": [
    "result = [item for item in result if item.strip()]\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ad75be3d-31df-4025-ab1a-70a874c5b164",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', ',', 'world', '.', 'Is', 'this', '--', 'a', 'test', '?']\n"
     ]
    }
   ],
   "source": [
    "text = \"Hello, world. Is this-- a test?\"\n",
    "result = re.split(r'([,.:;?_!\"()\\']|--|\\s)', text)\n",
    "result = [item.strip() for item in result if item.strip()]\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1a8fed2c-4bf5-4eb4-87f7-38db122df41f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of character: 20479\n",
      "I HAD always thought Jack Gisburn rather a cheap genius--though a good fellow enough--so it was no \n"
     ]
    }
   ],
   "source": [
    "with open(\"the-verdict.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    "print(\"Total number of character:\", len(raw_text))\n",
    "print(raw_text[:99])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "defcd660-b6da-4796-9207-f868a435f29a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4649\n"
     ]
    }
   ],
   "source": [
    "preprocessed = re.split(r'([,.?_!\"()\\']|--|\\s)', raw_text)\n",
    "preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "print(len(preprocessed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cc9b676a-c84b-4c45-adfc-fd8ed9a1cea9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'HAD', 'always', 'thought', 'Jack', 'Gisburn', 'rather', 'a', 'cheap', 'genius', '--', 'though', 'a', 'good', 'fellow', 'enough', '--', 'so', 'it', 'was', 'no', 'great', 'surprise', 'to', 'me', 'to', 'hear', 'that', ',', 'in']\n"
     ]
    }
   ],
   "source": [
    "print(preprocessed[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "46df9127-f230-4ae2-85c7-5bfa5da4dfc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1159\n"
     ]
    }
   ],
   "source": [
    "all_words = sorted(list(set(preprocessed)))\n",
    "vocab_size = len(all_words)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e883e3b-d8e8-40ce-8113-fca71f1353ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('!', 0)\n",
      "('\"', 1)\n",
      "(\"'\", 2)\n",
      "('(', 3)\n",
      "(')', 4)\n",
      "(',', 5)\n",
      "('--', 6)\n",
      "('.', 7)\n",
      "(':', 8)\n",
      "(';', 9)\n",
      "('?', 10)\n",
      "('A', 11)\n",
      "('Ah', 12)\n",
      "('Among', 13)\n",
      "('And', 14)\n",
      "('Are', 15)\n",
      "('Arrt', 16)\n",
      "('As', 17)\n",
      "('At', 18)\n",
      "('Be', 19)\n",
      "('Begin', 20)\n",
      "('Burlington', 21)\n",
      "('But', 22)\n",
      "('By', 23)\n",
      "('Carlo', 24)\n",
      "('Carlo;', 25)\n",
      "('Chicago', 26)\n",
      "('Claude', 27)\n",
      "('Come', 28)\n",
      "('Croft', 29)\n",
      "('Destroyed', 30)\n",
      "('Devonshire', 31)\n",
      "('Don', 32)\n",
      "('Dubarry', 33)\n",
      "('Emperors', 34)\n",
      "('Florence', 35)\n",
      "('For', 36)\n",
      "('Gallery', 37)\n",
      "('Gideon', 38)\n",
      "('Gisburn', 39)\n",
      "('Gisburns', 40)\n",
      "('Grafton', 41)\n",
      "('Greek', 42)\n",
      "('Grindle', 43)\n",
      "('Grindle:', 44)\n",
      "('Grindles', 45)\n",
      "('HAD', 46)\n",
      "('Had', 47)\n",
      "('Hang', 48)\n",
      "('Has', 49)\n",
      "('He', 50)\n",
      "('Her', 51)\n"
     ]
    }
   ],
   "source": [
    "vocab = {token:integer for integer,token in enumerate(all_words)}\n",
    "for i, item in enumerate(vocab.items()):\n",
    "    print(item)\n",
    "    if i > 50:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ae8bbb8b-72bc-49c7-b520-1b3fab4b213d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleTokenizerV1:\n",
    "    def __init__(self, vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = {i:s for s,i in vocab.items()}\n",
    "    \n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.?_!\"()\\']|--|\\s)', text)\n",
    "        preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "        \n",
    "    def decode(self, ids):\n",
    "        text = \" \".join([self.int_to_str[i] for i in ids]) \n",
    "        \n",
    "        text = re.sub(r'\\s+([,.?!\"()\\'])', r'\\1', text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3974974f-78d6-4512-8a72-9d02295adbf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 58, 2, 872, 1013, 615, 541, 763, 5, 1155, 608, 5, 1, 69, 7, 39, 873, 1136, 773, 812, 7]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = SimpleTokenizerV1(vocab)\n",
    " \n",
    "text = \"\"\"\"It's the last he painted, you know,\" Mrs. Gisburn said with pardonable pride.\"\"\"\n",
    "ids = tokenizer.encode(text)\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "689afdb1-0afa-43ad-867a-6478d04d2792",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\" It' s the last he painted, you know,\" Mrs. Gisburn said with pardonable pride.\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f1324c9e-a7fb-4791-9eb2-83228b03cb5d",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Hello'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[14], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHello, do you like tea?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[11], line 9\u001b[0m, in \u001b[0;36mSimpleTokenizerV1.encode\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m      8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstr_to_int[s] \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
      "Cell \u001b[0;32mIn[11], line 9\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      7\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m([,.?_!\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m()\u001b[39m\u001b[38;5;130;01m\\'\u001b[39;00m\u001b[38;5;124m]|--|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms)\u001b[39m\u001b[38;5;124m'\u001b[39m, text)\n\u001b[1;32m      8\u001b[0m preprocessed \u001b[38;5;241m=\u001b[39m [item\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m preprocessed \u001b[38;5;28;01mif\u001b[39;00m item\u001b[38;5;241m.\u001b[39mstrip()]\n\u001b[0;32m----> 9\u001b[0m ids \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr_to_int\u001b[49m\u001b[43m[\u001b[49m\u001b[43ms\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m preprocessed]\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ids\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Hello'"
     ]
    }
   ],
   "source": [
    "text = \"Hello, do you like tea?\"\n",
    "tokenizer.encode(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a20dc88-4c5b-4f35-9e3c-613016904e34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1161\n"
     ]
    }
   ],
   "source": [
    "all_tokens = sorted(list(set(preprocessed)))\n",
    "all_tokens.extend([\"<|endoftext|>\", \"<|unk|>\"])\n",
    "vocab = {token:integer for integer,token in enumerate(all_tokens)}\n",
    " \n",
    "print(len(vocab.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33bda269-b100-4211-b27c-ce64b2e8faa3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('younger', 1156)\n",
      "('your', 1157)\n",
      "('yourself', 1158)\n",
      "('<|endoftext|>', 1159)\n",
      "('<|unk|>', 1160)\n"
     ]
    }
   ],
   "source": [
    "for i, item in enumerate(list(vocab.items())[-5:]):\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89a4a61c-811c-4472-b6ea-e75414c50856",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleTokenizerV2:\n",
    "    def __init__(self, vocab):\n",
    "        self.str_to_int = vocab\n",
    "        self.int_to_str = { i:s for s,i in vocab.items()}\n",
    "    \n",
    "    def encode(self, text):\n",
    "        preprocessed = re.split(r'([,.?_!\"()\\']|--|\\s)', text)\n",
    "        preprocessed = [item.strip() for item in preprocessed if item.strip()]\n",
    "        preprocessed = [item if item in self.str_to_int\n",
    "                        else \"<|unk|>\" for item in preprocessed]\n",
    " \n",
    "        ids = [self.str_to_int[s] for s in preprocessed]\n",
    "        return ids\n",
    "        \n",
    "    def decode(self, ids):\n",
    "        text = \" \".join([self.int_to_str[i] for i in ids])\n",
    " \n",
    "        text = re.sub(r'\\s+([,.?!\"()\\'])', r'\\1', text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb3e178d-e9d7-4860-ba2c-4e5111bd8354",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello, do you like tea? <|endoftext|> In the sunlit terraces of the palace.\n"
     ]
    }
   ],
   "source": [
    "text1 = \"Hello, do you like tea?\"\n",
    "text2 = \"In the sunlit terraces of the palace.\"\n",
    "text = \" <|endoftext|> \".join((text1, text2))\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a30d5cbd-3c00-4f4c-bb00-c742b8a45221",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1160, 5, 362, 1155, 642, 1000, 10, 1159, 57, 1013, 981, 1009, 738, 1013, 1160, 7]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = SimpleTokenizerV2(vocab)\n",
    "print(tokenizer.encode(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ff63c01-473f-4a7a-b26e-c5a918ae7043",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|unk|>, do you like tea? <|endoftext|> In the sunlit terraces of the <|unk|>.\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(tokenizer.encode(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd74d78d-c794-4617-8634-64a659cd2ebf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tiktoken\n",
      "  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
      "Collecting regex>=2022.1.18 (from tiktoken)\n",
      "  Downloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.9/40.9 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.26.0 in /home/codespace/.local/lib/python3.10/site-packages (from tiktoken) (2.31.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2024.2.2)\n",
      "Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (775 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m775.1/775.1 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: regex, tiktoken\n",
      "Successfully installed regex-2024.5.15 tiktoken-0.7.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6450d9f-8d22-42c4-ac23-6ccb77ea7ff4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tiktoken version: 0.7.0\n"
     ]
    }
   ],
   "source": [
    "from importlib.metadata import version\n",
    "import tiktoken\n",
    "print(\"tiktoken version:\", version(\"tiktoken\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e3eb0d5-a981-45e3-8816-ccd381d7e4fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = tiktoken.get_encoding(\"gpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44195e18-d32c-4dd1-8641-c97845f689dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15496, 11, 466, 345, 588, 8887, 30, 220, 50256, 554, 262, 4252, 18250, 8812, 2114, 286, 617, 34680, 27271, 13]\n"
     ]
    }
   ],
   "source": [
    "text = \"Hello, do you like tea? <|endoftext|> In the sunlit terraces of someunknownPlace.\"\n",
    "integers = tokenizer.encode(text, allowed_special={\"<|endoftext|>\"})\n",
    "print(integers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8651405d-fdc5-4065-9f5b-b0f833f640dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello, do you like tea? <|endoftext|> In the sunlit terraces of someunknownPlace.\n"
     ]
    }
   ],
   "source": [
    "strings = tokenizer.decode(integers)\n",
    "print(strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5146e37f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5145\n"
     ]
    }
   ],
   "source": [
    "with open(\"the-verdict.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    " \n",
    "enc_text = tokenizer.encode(raw_text)\n",
    "print(len(enc_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cef76e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_sample = enc_text[50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29828fed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: [290, 4920, 2241, 287]\n",
      "y:      [4920, 2241, 287, 257]\n"
     ]
    }
   ],
   "source": [
    "context_size = 4\n",
    "\n",
    "x = enc_sample[:context_size]\n",
    "y = enc_sample[1:context_size+1]\n",
    "print(f\"x: {x}\")\n",
    "print(f\"y:      {y}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afcc5add",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[290] ----> 4920\n",
      "[290, 4920] ----> 2241\n",
      "[290, 4920, 2241] ----> 287\n",
      "[290, 4920, 2241, 287] ----> 257\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, context_size+1):\n",
    "    context = enc_sample[:i]\n",
    "    desired = enc_sample[i]\n",
    "    print(context, \"---->\", desired)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b8c9e1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " and ---->  established\n",
      " and established ---->  himself\n",
      " and established himself ---->  in\n",
      " and established himself in ---->  a\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, context_size+1):\n",
    "    context = enc_sample[:i]\n",
    "    desired = enc_sample[i]\n",
    "    print(tokenizer.decode(context), \"---->\", tokenizer.decode([desired]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d343e5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    " \n",
    "class GPTDatasetV1(Dataset):\n",
    "    def __init__(self, txt, tokenizer, max_length, stride):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.input_ids = []\n",
    "        self.target_ids = []\n",
    " \n",
    "        token_ids = tokenizer.encode(txt)\n",
    " \n",
    "        for i in range(0, len(token_ids) - max_length, stride):\n",
    "            input_chunk = token_ids[i:i + max_length]\n",
    "            target_chunk = token_ids[i + 1: i + max_length + 1]\n",
    "            self.input_ids.append(torch.tensor(input_chunk))\n",
    "            self.target_ids.append(torch.tensor(target_chunk))\n",
    " \n",
    "    def __len__(self):\n",
    "        return len(self.input_ids)\n",
    " \n",
    "    def __getitem__(self, idx):\n",
    "        return self.input_ids[idx], self.target_ids[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78aa65cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataloader_v1(txt, batch_size=4, \n",
    "        max_length=256, stride=128, shuffle=True, drop_last=True):\n",
    "    tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
    "    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)\n",
    "    dataloader = DataLoader(\n",
    "        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)\n",
    "    return dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4e5d45d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([[  40,  367, 2885, 1464]]), tensor([[ 367, 2885, 1464, 1807]])]\n"
     ]
    }
   ],
   "source": [
    "with open(\"the-verdict.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    " \n",
    "dataloader = create_dataloader_v1(\n",
    "    raw_text, batch_size=1, max_length=4, stride=1, shuffle=False)\n",
    "data_iter = iter(dataloader)\n",
    "first_batch = next(data_iter)\n",
    "print(first_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "554e2788",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    " \n",
    "class GPTDatasetV1(Dataset):\n",
    "    def __init__(self, txt, tokenizer, max_length, stride):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.input_ids = []\n",
    "        self.target_ids = []\n",
    " \n",
    "        token_ids = tokenizer.encode(txt)\n",
    " \n",
    "        for i in range(0, len(token_ids) - max_length, stride):\n",
    "            input_chunk = token_ids[i:i + max_length]\n",
    "            target_chunk = token_ids[i + 1: i + max_length + 1]\n",
    "            self.input_ids.append(torch.tensor(input_chunk))\n",
    "            self.target_ids.append(torch.tensor(target_chunk))\n",
    " \n",
    "    def __len__(self):\n",
    "        return len(self.input_ids)\n",
    " \n",
    "    def __getitem__(self, idx):\n",
    "        return self.input_ids[idx], self.target_ids[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c86b2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataloader_v1(txt, batch_size=4, \n",
    "        max_length=256, stride=128, shuffle=True, drop_last=True):\n",
    "    tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
    "    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)\n",
    "    dataloader = DataLoader(\n",
    "        dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)\n",
    "    return dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6656055a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([[  40,  367, 2885, 1464]]), tensor([[ 367, 2885, 1464, 1807]])]\n"
     ]
    }
   ],
   "source": [
    "with open(\"the-verdict.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    raw_text = f.read()\n",
    " \n",
    "dataloader = create_dataloader_v1(\n",
    "    raw_text, batch_size=1, max_length=4, stride=1, shuffle=False)\n",
    "data_iter = iter(dataloader)\n",
    "first_batch = next(data_iter)\n",
    "print(first_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8330a6a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([[ 367, 2885, 1464, 1807]]), tensor([[2885, 1464, 1807, 3619]])]\n"
     ]
    }
   ],
   "source": [
    "second_batch = next(data_iter)\n",
    "print(second_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3e12d6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inputs:\n",
      " tensor([[  276,   284,  3960,   503],\n",
      "        [ 2951,  6348, 23840,   284],\n",
      "        [  465,  3656,   338,  1263],\n",
      "        [  587,   973,   355,   257],\n",
      "        [ 2147,   438,    40,  1297],\n",
      "        [  625,   422, 22489, 40089],\n",
      "        [30960,   198,   198,     1],\n",
      "        [  262, 13476,   286,   616]])\n",
      "\n",
      "Targets:\n",
      " tensor([[  284,  3960,   503,    25],\n",
      "        [ 6348, 23840,   284,   262],\n",
      "        [ 3656,   338,  1263,  5236],\n",
      "        [  973,   355,   257,  8034],\n",
      "        [  438,    40,  1297,  9074],\n",
      "        [  422, 22489, 40089,    26],\n",
      "        [  198,   198,     1,  1026],\n",
      "        [13476,   286,   616, 12036]])\n"
     ]
    }
   ],
   "source": [
    "dataloader = create_dataloader_v1(raw_text, batch_size=8, max_length=4, stride=4)\n",
    " \n",
    "data_iter = iter(dataloader)\n",
    "inputs, targets = next(data_iter)\n",
    "print(\"Inputs:\\n\", inputs)\n",
    "print(\"\\nTargets:\\n\", targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e64a1873",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor([2, 3, 5, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cf38751",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 6\n",
    "output_dim = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de68e852",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[ 0.3374, -0.1778, -0.1690],\n",
      "        [ 0.9178,  1.5810,  1.3010],\n",
      "        [ 1.2753, -0.2010, -0.1606],\n",
      "        [-0.4015,  0.9666, -1.1481],\n",
      "        [-1.1589,  0.3255, -0.6315],\n",
      "        [-2.8400, -0.7849, -1.4096]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(123)\n",
    "embedding_layer = torch.nn.Embedding(vocab_size, output_dim)\n",
    "print(embedding_layer.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf99ce21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.4015,  0.9666, -1.1481]], grad_fn=<EmbeddingBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(embedding_layer(torch.tensor([3])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7c9e1b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dim = 256\n",
    "vocab_size = 50257\n",
    "token_embedding_layer = torch.nn.Embedding(vocab_size, output_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34bea2bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token IDs:\n",
      " tensor([[   40,   367,  2885,  1464],\n",
      "        [ 1807,  3619,   402,   271],\n",
      "        [10899,  2138,   257,  7026],\n",
      "        [15632,   438,  2016,   257],\n",
      "        [  922,  5891,  1576,   438],\n",
      "        [  568,   340,   373,   645],\n",
      "        [ 1049,  5975,   284,   502],\n",
      "        [  284,  3285,   326,    11]])\n",
      "\n",
      "Inputs shape:\n",
      " torch.Size([8, 4])\n"
     ]
    }
   ],
   "source": [
    "max_length = 4\n",
    "dataloader = create_dataloader_v1(\n",
    "    raw_text, batch_size=8, max_length=max_length, stride=max_length, shuffle=False)\n",
    "data_iter = iter(dataloader)\n",
    "inputs, targets = next(data_iter)\n",
    "print(\"Token IDs:\\n\", inputs)\n",
    "print(\"\\nInputs shape:\\n\", inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e18d3dd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 4, 256])\n"
     ]
    }
   ],
   "source": [
    "token_embeddings = token_embedding_layer(inputs)\n",
    "print(token_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a5fa99d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4, 256])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Size([8, 4, 256])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59a4428b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 256])\n"
     ]
    }
   ],
   "source": [
    "context_length = max_length\n",
    "pos_embedding_layer = torch.nn.Embedding(context_length, output_dim)\n",
    "pos_embeddings = pos_embedding_layer(torch.arange(context_length))\n",
    "print(pos_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad340a5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 256])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Size([4, 256])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2e1bd04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 4, 256])\n"
     ]
    }
   ],
   "source": [
    "input_embeddings = token_embeddings + pos_embeddings\n",
    "print(input_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "006affa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "inputs = torch.tensor(\n",
    "  [[0.43, 0.15, 0.89], # Your     (x^1)\n",
    "   [0.55, 0.87, 0.66], # journey  (x^2)\n",
    "   [0.57, 0.85, 0.64], # starts   (x^3)\n",
    "   [0.22, 0.58, 0.33], # with     (x^4)\n",
    "   [0.77, 0.25, 0.10], # one      (x^5)\n",
    "   [0.05, 0.80, 0.55]] # step     (x^6)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b71a0fcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.9544, 1.4950, 1.4754, 0.8434, 0.7070, 1.0865])\n"
     ]
    }
   ],
   "source": [
    "query = inputs[1]\n",
    "attn_scores_2 = torch.empty(inputs.shape[0])\n",
    "for i, x_i in enumerate(inputs):\n",
    "    attn_scores_2[i] = torch.dot(x_i, query)\n",
    "print(attn_scores_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fb38971",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.9544)\n",
      "tensor(0.9544)\n"
     ]
    }
   ],
   "source": [
    "res = 0.\n",
    "\n",
    "for idx, element in enumerate(inputs[0]):\n",
    "    res += inputs[0][idx] * query[idx]\n",
    "print(res)\n",
    "print(torch.dot(inputs[0], query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4003b6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights: tensor([0.1455, 0.2278, 0.2249, 0.1285, 0.1077, 0.1656])\n",
      "Sum: tensor(1.0000)\n"
     ]
    }
   ],
   "source": [
    "attn_weights_2_tmp = attn_scores_2 / attn_scores_2.sum()\n",
    "print(\"Attention weights:\", attn_weights_2_tmp)\n",
    "print(\"Sum:\", attn_weights_2_tmp.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dfa0cdf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights: tensor([0.1385, 0.2379, 0.2333, 0.1240, 0.1082, 0.1581])\n",
      "Sum: tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "def softmax_naive(x):\n",
    "    return torch.exp(x) / torch.exp(x).sum(dim=0)\n",
    " \n",
    "attn_weights_2_naive = softmax_naive(attn_scores_2)\n",
    "print(\"Attention weights:\", attn_weights_2_naive)\n",
    "print(\"Sum:\", attn_weights_2_naive.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3830225f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights: tensor([0.1385, 0.2379, 0.2333, 0.1240, 0.1082, 0.1581])\n",
      "Sum: tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "attn_weights_2 = torch.softmax(attn_scores_2, dim=0)\n",
    "print(\"Attention weights:\", attn_weights_2)\n",
    "print(\"Sum:\", attn_weights_2.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dde9bced",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4419, 0.6515, 0.5683])\n"
     ]
    }
   ],
   "source": [
    "query = inputs[1] # 2nd input token is the query\n",
    "context_vec_2 = torch.zeros(query.shape)\n",
    "for i,x_i in enumerate(inputs):\n",
    "    context_vec_2 += attn_weights_2[i]*x_i\n",
    "print(context_vec_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7e4dbfd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in /home/codespace/.local/lib/python3.10/site-packages (2.3.0+cpu)\n",
      "Requirement already satisfied: tensorflow in /usr/local/python/3.10.13/lib/python3.10/site-packages (2.16.1)\n",
      "Collecting tqdm\n",
      "  Downloading tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: filelock in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.14.0)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (4.11.0)\n",
      "Requirement already satisfied: sympy in /home/codespace/.local/lib/python3.10/site-packages (from torch) (1.12)\n",
      "Requirement already satisfied: networkx in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.3)\n",
      "Requirement already satisfied: jinja2 in /home/codespace/.local/lib/python3.10/site-packages (from torch) (3.1.4)\n",
      "Requirement already satisfied: fsspec in /home/codespace/.local/lib/python3.10/site-packages (from torch) (2024.3.1)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (2.1.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (24.3.25)\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (0.5.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: h5py>=3.10.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (3.11.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (18.1.1)\n",
      "Requirement already satisfied: ml-dtypes~=0.3.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (0.3.2)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: packaging in /home/codespace/.local/lib/python3.10/site-packages (from tensorflow) (24.0)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (4.25.3)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /home/codespace/.local/lib/python3.10/site-packages (from tensorflow) (2.31.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (68.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /home/codespace/.local/lib/python3.10/site-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (2.4.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (1.64.0)\n",
      "Requirement already satisfied: tensorboard<2.17,>=2.16 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (2.16.2)\n",
      "Requirement already satisfied: keras>=3.0.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (3.3.3)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorflow) (0.37.0)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /home/codespace/.local/lib/python3.10/site-packages (from tensorflow) (1.26.4)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from astunparse>=1.6.0->tensorflow) (0.43.0)\n",
      "Requirement already satisfied: rich in /usr/local/python/3.10.13/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow) (13.7.1)\n",
      "Requirement already satisfied: namex in /usr/local/python/3.10.13/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow) (0.0.8)\n",
      "Requirement already satisfied: optree in /usr/local/python/3.10.13/lib/python3.10/site-packages (from keras>=3.0.0->tensorflow) (0.11.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/codespace/.local/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/codespace/.local/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/codespace/.local/lib/python3.10/site-packages (from requests<3,>=2.21.0->tensorflow) (2024.2.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow) (3.6)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from tensorboard<2.17,>=2.16->tensorflow) (3.0.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/codespace/.local/lib/python3.10/site-packages (from jinja2->torch) (2.1.5)\n",
      "Requirement already satisfied: mpmath>=0.19 in /home/codespace/.local/lib/python3.10/site-packages (from sympy->torch) (1.3.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from rich->keras>=3.0.0->tensorflow) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/codespace/.local/lib/python3.10/site-packages (from rich->keras>=3.0.0->tensorflow) (2.18.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /usr/local/python/3.10.13/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.0.0->tensorflow) (0.1.2)\n",
      "Downloading tqdm-4.66.4-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.3/78.3 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: tqdm\n",
      "Successfully installed tqdm-4.66.4\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install torch tensorflow tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab74d685",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "import torch\n",
    "from previous_chapters import GPTModel\n",
    "\n",
    "GPT_CONFIG_124M = {\n",
    "    \"vocab_size\": 50257,   # Vocabulary size\n",
    "    \"context_length\": 256, # Shortened context length (orig: 1024)\n",
    "    \"emb_dim\": 768,        # Embedding dimension\n",
    "    \"n_heads\": 12,         # Number of attention heads\n",
    "    \"n_layers\": 12,        # Number of layers\n",
    "    \"drop_rate\": 0.1,      # Dropout rate\n",
    "    \"qkv_bias\": False      # Query-key-value bias\n",
    "}\n",
    "\n",
    "\n",
    "torch.manual_seed(123)\n",
    "\n",
    "tokenizer = tiktoken.get_encoding(\"gpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f2aaaf9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-05-30 02:31:09.675834: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-30 02:31:10.560742: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n",
      "2024-05-30 02:31:12.404580: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-05-30 02:31:15.513639: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "checkpoint: 100%|██████████| 77.0/77.0 [00:00<00:00, 115kiB/s]\n",
      "encoder.json: 100%|██████████| 1.04M/1.04M [00:00<00:00, 5.19MiB/s]\n",
      "hparams.json: 100%|██████████| 90.0/90.0 [00:00<00:00, 126kiB/s]\n",
      "model.ckpt.data-00000-of-00001: 100%|██████████| 498M/498M [00:09<00:00, 50.0MiB/s] \n",
      "model.ckpt.index: 100%|██████████| 5.21k/5.21k [00:00<00:00, 11.0MiB/s]\n",
      "model.ckpt.meta: 100%|██████████| 471k/471k [00:00<00:00, 3.30MiB/s]\n",
      "vocab.bpe: 100%|██████████| 456k/456k [00:00<00:00, 4.03MiB/s]\n",
      "2024-05-30 02:31:33.004250: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 154389504 exceeds 10% of free system memory.\n"
     ]
    }
   ],
   "source": [
    "from gpt_download import download_and_load_gpt2\n",
    "from gpt_generate import load_weights_into_gpt\n",
    "\n",
    "\n",
    "model_configs = {\n",
    "    \"gpt2-small (124M)\": {\"emb_dim\": 768, \"n_layers\": 12, \"n_heads\": 12},\n",
    "    \"gpt2-medium (355M)\": {\"emb_dim\": 1024, \"n_layers\": 24, \"n_heads\": 16},\n",
    "    \"gpt2-large (774M)\": {\"emb_dim\": 1280, \"n_layers\": 36, \"n_heads\": 20},\n",
    "    \"gpt2-xl (1558M)\": {\"emb_dim\": 1600, \"n_layers\": 48, \"n_heads\": 25},\n",
    "}\n",
    "\n",
    "model_name = \"gpt2-small (124M)\"\n",
    "NEW_CONFIG = GPT_CONFIG_124M.copy()\n",
    "NEW_CONFIG.update(model_configs[model_name])\n",
    "NEW_CONFIG.update({\"context_length\": 1024, \"qkv_bias\": True})\n",
    "\n",
    "gpt = GPTModel(NEW_CONFIG)\n",
    "gpt.eval()\n",
    "\n",
    "settings, params = download_and_load_gpt2(model_size=\"124M\", models_dir=\"gpt2\")\n",
    "load_weights_into_gpt(gpt, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "73bad1d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "from previous_chapters import create_dataloader_v1\n",
    "\n",
    "\n",
    "file_path = \"the-verdict.txt\"\n",
    "url = \"https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/main/ch02/01_main-chapter-code/the-verdict.txt\"\n",
    "\n",
    "if not os.path.exists(file_path):\n",
    "    with urllib.request.urlopen(url) as response:\n",
    "        text_data = response.read().decode('utf-8')\n",
    "    with open(file_path, \"w\", encoding=\"utf-8\") as file:\n",
    "        file.write(text_data)\n",
    "else:\n",
    "    with open(file_path, \"r\", encoding=\"utf-8\") as file:\n",
    "        text_data = file.read()\n",
    "\n",
    "\n",
    "# Train/validation ratio\n",
    "train_ratio = 0.90\n",
    "split_idx = int(train_ratio * len(text_data))\n",
    "train_data = text_data[:split_idx]\n",
    "val_data = text_data[split_idx:]\n",
    "\n",
    "\n",
    "torch.manual_seed(123)\n",
    "\n",
    "train_loader = create_dataloader_v1(\n",
    "    train_data,\n",
    "    batch_size=2,\n",
    "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
    "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
    "    drop_last=True,\n",
    "    shuffle=True,\n",
    "    num_workers=0\n",
    ")\n",
    "\n",
    "val_loader = create_dataloader_v1(\n",
    "    val_data,\n",
    "    batch_size=2,\n",
    "    max_length=GPT_CONFIG_124M[\"context_length\"],\n",
    "    stride=GPT_CONFIG_124M[\"context_length\"],\n",
    "    drop_last=False,\n",
    "    shuffle=False,\n",
    "    num_workers=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b86be1ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gpt_generate import assign, load_weights_into_gpt\n",
    "\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "load_weights_into_gpt(gpt, params)\n",
    "gpt.to(device);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd267192",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training loss: 3.7547490066952176\n",
      "Validation loss: 3.5596206188201904\n"
     ]
    }
   ],
   "source": [
    "from gpt_train import calc_loss_loader\n",
    "\n",
    "torch.manual_seed(123) # For reproducibility due to the shuffling in the data loader\n",
    "train_loss = calc_loss_loader(train_loader, gpt, device)\n",
    "val_loss = calc_loss_loader(val_loader, gpt, device)\n",
    "\n",
    "print(\"Training loss:\", train_loss)\n",
    "print(\"Validation loss:\", val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a9fcc392",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output text:\n",
      " What do you know about Wikipedia?\n",
      "\n",
      "Wikipedia is a website run from Wikia by Wikipedia Foundation, published by Wikipedia. Its primary mission is to provide the\n"
     ]
    }
   ],
   "source": [
    "from gpt_generate import generate, text_to_token_ids, token_ids_to_text\n",
    "from previous_chapters import generate_text_simple\n",
    "import torch\n",
    "\n",
    "torch.manual_seed(123)\n",
    "\n",
    "token_ids = generate(\n",
    "    model=gpt,\n",
    "    idx=text_to_token_ids(\"What do you know about Wikipedia?\", tokenizer),\n",
    "    max_new_tokens=25,\n",
    "    context_size=256,\n",
    "    top_k=50,\n",
    "    temperature=1\n",
    ")\n",
    "\n",
    "print(\"Output text:\\n\", token_ids_to_text(token_ids, tokenizer))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}