RAG_over_documents_using_Opensource_models_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9504de4bb06444d4987c63db433faacc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_283a739603df4572b95d5d2eb8ddbd74",
              "IPY_MODEL_c04dee5757c143338f45373a2617f9a7",
              "IPY_MODEL_90f7b541d579401b96f2f331160fba15"
            ],
            "layout": "IPY_MODEL_5f14f79471ed412d9614c031572cf062"
          }
        },
        "283a739603df4572b95d5d2eb8ddbd74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d1d05f54ae7446f49297fa3950db8d0d",
            "placeholder": "​",
            "style": "IPY_MODEL_fdc46a5ffc854a2baff46ac27db8c166",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "c04dee5757c143338f45373a2617f9a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1cf7e0b365d2403290c7e31bbcef9df6",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_90e95d938a30493b92e66aca951f5ab2",
            "value": 2
          }
        },
        "90f7b541d579401b96f2f331160fba15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf0a8c2b4f844babae54438dcc1ea404",
            "placeholder": "​",
            "style": "IPY_MODEL_3b7190810e084eb982eac8f9926042d1",
            "value": " 2/2 [01:03&lt;00:00, 29.05s/it]"
          }
        },
        "5f14f79471ed412d9614c031572cf062": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1d05f54ae7446f49297fa3950db8d0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdc46a5ffc854a2baff46ac27db8c166": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1cf7e0b365d2403290c7e31bbcef9df6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90e95d938a30493b92e66aca951f5ab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bf0a8c2b4f844babae54438dcc1ea404": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b7190810e084eb982eac8f9926042d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kyk-001/RAG_using_llamaindex/blob/main/RAG_over_documents_using_Opensource_models_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install llama-index transformers accelerate bitsandbytes pypdf"
      ],
      "metadata": {
        "id": "V1DwVkHCNDgT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3804cf9b-5d20-4ff0-924f-ad46834fc13c"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama-index\n",
            "  Downloading llama_index-0.9.3.post1-py3-none-any.whl (886 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m887.0/887.0 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Collecting accelerate\n",
            "  Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting bitsandbytes\n",
            "  Downloading bitsandbytes-0.41.2.post2-py3-none-any.whl (92.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index) (2.0.23)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index) (3.8.6)\n",
            "Collecting aiostream<0.6.0,>=0.5.2 (from llama-index)\n",
            "  Downloading aiostream-0.5.2-py3-none-any.whl (39 kB)\n",
            "Collecting beautifulsoup4<5.0.0,>=4.12.2 (from llama-index)\n",
            "  Downloading beautifulsoup4-4.12.2-py3-none-any.whl (142 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dataclasses-json<0.6.0,>=0.5.7 (from llama-index)\n",
            "  Downloading dataclasses_json-0.5.14-py3-none-any.whl (26 kB)\n",
            "Collecting deprecated>=1.2.9.3 (from llama-index)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index) (2023.6.0)\n",
            "Collecting httpx (from llama-index)\n",
            "  Downloading httpx-0.25.1-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.0/75.0 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index) (1.5.8)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index) (3.8.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from llama-index) (1.23.5)\n",
            "Collecting openai>=1.1.0 (from llama-index)\n",
            "  Downloading openai-1.3.3-py3-none-any.whl (220 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m220.3/220.3 kB\u001b[0m \u001b[31m30.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas[jinja2] in /usr/local/lib/python3.10/dist-packages (from llama-index) (1.5.3)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index) (8.2.3)\n",
            "Collecting tiktoken>=0.3.3 (from llama-index)\n",
            "  Downloading tiktoken-0.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m97.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index) (4.5.0)\n",
            "Collecting typing-inspect>=0.8.0 (from llama-index)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Collecting urllib3<2 (from llama-index)\n",
            "  Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu118)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (3.3.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.3.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.2->llama-index) (2.5)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.6.0,>=0.5.7->llama-index)\n",
            "  Downloading marshmallow-3.20.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.9.3->llama-index) (1.14.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index) (1.3.2)\n",
            "Requirement already satisfied: anyio<4,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index) (1.7.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index) (1.10.13)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index) (2023.7.22)\n",
            "Collecting httpcore (from httpx->llama-index)\n",
            "  Downloading httpcore-1.0.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index) (3.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index) (1.3.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index) (3.0.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "\u001b[33mWARNING: pandas 1.5.3 does not provide the extra 'jinja2'\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas[jinja2]->llama-index) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas[jinja2]->llama-index) (2023.3.post1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai>=1.1.0->llama-index) (1.1.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas[jinja2]->llama-index) (1.16.0)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore->httpx->llama-index)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: bitsandbytes, urllib3, mypy-extensions, marshmallow, h11, deprecated, beautifulsoup4, aiostream, typing-inspect, httpcore, tiktoken, httpx, dataclasses-json, openai, accelerate, llama-index\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 2.0.7\n",
            "    Uninstalling urllib3-2.0.7:\n",
            "      Successfully uninstalled urllib3-2.0.7\n",
            "  Attempting uninstall: beautifulsoup4\n",
            "    Found existing installation: beautifulsoup4 4.11.2\n",
            "    Uninstalling beautifulsoup4-4.11.2:\n",
            "      Successfully uninstalled beautifulsoup4-4.11.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed accelerate-0.24.1 aiostream-0.5.2 beautifulsoup4-4.12.2 bitsandbytes-0.41.2.post2 dataclasses-json-0.5.14 deprecated-1.2.14 h11-0.14.0 httpcore-1.0.2 httpx-0.25.1 llama-index-0.9.3.post1 marshmallow-3.20.1 mypy-extensions-1.0.0 openai-1.3.3 tiktoken-0.5.1 typing-inspect-0.9.0 urllib3-1.26.18\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setup"
      ],
      "metadata": {
        "id": "q1_IZvrVM_FQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data\n",
        "\n",
        "Here we are placing all our docs in Data folder\n"
      ],
      "metadata": {
        "id": "kkjVukvQO_5D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
        "\n",
        "documents = SimpleDirectoryReader(\"data\").load_data()"
      ],
      "metadata": {
        "id": "fJnvf-VAiCiL"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Since I'm using `meta-llama/Llama-2-7b-chat-hf` as my LLM, it requires my huggingface authentication."
      ],
      "metadata": {
        "id": "3lCdgK_sppkX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qj7XklBs6QVT",
        "outputId": "0e487b00-4fb0-45e8-99fc-ce4640aa9abe"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token: \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### LLM\n",
        "\n",
        "This should run on a T4 instance on the free tier"
      ],
      "metadata": {
        "id": "zXOJQa7FOGr5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "lMNaHDzPM68f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77,
          "referenced_widgets": [
            "9504de4bb06444d4987c63db433faacc",
            "283a739603df4572b95d5d2eb8ddbd74",
            "c04dee5757c143338f45373a2617f9a7",
            "90f7b541d579401b96f2f331160fba15",
            "5f14f79471ed412d9614c031572cf062",
            "d1d05f54ae7446f49297fa3950db8d0d",
            "fdc46a5ffc854a2baff46ac27db8c166",
            "1cf7e0b365d2403290c7e31bbcef9df6",
            "90e95d938a30493b92e66aca951f5ab2",
            "bf0a8c2b4f844babae54438dcc1ea404",
            "3b7190810e084eb982eac8f9926042d1"
          ]
        },
        "outputId": "d629b0c2-593f-4190-c706-927b1373b838"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9504de4bb06444d4987c63db433faacc"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import BitsAndBytesConfig\n",
        "from llama_index.prompts import PromptTemplate\n",
        "from llama_index.llms import HuggingFaceLLM\n",
        "\n",
        "#Without quantization, we will not be able to load the whole meta-llama/Llama-2-7b-chat-hf in the free tier of colab.\n",
        "quantization_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_compute_dtype=torch.float16,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_use_double_quant=True,\n",
        ")\n",
        "\n",
        "\n",
        "def messages_to_prompt(messages):\n",
        "  prompt = \"\"\n",
        "  for message in messages:\n",
        "    if message.role == 'system':\n",
        "      prompt += f\"<|system|>\\n{message.content}</s>\\n\"\n",
        "    elif message.role == 'user':\n",
        "      prompt += f\"<|user|>\\n{message.content}</s>\\n\"\n",
        "    elif message.role == 'assistant':\n",
        "      prompt += f\"<|assistant|>\\n{message.content}</s>\\n\"\n",
        "\n",
        "  # ensure we start with a system prompt, insert blank if needed\n",
        "  if not prompt.startswith(\"<|system|>\\n\"):\n",
        "    prompt = \"<|system|>\\n</s>\\n\" + prompt\n",
        "\n",
        "  # add final assistant prompt\n",
        "  prompt = prompt + \"<|assistant|>\\n\"\n",
        "\n",
        "  return prompt\n",
        "\n",
        "\n",
        "llm = HuggingFaceLLM(\n",
        "    model_name=\"meta-llama/Llama-2-7b-chat-hf\",\n",
        "    tokenizer_name=\"meta-llama/Llama-2-7b-chat-hf\",\n",
        "    query_wrapper_prompt=PromptTemplate(\"<|system|>\\n</s>\\n<|user|>\\n{query_str}</s>\\n<|assistant|>\\n\"),\n",
        "    context_window=3900,\n",
        "    max_new_tokens=256,\n",
        "    model_kwargs={\"quantization_config\": quantization_config},\n",
        "    # tokenizer_kwargs={},\n",
        "    generate_kwargs={\"temperature\": 0.3, \"top_k\": 50, \"top_p\": 0.95},\n",
        "    messages_to_prompt=messages_to_prompt,\n",
        "    device_map=\"auto\",\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import ServiceContext\n",
        "\n",
        "service_context = ServiceContext.from_defaults(llm=llm, embed_model=\"local:BAAI/bge-small-en-v1.5\")"
      ],
      "metadata": {
        "id": "rPawBPL0SwFj"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Index Setup"
      ],
      "metadata": {
        "id": "_iYeofUlS6_D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index import VectorStoreIndex\n",
        "\n",
        "vector_index = VectorStoreIndex.from_documents(documents, service_context=service_context)"
      ],
      "metadata": {
        "id": "Jz173lnJS8cZ"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Helpful Imports / Logging"
      ],
      "metadata": {
        "id": "pxFrAm_bTCYY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.response.notebook_utils import display_response"
      ],
      "metadata": {
        "id": "ghqk6C04TD3b"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import logging\n",
        "import sys\n",
        "\n",
        "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
        "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
      ],
      "metadata": {
        "id": "6YVB1ny-UjN4"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Basic Query Engine"
      ],
      "metadata": {
        "id": "K0l3oUS8NO-I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "query_engine = vector_index.as_query_engine(response_mode=\"compact\")\n",
        "\n",
        "response = query_engine.query(\"Hi?\")\n",
        "\n",
        "display_response(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 64
        },
        "id": "Lb9CsvvQNR9K",
        "outputId": "663919cc-048f-4e7f-a34e-88d305824233"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ],
            "text/markdown": "**`Final Response:`** Hi there! I'm here to help you with any questions or tasks you may have. What can I help you with today?"
          },
          "metadata": {}
        }
      ]
    }
  ]
}