GPTTextGeneration.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GPTTextGeneration.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPdvNG1whZp4yZe1yYLz23F",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "45ae523da3ac45b1ad39be79f3583330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_db4669cf31ce41869e7e7aaffc80d599",
              "IPY_MODEL_d6df185e251c44fda4f79485272d16ed",
              "IPY_MODEL_d519ef0994cf4a0a8f74ae6a9f348b93"
            ],
            "layout": "IPY_MODEL_cf2497b979a3462b9c3a3f6257b51507"
          }
        },
        "db4669cf31ce41869e7e7aaffc80d599": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d163b3d41e1549749b358b6d68149f7a",
            "placeholder": "​",
            "style": "IPY_MODEL_935307b86431407c9047c340dde2c6d6",
            "value": "Downloading: 100%"
          }
        },
        "d6df185e251c44fda4f79485272d16ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_04609032cff94c09a32cd883450ef4e8",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_972c579e281043d38a88dd06f07fe018",
            "value": 665
          }
        },
        "d519ef0994cf4a0a8f74ae6a9f348b93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77b1e3fa45c44a8f8e9feab91f5e6727",
            "placeholder": "​",
            "style": "IPY_MODEL_b20020576278452ba3cf86da31bef477",
            "value": " 665/665 [00:00&lt;00:00, 18.8kB/s]"
          }
        },
        "cf2497b979a3462b9c3a3f6257b51507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d163b3d41e1549749b358b6d68149f7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "935307b86431407c9047c340dde2c6d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "04609032cff94c09a32cd883450ef4e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "972c579e281043d38a88dd06f07fe018": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "77b1e3fa45c44a8f8e9feab91f5e6727": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b20020576278452ba3cf86da31bef477": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "625fe50be2c0493490e39626f52ec351": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33b8c22836774553bc33ee4262a9ec80",
              "IPY_MODEL_3208d1c27c064112b83d645a77494ad7",
              "IPY_MODEL_5545dce64a3247cc8b3d690fd55ef4a7"
            ],
            "layout": "IPY_MODEL_be4806a5a4cd4a4eb9fb0d5f73276375"
          }
        },
        "33b8c22836774553bc33ee4262a9ec80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e07b63bd06d4125b7c5a9be8fdb2126",
            "placeholder": "​",
            "style": "IPY_MODEL_028409dd1d4e49329b3baf0ea0048c5c",
            "value": "Downloading: 100%"
          }
        },
        "3208d1c27c064112b83d645a77494ad7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31ae85e1050f476697af408e1103841c",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a305a8ab3be4bb4ac4ceebfc9660480",
            "value": 1042301
          }
        },
        "5545dce64a3247cc8b3d690fd55ef4a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58d288982ccd41bd8b0e928c1f5b7f2b",
            "placeholder": "​",
            "style": "IPY_MODEL_fd9d20746eaa458283dde97f4f9e1185",
            "value": " 0.99M/0.99M [00:01&lt;00:00, 914kB/s]"
          }
        },
        "be4806a5a4cd4a4eb9fb0d5f73276375": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e07b63bd06d4125b7c5a9be8fdb2126": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "028409dd1d4e49329b3baf0ea0048c5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "31ae85e1050f476697af408e1103841c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a305a8ab3be4bb4ac4ceebfc9660480": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "58d288982ccd41bd8b0e928c1f5b7f2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd9d20746eaa458283dde97f4f9e1185": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dbb3f20f67a541f8849df1be1e43cd41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_73942ca9d6fe4e3a912ab904988481ef",
              "IPY_MODEL_54b2888861dd4a9e9eb8b7ed64146556",
              "IPY_MODEL_9561554d968549c99940f0376102fff5"
            ],
            "layout": "IPY_MODEL_e6867e9002e64526a1641d9a4a1db5d7"
          }
        },
        "73942ca9d6fe4e3a912ab904988481ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f4a690953c44de986b9bcf3f6a604f6",
            "placeholder": "​",
            "style": "IPY_MODEL_407a96ab53124693b16bd63111e43b20",
            "value": "Downloading: 100%"
          }
        },
        "54b2888861dd4a9e9eb8b7ed64146556": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd2b172417fd4823b9e770d6a3d7c875",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da8be14da2f846238472934370278191",
            "value": 456318
          }
        },
        "9561554d968549c99940f0376102fff5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aa57a2bf864545f4bef98b2b7d9f1421",
            "placeholder": "​",
            "style": "IPY_MODEL_e4b5af43a1734035b4d782d94c16fc93",
            "value": " 446k/446k [00:01&lt;00:00, 461kB/s]"
          }
        },
        "e6867e9002e64526a1641d9a4a1db5d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f4a690953c44de986b9bcf3f6a604f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "407a96ab53124693b16bd63111e43b20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd2b172417fd4823b9e770d6a3d7c875": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da8be14da2f846238472934370278191": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aa57a2bf864545f4bef98b2b7d9f1421": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4b5af43a1734035b4d782d94c16fc93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f29e50d227ab4c2ca7841c8044e95530": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_988930c37ba14039885b45a8b2a90970",
              "IPY_MODEL_7d72fdf4e29a45e28a660c6cdbf494cf",
              "IPY_MODEL_4aeffeec34f943849712b7d8dfde58c9"
            ],
            "layout": "IPY_MODEL_49a6461ffe0141c4a5582a10c8e28a52"
          }
        },
        "988930c37ba14039885b45a8b2a90970": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df3d4e374ed94925af5cfce37eb7343e",
            "placeholder": "​",
            "style": "IPY_MODEL_84a008b6606148e19cf39e2c9944a9be",
            "value": "Downloading: 100%"
          }
        },
        "7d72fdf4e29a45e28a660c6cdbf494cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ba91d1e72df4ee4aeeae63accae883b",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4ecdb1494e5f442ebc882e53ab7ab8bc",
            "value": 1355256
          }
        },
        "4aeffeec34f943849712b7d8dfde58c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52fc7885dfa74631b025d18cc569d957",
            "placeholder": "​",
            "style": "IPY_MODEL_ffab7deefcf14b0ab5a7503a0c27d03f",
            "value": " 1.29M/1.29M [00:01&lt;00:00, 972kB/s]"
          }
        },
        "49a6461ffe0141c4a5582a10c8e28a52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df3d4e374ed94925af5cfce37eb7343e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84a008b6606148e19cf39e2c9944a9be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ba91d1e72df4ee4aeeae63accae883b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ecdb1494e5f442ebc882e53ab7ab8bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "52fc7885dfa74631b025d18cc569d957": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ffab7deefcf14b0ab5a7503a0c27d03f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34239896a57444a9bbe5efa71f04989b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_814ead1aa4a1490aa79afaf8060a8a8e",
              "IPY_MODEL_686853005fba4fb491b3cfc2e679ece6",
              "IPY_MODEL_e5789f1e18c74a8da4a67488919eb429"
            ],
            "layout": "IPY_MODEL_3e5ef0100c0d4a7183d102bce0cbe6c7"
          }
        },
        "814ead1aa4a1490aa79afaf8060a8a8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10a01614d71344a3af1a2e83c99eed1a",
            "placeholder": "​",
            "style": "IPY_MODEL_c4e9533a5fe648df90adebd798e5373d",
            "value": "Downloading: 100%"
          }
        },
        "686853005fba4fb491b3cfc2e679ece6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d07bd1f28ef445e4bed042c8b0e63961",
            "max": 548118077,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_870801d715674169aa141146da1879f1",
            "value": 548118077
          }
        },
        "e5789f1e18c74a8da4a67488919eb429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a2167d7deee74a5a91d31e054b0da9d8",
            "placeholder": "​",
            "style": "IPY_MODEL_59b76ec316f04efeb19a0683b514e107",
            "value": " 523M/523M [00:09&lt;00:00, 63.4MB/s]"
          }
        },
        "3e5ef0100c0d4a7183d102bce0cbe6c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10a01614d71344a3af1a2e83c99eed1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4e9533a5fe648df90adebd798e5373d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d07bd1f28ef445e4bed042c8b0e63961": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "870801d715674169aa141146da1879f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a2167d7deee74a5a91d31e054b0da9d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59b76ec316f04efeb19a0683b514e107": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ahmedoumar/GPTTextGeneration/blob/main/GPTTextGeneration.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers "
      ],
      "metadata": {
        "id": "gCfRM1dEhZZn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b74948c-d8ed-447b-b27e-9c4442d5b6ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.20.1-py3-none-any.whl (4.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 39.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 59.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 42.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.8.1-py3-none-any.whl (101 kB)\n",
            "\u001b[K     |████████████████████████████████| 101 kB 12.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.12.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Installing collected packages: pyyaml, tokenizers, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.8.1 pyyaml-6.0 tokenizers-0.12.1 transformers-4.20.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N4LlL4jkhOvd"
      },
      "outputs": [],
      "source": [
        "# Import main packages\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get our model\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "model_name = 'gpt2'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModelForCausalLM.from_pretrained(model_name).to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "45ae523da3ac45b1ad39be79f3583330",
            "db4669cf31ce41869e7e7aaffc80d599",
            "d6df185e251c44fda4f79485272d16ed",
            "d519ef0994cf4a0a8f74ae6a9f348b93",
            "cf2497b979a3462b9c3a3f6257b51507",
            "d163b3d41e1549749b358b6d68149f7a",
            "935307b86431407c9047c340dde2c6d6",
            "04609032cff94c09a32cd883450ef4e8",
            "972c579e281043d38a88dd06f07fe018",
            "77b1e3fa45c44a8f8e9feab91f5e6727",
            "b20020576278452ba3cf86da31bef477",
            "625fe50be2c0493490e39626f52ec351",
            "33b8c22836774553bc33ee4262a9ec80",
            "3208d1c27c064112b83d645a77494ad7",
            "5545dce64a3247cc8b3d690fd55ef4a7",
            "be4806a5a4cd4a4eb9fb0d5f73276375",
            "8e07b63bd06d4125b7c5a9be8fdb2126",
            "028409dd1d4e49329b3baf0ea0048c5c",
            "31ae85e1050f476697af408e1103841c",
            "0a305a8ab3be4bb4ac4ceebfc9660480",
            "58d288982ccd41bd8b0e928c1f5b7f2b",
            "fd9d20746eaa458283dde97f4f9e1185",
            "dbb3f20f67a541f8849df1be1e43cd41",
            "73942ca9d6fe4e3a912ab904988481ef",
            "54b2888861dd4a9e9eb8b7ed64146556",
            "9561554d968549c99940f0376102fff5",
            "e6867e9002e64526a1641d9a4a1db5d7",
            "7f4a690953c44de986b9bcf3f6a604f6",
            "407a96ab53124693b16bd63111e43b20",
            "dd2b172417fd4823b9e770d6a3d7c875",
            "da8be14da2f846238472934370278191",
            "aa57a2bf864545f4bef98b2b7d9f1421",
            "e4b5af43a1734035b4d782d94c16fc93",
            "f29e50d227ab4c2ca7841c8044e95530",
            "988930c37ba14039885b45a8b2a90970",
            "7d72fdf4e29a45e28a660c6cdbf494cf",
            "4aeffeec34f943849712b7d8dfde58c9",
            "49a6461ffe0141c4a5582a10c8e28a52",
            "df3d4e374ed94925af5cfce37eb7343e",
            "84a008b6606148e19cf39e2c9944a9be",
            "0ba91d1e72df4ee4aeeae63accae883b",
            "4ecdb1494e5f442ebc882e53ab7ab8bc",
            "52fc7885dfa74631b025d18cc569d957",
            "ffab7deefcf14b0ab5a7503a0c27d03f",
            "34239896a57444a9bbe5efa71f04989b",
            "814ead1aa4a1490aa79afaf8060a8a8e",
            "686853005fba4fb491b3cfc2e679ece6",
            "e5789f1e18c74a8da4a67488919eb429",
            "3e5ef0100c0d4a7183d102bce0cbe6c7",
            "10a01614d71344a3af1a2e83c99eed1a",
            "c4e9533a5fe648df90adebd798e5373d",
            "d07bd1f28ef445e4bed042c8b0e63961",
            "870801d715674169aa141146da1879f1",
            "a2167d7deee74a5a91d31e054b0da9d8",
            "59b76ec316f04efeb19a0683b514e107"
          ]
        },
        "id": "SACQ75YahY3g",
        "outputId": "a3812452-b0e7-436a-8032-5335a5a7afb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "45ae523da3ac45b1ad39be79f3583330"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/0.99M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "625fe50be2c0493490e39626f52ec351"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dbb3f20f67a541f8849df1be1e43cd41"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f29e50d227ab4c2ca7841c8044e95530"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/523M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "34239896a57444a9bbe5efa71f04989b"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_txt = 'Transformers are the'\n",
        "input_ids = tokenizer(input_txt, return_tensors='pt')['input_ids'].to(device)\n",
        "iterations = []\n",
        "n_steps = 8\n",
        "choices_per_step = 5\n",
        "\n",
        "with torch.no_grad():\n",
        "    for _ in range(n_steps):\n",
        "        iteration = dict()\n",
        "        iteration['Input'] = tokenizer.decode(input_ids[0])\n",
        "        output = model(input_ids=input_ids)\n",
        "        # Select logits of the first batch and the last token and apply softmax\n",
        "        next_token_logits = output.logits[0, -1, :]\n",
        "        next_token_probs = torch.softmax(next_token_logits, dim=-1)\n",
        "        sorted_ids = torch.argsort(next_token_probs, dim=-1, descending=True)\n",
        "        # Store tokens with highest probabilities\n",
        "        for choice_idx in range(choices_per_step):\n",
        "            token_id = sorted_ids[choice_idx]\n",
        "            token_prob = next_token_probs[token_id].cpu().numpy()\n",
        "            token_choice = (\n",
        "                f'{tokenizer.decode(token_id)} ({100*token_prob:.2f}%)'\n",
        "            )\n",
        "            iteration[f'Choice {choice_idx+1}'] = token_choice\n",
        "            # Append predicted next token to input\n",
        "            input_ids = torch.cat([input_ids, sorted_ids[None, 0, None]], dim=-1)\n",
        "            iterations.append(iteration)\n",
        "\n",
        "pd.DataFrame(iterations)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "_p5zecMXiCCr",
        "outputId": "ad8f9caf-6113-4679-9a07-557cfe77059c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                Input            Choice 1  \\\n",
              "0                                Transformers are the        most (9.76%)   \n",
              "1                                Transformers are the        most (9.76%)   \n",
              "2                                Transformers are the        most (9.76%)   \n",
              "3                                Transformers are the        most (9.76%)   \n",
              "4                                Transformers are the        most (9.76%)   \n",
              "5       Transformers are the most most most most most   powerful (15.04%)   \n",
              "6       Transformers are the most most most most most   powerful (15.04%)   \n",
              "7       Transformers are the most most most most most   powerful (15.04%)   \n",
              "8       Transformers are the most most most most most   powerful (15.04%)   \n",
              "9       Transformers are the most most most most most   powerful (15.04%)   \n",
              "10  Transformers are the most most most most most ...       power (4.47%)   \n",
              "11  Transformers are the most most most most most ...       power (4.47%)   \n",
              "12  Transformers are the most most most most most ...       power (4.47%)   \n",
              "13  Transformers are the most most most most most ...       power (4.47%)   \n",
              "14  Transformers are the most most most most most ...       power (4.47%)   \n",
              "15  Transformers are the most most most most most ...           . (8.25%)   \n",
              "16  Transformers are the most most most most most ...           . (8.25%)   \n",
              "17  Transformers are the most most most most most ...           . (8.25%)   \n",
              "18  Transformers are the most most most most most ...           . (8.25%)   \n",
              "19  Transformers are the most most most most most ...           . (8.25%)   \n",
              "20  Transformers are the most most most most most ...           . (7.31%)   \n",
              "21  Transformers are the most most most most most ...           . (7.31%)   \n",
              "22  Transformers are the most most most most most ...           . (7.31%)   \n",
              "23  Transformers are the most most most most most ...           . (7.31%)   \n",
              "24  Transformers are the most most most most most ...           . (7.31%)   \n",
              "25  Transformers are the most most most most most ...          . (76.99%)   \n",
              "26  Transformers are the most most most most most ...          . (76.99%)   \n",
              "27  Transformers are the most most most most most ...          . (76.99%)   \n",
              "28  Transformers are the most most most most most ...          . (76.99%)   \n",
              "29  Transformers are the most most most most most ...          . (76.99%)   \n",
              "30  Transformers are the most most most most most ...          . (78.94%)   \n",
              "31  Transformers are the most most most most most ...          . (78.94%)   \n",
              "32  Transformers are the most most most most most ...          . (78.94%)   \n",
              "33  Transformers are the most most most most most ...          . (78.94%)   \n",
              "34  Transformers are the most most most most most ...          . (78.94%)   \n",
              "35  Transformers are the most most most most most ...          . (78.05%)   \n",
              "36  Transformers are the most most most most most ...          . (78.05%)   \n",
              "37  Transformers are the most most most most most ...          . (78.05%)   \n",
              "38  Transformers are the most most most most most ...          . (78.05%)   \n",
              "39  Transformers are the most most most most most ...          . (78.05%)   \n",
              "\n",
              "              Choice 2            Choice 3           Choice 4  \\\n",
              "0         same (2.94%)        only (2.87%)       best (2.38%)   \n",
              "1         same (2.94%)        only (2.87%)       best (2.38%)   \n",
              "2         same (2.94%)        only (2.87%)       best (2.38%)   \n",
              "3         same (2.94%)        only (2.87%)       best (2.38%)   \n",
              "4         same (2.94%)        only (2.87%)       best (2.38%)   \n",
              "5    efficient (4.53%)   important (4.48%)     common (4.41%)   \n",
              "6    efficient (4.53%)   important (4.48%)     common (4.41%)   \n",
              "7    efficient (4.53%)   important (4.48%)     common (4.41%)   \n",
              "8    efficient (4.53%)   important (4.48%)     common (4.41%)   \n",
              "9    efficient (4.53%)   important (4.48%)     common (4.41%)   \n",
              "10      weapon (3.91%)     ability (3.80%)   powerful (2.99%)   \n",
              "11      weapon (3.91%)     ability (3.80%)   powerful (2.99%)   \n",
              "12      weapon (3.91%)     ability (3.80%)   powerful (2.99%)   \n",
              "13      weapon (3.91%)     ability (3.80%)   powerful (2.99%)   \n",
              "14      weapon (3.91%)     ability (3.80%)   powerful (2.99%)   \n",
              "15        that (6.39%)           , (4.69%)      power (4.29%)   \n",
              "16        that (6.39%)           , (4.69%)      power (4.29%)   \n",
              "17        that (6.39%)           , (4.69%)      power (4.29%)   \n",
              "18        that (6.39%)           , (4.69%)      power (4.29%)   \n",
              "19        that (6.39%)           , (4.69%)      power (4.29%)   \n",
              "20           . (6.95%)         The (6.37%)         \\n (5.74%)   \n",
              "21           . (6.95%)         The (6.37%)         \\n (5.74%)   \n",
              "22           . (6.95%)         The (6.37%)         \\n (5.74%)   \n",
              "23           . (6.95%)         The (6.37%)         \\n (5.74%)   \n",
              "24           . (6.95%)         The (6.37%)         \\n (5.74%)   \n",
              "25           . (2.13%)          \\n (0.67%)          - (0.61%)   \n",
              "26           . (2.13%)          \\n (0.67%)          - (0.61%)   \n",
              "27           . (2.13%)          \\n (0.67%)          - (0.61%)   \n",
              "28           . (2.13%)          \\n (0.67%)          - (0.61%)   \n",
              "29           . (2.13%)          \\n (0.67%)          - (0.61%)   \n",
              "30          \\n (0.71%)           1 (0.48%)          - (0.45%)   \n",
              "31          \\n (0.71%)           1 (0.48%)          - (0.45%)   \n",
              "32          \\n (0.71%)           1 (0.48%)          - (0.45%)   \n",
              "33          \\n (0.71%)           1 (0.48%)          - (0.45%)   \n",
              "34          \\n (0.71%)           1 (0.48%)          - (0.45%)   \n",
              "35          \\n (0.91%)           1 (0.56%)          - (0.40%)   \n",
              "36          \\n (0.91%)           1 (0.56%)          - (0.40%)   \n",
              "37          \\n (0.91%)           1 (0.56%)          - (0.40%)   \n",
              "38          \\n (0.91%)           1 (0.56%)          - (0.40%)   \n",
              "39          \\n (0.91%)           1 (0.56%)          - (0.40%)   \n",
              "\n",
              "              Choice 5  \n",
              "0        first (1.77%)  \n",
              "1        first (1.77%)  \n",
              "2        first (1.77%)  \n",
              "3        first (1.77%)  \n",
              "4        first (1.77%)  \n",
              "5    expensive (2.72%)  \n",
              "6    expensive (2.72%)  \n",
              "7    expensive (2.72%)  \n",
              "8    expensive (2.72%)  \n",
              "9    expensive (2.72%)  \n",
              "10       being (2.62%)  \n",
              "11       being (2.62%)  \n",
              "12       being (2.62%)  \n",
              "13       being (2.62%)  \n",
              "14       being (2.62%)  \n",
              "15          in (4.21%)  \n",
              "16          in (4.21%)  \n",
              "17          in (4.21%)  \n",
              "18          in (4.21%)  \n",
              "19          in (4.21%)  \n",
              "20        They (3.07%)  \n",
              "21        They (3.07%)  \n",
              "22        They (3.07%)  \n",
              "23        They (3.07%)  \n",
              "24        They (3.07%)  \n",
              "25         The (0.54%)  \n",
              "26         The (0.54%)  \n",
              "27         The (0.54%)  \n",
              "28         The (0.54%)  \n",
              "29         The (0.54%)  \n",
              "30           , (0.30%)  \n",
              "31           , (0.30%)  \n",
              "32           , (0.30%)  \n",
              "33           , (0.30%)  \n",
              "34           , (0.30%)  \n",
              "35           4 (0.33%)  \n",
              "36           4 (0.33%)  \n",
              "37           4 (0.33%)  \n",
              "38           4 (0.33%)  \n",
              "39           4 (0.33%)  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-92eb1a58-4f9a-4a5f-bd0c-d6a8ec31bb3a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Input</th>\n",
              "      <th>Choice 1</th>\n",
              "      <th>Choice 2</th>\n",
              "      <th>Choice 3</th>\n",
              "      <th>Choice 4</th>\n",
              "      <th>Choice 5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (9.76%)</td>\n",
              "      <td>same (2.94%)</td>\n",
              "      <td>only (2.87%)</td>\n",
              "      <td>best (2.38%)</td>\n",
              "      <td>first (1.77%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (9.76%)</td>\n",
              "      <td>same (2.94%)</td>\n",
              "      <td>only (2.87%)</td>\n",
              "      <td>best (2.38%)</td>\n",
              "      <td>first (1.77%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (9.76%)</td>\n",
              "      <td>same (2.94%)</td>\n",
              "      <td>only (2.87%)</td>\n",
              "      <td>best (2.38%)</td>\n",
              "      <td>first (1.77%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (9.76%)</td>\n",
              "      <td>same (2.94%)</td>\n",
              "      <td>only (2.87%)</td>\n",
              "      <td>best (2.38%)</td>\n",
              "      <td>first (1.77%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (9.76%)</td>\n",
              "      <td>same (2.94%)</td>\n",
              "      <td>only (2.87%)</td>\n",
              "      <td>best (2.38%)</td>\n",
              "      <td>first (1.77%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Transformers are the most most most most most</td>\n",
              "      <td>powerful (15.04%)</td>\n",
              "      <td>efficient (4.53%)</td>\n",
              "      <td>important (4.48%)</td>\n",
              "      <td>common (4.41%)</td>\n",
              "      <td>expensive (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Transformers are the most most most most most</td>\n",
              "      <td>powerful (15.04%)</td>\n",
              "      <td>efficient (4.53%)</td>\n",
              "      <td>important (4.48%)</td>\n",
              "      <td>common (4.41%)</td>\n",
              "      <td>expensive (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Transformers are the most most most most most</td>\n",
              "      <td>powerful (15.04%)</td>\n",
              "      <td>efficient (4.53%)</td>\n",
              "      <td>important (4.48%)</td>\n",
              "      <td>common (4.41%)</td>\n",
              "      <td>expensive (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Transformers are the most most most most most</td>\n",
              "      <td>powerful (15.04%)</td>\n",
              "      <td>efficient (4.53%)</td>\n",
              "      <td>important (4.48%)</td>\n",
              "      <td>common (4.41%)</td>\n",
              "      <td>expensive (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Transformers are the most most most most most</td>\n",
              "      <td>powerful (15.04%)</td>\n",
              "      <td>efficient (4.53%)</td>\n",
              "      <td>important (4.48%)</td>\n",
              "      <td>common (4.41%)</td>\n",
              "      <td>expensive (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>power (4.47%)</td>\n",
              "      <td>weapon (3.91%)</td>\n",
              "      <td>ability (3.80%)</td>\n",
              "      <td>powerful (2.99%)</td>\n",
              "      <td>being (2.62%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>power (4.47%)</td>\n",
              "      <td>weapon (3.91%)</td>\n",
              "      <td>ability (3.80%)</td>\n",
              "      <td>powerful (2.99%)</td>\n",
              "      <td>being (2.62%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>power (4.47%)</td>\n",
              "      <td>weapon (3.91%)</td>\n",
              "      <td>ability (3.80%)</td>\n",
              "      <td>powerful (2.99%)</td>\n",
              "      <td>being (2.62%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>power (4.47%)</td>\n",
              "      <td>weapon (3.91%)</td>\n",
              "      <td>ability (3.80%)</td>\n",
              "      <td>powerful (2.99%)</td>\n",
              "      <td>being (2.62%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>power (4.47%)</td>\n",
              "      <td>weapon (3.91%)</td>\n",
              "      <td>ability (3.80%)</td>\n",
              "      <td>powerful (2.99%)</td>\n",
              "      <td>being (2.62%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (8.25%)</td>\n",
              "      <td>that (6.39%)</td>\n",
              "      <td>, (4.69%)</td>\n",
              "      <td>power (4.29%)</td>\n",
              "      <td>in (4.21%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (8.25%)</td>\n",
              "      <td>that (6.39%)</td>\n",
              "      <td>, (4.69%)</td>\n",
              "      <td>power (4.29%)</td>\n",
              "      <td>in (4.21%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (8.25%)</td>\n",
              "      <td>that (6.39%)</td>\n",
              "      <td>, (4.69%)</td>\n",
              "      <td>power (4.29%)</td>\n",
              "      <td>in (4.21%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (8.25%)</td>\n",
              "      <td>that (6.39%)</td>\n",
              "      <td>, (4.69%)</td>\n",
              "      <td>power (4.29%)</td>\n",
              "      <td>in (4.21%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (8.25%)</td>\n",
              "      <td>that (6.39%)</td>\n",
              "      <td>, (4.69%)</td>\n",
              "      <td>power (4.29%)</td>\n",
              "      <td>in (4.21%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (7.31%)</td>\n",
              "      <td>. (6.95%)</td>\n",
              "      <td>The (6.37%)</td>\n",
              "      <td>\\n (5.74%)</td>\n",
              "      <td>They (3.07%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (7.31%)</td>\n",
              "      <td>. (6.95%)</td>\n",
              "      <td>The (6.37%)</td>\n",
              "      <td>\\n (5.74%)</td>\n",
              "      <td>They (3.07%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (7.31%)</td>\n",
              "      <td>. (6.95%)</td>\n",
              "      <td>The (6.37%)</td>\n",
              "      <td>\\n (5.74%)</td>\n",
              "      <td>They (3.07%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (7.31%)</td>\n",
              "      <td>. (6.95%)</td>\n",
              "      <td>The (6.37%)</td>\n",
              "      <td>\\n (5.74%)</td>\n",
              "      <td>They (3.07%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (7.31%)</td>\n",
              "      <td>. (6.95%)</td>\n",
              "      <td>The (6.37%)</td>\n",
              "      <td>\\n (5.74%)</td>\n",
              "      <td>They (3.07%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (76.99%)</td>\n",
              "      <td>. (2.13%)</td>\n",
              "      <td>\\n (0.67%)</td>\n",
              "      <td>- (0.61%)</td>\n",
              "      <td>The (0.54%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (76.99%)</td>\n",
              "      <td>. (2.13%)</td>\n",
              "      <td>\\n (0.67%)</td>\n",
              "      <td>- (0.61%)</td>\n",
              "      <td>The (0.54%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (76.99%)</td>\n",
              "      <td>. (2.13%)</td>\n",
              "      <td>\\n (0.67%)</td>\n",
              "      <td>- (0.61%)</td>\n",
              "      <td>The (0.54%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (76.99%)</td>\n",
              "      <td>. (2.13%)</td>\n",
              "      <td>\\n (0.67%)</td>\n",
              "      <td>- (0.61%)</td>\n",
              "      <td>The (0.54%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (76.99%)</td>\n",
              "      <td>. (2.13%)</td>\n",
              "      <td>\\n (0.67%)</td>\n",
              "      <td>- (0.61%)</td>\n",
              "      <td>The (0.54%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.94%)</td>\n",
              "      <td>\\n (0.71%)</td>\n",
              "      <td>1 (0.48%)</td>\n",
              "      <td>- (0.45%)</td>\n",
              "      <td>, (0.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.94%)</td>\n",
              "      <td>\\n (0.71%)</td>\n",
              "      <td>1 (0.48%)</td>\n",
              "      <td>- (0.45%)</td>\n",
              "      <td>, (0.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.94%)</td>\n",
              "      <td>\\n (0.71%)</td>\n",
              "      <td>1 (0.48%)</td>\n",
              "      <td>- (0.45%)</td>\n",
              "      <td>, (0.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.94%)</td>\n",
              "      <td>\\n (0.71%)</td>\n",
              "      <td>1 (0.48%)</td>\n",
              "      <td>- (0.45%)</td>\n",
              "      <td>, (0.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.94%)</td>\n",
              "      <td>\\n (0.71%)</td>\n",
              "      <td>1 (0.48%)</td>\n",
              "      <td>- (0.45%)</td>\n",
              "      <td>, (0.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.05%)</td>\n",
              "      <td>\\n (0.91%)</td>\n",
              "      <td>1 (0.56%)</td>\n",
              "      <td>- (0.40%)</td>\n",
              "      <td>4 (0.33%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.05%)</td>\n",
              "      <td>\\n (0.91%)</td>\n",
              "      <td>1 (0.56%)</td>\n",
              "      <td>- (0.40%)</td>\n",
              "      <td>4 (0.33%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.05%)</td>\n",
              "      <td>\\n (0.91%)</td>\n",
              "      <td>1 (0.56%)</td>\n",
              "      <td>- (0.40%)</td>\n",
              "      <td>4 (0.33%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.05%)</td>\n",
              "      <td>\\n (0.91%)</td>\n",
              "      <td>1 (0.56%)</td>\n",
              "      <td>- (0.40%)</td>\n",
              "      <td>4 (0.33%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>Transformers are the most most most most most ...</td>\n",
              "      <td>. (78.05%)</td>\n",
              "      <td>\\n (0.91%)</td>\n",
              "      <td>1 (0.56%)</td>\n",
              "      <td>- (0.40%)</td>\n",
              "      <td>4 (0.33%)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-92eb1a58-4f9a-4a5f-bd0c-d6a8ec31bb3a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-92eb1a58-4f9a-4a5f-bd0c-d6a8ec31bb3a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-92eb1a58-4f9a-4a5f-bd0c-d6a8ec31bb3a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Now with the transformers lib built-in func: generate\n",
        "input_ids = tokenizer(input_txt, return_tensors='pt')['input_ids'].to(device)\n",
        "output = model.generate(input_ids, max_new_tokens=n_steps, do_sample=False)\n",
        "print(tokenizer.decode(output[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kKTxTyrZlFfL",
        "outputId": "0a3c60b2-0e73-42cb-bb8a-963252bb7db8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Transformers are the most common type of particle. They are\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 128\n",
        "input_txt = \"\"\"In a shocking finding, scientist discovered \\\n",
        "a herd of unicorns living in a remote, previously unexplored \\\n",
        "valley, in the Andes Mountains. Even more surprising to the \\\n",
        "researchers was the fact that the unicorns spoke perfect English.\\n\\n\n",
        "\"\"\"\n",
        "input_ids = tokenizer(input_txt, return_tensors='pt')['input_ids'].to(device)\n",
        "output_greedy = model.generate(input_ids, max_length=max_len,\n",
        "                               do_sample=False)\n",
        "print(tokenizer.decode(output_greedy[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pU5xcQGAmbv3",
        "outputId": "750343d1-9c70-432c-bb13-8057f0ebc006"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "\"The unicorns were very intelligent, and they were very intelligent,\" said Dr. David S. Siegel, a professor of anthropology at the University of California, Berkeley. \"They were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def log_probs_from_logits(logits, labels):\n",
        "    logp = F.log_softmax(logits, dim=-1)\n",
        "    logp_label = torch.gather(logp, 2, labels.unsqueeze(2)).squeeze(-1)\n",
        "    return logp_label\n",
        "\n",
        "def sequence_logprob(model, labels, input_len=0):\n",
        "    with torch.no_grad():\n",
        "        output = model(labels)\n",
        "        log_probs = log_probs_from_logits(\n",
        "            output.logits[:, :-1, :], labels[:, 1:]\n",
        "        )\n",
        "        seq_log_prob = torch.sum(log_probs[:, input_len:])\n",
        "    return seq_log_prob.cpu().numpy()"
      ],
      "metadata": {
        "id": "idvr7Tfq7-s3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "logp = sequence_logprob(model, output_greedy, input_len=len(input_ids[0]))\n",
        "print(tokenizer.decode(output_greedy[0]))\n",
        "print(f'\\nlog-prob: {logp:.2f}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QkZ3t-Wb9mBS",
        "outputId": "49dfde69-6b58-4b8e-9723-fb1defd6638e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "\"The unicorns were very intelligent, and they were very intelligent,\" said Dr. David S. Siegel, a professor of anthropology at the University of California, Berkeley. \"They were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very intelligent, and they were very\n",
            "\n",
            "log-prob: -83.33\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's generate text with beam search activated\n",
        "output_beam = model.generate(input_ids, max_length=max_len,\n",
        "                             num_beams=5, do_sample=False)\n",
        "logp = sequence_logprob(model, output_beam, input_len=len(input_ids[0]))\n",
        "print(tokenizer.decode(output_beam[0]))\n",
        "print(f'\\nlog-prob: {logp: .2f}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gtbf37WX_fS3",
        "outputId": "7dc41b8b-07a8-4e21-80dc-86fda6f702b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The researchers, from the University of California, San Diego, and the University of California, Santa Cruz, found that the unicorns were able to communicate with each other in a way that was similar to that of human speech.\n",
            "\n",
            "\n",
            "\"The unicorns were able to communicate with each other in a way that was similar to that of human speech,\" said study co-lead author Dr. David J.\n",
            "\n",
            "log-prob: -78.34\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output_temp = model.generate(input_ids, max_length=max_len, \n",
        "                             do_sample=True, temperature=.5, top_k=0)\n",
        "print(tokenizer.decode(output_temp[0]))"
      ],
      "metadata": {
        "id": "HfviWFPcAYUd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c62b28f7-012c-4e02-b115-f09cf14cee84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "\"We think that they were born in the wild,\" said lead author, Dr. David Schreiber, a researcher at the University of Washington in Seattle who was not involved in the study. \"They are also very intelligent.\"\n",
            "\n",
            "\n",
            "The researchers believe that the unicorns are being trained to speak English and that they have the ability to communicate with humans.\n",
            "\n",
            "\n",
            "The unicorns, which were born\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "eotSJDeEq5c7"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}