S10_assignement_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Welcome to Colaboratory",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ba3c167029fa4d62bdd42d42dcf66c25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e48d02d13e8e475eb9eded3ef091c8fa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dcf9b2c149374ee8ae292666d7b87d8e",
              "IPY_MODEL_1f7f722bfcff455e8b82031a7d25d2ea"
            ]
          }
        },
        "e48d02d13e8e475eb9eded3ef091c8fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dcf9b2c149374ee8ae292666d7b87d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_222970e3e83d4020b35dc06c7989670b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fda2dd63cff24abb88b642815e6591be"
          }
        },
        "1f7f722bfcff455e8b82031a7d25d2ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c81efda1f3674bcaa788505d481928bf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:20&lt;00:00, 1062237.94it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f33407078ecc48069fabdd45eee76451"
          }
        },
        "222970e3e83d4020b35dc06c7989670b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fda2dd63cff24abb88b642815e6591be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c81efda1f3674bcaa788505d481928bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f33407078ecc48069fabdd45eee76451": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "64a5242e91a848c6ac5d326c1f52f856": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d649f66dcc4a4a9faf0bc8ebdd7ae16b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2a86fe4a7a5345489bd3ae2e4fa201fd",
              "IPY_MODEL_77f01391c32a4e3b95f151c1be26d564"
            ]
          }
        },
        "d649f66dcc4a4a9faf0bc8ebdd7ae16b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a86fe4a7a5345489bd3ae2e4fa201fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a83515273fe149d1a28005c4b78ed7d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c95c59b030c4464a7ad1744e30aeb25"
          }
        },
        "77f01391c32a4e3b95f151c1be26d564": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_58528c0d5f9841378f221ba826b10344",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:01&lt;00:00, 23264.56it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4e048298a064e088e7783166e600c20"
          }
        },
        "a83515273fe149d1a28005c4b78ed7d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c95c59b030c4464a7ad1744e30aeb25": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58528c0d5f9841378f221ba826b10344": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4e048298a064e088e7783166e600c20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e7aa5fbe1ed343cc91eda4281c5d3233": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cbbf2e053cee4f63bef19fc8d02fa939",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b9b3feafb13e45fcb4f8e89804fb380f",
              "IPY_MODEL_15bf542ed9a24240abcc5e4fbb92e91f"
            ]
          }
        },
        "cbbf2e053cee4f63bef19fc8d02fa939": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9b3feafb13e45fcb4f8e89804fb380f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b9ef486db34d435aa0f1f477fc7111fd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb7a48349bdb4af0bd04b0d19576c174"
          }
        },
        "15bf542ed9a24240abcc5e4fbb92e91f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5d307540722d4a86ab13d9a2db3328c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:01&lt;00:00, 1473441.26it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f267f792158b4675a39a177eed8c5fb2"
          }
        },
        "b9ef486db34d435aa0f1f477fc7111fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb7a48349bdb4af0bd04b0d19576c174": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d307540722d4a86ab13d9a2db3328c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f267f792158b4675a39a177eed8c5fb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "759dc74c278048d3948c332b9829db13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_39381f38d16748b9a0350e526b7b2fac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_527c4cf5563743f9b94250f5fba456c2",
              "IPY_MODEL_2c142c16824c4784b841ebe64dc14f14"
            ]
          }
        },
        "39381f38d16748b9a0350e526b7b2fac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "527c4cf5563743f9b94250f5fba456c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_13405c2c8fc64be9b20cbb05708e9952",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f065bfbc7f9b406c80c595ae1338d073"
          }
        },
        "2c142c16824c4784b841ebe64dc14f14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dfedf222f55f425b87b60dff33e29b78",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 21391.58it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f10f0d5ef632495697c2cef1498414d3"
          }
        },
        "13405c2c8fc64be9b20cbb05708e9952": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f065bfbc7f9b406c80c595ae1338d073": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfedf222f55f425b87b60dff33e29b78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f10f0d5ef632495697c2cef1498414d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab1d595cb9b04d0ea197198c0823a538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1c72adb672544b26b1c0797a60e3b076",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ab00c4d6c0474ffd911d3ed57af3d705",
              "IPY_MODEL_b287e88b052148b8a401c458444905ea"
            ]
          }
        },
        "1c72adb672544b26b1c0797a60e3b076": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab00c4d6c0474ffd911d3ed57af3d705": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_900cea23f31843d3ae7e83ddd6ddb638",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_855c980b8d574d348019596bac0cf8a1"
          }
        },
        "b287e88b052148b8a401c458444905ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_126a782c33924372bda02529fb25602b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:17&lt;00:00, 52894581.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dcd380b55f644e6fb48d97b8ba8009a9"
          }
        },
        "900cea23f31843d3ae7e83ddd6ddb638": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "855c980b8d574d348019596bac0cf8a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "126a782c33924372bda02529fb25602b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dcd380b55f644e6fb48d97b8ba8009a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thamizhannal/eva5/blob/master/S10_assignement_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mG3TZhHt9rC_",
        "outputId": "b52b72eb-2f5c-476d-da30-981001af76ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "pwd\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rpazjeSHlOIj",
        "outputId": "df59e08d-2d5d-4206-8557-597148fccf00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        }
      },
      "source": [
        "!git clone https://github.com/theschoolof-ai/JEDI.git"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'JEDI'...\n",
            "remote: Enumerating objects: 33, done.\u001b[K\n",
            "remote: Counting objects: 100% (33/33), done.\u001b[K\n",
            "remote: Compressing objects: 100% (28/28), done.\u001b[K\n",
            "remote: Total 289 (delta 14), reused 11 (delta 3), pack-reused 256\u001b[K\n",
            "Receiving objects: 100% (289/289), 158.71 MiB | 38.14 MiB/s, done.\n",
            "Resolving deltas: 100% (117/117), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4PBsTWgA9ueL",
        "outputId": "64e44d10-52de-479d-85ed-320a01a8a8ca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cd /content/JEDI/tsai.jedi"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/JEDI/tsai.jedi\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IgJpNa66lj1U",
        "outputId": "d897cc21-3a4f-41de-9973-4a7feba4abf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "ls"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "aftereffects.py  dataloader.py         gradcam.py      \u001b[0m\u001b[01;34mModels\u001b[0m/\n",
            "batchnorm.py     datatransforms.py     main.py         \u001b[01;34m__pycache__\u001b[0m/\n",
            "config.py        Engine_train_test.py  \u001b[01;34mmodel_objects\u001b[0m/  README.md\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zE5Uz_Jglo56"
      },
      "source": [
        "import sys\n",
        "sys.path.append('Models/')\n",
        "from S8_resnet import ResNet18\n",
        "from S7 import model_summary"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2lmEpMC59wKE"
      },
      "source": [
        "import sys\n",
        "sys.path.append('Models/')\n",
        "from S7 import model_summary\n",
        "from S9_resnet import resnet18"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QtvINx2R9xz5",
        "outputId": "1eb813e6-a487-462f-fc21-41a7315af08d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 659
        }
      },
      "source": [
        "! pip install albumentations==0.4.6"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting albumentations==0.4.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/92/33/1c459c2c9a4028ec75527eff88bc4e2d256555189f42af4baf4d7bd89233/albumentations-0.4.6.tar.gz (117kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 4.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from albumentations==0.4.6) (1.18.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from albumentations==0.4.6) (1.4.1)\n",
            "Collecting imgaug>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/66/b1/af3142c4a85cba6da9f4ebb5ff4e21e2616309552caca5e8acefe9840622/imgaug-0.4.0-py2.py3-none-any.whl (948kB)\n",
            "\u001b[K     |████████████████████████████████| 952kB 14.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from albumentations==0.4.6) (3.13)\n",
            "Requirement already satisfied: opencv-python>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from albumentations==0.4.6) (4.1.2.30)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (3.2.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (1.15.0)\n",
            "Requirement already satisfied: Shapely in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (1.7.1)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (7.0.0)\n",
            "Requirement already satisfied: scikit-image>=0.14.2 in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (0.16.2)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.6/dist-packages (from imgaug>=0.4.0->albumentations==0.4.6) (2.4.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations==0.4.6) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations==0.4.6) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations==0.4.6) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations==0.4.6) (2.8.1)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image>=0.14.2->imgaug>=0.4.0->albumentations==0.4.6) (1.1.1)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image>=0.14.2->imgaug>=0.4.0->albumentations==0.4.6) (2.5)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.0->scikit-image>=0.14.2->imgaug>=0.4.0->albumentations==0.4.6) (4.4.2)\n",
            "Building wheels for collected packages: albumentations\n",
            "  Building wheel for albumentations (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for albumentations: filename=albumentations-0.4.6-cp36-none-any.whl size=65165 sha256=5b1cfec3eaca2de00f4be397a48904d4a154c56ca5fcafaa8b97e306876dbdcc\n",
            "  Stored in directory: /root/.cache/pip/wheels/c7/f4/89/56d1bee5c421c36c1a951eeb4adcc32fbb82f5344c086efa14\n",
            "Successfully built albumentations\n",
            "Installing collected packages: imgaug, albumentations\n",
            "  Found existing installation: imgaug 0.2.9\n",
            "    Uninstalling imgaug-0.2.9:\n",
            "      Successfully uninstalled imgaug-0.2.9\n",
            "  Found existing installation: albumentations 0.1.12\n",
            "    Uninstalling albumentations-0.1.12:\n",
            "      Successfully uninstalled albumentations-0.1.12\n",
            "Successfully installed albumentations-0.4.6 imgaug-0.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dz7p1dYodHJk"
      },
      "source": [
        "import albumentations as A\n",
        "from albumentations.pytorch import ToTensorV2\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "\n",
        "class train_transform_alb(Dataset):\n",
        "    def __init__(self, image_list, label):\n",
        "        self.image_list = image_list\n",
        "        self.label = label\n",
        "        self.aug = A.Compose([\n",
        "            A.Cutout(num_holes=8,  max_h_size=8, max_w_size=8, p=0.5),\n",
        "            A.RandomCrop(32, 32, p=0.5),\n",
        "            A.HorizontalFlip(p=0.5),\n",
        "            A.Rotate((-8.0, 8.0), p=0.5),\n",
        "            A.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))\n",
        "        ])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_list)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        image = Image.fromarray(self.image_list[i]).convert('RGB')\n",
        "        image = self.aug(image=np.array(image))['image']\n",
        "        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n",
        "        image = torch.tensor(image, dtype=torch.float)\n",
        "        label = self.label[i]\n",
        "        label = torch.tensor(label, dtype=torch.long)\n",
        "        return image, label\n",
        "\n",
        "\n",
        "class test_transform_alb(Dataset):\n",
        "    def __init__(self, image_list, label):\n",
        "        self.image_list = image_list\n",
        "        self.label = label\n",
        "        self.aug = A.Compose([\n",
        "            A.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))\n",
        "        ])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_list)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        image = Image.fromarray(self.image_list[i]).convert('RGB')\n",
        "        image = self.aug(image=np.array(image))['image']\n",
        "        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n",
        "        image = torch.tensor(image, dtype=torch.float)\n",
        "        label = self.label[i]\n",
        "        label = torch.tensor(label, dtype=torch.long)\n",
        "        return image, label"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "izjb77fec4ai",
        "outputId": "a758b2ca-c572-4723-c131-a578fa4349c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 508,
          "referenced_widgets": [
            "ba3c167029fa4d62bdd42d42dcf66c25",
            "e48d02d13e8e475eb9eded3ef091c8fa",
            "dcf9b2c149374ee8ae292666d7b87d8e",
            "1f7f722bfcff455e8b82031a7d25d2ea",
            "222970e3e83d4020b35dc06c7989670b",
            "fda2dd63cff24abb88b642815e6591be",
            "c81efda1f3674bcaa788505d481928bf",
            "f33407078ecc48069fabdd45eee76451",
            "64a5242e91a848c6ac5d326c1f52f856",
            "d649f66dcc4a4a9faf0bc8ebdd7ae16b",
            "2a86fe4a7a5345489bd3ae2e4fa201fd",
            "77f01391c32a4e3b95f151c1be26d564",
            "a83515273fe149d1a28005c4b78ed7d0",
            "0c95c59b030c4464a7ad1744e30aeb25",
            "58528c0d5f9841378f221ba826b10344",
            "a4e048298a064e088e7783166e600c20",
            "e7aa5fbe1ed343cc91eda4281c5d3233",
            "cbbf2e053cee4f63bef19fc8d02fa939",
            "b9b3feafb13e45fcb4f8e89804fb380f",
            "15bf542ed9a24240abcc5e4fbb92e91f",
            "b9ef486db34d435aa0f1f477fc7111fd",
            "fb7a48349bdb4af0bd04b0d19576c174",
            "5d307540722d4a86ab13d9a2db3328c9",
            "f267f792158b4675a39a177eed8c5fb2",
            "759dc74c278048d3948c332b9829db13",
            "39381f38d16748b9a0350e526b7b2fac",
            "527c4cf5563743f9b94250f5fba456c2",
            "2c142c16824c4784b841ebe64dc14f14",
            "13405c2c8fc64be9b20cbb05708e9952",
            "f065bfbc7f9b406c80c595ae1338d073",
            "dfedf222f55f425b87b60dff33e29b78",
            "f10f0d5ef632495697c2cef1498414d3",
            "ab1d595cb9b04d0ea197198c0823a538",
            "1c72adb672544b26b1c0797a60e3b076",
            "ab00c4d6c0474ffd911d3ed57af3d705",
            "b287e88b052148b8a401c458444905ea",
            "900cea23f31843d3ae7e83ddd6ddb638",
            "855c980b8d574d348019596bac0cf8a1",
            "126a782c33924372bda02529fb25602b",
            "dcd380b55f644e6fb48d97b8ba8009a9"
          ]
        }
      },
      "source": [
        "import sys\n",
        "sys.path.append(\"tsai.jedi\")\n",
        "#from __future__ import print_function\n",
        "import torch\n",
        "from torchvision import datasets, transforms\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from datatransforms import train_transform_alb, test_transform_alb\n",
        "import config\n",
        "\n",
        "torch.manual_seed(1)\n",
        "kwargs = {'num_workers': config.num_workers, 'pin_memory': config.pin_memory} if config.use_cuda else {}\n",
        "\n",
        "#MNIST\n",
        "train_loader_MNIST = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=True, download=True,\n",
        "                   transform=transforms.Compose([\n",
        "                       transforms.RandomRotation((-8.0, 8.0), fill=(1,)),\n",
        "                       transforms.ToTensor(),\n",
        "                       transforms.Normalize((0.1307,), (0.3081,))\n",
        "                   ])),\n",
        "    batch_size=config.batch_size, shuffle=True, **kwargs)\n",
        "test_loader_MNIST = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.1307,), (0.3081,))\n",
        "    ])),\n",
        "    batch_size=config.batch_size, shuffle=True, **kwargs)\n",
        "#CIFAR10\n",
        "train_loader_CIFAR10 = torch.utils.data.DataLoader(\n",
        "    datasets.CIFAR10('../data', train=True, download=True,\n",
        "                     transform=transforms.Compose([\n",
        "                         transforms.RandomCrop(32, padding=4),\n",
        "                         transforms.RandomHorizontalFlip(),\n",
        "                         transforms.RandomRotation((-8.0, 8.0)),\n",
        "                         transforms.ToTensor(),\n",
        "                         transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
        "                     ])),\n",
        "    batch_size=config.batch_size, shuffle=True, **kwargs)\n",
        "test_loader_CIFAR10 = torch.utils.data.DataLoader(\n",
        "    datasets.CIFAR10('../data', train=False, transform=transforms.Compose([\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
        "    ])),\n",
        "    batch_size=config.batch_size, shuffle=True, **kwargs)\n",
        "\n",
        "#CIFAR-10 Albumentation\n",
        "datasets.CIFAR10('../data', train=True, download=True)\n",
        "\n",
        "\n",
        "def load_cifar10_data(filename):\n",
        "    with open('../data/cifar-10-batches-py/' + filename, 'rb') as file:\n",
        "        batch = pickle.load(file, encoding='latin1')\n",
        "\n",
        "    features = batch['data']\n",
        "    labels = batch['labels']\n",
        "    return features, labels\n",
        "\n",
        "\n",
        "batch_1, labels_1 = load_cifar10_data('data_batch_1')\n",
        "batch_2, labels_2 = load_cifar10_data('data_batch_2')\n",
        "batch_3, labels_3 = load_cifar10_data('data_batch_3')\n",
        "batch_4, labels_4 = load_cifar10_data('data_batch_4')\n",
        "batch_5, labels_5 = load_cifar10_data('data_batch_5')\n",
        "\n",
        "test, label_test = load_cifar10_data('test_batch')\n",
        "\n",
        "X_train = np.concatenate([batch_1, batch_2, batch_3, batch_4, batch_5], 0)\n",
        "Y_train = np.concatenate([labels_1, labels_2, labels_3, labels_4, labels_5], 0)\n",
        "\n",
        "\n",
        "def return_photo(batch_file):\n",
        "    assert batch_file.shape[1] == 3072\n",
        "    dim = np.sqrt(1024).astype(int)\n",
        "    r = batch_file[:, 0:1024].reshape(batch_file.shape[0], dim, dim, 1)\n",
        "    g = batch_file[:, 1024:2048].reshape(batch_file.shape[0], dim, dim, 1)\n",
        "    b = batch_file[:, 2048:3072].reshape(batch_file.shape[0], dim, dim, 1)\n",
        "    photo = np.concatenate([r, g, b], -1)\n",
        "    return photo\n",
        "\n",
        "\n",
        "X_train = return_photo(X_train)\n",
        "X_test = return_photo(test)\n",
        "Y_test = np.array(label_test)\n",
        "\n",
        "classes_CIFAR10 = ('plane', 'car', 'bird', 'cat',\n",
        "                   'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
        "\n",
        "trainset = train_transform_alb(image_list=X_train, label=Y_train)\n",
        "testset = test_transform_alb(image_list=X_test, label=Y_test)\n",
        "\n",
        "train_loader_CIFAR10_alb = torch.utils.data.DataLoader(trainset,\n",
        "                                                   batch_size=config.batch_size, shuffle=True, **kwargs)\n",
        "\n",
        "test_loader_CIFAR10_alb = torch.utils.data.DataLoader(testset,\n",
        "                                                  batch_size=config.batch_size, shuffle=True, **kwargs)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ../data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ba3c167029fa4d62bdd42d42dcf66c25",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/train-images-idx3-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ../data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "64a5242e91a848c6ac5d326c1f52f856",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/train-labels-idx1-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ../data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e7aa5fbe1ed343cc91eda4281c5d3233",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/t10k-images-idx3-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ../data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "759dc74c278048d3948c332b9829db13",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ../data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n",
            "\n",
            "\n",
            "\n",
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchvision/datasets/mnist.py:469: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
            "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab1d595cb9b04d0ea197198c0823a538",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/cifar-10-python.tar.gz to ../data\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d244YvVB94Ov"
      },
      "source": [
        "sys.path.append(\".\")\n",
        "%matplotlib inline\n",
        "import torch\n",
        "import config\n",
        "import torch.optim as optim\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "from Engine_train_test import train, test\n",
        "#from dataloader import train_loader_CIFAR10_alb, test_loader_CIFAR10_alb"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lrpg-Fri_pRy",
        "outputId": "a7f18480-e11e-49bb-a954-92446568faa6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model_ = resnet18(num_classes = 10,pretrained = False).to(config.device)\n",
        "print(model_summary(model_, config.input_size_CIFAR10))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
            "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
            "              ReLU-3           [-1, 64, 32, 32]               0\n",
            "            Conv2d-4           [-1, 64, 32, 32]          36,864\n",
            "       BatchNorm2d-5           [-1, 64, 32, 32]             128\n",
            "              ReLU-6           [-1, 64, 32, 32]               0\n",
            "            Conv2d-7           [-1, 64, 32, 32]          36,864\n",
            "       BatchNorm2d-8           [-1, 64, 32, 32]             128\n",
            "              ReLU-9           [-1, 64, 32, 32]               0\n",
            "       BasicBlock-10           [-1, 64, 32, 32]               0\n",
            "           Conv2d-11           [-1, 64, 32, 32]          36,864\n",
            "      BatchNorm2d-12           [-1, 64, 32, 32]             128\n",
            "             ReLU-13           [-1, 64, 32, 32]               0\n",
            "           Conv2d-14           [-1, 64, 32, 32]          36,864\n",
            "      BatchNorm2d-15           [-1, 64, 32, 32]             128\n",
            "             ReLU-16           [-1, 64, 32, 32]               0\n",
            "       BasicBlock-17           [-1, 64, 32, 32]               0\n",
            "           Conv2d-18          [-1, 128, 16, 16]          73,728\n",
            "      BatchNorm2d-19          [-1, 128, 16, 16]             256\n",
            "             ReLU-20          [-1, 128, 16, 16]               0\n",
            "           Conv2d-21          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-22          [-1, 128, 16, 16]             256\n",
            "           Conv2d-23          [-1, 128, 16, 16]           8,192\n",
            "      BatchNorm2d-24          [-1, 128, 16, 16]             256\n",
            "             ReLU-25          [-1, 128, 16, 16]               0\n",
            "       BasicBlock-26          [-1, 128, 16, 16]               0\n",
            "           Conv2d-27          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-28          [-1, 128, 16, 16]             256\n",
            "             ReLU-29          [-1, 128, 16, 16]               0\n",
            "           Conv2d-30          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-31          [-1, 128, 16, 16]             256\n",
            "             ReLU-32          [-1, 128, 16, 16]               0\n",
            "       BasicBlock-33          [-1, 128, 16, 16]               0\n",
            "           Conv2d-34            [-1, 256, 8, 8]         294,912\n",
            "      BatchNorm2d-35            [-1, 256, 8, 8]             512\n",
            "             ReLU-36            [-1, 256, 8, 8]               0\n",
            "           Conv2d-37            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-38            [-1, 256, 8, 8]             512\n",
            "           Conv2d-39            [-1, 256, 8, 8]          32,768\n",
            "      BatchNorm2d-40            [-1, 256, 8, 8]             512\n",
            "             ReLU-41            [-1, 256, 8, 8]               0\n",
            "       BasicBlock-42            [-1, 256, 8, 8]               0\n",
            "           Conv2d-43            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-44            [-1, 256, 8, 8]             512\n",
            "             ReLU-45            [-1, 256, 8, 8]               0\n",
            "           Conv2d-46            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-47            [-1, 256, 8, 8]             512\n",
            "             ReLU-48            [-1, 256, 8, 8]               0\n",
            "       BasicBlock-49            [-1, 256, 8, 8]               0\n",
            "           Conv2d-50            [-1, 512, 4, 4]       1,179,648\n",
            "      BatchNorm2d-51            [-1, 512, 4, 4]           1,024\n",
            "             ReLU-52            [-1, 512, 4, 4]               0\n",
            "           Conv2d-53            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-54            [-1, 512, 4, 4]           1,024\n",
            "           Conv2d-55            [-1, 512, 4, 4]         131,072\n",
            "      BatchNorm2d-56            [-1, 512, 4, 4]           1,024\n",
            "             ReLU-57            [-1, 512, 4, 4]               0\n",
            "       BasicBlock-58            [-1, 512, 4, 4]               0\n",
            "           Conv2d-59            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-60            [-1, 512, 4, 4]           1,024\n",
            "             ReLU-61            [-1, 512, 4, 4]               0\n",
            "           Conv2d-62            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-63            [-1, 512, 4, 4]           1,024\n",
            "             ReLU-64            [-1, 512, 4, 4]               0\n",
            "       BasicBlock-65            [-1, 512, 4, 4]               0\n",
            "AdaptiveAvgPool2d-66            [-1, 512, 1, 1]               0\n",
            "           Linear-67                   [-1, 10]           5,130\n",
            "================================================================\n",
            "Total params: 11,173,962\n",
            "Trainable params: 11,173,962\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 15.50\n",
            "Params size (MB): 42.63\n",
            "Estimated Total Size (MB): 58.14\n",
            "----------------------------------------------------------------\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ge6j4ZW2Ebjy"
      },
      "source": [
        "import albumentations as A\n",
        "from albumentations.pytorch import ToTensorV2\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "\n",
        "class train_transform_alb(Dataset):\n",
        "    def __init__(self, image_list, label):\n",
        "        self.image_list = image_list\n",
        "        self.label = label\n",
        "        self.aug = A.Compose([\n",
        "            A.RandomCrop(32, 32, p=0.5),\n",
        "            A.Cutout(num_holes=8,  max_h_size=8, max_w_size=8, p=0.5),\n",
        "            A.HorizontalFlip(p=0.5),\n",
        "            A.Rotate((-8.0, 8.0), p=0.5),\n",
        "            A.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))\n",
        "        ])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_list)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        image = Image.fromarray(self.image_list[i]).convert('RGB')\n",
        "        image = self.aug(image=np.array(image))['image']\n",
        "        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n",
        "        image = torch.tensor(image, dtype=torch.float)\n",
        "        label = self.label[i]\n",
        "        label = torch.tensor(label, dtype=torch.long)\n",
        "        return image, label\n",
        "\n",
        "\n",
        "class test_transform_alb(Dataset):\n",
        "    def __init__(self, image_list, label):\n",
        "        self.image_list = image_list\n",
        "        self.label = label\n",
        "        self.aug = A.Compose([\n",
        "            A.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))\n",
        "        ])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_list)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        image = Image.fromarray(self.image_list[i]).convert('RGB')\n",
        "        image = self.aug(image=np.array(image))['image']\n",
        "        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n",
        "        image = torch.tensor(image, dtype=torch.float)\n",
        "        label = self.label[i]\n",
        "        label = torch.tensor(label, dtype=torch.long)\n",
        "        return image, label"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "66MbN2FmEA27",
        "outputId": "6e8e62b0-cee7-4330-f363-1b93a1e4e3d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "'''\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(model.parameters(), lr=1e-7, weight_decay=1e-2)\n",
        "lr_finder = LRFinder(model, optimizer, criterion, device=\"cuda\")\n",
        "lr_finder.range_test(trainloader, end_lr=100, num_iter=100, step_mode=\"exp\")\n",
        "'''\n",
        "%matplotlib inline\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from __future__ import print_function\n",
        "import torch.optim as optim\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "import sys\n",
        "\n",
        "\n",
        "\n",
        "import torch.optim as optim\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "\n",
        "accu = []\n",
        "loss_test = []\n",
        "\n",
        "# SGD\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model_.parameters(), lr=1e-7,  momentum=0.9, weight_decay=1e-2)\n",
        "lr_finder = LRFinder(model_, optimizer, criterion, device=\"cuda\")\n",
        "lr_finder.range_test(train_loader_CIFAR10_alb, end_lr=100, num_iter=100, step_mode=\"exp\")\n",
        "\n",
        "lr_finder.plot()\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-dd15ec485090>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0mcriterion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-7\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mmomentum\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mlr_finder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLRFinder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0mlr_finder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrange_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader_CIFAR10_alb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"exp\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'LRFinder' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zW_9LepFrRTy",
        "outputId": "5340e319-a744-4c1a-feac-6a00239edaab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
        "valid_acc = []\n",
        "loss_test_ = []\n",
        "l1_regularization = [0, 0]\n",
        "\n",
        "# https://pytorch.org/docs/stable/optim.html#torch.optim.lr_scheduler.ReduceLROnPlateau\n",
        "\n",
        "# Assign optimizer with suggested learning rate and run it with reduceonpleatue\n",
        "optimizer = optim.SGD(model_.parameters(), lr=5.34E-02,  momentum=0.9, weight_decay=1e-2)\n",
        "\n",
        "scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=10, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08, verbose=False)\n",
        "for epoch in range(50):\n",
        "    train(model_, config.device, train_loader_CIFAR10_alb, optimizer, epoch, l1_regularization=[0, 1])\n",
        "    #scheduler.step()\n",
        "    valid_a, valid_l = test(model_, config.device, test_loader_CIFAR10_alb)\n",
        "    # Appending to loss and accuracy lists\n",
        "    valid_acc.append(valid_a)\n",
        "    loss_test_.append(valid_l)\n",
        "    # Note that step should be called after validate()\n",
        "    scheduler.step(valid_l)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0,LR: 0.0534.\n",
            "Train set: train Average loss: 1.2298, train_Accuracy: 22109/50000 (44.2180%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.0989410877227783 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.79it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0138, Accuracy: 3843/10000 (38.4300%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0300, train_Accuracy: 30390/50000 (60.7800%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.2399870157241821 batch_id=0:   0%|          | 1/391 [00:00<01:13,  5.30it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0125, Accuracy: 5122/10000 (51.2200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 2,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0775, train_Accuracy: 32276/50000 (64.5520%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.979661226272583 batch_id=0:   0%|          | 1/391 [00:00<01:11,  5.44it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0125, Accuracy: 4895/10000 (48.9500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 3,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0079, train_Accuracy: 32827/50000 (65.6540%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.8037779927253723 batch_id=0:   0%|          | 1/391 [00:00<01:08,  5.69it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0191, Accuracy: 3610/10000 (36.1000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 4,LR: 0.0534.\n",
            "Train set: train Average loss: 0.9621, train_Accuracy: 33005/50000 (66.0100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9710895419120789 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.74it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0111, Accuracy: 5147/10000 (51.4700%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 5,LR: 0.0534.\n",
            "Train set: train Average loss: 1.1195, train_Accuracy: 33269/50000 (66.5380%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.8582000136375427 batch_id=0:   0%|          | 1/391 [00:00<01:08,  5.69it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0136, Accuracy: 4441/10000 (44.4100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 6,LR: 0.0534.\n",
            "Train set: train Average loss: 1.1478, train_Accuracy: 33125/50000 (66.2500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.0418254137039185 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.76it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0084, Accuracy: 6280/10000 (62.8000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 7,LR: 0.0534.\n",
            "Train set: train Average loss: 1.1649, train_Accuracy: 33323/50000 (66.6460%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9529560208320618 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.83it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0127, Accuracy: 4656/10000 (46.5600%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 8,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0770, train_Accuracy: 33246/50000 (66.4920%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9478516578674316 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.77it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0109, Accuracy: 5442/10000 (54.4200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 9,LR: 0.0534.\n",
            "Train set: train Average loss: 0.9703, train_Accuracy: 33282/50000 (66.5640%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.8390302062034607 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0119, Accuracy: 4711/10000 (47.1100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 10,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0655, train_Accuracy: 33470/50000 (66.9400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.0003242492675781 batch_id=0:   0%|          | 1/391 [00:00<01:09,  5.60it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0137, Accuracy: 4422/10000 (44.2200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 11,LR: 0.0534.\n",
            "Train set: train Average loss: 0.8651, train_Accuracy: 33441/50000 (66.8820%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9282758831977844 batch_id=0:   0%|          | 1/391 [00:00<01:10,  5.53it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0117, Accuracy: 5188/10000 (51.8800%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 12,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0572, train_Accuracy: 33188/50000 (66.3760%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.1229486465454102 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.92it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0109, Accuracy: 5180/10000 (51.8000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 13,LR: 0.0534.\n",
            "Train set: train Average loss: 0.8448, train_Accuracy: 33287/50000 (66.5740%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9964310526847839 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.83it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0121, Accuracy: 4953/10000 (49.5300%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 14,LR: 0.0534.\n",
            "Train set: train Average loss: 1.1191, train_Accuracy: 33266/50000 (66.5320%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.0732115507125854 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.75it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0140, Accuracy: 4565/10000 (45.6500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 15,LR: 0.0534.\n",
            "Train set: train Average loss: 1.2565, train_Accuracy: 33360/50000 (66.7200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9231301546096802 batch_id=0:   0%|          | 1/391 [00:00<01:09,  5.61it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0135, Accuracy: 4665/10000 (46.6500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 16,LR: 0.0534.\n",
            "Train set: train Average loss: 1.0341, train_Accuracy: 33245/50000 (66.4900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=1.1445574760437012 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.80it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0119, Accuracy: 4721/10000 (47.2100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 17,LR: 0.0534.\n",
            "Train set: train Average loss: 1.1209, train_Accuracy: 33077/50000 (66.1540%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.9291502833366394 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.95it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0130, Accuracy: 4674/10000 (46.7400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 18,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.6726, train_Accuracy: 38589/50000 (77.1780%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.6520923376083374 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.78it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0049, Accuracy: 7930/10000 (79.3000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 19,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.6470, train_Accuracy: 40260/50000 (80.5200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.5079575181007385 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 7928/10000 (79.2800%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 20,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5228, train_Accuracy: 40847/50000 (81.6940%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.5378351211547852 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.85it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 7964/10000 (79.6400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 21,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5922, train_Accuracy: 40946/50000 (81.8920%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.6097630858421326 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.84it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0057, Accuracy: 7624/10000 (76.2400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 22,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.3605, train_Accuracy: 41295/50000 (82.5900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.3557444214820862 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.96it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 7910/10000 (79.1000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 23,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5043, train_Accuracy: 41498/50000 (82.9960%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.666909396648407 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.88it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8149/10000 (81.4900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 24,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4168, train_Accuracy: 41667/50000 (83.3340%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4861440658569336 batch_id=0:   0%|          | 1/391 [00:00<01:10,  5.54it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0043, Accuracy: 8192/10000 (81.9200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 25,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5784, train_Accuracy: 41865/50000 (83.7300%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4139959514141083 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.91it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0050, Accuracy: 7881/10000 (78.8100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 26,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5803, train_Accuracy: 42109/50000 (84.2180%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4088798463344574 batch_id=0:   0%|          | 1/391 [00:00<01:08,  5.69it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0051, Accuracy: 7822/10000 (78.2200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 27,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4545, train_Accuracy: 42206/50000 (84.4120%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.5148842930793762 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 7996/10000 (79.9600%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 28,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4200, train_Accuracy: 42386/50000 (84.7720%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.5573347210884094 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.97it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 7962/10000 (79.6200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 29,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5209, train_Accuracy: 42489/50000 (84.9780%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.437851220369339 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.98it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0050, Accuracy: 7809/10000 (78.0900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 30,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4082, train_Accuracy: 42616/50000 (85.2320%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.387416809797287 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.77it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8126/10000 (81.2600%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 31,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5755, train_Accuracy: 42682/50000 (85.3640%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4216344356536865 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.96it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0053, Accuracy: 7725/10000 (77.2500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 32,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.3630, train_Accuracy: 42714/50000 (85.4280%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4483417868614197 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.91it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0055, Accuracy: 7644/10000 (76.4400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 33,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4161, train_Accuracy: 42821/50000 (85.6420%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.41062846779823303 batch_id=0:   0%|          | 1/391 [00:00<01:08,  5.71it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0045, Accuracy: 8116/10000 (81.1600%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 34,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4607, train_Accuracy: 42776/50000 (85.5520%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4481208622455597 batch_id=0:   0%|          | 1/391 [00:00<01:05,  5.99it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0041, Accuracy: 8271/10000 (82.7100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 35,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4086, train_Accuracy: 42939/50000 (85.8780%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.5187192559242249 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.82it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0046, Accuracy: 8019/10000 (80.1900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 36,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5370, train_Accuracy: 42995/50000 (85.9900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.3911520540714264 batch_id=0:   0%|          | 1/391 [00:00<01:05,  6.00it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0060, Accuracy: 7434/10000 (74.3400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 37,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.3657, train_Accuracy: 43198/50000 (86.3960%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.46527227759361267 batch_id=0:   0%|          | 1/391 [00:00<01:10,  5.50it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0054, Accuracy: 7780/10000 (77.8000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 38,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4088, train_Accuracy: 43094/50000 (86.1880%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.441404789686203 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.90it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8167/10000 (81.6700%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 39,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.5222, train_Accuracy: 43103/50000 (86.2060%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.38894230127334595 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.87it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0047, Accuracy: 8000/10000 (80.0000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 40,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4372, train_Accuracy: 43127/50000 (86.2540%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.3426204323768616 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.88it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0047, Accuracy: 7940/10000 (79.4000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 41,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.6896, train_Accuracy: 43195/50000 (86.3900%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4420810639858246 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.81it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0060, Accuracy: 7456/10000 (74.5600%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 42,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4554, train_Accuracy: 43353/50000 (86.7060%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.33869802951812744 batch_id=0:   0%|          | 1/391 [00:00<01:05,  6.00it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8111/10000 (81.1100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 43,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.3659, train_Accuracy: 43383/50000 (86.7660%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.33882012963294983 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.90it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0045, Accuracy: 8090/10000 (80.9000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 44,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.6373, train_Accuracy: 43340/50000 (86.6800%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.46461981534957886 batch_id=0:   0%|          | 1/391 [00:00<01:04,  6.00it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0054, Accuracy: 7690/10000 (76.9000%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 45,LR: 0.005340000000000001.\n",
            "Train set: train Average loss: 0.4954, train_Accuracy: 43394/50000 (86.7880%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.4479604959487915 batch_id=0:   0%|          | 1/391 [00:00<01:06,  5.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0047, Accuracy: 8025/10000 (80.2500%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 46,LR: 0.0005340000000000001.\n",
            "Train set: train Average loss: 0.2023, train_Accuracy: 45829/50000 (91.6580%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.2002590000629425 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.79it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0027, Accuracy: 8872/10000 (88.7200%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 47,LR: 0.0005340000000000001.\n",
            "Train set: train Average loss: 0.2268, train_Accuracy: 46771/50000 (93.5420%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.22352555394172668 batch_id=0:   0%|          | 1/391 [00:00<01:07,  5.75it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0025, Accuracy: 8951/10000 (89.5100%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 48,LR: 0.0005340000000000001.\n",
            "Train set: train Average loss: 0.2483, train_Accuracy: 47242/50000 (94.4840%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.126704141497612 batch_id=0:   0%|          | 1/391 [00:00<01:08,  5.72it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0025, Accuracy: 8984/10000 (89.8400%)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 49,LR: 0.0005340000000000001.\n",
            "Train set: train Average loss: 0.2102, train_Accuracy: 47549/50000 (95.0980%)\n",
            "\n",
            "\n",
            "Test set: Average loss: 0.0025, Accuracy: 8963/10000 (89.6300%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ISDAqABe5lh"
      },
      "source": [
        "# Save the model\n",
        "torch.save(model_, '/content/JEDI/tsai.jedi/model_objects/s10_resnet_albu_cutout_lrfinder_v1.pt')"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W7mBGpadqnIx"
      },
      "source": [
        "PATH='/content/JEDI/tsai.jedi/model_objects/s10_resnet_albu_cutout_lrfinder_v1.pt'\n",
        "model = torch.load(PATH)\n",
        "model.eval()\n",
        "\n",
        "tot = 0\n",
        "data_ = []\n",
        "target_ = []\n",
        "pred_ = []\n",
        "correct = 0\n",
        "tot_correct = 0\n",
        "indx = []\n",
        "\n",
        "for data, target in test_loader_CIFAR10_alb:\n",
        "  data, target = data.to(config.device), target.to(config.device)\n",
        "  output = model(data)\n",
        "  pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
        "  correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "  equals = pred.eq(target.view_as(pred)).tolist()\n",
        "    \n",
        "  for idx in range(0,len(equals)):\n",
        "    if equals[idx] == [False]:\n",
        "      indx.append(idx)\n",
        "      data_.append(data[idx])\n",
        "      target_.append(target[idx])\n",
        "      pred_.append(pred[idx])\n",
        "\n",
        "\n",
        "  if len(indx) > 25:\n",
        "    break\n",
        "\t"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kxxBN3n1qqHh",
        "outputId": "e832e806-2939-4c25-a136-369cba426f54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "# plots images with labels within jupyter notebook\n",
        "def plots(ims, figsize=(15,4), rows=2, interp=False,titles = None,actual_label=None):\n",
        "    f = plt.figure(figsize=figsize)\n",
        "    cols = len(ims)//rows if len(ims) % 2 == 0 else len(ims)//rows + 1\n",
        "    \n",
        "    for i in range(len(ims)):\n",
        "        sp = f.add_subplot(rows, cols, i+1)\n",
        "        if titles is not None and actual_label is not None:\n",
        "            sp.set_title(\"Pred:{} Actual:{}\".format(titles[i].tolist(),actual_label[i].tolist()), fontsize=12)\n",
        "\n",
        "        sp.axis('Off')\n",
        "        plt.imshow((torch.squeeze(ims[i]).tolist()))\n",
        "\n",
        "plots(data_,titles = pred_,actual_label=target_)\t"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-162f61c78b0f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mims\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mplots\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtitles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpred_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mactual_label\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-22-162f61c78b0f>\u001b[0m in \u001b[0;36mplots\u001b[0;34m(ims, figsize, rows, interp, titles, actual_label)\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0msp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Off'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mims\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mplots\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtitles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpred_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mactual_label\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, data, **kwargs)\u001b[0m\n\u001b[1;32m   2649\u001b[0m         \u001b[0mfilternorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilternorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilterrad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilterrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimlim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mimlim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2650\u001b[0m         resample=resample, url=url, **({\"data\": data} if data is not\n\u001b[0;32m-> 2651\u001b[0;31m         None else {}), **kwargs)\n\u001b[0m\u001b[1;32m   2652\u001b[0m     \u001b[0msci\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m__ret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2653\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m__ret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1563\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1565\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msanitize_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1567\u001b[0m         \u001b[0mbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_sig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/cbook/deprecation.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    356\u001b[0m                 \u001b[0;34mf\"%(removal)s.  If any parameter follows {name!r}, they \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m                 f\"should be pass as keyword, not positionally.\")\n\u001b[0;32m--> 358\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/cbook/deprecation.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    356\u001b[0m                 \u001b[0;34mf\"%(removal)s.  If any parameter follows {name!r}, they \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m                 f\"should be pass as keyword, not positionally.\")\n\u001b[0;32m--> 358\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(self, X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, **kwargs)\u001b[0m\n\u001b[1;32m   5624\u001b[0m                               resample=resample, **kwargs)\n\u001b[1;32m   5625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5626\u001b[0;31m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5627\u001b[0m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_alpha\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5628\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_clip_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/image.py\u001b[0m in \u001b[0;36mset_data\u001b[0;34m(self, A)\u001b[0m\n\u001b[1;32m    697\u001b[0m                 or self._A.ndim == 3 and self._A.shape[-1] in [3, 4]):\n\u001b[1;32m    698\u001b[0m             raise TypeError(\"Invalid shape {} for image data\"\n\u001b[0;32m--> 699\u001b[0;31m                             .format(self._A.shape))\n\u001b[0m\u001b[1;32m    700\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_A\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: Invalid shape (3, 32, 32) for image data"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAHMAAABQCAYAAAAqe0p+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAE50lEQVR4nO2ZW4hVVRjHf//KvI5EKkUyWjlGRUVjoglaVj6kdhGtwMILQWQaXUiKIsHMeogeeogkipA0woqsDEToIVBmHoKKzHva2Gij3Rh0xBiyr4e9jm1PM+fsM5xw+Ph+sGDvvW7fWr+z1trnHJkZgQ/OOdsBBPUjZDoiZDoiZDoiZDoiZDoiZDrirMqUtFbS6l7yFks6JalL0lUF29svqVvS+vpGWn8kmaSmerZZVaakNkkn06QeTQKG1TOICrSa2TAz25ViyQsupemlwmY2Dni5WqOSpqfJfKZoIGkeZvRlELUiaXPZGLslba9Wr+jKvNPMhgETgInA8z0EcF5tIfeZkuBS+rIPbSwC/gAW1je0+mBmM/NjBFqAD6vVq2mbNbPDwGbgGji9VSyTtA/Yl57dIelbSZ2SWiRdV6ovqVnS15KOS9oADKql/3ogaShwD7AMGC9pYln+Q5J2pRh3SpogaR0wBtiUVsrTaXUfKqt7evVKmiSpNc1Dh6TXJZ3fh3gvBaYB71YrW5NMSY3ALOCb3OM5wGTgaknNwDvAw8AI4E3gM0kD00A+AdYBF5J90uaVtd8paWqVMJol/SZpr6QVfdgR5gJdqf8tZKu01P+9wEqyFTscuAv43cwWAD+Rdigze6VAP6eAJ4GRwBTgNmBpTwUl3S/pu17aWQhsNbO2qj2aWcUEtJENvhM4CLwBDE55BtyaK7sGeLGs/h7gZuAm4GdAubwWYHUv/S4GtpU9uxy4jOxDeC2wE3i2rMxKYH2F8XwBvJau5wO/AgPS/Rbg8QrzMCN3Px04VKlMWd4TwMbcvQFNBeb/B2BxtXJmVnhlzjGzC8xsrJktNbOTubz23PVY4Km0wjoldQKNwCUpHbYz/6Y5WLB/AMzsgJn9aGZ/m9l2YBXZllmItLPcAryXHn1KttXPTveNwP5aYqrQ1xWSPpd0RNIxshezkTW2MRW4GPioSPl6fDXJy2kHXkriS2mImb0PdACjJSlXfkwd+lbVUv+ygGzMmyQdAQ6QySxtte3AuAp95TkBDCndSDoXGJXLXwPsBsab2XDguRpjJcX1sZl1FSlc7++ZbwFLJE1WxlBJsyU1AK3AX8BjkgZImgtMqqVxSTMlXZSurwRWkK2uoiwCXgCuz6V5wCxJI4C3geWSbkjxN0kam+oeJdvmS+wFBqXxDSB7wx+Yy28AjgFdKdZHahzrYOA+YG3hSgXPzN7Ogf/s+8DtwFdkZ2wH2YtGQ8qbSPbydBzYkNLqXN0uYFqFM/NVskk9QbaqVpHOu2pnJnAj8Ccwqoe8HcCj6XoJ2TnfBXwPNKfnd5O9BHUCy3MxdgC/AMvzc0X2jrA7tbM1xbqtp7kDHgB2lMU0n+wYUnm8vSWdeYT1HyQtIHsb7gamWPrhoEqdPcBo4AMze/B/DrHf0W9lBrUTP7Q7ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ6ImQ64h9bWqvhBMlOPQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7gI6jtdqqsTu",
        "outputId": "873b9bba-bf89-4529-f539-bff82b9216b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        }
      },
      "source": [
        "#################gradcam for misclassified images ################\n",
        "import config\n",
        "from gradcam import GradCam\n",
        "from gradcam import show_cam_on_image\n",
        "gradcam_img = []\n",
        "for idx in range(len(data_)): \n",
        "    #ip = data_.index_select(0,torch.tensor(idx))\n",
        "    ip = idx #data_[idx]\n",
        "    grad_cam = GradCam(model=model, feature_module=model.layer3,target_layer_names=[\"1\"], use_cuda=config.use_cuda)\n",
        "    mask = grad_cam(ip)\n",
        "    op_img = show_cam_on_image(ip.squeeze().permute(1,2,0), mask)\n",
        "    gradcam_img.append(op_img)\n",
        "#plots(gradcam_img,titles=target_t )\n",
        "plots(data_,titles = pred_,actual_label=target_)\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-26-fcf1ff860f1f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mip\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0midx\u001b[0m \u001b[0;31m#data_[idx]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mgrad_cam\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGradCam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_module\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtarget_layer_names\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"1\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_cuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrad_cam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mop_img\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mshow_cam_on_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mip\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mgradcam_img\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop_img\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/JEDI/tsai.jedi/gradcam.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, input, index)\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m             \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'int' object has no attribute 'cuda'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwuzdZ7sgAMR"
      },
      "source": [
        "import copy\n",
        "import os\n",
        "import torch\n",
        "import numpy as np\n",
        "from tqdm.autonotebook import tqdm\n",
        "from torch.optim.lr_scheduler import _LRScheduler\n",
        "import matplotlib.pyplot as plt\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "from packaging import version\n",
        "\n",
        "PYTORCH_VERSION = version.parse(torch.__version__)\n",
        "\n",
        "try:\n",
        "    from apex import amp\n",
        "\n",
        "    IS_AMP_AVAILABLE = True\n",
        "except ImportError:\n",
        "    IS_AMP_AVAILABLE = False\n",
        "\n",
        "\n",
        "class DataLoaderIter(object):\n",
        "    def __init__(self, data_loader):\n",
        "        self.data_loader = data_loader\n",
        "        self._iterator = iter(data_loader)\n",
        "\n",
        "    @property\n",
        "    def dataset(self):\n",
        "        return self.data_loader.dataset\n",
        "\n",
        "    def inputs_labels_from_batch(self, batch_data):\n",
        "        if not isinstance(batch_data, list) and not isinstance(batch_data, tuple):\n",
        "            raise ValueError(\n",
        "                \"Your batch type is not supported: {}. Please inherit from \"\n",
        "                \"`TrainDataLoaderIter` or `ValDataLoaderIter` and override the \"\n",
        "                \"`inputs_labels_from_batch` method.\".format(type(batch_data))\n",
        "            )\n",
        "\n",
        "        inputs, labels, *_ = batch_data\n",
        "\n",
        "        return inputs, labels\n",
        "\n",
        "    def __iter__(self):\n",
        "        return self\n",
        "\n",
        "    def __next__(self):\n",
        "        batch = next(self._iterator)\n",
        "        return self.inputs_labels_from_batch(batch)\n",
        "\n",
        "\n",
        "class TrainDataLoaderIter(DataLoaderIter):\n",
        "    def __init__(self, data_loader, auto_reset=True):\n",
        "        super().__init__(data_loader)\n",
        "        self.auto_reset = auto_reset\n",
        "\n",
        "    def __next__(self):\n",
        "        try:\n",
        "            batch = next(self._iterator)\n",
        "            inputs, labels = self.inputs_labels_from_batch(batch)\n",
        "        except StopIteration:\n",
        "            if not self.auto_reset:\n",
        "                raise\n",
        "            self._iterator = iter(self.data_loader)\n",
        "            batch = next(self._iterator)\n",
        "            inputs, labels = self.inputs_labels_from_batch(batch)\n",
        "\n",
        "        return inputs, labels\n",
        "\n",
        "\n",
        "class ValDataLoaderIter(DataLoaderIter):\n",
        "    \"\"\"This iterator will reset itself **only** when it is acquired by\n",
        "    the syntax of normal `iterator`. That is, this iterator just works\n",
        "    like a `torch.data.DataLoader`. If you want to restart it, you\n",
        "    should use it like:\n",
        "\n",
        "        ```\n",
        "        loader_iter = ValDataLoaderIter(data_loader)\n",
        "        for batch in loader_iter:\n",
        "            ...\n",
        "\n",
        "        # `loader_iter` should run out of values now, you can restart it by:\n",
        "        # 1. the way we use a `torch.data.DataLoader`\n",
        "        for batch in loader_iter:        # __iter__ is called implicitly\n",
        "            ...\n",
        "\n",
        "        # 2. passing it into `iter()` manually\n",
        "        loader_iter = iter(loader_iter)  # __iter__ is called by `iter()`\n",
        "        ```\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, data_loader):\n",
        "        super().__init__(data_loader)\n",
        "        self.run_limit = len(self.data_loader)\n",
        "        self.run_counter = 0\n",
        "\n",
        "    def __iter__(self):\n",
        "        if self.run_counter >= self.run_limit:\n",
        "            self._iterator = iter(self.data_loader)\n",
        "            self.run_counter = 0\n",
        "        return self\n",
        "\n",
        "    def __next__(self):\n",
        "        self.run_counter += 1\n",
        "        return super(ValDataLoaderIter, self).__next__()\n",
        "\n",
        "\n",
        "class LRFinder(object):\n",
        "    \"\"\"Learning rate range test.\n",
        "\n",
        "    The learning rate range test increases the learning rate in a pre-training run\n",
        "    between two boundaries in a linear or exponential manner. It provides valuable\n",
        "    information on how well the network can be trained over a range of learning rates\n",
        "    and what is the optimal learning rate.\n",
        "\n",
        "    Arguments:\n",
        "        model (torch.nn.Module): wrapped model.\n",
        "        optimizer (torch.optim.Optimizer): wrapped optimizer where the defined learning\n",
        "            is assumed to be the lower boundary of the range test.\n",
        "        criterion (torch.nn.Module): wrapped loss function.\n",
        "        device (str or torch.device, optional): a string (\"cpu\" or \"cuda\") with an\n",
        "            optional ordinal for the device type (e.g. \"cuda:X\", where is the ordinal).\n",
        "            Alternatively, can be an object representing the device on which the\n",
        "            computation will take place. Default: None, uses the same device as `model`.\n",
        "        memory_cache (boolean, optional): if this flag is set to True, `state_dict` of\n",
        "            model and optimizer will be cached in memory. Otherwise, they will be saved\n",
        "            to files under the `cache_dir`.\n",
        "        cache_dir (string, optional): path for storing temporary files. If no path is\n",
        "            specified, system-wide temporary directory is used. Notice that this\n",
        "            parameter will be ignored if `memory_cache` is True.\n",
        "\n",
        "    Example:\n",
        "        >>> lr_finder = LRFinder(net, optimizer, criterion, device=\"cuda\")\n",
        "        >>> lr_finder.range_test(dataloader, end_lr=100, num_iter=100)\n",
        "        >>> lr_finder.plot() # to inspect the loss-learning rate graph\n",
        "        >>> lr_finder.reset() # to reset the model and optimizer to their initial state\n",
        "\n",
        "    Reference:\n",
        "    Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186\n",
        "    fastai/lr_find: https://github.com/fastai/fastai\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        model,\n",
        "        optimizer,\n",
        "        criterion,\n",
        "        device=None,\n",
        "        memory_cache=True,\n",
        "        cache_dir=None,\n",
        "    ):\n",
        "        # Check if the optimizer is already attached to a scheduler\n",
        "        self.optimizer = optimizer\n",
        "        self._check_for_scheduler()\n",
        "\n",
        "        self.model = model\n",
        "        self.criterion = criterion\n",
        "        self.history = {\"lr\": [], \"loss\": []}\n",
        "        self.best_loss = None\n",
        "        self.memory_cache = memory_cache\n",
        "        self.cache_dir = cache_dir\n",
        "\n",
        "        # Save the original state of the model and optimizer so they can be restored if\n",
        "        # needed\n",
        "        self.model_device = next(self.model.parameters()).device\n",
        "        self.state_cacher = StateCacher(memory_cache, cache_dir=cache_dir)\n",
        "        self.state_cacher.store(\"model\", self.model.state_dict())\n",
        "        self.state_cacher.store(\"optimizer\", self.optimizer.state_dict())\n",
        "\n",
        "        # If device is None, use the same as the model\n",
        "        if device:\n",
        "            self.device = device\n",
        "        else:\n",
        "            self.device = self.model_device\n",
        "\n",
        "    def reset(self):\n",
        "        \"\"\"Restores the model and optimizer to their initial states.\"\"\"\n",
        "\n",
        "        self.model.load_state_dict(self.state_cacher.retrieve(\"model\"))\n",
        "        self.optimizer.load_state_dict(self.state_cacher.retrieve(\"optimizer\"))\n",
        "        self.model.to(self.model_device)\n",
        "\n",
        "    def range_test(\n",
        "        self,\n",
        "        train_loader,\n",
        "        val_loader=None,\n",
        "        start_lr=None,\n",
        "        end_lr=10,\n",
        "        num_iter=100,\n",
        "        step_mode=\"exp\",\n",
        "        smooth_f=0.05,\n",
        "        diverge_th=5,\n",
        "        accumulation_steps=1,\n",
        "        non_blocking_transfer=True,\n",
        "    ):\n",
        "        \"\"\"Performs the learning rate range test.\n",
        "\n",
        "        Arguments:\n",
        "            train_loader (`torch.utils.data.DataLoader`\n",
        "                or child of `TrainDataLoaderIter`, optional):\n",
        "                the training set data loader.\n",
        "                If your dataset (data loader) returns a tuple (inputs, labels,*) then\n",
        "                Pytorch data loader object can be provided. However, if a dataset\n",
        "                returns different outputs e.g. dicts, then you should inherit\n",
        "                from `TrainDataLoaderIter` class and redefine `inputs_labels_from_batch`\n",
        "                method so that it outputs (inputs, labels).\n",
        "            val_loader (`torch.utils.data.DataLoader`\n",
        "                or child of `ValDataLoaderIter`, optional): if `None` the range test\n",
        "                will only use the training loss. When given a data loader, the model is\n",
        "                evaluated after each iteration on that dataset and the evaluation loss\n",
        "                is used. Note that in this mode the test takes significantly longer but\n",
        "                generally produces more precise results.\n",
        "                Similarly to `train_loader`, if your dataset outputs are not standard\n",
        "                you should inherit from `ValDataLoaderIter` class and\n",
        "                redefine method `inputs_labels_from_batch` so that\n",
        "                it outputs (inputs, labels). Default: None.\n",
        "            start_lr (float, optional): the starting learning rate for the range test.\n",
        "                Default: None (uses the learning rate from the optimizer).\n",
        "            end_lr (float, optional): the maximum learning rate to test. Default: 10.\n",
        "            num_iter (int, optional): the number of iterations over which the test\n",
        "                occurs. Default: 100.\n",
        "            step_mode (str, optional): one of the available learning rate policies,\n",
        "                linear or exponential (\"linear\", \"exp\"). Default: \"exp\".\n",
        "            smooth_f (float, optional): the loss smoothing factor within the [0, 1[\n",
        "                interval. Disabled if set to 0, otherwise the loss is smoothed using\n",
        "                exponential smoothing. Default: 0.05.\n",
        "            diverge_th (int, optional): the test is stopped when the loss surpasses the\n",
        "                threshold:  diverge_th * best_loss. Default: 5.\n",
        "            accumulation_steps (int, optional): steps for gradient accumulation. If it\n",
        "                is 1, gradients are not accumulated. Default: 1.\n",
        "            non_blocking_transfer (bool, optional): when non_blocking_transfer is set,\n",
        "                tries to convert/move data to the device asynchronously if possible,\n",
        "                e.g., moving CPU Tensors with pinned memory to CUDA devices. Default: True.\n",
        "\n",
        "        Example (fastai approach):\n",
        "            >>> lr_finder = LRFinder(net, optimizer, criterion, device=\"cuda\")\n",
        "            >>> lr_finder.range_test(dataloader, end_lr=100, num_iter=100)\n",
        "\n",
        "        Example (Leslie Smith's approach):\n",
        "            >>> lr_finder = LRFinder(net, optimizer, criterion, device=\"cuda\")\n",
        "            >>> lr_finder.range_test(trainloader, val_loader=val_loader, end_lr=1, num_iter=100, step_mode=\"linear\")\n",
        "\n",
        "        Gradient accumulation is supported; example:\n",
        "            >>> train_data = ...    # prepared dataset\n",
        "            >>> desired_bs, real_bs = 32, 4         # batch size\n",
        "            >>> accumulation_steps = desired_bs // real_bs     # required steps for accumulation\n",
        "            >>> dataloader = torch.utils.data.DataLoader(train_data, batch_size=real_bs, shuffle=True)\n",
        "            >>> acc_lr_finder = LRFinder(net, optimizer, criterion, device=\"cuda\")\n",
        "            >>> acc_lr_finder.range_test(dataloader, end_lr=10, num_iter=100, accumulation_steps=accumulation_steps)\n",
        "\n",
        "        If your DataLoader returns e.g. dict, or other non standard output, intehit from TrainDataLoaderIter,\n",
        "        redefine method `inputs_labels_from_batch` so that it outputs (inputs, lables) data:\n",
        "            >>> import torch_lr_finder\n",
        "            >>> class TrainIter(torch_lr_finder.TrainDataLoaderIter):\n",
        "            >>>     def inputs_labels_from_batch(self, batch_data):\n",
        "            >>>         return (batch_data['user_features'], batch_data['user_history']), batch_data['y_labels']\n",
        "            >>> train_data_iter = TrainIter(train_dl)\n",
        "            >>> finder = torch_lr_finder.LRFinder(model, optimizer, partial(model._train_loss, need_one_hot=False))\n",
        "            >>> finder.range_test(train_data_iter, end_lr=10, num_iter=300, diverge_th=10)\n",
        "\n",
        "        Reference:\n",
        "        [Training Neural Nets on Larger Batches: Practical Tips for 1-GPU, Multi-GPU & Distributed setups](\n",
        "        https://medium.com/huggingface/ec88c3e51255)\n",
        "        [thomwolf/gradient_accumulation](https://gist.github.com/thomwolf/ac7a7da6b1888c2eeac8ac8b9b05d3d3)\n",
        "        \"\"\"\n",
        "\n",
        "        # Reset test results\n",
        "        self.history = {\"lr\": [], \"loss\": []}\n",
        "        self.best_loss = None\n",
        "\n",
        "        # Move the model to the proper device\n",
        "        self.model.to(self.device)\n",
        "\n",
        "        # Check if the optimizer is already attached to a scheduler\n",
        "        self._check_for_scheduler()\n",
        "\n",
        "        # Set the starting learning rate\n",
        "        if start_lr:\n",
        "            self._set_learning_rate(start_lr)\n",
        "\n",
        "        # Initialize the proper learning rate policy\n",
        "        if step_mode.lower() == \"exp\":\n",
        "            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)\n",
        "        elif step_mode.lower() == \"linear\":\n",
        "            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)\n",
        "        else:\n",
        "            raise ValueError(\"expected one of (exp, linear), got {}\".format(step_mode))\n",
        "\n",
        "        if smooth_f < 0 or smooth_f >= 1:\n",
        "            raise ValueError(\"smooth_f is outside the range [0, 1[\")\n",
        "\n",
        "        # Create an iterator to get data batch by batch\n",
        "        if isinstance(train_loader, DataLoader):\n",
        "            train_iter = TrainDataLoaderIter(train_loader)\n",
        "        elif isinstance(train_loader, TrainDataLoaderIter):\n",
        "            train_iter = train_loader\n",
        "        else:\n",
        "            raise ValueError(\n",
        "                \"`train_loader` has unsupported type: {}.\"\n",
        "                \"Expected types are `torch.utils.data.DataLoader`\"\n",
        "                \"or child of `TrainDataLoaderIter`.\".format(type(train_loader))\n",
        "            )\n",
        "\n",
        "        if val_loader:\n",
        "            if isinstance(val_loader, DataLoader):\n",
        "                val_iter = ValDataLoaderIter(val_loader)\n",
        "            elif isinstance(val_loader, ValDataLoaderIter):\n",
        "                val_iter = val_loader\n",
        "            else:\n",
        "                raise ValueError(\n",
        "                    \"`val_loader` has unsupported type: {}.\"\n",
        "                    \"Expected types are `torch.utils.data.DataLoader`\"\n",
        "                    \"or child of `ValDataLoaderIter`.\".format(type(val_loader))\n",
        "                )\n",
        "\n",
        "        for iteration in tqdm(range(num_iter)):\n",
        "            # Train on batch and retrieve loss\n",
        "            loss = self._train_batch(\n",
        "                train_iter,\n",
        "                accumulation_steps,\n",
        "                non_blocking_transfer=non_blocking_transfer,\n",
        "            )\n",
        "            if val_loader:\n",
        "                loss = self._validate(\n",
        "                    val_iter, non_blocking_transfer=non_blocking_transfer\n",
        "                )\n",
        "\n",
        "            # Update the learning rate\n",
        "            self.history[\"lr\"].append(lr_schedule.get_lr()[0])\n",
        "            lr_schedule.step()\n",
        "\n",
        "            # Track the best loss and smooth it if smooth_f is specified\n",
        "            if iteration == 0:\n",
        "                self.best_loss = loss\n",
        "            else:\n",
        "                if smooth_f > 0:\n",
        "                    loss = smooth_f * loss + (1 - smooth_f) * self.history[\"loss\"][-1]\n",
        "                if loss < self.best_loss:\n",
        "                    self.best_loss = loss\n",
        "\n",
        "            # Check if the loss has diverged; if it has, stop the test\n",
        "            self.history[\"loss\"].append(loss)\n",
        "            if loss > diverge_th * self.best_loss:\n",
        "                print(\"Stopping early, the loss has diverged\")\n",
        "                break\n",
        "\n",
        "        print(\"Learning rate search finished. See the graph with {finder_name}.plot()\")\n",
        "\n",
        "    def _set_learning_rate(self, new_lrs):\n",
        "        if not isinstance(new_lrs, list):\n",
        "            new_lrs = [new_lrs] * len(self.optimizer.param_groups)\n",
        "        if len(new_lrs) != len(self.optimizer.param_groups):\n",
        "            raise ValueError(\n",
        "                \"Length of `new_lrs` is not equal to the number of parameter groups \"\n",
        "                + \"in the given optimizer\"\n",
        "            )\n",
        "\n",
        "        for param_group, new_lr in zip(self.optimizer.param_groups, new_lrs):\n",
        "            param_group[\"lr\"] = new_lr\n",
        "\n",
        "    def _check_for_scheduler(self):\n",
        "        for param_group in self.optimizer.param_groups:\n",
        "            if \"initial_lr\" in param_group:\n",
        "                raise RuntimeError(\"Optimizer already has a scheduler attached to it\")\n",
        "\n",
        "    def _train_batch(self, train_iter, accumulation_steps, non_blocking_transfer=True):\n",
        "        self.model.train()\n",
        "        total_loss = None  # for late initialization\n",
        "\n",
        "        self.optimizer.zero_grad()\n",
        "        for i in range(accumulation_steps):\n",
        "            inputs, labels = next(train_iter)\n",
        "            inputs, labels = self._move_to_device(\n",
        "                inputs, labels, non_blocking=non_blocking_transfer\n",
        "            )\n",
        "\n",
        "            # Forward pass\n",
        "            outputs = self.model(inputs)\n",
        "            loss = self.criterion(outputs, labels)\n",
        "\n",
        "            # Loss should be averaged in each step\n",
        "            loss /= accumulation_steps\n",
        "\n",
        "            # Backward pass\n",
        "            if IS_AMP_AVAILABLE and hasattr(self.optimizer, \"_amp_stash\"):\n",
        "                # For minor performance optimization, see also:\n",
        "                # https://nvidia.github.io/apex/advanced.html#gradient-accumulation-across-iterations\n",
        "                delay_unscale = ((i + 1) % accumulation_steps) != 0\n",
        "\n",
        "                with amp.scale_loss(\n",
        "                    loss, self.optimizer, delay_unscale=delay_unscale\n",
        "                ) as scaled_loss:\n",
        "                    scaled_loss.backward()\n",
        "            else:\n",
        "                loss.backward()\n",
        "\n",
        "            if total_loss is None:\n",
        "                total_loss = loss\n",
        "            else:\n",
        "                total_loss += loss\n",
        "\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return total_loss.item()\n",
        "\n",
        "    def _move_to_device(self, inputs, labels, non_blocking=True):\n",
        "        def move(obj, device, non_blocking=True):\n",
        "            if hasattr(obj, \"to\"):\n",
        "                return obj.to(device, non_blocking=non_blocking)\n",
        "            elif isinstance(obj, tuple):\n",
        "                return tuple(move(o, device, non_blocking) for o in obj)\n",
        "            elif isinstance(obj, list):\n",
        "                return [move(o, device, non_blocking) for o in obj]\n",
        "            elif isinstance(obj, dict):\n",
        "                return {k: move(o, device, non_blocking) for k, o in obj.items()}\n",
        "            else:\n",
        "                return obj\n",
        "\n",
        "        inputs = move(inputs, self.device, non_blocking=non_blocking)\n",
        "        labels = move(labels, self.device, non_blocking=non_blocking)\n",
        "        return inputs, labels\n",
        "\n",
        "    def _validate(self, val_iter, non_blocking_transfer=True):\n",
        "        # Set model to evaluation mode and disable gradient computation\n",
        "        running_loss = 0\n",
        "        self.model.eval()\n",
        "        with torch.no_grad():\n",
        "            for inputs, labels in val_iter:\n",
        "                # Move data to the correct device\n",
        "                inputs, labels = self._move_to_device(\n",
        "                    inputs, labels, non_blocking=non_blocking_transfer\n",
        "                )\n",
        "\n",
        "                # Forward pass and loss computation\n",
        "                outputs = self.model(inputs)\n",
        "                loss = self.criterion(outputs, labels)\n",
        "                running_loss += loss.item() * len(labels)\n",
        "\n",
        "        return running_loss / len(val_iter.dataset)\n",
        "\n",
        "    def plot(\n",
        "        self,\n",
        "        skip_start=10,\n",
        "        skip_end=5,\n",
        "        log_lr=True,\n",
        "        show_lr=None,\n",
        "        ax=None,\n",
        "        suggest_lr=True,\n",
        "    ):\n",
        "        \"\"\"Plots the learning rate range test.\n",
        "\n",
        "        Arguments:\n",
        "            skip_start (int, optional): number of batches to trim from the start.\n",
        "                Default: 10.\n",
        "            skip_end (int, optional): number of batches to trim from the start.\n",
        "                Default: 5.\n",
        "            log_lr (bool, optional): True to plot the learning rate in a logarithmic\n",
        "                scale; otherwise, plotted in a linear scale. Default: True.\n",
        "            show_lr (float, optional): if set, adds a vertical line to visualize the\n",
        "                specified learning rate. Default: None.\n",
        "            ax (matplotlib.axes.Axes, optional): the plot is created in the specified\n",
        "                matplotlib axes object and the figure is not be shown. If `None`, then\n",
        "                the figure and axes object are created in this method and the figure is\n",
        "                shown . Default: None.\n",
        "            suggest_lr (bool, optional): suggest a learning rate by\n",
        "                - 'steepest': the point with steepest gradient (minimal gradient)\n",
        "                you can use that point as a first guess for an LR. Default: True.\n",
        "\n",
        "        Returns:\n",
        "            The matplotlib.axes.Axes object that contains the plot,\n",
        "            and the suggested learning rate (if set suggest_lr=True).\n",
        "        \"\"\"\n",
        "\n",
        "        if skip_start < 0:\n",
        "            raise ValueError(\"skip_start cannot be negative\")\n",
        "        if skip_end < 0:\n",
        "            raise ValueError(\"skip_end cannot be negative\")\n",
        "        if show_lr is not None and not isinstance(show_lr, float):\n",
        "            raise ValueError(\"show_lr must be float\")\n",
        "\n",
        "        # Get the data to plot from the history dictionary. Also, handle skip_end=0\n",
        "        # properly so the behaviour is the expected\n",
        "        lrs = self.history[\"lr\"]\n",
        "        losses = self.history[\"loss\"]\n",
        "        if skip_end == 0:\n",
        "            lrs = lrs[skip_start:]\n",
        "            losses = losses[skip_start:]\n",
        "        else:\n",
        "            lrs = lrs[skip_start:-skip_end]\n",
        "            losses = losses[skip_start:-skip_end]\n",
        "\n",
        "        # Create the figure and axes object if axes was not already given\n",
        "        fig = None\n",
        "        if ax is None:\n",
        "            fig, ax = plt.subplots()\n",
        "\n",
        "        # Plot loss as a function of the learning rate\n",
        "        ax.plot(lrs, losses)\n",
        "\n",
        "        # Plot the suggested LR\n",
        "        if suggest_lr:\n",
        "            # 'steepest': the point with steepest gradient (minimal gradient)\n",
        "            print(\"LR suggestion: steepest gradient\")\n",
        "            min_grad_idx = None\n",
        "            try:\n",
        "                min_grad_idx = (np.gradient(np.array(losses))).argmin()\n",
        "            except ValueError:\n",
        "                print(\n",
        "                    \"Failed to compute the gradients, there might not be enough points.\"\n",
        "                )\n",
        "            if min_grad_idx is not None:\n",
        "                print(\"Suggested LR: {:.2E}\".format(lrs[min_grad_idx]))\n",
        "                ax.scatter(\n",
        "                    lrs[min_grad_idx],\n",
        "                    losses[min_grad_idx],\n",
        "                    s=75,\n",
        "                    marker=\"o\",\n",
        "                    color=\"red\",\n",
        "                    zorder=3,\n",
        "                    label=\"steepest gradient\",\n",
        "                )\n",
        "                ax.legend()\n",
        "\n",
        "        if log_lr:\n",
        "            ax.set_xscale(\"log\")\n",
        "        ax.set_xlabel(\"Learning rate\")\n",
        "        ax.set_ylabel(\"Loss\")\n",
        "\n",
        "        if show_lr is not None:\n",
        "            ax.axvline(x=show_lr, color=\"red\")\n",
        "\n",
        "        # Show only if the figure was created internally\n",
        "        if fig is not None:\n",
        "            plt.show()\n",
        "\n",
        "        if suggest_lr and min_grad_idx:\n",
        "            return ax, lrs[min_grad_idx]\n",
        "        else:\n",
        "            return ax\n",
        "\n",
        "\n",
        "class LinearLR(_LRScheduler):\n",
        "    \"\"\"Linearly increases the learning rate between two boundaries over a number of\n",
        "    iterations.\n",
        "\n",
        "    Arguments:\n",
        "        optimizer (torch.optim.Optimizer): wrapped optimizer.\n",
        "        end_lr (float): the final learning rate.\n",
        "        num_iter (int): the number of iterations over which the test occurs.\n",
        "        last_epoch (int, optional): the index of last epoch. Default: -1.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n",
        "        self.end_lr = end_lr\n",
        "\n",
        "        if num_iter <= 1:\n",
        "            raise ValueError(\"`num_iter` must be larger than 1\")\n",
        "        self.num_iter = num_iter\n",
        "\n",
        "        super(LinearLR, self).__init__(optimizer, last_epoch)\n",
        "\n",
        "    def get_lr(self):\n",
        "        # In earlier Pytorch versions last_epoch starts at -1, while in recent versions\n",
        "        # it starts at 0. We need to adjust the math a bit to handle this. See\n",
        "        # discussion at: https://github.com/davidtvs/pytorch-lr-finder/pull/42\n",
        "        if PYTORCH_VERSION < version.parse(\"1.1.0\"):\n",
        "            curr_iter = self.last_epoch + 1\n",
        "            r = curr_iter / (self.num_iter - 1)\n",
        "        else:\n",
        "            r = self.last_epoch / (self.num_iter - 1)\n",
        "\n",
        "        return [base_lr + r * (self.end_lr - base_lr) for base_lr in self.base_lrs]\n",
        "\n",
        "\n",
        "class ExponentialLR(_LRScheduler):\n",
        "    \"\"\"Exponentially increases the learning rate between two boundaries over a number of\n",
        "    iterations.\n",
        "\n",
        "    Arguments:\n",
        "        optimizer (torch.optim.Optimizer): wrapped optimizer.\n",
        "        end_lr (float): the final learning rate.\n",
        "        num_iter (int): the number of iterations over which the test occurs.\n",
        "        last_epoch (int, optional): the index of last epoch. Default: -1.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n",
        "        self.end_lr = end_lr\n",
        "\n",
        "        if num_iter <= 1:\n",
        "            raise ValueError(\"`num_iter` must be larger than 1\")\n",
        "        self.num_iter = num_iter\n",
        "\n",
        "        super(ExponentialLR, self).__init__(optimizer, last_epoch)\n",
        "\n",
        "    def get_lr(self):\n",
        "        # In earlier Pytorch versions last_epoch starts at -1, while in recent versions\n",
        "        # it starts at 0. We need to adjust the math a bit to handle this. See\n",
        "        # discussion at: https://github.com/davidtvs/pytorch-lr-finder/pull/42\n",
        "        if PYTORCH_VERSION < version.parse(\"1.1.0\"):\n",
        "            curr_iter = self.last_epoch + 1\n",
        "            r = curr_iter / (self.num_iter - 1)\n",
        "        else:\n",
        "            r = self.last_epoch / (self.num_iter - 1)\n",
        "\n",
        "        return [base_lr * (self.end_lr / base_lr) ** r for base_lr in self.base_lrs]\n",
        "\n",
        "\n",
        "class StateCacher(object):\n",
        "    def __init__(self, in_memory, cache_dir=None):\n",
        "        self.in_memory = in_memory\n",
        "        self.cache_dir = cache_dir\n",
        "\n",
        "        if self.cache_dir is None:\n",
        "            import tempfile\n",
        "\n",
        "            self.cache_dir = tempfile.gettempdir()\n",
        "        else:\n",
        "            if not os.path.isdir(self.cache_dir):\n",
        "                raise ValueError(\"Given `cache_dir` is not a valid directory.\")\n",
        "\n",
        "        self.cached = {}\n",
        "\n",
        "    def store(self, key, state_dict):\n",
        "        if self.in_memory:\n",
        "            self.cached.update({key: copy.deepcopy(state_dict)})\n",
        "        else:\n",
        "            fn = os.path.join(self.cache_dir, \"state_{}_{}.pt\".format(key, id(self)))\n",
        "            self.cached.update({key: fn})\n",
        "            torch.save(state_dict, fn)\n",
        "\n",
        "    def retrieve(self, key):\n",
        "        if key not in self.cached:\n",
        "            raise KeyError(\"Target {} was not cached.\".format(key))\n",
        "\n",
        "        if self.in_memory:\n",
        "            return self.cached.get(key)\n",
        "        else:\n",
        "            fn = self.cached.get(key)\n",
        "            if not os.path.exists(fn):\n",
        "                raise RuntimeError(\n",
        "                    \"Failed to load state in {}. File doesn't exist anymore.\".format(fn)\n",
        "                )\n",
        "            state_dict = torch.load(fn, map_location=lambda storage, location: storage)\n",
        "            return state_dict\n",
        "\n",
        "    def __del__(self):\n",
        "        \"\"\"Check whether there are unused cached files existing in `cache_dir` before\n",
        "        this instance being destroyed.\"\"\"\n",
        "\n",
        "        if self.in_memory:\n",
        "            return\n",
        "\n",
        "        for k in self.cached:\n",
        "            if os.path.exists(self.cached[k]):\n",
        "                os.remove(self.cached[k])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PeQObsgigAzF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}