tutorials/05_FastQuestionAnsweringWithBertQA/BertQAInferentia1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8fea1a0e-0485-46da-91e9-83e525e43d99",
   "metadata": {},
   "source": [
    "# Deploy BertQA to Inferentia1 + SageMaker\n",
    "\n",
    "http://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad\n",
    "\n",
    "**SageMaker Studio Kernel**: Python 3 (PyTorch 1.13 Python 3.9 CPU Optimized)  \n",
    "**Instance**: ml.t3.medium"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f9b837a-0c65-4629-88f4-894fe49261ab",
   "metadata": {},
   "source": [
    "## 1) Update SageMaker SDK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52faabec-a7a9-49d6-90f7-3fa29bc76da3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4e5ccb4-eec0-4465-8a82-6d9f1d137543",
   "metadata": {},
   "source": [
    "## 2) Initialize session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd7e49f7-fa16-4141-8f8a-584736dd805f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import sagemaker\n",
    "\n",
    "print(sagemaker.__version__)\n",
    "if not sagemaker.__version__ >= \"2.196.0\": print(\"You need to upgrade or restart the kernel if you already upgraded\")\n",
    "\n",
    "os.makedirs(\"src\", exist_ok=True)\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "region = sess.boto_region_name\n",
    "bucket = sess.default_bucket()\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {bucket}\")\n",
    "print(f\"sagemaker session region: {region}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04b2ed26-ccb8-4ce2-94c8-7338ebb63311",
   "metadata": {},
   "source": [
    "## 3) Create artifacts to compile & run the model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "298b9c52-74be-47c8-ace4-e753ecd5096f",
   "metadata": {},
   "source": [
    "### 3.1) Dependencies file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2a71b5a-bd53-43ad-9724-32aca9a47bf5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile src/requirements.txt\n",
    "--extra-index-url https://pip.repos.neuron.amazonaws.com\n",
    "torchvision\n",
    "numpy==1.22.2\n",
    "accelerate==0.20.3\n",
    "transformers==4.34.1\n",
    "torch-neuron==1.13.1.2.9.6.0\n",
    "neuron-cc[tensorflow]==1.20.3.0+ed6db4a2e"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ba96cc8-fde4-48ca-8e29-eb9d4a0c113d",
   "metadata": {},
   "source": [
    "### 3.2) Python script for compiling and deploying the model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da469206-98ca-4fbc-af5a-2e7ae9a0b079",
   "metadata": {},
   "source": [
    "This script will download model weights from HF, compile each module to inf1 and save the compiled artifacts to S3\n",
    "\n",
    "The envvar **NEURON_RT_NUM_CORES** controls how many NeuronCores are allocated per process. SageMaker can launch multiple processess in just one Endpoint. It means you can increase throughput of your endpoint by deploying multiple copies of your model to different cores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99aca2e1-b5dc-4ebe-82c8-174de491fbd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile src/inference.py\n",
    "# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.\n",
    "# SPDX-License-Identifier: MIT-0\n",
    "\n",
    "import os\n",
    "# To use one neuron core per worker\n",
    "os.environ[\"NEURON_RT_NUM_CORES\"] = os.environ.get(\"TP_DEGREE\", \"1\")\n",
    "import time\n",
    "import json\n",
    "import torch\n",
    "import shutil\n",
    "import argparse\n",
    "import torch.neuron\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig\n",
    "from transformers import BertForQuestionAnswering, BertTokenizer\n",
    "from filelock import Timeout, FileLock\n",
    "\n",
    "lock_path='/tmp/new_packages.lock'\n",
    "lock = FileLock(lock_path)\n",
    "\n",
    "def model_fn(model_dir):\n",
    "    print(\"Waiting for the lock acquire...\")    \n",
    "    lock.acquire()\n",
    "    # this lock is necessary to load one worker at a time and avoid OOM\n",
    "    t=time.time()\n",
    "    print(\"Loading model...\")\n",
    "    # load tokenizer and neuron model from model_dir\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_dir)\n",
    "    model = torch.jit.load(os.path.join(model_dir, \"model.pt\"))\n",
    "    print(f\"Model loaded. Elapsed: {time.time()-t}s\")\n",
    "    lock.release()\n",
    "    return model, tokenizer\n",
    "\n",
    "def predict_fn(data, model_tokenizer):\n",
    "    # destruct model, tokenizer and model config\n",
    "    model, tokenizer = model_tokenizer\n",
    "\n",
    "    # create embeddings for inputs\n",
    "    # Process the input data (tokenization, input tensors, etc.)\n",
    "    input_question = data.get(\"question\")\n",
    "    input_context = data.get(\"context\")\n",
    "\n",
    "    inputs = tokenizer.encode_plus(input_question, input_context, return_tensors=\"pt\", max_length=384, padding='max_length', truncation=True)\n",
    "    inputs_pr = inputs['input_ids'], inputs['attention_mask'], inputs['token_type_ids']\n",
    "    input_ids = inputs[\"input_ids\"]\n",
    "    attention_mask = inputs[\"attention_mask\"]\n",
    "    # Perform inference with the compiled model\n",
    "    with torch.no_grad():\n",
    "        output = model(*inputs_pr)\n",
    "\n",
    "    # Process the output as needed (e.g., extract answer)\n",
    "    # Ensure you adapt this part to your specific model's output structure\n",
    "    start_scores = output['start_logits']\n",
    "    end_scores = output['end_logits']\n",
    "\n",
    "    start_index = torch.argmax(start_scores)\n",
    "    end_index = torch.argmax(end_scores)\n",
    "\n",
    "    # Convert token indices to answer text\n",
    "    answer_tokens = input_ids[0][start_index:end_index + 1]\n",
    "    answer_text = tokenizer.decode(answer_tokens)\n",
    "\n",
    "    return {\"answer\": answer_text}\n",
    "\n",
    "def input_fn(input_data, content_type, context=None):\n",
    "    if content_type == 'application/json':\n",
    "        return json.loads(input_data)\n",
    "    else:\n",
    "        raise Exception(f\"Unsupported mime type: {content_type}. Supported: application/json\")    \n",
    "\n",
    "def output_fn(prediction, content_type, context=None):\n",
    "    if content_type==\"application/json\":\n",
    "        return json.dumps(prediction)\n",
    "    else:\n",
    "        raise Exception(f\"Invalid content-type: {content_type}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    parser = argparse.ArgumentParser()\n",
    "\n",
    "    # hyperparameters sent by the client are passed as command-line arguments to the script.    \n",
    "    parser.add_argument(\"--model_name\", type=str, default=\"bert-large-uncased-whole-word-masking-finetuned-squad\")\n",
    "    parser.add_argument(\"--input_question\", type=str, default=\"What is the capital of France?\")\n",
    "    parser.add_argument(\"--input_context\", type=str, default=\"Paris is the capital of France.\")    \n",
    "    parser.add_argument(\"--max_len\", type=int, default=384)\n",
    "    \n",
    "    parser.add_argument(\"--model_dir\", type=str, default=os.environ[\"SM_MODEL_DIR\"])    \n",
    "    \n",
    "    args, _ = parser.parse_known_args()\n",
    "    \n",
    "    # Build tokenizer and model\n",
    "    model = BertForQuestionAnswering.from_pretrained(args.model_name)\n",
    "    tokenizer = BertTokenizer.from_pretrained(args.model_name)\n",
    "\n",
    "    # Tokenize and format the input\n",
    "    inputs = tokenizer.encode_plus(\n",
    "        args.input_question,\n",
    "        args.input_context,\n",
    "        return_tensors=\"pt\",\n",
    "        max_length=args.max_len,\n",
    "        padding='max_length',\n",
    "        truncation=True\n",
    "    )\n",
    "\n",
    "    # Convert example inputs to a format that is compatible with TorchScript tracing\n",
    "    example_inputs = inputs['input_ids'], inputs['attention_mask'], inputs['token_type_ids']\n",
    "\n",
    "    y = model(**inputs) # warmup the model\n",
    "    try:\n",
    "        traced_model = torch.jit.trace(model, example_inputs, strict=False)\n",
    "        print(\"Cool! Model is jit traceable\")\n",
    "    except Exception as e:\n",
    "        print(\"Ops. Something went wrong. Model is not traceable\")\n",
    "\n",
    "    neuron_model = torch.neuron.trace(model, example_inputs, strict=False)\n",
    "    neuron_model.save(os.path.join(args.model_dir, \"model.pt\"))\n",
    "    tokenizer.save_pretrained(args.model_dir)\n",
    "    \n",
    "    code_path=os.path.join(args.model_dir, \"code\")\n",
    "    os.makedirs(code_path, exist_ok=True)\n",
    "    shutil.copy(\"inference.py\", os.path.join(code_path, \"inference.py\"))\n",
    "    shutil.copy(\"requirements.txt\", os.path.join(code_path, \"requirements.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "413a9603-fb4b-481e-a911-65a7f2cd2d96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import logging\n",
    "from sagemaker.pytorch import PyTorch\n",
    "\n",
    "estimator = PyTorch(\n",
    "    entry_point=\"inference.py\", # Specify your train script\n",
    "    source_dir=\"src\",\n",
    "    role=role,\n",
    "    sagemaker_session=sess,\n",
    "    container_log_level=logging.DEBUG,\n",
    "    instance_count=1,\n",
    "    instance_type='ml.c5.9xlarge',   \n",
    "    disable_profiler=True,\n",
    "    env={\n",
    "        \"TP_DEGREE\": \"1\"\n",
    "    },\n",
    "    image_uri=f\"763104351884.dkr.ecr.{region}.amazonaws.com/pytorch-training:1.13.1-cpu-py39-ubuntu20.04-sagemaker\",\n",
    "    \n",
    "    volume_size = 30,\n",
    "    hyperparameters={\n",
    "        \"max_len\": 384\n",
    "    }\n",
    ")\n",
    "estimator.framework_version = '1.13.1' # workround when using image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0def49f4-3198-4782-9961-138f72c0900d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "estimator.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db234d91-e8a8-42d4-8936-65ea34d64863",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\"\"\n",
    "If you decide to run this notebook again, you don't need to re-compile the model.\n",
    "Just keep the following path and use it to deploy the model next time.\n",
    "\"\"\")\n",
    "print(estimator.model_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2c938af-cedf-4491-bc1e-f436101e771d",
   "metadata": {},
   "source": [
    "## 5) Deploy the compiled model to a SageMaker endpoint on inf1\n",
    "SageMaker can launch multiple workers, depending on the size of the Inf1 instance. A worker is a standalone Python process that manages one copy of the model. SageMaker puts a load balancer on top of all these processes and distributes the load automatically for your clients. It means that you can increase throughput by launching multiple workers, which serve different clients in parallel.\n",
    "\n",
    "For instance. If you deploy the model to a **ml.inf1.xlarge**, SageMaker can launch 4 workers with 4 copies of the model. This instance has 4 cores and each copy of the model utilizes 1 core. Then, you can have 4 simultaneous clients invoking the endpoint and being served at the same time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29611b6b-0935-4811-97c6-92eaea69fab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from sagemaker.utils import name_from_base\n",
    "from sagemaker.pytorch.model import PyTorchModel\n",
    "\n",
    "# depending on the inf1 instance you deploy the model you'll have more or less accelerators\n",
    "# we'll ask SageMaker to launch 1 worker per core\n",
    "\n",
    "num_workers=4\n",
    "pytorch_model = PyTorchModel(    \n",
    "    image_uri=f\"763104351884.dkr.ecr.{region}.amazonaws.com/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.13.2-ubuntu20.04\",\n",
    "    model_data=estimator.model_data,\n",
    "    role=role,\n",
    "    name=name_from_base('bert-large-qa'),\n",
    "    sagemaker_session=sess,\n",
    "    container_log_level=logging.DEBUG,\n",
    "    model_server_workers=num_workers, # 1 worker per core\n",
    "    framework_version=\"1.13.1\",\n",
    "    env = {\n",
    "        'TP_DEGREE': '1',\n",
    "        'SAGEMAKER_MODEL_SERVER_TIMEOUT' : '3600' \n",
    "    }\n",
    "    # for production it is important to define vpc_config and use a vpc_endpoint\n",
    "    #vpc_config={\n",
    "    #    'Subnets': ['<SUBNET1>', '<SUBNET2>'],\n",
    "    #    'SecurityGroupIds': ['<SECURITYGROUP1>', '<DEFAULTSECURITYGROUP>']\n",
    "    #}\n",
    ")\n",
    "pytorch_model._is_compiled_model = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75bdff9c-163a-46bd-9567-979ac27175d1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "predictor = pytorch_model.deploy(\n",
    "    initial_instance_count=1,\n",
    "    instance_type='ml.inf1.xlarge'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05d9a055-8708-4c9c-892b-cb20d00b1f28",
   "metadata": {},
   "source": [
    "## 6) Run a simple test to check the endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35096e35-9a33-4d8e-9658-6ae73df9d809",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.serializers import JSONSerializer\n",
    "from sagemaker.deserializers import JSONDeserializer\n",
    "predictor.serializer = JSONSerializer()\n",
    "predictor.deserializer = JSONDeserializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dbc72bf9-9b02-414d-b959-3c7b8bbefe3b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer': 'paris'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.predict({\"question\": \"What is the capital of France?\", \"context\": \"Paris is the capital of France.\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "be3cc277-cc7d-4b19-aa44-52ec77544818",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total elapsed time for 4 workers: 0.08790946006774902\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from multiprocessing.pool import ThreadPool\n",
    "\n",
    "q = {\"question\": \"What is the capital of France?\", \"context\": \"Paris is the capital of France.\"}\n",
    "\n",
    "with ThreadPool(num_workers) as p:\n",
    "    t=time.time()\n",
    "    resp = p.map(predictor.predict, [q] * num_workers)\n",
    "    elapsed=time.time()-t\n",
    "    print(f\"Total elapsed time for {num_workers} workers: {elapsed}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18d44c10-9033-4ce8-a08c-370a18b54fbc",
   "metadata": {},
   "source": [
    "## 7) Cleanup\n",
    "Delete the endpoint to stop paying for the provisioned resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d55aea1-1de2-4606-8d8c-c867d701a9ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "predictor.delete_endpoint()"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}