challenge_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 1: landcover classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve S2 embeddings from an S3 bucket over California"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "import s3fs\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "file_path = 'models/land_cover_model.h5'\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "    # S3 bucket and prefix\n",
    "    s3_bucket = 'clay-worker-bucket-dev-small-tasks'\n",
    "    s3_prefix = '_data/gpq/87/'#51/' #2022 S2 Cali embeddings\n",
    "\n",
    "    # Initialize s3fs filesystem\n",
    "    fs = s3fs.S3FileSystem()\n",
    "\n",
    "    # List all the GeoParquet files in the specified S3 directory\n",
    "    geo_parquet_files = fs.glob(f's3://{s3_bucket}/{s3_prefix}*.gpq')\n",
    "\n",
    "    # List to store GeoDataFrames\n",
    "    gdfs = []\n",
    "\n",
    "    # Load each GeoParquet file into a GeoDataFrame and append to the list\n",
    "    for file in tqdm(geo_parquet_files):\n",
    "        gdf = gpd.read_parquet(\"s3://\"+file, storage_options={\"anon\": False, \"client_kwargs\": {\"region_name\": \"us-west-2\"}})\n",
    "        gdfs.append(gdf)\n",
    "\n",
    "    # Concatenate all GeoDataFrames into a single GeoDataFrame\n",
    "    combined_gdf = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True))\n",
    "\n",
    "    # Display the concatenated GeoDataFrame\n",
    "    combined_gdf\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join most common land use value that corresponds to each chip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import ee\n",
    "\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "    # Initialize Earth Engine\n",
    "    ee.Initialize()\n",
    "\n",
    "    # Load the ESRI Global LULC dataset\n",
    "    lulc = ee.ImageCollection(\"projects/sat-io/open-datasets/landcover/ESRI_Global-LULC_10m_TS\").mosaic()\n",
    "\n",
    "    def get_most_common_lulc(geometry):\n",
    "        # Convert the GeoPandas geometry to an Earth Engine geometry\n",
    "        ee_geometry = ee.Geometry.Rectangle(geometry.bounds)\n",
    "        \n",
    "        # Get the LULC values within the bounding box\n",
    "        lulc_values = lulc.reduceRegion(\n",
    "            reducer=ee.Reducer.frequencyHistogram(),\n",
    "            geometry=ee_geometry,\n",
    "            scale=10,\n",
    "            maxPixels=1e9\n",
    "        ).get('b1')\n",
    "        \n",
    "        # Find the most common LULC value\n",
    "        lulc_dict = ee.Dictionary(lulc_values)\n",
    "        most_common = lulc_dict.keys().sort(lulc_dict.values()).get(-1)\n",
    "        \n",
    "        # Return the result\n",
    "        return int(most_common.getInfo())\n",
    "\n",
    "    def process_geometries(combined_gdf):\n",
    "        # Get the total number of rows in the GeoDataFrame\n",
    "        total_rows = len(combined_gdf)\n",
    "\n",
    "        # Determine the number of threads to use\n",
    "        max_threads = 10  # Adjust this based on your system and Earth Engine quota\n",
    "        num_threads = min(total_rows, max_threads)\n",
    "\n",
    "        results = []\n",
    "\n",
    "        with ThreadPoolExecutor(max_workers=num_threads) as executor:\n",
    "            # Submit all tasks\n",
    "            future_to_index = {executor.submit(get_most_common_lulc, row.geometry): index \n",
    "                            for index, row in combined_gdf.iterrows()}\n",
    "            \n",
    "            # Process as they complete with a progress bar\n",
    "            with tqdm(total=total_rows, desc=\"Processing bounding boxes\") as pbar:\n",
    "                for future in as_completed(future_to_index):\n",
    "                    index = future_to_index[future]\n",
    "                    try:\n",
    "                        result = future.result()\n",
    "                    except Exception as exc:\n",
    "                        print(f'Generated an exception: {exc}')\n",
    "                        result = None\n",
    "                    results.append((index, result))\n",
    "                    pbar.update(1)\n",
    "\n",
    "        # Sort results by index and extract only the values\n",
    "        sorted_results = [r[1] for r in sorted(results, key=lambda x: x[0])]\n",
    "        \n",
    "        return sorted_results\n",
    "\n",
    "    if __name__ == '__main__':\n",
    "        # Process the geometries\n",
    "        results = process_geometries(combined_gdf)\n",
    "\n",
    "        # Add the results as a new column to the GeoDataFrame\n",
    "        combined_gdf['most_common_lulc'] = results\n",
    "\n",
    "        # Save as GeoJSON\n",
    "        combined_gdf[['geometry', 'most_common_lulc']].to_file(\"test_lulc.geojson\", driver=\"GeoJSON\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tune hyperparameters for land use classifier using Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/claymodel-latest-v2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-08-06 22:51:35.185053: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-08-06 22:51:35.410486: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-08-06 22:51:35.485016: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-08-06 22:51:35.980377: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-06 22:51:37.744057: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import optuna\n",
    "import tensorflow as tf\n",
    "import joblib\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn.metrics import f1_score\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Input\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "class NeuralNetwork:\n",
    "    def __init__(self, input_shape, layers, dropout_rate, learning_rate, num_classes, device):\n",
    "        self.input_shape = input_shape\n",
    "        self.layers = layers\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.learning_rate = learning_rate\n",
    "        self.num_classes = num_classes\n",
    "        self.device = device\n",
    "        self.model = self.build_model()\n",
    "\n",
    "    def build_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Input(shape=(self.input_shape,)))\n",
    "        for layer_size in self.layers:\n",
    "            model.add(Dense(layer_size, activation='relu'))\n",
    "            model.add(Dropout(self.dropout_rate))\n",
    "        model.add(Dense(self.num_classes, activation='softmax'))\n",
    "        return model\n",
    "\n",
    "    def compile_model(self):\n",
    "        optimizer = Adam(learning_rate=self.learning_rate)\n",
    "        self.model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    def train_model(self, X_train, y_train, epochs=20, batch_size=32, validation_split=0.2):\n",
    "        with tf.device(self.device):\n",
    "            history = self.model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=validation_split)\n",
    "        return history\n",
    "\n",
    "    def evaluate_model(self, X_test, y_test):\n",
    "        with tf.device(self.device):\n",
    "            loss, accuracy = self.model.evaluate(X_test, y_test)\n",
    "        return loss, accuracy\n",
    "\n",
    "    def predict(self, X_test):\n",
    "        with tf.device(self.device):\n",
    "            predictions = self.model.predict(X_test)\n",
    "        return np.argmax(predictions, axis=1)\n",
    "\n",
    "    def calculate_f1(self, y_test, predictions):\n",
    "        return f1_score(np.argmax(y_test, axis=1), predictions, average='weighted')\n",
    "    \n",
    "    def save_model(self, filename):\n",
    "        self.model.save(filename)\n",
    "\n",
    "    @classmethod\n",
    "    def load_model(cls, filename, input_shape, num_classes, device):\n",
    "        loaded_model = tf.keras.models.load_model(filename)\n",
    "        nn = cls(input_shape, [], 0, 0, num_classes, device)  # Dummy values for layers, dropout_rate, and learning_rate\n",
    "        nn.model = loaded_model\n",
    "        return nn\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "    layers = []\n",
    "    for i in range(trial.suggest_int('n_layers', 1, 3)):\n",
    "        layers.append(trial.suggest_int(f'n_units_l{i}', 64, 512))\n",
    "    \n",
    "    dropout_rate = trial.suggest_float('dropout_rate', 0.2, 0.5)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-2)\n",
    "    \n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate, num_classes=num_classes)\n",
    "    nn.compile_model()\n",
    "    \n",
    "    nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    \n",
    "    predictions = nn.predict(X_test)\n",
    "    f1 = nn.calculate_f1(y_test, predictions)\n",
    "    \n",
    "    return f1\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "    # Assuming combined_gdf is already loaded\n",
    "    X = combined_gdf['embeddings'].tolist()\n",
    "    X = pd.DataFrame(X)\n",
    "    y = combined_gdf['most_common_lulc']\n",
    "\n",
    "    # Encode the target variable\n",
    "    label_encoder = LabelEncoder()\n",
    "    y_encoded = label_encoder.fit_transform(y)\n",
    "    num_classes = len(label_encoder.classes_)\n",
    "\n",
    "    # Convert to one-hot encoding\n",
    "    y_onehot = to_categorical(y_encoded)\n",
    "\n",
    "    # Split data into training and testing sets\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_onehot, test_size=0.2, random_state=42)\n",
    "\n",
    "    # Scale the features\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n",
    "\n",
    "    # Optimize the hyperparameters\n",
    "    study = optuna.create_study(direction='maximize')\n",
    "    study.optimize(objective, n_trials=20)\n",
    "\n",
    "    # Print the best hyperparameters\n",
    "    print(study.best_params)\n",
    "\n",
    "    # Example usage with the best hyperparameters\n",
    "    best_params = study.best_params\n",
    "    layers = [best_params[f'n_units_l{i}'] for i in range(best_params['n_layers'])]\n",
    "    dropout_rate = best_params['dropout_rate']\n",
    "    learning_rate = best_params['learning_rate']\n",
    "\n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate, num_classes=num_classes)\n",
    "    nn.compile_model()\n",
    "    history = nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    loss, accuracy = nn.evaluate_model(X_test, y_test)\n",
    "    print(f'Test Accuracy: {accuracy:.4f}')\n",
    "    predictions = nn.predict(X_test)\n",
    "    f1 = nn.calculate_f1(y_test, predictions)\n",
    "    print(f'Test F1 Score: {f1:.4f}')\n",
    "\n",
    "    # Save the model\n",
    "    nn.save_model('models/land_cover_model.h5')\n",
    "\n",
    "    # Save scaler and label_encoder\n",
    "    joblib.dump(scaler, 'models/scaler.joblib')\n",
    "    joblib.dump(label_encoder, 'models/label_encoder.joblib')\n",
    "\n",
    "    # If you want to get the actual class labels\n",
    "    predicted_classes = label_encoder.inverse_transform(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on test set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load in test set geojson and turn into grid of points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POLYGON ((-106.08092 35.78627, -106.08092 35.4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            geometry\n",
       "0  POLYGON ((-106.08092 35.78627, -106.08092 35.4..."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "\n",
    "year = 2022\n",
    "\n",
    "# Load the GeoJSON file\n",
    "geojson_path = 'test_data/challenge_1_bb.geojson'\n",
    "gdf = gpd.read_file(geojson_path)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POLYGON ((402315.263 3960781.699, 401878.759 3...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            geometry\n",
       "0  POLYGON ((402315.263 3960781.699, 401878.759 3..."
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyproj\n",
    "\n",
    "def get_utm_zone(longitude):\n",
    "    return int((longitude + 180) / 6) + 1\n",
    "\n",
    "# Get the bounds of the geometry\n",
    "minx, miny, maxx, maxy = gdf.geometry.bounds.iloc[0]\n",
    "\n",
    "# Calculate UTM zone\n",
    "utm_zone = get_utm_zone(minx)\n",
    "\n",
    "# Check for a suitable projection using pyproj\n",
    "proj = pyproj.Proj(proj='utm', zone=utm_zone, ellps='WGS84')\n",
    "\n",
    "# Get the corresponding EPSG code for the UTM zone using pyproj\n",
    "utm_crs = pyproj.CRS(f\"+proj=utm +zone={utm_zone} +datum=WGS84\")\n",
    "epsg_code = utm_crs.to_epsg()\n",
    "\n",
    "# Reproject the GeoDataFrame to the chosen EPSG code\n",
    "gdf = gdf.to_crs(epsg=epsg_code)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POINT (401878.759 3920624.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>POINT (404438.759 3920624.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>POINT (406998.759 3920624.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>POINT (409558.759 3920624.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>POINT (412118.759 3920624.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>POINT (442838.759 3959024.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>POINT (445398.759 3959024.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>POINT (447958.759 3959024.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>POINT (450518.759 3959024.607)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>POINT (453078.759 3959024.607)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>336 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           geometry\n",
       "0    POINT (401878.759 3920624.607)\n",
       "1    POINT (404438.759 3920624.607)\n",
       "2    POINT (406998.759 3920624.607)\n",
       "3    POINT (409558.759 3920624.607)\n",
       "4    POINT (412118.759 3920624.607)\n",
       "..                              ...\n",
       "331  POINT (442838.759 3959024.607)\n",
       "332  POINT (445398.759 3959024.607)\n",
       "333  POINT (447958.759 3959024.607)\n",
       "334  POINT (450518.759 3959024.607)\n",
       "335  POINT (453078.759 3959024.607)\n",
       "\n",
       "[336 rows x 1 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Create a grid of points 5120m apart\n",
    "x = np.arange(gdf.total_bounds[0], gdf.total_bounds[2], 2560)\n",
    "y = np.arange(gdf.total_bounds[1], gdf.total_bounds[3], 2560)\n",
    "xx, yy = np.meshgrid(x, y)\n",
    "points = np.vstack([xx.ravel(), yy.ravel()]).T\n",
    "\n",
    "grid = gpd.GeoDataFrame(geometry=gpd.points_from_xy(points[:, 0], points[:, 1], crs=gdf.crs))\n",
    "grid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download S2 data for the grid defined above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stackstac\n",
    "# import rasterio\n",
    "# import pystac_client\n",
    "# import rioxarray\n",
    "\n",
    "# import warnings\n",
    "# warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# from shapely.geometry import Point\n",
    "\n",
    "# def download_stack_images(start_date, end_date, grid, output_directory):\n",
    "#     os.makedirs(output_directory, exist_ok=True)\n",
    "    \n",
    "#     # Band groups updated with new bands\n",
    "#     BAND_GROUPS = {\n",
    "#         \"rgb\": [\"red\", \"green\", \"blue\"],\n",
    "#         \"rededge\": [\"rededge1\", \"rededge2\", \"rededge3\", \"nir08\"],\n",
    "#         \"nir\": [\"nir\"],\n",
    "#         \"swir\": [\"swir16\", \"swir22\"],\n",
    "#         \"sar\": [\"vv\", \"vh\"],\n",
    "#     }\n",
    "\n",
    "#     # STAC API and Collection details\n",
    "#     STAC_API = \"https://earth-search.aws.element84.com/v1\"\n",
    "#     COLLECTION = \"sentinel-2-l2a\"\n",
    "\n",
    "#     # Initialize STAC client\n",
    "#     catalog = pystac_client.Client.open(STAC_API)\n",
    "\n",
    "#     # Points of Interest as a list of tuples (longitude, latitude)\n",
    "#     points = grid.to_crs(\"EPSG:4326\").geometry.apply(lambda x: (x.x, x.y)).tolist()\n",
    "\n",
    "#     # Iterate over each point of interest\n",
    "#     cnt = 0\n",
    "#     indices_for_join = []\n",
    "#     for lon, lat in tqdm(points):\n",
    "#         bbox = [lon - 1e-5, lat - 1e-5, lon + 1e-5, lat + 1e-5]\n",
    "\n",
    "#         # Search for items with low cloud cover, without sorting by cloud cover\n",
    "#         search = catalog.search(\n",
    "#             collections=[COLLECTION],\n",
    "#             datetime=f\"{start_date}/{end_date}\",\n",
    "#             bbox=bbox,\n",
    "#             max_items=10,\n",
    "#             query={\"eo:cloud_cover\": {\"lt\": 20}}  # Assuming 'eo:cloud_cover' is valid for filtering\n",
    "#         )\n",
    "\n",
    "#         items = list(search.get_items())\n",
    "#         if not items:\n",
    "#             print(\"No items found with low cloud cover.\")\n",
    "#             continue\n",
    "\n",
    "#         # Optionally, sort items manually by cloud cover if it's available in the properties\n",
    "#         items = sorted(items, key=lambda x: x.properties.get('eo:cloud_cover', float('inf')))\n",
    "\n",
    "#         # Select the item with the lowest cloud cover\n",
    "#         lowest_cloud_item = items[0]\n",
    "\n",
    "#         # Convert point into the image projection of the selected item\n",
    "#         epsg = lowest_cloud_item.properties[\"proj:epsg\"]\n",
    "#         point_gdf = gpd.GeoDataFrame(\n",
    "#             [{'geometry': Point(lon, lat)}], \n",
    "#             crs='EPSG:4326'\n",
    "#         ).to_crs(epsg=epsg)\n",
    "#         coords = list(point_gdf.iloc[0].geometry.coords)[0]\n",
    "\n",
    "#         bounds = (\n",
    "#             coords[0] - 1280, coords[1] - 1280,\n",
    "#             coords[0] + 1280, coords[1] + 1280,\n",
    "#         )\n",
    "\n",
    "#         # Process and visualize the selected image\n",
    "#         stack = stackstac.stack(\n",
    "#             [lowest_cloud_item],\n",
    "#             bounds=bounds,\n",
    "#             snap_bounds=False,\n",
    "#             epsg=epsg,\n",
    "#             resolution=10,\n",
    "#             dtype=\"float32\",\n",
    "#             rescale=False,\n",
    "#             fill_value=np.nan,\n",
    "#             assets=BAND_GROUPS[\"rgb\"] + BAND_GROUPS[\"nir\"],  # TODO: Add more/all of the Sentinel 2 bands to see if embeddings improve\n",
    "#         ).compute()\n",
    "\n",
    "#         # Assuming 'stack' is your xarray DataArray loaded with stackstac\n",
    "#         stack = stack.rio.write_crs(\"EPSG:4326\")  # You can change to your specific EPSG code\n",
    "\n",
    "#         mgrs = str(stack.coords[\"grid:code\"].values).split(\"-\")[1]\n",
    "#         date = str(stack.time.values)[2:11]\n",
    "\n",
    "#         output_path = os.path.join(output_directory, f\"stack_{lon}_{lat}_{cnt}.tif\")\n",
    "#         indices_for_join.append(cnt)\n",
    "\n",
    "#         if 'time' in stack.dims:\n",
    "#             stack = stack.isel(time=0)\n",
    "\n",
    "#         # Write the stack to a TIFF file\n",
    "#         with rasterio.open(\n",
    "#                 output_path, 'w',\n",
    "#                 driver='GTiff',\n",
    "#                 height=stack.shape[1],\n",
    "#                 width=stack.shape[2],\n",
    "#                 count=len(stack.band),  # Number of bands\n",
    "#                 dtype=str(stack.dtype),\n",
    "#                 crs=epsg, # TODO: Confirm this\n",
    "#                 transform=stack.rio.transform()\n",
    "#             ) as tif:\n",
    "#             for i, band in enumerate(stack.band, start=1):\n",
    "#                 tif.write(stack.sel(band=band).values, i)\n",
    "\n",
    "#         # Reopen the file to add metadata\n",
    "#         with rasterio.open(output_path, \"r+\") as rst:\n",
    "#             rst.update_tags(date=date)\n",
    "\n",
    "#         cnt += 1\n",
    "\n",
    "#     grid[\"join_index\"] = indices_for_join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download_stack_images(f\"{year}-01-01\", f\"{year}-12-31\", grid, \"test_data/embeddings/challenge_1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Join LULC data on grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing bounding boxes: 100%|██████████| 336/336 [00:05<00:00, 66.64it/s]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Earth Engine\n",
    "ee.Initialize()\n",
    "\n",
    "# Load the ESRI Global LULC dataset\n",
    "lulc = ee.ImageCollection(\"projects/sat-io/open-datasets/landcover/ESRI_Global-LULC_10m_TS\").mosaic()\n",
    "\n",
    "def get_most_common_lulc(geometry):\n",
    "    # Convert the GeoPandas geometry to an Earth Engine geometry\n",
    "    ee_geometry = ee.Geometry.Rectangle(geometry.bounds)\n",
    "    \n",
    "    # Get the LULC values within the bounding box\n",
    "    lulc_values = lulc.reduceRegion(\n",
    "        reducer=ee.Reducer.frequencyHistogram(),\n",
    "        geometry=ee_geometry,\n",
    "        scale=10,\n",
    "        maxPixels=1e9\n",
    "    ).get('b1')\n",
    "    \n",
    "    # Find the most common LULC value\n",
    "    lulc_dict = ee.Dictionary(lulc_values)\n",
    "    most_common = lulc_dict.keys().sort(lulc_dict.values()).get(-1)\n",
    "    \n",
    "    # Return the result\n",
    "    return int(most_common.getInfo())\n",
    "\n",
    "def process_geometries(combined_gdf):\n",
    "    # Get the total number of rows in the GeoDataFrame\n",
    "    total_rows = len(combined_gdf)\n",
    "\n",
    "    # Determine the number of threads to use\n",
    "    max_threads = 10  # Adjust this based on your system and Earth Engine quota\n",
    "    num_threads = min(total_rows, max_threads)\n",
    "\n",
    "    results = []\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_threads) as executor:\n",
    "        # Submit all tasks\n",
    "        future_to_index = {executor.submit(get_most_common_lulc, row.geometry): index \n",
    "                        for index, row in combined_gdf.iterrows()}\n",
    "        \n",
    "        # Process as they complete with a progress bar\n",
    "        with tqdm(total=total_rows, desc=\"Processing bounding boxes\") as pbar:\n",
    "            for future in as_completed(future_to_index):\n",
    "                index = future_to_index[future]\n",
    "                try:\n",
    "                    result = future.result()\n",
    "                except Exception as exc:\n",
    "                    print(f'Generated an exception: {exc}')\n",
    "                    result = None\n",
    "                results.append((index, result))\n",
    "                pbar.update(1)\n",
    "\n",
    "    # Sort results by index and extract only the values\n",
    "    sorted_results = [r[1] for r in sorted(results, key=lambda x: x[0])]\n",
    "    \n",
    "    return sorted_results\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # Process the geometries\n",
    "    results = process_geometries(grid.to_crs(\"EPSG:4326\"))\n",
    "\n",
    "    # Add the results as a new column to the GeoDataFrame\n",
    "    grid['most_common_lulc'] = results\n",
    "\n",
    "    # Save as GeoJSON\n",
    "    grid[['geometry', 'most_common_lulc']].to_file(\"test_lulc_v2.geojson\", driver=\"GeoJSON\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate embeddings for the images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POLYGON ((-123.28134 47.75348, -123.28134 46.2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            geometry\n",
       "0  POLYGON ((-123.28134 47.75348, -123.28134 46.2..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "\n",
    "year = 2022\n",
    "\n",
    "# Load the GeoJSON file\n",
    "geojson_path = 'train_data/challenge_1_bb_TRAIN.geojson'\n",
    "gdf = gpd.read_file(geojson_path)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POLYGON ((478913.685 5288938.981, 478311.614 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            geometry\n",
       "0  POLYGON ((478913.685 5288938.981, 478311.614 5..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyproj\n",
    "\n",
    "def get_utm_zone(longitude):\n",
    "    return int((longitude + 180) / 6) + 1\n",
    "\n",
    "# Get the bounds of the geometry\n",
    "minx, miny, maxx, maxy = gdf.geometry.bounds.iloc[0]\n",
    "\n",
    "# Calculate UTM zone\n",
    "utm_zone = get_utm_zone(minx)\n",
    "\n",
    "# Check for a suitable projection using pyproj\n",
    "proj = pyproj.Proj(proj='utm', zone=utm_zone, ellps='WGS84')\n",
    "\n",
    "# Get the corresponding EPSG code for the UTM zone using pyproj\n",
    "utm_crs = pyproj.CRS(f\"+proj=utm +zone={utm_zone} +datum=WGS84\")\n",
    "epsg_code = utm_crs.to_epsg()\n",
    "\n",
    "# Reproject the GeoDataFrame to the chosen EPSG code\n",
    "gdf = gdf.to_crs(epsg=epsg_code)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>POINT (478311.614 5121278.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>POINT (480871.614 5121278.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>POINT (483431.614 5121278.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>POINT (485991.614 5121278.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>POINT (488551.614 5121278.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6659</th>\n",
       "      <td>POINT (716391.614 5292798.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6660</th>\n",
       "      <td>POINT (718951.614 5292798.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6661</th>\n",
       "      <td>POINT (721511.614 5292798.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6662</th>\n",
       "      <td>POINT (724071.614 5292798.860)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6663</th>\n",
       "      <td>POINT (726631.614 5292798.860)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6664 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            geometry\n",
       "0     POINT (478311.614 5121278.860)\n",
       "1     POINT (480871.614 5121278.860)\n",
       "2     POINT (483431.614 5121278.860)\n",
       "3     POINT (485991.614 5121278.860)\n",
       "4     POINT (488551.614 5121278.860)\n",
       "...                              ...\n",
       "6659  POINT (716391.614 5292798.860)\n",
       "6660  POINT (718951.614 5292798.860)\n",
       "6661  POINT (721511.614 5292798.860)\n",
       "6662  POINT (724071.614 5292798.860)\n",
       "6663  POINT (726631.614 5292798.860)\n",
       "\n",
       "[6664 rows x 1 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Create a grid of points 5120m apart\n",
    "x = np.arange(gdf.total_bounds[0], gdf.total_bounds[2], 2560)\n",
    "y = np.arange(gdf.total_bounds[1], gdf.total_bounds[3], 2560)\n",
    "xx, yy = np.meshgrid(x, y)\n",
    "points = np.vstack([xx.ravel(), yy.ravel()]).T\n",
    "\n",
    "grid = gpd.GeoDataFrame(geometry=gpd.points_from_xy(points[:, 0], points[:, 1], crs=gdf.crs))\n",
    "grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing bounding boxes: 100%|██████████| 6664/6664 [00:54<00:00, 123.33it/s]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Earth Engine\n",
    "ee.Initialize()\n",
    "\n",
    "# Load the ESRI Global LULC dataset\n",
    "lulc = ee.ImageCollection(\"projects/sat-io/open-datasets/landcover/ESRI_Global-LULC_10m_TS\").mosaic()\n",
    "\n",
    "def get_most_common_lulc(geometry):\n",
    "    # Convert the GeoPandas geometry to an Earth Engine geometry\n",
    "    ee_geometry = ee.Geometry.Rectangle(geometry.bounds)\n",
    "    \n",
    "    # Get the LULC values within the bounding box\n",
    "    lulc_values = lulc.reduceRegion(\n",
    "        reducer=ee.Reducer.frequencyHistogram(),\n",
    "        geometry=ee_geometry,\n",
    "        scale=10,\n",
    "        maxPixels=1e9\n",
    "    ).get('b1')\n",
    "    \n",
    "    # Find the most common LULC value\n",
    "    lulc_dict = ee.Dictionary(lulc_values)\n",
    "    most_common = lulc_dict.keys().sort(lulc_dict.values()).get(-1)\n",
    "    \n",
    "    # Return the result\n",
    "    return int(most_common.getInfo())\n",
    "\n",
    "def process_geometries(combined_gdf):\n",
    "    # Get the total number of rows in the GeoDataFrame\n",
    "    total_rows = len(combined_gdf)\n",
    "\n",
    "    # Determine the number of threads to use\n",
    "    max_threads = 10  # Adjust this based on your system and Earth Engine quota\n",
    "    num_threads = min(total_rows, max_threads)\n",
    "\n",
    "    results = []\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_threads) as executor:\n",
    "        # Submit all tasks\n",
    "        future_to_index = {executor.submit(get_most_common_lulc, row.geometry): index \n",
    "                        for index, row in combined_gdf.iterrows()}\n",
    "        \n",
    "        # Process as they complete with a progress bar\n",
    "        with tqdm(total=total_rows, desc=\"Processing bounding boxes\") as pbar:\n",
    "            for future in as_completed(future_to_index):\n",
    "                index = future_to_index[future]\n",
    "                try:\n",
    "                    result = future.result()\n",
    "                except Exception as exc:\n",
    "                    print(f'Generated an exception: {exc}')\n",
    "                    result = None\n",
    "                results.append((index, result))\n",
    "                pbar.update(1)\n",
    "\n",
    "    # Sort results by index and extract only the values\n",
    "    sorted_results = [r[1] for r in sorted(results, key=lambda x: x[0])]\n",
    "    \n",
    "    return sorted_results\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # Process the geometries\n",
    "    results = process_geometries(grid.to_crs(\"EPSG:4326\"))\n",
    "\n",
    "    # Add the results as a new column to the GeoDataFrame\n",
    "    grid['most_common_lulc'] = results\n",
    "\n",
    "    # Save as GeoJSON\n",
    "    grid[['geometry', 'most_common_lulc']].to_file(\"TRAIN_lulc.geojson\", driver=\"GeoJSON\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6664/6664 [1:29:34<00:00,  1.24it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>most_common_lulc</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-123.281338</td>\n",
       "      <td>46.244740</td>\n",
       "      <td>[[0.0954321, -0.0801714, -0.029764883, -0.0657...</td>\n",
       "      <td>2</td>\n",
       "      <td>POINT (-123.28134 46.24474)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-123.248130</td>\n",
       "      <td>46.244817</td>\n",
       "      <td>[[0.07401827, -0.089014255, -0.008051335, -0.0...</td>\n",
       "      <td>2</td>\n",
       "      <td>POINT (-123.24813 46.24482)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-123.214923</td>\n",
       "      <td>46.244884</td>\n",
       "      <td>[[0.053021867, -0.047577165, -0.024503708, -0....</td>\n",
       "      <td>2</td>\n",
       "      <td>POINT (-123.21492 46.24488)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-123.181715</td>\n",
       "      <td>46.244942</td>\n",
       "      <td>[[0.026906254, -0.064052366, -0.030531697, -0....</td>\n",
       "      <td>2</td>\n",
       "      <td>POINT (-123.18171 46.24494)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-123.148507</td>\n",
       "      <td>46.244990</td>\n",
       "      <td>[[0.05992042, -0.044143558, -0.01674647, -0.01...</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-123.14851 46.24499)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          lon        lat                                         embeddings  \\\n",
       "0 -123.281338  46.244740  [[0.0954321, -0.0801714, -0.029764883, -0.0657...   \n",
       "1 -123.248130  46.244817  [[0.07401827, -0.089014255, -0.008051335, -0.0...   \n",
       "2 -123.214923  46.244884  [[0.053021867, -0.047577165, -0.024503708, -0....   \n",
       "3 -123.181715  46.244942  [[0.026906254, -0.064052366, -0.030531697, -0....   \n",
       "4 -123.148507  46.244990  [[0.05992042, -0.044143558, -0.01674647, -0.01...   \n",
       "\n",
       "   most_common_lulc                     geometry  \n",
       "0                 2  POINT (-123.28134 46.24474)  \n",
       "1                 2  POINT (-123.24813 46.24482)  \n",
       "2                 2  POINT (-123.21492 46.24488)  \n",
       "3                 2  POINT (-123.18171 46.24494)  \n",
       "4                11  POINT (-123.14851 46.24499)  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "from shapely.geometry import Point\n",
    "import pystac_client\n",
    "import stackstac\n",
    "import torch\n",
    "from torchvision import transforms as v2\n",
    "from box import Box\n",
    "import yaml\n",
    "import math\n",
    "from rasterio.enums import Resampling\n",
    "from tqdm import tqdm\n",
    "import rasterio\n",
    "import warnings\n",
    "import os\n",
    "import numpy as np\n",
    "import rioxarray  # Make sure to import rioxarray to extend xarray\n",
    "\n",
    "from src.model import ClayMAEModule\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "STAC_API = \"https://earth-search.aws.element84.com/v1\"\n",
    "COLLECTION = \"sentinel-2-l2a\"\n",
    "\n",
    "# Load the model and metadata\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "ckpt = \"https://clay-model-ckpt.s3.amazonaws.com/v0.5.7/mae_v0.5.7_epoch-13_val-loss-0.3098.ckpt\"\n",
    "torch.set_default_device(device)\n",
    "\n",
    "torch.cuda.empty_cache()  # Clear GPU cache\n",
    "\n",
    "# Assuming grid is a GeoDataFrame with the points\n",
    "points = grid.to_crs(\"EPSG:4326\").geometry.apply(lambda x: (x.x, x.y)).tolist()\n",
    "\n",
    "model = ClayMAEModule.load_from_checkpoint(\n",
    "    ckpt, metadata_path=\"configs/metadata.yaml\", shuffle=False, mask_ratio=0\n",
    ")\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "\n",
    "metadata = Box(yaml.safe_load(open(\"configs/metadata.yaml\")))\n",
    "\n",
    "# Function to normalize timestamp\n",
    "def normalize_timestamp(date):\n",
    "    week = date.isocalendar().week * 2 * np.pi / 52\n",
    "    hour = date.hour * 2 * np.pi / 24\n",
    "    return (math.sin(week), math.cos(week)), (math.sin(hour), math.cos(hour))\n",
    "\n",
    "# Function to normalize lat/lon\n",
    "def normalize_latlon(lat, lon):\n",
    "    lat = lat * np.pi / 180\n",
    "    lon = lon * np.pi / 180\n",
    "    return (math.sin(lat), math.cos(lat)), (math.sin(lon), math.cos(lon))\n",
    "\n",
    "def to_device(data, device):\n",
    "    if isinstance(data, torch.Tensor):\n",
    "        return data.to(device)\n",
    "    elif isinstance(data, dict):\n",
    "        return {k: to_device(v, device) for k, v in data.items()}\n",
    "    elif isinstance(data, list):\n",
    "        return [to_device(v, device) for v in data]\n",
    "    return data\n",
    "\n",
    "def process_point(lon, lat, model, metadata, year, device, j):\n",
    "    model.to(device)  # Ensure the model is on the correct device\n",
    "    catalog = pystac_client.Client.open(STAC_API)\n",
    "    search = catalog.search(\n",
    "        collections=[COLLECTION],\n",
    "        datetime=f\"{year}-01-01/{year}-12-31\",\n",
    "        bbox=(lon - 1e-5, lat - 1e-5, lon + 1e-5, lat + 1e-5),\n",
    "        max_items=10,\n",
    "        query={\"eo:cloud_cover\": {\"lt\": 80}},\n",
    "    )\n",
    "\n",
    "    all_items = search.get_all_items()\n",
    "    items = list(all_items)\n",
    "    if not items:\n",
    "        return None\n",
    "    \n",
    "    items = sorted(items, key=lambda x: x.properties.get('eo:cloud_cover', float('inf')))\n",
    "    lowest_cloud_item = items[0]\n",
    "\n",
    "    epsg = lowest_cloud_item.properties[\"proj:epsg\"]\n",
    "\n",
    "    poidf = gpd.GeoDataFrame(\n",
    "        pd.DataFrame(),\n",
    "        crs=\"EPSG:4326\",\n",
    "        geometry=[Point(lon, lat)],\n",
    "    ).to_crs(epsg)\n",
    "\n",
    "    coords = poidf.iloc[0].geometry.coords[0]\n",
    "\n",
    "    size = 256\n",
    "    gsd = 10\n",
    "    bounds = (\n",
    "        coords[0] - (size * gsd) // 2,\n",
    "        coords[1] - (size * gsd) // 2,\n",
    "        coords[0] + (size * gsd) // 2,\n",
    "        coords[1] + (size * gsd) // 2,\n",
    "    )\n",
    "\n",
    "    stack = stackstac.stack(\n",
    "        lowest_cloud_item,\n",
    "        bounds=bounds,\n",
    "        snap_bounds=False,\n",
    "        epsg=epsg,\n",
    "        resolution=gsd,\n",
    "        dtype=\"float32\",\n",
    "        rescale=False,\n",
    "        fill_value=0,\n",
    "        assets=[\"blue\", \"green\", \"red\", \"nir\"],\n",
    "        resampling=Resampling.nearest,\n",
    "    )\n",
    "\n",
    "    stack = stack.compute()\n",
    "\n",
    "    items = []\n",
    "    dates = []\n",
    "    for item in all_items:\n",
    "        if item.datetime.date() not in dates:\n",
    "            items.append(item)\n",
    "            dates.append(item.datetime.date())\n",
    "\n",
    "    # date = str(stack.time.values)[2:11]\n",
    "\n",
    "    # output_path = os.path.join(\"train_data/embeddings/challenge_1/\", f\"stack_{lon}_{lat}_{j}.tif\")\n",
    "    \n",
    "    # # Write the stack to a TIFF file\n",
    "    # with rasterio.open(\n",
    "    #         output_path, 'w',\n",
    "    #         driver='GTiff',\n",
    "    #         height=stack.shape[2],\n",
    "    #         width=stack.shape[3],\n",
    "    #         count=len(stack.band),  # Number of bands\n",
    "    #         dtype=str(stack.dtype),\n",
    "    #         crs=epsg,\n",
    "    #         transform=stack.rio.transform()\n",
    "    #     ) as tif:\n",
    "    #     for i, band in enumerate(stack.band, start=1):\n",
    "    #         tif.write(np.squeeze(stack.sel(band=band).values), i)\n",
    "\n",
    "    # # Reopen the file to add metadata\n",
    "    # with rasterio.open(output_path, \"r+\") as rst:\n",
    "    #     rst.update_tags(date=date)\n",
    "\n",
    "    platform = \"sentinel-2-l2a\"\n",
    "    mean = []\n",
    "    std = []\n",
    "    waves = []\n",
    "    for band in stack.band:\n",
    "        mean.append(metadata[platform].bands.mean[str(band.values)])\n",
    "        std.append(metadata[platform].bands.std[str(band.values)])\n",
    "        waves.append(metadata[platform].bands.wavelength[str(band.values)])\n",
    "\n",
    "    transform = v2.Compose([v2.Normalize(mean=mean, std=std)])\n",
    "\n",
    "    datetimes = stack.time.values.astype(\"datetime64[s]\").tolist()\n",
    "    times = [normalize_timestamp(dat) for dat in datetimes]\n",
    "    week_norm = [dat[0] for dat in times]\n",
    "    hour_norm = [dat[1] for dat in times]\n",
    "\n",
    "    latlons = [normalize_latlon(lat, lon)] * len(times)\n",
    "    lat_norm = [dat[0] for dat in latlons]\n",
    "    lon_norm = [dat[1] for dat in latlons]\n",
    "\n",
    "    pixels = torch.from_numpy(stack.data.astype(np.float32)).to(device)\n",
    "    pixels = transform(pixels)\n",
    "\n",
    "    batch_size = 16\n",
    "    num_batches = math.ceil(len(stack) / batch_size)\n",
    "    \n",
    "    embeddings_list = []\n",
    "    for i in range(num_batches):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = min((i + 1) * batch_size, len(stack))\n",
    "        \n",
    "        batch_pixels = pixels[start_idx:end_idx].to(device)\n",
    "        batch_time = torch.tensor(np.hstack((week_norm, hour_norm))[start_idx:end_idx], dtype=torch.float32).to(device)\n",
    "        batch_latlon = torch.tensor(np.hstack((lat_norm, lon_norm))[start_idx:end_idx], dtype=torch.float32).to(device)\n",
    "        \n",
    "        batch_datacube = {\n",
    "            \"platform\": platform,\n",
    "            \"time\": batch_time,\n",
    "            \"latlon\": batch_latlon,\n",
    "            \"pixels\": batch_pixels,\n",
    "            \"gsd\": torch.tensor(stack.gsd.values).to(device),\n",
    "            \"waves\": torch.tensor(waves).to(device),\n",
    "        }\n",
    "\n",
    "        batch_datacube = to_device(batch_datacube, device)\n",
    "\n",
    "        try:\n",
    "            model = model.to(device)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                unmsk_patch, _, _, _ = model.model.encoder(batch_datacube)\n",
    "            batch_embeddings = unmsk_patch[:, 0, :].cpu().numpy()\n",
    "            embeddings_list.append(batch_embeddings)\n",
    "        except RuntimeError as e:\n",
    "            if \"out of memory\" in str(e):\n",
    "                print(f\"GPU OOM for point ({lon}, {lat}), batch {i+1}/{num_batches}. Trying CPU...\")\n",
    "                device = torch.device(\"cpu\")\n",
    "                batch_datacube = to_device(batch_datacube, device)\n",
    "                model = model.to(device)\n",
    "                with torch.no_grad():\n",
    "                    unmsk_patch, _, _, _ = model.model.encoder(batch_datacube)\n",
    "                batch_embeddings = unmsk_patch[:, 0, :].numpy()\n",
    "                embeddings_list.append(batch_embeddings)\n",
    "                device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "            else:\n",
    "                raise e\n",
    "\n",
    "    embeddings = np.concatenate(embeddings_list, axis=0)\n",
    "    return embeddings\n",
    "\n",
    "# Specify the year for the datetime range in the search\n",
    "year = 2022\n",
    "\n",
    "# Store results in a list\n",
    "results = []\n",
    "\n",
    "# Iterate through the points and process each one\n",
    "for i, point in enumerate(tqdm(points)):\n",
    "    lon, lat = point\n",
    "    embeddings = process_point(lon, lat, model, metadata, year, device, i)\n",
    "    if embeddings is not None:\n",
    "        results.append((lon, lat, embeddings, grid.loc[i, 'most_common_lulc']))\n",
    "\n",
    "# Create a DataFrame from the results\n",
    "df = pd.DataFrame(results, columns=[\"lon\", \"lat\", \"embeddings\", \"most_common_lulc\"])\n",
    "\n",
    "# Convert to a GeoDataFrame\n",
    "gdf_results = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.lon, df.lat))\n",
    "\n",
    "# Output the resulting GeoDataFrame\n",
    "gdf_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1722999059.663877    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "[I 2024-08-07 02:50:59,682] A new study created in memory with name: no-name-53fd32fe-de38-462b-aaa6-d3745f900a31\n",
      "I0000 00:00:1722999059.678687    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.680160    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.692790    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.694282    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.695721    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.697233    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.698673    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "I0000 00:00:1722999059.700124    3069 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-08-07 02:50:59.702702: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 17057 MB memory:  -> device: 0, name: NVIDIA A10G, pci bus id: 0000:00:1e.0, compute capability: 8.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1722999060.585835    3894 service.cc:146] XLA service 0x7f81d000a840 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1722999060.585884    3894 service.cc:154]   StreamExecutor device (0): NVIDIA A10G, Compute Capability 8.6\n",
      "2024-08-07 02:51:00.675019: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2024-08-07 02:51:00.847610: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m117/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━━━\u001b[0m \u001b[1m0s\u001b[0m 868us/step - accuracy: 0.6615 - loss: 3.7767"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0000 00:00:1722999062.203440    3894 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 9ms/step - accuracy: 0.6666 - loss: 3.6759  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:51:04.195547: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_47', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 29ms/step - accuracy: 0.6669 - loss: 3.6704 - val_accuracy: 0.6935 - val_loss: 1.9502\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7244 - loss: 2.0777 - val_accuracy: 0.7301 - val_loss: 2.8380\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7555 - loss: 2.0405 - val_accuracy: 0.7554 - val_loss: 2.6633\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7274 - loss: 2.8781 - val_accuracy: 0.7067 - val_loss: 2.4844\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7324 - loss: 2.9547 - val_accuracy: 0.7348 - val_loss: 2.7956\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7326 - loss: 2.6526 - val_accuracy: 0.7301 - val_loss: 2.5576\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7408 - loss: 4.1304 - val_accuracy: 0.6851 - val_loss: 3.8843\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7311 - loss: 3.5946 - val_accuracy: 0.7657 - val_loss: 3.1817\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7626 - loss: 2.3963 - val_accuracy: 0.7723 - val_loss: 3.8773\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7532 - loss: 2.7555 - val_accuracy: 0.7666 - val_loss: 2.7545\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7506 - loss: 2.8434 - val_accuracy: 0.7404 - val_loss: 4.1258\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7618 - loss: 3.7461 - val_accuracy: 0.7882 - val_loss: 3.9408\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7648 - loss: 3.6247 - val_accuracy: 0.7526 - val_loss: 3.8407\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7528 - loss: 3.5997 - val_accuracy: 0.7910 - val_loss: 4.4842\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7557 - loss: 4.8375 - val_accuracy: 0.8013 - val_loss: 5.2585\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7855 - loss: 3.0204 - val_accuracy: 0.7245 - val_loss: 3.6502\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7693 - loss: 4.5460 - val_accuracy: 0.7816 - val_loss: 4.5582\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7888 - loss: 3.2712 - val_accuracy: 0.7573 - val_loss: 5.3459\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7849 - loss: 3.4423 - val_accuracy: 0.8013 - val_loss: 4.2786\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7960 - loss: 2.6691 - val_accuracy: 0.7123 - val_loss: 5.0664\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 27ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:51:11,034] Trial 0 finished with value: 0.7130169484692862 and parameters: {'n_layers': 1, 'n_units_l0': 477, 'dropout_rate': 0.29073284384135306, 'learning_rate': 0.007900485170528578}. Best is trial 0 with value: 0.7130169484692862.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.5974 - loss: 1.5838 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:51:17.471249: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_65', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 54ms/step - accuracy: 0.5975 - loss: 1.5821 - val_accuracy: 0.6439 - val_loss: 1.0441\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6125 - loss: 1.1442 - val_accuracy: 0.5858 - val_loss: 1.2095\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5631 - loss: 1.3083 - val_accuracy: 0.5351 - val_loss: 1.2581\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5770 - loss: 1.1962 - val_accuracy: 0.5351 - val_loss: 1.2991\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5747 - loss: 1.2110 - val_accuracy: 0.5351 - val_loss: 1.2867\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5579 - loss: 1.2285 - val_accuracy: 0.5351 - val_loss: 1.3024\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5652 - loss: 1.2284 - val_accuracy: 0.5351 - val_loss: 1.3093\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5737 - loss: 1.2101 - val_accuracy: 0.5351 - val_loss: 1.3085\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5809 - loss: 1.2001 - val_accuracy: 0.5351 - val_loss: 1.2938\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5711 - loss: 1.2481 - val_accuracy: 0.5351 - val_loss: 1.2994\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5663 - loss: 1.2222 - val_accuracy: 0.5351 - val_loss: 1.3064\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5647 - loss: 1.2194 - val_accuracy: 0.5351 - val_loss: 1.2959\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5585 - loss: 1.2326 - val_accuracy: 0.5351 - val_loss: 1.2935\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5685 - loss: 1.2100 - val_accuracy: 0.5351 - val_loss: 1.2910\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5766 - loss: 1.2092 - val_accuracy: 0.5351 - val_loss: 1.3110\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5696 - loss: 1.2359 - val_accuracy: 0.5351 - val_loss: 1.3037\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5783 - loss: 1.2061 - val_accuracy: 0.5351 - val_loss: 1.3033\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5812 - loss: 1.1951 - val_accuracy: 0.5351 - val_loss: 1.2974\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5669 - loss: 1.2203 - val_accuracy: 0.5351 - val_loss: 1.2949\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5711 - loss: 1.2077 - val_accuracy: 0.5351 - val_loss: 1.2975\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m35s\u001b[0m 877ms/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:51:27.838271: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_18', 16 bytes spill stores, 16 bytes spill loads\n",
      "\n",
      "2024-08-07 02:51:27.894932: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_18', 96 bytes spill stores, 96 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 63ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:51:28,934] Trial 1 finished with value: 0.3928212599307391 and parameters: {'n_layers': 3, 'n_units_l0': 309, 'n_units_l1': 83, 'n_units_l2': 511, 'dropout_rate': 0.3653050205603694, 'learning_rate': 0.007842042668730675}. Best is trial 0 with value: 0.7130169484692862.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 33ms/step - accuracy: 0.6655 - loss: 1.7158 - val_accuracy: 0.7835 - val_loss: 0.7106\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7536 - loss: 0.8385 - val_accuracy: 0.7854 - val_loss: 0.7006\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7619 - loss: 0.8323 - val_accuracy: 0.7619 - val_loss: 0.7352\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7443 - loss: 0.8285 - val_accuracy: 0.7498 - val_loss: 0.7704\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7402 - loss: 0.8320 - val_accuracy: 0.7432 - val_loss: 0.7714\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7399 - loss: 0.8101 - val_accuracy: 0.7301 - val_loss: 0.8616\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7631 - loss: 0.7899 - val_accuracy: 0.6879 - val_loss: 0.8823\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7109 - loss: 0.8929 - val_accuracy: 0.7310 - val_loss: 0.8563\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7335 - loss: 0.8350 - val_accuracy: 0.7291 - val_loss: 0.9013\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7391 - loss: 0.8301 - val_accuracy: 0.6935 - val_loss: 0.9370\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7222 - loss: 0.8909 - val_accuracy: 0.6645 - val_loss: 1.0228\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6895 - loss: 1.0112 - val_accuracy: 0.6523 - val_loss: 1.2021\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6853 - loss: 0.9629 - val_accuracy: 0.6176 - val_loss: 1.2150\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6290 - loss: 1.1749 - val_accuracy: 0.6270 - val_loss: 1.0882\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6457 - loss: 1.0304 - val_accuracy: 0.6532 - val_loss: 1.0148\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6728 - loss: 0.9695 - val_accuracy: 0.6682 - val_loss: 0.9876\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6707 - loss: 0.9405 - val_accuracy: 0.6270 - val_loss: 1.1366\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6627 - loss: 0.9850 - val_accuracy: 0.6485 - val_loss: 1.0869\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6483 - loss: 0.9968 - val_accuracy: 0.6401 - val_loss: 1.1939\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6607 - loss: 1.0069 - val_accuracy: 0.6261 - val_loss: 1.1404\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 33ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:51:41,278] Trial 2 finished with value: 0.5816451673780522 and parameters: {'n_layers': 2, 'n_units_l0': 235, 'n_units_l1': 444, 'dropout_rate': 0.2807943200848955, 'learning_rate': 0.00788201241617743}. Best is trial 0 with value: 0.7130169484692862.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 56ms/step - accuracy: 0.5755 - loss: 2.0652 - val_accuracy: 0.7338 - val_loss: 0.8257\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6803 - loss: 1.0904 - val_accuracy: 0.5886 - val_loss: 0.9488\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6467 - loss: 1.1744 - val_accuracy: 0.6560 - val_loss: 0.8998\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6265 - loss: 1.1334 - val_accuracy: 0.6326 - val_loss: 1.0785\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6341 - loss: 1.1603 - val_accuracy: 0.5764 - val_loss: 1.1946\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6071 - loss: 1.2068 - val_accuracy: 0.5661 - val_loss: 1.2450\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5735 - loss: 1.3115 - val_accuracy: 0.5351 - val_loss: 1.5589\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5525 - loss: 1.3003 - val_accuracy: 0.5351 - val_loss: 1.2960\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5523 - loss: 1.2453 - val_accuracy: 0.5351 - val_loss: 1.2964\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5667 - loss: 1.2442 - val_accuracy: 0.5351 - val_loss: 1.2962\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5775 - loss: 1.2166 - val_accuracy: 0.5351 - val_loss: 1.2936\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5699 - loss: 1.2200 - val_accuracy: 0.4705 - val_loss: 1.4718\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5721 - loss: 1.7792 - val_accuracy: 0.5351 - val_loss: 1.2952\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5696 - loss: 1.2099 - val_accuracy: 0.5351 - val_loss: 1.2963\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5662 - loss: 1.2009 - val_accuracy: 0.5351 - val_loss: 1.2975\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5547 - loss: 1.2341 - val_accuracy: 0.5351 - val_loss: 1.2964\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5640 - loss: 1.2515 - val_accuracy: 0.5351 - val_loss: 1.2959\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5758 - loss: 1.1975 - val_accuracy: 0.5351 - val_loss: 1.2948\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5736 - loss: 1.2009 - val_accuracy: 0.5351 - val_loss: 1.2945\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5658 - loss: 1.2250 - val_accuracy: 0.5351 - val_loss: 1.2960\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m55s\u001b[0m 1s/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:51:58.178497: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_18', 28 bytes spill stores, 28 bytes spill loads\n",
      "\n",
      "2024-08-07 02:51:58.624896: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_18', 20 bytes spill stores, 20 bytes spill loads\n",
      "\n",
      "2024-08-07 02:51:58.813364: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_25', 112 bytes spill stores, 112 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 82ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:52:00,603] Trial 3 finished with value: 0.3928212599307391 and parameters: {'n_layers': 3, 'n_units_l0': 369, 'n_units_l1': 359, 'n_units_l2': 131, 'dropout_rate': 0.48547006827504047, 'learning_rate': 0.007051529295035562}. Best is trial 0 with value: 0.7130169484692862.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 24ms/step - accuracy: 0.6906 - loss: 2.3912 - val_accuracy: 0.7132 - val_loss: 1.2524\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7407 - loss: 1.1035 - val_accuracy: 0.8107 - val_loss: 0.9237\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7533 - loss: 1.0289 - val_accuracy: 0.7460 - val_loss: 1.0348\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7480 - loss: 1.3651 - val_accuracy: 0.7226 - val_loss: 1.8208\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7314 - loss: 1.4317 - val_accuracy: 0.7769 - val_loss: 1.2932\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7485 - loss: 1.4722 - val_accuracy: 0.7582 - val_loss: 1.9281\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7611 - loss: 1.2859 - val_accuracy: 0.7966 - val_loss: 1.4835\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7569 - loss: 1.2823 - val_accuracy: 0.7573 - val_loss: 1.4084\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7743 - loss: 1.2507 - val_accuracy: 0.7423 - val_loss: 1.6333\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7565 - loss: 1.6199 - val_accuracy: 0.7010 - val_loss: 1.7825\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7653 - loss: 1.4508 - val_accuracy: 0.7807 - val_loss: 1.8915\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7653 - loss: 1.8114 - val_accuracy: 0.7601 - val_loss: 1.8569\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7907 - loss: 1.2423 - val_accuracy: 0.6898 - val_loss: 2.0032\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7565 - loss: 1.8569 - val_accuracy: 0.7929 - val_loss: 1.9850\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8024 - loss: 1.2728 - val_accuracy: 0.7207 - val_loss: 1.9619\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7736 - loss: 1.5505 - val_accuracy: 0.7816 - val_loss: 2.4030\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7995 - loss: 1.6429 - val_accuracy: 0.7170 - val_loss: 2.0645\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7964 - loss: 1.2866 - val_accuracy: 0.7685 - val_loss: 2.3737\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7992 - loss: 1.3585 - val_accuracy: 0.7666 - val_loss: 1.9402\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7996 - loss: 1.4428 - val_accuracy: 0.7826 - val_loss: 2.5086\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 31ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:52:10,692] Trial 4 finished with value: 0.7669589469504712 and parameters: {'n_layers': 1, 'n_units_l0': 384, 'dropout_rate': 0.2316262031414833, 'learning_rate': 0.006428799974731112}. Best is trial 4 with value: 0.7669589469504712.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 40ms/step - accuracy: 0.6605 - loss: 1.9300 - val_accuracy: 0.8013 - val_loss: 0.7324\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7525 - loss: 0.8042 - val_accuracy: 0.7479 - val_loss: 0.7247\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7590 - loss: 0.7341 - val_accuracy: 0.7873 - val_loss: 0.6911\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7583 - loss: 0.8431 - val_accuracy: 0.7619 - val_loss: 0.8154\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7411 - loss: 0.8210 - val_accuracy: 0.7732 - val_loss: 0.9244\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7360 - loss: 0.8589 - val_accuracy: 0.7048 - val_loss: 0.9254\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7185 - loss: 0.9065 - val_accuracy: 0.7310 - val_loss: 0.8338\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7175 - loss: 0.8915 - val_accuracy: 0.7235 - val_loss: 1.0146\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6812 - loss: 1.0090 - val_accuracy: 0.6982 - val_loss: 0.9388\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7035 - loss: 0.9633 - val_accuracy: 0.6692 - val_loss: 1.0276\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6506 - loss: 1.1597 - val_accuracy: 0.5979 - val_loss: 1.1826\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6252 - loss: 1.1301 - val_accuracy: 0.6307 - val_loss: 1.1594\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6599 - loss: 1.0608 - val_accuracy: 0.6354 - val_loss: 1.1469\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6366 - loss: 1.1290 - val_accuracy: 0.6242 - val_loss: 1.2012\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6418 - loss: 1.1053 - val_accuracy: 0.6195 - val_loss: 1.3038\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6436 - loss: 1.1530 - val_accuracy: 0.6326 - val_loss: 1.2874\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6622 - loss: 1.0882 - val_accuracy: 0.6420 - val_loss: 1.2131\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6456 - loss: 1.3082 - val_accuracy: 0.6429 - val_loss: 1.1955\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6578 - loss: 1.0447 - val_accuracy: 0.6120 - val_loss: 1.1291\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6352 - loss: 1.0888 - val_accuracy: 0.6373 - val_loss: 1.2076\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 44ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:52:24,911] Trial 5 finished with value: 0.5942317622249029 and parameters: {'n_layers': 2, 'n_units_l0': 346, 'n_units_l1': 213, 'dropout_rate': 0.23717394051247603, 'learning_rate': 0.008961900686427363}. Best is trial 4 with value: 0.7669589469504712.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 40ms/step - accuracy: 0.5889 - loss: 1.2650 - val_accuracy: 0.8079 - val_loss: 0.5694\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7602 - loss: 0.7039 - val_accuracy: 0.8144 - val_loss: 0.5532\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7989 - loss: 0.6023 - val_accuracy: 0.8154 - val_loss: 0.5279\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7932 - loss: 0.5908 - val_accuracy: 0.8257 - val_loss: 0.5185\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8074 - loss: 0.5384 - val_accuracy: 0.8247 - val_loss: 0.5202\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7980 - loss: 0.5436 - val_accuracy: 0.8238 - val_loss: 0.5172\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8223 - loss: 0.5003 - val_accuracy: 0.8257 - val_loss: 0.5122\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8202 - loss: 0.4986 - val_accuracy: 0.8257 - val_loss: 0.5066\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8186 - loss: 0.4891 - val_accuracy: 0.8219 - val_loss: 0.5157\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8242 - loss: 0.4877 - val_accuracy: 0.8304 - val_loss: 0.5050\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8358 - loss: 0.4502 - val_accuracy: 0.8276 - val_loss: 0.5104\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8389 - loss: 0.4330 - val_accuracy: 0.8229 - val_loss: 0.5065\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8483 - loss: 0.4200 - val_accuracy: 0.8304 - val_loss: 0.5061\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8345 - loss: 0.4436 - val_accuracy: 0.8360 - val_loss: 0.4952\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8410 - loss: 0.4289 - val_accuracy: 0.8229 - val_loss: 0.5022\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8576 - loss: 0.3796 - val_accuracy: 0.8191 - val_loss: 0.5030\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8454 - loss: 0.3924 - val_accuracy: 0.8294 - val_loss: 0.5046\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8498 - loss: 0.4048 - val_accuracy: 0.8107 - val_loss: 0.5131\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8527 - loss: 0.3829 - val_accuracy: 0.8341 - val_loss: 0.5146\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8623 - loss: 0.3526 - val_accuracy: 0.8341 - val_loss: 0.5079\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 44ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:52:39,060] Trial 6 finished with value: 0.7871345736449633 and parameters: {'n_layers': 2, 'n_units_l0': 462, 'n_units_l1': 271, 'dropout_rate': 0.391496239477795, 'learning_rate': 0.00014162268637782355}. Best is trial 6 with value: 0.7871345736449633.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 35ms/step - accuracy: 0.6783 - loss: 1.3778 - val_accuracy: 0.7948 - val_loss: 0.6426\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7660 - loss: 0.7152 - val_accuracy: 0.7788 - val_loss: 0.6256\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7704 - loss: 0.6699 - val_accuracy: 0.7629 - val_loss: 0.6669\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7695 - loss: 0.7156 - val_accuracy: 0.7957 - val_loss: 0.5950\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7804 - loss: 0.6622 - val_accuracy: 0.7666 - val_loss: 0.6243\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7830 - loss: 0.6445 - val_accuracy: 0.7957 - val_loss: 0.5977\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7776 - loss: 0.6438 - val_accuracy: 0.7910 - val_loss: 0.6106\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7860 - loss: 0.6374 - val_accuracy: 0.7816 - val_loss: 0.6400\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7898 - loss: 0.6171 - val_accuracy: 0.7891 - val_loss: 0.7005\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7771 - loss: 0.6715 - val_accuracy: 0.7798 - val_loss: 0.6699\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7773 - loss: 0.6587 - val_accuracy: 0.7957 - val_loss: 0.6557\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7887 - loss: 0.6493 - val_accuracy: 0.7657 - val_loss: 0.6584\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7796 - loss: 0.6345 - val_accuracy: 0.7816 - val_loss: 0.6889\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7859 - loss: 0.6252 - val_accuracy: 0.7741 - val_loss: 0.7497\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7889 - loss: 0.6426 - val_accuracy: 0.7779 - val_loss: 0.7002\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7777 - loss: 0.6779 - val_accuracy: 0.7854 - val_loss: 0.7385\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7825 - loss: 0.6431 - val_accuracy: 0.7929 - val_loss: 0.7051\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7979 - loss: 0.6544 - val_accuracy: 0.7516 - val_loss: 0.7242\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7829 - loss: 0.6143 - val_accuracy: 0.7976 - val_loss: 0.6872\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8007 - loss: 0.6074 - val_accuracy: 0.7807 - val_loss: 0.7200\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 35ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:52:51,985] Trial 7 finished with value: 0.7517940033158698 and parameters: {'n_layers': 2, 'n_units_l0': 282, 'n_units_l1': 286, 'dropout_rate': 0.3354047338926447, 'learning_rate': 0.003984781653061122}. Best is trial 6 with value: 0.7871345736449633.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 27ms/step - accuracy: 0.6983 - loss: 1.1617 - val_accuracy: 0.7919 - val_loss: 0.6829\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7879 - loss: 0.6591 - val_accuracy: 0.8051 - val_loss: 0.6338\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8022 - loss: 0.5792 - val_accuracy: 0.7994 - val_loss: 0.5852\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8108 - loss: 0.5361 - val_accuracy: 0.8088 - val_loss: 0.6135\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8066 - loss: 0.5653 - val_accuracy: 0.8154 - val_loss: 0.5751\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8183 - loss: 0.4936 - val_accuracy: 0.8013 - val_loss: 0.5907\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8280 - loss: 0.4850 - val_accuracy: 0.7976 - val_loss: 0.6309\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8158 - loss: 0.5036 - val_accuracy: 0.8201 - val_loss: 0.6008\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8303 - loss: 0.4504 - val_accuracy: 0.8144 - val_loss: 0.6222\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8308 - loss: 0.4577 - val_accuracy: 0.8144 - val_loss: 0.6287\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8493 - loss: 0.4282 - val_accuracy: 0.7985 - val_loss: 0.6668\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8211 - loss: 0.4707 - val_accuracy: 0.8060 - val_loss: 0.6972\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8430 - loss: 0.4208 - val_accuracy: 0.8060 - val_loss: 0.6436\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8398 - loss: 0.4423 - val_accuracy: 0.8022 - val_loss: 0.6883\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8543 - loss: 0.4206 - val_accuracy: 0.8107 - val_loss: 0.7123\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8558 - loss: 0.3913 - val_accuracy: 0.8051 - val_loss: 0.6414\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8692 - loss: 0.3550 - val_accuracy: 0.7826 - val_loss: 0.7108\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8476 - loss: 0.3967 - val_accuracy: 0.7882 - val_loss: 0.7340\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8660 - loss: 0.3706 - val_accuracy: 0.8069 - val_loss: 0.7564\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8664 - loss: 0.3831 - val_accuracy: 0.8041 - val_loss: 0.7201\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 22ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:53:02,455] Trial 8 finished with value: 0.7791397706257758 and parameters: {'n_layers': 1, 'n_units_l0': 270, 'dropout_rate': 0.3587958401036683, 'learning_rate': 0.0019058143286411963}. Best is trial 6 with value: 0.7871345736449633.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 41ms/step - accuracy: 0.6290 - loss: 1.3881 - val_accuracy: 0.7694 - val_loss: 0.7675\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7087 - loss: 1.0266 - val_accuracy: 0.7310 - val_loss: 0.8105\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7136 - loss: 0.9799 - val_accuracy: 0.7329 - val_loss: 0.7768\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6990 - loss: 0.9787 - val_accuracy: 0.7685 - val_loss: 0.7923\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7302 - loss: 0.8998 - val_accuracy: 0.7470 - val_loss: 0.8232\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7065 - loss: 0.9477 - val_accuracy: 0.6813 - val_loss: 0.9135\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6993 - loss: 1.0361 - val_accuracy: 0.7526 - val_loss: 0.7723\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7004 - loss: 0.9738 - val_accuracy: 0.6457 - val_loss: 0.8950\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6472 - loss: 1.0305 - val_accuracy: 0.6270 - val_loss: 0.9568\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6578 - loss: 1.0760 - val_accuracy: 0.6176 - val_loss: 1.0610\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6238 - loss: 1.2937 - val_accuracy: 0.6279 - val_loss: 1.0112\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.6317 - loss: 1.2258 - val_accuracy: 0.5614 - val_loss: 1.0769\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5901 - loss: 1.1078 - val_accuracy: 0.5548 - val_loss: 1.1104\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5909 - loss: 1.1038 - val_accuracy: 0.5576 - val_loss: 1.0330\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5826 - loss: 1.1563 - val_accuracy: 0.5380 - val_loss: 1.1449\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5752 - loss: 1.1765 - val_accuracy: 0.5567 - val_loss: 1.0935\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5890 - loss: 1.1218 - val_accuracy: 0.5464 - val_loss: 1.0941\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5780 - loss: 1.1300 - val_accuracy: 0.5455 - val_loss: 1.2017\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5786 - loss: 1.2037 - val_accuracy: 0.5370 - val_loss: 1.2272\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.5781 - loss: 1.1572 - val_accuracy: 0.5351 - val_loss: 1.2080\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 38ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:53:17,274] Trial 9 finished with value: 0.3928212599307391 and parameters: {'n_layers': 3, 'n_units_l0': 115, 'n_units_l1': 135, 'n_units_l2': 259, 'dropout_rate': 0.35783171296437455, 'learning_rate': 0.008067179084199756}. Best is trial 6 with value: 0.7871345736449633.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - accuracy: 0.6304 - loss: 1.1930 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:53:23.237880: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 34ms/step - accuracy: 0.6310 - loss: 1.1908 - val_accuracy: 0.7891 - val_loss: 0.5824\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7838 - loss: 0.6572 - val_accuracy: 0.8079 - val_loss: 0.5441\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7825 - loss: 0.6110 - val_accuracy: 0.7985 - val_loss: 0.5469\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8051 - loss: 0.5624 - val_accuracy: 0.8191 - val_loss: 0.5139\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8081 - loss: 0.5265 - val_accuracy: 0.8201 - val_loss: 0.5193\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8242 - loss: 0.4802 - val_accuracy: 0.8201 - val_loss: 0.5181\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8174 - loss: 0.5001 - val_accuracy: 0.8135 - val_loss: 0.5155\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8248 - loss: 0.4932 - val_accuracy: 0.8229 - val_loss: 0.5029\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8321 - loss: 0.4593 - val_accuracy: 0.8172 - val_loss: 0.5144\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8418 - loss: 0.4419 - val_accuracy: 0.8247 - val_loss: 0.4944\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8382 - loss: 0.4331 - val_accuracy: 0.8201 - val_loss: 0.5043\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8365 - loss: 0.4398 - val_accuracy: 0.8172 - val_loss: 0.5100\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8465 - loss: 0.4033 - val_accuracy: 0.8201 - val_loss: 0.5108\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8521 - loss: 0.4124 - val_accuracy: 0.8238 - val_loss: 0.4954\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8392 - loss: 0.4255 - val_accuracy: 0.8201 - val_loss: 0.5088\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8538 - loss: 0.3895 - val_accuracy: 0.8116 - val_loss: 0.5269\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8519 - loss: 0.3933 - val_accuracy: 0.8097 - val_loss: 0.5193\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8597 - loss: 0.3737 - val_accuracy: 0.8135 - val_loss: 0.5086\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8584 - loss: 0.3669 - val_accuracy: 0.8154 - val_loss: 0.5153\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8632 - loss: 0.3603 - val_accuracy: 0.8238 - val_loss: 0.5281\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 26ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:53:29,803] Trial 10 finished with value: 0.7941815048681673 and parameters: {'n_layers': 2, 'n_units_l0': 512, 'n_units_l1': 474, 'dropout_rate': 0.4468147287490691, 'learning_rate': 0.00026551980594566923}. Best is trial 10 with value: 0.7941815048681673.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - accuracy: 0.5594 - loss: 1.3519 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:53:34.405188: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n",
      "2024-08-07 02:53:34.707152: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_49', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 38ms/step - accuracy: 0.5602 - loss: 1.3493 - val_accuracy: 0.7938 - val_loss: 0.5966\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7661 - loss: 0.6934 - val_accuracy: 0.8116 - val_loss: 0.5496\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7787 - loss: 0.6402 - val_accuracy: 0.8144 - val_loss: 0.5302\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7931 - loss: 0.5770 - val_accuracy: 0.8163 - val_loss: 0.5148\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8035 - loss: 0.5662 - val_accuracy: 0.8191 - val_loss: 0.5125\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8010 - loss: 0.5511 - val_accuracy: 0.8257 - val_loss: 0.5054\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8150 - loss: 0.5126 - val_accuracy: 0.8313 - val_loss: 0.5035\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8188 - loss: 0.5008 - val_accuracy: 0.8276 - val_loss: 0.5066\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8159 - loss: 0.4791 - val_accuracy: 0.8304 - val_loss: 0.4969\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8175 - loss: 0.4829 - val_accuracy: 0.8276 - val_loss: 0.4946\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8303 - loss: 0.4621 - val_accuracy: 0.8219 - val_loss: 0.5047\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8173 - loss: 0.4837 - val_accuracy: 0.8219 - val_loss: 0.4946\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8317 - loss: 0.4532 - val_accuracy: 0.8322 - val_loss: 0.4929\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8309 - loss: 0.4601 - val_accuracy: 0.8285 - val_loss: 0.4998\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8379 - loss: 0.4299 - val_accuracy: 0.8313 - val_loss: 0.4883\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8549 - loss: 0.4020 - val_accuracy: 0.8276 - val_loss: 0.4939\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8454 - loss: 0.4219 - val_accuracy: 0.8369 - val_loss: 0.4871\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8315 - loss: 0.4311 - val_accuracy: 0.8304 - val_loss: 0.4995\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8517 - loss: 0.3960 - val_accuracy: 0.8322 - val_loss: 0.4906\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8492 - loss: 0.3868 - val_accuracy: 0.8322 - val_loss: 0.4927\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m39s\u001b[0m 964ms/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:53:43.002485: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_16', 48 bytes spill stores, 48 bytes spill loads\n",
      "\n",
      "2024-08-07 02:53:43.191679: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_16', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 33ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:53:43,590] Trial 11 finished with value: 0.7896297468667736 and parameters: {'n_layers': 2, 'n_units_l0': 503, 'n_units_l1': 493, 'dropout_rate': 0.44818899473839335, 'learning_rate': 0.00012162431199714695}. Best is trial 10 with value: 0.7941815048681673.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - accuracy: 0.6606 - loss: 1.1031 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:53:48.258032: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_56', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n",
      "2024-08-07 02:53:48.550683: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_49', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 39ms/step - accuracy: 0.6611 - loss: 1.1014 - val_accuracy: 0.8097 - val_loss: 0.5496\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7756 - loss: 0.6797 - val_accuracy: 0.8163 - val_loss: 0.5523\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7858 - loss: 0.6129 - val_accuracy: 0.8069 - val_loss: 0.5370\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8092 - loss: 0.5435 - val_accuracy: 0.8172 - val_loss: 0.5302\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8044 - loss: 0.5629 - val_accuracy: 0.8201 - val_loss: 0.5240\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8110 - loss: 0.5200 - val_accuracy: 0.8135 - val_loss: 0.5265\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8214 - loss: 0.4865 - val_accuracy: 0.8069 - val_loss: 0.5192\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8184 - loss: 0.4811 - val_accuracy: 0.8182 - val_loss: 0.5262\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8279 - loss: 0.4596 - val_accuracy: 0.8182 - val_loss: 0.5189\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8252 - loss: 0.4802 - val_accuracy: 0.8135 - val_loss: 0.5024\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8383 - loss: 0.4321 - val_accuracy: 0.8219 - val_loss: 0.5005\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8359 - loss: 0.4391 - val_accuracy: 0.8229 - val_loss: 0.5090\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8338 - loss: 0.4341 - val_accuracy: 0.8135 - val_loss: 0.5223\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8308 - loss: 0.4518 - val_accuracy: 0.8219 - val_loss: 0.5169\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8466 - loss: 0.3924 - val_accuracy: 0.8229 - val_loss: 0.5422\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8458 - loss: 0.4297 - val_accuracy: 0.8182 - val_loss: 0.5240\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8348 - loss: 0.4245 - val_accuracy: 0.8285 - val_loss: 0.5126\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8501 - loss: 0.3959 - val_accuracy: 0.8238 - val_loss: 0.5239\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8507 - loss: 0.3871 - val_accuracy: 0.8276 - val_loss: 0.5312\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8440 - loss: 0.3902 - val_accuracy: 0.8257 - val_loss: 0.5240\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m31s\u001b[0m 774ms/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:53:56.695399: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_16', 8 bytes spill stores, 8 bytes spill loads\n",
      "\n",
      "2024-08-07 02:53:56.713241: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_16', 32 bytes spill stores, 32 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 35ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:53:57,330] Trial 12 finished with value: 0.7961813683060185 and parameters: {'n_layers': 2, 'n_units_l0': 511, 'n_units_l1': 505, 'dropout_rate': 0.4639565318456878, 'learning_rate': 0.0004228409406485573}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m7s\u001b[0m 37ms/step - accuracy: 0.6553 - loss: 1.4463 - val_accuracy: 0.7985 - val_loss: 0.6439\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7722 - loss: 0.7132 - val_accuracy: 0.8060 - val_loss: 0.6122\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7748 - loss: 0.6739 - val_accuracy: 0.7994 - val_loss: 0.6013\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7816 - loss: 0.6488 - val_accuracy: 0.7938 - val_loss: 0.6319\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7757 - loss: 0.6568 - val_accuracy: 0.7985 - val_loss: 0.5764\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7847 - loss: 0.6627 - val_accuracy: 0.8126 - val_loss: 0.6185\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7787 - loss: 0.6724 - val_accuracy: 0.7788 - val_loss: 0.6429\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7813 - loss: 0.6673 - val_accuracy: 0.8069 - val_loss: 0.6226\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7861 - loss: 0.6573 - val_accuracy: 0.7844 - val_loss: 0.6683\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7797 - loss: 0.6681 - val_accuracy: 0.7648 - val_loss: 0.7101\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7698 - loss: 0.7150 - val_accuracy: 0.7788 - val_loss: 0.6935\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7866 - loss: 0.6670 - val_accuracy: 0.8051 - val_loss: 0.6308\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7815 - loss: 0.6723 - val_accuracy: 0.8088 - val_loss: 0.6332\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7821 - loss: 0.6636 - val_accuracy: 0.7919 - val_loss: 0.6303\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7838 - loss: 0.6342 - val_accuracy: 0.7807 - val_loss: 0.6649\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7885 - loss: 0.6430 - val_accuracy: 0.7788 - val_loss: 0.6714\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7829 - loss: 0.6480 - val_accuracy: 0.7948 - val_loss: 0.6434\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7721 - loss: 0.7060 - val_accuracy: 0.7854 - val_loss: 0.6732\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7959 - loss: 0.6053 - val_accuracy: 0.8013 - val_loss: 0.6852\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7707 - loss: 0.6869 - val_accuracy: 0.7723 - val_loss: 0.6991\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m24s\u001b[0m 604ms/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:54:09.823207: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_16', 72 bytes spill stores, 72 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 36ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:54:10,515] Trial 13 finished with value: 0.7378198296700478 and parameters: {'n_layers': 2, 'n_units_l0': 429, 'n_units_l1': 406, 'dropout_rate': 0.4359508188415538, 'learning_rate': 0.002896529424542867}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 23ms/step - accuracy: 0.6766 - loss: 1.2171 - val_accuracy: 0.7976 - val_loss: 0.6267\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7666 - loss: 0.7266 - val_accuracy: 0.8116 - val_loss: 0.5922\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7977 - loss: 0.5753 - val_accuracy: 0.8088 - val_loss: 0.5541\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8097 - loss: 0.5545 - val_accuracy: 0.8116 - val_loss: 0.5192\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8067 - loss: 0.5408 - val_accuracy: 0.8032 - val_loss: 0.5715\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8110 - loss: 0.5257 - val_accuracy: 0.8116 - val_loss: 0.5409\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8307 - loss: 0.4869 - val_accuracy: 0.8163 - val_loss: 0.5327\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8122 - loss: 0.4907 - val_accuracy: 0.8126 - val_loss: 0.5754\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8202 - loss: 0.4823 - val_accuracy: 0.8182 - val_loss: 0.5516\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8172 - loss: 0.4872 - val_accuracy: 0.8229 - val_loss: 0.5595\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8333 - loss: 0.4453 - val_accuracy: 0.8219 - val_loss: 0.5502\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8351 - loss: 0.4380 - val_accuracy: 0.8247 - val_loss: 0.5644\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8358 - loss: 0.4332 - val_accuracy: 0.8126 - val_loss: 0.5489\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8209 - loss: 0.4294 - val_accuracy: 0.8276 - val_loss: 0.5630\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8420 - loss: 0.4166 - val_accuracy: 0.8219 - val_loss: 0.6101\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8297 - loss: 0.4438 - val_accuracy: 0.8257 - val_loss: 0.5628\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8526 - loss: 0.3931 - val_accuracy: 0.8229 - val_loss: 0.5953\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8598 - loss: 0.3736 - val_accuracy: 0.8163 - val_loss: 0.5892\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8450 - loss: 0.4132 - val_accuracy: 0.8201 - val_loss: 0.5959\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8403 - loss: 0.4152 - val_accuracy: 0.8210 - val_loss: 0.5990\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:54:20,214] Trial 14 finished with value: 0.7956795160966945 and parameters: {'n_layers': 1, 'n_units_l0': 191, 'dropout_rate': 0.4979503490692606, 'learning_rate': 0.0015983288561742088}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 22ms/step - accuracy: 0.6656 - loss: 1.3724 - val_accuracy: 0.8097 - val_loss: 0.6466\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7703 - loss: 0.7226 - val_accuracy: 0.8107 - val_loss: 0.5633\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7936 - loss: 0.5896 - val_accuracy: 0.7948 - val_loss: 0.5560\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7948 - loss: 0.5578 - val_accuracy: 0.8079 - val_loss: 0.5715\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8180 - loss: 0.5359 - val_accuracy: 0.7882 - val_loss: 0.6061\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8194 - loss: 0.5087 - val_accuracy: 0.8247 - val_loss: 0.5436\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8066 - loss: 0.5244 - val_accuracy: 0.7873 - val_loss: 0.5831\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8090 - loss: 0.5208 - val_accuracy: 0.8060 - val_loss: 0.5749\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8178 - loss: 0.4948 - val_accuracy: 0.8097 - val_loss: 0.6036\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8265 - loss: 0.4986 - val_accuracy: 0.8116 - val_loss: 0.5703\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8198 - loss: 0.4937 - val_accuracy: 0.8144 - val_loss: 0.5724\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8286 - loss: 0.4720 - val_accuracy: 0.8210 - val_loss: 0.5858\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8126 - loss: 0.4770 - val_accuracy: 0.8079 - val_loss: 0.5790\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8363 - loss: 0.4271 - val_accuracy: 0.8032 - val_loss: 0.5797\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8278 - loss: 0.4656 - val_accuracy: 0.8041 - val_loss: 0.5611\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8327 - loss: 0.4433 - val_accuracy: 0.8041 - val_loss: 0.6119\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8256 - loss: 0.4597 - val_accuracy: 0.8191 - val_loss: 0.5819\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8470 - loss: 0.4104 - val_accuracy: 0.8238 - val_loss: 0.6196\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8337 - loss: 0.4364 - val_accuracy: 0.8060 - val_loss: 0.6021\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8337 - loss: 0.4269 - val_accuracy: 0.8088 - val_loss: 0.6492\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:54:29,580] Trial 15 finished with value: 0.7848788969824153 and parameters: {'n_layers': 1, 'n_units_l0': 178, 'dropout_rate': 0.49357739532310296, 'learning_rate': 0.001976978759382274}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 19ms/step - accuracy: 0.6666 - loss: 1.4074 - val_accuracy: 0.7619 - val_loss: 0.7255\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7680 - loss: 0.6751 - val_accuracy: 0.8107 - val_loss: 0.5810\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7772 - loss: 0.6381 - val_accuracy: 0.8069 - val_loss: 0.5243\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7770 - loss: 0.6343 - val_accuracy: 0.7985 - val_loss: 0.6200\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7917 - loss: 0.6017 - val_accuracy: 0.8088 - val_loss: 0.5983\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8057 - loss: 0.5642 - val_accuracy: 0.8088 - val_loss: 0.5704\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7829 - loss: 0.5832 - val_accuracy: 0.7957 - val_loss: 0.6495\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7968 - loss: 0.5999 - val_accuracy: 0.8069 - val_loss: 0.6131\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8015 - loss: 0.5655 - val_accuracy: 0.8069 - val_loss: 0.5880\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7874 - loss: 0.5746 - val_accuracy: 0.8097 - val_loss: 0.6789\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7931 - loss: 0.5872 - val_accuracy: 0.8191 - val_loss: 0.5998\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7973 - loss: 0.5560 - val_accuracy: 0.8022 - val_loss: 0.6555\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7986 - loss: 0.5613 - val_accuracy: 0.8004 - val_loss: 0.6379\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8094 - loss: 0.5468 - val_accuracy: 0.7938 - val_loss: 0.7227\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8148 - loss: 0.5615 - val_accuracy: 0.8126 - val_loss: 0.6142\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7942 - loss: 0.5471 - val_accuracy: 0.8126 - val_loss: 0.6530\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8009 - loss: 0.5954 - val_accuracy: 0.8116 - val_loss: 0.6243\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8182 - loss: 0.5308 - val_accuracy: 0.7966 - val_loss: 0.6603\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8174 - loss: 0.5124 - val_accuracy: 0.7919 - val_loss: 0.6749\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8032 - loss: 0.5377 - val_accuracy: 0.7910 - val_loss: 0.7018\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:54:38,319] Trial 16 finished with value: 0.7633267435076767 and parameters: {'n_layers': 1, 'n_units_l0': 71, 'dropout_rate': 0.4199948135611185, 'learning_rate': 0.0050907310362635055}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 22ms/step - accuracy: 0.6832 - loss: 1.2286 - val_accuracy: 0.8060 - val_loss: 0.6111\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7744 - loss: 0.6778 - val_accuracy: 0.7882 - val_loss: 0.6263\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7958 - loss: 0.5805 - val_accuracy: 0.8154 - val_loss: 0.5707\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8172 - loss: 0.5251 - val_accuracy: 0.7994 - val_loss: 0.5837\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7983 - loss: 0.5703 - val_accuracy: 0.8191 - val_loss: 0.5542\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8271 - loss: 0.4848 - val_accuracy: 0.8266 - val_loss: 0.5371\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8207 - loss: 0.4990 - val_accuracy: 0.8163 - val_loss: 0.5401\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8248 - loss: 0.4574 - val_accuracy: 0.8144 - val_loss: 0.5716\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8207 - loss: 0.4879 - val_accuracy: 0.8238 - val_loss: 0.5683\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8401 - loss: 0.4340 - val_accuracy: 0.8116 - val_loss: 0.5808\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8281 - loss: 0.4576 - val_accuracy: 0.8107 - val_loss: 0.5795\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8326 - loss: 0.4527 - val_accuracy: 0.8079 - val_loss: 0.5796\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8463 - loss: 0.4055 - val_accuracy: 0.8032 - val_loss: 0.6059\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8391 - loss: 0.4302 - val_accuracy: 0.8257 - val_loss: 0.5566\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8488 - loss: 0.4035 - val_accuracy: 0.8229 - val_loss: 0.5821\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8531 - loss: 0.3880 - val_accuracy: 0.8163 - val_loss: 0.5985\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8533 - loss: 0.3927 - val_accuracy: 0.8276 - val_loss: 0.5804\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8447 - loss: 0.4033 - val_accuracy: 0.8060 - val_loss: 0.6062\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8513 - loss: 0.3955 - val_accuracy: 0.8238 - val_loss: 0.6164\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8485 - loss: 0.3994 - val_accuracy: 0.8182 - val_loss: 0.6263\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:54:47,672] Trial 17 finished with value: 0.7853987521094159 and parameters: {'n_layers': 1, 'n_units_l0': 182, 'dropout_rate': 0.4738943213518657, 'learning_rate': 0.0016217646407937147}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 47ms/step - accuracy: 0.6512 - loss: 1.2626 - val_accuracy: 0.7994 - val_loss: 0.6519\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7533 - loss: 0.7977 - val_accuracy: 0.7985 - val_loss: 0.6209\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7651 - loss: 0.7594 - val_accuracy: 0.7685 - val_loss: 0.7074\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7517 - loss: 0.7783 - val_accuracy: 0.8079 - val_loss: 0.6881\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7510 - loss: 0.7864 - val_accuracy: 0.7826 - val_loss: 0.7533\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7706 - loss: 0.7835 - val_accuracy: 0.7498 - val_loss: 0.7649\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7447 - loss: 0.8389 - val_accuracy: 0.7601 - val_loss: 0.7104\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7538 - loss: 0.7719 - val_accuracy: 0.7694 - val_loss: 0.7036\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7551 - loss: 0.7889 - val_accuracy: 0.7554 - val_loss: 0.6806\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7506 - loss: 0.7537 - val_accuracy: 0.7470 - val_loss: 0.7789\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7487 - loss: 0.7817 - val_accuracy: 0.7966 - val_loss: 0.7749\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7502 - loss: 0.8518 - val_accuracy: 0.7985 - val_loss: 0.6847\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7597 - loss: 0.7981 - val_accuracy: 0.7826 - val_loss: 0.7147\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7332 - loss: 0.8369 - val_accuracy: 0.7704 - val_loss: 0.7160\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7533 - loss: 0.7908 - val_accuracy: 0.7732 - val_loss: 0.7497\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7490 - loss: 0.8001 - val_accuracy: 0.7788 - val_loss: 0.7735\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7590 - loss: 0.7658 - val_accuracy: 0.7432 - val_loss: 0.7422\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7602 - loss: 0.7481 - val_accuracy: 0.7404 - val_loss: 0.8942\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7590 - loss: 0.7598 - val_accuracy: 0.7573 - val_loss: 0.7117\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7526 - loss: 0.7451 - val_accuracy: 0.7685 - val_loss: 0.7584\n",
      "\u001b[1m 1/42\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m48s\u001b[0m 1s/step"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-07 02:55:02.695642: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_25', 4 bytes spill stores, 4 bytes spill loads\n",
      "\n",
      "2024-08-07 02:55:02.924265: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_25', 32 bytes spill stores, 32 bytes spill loads\n",
      "\n",
      "2024-08-07 02:55:03.030161: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_25', 84 bytes spill stores, 84 bytes spill loads\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 48ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:55:04,049] Trial 18 finished with value: 0.7267600849095486 and parameters: {'n_layers': 3, 'n_units_l0': 204, 'n_units_l1': 383, 'n_units_l2': 511, 'dropout_rate': 0.40580632576895614, 'learning_rate': 0.0034580856439597497}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 16ms/step - accuracy: 0.6631 - loss: 1.7285 - val_accuracy: 0.8135 - val_loss: 0.7503\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7637 - loss: 0.7854 - val_accuracy: 0.8041 - val_loss: 0.6198\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7686 - loss: 0.6798 - val_accuracy: 0.7938 - val_loss: 0.6679\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7828 - loss: 0.6681 - val_accuracy: 0.8088 - val_loss: 0.6604\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7790 - loss: 0.6549 - val_accuracy: 0.7966 - val_loss: 0.6028\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7794 - loss: 0.6437 - val_accuracy: 0.8154 - val_loss: 0.5883\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7800 - loss: 0.6575 - val_accuracy: 0.8219 - val_loss: 0.5833\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7775 - loss: 0.6571 - val_accuracy: 0.7844 - val_loss: 0.7500\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7798 - loss: 0.7148 - val_accuracy: 0.7957 - val_loss: 0.6596\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7864 - loss: 0.6386 - val_accuracy: 0.7938 - val_loss: 0.7028\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7779 - loss: 0.6565 - val_accuracy: 0.8247 - val_loss: 0.6623\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7756 - loss: 0.6524 - val_accuracy: 0.7948 - val_loss: 0.6340\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7996 - loss: 0.6053 - val_accuracy: 0.7976 - val_loss: 0.7361\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7900 - loss: 0.6364 - val_accuracy: 0.8051 - val_loss: 0.6953\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7935 - loss: 0.6387 - val_accuracy: 0.7938 - val_loss: 0.7371\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7824 - loss: 0.6527 - val_accuracy: 0.8154 - val_loss: 0.6485\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7867 - loss: 0.6310 - val_accuracy: 0.8219 - val_loss: 0.6378\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7890 - loss: 0.7176 - val_accuracy: 0.7844 - val_loss: 0.7454\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7944 - loss: 0.6362 - val_accuracy: 0.7985 - val_loss: 0.8103\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8094 - loss: 0.5912 - val_accuracy: 0.8069 - val_loss: 0.7383\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 9ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-07 02:55:11,931] Trial 19 finished with value: 0.7743571467374116 and parameters: {'n_layers': 1, 'n_units_l0': 131, 'dropout_rate': 0.4608146201791311, 'learning_rate': 0.004806784092794364}. Best is trial 12 with value: 0.7961813683060185.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_layers': 2, 'n_units_l0': 511, 'n_units_l1': 505, 'dropout_rate': 0.4639565318456878, 'learning_rate': 0.0004228409406485573}\n",
      "Epoch 1/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 22ms/step - accuracy: 0.6513 - loss: 1.1039 - val_accuracy: 0.8004 - val_loss: 0.5610\n",
      "Epoch 2/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7728 - loss: 0.6734 - val_accuracy: 0.8144 - val_loss: 0.5343\n",
      "Epoch 3/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.7959 - loss: 0.5881 - val_accuracy: 0.7938 - val_loss: 0.5542\n",
      "Epoch 4/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8081 - loss: 0.5458 - val_accuracy: 0.8163 - val_loss: 0.5225\n",
      "Epoch 5/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8145 - loss: 0.5169 - val_accuracy: 0.8116 - val_loss: 0.5206\n",
      "Epoch 6/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8113 - loss: 0.5279 - val_accuracy: 0.8247 - val_loss: 0.5012\n",
      "Epoch 7/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8365 - loss: 0.4783 - val_accuracy: 0.8276 - val_loss: 0.5276\n",
      "Epoch 8/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8183 - loss: 0.4795 - val_accuracy: 0.8266 - val_loss: 0.4954\n",
      "Epoch 9/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8203 - loss: 0.4794 - val_accuracy: 0.8257 - val_loss: 0.5007\n",
      "Epoch 10/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8243 - loss: 0.4728 - val_accuracy: 0.8182 - val_loss: 0.5101\n",
      "Epoch 11/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8343 - loss: 0.4470 - val_accuracy: 0.8210 - val_loss: 0.5157\n",
      "Epoch 12/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8338 - loss: 0.4639 - val_accuracy: 0.8097 - val_loss: 0.5247\n",
      "Epoch 13/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8387 - loss: 0.4237 - val_accuracy: 0.8247 - val_loss: 0.5140\n",
      "Epoch 14/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8369 - loss: 0.4310 - val_accuracy: 0.8219 - val_loss: 0.5171\n",
      "Epoch 15/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8415 - loss: 0.4114 - val_accuracy: 0.8172 - val_loss: 0.5356\n",
      "Epoch 16/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8443 - loss: 0.4080 - val_accuracy: 0.8257 - val_loss: 0.5167\n",
      "Epoch 17/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8361 - loss: 0.4045 - val_accuracy: 0.8294 - val_loss: 0.5138\n",
      "Epoch 18/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8404 - loss: 0.4004 - val_accuracy: 0.8135 - val_loss: 0.5243\n",
      "Epoch 19/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8680 - loss: 0.3662 - val_accuracy: 0.8163 - val_loss: 0.5444\n",
      "Epoch 20/20\n",
      "\u001b[1m134/134\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - accuracy: 0.8498 - loss: 0.3769 - val_accuracy: 0.8313 - val_loss: 0.5349\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.7863 - loss: 0.5817\n",
      "Test Accuracy: 0.7899\n",
      "\u001b[1m42/42\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 8ms/step \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test F1 Score: 0.7863\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import optuna\n",
    "import tensorflow as tf\n",
    "import joblib\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn.metrics import f1_score\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Input\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "class NeuralNetwork:\n",
    "    def __init__(self, input_shape, layers, dropout_rate, learning_rate, num_classes, device):\n",
    "        self.input_shape = input_shape\n",
    "        self.layers = layers\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.learning_rate = learning_rate\n",
    "        self.num_classes = num_classes\n",
    "        self.device = device\n",
    "        self.model = self.build_model()\n",
    "\n",
    "    def build_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Input(shape=(self.input_shape,)))\n",
    "        for layer_size in self.layers:\n",
    "            model.add(Dense(layer_size, activation='relu'))\n",
    "            model.add(Dropout(self.dropout_rate))\n",
    "        model.add(Dense(self.num_classes, activation='softmax'))\n",
    "        return model\n",
    "\n",
    "    def compile_model(self):\n",
    "        optimizer = Adam(learning_rate=self.learning_rate)\n",
    "        self.model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    def train_model(self, X_train, y_train, epochs=20, batch_size=32, validation_split=0.2):\n",
    "        with tf.device(self.device):\n",
    "            history = self.model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=validation_split)\n",
    "        return history\n",
    "\n",
    "    def evaluate_model(self, X_test, y_test):\n",
    "        with tf.device(self.device):\n",
    "            loss, accuracy = self.model.evaluate(X_test, y_test)\n",
    "        return loss, accuracy\n",
    "\n",
    "    def predict(self, X_test):\n",
    "        with tf.device(self.device):\n",
    "            predictions = self.model.predict(X_test)\n",
    "        return np.argmax(predictions, axis=1)\n",
    "\n",
    "    def calculate_f1(self, y_test, predictions):\n",
    "        return f1_score(np.argmax(y_test, axis=1), predictions, average='weighted')\n",
    "    \n",
    "    def save_model(self, filename):\n",
    "        self.model.save(filename)\n",
    "\n",
    "    @classmethod\n",
    "    def load_model(cls, filename, input_shape, num_classes, device):\n",
    "        loaded_model = tf.keras.models.load_model(filename)\n",
    "        nn = cls(input_shape, [], 0, 0, num_classes, device)  # Dummy values for layers, dropout_rate, and learning_rate\n",
    "        nn.model = loaded_model\n",
    "        return nn\n",
    "\n",
    "\n",
    "def objective(trial):\n",
    "    layers = []\n",
    "    for i in range(trial.suggest_int('n_layers', 1, 3)):\n",
    "        layers.append(trial.suggest_int(f'n_units_l{i}', 64, 512))\n",
    "    \n",
    "    dropout_rate = trial.suggest_float('dropout_rate', 0.2, 0.5)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-2)\n",
    "    \n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate, num_classes=num_classes)\n",
    "    nn.compile_model()\n",
    "    \n",
    "    nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    \n",
    "    predictions = nn.predict(X_test)\n",
    "    f1 = nn.calculate_f1(y_test, predictions)\n",
    "    \n",
    "    return f1\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "# Assuming combined_gdf is already loaded\n",
    "    X = gdf_results['embeddings'].tolist()\n",
    "    X = pd.DataFrame(np.squeeze(X))\n",
    "    y = gdf_results['most_common_lulc']\n",
    "\n",
    "    # Encode the target variable\n",
    "    label_encoder = LabelEncoder()\n",
    "    y_encoded = label_encoder.fit_transform(y)\n",
    "    num_classes = len(label_encoder.classes_)\n",
    "\n",
    "    # Convert to one-hot encoding\n",
    "    y_onehot = to_categorical(y_encoded)\n",
    "\n",
    "    # Split data into training and testing sets\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_onehot, test_size=0.2, random_state=42)\n",
    "\n",
    "    # Scale the features\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n",
    "\n",
    "    # Optimize the hyperparameters\n",
    "    study = optuna.create_study(direction='maximize')\n",
    "    study.optimize(objective, n_trials=20)\n",
    "\n",
    "    # Print the best hyperparameters\n",
    "    print(study.best_params)\n",
    "\n",
    "    # Example usage with the best hyperparameters\n",
    "    best_params = study.best_params\n",
    "    layers = [best_params[f'n_units_l{i}'] for i in range(best_params['n_layers'])]\n",
    "    dropout_rate = best_params['dropout_rate']\n",
    "    learning_rate = best_params['learning_rate']\n",
    "\n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate, num_classes=num_classes)\n",
    "    nn.compile_model()\n",
    "    history = nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    loss, accuracy = nn.evaluate_model(X_test, y_test)\n",
    "    print(f'Test Accuracy: {accuracy:.4f}')\n",
    "    predictions = nn.predict(X_test)\n",
    "    f1 = nn.calculate_f1(y_test, predictions)\n",
    "    print(f'Test F1 Score: {f1:.4f}')\n",
    "\n",
    "    # Save the model\n",
    "    nn.save_model('models/land_cover_model.h5')\n",
    "\n",
    "    # Save scaler and label_encoder\n",
    "    joblib.dump(scaler, 'models/scaler.joblib')\n",
    "    joblib.dump(label_encoder, 'models/label_encoder.joblib')\n",
    "\n",
    "    # If you want to get the actual class labels\n",
    "    predicted_classes = label_encoder.inverse_transform(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 336/336 [04:35<00:00,  1.22it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lon</th>\n",
       "      <th>lat</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>most_common_lulc</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-106.080869</td>\n",
       "      <td>35.424211</td>\n",
       "      <td>[[0.040777754, -0.022916876, 0.07173511, 0.078...</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-106.08087 35.42421)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-106.052673</td>\n",
       "      <td>35.424460</td>\n",
       "      <td>[[0.03793185, 0.009665264, 0.10715726, 0.04229...</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-106.05267 35.42446)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-106.024477</td>\n",
       "      <td>35.424703</td>\n",
       "      <td>[[0.025163846, 0.013742316, 0.124567054, 0.047...</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-106.02448 35.42470)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-105.996280</td>\n",
       "      <td>35.424939</td>\n",
       "      <td>[[0.0464423, 0.015486966, 0.08057992, 0.064075...</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-105.99628 35.42494)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-105.968084</td>\n",
       "      <td>35.425168</td>\n",
       "      <td>[[0.0056447657, -0.0030673319, 0.074692115, 0....</td>\n",
       "      <td>11</td>\n",
       "      <td>POINT (-105.96808 35.42517)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          lon        lat                                         embeddings  \\\n",
       "0 -106.080869  35.424211  [[0.040777754, -0.022916876, 0.07173511, 0.078...   \n",
       "1 -106.052673  35.424460  [[0.03793185, 0.009665264, 0.10715726, 0.04229...   \n",
       "2 -106.024477  35.424703  [[0.025163846, 0.013742316, 0.124567054, 0.047...   \n",
       "3 -105.996280  35.424939  [[0.0464423, 0.015486966, 0.08057992, 0.064075...   \n",
       "4 -105.968084  35.425168  [[0.0056447657, -0.0030673319, 0.074692115, 0....   \n",
       "\n",
       "   most_common_lulc                     geometry  \n",
       "0                11  POINT (-106.08087 35.42421)  \n",
       "1                11  POINT (-106.05267 35.42446)  \n",
       "2                11  POINT (-106.02448 35.42470)  \n",
       "3                11  POINT (-105.99628 35.42494)  \n",
       "4                11  POINT (-105.96808 35.42517)  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "from shapely.geometry import Point\n",
    "import pystac_client\n",
    "import stackstac\n",
    "import torch\n",
    "from torchvision import transforms as v2\n",
    "from box import Box\n",
    "import yaml\n",
    "import math\n",
    "from rasterio.enums import Resampling\n",
    "from tqdm import tqdm\n",
    "import rasterio\n",
    "import warnings\n",
    "import os\n",
    "import numpy as np\n",
    "import rioxarray  # Make sure to import rioxarray to extend xarray\n",
    "\n",
    "from src.model import ClayMAEModule\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "STAC_API = \"https://earth-search.aws.element84.com/v1\"\n",
    "COLLECTION = \"sentinel-2-l2a\"\n",
    "\n",
    "# Load the model and metadata\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "ckpt = \"https://clay-model-ckpt.s3.amazonaws.com/v0.5.7/mae_v0.5.7_epoch-13_val-loss-0.3098.ckpt\"\n",
    "torch.set_default_device(device)\n",
    "\n",
    "torch.cuda.empty_cache()  # Clear GPU cache\n",
    "\n",
    "# Assuming grid is a GeoDataFrame with the points\n",
    "points = grid.to_crs(\"EPSG:4326\").geometry.apply(lambda x: (x.x, x.y)).tolist()\n",
    "\n",
    "model = ClayMAEModule.load_from_checkpoint(\n",
    "    ckpt, metadata_path=\"configs/metadata.yaml\", shuffle=False, mask_ratio=0\n",
    ")\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "\n",
    "metadata = Box(yaml.safe_load(open(\"configs/metadata.yaml\")))\n",
    "\n",
    "# Function to normalize timestamp\n",
    "def normalize_timestamp(date):\n",
    "    week = date.isocalendar().week * 2 * np.pi / 52\n",
    "    hour = date.hour * 2 * np.pi / 24\n",
    "    return (math.sin(week), math.cos(week)), (math.sin(hour), math.cos(hour))\n",
    "\n",
    "# Function to normalize lat/lon\n",
    "def normalize_latlon(lat, lon):\n",
    "    lat = lat * np.pi / 180\n",
    "    lon = lon * np.pi / 180\n",
    "    return (math.sin(lat), math.cos(lat)), (math.sin(lon), math.cos(lon))\n",
    "\n",
    "def to_device(data, device):\n",
    "    if isinstance(data, torch.Tensor):\n",
    "        return data.to(device)\n",
    "    elif isinstance(data, dict):\n",
    "        return {k: to_device(v, device) for k, v in data.items()}\n",
    "    elif isinstance(data, list):\n",
    "        return [to_device(v, device) for v in data]\n",
    "    return data\n",
    "\n",
    "def process_point(lon, lat, model, metadata, year, device, j):\n",
    "    model.to(device)  # Ensure the model is on the correct device\n",
    "    catalog = pystac_client.Client.open(STAC_API)\n",
    "    search = catalog.search(\n",
    "        collections=[COLLECTION],\n",
    "        datetime=f\"{year}-01-01/{year}-12-31\",\n",
    "        bbox=(lon - 1e-5, lat - 1e-5, lon + 1e-5, lat + 1e-5),\n",
    "        max_items=10,\n",
    "        query={\"eo:cloud_cover\": {\"lt\": 80}},\n",
    "    )\n",
    "\n",
    "    all_items = search.get_all_items()\n",
    "    items = list(all_items)\n",
    "    if not items:\n",
    "        return None\n",
    "    \n",
    "    items = sorted(items, key=lambda x: x.properties.get('eo:cloud_cover', float('inf')))\n",
    "    lowest_cloud_item = items[0]\n",
    "\n",
    "    epsg = lowest_cloud_item.properties[\"proj:epsg\"]\n",
    "\n",
    "    poidf = gpd.GeoDataFrame(\n",
    "        pd.DataFrame(),\n",
    "        crs=\"EPSG:4326\",\n",
    "        geometry=[Point(lon, lat)],\n",
    "    ).to_crs(epsg)\n",
    "\n",
    "    coords = poidf.iloc[0].geometry.coords[0]\n",
    "\n",
    "    size = 256\n",
    "    gsd = 10\n",
    "    bounds = (\n",
    "        coords[0] - (size * gsd) // 2,\n",
    "        coords[1] - (size * gsd) // 2,\n",
    "        coords[0] + (size * gsd) // 2,\n",
    "        coords[1] + (size * gsd) // 2,\n",
    "    )\n",
    "\n",
    "    stack = stackstac.stack(\n",
    "        lowest_cloud_item,\n",
    "        bounds=bounds,\n",
    "        snap_bounds=False,\n",
    "        epsg=epsg,\n",
    "        resolution=gsd,\n",
    "        dtype=\"float32\",\n",
    "        rescale=False,\n",
    "        fill_value=0,\n",
    "        assets=[\"blue\", \"green\", \"red\", \"nir\"],\n",
    "        resampling=Resampling.nearest,\n",
    "    )\n",
    "\n",
    "    stack = stack.compute()\n",
    "\n",
    "    items = []\n",
    "    dates = []\n",
    "    for item in all_items:\n",
    "        if item.datetime.date() not in dates:\n",
    "            items.append(item)\n",
    "            dates.append(item.datetime.date())\n",
    "\n",
    "    date = str(stack.time.values)[2:11]\n",
    "\n",
    "    output_path = os.path.join(\"test_data/embeddings/challenge_1/\", f\"stack_{lon}_{lat}_{j}.tif\")\n",
    "    \n",
    "    # Write the stack to a TIFF file\n",
    "    with rasterio.open(\n",
    "            output_path, 'w',\n",
    "            driver='GTiff',\n",
    "            height=stack.shape[2],\n",
    "            width=stack.shape[3],\n",
    "            count=len(stack.band),  # Number of bands\n",
    "            dtype=str(stack.dtype),\n",
    "            crs=epsg,\n",
    "            transform=stack.rio.transform()\n",
    "        ) as tif:\n",
    "        for i, band in enumerate(stack.band, start=1):\n",
    "            tif.write(np.squeeze(stack.sel(band=band).values), i)\n",
    "\n",
    "    # Reopen the file to add metadata\n",
    "    with rasterio.open(output_path, \"r+\") as rst:\n",
    "        rst.update_tags(date=date)\n",
    "\n",
    "    platform = \"sentinel-2-l2a\"\n",
    "    mean = []\n",
    "    std = []\n",
    "    waves = []\n",
    "    for band in stack.band:\n",
    "        mean.append(metadata[platform].bands.mean[str(band.values)])\n",
    "        std.append(metadata[platform].bands.std[str(band.values)])\n",
    "        waves.append(metadata[platform].bands.wavelength[str(band.values)])\n",
    "\n",
    "    transform = v2.Compose([v2.Normalize(mean=mean, std=std)])\n",
    "\n",
    "    datetimes = stack.time.values.astype(\"datetime64[s]\").tolist()\n",
    "    times = [normalize_timestamp(dat) for dat in datetimes]\n",
    "    week_norm = [dat[0] for dat in times]\n",
    "    hour_norm = [dat[1] for dat in times]\n",
    "\n",
    "    latlons = [normalize_latlon(lat, lon)] * len(times)\n",
    "    lat_norm = [dat[0] for dat in latlons]\n",
    "    lon_norm = [dat[1] for dat in latlons]\n",
    "\n",
    "    pixels = torch.from_numpy(stack.data.astype(np.float32)).to(device)\n",
    "    pixels = transform(pixels)\n",
    "\n",
    "    batch_size = 16\n",
    "    num_batches = math.ceil(len(stack) / batch_size)\n",
    "    \n",
    "    embeddings_list = []\n",
    "    for i in range(num_batches):\n",
    "        start_idx = i * batch_size\n",
    "        end_idx = min((i + 1) * batch_size, len(stack))\n",
    "        \n",
    "        batch_pixels = pixels[start_idx:end_idx].to(device)\n",
    "        batch_time = torch.tensor(np.hstack((week_norm, hour_norm))[start_idx:end_idx], dtype=torch.float32).to(device)\n",
    "        batch_latlon = torch.tensor(np.hstack((lat_norm, lon_norm))[start_idx:end_idx], dtype=torch.float32).to(device)\n",
    "        \n",
    "        batch_datacube = {\n",
    "            \"platform\": platform,\n",
    "            \"time\": batch_time,\n",
    "            \"latlon\": batch_latlon,\n",
    "            \"pixels\": batch_pixels,\n",
    "            \"gsd\": torch.tensor(stack.gsd.values).to(device),\n",
    "            \"waves\": torch.tensor(waves).to(device),\n",
    "        }\n",
    "\n",
    "        batch_datacube = to_device(batch_datacube, device)\n",
    "\n",
    "        try:\n",
    "            model = model.to(device)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                unmsk_patch, _, _, _ = model.model.encoder(batch_datacube)\n",
    "            batch_embeddings = unmsk_patch[:, 0, :].cpu().numpy()\n",
    "            embeddings_list.append(batch_embeddings)\n",
    "        except RuntimeError as e:\n",
    "            if \"out of memory\" in str(e):\n",
    "                print(f\"GPU OOM for point ({lon}, {lat}), batch {i+1}/{num_batches}. Trying CPU...\")\n",
    "                device = torch.device(\"cpu\")\n",
    "                batch_datacube = to_device(batch_datacube, device)\n",
    "                model = model.to(device)\n",
    "                with torch.no_grad():\n",
    "                    unmsk_patch, _, _, _ = model.model.encoder(batch_datacube)\n",
    "                batch_embeddings = unmsk_patch[:, 0, :].numpy()\n",
    "                embeddings_list.append(batch_embeddings)\n",
    "                device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "            else:\n",
    "                raise e\n",
    "\n",
    "    embeddings = np.concatenate(embeddings_list, axis=0)\n",
    "    return embeddings\n",
    "\n",
    "# Specify the year for the datetime range in the search\n",
    "year = 2022\n",
    "\n",
    "# Store results in a list\n",
    "results = []\n",
    "\n",
    "# Iterate through the points and process each one\n",
    "for i, point in enumerate(tqdm(points)):\n",
    "    lon, lat = point\n",
    "    embeddings = process_point(lon, lat, model, metadata, year, device, i)\n",
    "    if embeddings is not None:\n",
    "        results.append((lon, lat, embeddings, grid.loc[i, 'most_common_lulc']))\n",
    "\n",
    "# Create a DataFrame from the results\n",
    "df = pd.DataFrame(results, columns=[\"lon\", \"lat\", \"embeddings\", \"most_common_lulc\"])\n",
    "\n",
    "# Convert to a GeoDataFrame\n",
    "gdf_results = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.lon, df.lat))\n",
    "\n",
    "# Output the resulting GeoDataFrame\n",
    "gdf_results.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m11/11\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 141ms/step\n"
     ]
    }
   ],
   "source": [
    "# Detect if GPU is available\n",
    "device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n",
    "\n",
    "# Load the model\n",
    "loaded_nn = NeuralNetwork.load_model('models/land_cover_model.h5', input_shape=768, num_classes=11, device=device)\n",
    "\n",
    "# Prepare your new data (assuming it's in the same format as your training data)\n",
    "new_data = np.squeeze(gdf_results['embeddings'].tolist())\n",
    "new_data = pd.DataFrame(new_data)  # Ensure the new data is in DataFrame format\n",
    "\n",
    "# Standardize the new data using the saved scaler\n",
    "scaler = joblib.load('models/scaler.joblib')\n",
    "new_data_scaled = scaler.transform(new_data)\n",
    "\n",
    "# Make predictions\n",
    "new_predictions = loaded_nn.predict(new_data_scaled)\n",
    "\n",
    "# Load the label encoder\n",
    "label_encoder = joblib.load('models/label_encoder.joblib')\n",
    "\n",
    "# If you want to convert the predictions back to the original class labels\n",
    "gdf_results['pred_lulc'] = label_encoder.inverse_transform(new_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.7232142857142857\n"
     ]
    }
   ],
   "source": [
    "print(\"Test set accuracy:\", np.mean(gdf_results['pred_lulc']==gdf_results['most_common_lulc']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1.0\n",
       "1      1.0\n",
       "2      1.0\n",
       "3      1.0\n",
       "4      1.0\n",
       "      ... \n",
       "331    1.0\n",
       "332    1.0\n",
       "333    1.0\n",
       "334    1.0\n",
       "335    1.0\n",
       "Name: pred_lulc, Length: 336, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf_results['pred_lulc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ignore everything after here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make predictions for the given grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:01<00:00,  1.08s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>embeddings</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-0.10595816, 0.049139716, -0.013616198, 0.103...</td>\n",
       "      <td>POLYGON ((-106.06225 35.59601, -106.06225 35.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[-0.11944625, 0.04765699, 0.0012937918, 0.0989...</td>\n",
       "      <td>POLYGON ((-106.04359 35.59601, -106.04359 35.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.1352622, 0.03729467, 0.025111463, 0.089445...</td>\n",
       "      <td>POLYGON ((-106.06225 35.61468, -106.06225 35.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.11955425, 0.047935113, 0.012492391, 0.0986...</td>\n",
       "      <td>POLYGON ((-106.04359 35.63335, -106.04359 35.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[-0.110916734, 0.043892846, 0.00046651432, 0.1...</td>\n",
       "      <td>POLYGON ((-106.04359 35.61468, -106.04359 35.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635</th>\n",
       "      <td>[-0.10154818, 0.04700841, -0.058602504, 0.1337...</td>\n",
       "      <td>POLYGON ((-105.48359 35.76401, -105.48359 35.7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>[-0.10063921, 0.0487299, -0.056069173, 0.12934...</td>\n",
       "      <td>POLYGON ((-105.48359 35.46535, -105.48359 35.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>[-0.119740516, 0.050605718, -0.021452673, 0.12...</td>\n",
       "      <td>POLYGON ((-105.48359 35.78268, -105.48359 35.8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>[-0.10471448, 0.04887444, -0.05049323, 0.11969...</td>\n",
       "      <td>POLYGON ((-105.48359 35.44668, -105.48359 35.4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>[-0.09382948, 0.051646754, -0.042943284, 0.118...</td>\n",
       "      <td>POLYGON ((-105.48359 35.42801, -105.48359 35.4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>640 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            embeddings  \\\n",
       "0    [-0.10595816, 0.049139716, -0.013616198, 0.103...   \n",
       "1    [-0.11944625, 0.04765699, 0.0012937918, 0.0989...   \n",
       "2    [-0.1352622, 0.03729467, 0.025111463, 0.089445...   \n",
       "3    [-0.11955425, 0.047935113, 0.012492391, 0.0986...   \n",
       "4    [-0.110916734, 0.043892846, 0.00046651432, 0.1...   \n",
       "..                                                 ...   \n",
       "635  [-0.10154818, 0.04700841, -0.058602504, 0.1337...   \n",
       "636  [-0.10063921, 0.0487299, -0.056069173, 0.12934...   \n",
       "637  [-0.119740516, 0.050605718, -0.021452673, 0.12...   \n",
       "638  [-0.10471448, 0.04887444, -0.05049323, 0.11969...   \n",
       "639  [-0.09382948, 0.051646754, -0.042943284, 0.118...   \n",
       "\n",
       "                                              geometry  \n",
       "0    POLYGON ((-106.06225 35.59601, -106.06225 35.6...  \n",
       "1    POLYGON ((-106.04359 35.59601, -106.04359 35.6...  \n",
       "2    POLYGON ((-106.06225 35.61468, -106.06225 35.6...  \n",
       "3    POLYGON ((-106.04359 35.63335, -106.04359 35.6...  \n",
       "4    POLYGON ((-106.04359 35.61468, -106.04359 35.6...  \n",
       "..                                                 ...  \n",
       "635  POLYGON ((-105.48359 35.76401, -105.48359 35.7...  \n",
       "636  POLYGON ((-105.48359 35.46535, -105.48359 35.4...  \n",
       "637  POLYGON ((-105.48359 35.78268, -105.48359 35.8...  \n",
       "638  POLYGON ((-105.48359 35.44668, -105.48359 35.4...  \n",
       "639  POLYGON ((-105.48359 35.42801, -105.48359 35.4...  \n",
       "\n",
       "[640 rows x 2 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "import s3fs\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "\n",
    "# S3 bucket and prefix\n",
    "s3_bucket = 'clay-worker-bucket-dev-small-tasks'\n",
    "s3_prefix = '_data/gpq/86/' #2022 S2 embeddings over test set\n",
    "\n",
    "# Initialize s3fs filesystem\n",
    "fs = s3fs.S3FileSystem()\n",
    "\n",
    "# List all the GeoParquet files in the specified S3 directory\n",
    "geo_parquet_files = fs.glob(f's3://{s3_bucket}/{s3_prefix}*.gpq')\n",
    "\n",
    "# List to store GeoDataFrames\n",
    "gdfs = []\n",
    "\n",
    "# Load each GeoParquet file into a GeoDataFrame and append to the list\n",
    "for file in tqdm(geo_parquet_files):\n",
    "    gdf = gpd.read_parquet(\"s3://\"+file, storage_options={\"anon\": False, \"client_kwargs\": {\"region_name\": \"us-west-2\"}})\n",
    "    gdfs.append(gdf)\n",
    "\n",
    "# Concatenate all GeoDataFrames into a single GeoDataFrame\n",
    "gdf_test_embed = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True))\n",
    "\n",
    "# Display the concatenated GeoDataFrame\n",
    "gdf_test_embed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing bounding boxes: 100%|██████████| 640/640 [00:16<00:00, 38.75it/s]\n"
     ]
    }
   ],
   "source": [
    "# Initialize Earth Engine\n",
    "ee.Initialize()\n",
    "\n",
    "# Load the ESRI Global LULC dataset\n",
    "lulc = ee.ImageCollection(\"projects/sat-io/open-datasets/landcover/ESRI_Global-LULC_10m_TS\").mosaic()\n",
    "\n",
    "def get_most_common_lulc(geometry):\n",
    "    # Convert the GeoPandas geometry to an Earth Engine geometry\n",
    "    ee_geometry = ee.Geometry.Rectangle(geometry.bounds)\n",
    "    \n",
    "    # Get the LULC values within the bounding box\n",
    "    lulc_values = lulc.reduceRegion(\n",
    "        reducer=ee.Reducer.frequencyHistogram(),\n",
    "        geometry=ee_geometry,\n",
    "        scale=10,\n",
    "        maxPixels=1e9\n",
    "    ).get('b1')\n",
    "    \n",
    "    # Find the most common LULC value\n",
    "    lulc_dict = ee.Dictionary(lulc_values)\n",
    "    most_common = lulc_dict.keys().sort(lulc_dict.values()).get(-1)\n",
    "    \n",
    "    # Return the result\n",
    "    return int(most_common.getInfo())\n",
    "\n",
    "def process_geometries(combined_gdf):\n",
    "    # Get the total number of rows in the GeoDataFrame\n",
    "    total_rows = len(combined_gdf)\n",
    "\n",
    "    # Determine the number of threads to use\n",
    "    max_threads = 10  # Adjust this based on your system and Earth Engine quota\n",
    "    num_threads = min(total_rows, max_threads)\n",
    "\n",
    "    results = []\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_threads) as executor:\n",
    "        # Submit all tasks\n",
    "        future_to_index = {executor.submit(get_most_common_lulc, row.geometry): index \n",
    "                        for index, row in combined_gdf.iterrows()}\n",
    "        \n",
    "        # Process as they complete with a progress bar\n",
    "        with tqdm(total=total_rows, desc=\"Processing bounding boxes\") as pbar:\n",
    "            for future in as_completed(future_to_index):\n",
    "                index = future_to_index[future]\n",
    "                try:\n",
    "                    result = future.result()\n",
    "                except Exception as exc:\n",
    "                    print(f'Generated an exception: {exc}')\n",
    "                    result = None\n",
    "                results.append((index, result))\n",
    "                pbar.update(1)\n",
    "\n",
    "    # Sort results by index and extract only the values\n",
    "    sorted_results = [r[1] for r in sorted(results, key=lambda x: x[0])]\n",
    "    \n",
    "    return sorted_results\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # Process the geometries\n",
    "    results = process_geometries(gdf_test_embed.to_crs(\"EPSG:4326\"))\n",
    "\n",
    "    # Add the results as a new column to the GeoDataFrame\n",
    "    gdf_test_embed['most_common_lulc'] = results\n",
    "\n",
    "    # Save as GeoJSON\n",
    "    gdf_test_embed[['geometry', 'most_common_lulc']].to_file(\"test_lulc_v2.geojson\", driver=\"GeoJSON\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m20/20\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 758us/step\n"
     ]
    }
   ],
   "source": [
    "# Detect if GPU is available\n",
    "device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n",
    "\n",
    "# Load the model\n",
    "loaded_nn = NeuralNetwork.load_model('models/land_cover_model.h5', input_shape=768, num_classes=11, device=device)\n",
    "\n",
    "# Prepare your new data (assuming it's in the same format as your training data)\n",
    "new_data = gdf_test_embed['embeddings'].tolist()\n",
    "new_data = pd.DataFrame(new_data)  # Ensure the new data is in DataFrame format\n",
    "\n",
    "# Standardize the new data using the saved scaler\n",
    "scaler = joblib.load('models/scaler.joblib')\n",
    "new_data_scaled = scaler.transform(new_data)\n",
    "\n",
    "# Make predictions\n",
    "new_predictions = loaded_nn.predict(new_data_scaled)\n",
    "\n",
    "# Load the label encoder\n",
    "label_encoder = joblib.load('models/label_encoder.joblib')\n",
    "\n",
    "# If you want to convert the predictions back to the original class labels\n",
    "gdf_test_embed['pred_lulc'] = label_encoder.inverse_transform(new_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.6546875\n"
     ]
    }
   ],
   "source": [
    "print(\"Test set accuracy:\", np.mean(gdf_test_embed['pred_lulc']==gdf_test_embed['most_common_lulc']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 3: Above ground stock regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve S2 embeddings from an S3 bucket over California"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "import s3fs\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "file_path = 'models/agb_regression_model.h5'\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "    # S3 bucket and prefix\n",
    "    s3_bucket = 'clay-worker-bucket-dev-small-tasks'\n",
    "    s3_prefix = '_data/gpq/51/' #2022 S2 Cali embeddings\n",
    "\n",
    "    # Initialize s3fs filesystem\n",
    "    fs = s3fs.S3FileSystem()\n",
    "\n",
    "    # List all the GeoParquet files in the specified S3 directory\n",
    "    geo_parquet_files = fs.glob(f's3://{s3_bucket}/{s3_prefix}*.gpq')\n",
    "\n",
    "    # List to store GeoDataFrames\n",
    "    gdfs = []\n",
    "\n",
    "    # Load each GeoParquet file into a GeoDataFrame and append to the list\n",
    "    for file in tqdm(geo_parquet_files):\n",
    "        gdf = gpd.read_parquet(\"s3://\"+file, storage_options={\"anon\": False, \"client_kwargs\": {\"region_name\": \"us-west-2\"}})\n",
    "        gdfs.append(gdf)\n",
    "\n",
    "    # Concatenate all GeoDataFrames into a single GeoDataFrame\n",
    "    combined_gdf = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True))\n",
    "\n",
    "    # Display the concatenated GeoDataFrame\n",
    "    combined_gdf\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join above ground biomass for each chip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import ee\n",
    "import geopandas as gpd\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import os\n",
    "\n",
    "if os.path.exists(file_path):\n",
    "    print(\"Model already exists\")\n",
    "    combined_gdf = gpd.read_file(\"agb.geojson\")\n",
    "else:\n",
    "    # Initialize Earth Engine\n",
    "    ee.Initialize()\n",
    "\n",
    "    # Load the NASA/ORNL biomass carbon density dataset\n",
    "    biomass = ee.ImageCollection(\"NASA/ORNL/biomass_carbon_density/v1\").mosaic()\n",
    "\n",
    "    def get_agb(geometry):\n",
    "        # Convert the GeoPandas geometry to an Earth Engine geometry\n",
    "        ee_geometry = ee.Geometry.Polygon(list(geometry.exterior.coords))\n",
    "\n",
    "        # Get the mean AGB value within the geometry\n",
    "        agb_value = biomass.reduceRegion(\n",
    "            reducer=ee.Reducer.mean(),\n",
    "            geometry=ee_geometry,\n",
    "            scale=300,\n",
    "            maxPixels=1e9\n",
    "        ).get('agb')\n",
    "        \n",
    "        # Return the result\n",
    "        return agb_value.getInfo() if agb_value is not None else None\n",
    "\n",
    "    def process_geometries(combined_gdf):\n",
    "        # Get the total number of rows in the GeoDataFrame\n",
    "        total_rows = len(combined_gdf)\n",
    "\n",
    "        # Determine the number of threads to use\n",
    "        max_threads = 10  # Adjust this based on your system and Earth Engine quota\n",
    "        num_threads = min(total_rows, max_threads)\n",
    "\n",
    "        results = []\n",
    "\n",
    "        with ThreadPoolExecutor(max_workers=num_threads) as executor:\n",
    "            # Submit all tasks\n",
    "            future_to_index = {executor.submit(get_agb, row.geometry): index \n",
    "                            for index, row in combined_gdf.iterrows()}\n",
    "            \n",
    "            # Process as they complete with a progress bar\n",
    "            with tqdm(total=total_rows, desc=\"Processing geometries\") as pbar:\n",
    "                for future in as_completed(future_to_index):\n",
    "                    index = future_to_index[future]\n",
    "                    try:\n",
    "                        result = future.result()\n",
    "                    except Exception as exc:\n",
    "                        print(f'Generated an exception: {exc}')\n",
    "                        result = None\n",
    "                    results.append((index, result))\n",
    "                    pbar.update(1)\n",
    "\n",
    "        # Sort results by index and extract only the values\n",
    "        sorted_results = [r[1] for r in sorted(results, key=lambda x: x[0])]\n",
    "        \n",
    "        return sorted_results\n",
    "\n",
    "    if __name__ == '__main__':\n",
    "        # Process the geometries\n",
    "        results = process_geometries(combined_gdf)\n",
    "\n",
    "        # Add the results as a new column to the GeoDataFrame\n",
    "        combined_gdf['mean_agb'] = results\n",
    "\n",
    "        # Save as GeoJSON\n",
    "        combined_gdf[['geometry', 'mean_agb']].to_file(\"agb.geojson\", driver=\"GeoJSON\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tune hyperparameters for above ground biomass regressor using Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/claymodel-latest-v2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-07-26 15:29:41.264463: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-07-26 15:29:41.530251: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-07-26 15:29:41.603764: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-07-26 15:29:42.194784: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-07-26 15:29:44.148003: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model already exists\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import optuna\n",
    "import tensorflow as tf\n",
    "import joblib\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Input\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "class NeuralNetwork:\n",
    "    def __init__(self, input_shape, layers, dropout_rate, learning_rate, device):\n",
    "        self.input_shape = input_shape\n",
    "        self.layers = layers\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.learning_rate = learning_rate\n",
    "        self.device = device\n",
    "        self.model = self.build_model()\n",
    "\n",
    "    def build_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Input(shape=(self.input_shape,)))\n",
    "        for layer_size in self.layers:\n",
    "            model.add(Dense(layer_size, activation='relu'))\n",
    "            model.add(Dropout(self.dropout_rate))\n",
    "        model.add(Dense(1, activation='linear'))  # Output layer for regression\n",
    "        return model\n",
    "\n",
    "    def compile_model(self):\n",
    "        optimizer = Adam(learning_rate=self.learning_rate)\n",
    "        self.model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['mae'])  # Use MSE for regression\n",
    "\n",
    "    def train_model(self, X_train, y_train, epochs=20, batch_size=32, validation_split=0.2):\n",
    "        with tf.device(self.device):\n",
    "            history = self.model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=validation_split)\n",
    "        return history\n",
    "\n",
    "    def evaluate_model(self, X_test, y_test):\n",
    "        with tf.device(self.device):\n",
    "            loss, mae = self.model.evaluate(X_test, y_test)\n",
    "        return loss, mae\n",
    "\n",
    "    def predict(self, X_test):\n",
    "        with tf.device(self.device):\n",
    "            predictions = self.model.predict(X_test)\n",
    "        return predictions.flatten()\n",
    "\n",
    "    def calculate_rmse(self, y_test, predictions):\n",
    "        return np.sqrt(mean_squared_error(y_test, predictions))\n",
    "    \n",
    "    def save_model(self, filename):\n",
    "        self.model.save(filename)\n",
    "\n",
    "    @classmethod\n",
    "    def load_model(cls, filename, input_shape, device):\n",
    "        loaded_model = tf.keras.models.load_model(filename)\n",
    "        nn = cls(input_shape, [], 0, 0, device)  # Dummy values for layers, dropout_rate, and learning_rate\n",
    "        nn.model = loaded_model\n",
    "        return nn\n",
    "\n",
    "def objective(trial):\n",
    "    layers = []\n",
    "    for i in range(trial.suggest_int('n_layers', 1, 3)):\n",
    "        layers.append(trial.suggest_int(f'n_units_l{i}', 64, 512))\n",
    "    \n",
    "    dropout_rate = trial.suggest_float('dropout_rate', 0.2, 0.5)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-2)\n",
    "    \n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate)\n",
    "    nn.compile_model()\n",
    "    \n",
    "    nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    \n",
    "    predictions = nn.predict(X_test)\n",
    "    predictions_exp = np.exp(predictions)  # Inverse log transform predictions\n",
    "    rmse = nn.calculate_rmse(y_test_exp, predictions_exp)\n",
    "    \n",
    "    return rmse\n",
    "\n",
    "if os.path.exists('models/agb_regression_model.h5'):\n",
    "    print(\"Model already exists\")\n",
    "else:\n",
    "    # Assuming combined_gdf is already loaded\n",
    "    combined_gdf_filtered = combined_gdf.dropna()\n",
    "    X = combined_gdf_filtered['embeddings'].tolist()\n",
    "    X = pd.DataFrame(X)\n",
    "    y = combined_gdf_filtered['mean_agb']\n",
    "\n",
    "    # Log transform the target variable\n",
    "    y_log = np.log(y + 1)  # Adding 1 to avoid log(0)\n",
    "\n",
    "    # Split data into training and testing sets\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_log, test_size=0.2, random_state=42)\n",
    "\n",
    "    # Scale the features\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    # Inverse transform for evaluation\n",
    "    y_test_exp = np.exp(y_test) - 1  # Subtract 1 to get the original scale\n",
    "\n",
    "    device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n",
    "\n",
    "    # Optimize the hyperparameters\n",
    "    study = optuna.create_study(direction='minimize')\n",
    "    study.optimize(objective, n_trials=5)\n",
    "\n",
    "    # Print the best hyperparameters\n",
    "    print(study.best_params)\n",
    "\n",
    "    # Example usage with the best hyperparameters\n",
    "    best_params = study.best_params\n",
    "    layers = [best_params[f'n_units_l{i}'] for i in range(best_params['n_layers'])]\n",
    "    dropout_rate = best_params['dropout_rate']\n",
    "    learning_rate = best_params['learning_rate']\n",
    "\n",
    "    nn = NeuralNetwork(input_shape=X_train.shape[1], device=device, layers=layers, dropout_rate=dropout_rate, learning_rate=learning_rate)\n",
    "    nn.compile_model()\n",
    "    history = nn.train_model(X_train, y_train, epochs=20, batch_size=32, validation_split=0.2)\n",
    "    loss, mae = nn.evaluate_model(X_test, y_test)\n",
    "    print(f'Test MAE: {mae:.4f}')\n",
    "    predictions = nn.predict(X_test)\n",
    "    predictions_exp = np.exp(predictions) - 1  # Inverse log transform\n",
    "    rmse = nn.calculate_rmse(y_test_exp, predictions_exp)\n",
    "    print(f'Test RMSE: {rmse:.4f}')\n",
    "\n",
    "    # Save the model\n",
    "    nn.save_model('models/agb_regression_model.h5')\n",
    "\n",
    "    # Save scaler\n",
    "    joblib.dump(scaler, 'models/3_scaler.joblib')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>embeddings</th>\n",
       "      <th>geometry</th>\n",
       "      <th>mean_agb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-0.1340062, 0.077923074, -0.0016371261, 0.083...</td>\n",
       "      <td>POLYGON ((-116.13800 34.50819, -116.13800 34.5...</td>\n",
       "      <td>0.336093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[-0.12408799, 0.06986915, -0.0014471954, 0.080...</td>\n",
       "      <td>POLYGON ((-116.15667 34.50819, -116.15667 34.5...</td>\n",
       "      <td>0.315462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.13270076, 0.067874126, -0.0058731856, 0.08...</td>\n",
       "      <td>POLYGON ((-116.13800 34.48952, -116.13800 34.5...</td>\n",
       "      <td>0.308832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.11633471, 0.07077362, -0.009755048, 0.0875...</td>\n",
       "      <td>POLYGON ((-116.13800 34.47086, -116.13800 34.4...</td>\n",
       "      <td>0.263030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[-0.1335075, 0.0719392, -0.0011160049, 0.08034...</td>\n",
       "      <td>POLYGON ((-116.11934 34.48952, -116.11934 34.5...</td>\n",
       "      <td>0.281872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125141</th>\n",
       "      <td>[-0.09712396, 0.06625347, -0.056792237, 0.1275...</td>\n",
       "      <td>POLYGON ((-123.32467 40.51886, -123.32467 40.5...</td>\n",
       "      <td>95.744820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125142</th>\n",
       "      <td>[-0.09359632, 0.06506389, -0.053921893, 0.1259...</td>\n",
       "      <td>POLYGON ((-123.34334 40.53752, -123.34334 40.5...</td>\n",
       "      <td>90.395423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125143</th>\n",
       "      <td>[-0.093709834, 0.06074024, -0.07075437, 0.1278...</td>\n",
       "      <td>POLYGON ((-123.23134 39.32419, -123.23134 39.3...</td>\n",
       "      <td>95.926339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125144</th>\n",
       "      <td>[-0.09378435, 0.059857186, -0.055076838, 0.128...</td>\n",
       "      <td>POLYGON ((-123.45534 39.10019, -123.45534 39.1...</td>\n",
       "      <td>77.902087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125145</th>\n",
       "      <td>[-0.11787722, 0.04442746, -0.058926195, 0.1103...</td>\n",
       "      <td>POLYGON ((-123.23134 40.40686, -123.23134 40.4...</td>\n",
       "      <td>83.995837</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120508 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               embeddings  \\\n",
       "0       [-0.1340062, 0.077923074, -0.0016371261, 0.083...   \n",
       "1       [-0.12408799, 0.06986915, -0.0014471954, 0.080...   \n",
       "2       [-0.13270076, 0.067874126, -0.0058731856, 0.08...   \n",
       "3       [-0.11633471, 0.07077362, -0.009755048, 0.0875...   \n",
       "4       [-0.1335075, 0.0719392, -0.0011160049, 0.08034...   \n",
       "...                                                   ...   \n",
       "125141  [-0.09712396, 0.06625347, -0.056792237, 0.1275...   \n",
       "125142  [-0.09359632, 0.06506389, -0.053921893, 0.1259...   \n",
       "125143  [-0.093709834, 0.06074024, -0.07075437, 0.1278...   \n",
       "125144  [-0.09378435, 0.059857186, -0.055076838, 0.128...   \n",
       "125145  [-0.11787722, 0.04442746, -0.058926195, 0.1103...   \n",
       "\n",
       "                                                 geometry   mean_agb  \n",
       "0       POLYGON ((-116.13800 34.50819, -116.13800 34.5...   0.336093  \n",
       "1       POLYGON ((-116.15667 34.50819, -116.15667 34.5...   0.315462  \n",
       "2       POLYGON ((-116.13800 34.48952, -116.13800 34.5...   0.308832  \n",
       "3       POLYGON ((-116.13800 34.47086, -116.13800 34.4...   0.263030  \n",
       "4       POLYGON ((-116.11934 34.48952, -116.11934 34.5...   0.281872  \n",
       "...                                                   ...        ...  \n",
       "125141  POLYGON ((-123.32467 40.51886, -123.32467 40.5...  95.744820  \n",
       "125142  POLYGON ((-123.34334 40.53752, -123.34334 40.5...  90.395423  \n",
       "125143  POLYGON ((-123.23134 39.32419, -123.23134 39.3...  95.926339  \n",
       "125144  POLYGON ((-123.45534 39.10019, -123.45534 39.1...  77.902087  \n",
       "125145  POLYGON ((-123.23134 40.40686, -123.23134 40.4...  83.995837  \n",
       "\n",
       "[120508 rows x 3 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_gdf_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/claymodel-latest-v2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/opt/conda/envs/claymodel-latest-v2/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import glob\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import requests\n",
    "\n",
    "import geopandas as gpd\n",
    "import lancedb\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pystac_client\n",
    "import shapely\n",
    "from shapely.geometry import box, Polygon\n",
    "import torch\n",
    "import yaml\n",
    "from box import Box\n",
    "from pyproj import Transformer\n",
    "from rasterio.io import MemoryFile\n",
    "from stacchip.chipper import Chipper\n",
    "from stacchip.indexer import Sentinel2Indexer\n",
    "from stacchip.processors.prechip import normalize_timestamp\n",
    "from torchvision.transforms import v2\n",
    "\n",
    "from src.model import ClayMAEModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/claymodel-latest-v2/lib/python3.11/site-packages/pystac_client/item_search.py:851: FutureWarning: get_all_items() is deprecated, use item_collection() instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 23 items\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Optimize GDAL settings for cloud optimized reading\n",
    "os.environ[\"GDAL_DISABLE_READDIR_ON_OPEN\"] = \"EMPTY_DIR\"\n",
    "os.environ[\"AWS_REQUEST_PAYER\"] = \"requester\"\n",
    "\n",
    "STAC_API = \"https://earth-search.aws.element84.com/v1\"\n",
    "COLLECTION = \"sentinel-2-l2a\"\n",
    "YEAR = 2022\n",
    "\n",
    "# Search the catalogue\n",
    "catalog = pystac_client.Client.open(STAC_API)\n",
    "search = catalog.search(\n",
    "    collections=[COLLECTION],\n",
    "    datetime=f\"{YEAR}-01-01T00:00:00Z/{YEAR+1}-01-01T00:00:00Z\",\n",
    "    bbox=[-123.30442043188332, 46.875515943543945, -121.65107564263492, 48.012480708758716],\n",
    "    max_items=100,\n",
    "    query={\"eo:cloud_cover\": {\"lt\": 80}},\n",
    ")\n",
    "\n",
    "all_items = search.get_all_items()\n",
    "\n",
    "# Reduce to one per date (there might be some duplicates\n",
    "# based on the location)\n",
    "items = []\n",
    "dates = []\n",
    "for item in all_items:\n",
    "    if item.datetime.date() not in dates:\n",
    "        items.append(item)\n",
    "        dates.append(item.datetime.date())\n",
    "\n",
    "print(f\"Found {len(items)} items\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on <Item id=S2B_10TDS_20221231_0_L2A>\n",
      "Working on <Item id=S2B_10TDT_20221228_0_L2A>\n",
      "Working on <Item id=S2A_10TFS_20221226_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221221_0_L2A>\n",
      "Working on <Item id=S2B_10TFS_20221218_0_L2A>\n",
      "Working on <Item id=S2A_10TDS_20221216_0_L2A>\n",
      "Working on <Item id=S2A_10TES_20221213_0_L2A>\n",
      "Working on <Item id=S2B_10TDT_20221211_0_L2A>\n",
      "Working on <Item id=S2A_10TFS_20221206_0_L2A>\n",
      "Working on <Item id=S2A_10TES_20221203_0_L2A>\n",
      "Working on <Item id=S2B_10TDT_20221201_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221128_0_L2A>\n",
      "Working on <Item id=S2A_10TDS_20221126_0_L2A>\n",
      "Working on <Item id=S2A_10TDS_20221123_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221121_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221118_0_L2A>\n",
      "Working on <Item id=S2A_10TDS_20221116_0_L2A>\n",
      "Working on <Item id=S2A_10TFS_20221113_0_L2A>\n",
      "Working on <Item id=S2B_10TFS_20221111_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221108_0_L2A>\n",
      "Working on <Item id=S2A_10TDT_20221106_0_L2A>\n",
      "Working on <Item id=S2B_10TFS_20221101_0_L2A>\n",
      "Working on <Item id=S2B_10TDS_20221029_0_L2A>\n"
     ]
    }
   ],
   "source": [
    "chips = []\n",
    "datetimes = []\n",
    "bboxs = []\n",
    "chip_ids = []\n",
    "item_ids = []\n",
    "\n",
    "for item in items:\n",
    "    print(f\"Working on {item}\")\n",
    "\n",
    "    # Index the chips in the item\n",
    "    indexer = Sentinel2Indexer(item)\n",
    "\n",
    "    # Instanciate the chipper\n",
    "    chipper = Chipper(indexer, assets=[\"red\", \"green\", \"blue\", \"nir\", \"scl\"])\n",
    "\n",
    "    # Get first chip for the \"image\" asset key\n",
    "    for idx, (x, y, chip) in enumerate(chipper):\n",
    "        if idx > 2:\n",
    "            break\n",
    "        del chip[\"scl\"]\n",
    "        chips.append(chip)\n",
    "        datetimes.append(item.datetime)\n",
    "        bboxs.append(indexer.get_chip_bbox(x, y))\n",
    "        chip_ids.append((x, y))\n",
    "        item_ids.append(item.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(69, 4, 224, 224)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pixels = np.array([np.array(list(chip.values())).squeeze() for chip in chips])\n",
    "pixels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract mean, std, and wavelengths from metadata\n",
    "platform = \"sentinel-2-l2a\"\n",
    "# Retrieve the file content from the URL\n",
    "\n",
    "url = (\n",
    "    \"https://raw.githubusercontent.com/Clay-foundation/model/main/configs/metadata.yaml\"\n",
    ")\n",
    "response = requests.get(url, allow_redirects=True)\n",
    "\n",
    "# Convert bytes to string\n",
    "content = response.content.decode(\"utf-8\")\n",
    "\n",
    "# Load the yaml\n",
    "content = yaml.safe_load(content)\n",
    "\n",
    "metadata = Box(content)\n",
    "mean = []\n",
    "std = []\n",
    "waves = []\n",
    "# Use the band names to get the correct values in the correct order.\n",
    "for band in chips[0].keys():\n",
    "    mean.append(metadata[platform].bands.mean[band])\n",
    "    std.append(metadata[platform].bands.std[band])\n",
    "    waves.append(metadata[platform].bands.wavelength[band])\n",
    "\n",
    "# Prepare the normalization transform function using the mean and std values.\n",
    "transform = v2.Compose(\n",
    "    [\n",
    "        v2.Normalize(mean=mean, std=std),\n",
    "    ]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "claymodel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}