DATA01_preprocess_GEFS_opt_members.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b97c5d83-9537-4f73-8256-d97a8ff7f2f1",
   "metadata": {},
   "source": [
    "# Operational GEFS preprocessing\n",
    "\n",
    "This notebook converts operatoinal GEFS enemble mean grib2 files into h5 format\n",
    "\n",
    "* `$camp_dir/GFS/geave_y{}.hdf`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f976b7bd-6e84-4652-b7f6-0fc4afa7e594",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import h5py\n",
    "import pygrib\n",
    "\n",
    "import numpy as np\n",
    "# import netCDF4 as nc\n",
    "from glob import glob\n",
    "\n",
    "from scipy.interpolate import RegularGridInterpolator\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "39c5cd2a-015f-47e6-85c7-e2e4852a4f30",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '/glade/u/home/ksha/GAN_proj/')\n",
    "sys.path.insert(0, '/glade/u/home/ksha/GAN_proj/libs/')\n",
    "\n",
    "from namelist import *\n",
    "import data_utils as du"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0312ec81-4653-4dee-90c2-eb20b251089e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cf5408d3-e0eb-4e2b-bb27-b1a5c8227f8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# base = datetime(2021, 1, 1)\n",
    "# date_list = [base + timedelta(days=day) for day in range(365)]\n",
    "\n",
    "base = datetime(2020, 12, 1)\n",
    "date_list = [base + timedelta(days=day) for day in range(31)]\n",
    "\n",
    "LEADs = np.arange(6, 144+6, 6)\n",
    "\n",
    "with h5py.File(save_dir+'CCPA_domain.hdf', 'r') as h5io:\n",
    "    lon_CCPA = h5io['lon_CCPA'][...]\n",
    "    lat_CCPA = h5io['lat_CCPA'][...]\n",
    "\n",
    "lon_CCPA += 360"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a6b408a4-3d05-4edc-b8e1-10134e9ab93d",
   "metadata": {},
   "outputs": [],
   "source": [
    "GEFS_dir = camp_dir+'wget_GEFSv12_members/'\n",
    "filenames = sorted(glob(GEFS_dir+'*f*'))\n",
    "\n",
    "with pygrib.open(filenames[0]) as grbio:\n",
    "    lat_GFS, lon_GFS = grbio[1].latlons()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c463caba-4bfb-498c-8302-9420c5eff760",
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_names = ['gec00',]\n",
    "\n",
    "for i in range(1, 31, 1):\n",
    "    ens_names.append('gep{:02d}'.format(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9920f177-67ff-417c-a552-fd8e9b28f32c",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_shape = lon_CCPA.shape\n",
    "N_days = len(date_list)\n",
    "N_leads = len(LEADs)\n",
    "N_ens = len(ens_names)\n",
    "\n",
    "apcp_save = np.empty((N_leads, N_ens,)+grid_shape); apcp_save[...] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dbfa1a90-b5b0-4f0d-b10b-9a075a4350c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_base = camp_dir+'wget_GEFSv12_members/{}_00_{}.t00z.pgrb2s.0p25.f{:03d}'\n",
    "\n",
    "for i, dt in enumerate(date_list):\n",
    "    dt_str = datetime.strftime(dt, '%Y%m%d')\n",
    "    apcp_save[...] = np.nan\n",
    "    \n",
    "    name_check = camp_dir+'GFS/{}.hdf'.format(dt_str)\n",
    "\n",
    "    if os.path.isfile(name_check) is False:\n",
    "        print('Missing {}'.format(name_check))\n",
    "        # get the current member name\n",
    "        flag_all_member_ready = True\n",
    "        for ilead, lead in enumerate(LEADs):    \n",
    "            for iens, ens_name in enumerate(ens_names):\n",
    "                filename_ = filename_base.format(dt_str, ens_name, lead)\n",
    "                # if this member does not exist, skip\n",
    "                if os.path.isfile(filename_) is False:\n",
    "                    print('Missing {}'.format(filename_))\n",
    "                    flag_all_member_ready = False\n",
    "                    break;\n",
    "                    \n",
    "        #if flag_all_member_ready:\n",
    "        for iens, ens_name in enumerate(ens_names):\n",
    "            for ilead, lead in enumerate(LEADs):\n",
    "                filename_ = filename_base.format(dt_str, ens_name, lead)\n",
    "                try:\n",
    "                    with pygrib.open(filename_) as grbio:\n",
    "                        apcp = grbio[1].values\n",
    "                \n",
    "                    lr_to_hr = RegularGridInterpolator((lat_GFS[:, 0], lon_GFS[0, :]), apcp, \n",
    "                                                       bounds_error=False, fill_value=None)\n",
    "                    apcp_hr0 = lr_to_hr((lat_CCPA, lon_CCPA))\n",
    "                    \n",
    "                    apcp_save[ilead, iens, ...] = apcp_hr0\n",
    "                except:\n",
    "                    print('Empty or failed file: {}'.format(filename_))\n",
    "                    apcp_save[ilead, iens, ...] = np.nan\n",
    "        \n",
    "        # tuple_save = (apcp_save,)\n",
    "        # label_save = ['apcp',]\n",
    "        # du.save_hdf5(tuple_save, label_save, camp_dir+'GFS/', \n",
    "        #              '{}.hdf'.format(dt_str))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c098bda4-21b8-4ce5-a7e9-8298234616bc",
   "metadata": {},
   "source": [
    "## Merge single day files as one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "221c5ae8-2bb8-414d-8507-f80fb28907f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "apcp_all = np.empty((N_days, N_leads, N_ens)+grid_shape); apcp_all[...] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5b2c7441-d45a-4287-bb91-0c190008a59c",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_base = camp_dir+'wget_GEFSv12_members/{}_00_{}.t00z.pgrb2s.0p25.f{:03d}'\n",
    "\n",
    "for i, dt in enumerate(date_list):\n",
    "    \n",
    "    dt_str = datetime.strftime(dt, '%Y%m%d')\n",
    "    print(dt_str)\n",
    "    \n",
    "    name_load = camp_dir+'GFS/{}.hdf'.format(dt_str)\n",
    "    with h5py.File(name_load, 'r') as h5io:\n",
    "        apcp_all[i, ...] = h5io['apcp'][...]\n",
    "            \n",
    "# tuple_save = (apcp_all,)\n",
    "# label_save = ['apcp',]\n",
    "# du.save_hdf5(tuple_save, label_save, camp_dir+'GFS/', \n",
    "#              'GEFS_OPT_MEMBERS_2020_DEC.hdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "167774f6-ce43-427f-8fd0-8878c465bc1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "with h5py.File(camp_dir+'GFS/GEFS_OPT_MEMBERS_2020_DEC.hdf', 'r') as h5io:\n",
    "    apcp = h5io['apcp'][...]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8154879f-ec25-4f9b-b919-ac37980d8b98",
   "metadata": {},
   "outputs": [],
   "source": [
    "apcp_mean = np.nanmean(apcp, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41f8822b-6b21-4f42-b95b-5a1825117dd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "apcp_full = np.empty((366, 24, 31, 224, 464)); apcp_full[...] = np.nan\n",
    "apcp_full[-31:, ...] = apcp\n",
    "\n",
    "apcp_mean_full = np.empty((366, 24, 224, 464)); apcp_mean_full[...] = np.nan\n",
    "apcp_mean_full[-31:, ...] = apcp_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3f051c99-8de7-47a7-9273-6b35713c573f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tuple_save = (apcp_full, apcp_mean_full)\n",
    "# label_save = ['apcp', 'apcp_mean']\n",
    "# du.save_hdf5(tuple_save, label_save, camp_dir+'GFS/', \n",
    "#              'GEFS_OPT_MEMBERS_2020_DEC.hdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbca98ce-6040-47cf-bb23-336a35136692",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}