DATA00_wget_GEFSv12_reforecast.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a0d95b56-6124-4f90-8183-c17f90af1aa1",
   "metadata": {},
   "source": [
    "# GEFS reforecast wget cmd generator\n",
    "\n",
    "This notebook generates qsub scripts to download GEFS reforecasts\n",
    "* Source: https://noaa-gefs-retrospective.s3.amazonaws.com/GEFSv12/reforecast/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4e88911-1838-477d-934a-391f92d0d53f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6817238-9ace-4100-9a67-57c0ad373611",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '/glade/u/home/ksha/GAN_proj/')\n",
    "from namelist import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae3f28b0-53c0-4c8f-8210-37ee183d9e36",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = ['c00', 'p01', 'p02', 'p03', 'p04']\n",
    "# /Days:1-10\n",
    "vars = ['apcp_sfc_{}_{}.grib2',]# 'cape_sfc_{}_{}.grib2', 'pwat_eatm_{}_{}.grib2', \n",
    "             #'tmp_2m_{}_{}.grib2', 'spfh_2m_{}_{}.grib2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "78220459-ed92-45ce-9ab8-c18fa4c3a000",
   "metadata": {},
   "outputs": [],
   "source": [
    "base = datetime(2002, 1, 1)\n",
    "date_list = [base + timedelta(days=day) for day in range(365*18+4)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64025dab-5d2f-477e-ad2e-63221f6841f4",
   "metadata": {},
   "source": [
    "## Fill-in-the-gap version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "14921191-abd5-4e15-ae37-ea6780b4db41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if files exist\n",
    "date_list_fill = []\n",
    "\n",
    "for dt in date_list:\n",
    "    \n",
    "    dt_str = datetime.strftime(dt, '%Y%m%d00')\n",
    "    filename = camp_dir+'wget_GEFS_reforecast/{}_done.txt'.format(dt_str)\n",
    "    \n",
    "    if os.path.isfile(filename):\n",
    "        continue;\n",
    "    else:\n",
    "        date_list_fill.append(dt)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ca3c5879-bfe7-4a8b-b25a-a313f8075231",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "need to download 0 files\n"
     ]
    }
   ],
   "source": [
    "L_fill = len(date_list_fill)\n",
    "print('need to download {} files'.format(L_fill))\n",
    "#date_list_fill = date_list_fill[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5af283c6-43d3-452f-b4b4-d9bb758661bd",
   "metadata": {},
   "source": [
    "## Generate qsub scripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2146c7b7-fa47-487e-8ff2-206ea4fb25d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = int(L_fill/4) + 1\n",
    "\n",
    "index = np.arange(0, L_fill, N)\n",
    "\n",
    "for i_, i_start in enumerate(index):\n",
    "\n",
    "    f = open(work_dir+'qsub/wget_reforecast_{:03d}.sh'.format(i_), 'w') \n",
    "    \n",
    "    heads = '''\n",
    "    #!/bin/bash -l\n",
    "    \n",
    "    #PBS -N wget_reforecast\n",
    "    #PBS -A NAML0001\n",
    "    #PBS -l walltime=23:59:59\n",
    "    #PBS -l select=1:ncpus=4:mem=12GB\n",
    "    #PBS -q casper\n",
    "    #PBS -o wget_reforecast.log\n",
    "    #PBS -e wget_reforecast.err\n",
    "    \n",
    "    cd {}wget_GEFS_reforecast/\n",
    "    \n",
    "    '''.format(camp_dir)\n",
    "    \n",
    "    print(heads, file=f)\n",
    "    \n",
    "    for i in range(i_start, i_start+N, 1):\n",
    "        if i < L_fill:\n",
    "            dt = date_list_fill[i]\n",
    "            dt_str = datetime.strftime(dt, '%Y%m%d%H')\n",
    "            year = dt.year\n",
    "            \n",
    "            for member in keys:\n",
    "                for var in vars:\n",
    "                    var_name = var.format(dt_str, member)\n",
    "                    download_link = 'https://noaa-gefs-retrospective.s3.amazonaws.com/GEFSv12/reforecast/{}/{}/{}/Days%3A1-10/{}'.format(\n",
    "                        year, dt_str, member, var_name)\n",
    "                    \n",
    "                    save_name = var_name\n",
    "                    \n",
    "                    print('wget -L -O {} {}'.format(save_name, download_link), file=f)\n",
    "                    \n",
    "            print('touch {}_done.txt'.format(dt_str), file=f)\n",
    "    \n",
    "    f.close()\n",
    "\n",
    "f = open(work_dir+'qsub/wget_reforecast_all.sh', 'w')\n",
    "\n",
    "for i_, i_start in enumerate(index):\n",
    "    print('qsub wget_reforecast_{:03d}.sh'.format(i_), file=f)\n",
    "    \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "282a4c16-d2ed-46eb-832d-64e3d18eb241",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}