LakeOccurrence.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ericslevenson/arctic-surface-water/blob/main/LakeOccurrence.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u8Bxr0e8HM0Y"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "author: @ericslevenson\n",
        "date: Nov 6, 2022\n",
        "description: Ingest map of buffered lakes, roi, timeframe. Export lake occurrence map\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "raErdXDCB-vO",
        "outputId": "59bf05c4-c845-477f-9589-83e03b87bdc0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "To authorize access needed by Earth Engine, open the following URL in a web browser and follow the instructions. If the web browser does not start automatically, please manually browse the URL below.\n",
            "\n",
            "    https://code.earthengine.google.com/client-auth?scopes=https%3A//www.googleapis.com/auth/earthengine%20https%3A//www.googleapis.com/auth/devstorage.full_control&request_id=tH2cz0Mh8s_rSekw2uBE2QbGe51_FECJK31VpK5gK3Y&tc=FVl1fir87X3_1YfAiULouJUBZE6c8FefpPYHAOHqp9Y&cc=uLay-_qlkJcdXxX5nXBnufFqDuSPCu_VQazI6MimYbs\n",
            "\n",
            "The authorization workflow will generate a code, which you should paste in the box below.\n",
            "Enter verification code: 4/1AbUR2VOluYI6ZSDswa-R3nv1YT5L3sIHe9B0Q5OYwDMr54xU5gwGQeotjP8\n",
            "\n",
            "Successfully saved authorization token.\n",
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Authenticate private account (only required for exporting to drive/gee/gcp)\n",
        "from google.colab import auth \n",
        "auth.authenticate_user()\n",
        "\n",
        "# Earth Engine setup\n",
        "import ee # Trigger the authentication flow.\n",
        "ee.Authenticate()\n",
        "ee.Initialize() # Initialize the library.\n",
        "\n",
        "# Google Drive setup (if needed)\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Some common imports\n",
        "from IPython.display import Image\n",
        "import folium"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "_nb_LpkAC_x3"
      },
      "outputs": [],
      "source": [
        "## ***INPUTS***\n",
        "\n",
        "## Regularly Adjusted Inputs ##\n",
        "\n",
        "# Tile of interest ID\n",
        "id = 63\n",
        "\n",
        "# Timeframe\n",
        "start = '2016-05-22'\n",
        "finish = '2021-09-30'\n",
        "\n",
        "## Periodically Adjusted Inputs ##\n",
        "# Lake Shapefile\n",
        "lakes = ee.FeatureCollection('projects/ee-eric-levenson/assets/OccurrenceMask/uninspected_bufferedLakes_5N_round2')\n",
        "# Tiled ROI\n",
        "tiles = ee.FeatureCollection('projects/ee-eric-levenson/assets/OccurrenceMask/GEE_processingGrid_5N') # TODO: non-overlapping tiles\n",
        "# Export settings\n",
        "directory = 'S2LakeOccurrence' # Folder\n",
        "description = str(id) + '_5N_lakeOccurrence_2016-2021' # filename\n",
        "\n",
        "## Rarely Adjusted Input ##\n",
        "# Image scale\n",
        "pixScale = 10\n",
        "\n",
        "## ***EARTH ENGINE-IFY***\n",
        "eeroi = tiles.filter(ee.Filter.eq('id', id)).first() # Filter grid to tile to define roi\n",
        "roi = ee.Geometry.Polygon(eeroi.geometry().getInfo()['coordinates'][0]) # define roi as geometry variable\n",
        "startDoy = ee.Date(start).getRelative('day', 'year')\n",
        "endDoy = ee.Date(finish).getRelative('day', 'year')\n",
        "eestart = ee.Date(start)\n",
        "eefinish = ee.Date(finish)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2uS89XFHEUSZ"
      },
      "source": [
        "### Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "QVf338acEVwm"
      },
      "outputs": [],
      "source": [
        "## ***IMAGE PRE-PROCESSING METHODS***\n",
        "\n",
        "# Mask clouds in Sentinel-2\n",
        "def maskS2clouds(image):\n",
        "  '''Takes an input and adds two bands: cloud mask and clear mask'''\n",
        "  qa = image.select('QA60')\n",
        "  cloudBitMask = 1 << 10\n",
        "  cirrusBitMask = 1 << 11\n",
        "  clear_mask = qa.bitwiseAnd(cloudBitMask).eq(0).And(qa.bitwiseAnd(cirrusBitMask).eq(0)).rename('clear_mask')\n",
        "  cloud_mask = qa.bitwiseAnd(cloudBitMask).eq(1).And(qa.bitwiseAnd(cirrusBitMask).eq(1)).rename('cloud_mask')\n",
        "  return image.addBands([cloud_mask,clear_mask])\n",
        "\n",
        "# Clip image\n",
        "def clip_image(image):\n",
        "  '''Clips to the roi defined at the beginning of the script'''\n",
        "  return image.clip(roi)\n",
        "\n",
        "def clip2lakes(image):\n",
        "  '''Clips an image based on the lake boundaries'''\n",
        "  return image.clip(lakes)\n",
        "\n",
        "# Get percentile cover   \n",
        "def getCover(image):\n",
        "  '''calculates percentage of the roi covered by the clear mask. NOTE: this function\n",
        "  calls the global totPixels variable that needs to be calculated in the main script.'''\n",
        "  actPixels = ee.Number(image.updateMask(image.select('clear_mask')).reduceRegion(\n",
        "      reducer = ee.Reducer.count(),\n",
        "      scale = 100,\n",
        "      geometry = roi,\n",
        "      maxPixels=1e12,\n",
        "      ).values().get(0))\n",
        "  # calculate the perc of cover OF CLEAR PIXELS \n",
        "  percCover = actPixels.divide(totPixels).multiply(100).round()\n",
        "  # number as output\n",
        "  return image.set('percCover', percCover,'actPixels',actPixels)\n",
        "  \n",
        "# Mosaic images by date, orbit, - basically combines images together that were taken on the same day \n",
        "def mosaicBy(imcol):\n",
        "  '''Takes an image collection (imcol) and creates a mosaic for each day\n",
        "  Returns: An image collection of daily mosaics'''\n",
        "  #return the collection as a list of images (not an image collection)\n",
        "  imlist = imcol.toList(imcol.size())\n",
        "  # Get all the dates as list\n",
        "  def imdate(im):\n",
        "    date = ee.Image(im).date().format(\"YYYY-MM-dd\")\n",
        "    return date\n",
        "  all_dates = imlist.map(imdate)\n",
        "  # get all orbits as list\n",
        "  def orbitId(im):\n",
        "    orb = ee.Image(im).get('SENSING_ORBIT_NUMBER')\n",
        "    return orb\n",
        "  all_orbits = imlist.map(orbitId)\n",
        "  # get all spacecraft names as list\n",
        "  def spacecraft(im):\n",
        "    return ee.Image(im).get('SPACECRAFT_NAME')\n",
        "  all_spNames = imlist.map(spacecraft)\n",
        "  # this puts dates, orbits and names into a nested list\n",
        "  concat_all = all_dates.zip(all_orbits).zip(all_spNames);\n",
        "  # here we unnest the list with flatten, and then concatenate the list elements with \" \"\n",
        "  def concat(el):\n",
        "    return ee.List(el).flatten().join(\" \")\n",
        "  concat_all = concat_all.map(concat)\n",
        "  # here, just get distinct combintations of date, orbit and name\n",
        "  concat_unique = concat_all.distinct()\n",
        "  # mosaic\n",
        "  def mosaicIms(d):\n",
        "    d1 = ee.String(d).split(\" \")\n",
        "    date1 = ee.Date(d1.get(0))\n",
        "    orbit = ee.Number.parse(d1.get(1)).toInt()\n",
        "    spName = ee.String(d1.get(2))\n",
        "    im = imcol.filterDate(date1, date1.advance(1, \"day\")).filterMetadata('SPACECRAFT_NAME', 'equals', spName).filterMetadata('SENSING_ORBIT_NUMBER','equals', orbit).mosaic()\n",
        "    return im.set(\n",
        "        \"system:time_start\", date1.millis(),\n",
        "        \"system:date\", date1.format(\"YYYY-MM-dd\"),\n",
        "        \"system:id\", d1)\n",
        "  mosaic_imlist = concat_unique.map(mosaicIms)\n",
        "  return ee.ImageCollection(mosaic_imlist)\n",
        "\n",
        "def reprojectMosaic(image):\n",
        "  '''Reproject to UTM. A future function should take the image location and return\n",
        "  the UTM zone. For now, I'm manually entering the EPSG code.'''\n",
        "  image_projected = image.reproject(epsg)\n",
        "  return image_projected\n",
        "\n",
        "###########################################################################\n",
        "## ***WATER CLASSIFICATION METHODS***\n",
        "\n",
        "# Define NDWI image\n",
        "def ndwi(image):\n",
        "  '''Adds an NDWI band to the input image'''\n",
        "  return image.normalizedDifference(['B3', 'B8']).rename('NDWI').multiply(1000)\n",
        "\n",
        "# Basic ndwi classification  \n",
        "def ndwi_classify(image):\n",
        "  '''Creates a binary image based on an NDWI threshold of 0'''\n",
        "  ndwimask = image.select('NDWI')\n",
        "  water = ndwimask.gte(0)\n",
        "  land = ndwimask.lt(0)\n",
        "  return(water)\n",
        "\n",
        "# OTSU thresholding from histogram\n",
        "def otsu(histogram):\n",
        "  '''Returns the NDWI threshold for binary water classification'''\n",
        "  counts = ee.Array(ee.Dictionary(histogram).get('histogram'))\n",
        "  means = ee.Array(ee.Dictionary(histogram).get('bucketMeans'))\n",
        "  size = means.length().get([0])\n",
        "  total = counts.reduce(ee.Reducer.sum(), [0]).get([0])\n",
        "  sum = means.multiply(counts).reduce(ee.Reducer.sum(), [0]).get([0])\n",
        "  mean = sum.divide(total)\n",
        "  indices = ee.List.sequence(1, size)\n",
        "  def func_xxx(i):\n",
        "    '''Compute between sum of squares, where each mean partitions the data.'''\n",
        "    aCounts = counts.slice(0, 0, i)\n",
        "    aCount = aCounts.reduce(ee.Reducer.sum(), [0]).get([0])\n",
        "    aMeans = means.slice(0, 0, i)\n",
        "    aMean = aMeans.multiply(aCounts) \\\n",
        "        .reduce(ee.Reducer.sum(), [0]).get([0]) \\\n",
        "        .divide(aCount)\n",
        "    bCount = total.subtract(aCount)\n",
        "    bMean = sum.subtract(aCount.multiply(aMean)).divide(bCount)\n",
        "    return aCount.multiply(aMean.subtract(mean).pow(2)).add(\n",
        "           bCount.multiply(bMean.subtract(mean).pow(2)))\n",
        "  bss = indices.map(func_xxx)\n",
        "  # Return the mean value corresponding to the maximum BSS.\n",
        "  return means.sort(bss).get([-1])\n",
        "\n",
        "# OTSU thresholding for an image\n",
        "def otsu_thresh(water_image):\n",
        "  '''Calculate NDWI and create histogram. Return the OTSU threshold.'''\n",
        "  NDWI = ndwi(water_image).select('NDWI').updateMask(water_image.select('clear_mask'))\n",
        "  histogram = ee.Dictionary(NDWI.reduceRegion(\n",
        "    geometry = roi,\n",
        "    reducer = ee.Reducer.histogram(255, 2).combine('mean', None, True).combine('variance', None, True),\n",
        "    scale = pixScale,\n",
        "    maxPixels = 1e12\n",
        "  ))\n",
        "  return otsu(histogram.get('NDWI_histogram'))\n",
        "\n",
        "# Classify an image using OTSU threshold.\n",
        "def otsu_classify(water_image):\n",
        "  '''(1) Calculate NDWI and create histogram. (2) Calculate NDWI threshold for \n",
        "  binary classification using OTSU method. (3) Classify image and add layer to input image.\n",
        "  '''\n",
        "  NDWI = ndwi(water_image).select('NDWI')\n",
        "  histogram = ee.Dictionary(NDWI.reduceRegion(\n",
        "    geometry = roi,\n",
        "    reducer = ee.Reducer.histogram(255, 2).combine('mean', None, True).combine('variance', None, True),\n",
        "    scale = pixScale,\n",
        "    maxPixels = 1e12\n",
        "  ))\n",
        "  threshold = otsu(histogram.get('NDWI_histogram'))\n",
        "  otsu_classed = NDWI.gt(ee.Number(threshold)).And(water_image.select('B8').lt(2000)).rename('otsu_classed')\n",
        "  return water_image.addBands([otsu_classed])\n",
        "\n",
        "def adaptive_thresholding(water_image):\n",
        "  '''Takes an image clipped to lakes and returns the water mask'''\n",
        "  NDWI = ndwi(water_image).select('NDWI')#.updateMask(water_image.select('clear_mask')) # get NDWI **TURNED OFF CLOUD MASK, SHOULD THIS STAY OFF?**\n",
        "  threshold = ee.Number(otsu_thresh(water_image)) \n",
        "  threshold = threshold.divide(10).round().multiply(10)\n",
        "  # get fixed histogram\n",
        "  histo = NDWI.reduceRegion(\n",
        "      geometry = roi,\n",
        "      reducer = ee.Reducer.fixedHistogram(-1000, 1000, 200),\n",
        "      scale = pixScale, # This was 30, keep at 10!?!?\n",
        "      maxPixels = 1e12\n",
        "  )\n",
        "  hist = ee.Array(histo.get('NDWI'))\n",
        "  counts = hist.cut([-1,1])\n",
        "  buckets = hist.cut([-1,0])\n",
        "  #find split points from otsu threshold\n",
        "  threshold = ee.Array([threshold]).toList()\n",
        "  buckets_list = buckets.toList()\n",
        "  split = buckets_list.indexOf(threshold)\n",
        "  # split into land and water slices\n",
        "  land_slice = counts.slice(0,0,split)\n",
        "  water_slice = counts.slice(0,split.add(1),-1)\n",
        "  # find max of land and water slices\n",
        "  land_max = land_slice.reduce(ee.Reducer.max(),[0])\n",
        "  water_max = water_slice.reduce(ee.Reducer.max(),[0])\n",
        "  land_max = land_max.toList().get(0)\n",
        "  water_max = water_max.toList().get(0)\n",
        "  land_max = ee.List(land_max).getNumber(0)\n",
        "  water_max = ee.List(water_max).getNumber(0)\n",
        "  #find difference between land, water and otsu val\n",
        "  counts_list = counts.toList()\n",
        "  otsu_val = ee.Number(counts_list.get(split))\n",
        "  otsu_val = ee.List(otsu_val).getNumber(0)\n",
        "  land_prom = ee.Number(land_max).subtract(otsu_val)\n",
        "  water_prom = ee.Number(water_max).subtract(otsu_val)\n",
        "  #find land and water buckets corresponding to 0.9 times the prominence\n",
        "  land_thresh = ee.Number(land_max).subtract((land_prom).multiply(ee.Number(0.9)))\n",
        "  water_thresh = ee.Number(water_max).subtract((water_prom).multiply(ee.Number(0.9)))\n",
        "  land_max_ind = land_slice.argmax().get(0)\n",
        "  water_max_ind = water_slice.argmax().get(0)\n",
        "  li = ee.Number(land_max_ind).subtract(1)\n",
        "  li = li.max(ee.Number(1))\n",
        "  wi = ee.Number(water_max_ind).add(1)\n",
        "  wi = wi.min(ee.Number(199))\n",
        "  land_slice2 = land_slice.slice(0,li,-1).subtract(land_thresh)\n",
        "  water_slice2 = water_slice.slice(0,0,wi).subtract(water_thresh)\n",
        "  land_slice2  = land_slice2.abs().multiply(-1)\n",
        "  water_slice2 = water_slice2.abs().multiply(-1)\n",
        "  land_index = ee.Number(land_slice2.argmax().get(0)).add(land_max_ind)\n",
        "  water_index = ee.Number(water_slice2.argmax().get(0)).add(split)\n",
        "  land_level = ee.Number(buckets_list.get(land_index))\n",
        "  water_level = ee.Number(buckets_list.get(water_index))\n",
        "  land_level = ee.Number(ee.List(land_level).get(0)).add(5)\n",
        "  water_level = ee.Number(ee.List(water_level).get(0)).add(5)\n",
        "  #calculate water fraction and classify\n",
        "  water_fraction = (NDWI.subtract(land_level)).divide(water_level.subtract(land_level)).multiply(100).rename('water_fraction')\n",
        "  #water_fraction = conditional(water_fraction) #sets values less than 0 to 0 and greater than 100 to 100\n",
        "  water_75 = water_fraction.gte(75).rename('water_75'); #note, this is a non-binary classification, so we use 75% water as \"water\"\n",
        "  all_mask = water_image.select('B2').gt(5).rename('all_mask')\n",
        "  cloud_mask_ed = water_image.select('clear_mask').neq(1).rename('cloud_mask_ed')\n",
        "  return water_image.addBands([water_fraction,water_75,NDWI,cloud_mask_ed])\n",
        "# Apply cloud mask to other bands\n",
        "def applyMask(image):\n",
        "  img = image.updateMask(image.select('clear_mask'))\n",
        "  return img\n",
        "def binaryImage(image):\n",
        "  '''takes a multiband image and returns just the binary water_75 band'''\n",
        "  img = image.select('water_75')\n",
        "  return img\n",
        "def waterImage(image):\n",
        "  '''takes a multiband image and returns just the water fraction band'''\n",
        "  img = image.select('water_fraction')\n",
        "  return img\n",
        "###############################################################################\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M17JcGSkEsUc"
      },
      "source": [
        "### Main"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jk6QSMoVEtWq",
        "outputId": "e3278e7e-b8f9-437a-8747-68c1ada42cb3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n",
            "Polling for task (id: PLHQI3ST4J7R3RIY2EZB4GJ2).\n"
          ]
        }
      ],
      "source": [
        "## ***Lake Occurrence Raster***\n",
        "\n",
        "# (1) image pre-process:\n",
        "\n",
        "# Get images and filter images by cloudiness, roi, time period, and month range\n",
        "images = ee.ImageCollection('COPERNICUS/S2').filterBounds(roi).filterDate(start,finish).filter(ee.Filter.calendarRange(startDoy, endDoy, 'day_of_year')).filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE',50)) \n",
        "# Mosaic images and add cloud/clear masks\n",
        "images_all = mosaicBy(images)\n",
        "images_all = images_all.map(maskS2clouds)\n",
        "# Clip mosaics to roi\n",
        "images_all = images_all.map(clip_image)\n",
        "# Get percent cover for each mosaic\n",
        "image_mask = images.select('B2').mean().clip(roi).gte(0) #First, calculate total number of pixels\n",
        "totPixels = ee.Number(image_mask.reduceRegion(\n",
        "    reducer = ee.Reducer.count(),\n",
        "    scale = 100,\n",
        "    geometry = roi,\n",
        "    maxPixels = 1e12\n",
        "    ).values().get(0))\n",
        "images_all = images_all.map(getCover)\n",
        "# Filter by percent cover\n",
        "images = images_all.filterMetadata('percCover','greater_than',70) # remove images covering less than 50% of the ROI)\n",
        "# Clip remaining mosaics to buffered lake shapefile\n",
        "lakeimages = images.map(clip2lakes) # Clip images to buffered lake mask\n",
        "\n",
        "# (2) water classification:\n",
        "\n",
        "# Apply cloud mask to other bands\n",
        "lakeimages = lakeimages.map(applyMask) \n",
        "# adaptive thresholding on every lake image\n",
        "lakeimages = lakeimages.map(adaptive_thresholding)\n",
        "# create image collection of just binary water images\n",
        "lakeimages = lakeimages.map(binaryImage)\n",
        "\n",
        "# (3) Lake occurrence for entire date range (2016-2021)\n",
        "\n",
        "# reduce image collection to mean value\n",
        "lakeimage = lakeimages.reduce(ee.Reducer.mean()) # this accounts for masked pixels\n",
        "\n",
        "# (4) Export lake occurrence mask to drive\n",
        "task = ee.batch.Export.image.toDrive(**{\n",
        "    'image': lakeimage,\n",
        "    'description': str(id) + '_5N_lakeOccurrence_2016-2021',\n",
        "    'folder': directory,\n",
        "    'fileFormat': 'GeoTIFF',\n",
        "    'scale': 10,\n",
        "    'region': roi,\n",
        "    'maxPixels': 1e12\n",
        "})\n",
        "task.start()\n",
        "import time \n",
        "while task.active():\n",
        "  print('Polling for task (id: {}).'.format(task.id))\n",
        "  time.sleep(5)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "2uS89XFHEUSZ"
      ],
      "provenance": [],
      "authorship_tag": "ABX9TyNMEMz2Q9LuX2JsCL8o3VrM",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}