RXCADRE_DATALOADER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JIDt2Lf9RWLl"
   },
   "source": [
    "# RxCadre Dataset Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AHGN6nU9RcyF"
   },
   "source": [
    "Our goal is to be able to extract information from the RxCadre datasets to mesh it with an external NASA dataset. This is an introductory notebook to give some foundation to this procedure."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ei2YzfeZR9JD"
   },
   "source": [
    "## RxCadre Ignition Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kYyBsrHoR_gH"
   },
   "source": [
    "https://www.fs.usda.gov/rds/archive/catalog/RDS-2017-0065\n",
    "\n",
    "This dataset actually describes how the ignition devices move so that we can test to see how we can join this table with the NASA one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ejYSQNgPRPMf",
    "outputId": "f44a10ad-0dc2-4eb9-f0fc-0636123e6947"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: geopandas in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (0.10.2)\n",
      "Requirement already satisfied: pandas>=0.25.0 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from geopandas) (1.2.3)\n",
      "Requirement already satisfied: fiona>=1.8 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from geopandas) (1.8.21)\n",
      "Requirement already satisfied: pyproj>=2.2.0 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from geopandas) (3.2.1)\n",
      "Requirement already satisfied: shapely>=1.6 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from geopandas) (1.8.2)\n",
      "Requirement already satisfied: certifi in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2021.10.8)\n",
      "Requirement already satisfied: click>=4.0 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (7.1.2)\n",
      "Requirement already satisfied: cligj>=0.5 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n",
      "Requirement already satisfied: six>=1.7 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.16.0)\n",
      "Requirement already satisfied: munch in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (2.5.0)\n",
      "Requirement already satisfied: click-plugins>=1.0 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n",
      "Requirement already satisfied: setuptools in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (52.0.0.post20210125)\n",
      "Requirement already satisfied: attrs>=17 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from fiona>=1.8->geopandas) (20.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (2021.3)\n",
      "Requirement already satisfied: numpy>=1.16.5 in /Users/ritikbatra/opt/anaconda3/lib/python3.7/site-packages (from pandas>=0.25.0->geopandas) (1.21.4)\n"
     ]
    }
   ],
   "source": [
    "# We will first be installing geopandas to be able to read the gdb files that come from RxCadre\n",
    "!pip3 install geopandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TEbOaxmYR4hR",
    "outputId": "e545b505-aa68-472e-dac9-fdc907835b05"
   },
   "outputs": [],
   "source": [
    "# import statements\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n",
    "import geopandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 511
    },
    "id": "9EsjsbCjScUh",
    "outputId": "e769ac44-99b1-4b02-e682-2b631060c8cc"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ALTITUDE</th>\n",
       "      <th>EASTING</th>\n",
       "      <th>NORTHING</th>\n",
       "      <th>UTCDATE</th>\n",
       "      <th>UTCTIME</th>\n",
       "      <th>SOG</th>\n",
       "      <th>COG</th>\n",
       "      <th>SATS_USED</th>\n",
       "      <th>HDOP</th>\n",
       "      <th>VDOP</th>\n",
       "      <th>PDOP</th>\n",
       "      <th>QUALITY</th>\n",
       "      <th>DIFF_AGE</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30.525149</td>\n",
       "      <td>-86.736444</td>\n",
       "      <td>18.5</td>\n",
       "      <td>525284.544</td>\n",
       "      <td>3377008.024</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183122</td>\n",
       "      <td>0.1</td>\n",
       "      <td>141.800003</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>POINT Z (525284.544 3377008.024 18.500)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>30.525151</td>\n",
       "      <td>-86.736447</td>\n",
       "      <td>18.4</td>\n",
       "      <td>525284.224</td>\n",
       "      <td>3377008.208</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183123</td>\n",
       "      <td>0.1</td>\n",
       "      <td>18.200001</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>POINT Z (525284.224 3377008.208 18.400)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>30.525158</td>\n",
       "      <td>-86.736455</td>\n",
       "      <td>18.3</td>\n",
       "      <td>525283.423</td>\n",
       "      <td>3377008.945</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183124</td>\n",
       "      <td>2.1</td>\n",
       "      <td>304.100006</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>POINT Z (525283.423 3377008.944 18.300)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>30.525164</td>\n",
       "      <td>-86.736469</td>\n",
       "      <td>18.1</td>\n",
       "      <td>525282.142</td>\n",
       "      <td>3377009.680</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183125</td>\n",
       "      <td>5.6</td>\n",
       "      <td>303.600006</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>POINT Z (525282.142 3377009.680 18.100)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>30.525169</td>\n",
       "      <td>-86.736482</td>\n",
       "      <td>18.0</td>\n",
       "      <td>525280.861</td>\n",
       "      <td>3377010.231</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183126</td>\n",
       "      <td>4.9</td>\n",
       "      <td>301.000000</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>POINT Z (525280.861 3377010.231 18.000)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OBJECTID   LATITUDE  LONGITUDE  ALTITUDE     EASTING     NORTHING  \\\n",
       "0         1  30.525149 -86.736444      18.5  525284.544  3377008.024   \n",
       "1         2  30.525151 -86.736447      18.4  525284.224  3377008.208   \n",
       "2         3  30.525158 -86.736455      18.3  525283.423  3377008.945   \n",
       "3         4  30.525164 -86.736469      18.1  525282.142  3377009.680   \n",
       "4         5  30.525169 -86.736482      18.0  525280.861  3377010.231   \n",
       "\n",
       "                     UTCDATE UTCTIME  SOG         COG  SATS_USED  HDOP  VDOP  \\\n",
       "0  2012-11-04T00:00:00+00:00  183122  0.1  141.800003          8   1.1   1.6   \n",
       "1  2012-11-04T00:00:00+00:00  183123  0.1   18.200001          8   1.1   1.6   \n",
       "2  2012-11-04T00:00:00+00:00  183124  2.1  304.100006          8   1.1   1.6   \n",
       "3  2012-11-04T00:00:00+00:00  183125  5.6  303.600006          8   1.1   1.6   \n",
       "4  2012-11-04T00:00:00+00:00  183126  4.9  301.000000          8   1.1   1.6   \n",
       "\n",
       "   PDOP  QUALITY  DIFF_AGE                                 geometry  \n",
       "0   1.9        1       0.0  POINT Z (525284.544 3377008.024 18.500)  \n",
       "1   1.9        2       1.0  POINT Z (525284.224 3377008.208 18.400)  \n",
       "2   1.9        2       1.0  POINT Z (525283.423 3377008.944 18.300)  \n",
       "3   1.9        2       2.0  POINT Z (525282.142 3377009.680 18.100)  \n",
       "4   1.9        2       3.0  POINT Z (525280.861 3377010.231 18.000)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# read in the gdb file provided by the database\n",
    "data = geopandas.read_file(\"data.gdb\") # make sure to update to your path\n",
    "display(data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KYBiM-IVSp-N"
   },
   "source": [
    "For some reason reading the gdb file like this only shows data from the 2012-11-04 dataset (L1G). So let's dive into this deeper and explore each gdbtable individually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4f7tZwS2SkKb",
    "outputId": "7992d10a-c0e1-441b-de5f-a526e8d677fc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of tables: 0\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "path = \"/data.gdb/*\" # make sure to update to your path\n",
    "dfs = {}\n",
    "count = 0\n",
    "booly = True\n",
    "for file in glob.glob(path):\n",
    "    if \".gdbtable\" in file:\n",
    "        gdbtable = geopandas.read_file(file)\n",
    "        count += 1\n",
    "        cols = gdbtable.columns\n",
    "    if str(cols) in dfs:\n",
    "        dfs[str(cols)].append(gdbtable)\n",
    "    else:\n",
    "        dfs[str(cols)] = [gdbtable]\n",
    "\n",
    "print(\"Total number of tables:\", count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 528
    },
    "id": "SUOnHoUBTFW1",
    "outputId": "94ab8771-7b6d-4748-f122-aa90bb1dbbaa"
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "object of type 'NoneType' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-1740e8e5c25c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mdf_2012\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Number of rows:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_2012\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mdf_2012\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# check to see HHMMSS format\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
     ]
    }
   ],
   "source": [
    "# Let's extract only the data with SOG (Speed over ground feet per second) for the ATVs in 2012\n",
    "# I realized that in 2011, they only used aerial devices and it seems that the data of their locations is not available to us in the dataset. Therefore, let's focus on 2012\n",
    "# for all intensive purposes\n",
    "\n",
    "df_2012 = None\n",
    "for key in dfs.keys():\n",
    "    if 'SOG' in key:\n",
    "        df_2012 = pd.concat(dfs[key])\n",
    "        break\n",
    "print(\"Number of rows:\", len(df_2012))\n",
    "df_2012.head() # check to see HHMMSS format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5DThygaOW5xP"
   },
   "source": [
    "Okay, that seems like a lot let's see if we can now use the date and time to connect with another dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9JjCPCS9W65a"
   },
   "source": [
    "## NASA Aura Dataset (not working)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uN8MXTEXZS0a",
    "outputId": "29d2840a-d89f-4136-c9b1-51fc5acdd8a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pydap\n",
      "  Downloading Pydap-3.2.2-py3-none-any.whl (2.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.3 MB 3.7 MB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from pydap) (1.21.6)\n",
      "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from pydap) (0.6.2)\n",
      "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from pydap) (1.15.0)\n",
      "Collecting Webob\n",
      "  Downloading WebOb-1.8.7-py2.py3-none-any.whl (114 kB)\n",
      "\u001b[K     |████████████████████████████████| 114 kB 68.9 MB/s \n",
      "\u001b[?25hRequirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from pydap) (4.6.3)\n",
      "Requirement already satisfied: Jinja2 in /usr/local/lib/python3.7/dist-packages (from pydap) (2.11.3)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2->pydap) (2.0.1)\n",
      "Installing collected packages: Webob, pydap\n",
      "Successfully installed Webob-1.8.7 pydap-3.2.2\n"
     ]
    }
   ],
   "source": [
    "!pip install pydap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 336
    },
    "id": "vL2pIcSLUq-w",
    "outputId": "67b5034b-7d17-409b-ba94-73018dba9a1f"
   },
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-1f10a92fbbb9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mdataset_url\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"https://acdisc.gesdisc.eosdis.nasa.gov/data/Aura_OMI_Level3/OMHCHOd.003/2012/OMI-Aura_L3-OMHCHOd_2012m1108_v003-2019m0726t083354.nc\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msetup_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpassword\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_url\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdataset_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_url\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/client.py\u001b[0m in \u001b[0;36mopen_url\u001b[0;34m(url, application, session, output_grid, timeout)\u001b[0m\n\u001b[1;32m     65\u001b[0m     \"\"\"\n\u001b[1;32m     66\u001b[0m     dataset = DAPHandler(url, application, session, output_grid,\n\u001b[0;32m---> 67\u001b[0;31m                          timeout).dataset\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;31m# attach server-side functions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/handlers/dap.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, url, application, session, output_grid, timeout)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0;31m# build the dataset from the DDS and add attributes from the DAS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m         \u001b[0madd_attributes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_das\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/parsers/dds.py\u001b[0m in \u001b[0;36mbuild_dataset\u001b[0;34m(dds)\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbuild_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m     \u001b[0;34m\"\"\"Return a dataset object from a DDS representation.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mDDSParser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/parsers/dds.py\u001b[0m in \u001b[0;36mparse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDatasetType\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'nameless'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'dataset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpeek\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/parsers/dds.py\u001b[0m in \u001b[0;36mconsume\u001b[0;34m(self, regexp)\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;34m\"\"\"Consume and return a token.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m         \u001b[0mtoken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDDSParser\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconsume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mregexp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pydap/parsers/__init__.py\u001b[0m in \u001b[0;36mconsume\u001b[0;34m(self, regexp)\u001b[0m\n\u001b[1;32m    180\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unable to parse token: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mException\u001b[0m: Unable to parse token: <!DOCTYPE "
     ]
    }
   ],
   "source": [
    "from pydap.client import open_url\n",
    "from pydap.cas.urs import setup_session\n",
    "\n",
    "username = \"\"\n",
    "password = \"\"\n",
    "dataset_url = \"https://acdisc.gesdisc.eosdis.nasa.gov/data/Aura_OMI_Level3/OMHCHOd.003/2012/OMI-Aura_L3-OMHCHOd_2012m1108_v003-2019m0726t083354.nc\"\n",
    "session = setup_session(username, password, check_url=dataset_url)\n",
    "dataset = open_url(dataset_url, session=session)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TqeEJhe3Zakh"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "with requests.Session() as session:\n",
    "\n",
    "  # s.auth = (username, password)\n",
    "\n",
    "  r1 = session.request('get', dataset_url)\n",
    "\n",
    "  r = session.get(r1.url, auth=(username, password))\n",
    "\n",
    "  if r.ok:\n",
    "\n",
    "    print(r.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4FBdTEkafwI6"
   },
   "source": [
    "## Amemometer Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "M2CVHBANf0pd"
   },
   "outputs": [],
   "source": [
    "# https://www.fs.usda.gov/rds/archive/catalog/RDS-2016-0038\n",
    "amem = pd.read_csv(\"/content/drive/MyDrive/research/rxcadre/amemometer.csv\", encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 268
    },
    "id": "XJYeUmrqgoA9",
    "outputId": "1cabd951-04c5-41c5-f452-b2d9d31aa287"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-d1cd2bd4-96cb-4c3a-a07d-cbd946dccdfb\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FileRecNo</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time GMT+00:00</th>\n",
       "      <th>WindSpeed mph</th>\n",
       "      <th>WindGust mph</th>\n",
       "      <th>WindDirection Degrees_from_north</th>\n",
       "      <th>PlotID</th>\n",
       "      <th>InstrumentID</th>\n",
       "      <th>TagID</th>\n",
       "      <th>Latitude (Y)</th>\n",
       "      <th>Longitude (X)</th>\n",
       "      <th>Easting (X)</th>\n",
       "      <th>Northing (Y)</th>\n",
       "      <th>EllipsoidHt(Z)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>11/03/12</td>\n",
       "      <td>08:35:35 PM</td>\n",
       "      <td>2.48</td>\n",
       "      <td>18.68</td>\n",
       "      <td>292.0</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>11/03/12</td>\n",
       "      <td>08:35:38 PM</td>\n",
       "      <td>7.05</td>\n",
       "      <td>7.05</td>\n",
       "      <td>240.1</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>11/03/12</td>\n",
       "      <td>08:35:41 PM</td>\n",
       "      <td>8.30</td>\n",
       "      <td>8.30</td>\n",
       "      <td>237.3</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>11/03/12</td>\n",
       "      <td>08:35:44 PM</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>237.3</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>11/03/12</td>\n",
       "      <td>08:35:47 PM</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>240.1</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d1cd2bd4-96cb-4c3a-a07d-cbd946dccdfb')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-d1cd2bd4-96cb-4c3a-a07d-cbd946dccdfb button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-d1cd2bd4-96cb-4c3a-a07d-cbd946dccdfb');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "   FileRecNo      Date  Time GMT+00:00   WindSpeed mph   WindGust mph  \\\n",
       "0          1  11/03/12     08:35:35 PM            2.48          18.68   \n",
       "1          2  11/03/12     08:35:38 PM            7.05           7.05   \n",
       "2          3  11/03/12     08:35:41 PM            8.30           8.30   \n",
       "3          4  11/03/12     08:35:44 PM            8.72           8.72   \n",
       "4          5  11/03/12     08:35:47 PM            8.72           8.72   \n",
       "\n",
       "    WindDirection Degrees_from_north  PlotID  InstrumentID   TagID  \\\n",
       "0                              292.0     L1G            A2      21   \n",
       "1                              240.1     L1G            A2      21   \n",
       "2                              237.3     L1G            A2      21   \n",
       "3                              237.3     L1G            A2      21   \n",
       "4                              240.1     L1G            A2      21   \n",
       "\n",
       "     Latitude (Y)    Longitude (X)   Easting (X)   Northing (Y)  \\\n",
       "0   30°31'47.602\"   -86°44'20.271\"      525041.5     3377532.74   \n",
       "1   30°31'47.602\"   -86°44'20.271\"      525041.5     3377532.74   \n",
       "2   30°31'47.602\"   -86°44'20.271\"      525041.5     3377532.74   \n",
       "3   30°31'47.602\"   -86°44'20.271\"      525041.5     3377532.74   \n",
       "4   30°31'47.602\"   -86°44'20.271\"      525041.5     3377532.74   \n",
       "\n",
       "    EllipsoidHt(Z)  \n",
       "0            14.51  \n",
       "1            14.51  \n",
       "2            14.51  \n",
       "3            14.51  \n",
       "4            14.51  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amem.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "s6B_2lJjhBmo"
   },
   "outputs": [],
   "source": [
    "# Let's hastily try to combine with the previous dataset. First we need to clean a little\n",
    "import re\n",
    "\n",
    "amem.columns = amem.columns.str.replace(\" \", \"\")\n",
    "amem['Date']= pd.to_datetime(amem['Date'], utc=True)\n",
    "# amem['Time'] = pd.to_datetime(amem['TimeGMT+00:00'], utc=True)\n",
    "\n",
    "def dms2dd(s):\n",
    "  # example: s = \"\"\"0°51'56.29\"S\"\"\"\n",
    "  degrees, minutes, seconds, direction = re.split('[°\\'\"]+', s)\n",
    "  dd = float(degrees) + float(minutes)/60 + float(seconds)/(60*60);\n",
    "  if direction in ('S','W'):\n",
    "      dd*= -1\n",
    "  return dd\n",
    "\n",
    "amem[\"LATITUDE\"] = amem[\"Latitude(Y)\"].apply(dms2dd)\n",
    "amem[\"LONGITUDE\"] = amem[\"Longitude(X)\"].apply(dms2dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 357
    },
    "id": "f_8GL6nghYh-",
    "outputId": "fcd816b6-f68c-4160-824e-f7a188281d2f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-64943f80-c531-436b-af46-4520cbd64da7\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FileRecNo</th>\n",
       "      <th>Date</th>\n",
       "      <th>TimeGMT+00:00</th>\n",
       "      <th>WindSpeedmph</th>\n",
       "      <th>WindGustmph</th>\n",
       "      <th>WindDirectionDegrees_from_north</th>\n",
       "      <th>PlotID</th>\n",
       "      <th>InstrumentID</th>\n",
       "      <th>TagID</th>\n",
       "      <th>Latitude(Y)</th>\n",
       "      <th>Longitude(X)</th>\n",
       "      <th>Easting(X)</th>\n",
       "      <th>Northing(Y)</th>\n",
       "      <th>EllipsoidHt(Z)</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2012-11-03 00:00:00+00:00</td>\n",
       "      <td>08:35:35 PM</td>\n",
       "      <td>2.48</td>\n",
       "      <td>18.68</td>\n",
       "      <td>292.0</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "      <td>30.529889</td>\n",
       "      <td>-85.261036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2012-11-03 00:00:00+00:00</td>\n",
       "      <td>08:35:38 PM</td>\n",
       "      <td>7.05</td>\n",
       "      <td>7.05</td>\n",
       "      <td>240.1</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "      <td>30.529889</td>\n",
       "      <td>-85.261036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2012-11-03 00:00:00+00:00</td>\n",
       "      <td>08:35:41 PM</td>\n",
       "      <td>8.30</td>\n",
       "      <td>8.30</td>\n",
       "      <td>237.3</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "      <td>30.529889</td>\n",
       "      <td>-85.261036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2012-11-03 00:00:00+00:00</td>\n",
       "      <td>08:35:44 PM</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>237.3</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "      <td>30.529889</td>\n",
       "      <td>-85.261036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2012-11-03 00:00:00+00:00</td>\n",
       "      <td>08:35:47 PM</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>240.1</td>\n",
       "      <td>L1G</td>\n",
       "      <td>A2</td>\n",
       "      <td>21</td>\n",
       "      <td>30°31'47.602\"</td>\n",
       "      <td>-86°44'20.271\"</td>\n",
       "      <td>525041.5</td>\n",
       "      <td>3377532.74</td>\n",
       "      <td>14.51</td>\n",
       "      <td>30.529889</td>\n",
       "      <td>-85.261036</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-64943f80-c531-436b-af46-4520cbd64da7')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-64943f80-c531-436b-af46-4520cbd64da7 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-64943f80-c531-436b-af46-4520cbd64da7');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "   FileRecNo                      Date TimeGMT+00:00  WindSpeedmph  \\\n",
       "0          1 2012-11-03 00:00:00+00:00   08:35:35 PM          2.48   \n",
       "1          2 2012-11-03 00:00:00+00:00   08:35:38 PM          7.05   \n",
       "2          3 2012-11-03 00:00:00+00:00   08:35:41 PM          8.30   \n",
       "3          4 2012-11-03 00:00:00+00:00   08:35:44 PM          8.72   \n",
       "4          5 2012-11-03 00:00:00+00:00   08:35:47 PM          8.72   \n",
       "\n",
       "   WindGustmph  WindDirectionDegrees_from_north PlotID InstrumentID  TagID  \\\n",
       "0        18.68                            292.0    L1G           A2     21   \n",
       "1         7.05                            240.1    L1G           A2     21   \n",
       "2         8.30                            237.3    L1G           A2     21   \n",
       "3         8.72                            237.3    L1G           A2     21   \n",
       "4         8.72                            240.1    L1G           A2     21   \n",
       "\n",
       "      Latitude(Y)     Longitude(X)  Easting(X)  Northing(Y)  EllipsoidHt(Z)  \\\n",
       "0   30°31'47.602\"   -86°44'20.271\"    525041.5   3377532.74           14.51   \n",
       "1   30°31'47.602\"   -86°44'20.271\"    525041.5   3377532.74           14.51   \n",
       "2   30°31'47.602\"   -86°44'20.271\"    525041.5   3377532.74           14.51   \n",
       "3   30°31'47.602\"   -86°44'20.271\"    525041.5   3377532.74           14.51   \n",
       "4   30°31'47.602\"   -86°44'20.271\"    525041.5   3377532.74           14.51   \n",
       "\n",
       "    LATITUDE  LONGITUDE  \n",
       "0  30.529889 -85.261036  \n",
       "1  30.529889 -85.261036  \n",
       "2  30.529889 -85.261036  \n",
       "3  30.529889 -85.261036  \n",
       "4  30.529889 -85.261036  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amem.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "bABSxW0fiK4b"
   },
   "outputs": [],
   "source": [
    "devices_df = df_2012\n",
    "devices_df[\"Date\"] = pd.to_datetime(devices_df[\"UTCDATE\"], utc=True)\n",
    "# devices_df[\"Time\"] = pd.to_datetime(devices_df[\"UTCTIME\"], utc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 531
    },
    "id": "Lf-kgz1FiZyE",
    "outputId": "dc239752-0716-47e6-f6b3-7818fc5aa96d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e815f740-2da4-49dc-a2ee-2f2abb39a13e\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ALTITUDE</th>\n",
       "      <th>EASTING</th>\n",
       "      <th>NORTHING</th>\n",
       "      <th>UTCDATE</th>\n",
       "      <th>UTCTIME</th>\n",
       "      <th>SOG</th>\n",
       "      <th>COG</th>\n",
       "      <th>SATS_USED</th>\n",
       "      <th>HDOP</th>\n",
       "      <th>VDOP</th>\n",
       "      <th>PDOP</th>\n",
       "      <th>QUALITY</th>\n",
       "      <th>DIFF_AGE</th>\n",
       "      <th>geometry</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30.525149</td>\n",
       "      <td>-86.736444</td>\n",
       "      <td>18.5</td>\n",
       "      <td>525284.544</td>\n",
       "      <td>3377008.024</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183122</td>\n",
       "      <td>0.1</td>\n",
       "      <td>141.800003</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>POINT Z (525284.544 3377008.024 18.500)</td>\n",
       "      <td>2012-11-04 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>30.525151</td>\n",
       "      <td>-86.736447</td>\n",
       "      <td>18.4</td>\n",
       "      <td>525284.224</td>\n",
       "      <td>3377008.208</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183123</td>\n",
       "      <td>0.1</td>\n",
       "      <td>18.200001</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>POINT Z (525284.224 3377008.208 18.400)</td>\n",
       "      <td>2012-11-04 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>30.525158</td>\n",
       "      <td>-86.736455</td>\n",
       "      <td>18.3</td>\n",
       "      <td>525283.423</td>\n",
       "      <td>3377008.945</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183124</td>\n",
       "      <td>2.1</td>\n",
       "      <td>304.100006</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>POINT Z (525283.423 3377008.944 18.300)</td>\n",
       "      <td>2012-11-04 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>30.525164</td>\n",
       "      <td>-86.736469</td>\n",
       "      <td>18.1</td>\n",
       "      <td>525282.142</td>\n",
       "      <td>3377009.680</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183125</td>\n",
       "      <td>5.6</td>\n",
       "      <td>303.600006</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>POINT Z (525282.142 3377009.680 18.100)</td>\n",
       "      <td>2012-11-04 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>30.525169</td>\n",
       "      <td>-86.736482</td>\n",
       "      <td>18.0</td>\n",
       "      <td>525280.861</td>\n",
       "      <td>3377010.231</td>\n",
       "      <td>2012-11-04T00:00:00+00:00</td>\n",
       "      <td>183126</td>\n",
       "      <td>4.9</td>\n",
       "      <td>301.000000</td>\n",
       "      <td>8</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>POINT Z (525280.861 3377010.231 18.000)</td>\n",
       "      <td>2012-11-04 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e815f740-2da4-49dc-a2ee-2f2abb39a13e')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-e815f740-2da4-49dc-a2ee-2f2abb39a13e button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-e815f740-2da4-49dc-a2ee-2f2abb39a13e');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "   OBJECTID   LATITUDE  LONGITUDE  ALTITUDE     EASTING     NORTHING  \\\n",
       "0         1  30.525149 -86.736444      18.5  525284.544  3377008.024   \n",
       "1         2  30.525151 -86.736447      18.4  525284.224  3377008.208   \n",
       "2         3  30.525158 -86.736455      18.3  525283.423  3377008.945   \n",
       "3         4  30.525164 -86.736469      18.1  525282.142  3377009.680   \n",
       "4         5  30.525169 -86.736482      18.0  525280.861  3377010.231   \n",
       "\n",
       "                     UTCDATE UTCTIME  SOG         COG  SATS_USED  HDOP  VDOP  \\\n",
       "0  2012-11-04T00:00:00+00:00  183122  0.1  141.800003          8   1.1   1.6   \n",
       "1  2012-11-04T00:00:00+00:00  183123  0.1   18.200001          8   1.1   1.6   \n",
       "2  2012-11-04T00:00:00+00:00  183124  2.1  304.100006          8   1.1   1.6   \n",
       "3  2012-11-04T00:00:00+00:00  183125  5.6  303.600006          8   1.1   1.6   \n",
       "4  2012-11-04T00:00:00+00:00  183126  4.9  301.000000          8   1.1   1.6   \n",
       "\n",
       "   PDOP  QUALITY  DIFF_AGE                                 geometry  \\\n",
       "0   1.9        1       0.0  POINT Z (525284.544 3377008.024 18.500)   \n",
       "1   1.9        2       1.0  POINT Z (525284.224 3377008.208 18.400)   \n",
       "2   1.9        2       1.0  POINT Z (525283.423 3377008.944 18.300)   \n",
       "3   1.9        2       2.0  POINT Z (525282.142 3377009.680 18.100)   \n",
       "4   1.9        2       3.0  POINT Z (525280.861 3377010.231 18.000)   \n",
       "\n",
       "                       Date  \n",
       "0 2012-11-04 00:00:00+00:00  \n",
       "1 2012-11-04 00:00:00+00:00  \n",
       "2 2012-11-04 00:00:00+00:00  \n",
       "3 2012-11-04 00:00:00+00:00  \n",
       "4 2012-11-04 00:00:00+00:00  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "devices_df.head() # Exploring what the time is?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hnUdJUTSKExn"
   },
   "source": [
    "# Reading Fire Behavior Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "9PA1BCgbKJuD"
   },
   "outputs": [],
   "source": [
    "# https://www.fs.usda.gov/rds/archive/catalog/RDS-2016-0038\n",
    "import zipfile\n",
    "with zipfile.ZipFile(\"/content/drive/MyDrive/research/rxcadre/FireBehavior.zip\", 'r') as zip_ref:\n",
    "    zip_ref.extractall(\"/content/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 490
    },
    "id": "UsBLcB00LIxT",
    "outputId": "3d00517e-8c85-48fd-f943-7946fe5b9349"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e754b2c5-49c1-4d0c-b217-8c1db7fb36af\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Date</th>\n",
       "      <th>UTC Time</th>\n",
       "      <th>Time After NAR Signal Detection (s)</th>\n",
       "      <th>Temperature (degC)</th>\n",
       "      <th>MT-total(kW/m^2)</th>\n",
       "      <th>MT-rad(kW/m^2)</th>\n",
       "      <th>Convective(kW/m^2)</th>\n",
       "      <th>Total(kW/m^2)</th>\n",
       "      <th>NAR(kW/m^2)</th>\n",
       "      <th>...</th>\n",
       "      <th>Medtherm Body Temp</th>\n",
       "      <th>Vert. Wind(m/s)</th>\n",
       "      <th>Horiz. Wind(m/s)</th>\n",
       "      <th>Zeroed Vert. Wind(m/s)</th>\n",
       "      <th>Zeroed Horiz. Wind(m/s)</th>\n",
       "      <th>Battery</th>\n",
       "      <th>Vert. Wind 30 sec. avg.(m/s)</th>\n",
       "      <th>Horiz. Wind 30 sec. avg.(m/s)</th>\n",
       "      <th>Zeroed Vert. Wind 30 sec. avg.(m/s)</th>\n",
       "      <th>Zeroed Horiz. Wind 30 sec. avg.(m/s)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>2012-11-04</td>\n",
       "      <td>17:39:48.800000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.57679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NAN</td>\n",
       "      <td>2.666134</td>\n",
       "      <td>2.811149</td>\n",
       "      <td>-0.403326</td>\n",
       "      <td>3.102748</td>\n",
       "      <td>12.34775</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>41027</td>\n",
       "      <td>2012-11-04</td>\n",
       "      <td>17:39:48.900000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>26.57679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NAN</td>\n",
       "      <td>2.715232</td>\n",
       "      <td>2.026845</td>\n",
       "      <td>0.479193</td>\n",
       "      <td>2.562644</td>\n",
       "      <td>12.34227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>42309</td>\n",
       "      <td>2012-11-04</td>\n",
       "      <td>17:39:49</td>\n",
       "      <td>0.2</td>\n",
       "      <td>26.57679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NAN</td>\n",
       "      <td>2.616603</td>\n",
       "      <td>2.165843</td>\n",
       "      <td>-0.781293</td>\n",
       "      <td>2.627834</td>\n",
       "      <td>12.34775</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18:48:11.400000</td>\n",
       "      <td>2012-11-04</td>\n",
       "      <td>17:39:49.100000</td>\n",
       "      <td>0.3</td>\n",
       "      <td>26.57679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NAN</td>\n",
       "      <td>2.617048</td>\n",
       "      <td>2.490635</td>\n",
       "      <td>-0.781466</td>\n",
       "      <td>2.815762</td>\n",
       "      <td>12.34227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18:50:19.600000</td>\n",
       "      <td>2012-11-04</td>\n",
       "      <td>17:39:49.200000</td>\n",
       "      <td>0.4</td>\n",
       "      <td>26.57679</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NAN</td>\n",
       "      <td>2.762049</td>\n",
       "      <td>2.810720</td>\n",
       "      <td>0.767959</td>\n",
       "      <td>3.102236</td>\n",
       "      <td>12.35322</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e754b2c5-49c1-4d0c-b217-8c1db7fb36af')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-e754b2c5-49c1-4d0c-b217-8c1db7fb36af button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-e754b2c5-49c1-4d0c-b217-8c1db7fb36af');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "        Unnamed: 0       Date         UTC Time  \\\n",
       "0              0.1 2012-11-04  17:39:48.800000   \n",
       "1            41027 2012-11-04  17:39:48.900000   \n",
       "2            42309 2012-11-04         17:39:49   \n",
       "3  18:48:11.400000 2012-11-04  17:39:49.100000   \n",
       "4  18:50:19.600000 2012-11-04  17:39:49.200000   \n",
       "\n",
       "   Time After NAR Signal Detection (s)  Temperature (degC)  MT-total(kW/m^2)  \\\n",
       "0                                  0.0            26.57679               0.0   \n",
       "1                                  0.1            26.57679               0.0   \n",
       "2                                  0.2            26.57679               0.0   \n",
       "3                                  0.3            26.57679               0.0   \n",
       "4                                  0.4            26.57679               0.0   \n",
       "\n",
       "   MT-rad(kW/m^2)  Convective(kW/m^2)  Total(kW/m^2)  NAR(kW/m^2)  ...  \\\n",
       "0             0.0                 0.0            0.0          0.0  ...   \n",
       "1             0.0                 0.0            0.0          0.0  ...   \n",
       "2             0.0                 0.0            0.0          0.0  ...   \n",
       "3             0.0                 0.0            0.0          0.0  ...   \n",
       "4             0.0                 0.0            0.0          0.0  ...   \n",
       "\n",
       "   Medtherm Body Temp  Vert. Wind(m/s)  Horiz. Wind(m/s)  \\\n",
       "0                 NAN         2.666134          2.811149   \n",
       "1                 NAN         2.715232          2.026845   \n",
       "2                 NAN         2.616603          2.165843   \n",
       "3                 NAN         2.617048          2.490635   \n",
       "4                 NAN         2.762049          2.810720   \n",
       "\n",
       "   Zeroed Vert. Wind(m/s)  Zeroed Horiz. Wind(m/s)   Battery  \\\n",
       "0               -0.403326                 3.102748  12.34775   \n",
       "1                0.479193                 2.562644  12.34227   \n",
       "2               -0.781293                 2.627834  12.34775   \n",
       "3               -0.781466                 2.815762  12.34227   \n",
       "4                0.767959                 3.102236  12.35322   \n",
       "\n",
       "   Vert. Wind 30 sec. avg.(m/s)  Horiz. Wind 30 sec. avg.(m/s)  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   Zeroed Vert. Wind 30 sec. avg.(m/s)  Zeroed Horiz. Wind 30 sec. avg.(m/s)  \n",
       "0                                  NaN                                   NaN  \n",
       "1                                  NaN                                   NaN  \n",
       "2                                  NaN                                   NaN  \n",
       "3                                  NaN                                   NaN  \n",
       "4                                  NaN                                   NaN  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# focus only on L1G and create big pandas\n",
    "import os\n",
    "\n",
    "xcels = []\n",
    "for file in os.listdir(\"/content/FireBehavior/l1g\"):\n",
    "  # making this smaller for proof of concept optimization\n",
    "  if len(xcels) > 2:\n",
    "    break\n",
    "  if \"~$\" in file:\n",
    "    continue\n",
    "  xcels.append(pd.read_excel(\"/content/FireBehavior/l1g/\" + file, sheet_name=\"burn1\"))\n",
    "\n",
    "firebehavior = pd.concat(xcels)\n",
    "firebehavior.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "dkXckrfKOv8O",
    "outputId": "e5cdcaa4-8136-455f-c935-ee56a7da3ddd"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-dcdc8ba7-1e54-4e20-ab4d-180e8d126c26\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11-04-2012</td>\n",
       "      <td>17:39:49.200000</td>\n",
       "      <td>-0.023942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11-04-2012</td>\n",
       "      <td>17:39:49.300000</td>\n",
       "      <td>-0.025926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11-04-2012</td>\n",
       "      <td>17:39:49.400000</td>\n",
       "      <td>-0.027911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11-04-2012</td>\n",
       "      <td>17:39:49.500000</td>\n",
       "      <td>-0.029896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>11-04-2012</td>\n",
       "      <td>17:39:49.600000</td>\n",
       "      <td>-0.031881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dcdc8ba7-1e54-4e20-ab4d-180e8d126c26')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-dcdc8ba7-1e54-4e20-ab4d-180e8d126c26 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-dcdc8ba7-1e54-4e20-ab4d-180e8d126c26');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "         date             time      rate\n",
       "4  11-04-2012  17:39:49.200000 -0.023942\n",
       "5  11-04-2012  17:39:49.300000 -0.025926\n",
       "6  11-04-2012  17:39:49.400000 -0.027911\n",
       "7  11-04-2012  17:39:49.500000 -0.029896\n",
       "8  11-04-2012  17:39:49.600000 -0.031881"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rateOfSpread = firebehavior[[\"Date\", \"UTC Time\", \"Conv. 1 sec. avg.(kW/m^2)\"]]\n",
    "rateOfSpread.columns = [\"date\", \"time\", \"rate\"]\n",
    "rateOfSpread = rateOfSpread.dropna()\n",
    "rateOfSpread[\"date\"] = [date.strftime('%m-%d-%Y') for date in list(rateOfSpread['date'])]\n",
    "rateOfSpread[\"time\"] = [time.strftime('%H:%M:%S.%f') for time in list(rateOfSpread['time'])]\n",
    "\n",
    "rateOfSpread.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "cY6cgHLDQNv-",
    "outputId": "5fe651dc-a994-4b9b-a5d5-9b05242a3614"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-8586785d-3d9a-4474-ba4c-1ff26cc7d0d8\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate</th>\n",
       "      <th>utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.023942</td>\n",
       "      <td>11-04-2012 17:39:49.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.025926</td>\n",
       "      <td>11-04-2012 17:39:49.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.027911</td>\n",
       "      <td>11-04-2012 17:39:49.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.029896</td>\n",
       "      <td>11-04-2012 17:39:49.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.031881</td>\n",
       "      <td>11-04-2012 17:39:49.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8586785d-3d9a-4474-ba4c-1ff26cc7d0d8')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-8586785d-3d9a-4474-ba4c-1ff26cc7d0d8 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-8586785d-3d9a-4474-ba4c-1ff26cc7d0d8');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "       rate                         utc\n",
       "4 -0.023942  11-04-2012 17:39:49.200000\n",
       "5 -0.025926  11-04-2012 17:39:49.300000\n",
       "6 -0.027911  11-04-2012 17:39:49.400000\n",
       "7 -0.029896  11-04-2012 17:39:49.500000\n",
       "8 -0.031881  11-04-2012 17:39:49.600000"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rateOfSpread[\"utc\"] = rateOfSpread['date'] + ' ' + rateOfSpread['time']\n",
    "rateOfSpread = rateOfSpread.drop(columns=[\"date\", \"time\"])\n",
    "rateOfSpread.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JXNLpNzCMNSt"
   },
   "source": [
    "# Combine Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "AtNBCqbIMNCK",
    "outputId": "e38e07d8-7a20-4f98-f0a6-3405a1405e74"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-a974f183-ba57-4092-885c-5e0eaf6d6c71\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>windspeed</th>\n",
       "      <th>windgust</th>\n",
       "      <th>winddirection</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11/03/2012</td>\n",
       "      <td>20:35:35.000</td>\n",
       "      <td>2.48</td>\n",
       "      <td>18.68</td>\n",
       "      <td>292.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11/03/2012</td>\n",
       "      <td>20:35:38.000</td>\n",
       "      <td>7.05</td>\n",
       "      <td>7.05</td>\n",
       "      <td>240.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11/03/2012</td>\n",
       "      <td>20:35:41.000</td>\n",
       "      <td>8.30</td>\n",
       "      <td>8.30</td>\n",
       "      <td>237.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11/03/2012</td>\n",
       "      <td>20:35:44.000</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>237.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11/03/2012</td>\n",
       "      <td>20:35:47.000</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>240.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a974f183-ba57-4092-885c-5e0eaf6d6c71')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-a974f183-ba57-4092-885c-5e0eaf6d6c71 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-a974f183-ba57-4092-885c-5e0eaf6d6c71');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "         date          time  windspeed  windgust  winddirection\n",
       "0  11/03/2012  20:35:35.000       2.48     18.68          292.0\n",
       "1  11/03/2012  20:35:38.000       7.05      7.05          240.1\n",
       "2  11/03/2012  20:35:41.000       8.30      8.30          237.3\n",
       "3  11/03/2012  20:35:44.000       8.72      8.72          237.3\n",
       "4  11/03/2012  20:35:47.000       8.72      8.72          240.1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Need to only focus on L1G for now\n",
    "# https://www.fs.usda.gov/rds/archive/catalog/RDS-2016-0038\n",
    "amem_l1g = pd.read_csv(\"/content/drive/MyDrive/research/rxcadre/amem_l1g.csv\", encoding = \"ISO-8859-1\")\n",
    "# print(amem_l1g.columns)\n",
    "amem_l1g = amem_l1g[[\"_Date\", \"_Time_GMT_00_00\", \"_WindSpeed_mph\", \"_WindGust_mph\", \"_WindDirection_Degrees_from_nort\"]]\n",
    "amem_l1g.columns = [\"date\", \"time\", \"windspeed\", \"windgust\", \"winddirection\"]\n",
    "amem_l1g.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "AHDHJSI1jN4l",
    "outputId": "4af6df45-977c-465d-8a6a-d79db54535a3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-7cafd531-c0a4-4709-84d9-686ac709e23a\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>windspeed</th>\n",
       "      <th>windgust</th>\n",
       "      <th>winddirection</th>\n",
       "      <th>utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.48</td>\n",
       "      <td>18.68</td>\n",
       "      <td>292.0</td>\n",
       "      <td>11/03/2012 20:35:35.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.05</td>\n",
       "      <td>7.05</td>\n",
       "      <td>240.1</td>\n",
       "      <td>11/03/2012 20:35:38.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.30</td>\n",
       "      <td>8.30</td>\n",
       "      <td>237.3</td>\n",
       "      <td>11/03/2012 20:35:41.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>237.3</td>\n",
       "      <td>11/03/2012 20:35:44.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>240.1</td>\n",
       "      <td>11/03/2012 20:35:47.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7cafd531-c0a4-4709-84d9-686ac709e23a')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-7cafd531-c0a4-4709-84d9-686ac709e23a button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-7cafd531-c0a4-4709-84d9-686ac709e23a');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "   windspeed  windgust  winddirection                      utc\n",
       "0       2.48     18.68          292.0  11/03/2012 20:35:35.000\n",
       "1       7.05      7.05          240.1  11/03/2012 20:35:38.000\n",
       "2       8.30      8.30          237.3  11/03/2012 20:35:41.000\n",
       "3       8.72      8.72          237.3  11/03/2012 20:35:44.000\n",
       "4       8.72      8.72          240.1  11/03/2012 20:35:47.000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amem_l1g[\"utc\"] = amem_l1g['date'] + ' ' + amem_l1g['time']\n",
    "amem_l1g = amem_l1g.drop(columns=[\"time\", \"date\"])\n",
    "amem_l1g.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "NOTltjEgYKyi"
   },
   "outputs": [],
   "source": [
    "# convert utc columns to datetime\n",
    "amem_l1g[\"utc\"] = pd.to_datetime(amem_l1g[\"utc\"])\n",
    "rateOfSpread[\"utc\"] = pd.to_datetime(rateOfSpread[\"utc\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "Z0g1lPGJZ7va"
   },
   "outputs": [],
   "source": [
    "# We are setting the index\n",
    "am = amem_l1g\n",
    "ros = rateOfSpread\n",
    "am = am.set_index(\"utc\")\n",
    "ros = ros.set_index(\"utc\")\n",
    "am = am.sort_index()\n",
    "ros = ros.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "id": "oJ3uKdPhYhOd",
    "outputId": "f68c6622-1b77-4eb1-db0d-f94e2b631b18"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-a2b6b7af-4271-4816-b6fd-424458c2b061\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>utc</th>\n",
       "      <th>windspeed</th>\n",
       "      <th>windgust</th>\n",
       "      <th>winddirection</th>\n",
       "      <th>rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1927333</th>\n",
       "      <td>2012-11-04 17:39:50</td>\n",
       "      <td>9.13</td>\n",
       "      <td>9.13</td>\n",
       "      <td>304.7</td>\n",
       "      <td>-0.037913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1927334</th>\n",
       "      <td>2012-11-04 17:39:50</td>\n",
       "      <td>8.30</td>\n",
       "      <td>8.30</td>\n",
       "      <td>320.1</td>\n",
       "      <td>-0.037913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1927335</th>\n",
       "      <td>2012-11-04 17:39:50</td>\n",
       "      <td>2.82</td>\n",
       "      <td>2.82</td>\n",
       "      <td>307.5</td>\n",
       "      <td>-0.037913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1927336</th>\n",
       "      <td>2012-11-04 17:39:50</td>\n",
       "      <td>6.22</td>\n",
       "      <td>6.22</td>\n",
       "      <td>353.8</td>\n",
       "      <td>-0.037913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1927337</th>\n",
       "      <td>2012-11-04 17:39:50</td>\n",
       "      <td>7.05</td>\n",
       "      <td>7.05</td>\n",
       "      <td>278.0</td>\n",
       "      <td>-0.037913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064970</th>\n",
       "      <td>2012-11-04 19:09:47</td>\n",
       "      <td>9.55</td>\n",
       "      <td>9.55</td>\n",
       "      <td>248.5</td>\n",
       "      <td>-0.084964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064971</th>\n",
       "      <td>2012-11-04 19:09:47</td>\n",
       "      <td>14.94</td>\n",
       "      <td>14.94</td>\n",
       "      <td>296.2</td>\n",
       "      <td>-0.084964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064972</th>\n",
       "      <td>2012-11-04 19:09:47</td>\n",
       "      <td>7.47</td>\n",
       "      <td>7.47</td>\n",
       "      <td>318.7</td>\n",
       "      <td>-0.084964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064973</th>\n",
       "      <td>2012-11-04 19:09:47</td>\n",
       "      <td>8.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>292.0</td>\n",
       "      <td>-0.084964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2064974</th>\n",
       "      <td>2012-11-04 19:09:47</td>\n",
       "      <td>10.38</td>\n",
       "      <td>10.38</td>\n",
       "      <td>304.7</td>\n",
       "      <td>-0.084964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137642 rows × 5 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a2b6b7af-4271-4816-b6fd-424458c2b061')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-a2b6b7af-4271-4816-b6fd-424458c2b061 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-a2b6b7af-4271-4816-b6fd-424458c2b061');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                        utc  windspeed  windgust  winddirection      rate\n",
       "1927333 2012-11-04 17:39:50       9.13      9.13          304.7 -0.037913\n",
       "1927334 2012-11-04 17:39:50       8.30      8.30          320.1 -0.037913\n",
       "1927335 2012-11-04 17:39:50       2.82      2.82          307.5 -0.037913\n",
       "1927336 2012-11-04 17:39:50       6.22      6.22          353.8 -0.037913\n",
       "1927337 2012-11-04 17:39:50       7.05      7.05          278.0 -0.037913\n",
       "...                     ...        ...       ...            ...       ...\n",
       "2064970 2012-11-04 19:09:47       9.55      9.55          248.5 -0.084964\n",
       "2064971 2012-11-04 19:09:47      14.94     14.94          296.2 -0.084964\n",
       "2064972 2012-11-04 19:09:47       7.47      7.47          318.7 -0.084964\n",
       "2064973 2012-11-04 19:09:47       8.72      8.72          292.0 -0.084964\n",
       "2064974 2012-11-04 19:09:47      10.38     10.38          304.7 -0.084964\n",
       "\n",
       "[137642 rows x 5 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now let's merge the two\n",
    "merged = pd.merge_asof(am, ros, on=\"utc\", tolerance=pd.Timedelta('5min'))\n",
    "merged = merged.dropna()\n",
    "merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GVsFFf-Db4sb"
   },
   "source": [
    "# Basic DL Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "YWHMa9EWnWkH"
   },
   "outputs": [],
   "source": [
    "import torch as th\n",
    "\n",
    "# convert variables to PyTorch tensor \n",
    "X1 = th.tensor(merged[['windspeed']].values)\n",
    "X2 = th.tensor(merged[[\"windgust\"]].values)\n",
    "X3 = th.tensor(merged[[\"winddirection\"]].values)\n",
    "X = th.stack([X1, X2, X3], 2)\n",
    "y = th.tensor(merged[['rate']].values, dtype=th.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "4yESUCkfcvJa"
   },
   "outputs": [],
   "source": [
    "in_features = 3 # number of independent variables\n",
    "out_features = 1 # dimension of predicted variables\n",
    "# bias is default true and can be skipped\n",
    "model = th.nn.Linear(in_features=in_features, out_features=out_features, bias=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "GrgZlIiCdmTw"
   },
   "outputs": [],
   "source": [
    "mse_loss = th.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "avFH6Mmndodr"
   },
   "outputs": [],
   "source": [
    "optimizer = th.optim.SGD(model.parameters(), lr=0.002)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KA9mu8pEdqBi",
    "outputId": "b29afcb1-0967-42e8-f586-5e0bbc13b2a7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:529: UserWarning: Using a target size (torch.Size([137642, 1])) that is different to the input size (torch.Size([137642, 1, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
      "  return F.mse_loss(input, target, reduction=self.reduction)\n"
     ]
    }
   ],
   "source": [
    "# set epochs\n",
    "n_epoch = 10\n",
    "for i in range(n_epoch):\n",
    "    # predict model with current regression parameters\n",
    "    # forward pass (feed the data to model)\n",
    "    y_pred = model(X.float())\n",
    "    # calculate loss function\n",
    "    step_loss = mse_loss(y_pred, y)\n",
    "        \n",
    "    # Backward to find the derivatives of the loss function with respect to regression parameters\n",
    "    # make any stored gradients to zero\n",
    "    # backward pass (go back and update the regression parameters to minimize the loss)\n",
    "    optimizer.zero_grad()\n",
    "    step_loss.backward()\n",
    "    # update with current step regression parameters \n",
    "    optimizer.step()\n",
    "    print ('epoch [{}], Loss: {:.2f}'.format(i, step_loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rECnjHaldwCB"
   },
   "outputs": [],
   "source": [
    "# ACtion items\n",
    "# share the code with meng students\n",
    "# run this locally if doesnt work\n",
    "# combine with Rohans"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Squishy_Wildfire_Analysis.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}