2022-07-13 IMD Gridded data extraction sample flow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0e3b20f9",
   "metadata": {},
   "source": [
    "## 2022-07-13 IMD Gridded data extraction sample flow\n",
    "\n",
    "by Nikhil VJ, https://nikhilvj.co.in\n",
    "\n",
    "Processing weather data from https://imdpune.gov.in/Clim_Pred_LRF_New/Grided_Data_Download.html\n",
    "\n",
    "Using imdblib package, https://imdlib.readthedocs.io/en/latest/Usage.html#processing   \n",
    "\n",
    "Got imdblib reference from here: https://www.youtube.com/watch?v=40Dvj6MwGTs  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "79eb1649",
   "metadata": {},
   "outputs": [],
   "source": [
    "import imdlib as imd\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "86e3c30f",
   "metadata": {},
   "outputs": [],
   "source": [
    "settings = {\n",
    "    'tmax': {\n",
    "        'start_yr': 1951, 'end_yr': 2020, 'junk': 99, 'valid_criteria': 'lt'\n",
    "    },\n",
    "    'tmin': {\n",
    "        'start_yr': 1951, 'end_yr': 2020, 'junk': 99, 'valid_criteria': 'lt'\n",
    "    },\n",
    "    'rain': {\n",
    "        'start_yr': 1901, 'end_yr': 2020, 'junk': -900, 'valid_criteria': 'gt'\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7c3746ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "limit = 9\n",
    "start_yr = 1951"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "60457db8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: maxtemp for year 1951\n",
      "Downloading: maxtemp for year 1952\n",
      "Downloading: maxtemp for year 1953\n",
      "Downloading: maxtemp for year 1954\n",
      "Downloading: maxtemp for year 1955\n",
      "Downloading: maxtemp for year 1956\n",
      "Downloading: maxtemp for year 1957\n",
      "Downloading: maxtemp for year 1958\n",
      "Downloading: maxtemp for year 1959\n",
      "Downloading: maxtemp for year 1960\n",
      "Downloading: maxtemp for year 1961\n",
      "Download Successful !!!\n"
     ]
    }
   ],
   "source": [
    "# tmax\n",
    "variable = 'tmax'\n",
    "res = imd.get_data(variable, start_yr, start_yr+limit, fn_format='yearwise')\n",
    "data = imd.open_data(variable, start_yr, start_yr+limit,'yearwise')\n",
    "df_tmax1 = data.get_xarray().to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1582bd01",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tmax2 = df_tmax1[df_tmax1[variable] < settings[variable]['junk']].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2f411ac9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65.1% of 3861k rows data was junk, discarded.\n"
     ]
    }
   ],
   "source": [
    "print(f\"{round(100*(len(df_tmax1) - len(df_tmax2))/len(df_tmax1),1)}% of {len(df_tmax1)//1000}k rows data was junk, discarded.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9457fb45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>tmax</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>31.450001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>30.020000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>31.190001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>30.170000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>9.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>30.180000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346944</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>9.960000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346945</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>79.5</td>\n",
       "      <td>11.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346946</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>8.970000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346947</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>8.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346948</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>8.380000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1346949 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time   lat   lon       tmax\n",
       "0       1951-01-01   8.5  73.5  31.450001\n",
       "1       1951-01-01   8.5  76.5  30.020000\n",
       "2       1951-01-01   8.5  77.5  31.190001\n",
       "3       1951-01-01   8.5  78.5  30.170000\n",
       "4       1951-01-01   9.5  76.5  30.180000\n",
       "...            ...   ...   ...        ...\n",
       "1346944 1961-12-31  33.5  78.5   9.960000\n",
       "1346945 1961-12-31  33.5  79.5  11.090000\n",
       "1346946 1961-12-31  34.5  76.5   8.970000\n",
       "1346947 1961-12-31  34.5  77.5   8.500000\n",
       "1346948 1961-12-31  34.5  78.5   8.380000\n",
       "\n",
       "[1346949 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tmax2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2d3235d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "23317dda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: mintemp for year 1951\n",
      "Downloading: mintemp for year 1952\n",
      "Downloading: mintemp for year 1953\n",
      "Downloading: mintemp for year 1954\n",
      "Downloading: mintemp for year 1955\n",
      "Downloading: mintemp for year 1956\n",
      "Downloading: mintemp for year 1957\n",
      "Downloading: mintemp for year 1958\n",
      "Downloading: mintemp for year 1959\n",
      "Downloading: mintemp for year 1960\n",
      "Downloading: mintemp for year 1961\n",
      "Download Successful !!!\n"
     ]
    }
   ],
   "source": [
    "# tmin\n",
    "variable = 'tmin'\n",
    "res = imd.get_data(variable, start_yr, start_yr+limit, fn_format='yearwise')\n",
    "data = imd.open_data(variable, start_yr, start_yr+limit,'yearwise')\n",
    "df_tmin1 = data.get_xarray().to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ab78fe9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tmin2 = df_tmin1[df_tmin1[variable] < settings[variable]['junk']].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f5b42e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65.1% of 3861k rows data was junk, discarded.\n"
     ]
    }
   ],
   "source": [
    "print(f\"{round(100*(len(df_tmin1) - len(df_tmin2))/len(df_tmin1),1)}% of {len(df_tmin1)//1000}k rows data was junk, discarded.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bdcfd3e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>tmin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>22.469999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>21.760000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>22.930000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>22.990000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>9.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>20.879999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346944</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>0.450000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346945</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>79.5</td>\n",
       "      <td>1.270000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346946</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>-0.720000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346947</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>-1.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346948</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>-1.460000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1346949 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time   lat   lon       tmin\n",
       "0       1951-01-01   8.5  73.5  22.469999\n",
       "1       1951-01-01   8.5  76.5  21.760000\n",
       "2       1951-01-01   8.5  77.5  22.930000\n",
       "3       1951-01-01   8.5  78.5  22.990000\n",
       "4       1951-01-01   9.5  76.5  20.879999\n",
       "...            ...   ...   ...        ...\n",
       "1346944 1961-12-31  33.5  78.5   0.450000\n",
       "1346945 1961-12-31  33.5  79.5   1.270000\n",
       "1346946 1961-12-31  34.5  76.5  -0.720000\n",
       "1346947 1961-12-31  34.5  77.5  -1.650000\n",
       "1346948 1961-12-31  34.5  78.5  -1.460000\n",
       "\n",
       "[1346949 rows x 4 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tmin2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fe88211",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine tmax and tmin data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f5cdaf6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>tmax</th>\n",
       "      <th>tmin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>31.450001</td>\n",
       "      <td>22.469999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>30.020000</td>\n",
       "      <td>21.760000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>31.190001</td>\n",
       "      <td>22.930000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>30.170000</td>\n",
       "      <td>22.990000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>9.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>30.180000</td>\n",
       "      <td>20.879999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346944</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>9.960000</td>\n",
       "      <td>0.450000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346945</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>33.5</td>\n",
       "      <td>79.5</td>\n",
       "      <td>11.090000</td>\n",
       "      <td>1.270000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346946</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>76.5</td>\n",
       "      <td>8.970000</td>\n",
       "      <td>-0.720000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346947</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>77.5</td>\n",
       "      <td>8.500000</td>\n",
       "      <td>-1.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346948</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>34.5</td>\n",
       "      <td>78.5</td>\n",
       "      <td>8.380000</td>\n",
       "      <td>-1.460000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1346949 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time   lat   lon       tmax       tmin\n",
       "0       1951-01-01   8.5  73.5  31.450001  22.469999\n",
       "1       1951-01-01   8.5  76.5  30.020000  21.760000\n",
       "2       1951-01-01   8.5  77.5  31.190001  22.930000\n",
       "3       1951-01-01   8.5  78.5  30.170000  22.990000\n",
       "4       1951-01-01   9.5  76.5  30.180000  20.879999\n",
       "...            ...   ...   ...        ...        ...\n",
       "1346944 1961-12-31  33.5  78.5   9.960000   0.450000\n",
       "1346945 1961-12-31  33.5  79.5  11.090000   1.270000\n",
       "1346946 1961-12-31  34.5  76.5   8.970000  -0.720000\n",
       "1346947 1961-12-31  34.5  77.5   8.500000  -1.650000\n",
       "1346948 1961-12-31  34.5  78.5   8.380000  -1.460000\n",
       "\n",
       "[1346949 rows x 5 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dft1 = pd.merge(df_tmax2, df_tmin2, how='left', on=['time','lat','lon'] )\n",
    "dft1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "737e111e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>tmax</th>\n",
       "      <th>tmin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.090000</td>\n",
       "      <td>17.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>1951-01-02</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.629999</td>\n",
       "      <td>16.719999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>1951-01-03</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>29.160000</td>\n",
       "      <td>15.470000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1072</th>\n",
       "      <td>1951-01-04</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>29.020000</td>\n",
       "      <td>15.430000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1406</th>\n",
       "      <td>1951-01-05</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.770000</td>\n",
       "      <td>15.010000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345344</th>\n",
       "      <td>1961-12-27</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.730000</td>\n",
       "      <td>15.690000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345679</th>\n",
       "      <td>1961-12-28</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>29.260000</td>\n",
       "      <td>14.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346014</th>\n",
       "      <td>1961-12-29</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.379999</td>\n",
       "      <td>13.380000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346349</th>\n",
       "      <td>1961-12-30</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>29.219999</td>\n",
       "      <td>13.370000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346684</th>\n",
       "      <td>1961-12-31</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.5</td>\n",
       "      <td>28.320000</td>\n",
       "      <td>14.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4018 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time   lat   lon       tmax       tmin\n",
       "70      1951-01-01  18.5  73.5  28.090000  17.250000\n",
       "404     1951-01-02  18.5  73.5  28.629999  16.719999\n",
       "738     1951-01-03  18.5  73.5  29.160000  15.470000\n",
       "1072    1951-01-04  18.5  73.5  29.020000  15.430000\n",
       "1406    1951-01-05  18.5  73.5  28.770000  15.010000\n",
       "...            ...   ...   ...        ...        ...\n",
       "1345344 1961-12-27  18.5  73.5  28.730000  15.690000\n",
       "1345679 1961-12-28  18.5  73.5  29.260000  14.600000\n",
       "1346014 1961-12-29  18.5  73.5  28.379999  13.380000\n",
       "1346349 1961-12-30  18.5  73.5  29.219999  13.370000\n",
       "1346684 1961-12-31  18.5  73.5  28.320000  14.400000\n",
       "\n",
       "[4018 rows x 5 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# nr pune: at .5 res, 18.5, 73.5 (tried for 74.0 which is closer, but no data available on tht)\n",
    "dft1_pune = dft1[(dft1['lat']==18.5) & (dft1['lon']==73.5)].copy()\n",
    "dft1_pune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "25b08438",
   "metadata": {},
   "outputs": [],
   "source": [
    "dft1_pune.to_csv('temp_pune.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f7b09d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f065d737",
   "metadata": {},
   "source": [
    "## Rain data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "69327c33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: rain for year 1951\n",
      "Downloading: rain for year 1952\n",
      "Downloading: rain for year 1953\n",
      "Download Successful !!!\n"
     ]
    }
   ],
   "source": [
    "# rain\n",
    "limit = 2 # seems its a lot of data - one yr's file is 24.2MB\n",
    "variable = 'rain'\n",
    "res = imd.get_data(variable, start_yr, start_yr+limit, fn_format='yearwise')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "27cb73dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = imd.open_data(variable, start_yr, start_yr+limit,'yearwise')\n",
    "df_rain1 = data.get_xarray().to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "96296cfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rain2 = df_rain1[df_rain1[variable] > settings[variable]['junk']].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "aa757d7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    13646296.0\n",
       "mean         -999.0\n",
       "std             0.0\n",
       "min          -999.0\n",
       "25%          -999.0\n",
       "50%          -999.0\n",
       "75%          -999.0\n",
       "max          -999.0\n",
       "Name: rain, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ensuring that the discarded data is junk only\n",
    "df_rain1[df_rain1[variable] < settings[variable]['junk']][variable].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dcb2a206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71.5% of 19M rows data was junk, discarded.\n"
     ]
    }
   ],
   "source": [
    "print(f\"{round(100*(len(df_rain1) - len(df_rain2))/len(df_rain1),1)}% of {len(df_rain1)//pow(10,6)}M rows data was junk, discarded.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9ff44d8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.25</td>\n",
       "      <td>77.00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.25</td>\n",
       "      <td>77.25</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.25</td>\n",
       "      <td>77.50</td>\n",
       "      <td>5.228102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.25</td>\n",
       "      <td>77.75</td>\n",
       "      <td>2.281837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>8.50</td>\n",
       "      <td>76.75</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440539</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>37.00</td>\n",
       "      <td>75.50</td>\n",
       "      <td>6.498086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440540</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>37.25</td>\n",
       "      <td>74.50</td>\n",
       "      <td>6.639433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440541</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>37.25</td>\n",
       "      <td>74.75</td>\n",
       "      <td>7.242740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440542</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>37.25</td>\n",
       "      <td>75.00</td>\n",
       "      <td>7.362216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440543</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>37.25</td>\n",
       "      <td>75.25</td>\n",
       "      <td>7.173277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5440544 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time    lat    lon      rain\n",
       "0       1951-01-01   8.25  77.00  0.000000\n",
       "1       1951-01-01   8.25  77.25  0.000000\n",
       "2       1951-01-01   8.25  77.50  5.228102\n",
       "3       1951-01-01   8.25  77.75  2.281837\n",
       "4       1951-01-01   8.50  76.75  0.000000\n",
       "...            ...    ...    ...       ...\n",
       "5440539 1953-12-31  37.00  75.50  6.498086\n",
       "5440540 1953-12-31  37.25  74.50  6.639433\n",
       "5440541 1953-12-31  37.25  74.75  7.242740\n",
       "5440542 1953-12-31  37.25  75.00  7.362216\n",
       "5440543 1953-12-31  37.25  75.25  7.173277\n",
       "\n",
       "[5440544 rows x 4 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rain2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "05f07c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>rain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>954</th>\n",
       "      <td>1951-01-01</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5918</th>\n",
       "      <td>1951-01-02</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10882</th>\n",
       "      <td>1951-01-03</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15846</th>\n",
       "      <td>1951-01-04</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20810</th>\n",
       "      <td>1951-01-05</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5416678</th>\n",
       "      <td>1953-12-27</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5421642</th>\n",
       "      <td>1953-12-28</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5426606</th>\n",
       "      <td>1953-12-29</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5431570</th>\n",
       "      <td>1953-12-30</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5436534</th>\n",
       "      <td>1953-12-31</td>\n",
       "      <td>18.5</td>\n",
       "      <td>73.75</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1096 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              time   lat    lon  rain\n",
       "954     1951-01-01  18.5  73.75   0.0\n",
       "5918    1951-01-02  18.5  73.75   0.0\n",
       "10882   1951-01-03  18.5  73.75   0.0\n",
       "15846   1951-01-04  18.5  73.75   0.0\n",
       "20810   1951-01-05  18.5  73.75   0.0\n",
       "...            ...   ...    ...   ...\n",
       "5416678 1953-12-27  18.5  73.75   0.0\n",
       "5421642 1953-12-28  18.5  73.75   0.0\n",
       "5426606 1953-12-29  18.5  73.75   0.0\n",
       "5431570 1953-12-30  18.5  73.75   0.0\n",
       "5436534 1953-12-31  18.5  73.75   0.0\n",
       "\n",
       "[1096 rows x 4 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pune: if .25 res, then 18.5, 73.75\n",
    "# nr pune\n",
    "dfr1_pune = df_rain2[(df_rain2['lat']==18.5) & (df_rain2['lon']==73.75)].copy()\n",
    "dfr1_pune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f7380d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr1_pune.to_csv('rain_pune.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c643b25b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d9b42ae5",
   "metadata": {},
   "source": [
    "## Visualized here:\n",
    "https://docs.google.com/spreadsheets/d/13j3kMvGdr52_dX-4nG9YDFCu3ZO_sYadFmYeiZJVgWo/edit?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0732463b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}