viirs_export_csv.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "viirs_export_csv.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oYM01t0D4gLJ"
      },
      "source": [
        "\n",
        "**Module:** viirs_export_csv.ipynb\n",
        "\n",
        "**Disclaimer**: The code is for demonstration purposes only. Users are responsible to check for accuracy and revise to fit their objective.\n",
        "\n",
        "**Organization**: NASA ARSET\n",
        "\n",
        "**Author**: Justin Roberts-Pierel and Pawan Gupta, 2015.\n",
        "\n",
        "**Modified to work with netCDF** : Vikalp Mishra, 2019 \n",
        "\n",
        "**Modified to work with VIIRS data**: Aavash Thapa, 2020\n",
        "\n",
        "**Purpose**: To save data into a csv file from a VIIRS Deep Blue netcdf4 file\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nv3UhsBrKaNW"
      },
      "source": [
        "#Mount drive to save files there\n",
        "#clone the repository to access files from there\n",
        "#pull the latest\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "! git clone https://github.com/NASAARSET/VIIRS_NASA.git\n",
        "! git -C VIIRS_NASA/ pull"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTxkRB897HJ9"
      },
      "source": [
        "! pip install netCDF4\n",
        "from netCDF4 import Dataset\n",
        "import numpy as np\n",
        "import sys\n",
        "import time\n",
        "import calendar\n",
        "import datetime as dt\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wpy4FnpQvFtI"
      },
      "source": [
        "\n",
        "#!/usr/bin/python      \n",
        "\n",
        "#This finds the user's current path so that all hdf4 files can be found\n",
        "try:\n",
        "    fileList = open('VIIRS_NASA/fileList.txt', 'r')\n",
        "\n",
        "except:\n",
        "    print('Did not find a text file containing file names (perhaps name does not match)')\n",
        "    sys.exit()\n",
        "\n",
        "#loops through all files listed in the text file\n",
        "for FILE_NAME in fileList:\n",
        "    FILE_NAME=FILE_NAME.strip()\n",
        "    user_input=input('\\nWould you like to process\\n' + FILE_NAME + '\\n\\n(Y/N)')\n",
        "    if (user_input == 'N' or user_input == 'n'):\n",
        "        print('Skipping...')\n",
        "        continue\n",
        "    else:\n",
        "        file = Dataset('VIIRS_NASA/' + FILE_NAME, 'r')\n",
        "# read the data\n",
        "        if 'AERDB' in FILE_NAME:\n",
        "            print('This is a VIIRS Deep Blue file.')\n",
        "            #this is how you access the data tree in an hdf5 file\n",
        "            SDS_NAME='Aerosol_Optical_Thickness_550_Land_Best_Estimate'    \n",
        "        ds=file  \n",
        "        lat= ds.variables['Latitude'][:][:]\n",
        "        lon= ds.variables['Longitude'][:][:]\n",
        "        data= ds.variables[SDS_NAME]\n",
        "\n",
        "        #get necessary attributes \n",
        "        fv=data._FillValue\n",
        "          \n",
        "        fileparts=FILE_NAME.split('.')\n",
        "\n",
        "        #There are some columns that are going to be the same\n",
        "        #like the year, month and so on listed below.\n",
        "        #Therefore, we can make the columns for them to store\n",
        "        #the data for every row.\n",
        "        year = np.zeros(lat.shape)\n",
        "        mth = np.zeros(lat.shape)\n",
        "        doy = np.zeros(lat.shape)\n",
        "        hr = np.zeros(lat.shape)\n",
        "        mn = np.zeros(lat.shape)\n",
        "        \n",
        "        for i in range(0,lat.shape[0]):\n",
        "            y= fileparts[1][1:5]\n",
        "            h = fileparts[2][0:2]\n",
        "            m = fileparts[2][2:4]\n",
        "            date = y + ',' + fileparts[1][5:8] + ',' + h + ',' + m\n",
        "            t2 = dt.datetime.strptime(date,'%Y,%j,%H,%M')\n",
        "           \n",
        "            mt = t2.month\n",
        "            d = t2.day\n",
        "            \n",
        "            year[i][:] = y\n",
        "            mth[i][:] = mt\n",
        "            doy[i][:] = d\n",
        "            hr[i][:] = h\n",
        "            mn[i][:] = m\n",
        "            \n",
        "        vlist = list(file.variables.keys())\n",
        "        \n",
        "        #create the dataframe and enter the values here\n",
        "        df = pd.DataFrame()\n",
        "        df['Year'] = year.ravel()\n",
        "        df['Month'] = mth.ravel()\n",
        "        df['Day'] = doy.ravel()\n",
        "        df['Hour'] = hr.ravel()\n",
        "        df['Minute'] = mn.ravel()\n",
        "        \n",
        "        #0-->Aerosol_Optical_Thickness_550_Land\n",
        "        #3-->Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate\n",
        "        #8-->Aerosol_Optical_Thickness_QA_Flag_Land\n",
        "        #11-->Aerosol_Type_Land_Ocean\n",
        "        #18-->Angstrom_Exponent_Land_Ocean_Best_Estimate\n",
        "        sds_lst = [ 'Aerosol_Optical_Thickness_550_Land',\n",
        "                   'Aerosol_Optical_Thickness_550_Land_Ocean_Best_Estimate',\n",
        "                   'Aerosol_Optical_Thickness_QA_Flag_Land',\n",
        "                   'Aerosol_Type_Land_Ocean',\n",
        "                   'Angstrom_Exponent_Land_Ocean_Best_Estimate']\n",
        "        \n",
        "        #This for loop saves all of the SDS in the dictionary at the top (dependent on file type) to the array (with titles)\n",
        "        #All the sds that we need seem to be contained in this range.\n",
        "        #Can extend this range to loop through more sds variables in the NC file.\n",
        "        for i in range(0,20):\n",
        "            SDS_NAME=vlist[(i)] # The name of the sds to read\n",
        "            \n",
        "            if SDS_NAME in sds_lst:\n",
        "                print('SDS_NAME', SDS_NAME)\n",
        "                #try:\n",
        "                sds=ds.variables[SDS_NAME]\n",
        "               \n",
        "                scale = 1.0\n",
        "                fv=sds._FillValue\n",
        "                #get SDS data as a vector\n",
        "                data=sds[:].ravel()\n",
        "               #The next few lines change fill value/missing value to NaN so that we can multiply valid values by the scale factor, then back to fill values for saving\n",
        "                data=data.astype(float)\n",
        "                data=(data)*scale  \n",
        "                data[np.isnan(data)]=fv\n",
        "                data[data==float(fv)]=np.nan\n",
        "                data=np.array(data[:])\n",
        "                df[SDS_NAME] = data\n",
        "    \n",
        "    outfilename=FILE_NAME[:-3]+'.csv'    \n",
        "    df.to_csv(\"drive/My Drive/Colab Notebooks/\" + outfilename, index = False) \n",
        "    print('\\nAll files have been saved successfully.')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}