function_tester.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "%autoreload\n",
    "\n",
    "import os\n",
    "\n",
    "sys.path.append('../')\n",
    "\n",
    "from utilities import create_data as create\n",
    "\n",
    "home_dir = os.path.join(os.path.expanduser('~'), 'Documents/TDT4173-ML')\n",
    "saved_data_path = os.path.join(home_dir, 'data_processed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impute A successful\n",
      "Impute B successful\n",
      "Impute C successful\n",
      "absolute_humidity_2m:gm3          0\n",
      "air_density_2m:kgm3               0\n",
      "clear_sky_energy_1h:J             0\n",
      "clear_sky_rad:W                   0\n",
      "dew_or_rime:idx                   0\n",
      "dew_point_2m:K                    0\n",
      "diffuse_rad:W                     0\n",
      "diffuse_rad_1h:J                  0\n",
      "direct_rad:W                      0\n",
      "direct_rad_1h:J                   0\n",
      "effective_cloud_cover:p           0\n",
      "elevation:m                       0\n",
      "fresh_snow_12h:cm                 0\n",
      "fresh_snow_1h:cm                  0\n",
      "fresh_snow_24h:cm                 0\n",
      "fresh_snow_3h:cm                  0\n",
      "fresh_snow_6h:cm                  0\n",
      "is_day:idx                        0\n",
      "is_in_shadow:idx                  0\n",
      "msl_pressure:hPa                  0\n",
      "precip_5min:mm                    0\n",
      "precip_type_5min:idx              0\n",
      "pressure_100m:hPa                 0\n",
      "pressure_50m:hPa                  0\n",
      "prob_rime:p                       0\n",
      "rain_water:kgm2                   0\n",
      "relative_humidity_1000hPa:p       0\n",
      "sfc_pressure:hPa                  0\n",
      "snow_depth:cm                     0\n",
      "snow_drift:idx                    0\n",
      "snow_melt_10min:mm                0\n",
      "snow_water:kgm2                   0\n",
      "sun_azimuth:d                     0\n",
      "sun_elevation:d                   0\n",
      "super_cooled_liquid_water:kgm2    0\n",
      "t_1000hPa:K                       0\n",
      "total_cloud_cover:p               0\n",
      "visibility:m                      0\n",
      "wind_speed_10m:ms                 0\n",
      "wind_speed_u_10m:ms               0\n",
      "wind_speed_v_10m:ms               0\n",
      "wind_speed_w_1000hPa:ms           0\n",
      "ceiling_height_agl:m              0\n",
      "cloud_base_agl:m                  0\n",
      "pv_measurement                    0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "data = create.CreateData()\n",
    "\n",
    "train = data.create_training_data(use_mean_values=True, impute_data=True)\n",
    "\n",
    "print(train.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(92951, 45)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train.to_csv(os.path.join(saved_data_path, 'train.csv'), index=False)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impute A successfull\n",
      "Impute B successfull\n",
      "Impute C successfull\n",
      "absolute_humidity_2m:gm3          0\n",
      "air_density_2m:kgm3               0\n",
      "clear_sky_energy_1h:J             0\n",
      "clear_sky_rad:W                   0\n",
      "dew_or_rime:idx                   0\n",
      "dew_point_2m:K                    0\n",
      "diffuse_rad:W                     0\n",
      "diffuse_rad_1h:J                  0\n",
      "direct_rad:W                      0\n",
      "direct_rad_1h:J                   0\n",
      "effective_cloud_cover:p           0\n",
      "elevation:m                       0\n",
      "fresh_snow_12h:cm                 0\n",
      "fresh_snow_1h:cm                  0\n",
      "fresh_snow_24h:cm                 0\n",
      "fresh_snow_3h:cm                  0\n",
      "fresh_snow_6h:cm                  0\n",
      "is_day:idx                        0\n",
      "is_in_shadow:idx                  0\n",
      "msl_pressure:hPa                  0\n",
      "precip_5min:mm                    0\n",
      "precip_type_5min:idx              0\n",
      "pressure_100m:hPa                 0\n",
      "pressure_50m:hPa                  0\n",
      "prob_rime:p                       0\n",
      "rain_water:kgm2                   0\n",
      "relative_humidity_1000hPa:p       0\n",
      "sfc_pressure:hPa                  0\n",
      "snow_depth:cm                     0\n",
      "snow_drift:idx                    0\n",
      "snow_melt_10min:mm                0\n",
      "snow_water:kgm2                   0\n",
      "sun_azimuth:d                     0\n",
      "sun_elevation:d                   0\n",
      "super_cooled_liquid_water:kgm2    0\n",
      "t_1000hPa:K                       0\n",
      "total_cloud_cover:p               0\n",
      "visibility:m                      0\n",
      "wind_speed_10m:ms                 0\n",
      "wind_speed_u_10m:ms               0\n",
      "wind_speed_v_10m:ms               0\n",
      "wind_speed_w_1000hPa:ms           0\n",
      "ceiling_height_agl:m              0\n",
      "cloud_base_agl:m                  0\n",
      "pv_measurement                    0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "data1 = create.CreateData()\n",
    "\n",
    "train1 = data1.create_training_data(use_mean_values=True, impute_data=True)\n",
    "\n",
    "print(train1.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(92951, 45)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impute A successfull\n",
      "Impute B successfull\n",
      "Impute C successfull\n",
      "date_calc                         0\n",
      "absolute_humidity_2m:gm3          0\n",
      "air_density_2m:kgm3               0\n",
      "clear_sky_energy_1h:J             0\n",
      "clear_sky_rad:W                   0\n",
      "dew_or_rime:idx                   0\n",
      "dew_point_2m:K                    0\n",
      "diffuse_rad:W                     0\n",
      "diffuse_rad_1h:J                  0\n",
      "direct_rad:W                      0\n",
      "direct_rad_1h:J                   0\n",
      "effective_cloud_cover:p           0\n",
      "elevation:m                       0\n",
      "fresh_snow_12h:cm                 0\n",
      "fresh_snow_1h:cm                  0\n",
      "fresh_snow_24h:cm                 0\n",
      "fresh_snow_3h:cm                  0\n",
      "fresh_snow_6h:cm                  0\n",
      "is_day:idx                        0\n",
      "is_in_shadow:idx                  0\n",
      "msl_pressure:hPa                  0\n",
      "precip_5min:mm                    0\n",
      "precip_type_5min:idx              0\n",
      "pressure_100m:hPa                 0\n",
      "pressure_50m:hPa                  0\n",
      "prob_rime:p                       0\n",
      "rain_water:kgm2                   0\n",
      "relative_humidity_1000hPa:p       0\n",
      "sfc_pressure:hPa                  0\n",
      "snow_depth:cm                     0\n",
      "snow_drift:idx                    0\n",
      "snow_melt_10min:mm                0\n",
      "snow_water:kgm2                   0\n",
      "sun_azimuth:d                     0\n",
      "sun_elevation:d                   0\n",
      "super_cooled_liquid_water:kgm2    0\n",
      "t_1000hPa:K                       0\n",
      "total_cloud_cover:p               0\n",
      "visibility:m                      0\n",
      "wind_speed_10m:ms                 0\n",
      "wind_speed_u_10m:ms               0\n",
      "wind_speed_v_10m:ms               0\n",
      "wind_speed_w_1000hPa:ms           0\n",
      "ceiling_height_agl:m              0\n",
      "cloud_base_agl:m                  0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "data2 = create.CreateData()\n",
    "\n",
    "test = data2.create_test_data(use_mean_values=True, impute_data=True)\n",
    "\n",
    "print(test.isna().sum())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2160, 45)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_calc</th>\n",
       "      <th>absolute_humidity_2m:gm3</th>\n",
       "      <th>air_density_2m:kgm3</th>\n",
       "      <th>clear_sky_energy_1h:J</th>\n",
       "      <th>clear_sky_rad:W</th>\n",
       "      <th>dew_or_rime:idx</th>\n",
       "      <th>dew_point_2m:K</th>\n",
       "      <th>diffuse_rad:W</th>\n",
       "      <th>diffuse_rad_1h:J</th>\n",
       "      <th>direct_rad:W</th>\n",
       "      <th>...</th>\n",
       "      <th>super_cooled_liquid_water:kgm2</th>\n",
       "      <th>t_1000hPa:K</th>\n",
       "      <th>total_cloud_cover:p</th>\n",
       "      <th>visibility:m</th>\n",
       "      <th>wind_speed_10m:ms</th>\n",
       "      <th>wind_speed_u_10m:ms</th>\n",
       "      <th>wind_speed_v_10m:ms</th>\n",
       "      <th>wind_speed_w_1000hPa:ms</th>\n",
       "      <th>ceiling_height_agl:m</th>\n",
       "      <th>cloud_base_agl:m</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_forecast</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-05-01 00:00:00</th>\n",
       "      <td>2023-04-30 07:00:04</td>\n",
       "      <td>4.325</td>\n",
       "      <td>1.28675</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>271.650024</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>273.799988</td>\n",
       "      <td>74.949997</td>\n",
       "      <td>29907.500000</td>\n",
       "      <td>3.950</td>\n",
       "      <td>2.100</td>\n",
       "      <td>3.35</td>\n",
       "      <td>0.0</td>\n",
       "      <td>912.700012</td>\n",
       "      <td>1061.550049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-05-01 01:00:00</th>\n",
       "      <td>2023-04-30 07:00:04</td>\n",
       "      <td>4.275</td>\n",
       "      <td>1.28600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>271.450012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>273.799988</td>\n",
       "      <td>77.474998</td>\n",
       "      <td>29519.074219</td>\n",
       "      <td>3.825</td>\n",
       "      <td>1.925</td>\n",
       "      <td>3.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1571.974976</td>\n",
       "      <td>1075.100098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-05-01 02:00:00</th>\n",
       "      <td>2023-04-30 07:00:04</td>\n",
       "      <td>4.150</td>\n",
       "      <td>1.28375</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>271.049988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>273.849976</td>\n",
       "      <td>88.099998</td>\n",
       "      <td>31009.125000</td>\n",
       "      <td>3.650</td>\n",
       "      <td>1.750</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1791.300049</td>\n",
       "      <td>1200.400024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-05-01 03:00:00</th>\n",
       "      <td>2023-04-30 07:00:04</td>\n",
       "      <td>4.025</td>\n",
       "      <td>1.28200</td>\n",
       "      <td>10124.424805</td>\n",
       "      <td>11.675</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270.649994</td>\n",
       "      <td>9.375000</td>\n",
       "      <td>16845.226562</td>\n",
       "      <td>2.100000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>273.899994</td>\n",
       "      <td>68.599998</td>\n",
       "      <td>34552.500000</td>\n",
       "      <td>3.500</td>\n",
       "      <td>1.450</td>\n",
       "      <td>3.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2312.875000</td>\n",
       "      <td>1179.849976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-05-01 04:00:00</th>\n",
       "      <td>2023-04-30 07:00:04</td>\n",
       "      <td>3.900</td>\n",
       "      <td>1.28100</td>\n",
       "      <td>141748.593750</td>\n",
       "      <td>76.875</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270.375000</td>\n",
       "      <td>47.400002</td>\n",
       "      <td>102209.703125</td>\n",
       "      <td>25.450001</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>273.924988</td>\n",
       "      <td>66.300003</td>\n",
       "      <td>35483.875000</td>\n",
       "      <td>3.325</td>\n",
       "      <td>1.300</td>\n",
       "      <td>3.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2198.299805</td>\n",
       "      <td>920.049988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              date_calc  absolute_humidity_2m:gm3  \\\n",
       "date_forecast                                                       \n",
       "2023-05-01 00:00:00 2023-04-30 07:00:04                     4.325   \n",
       "2023-05-01 01:00:00 2023-04-30 07:00:04                     4.275   \n",
       "2023-05-01 02:00:00 2023-04-30 07:00:04                     4.150   \n",
       "2023-05-01 03:00:00 2023-04-30 07:00:04                     4.025   \n",
       "2023-05-01 04:00:00 2023-04-30 07:00:04                     3.900   \n",
       "\n",
       "                     air_density_2m:kgm3  clear_sky_energy_1h:J  \\\n",
       "date_forecast                                                     \n",
       "2023-05-01 00:00:00              1.28675               0.000000   \n",
       "2023-05-01 01:00:00              1.28600               0.000000   \n",
       "2023-05-01 02:00:00              1.28375               0.000000   \n",
       "2023-05-01 03:00:00              1.28200           10124.424805   \n",
       "2023-05-01 04:00:00              1.28100          141748.593750   \n",
       "\n",
       "                     clear_sky_rad:W  dew_or_rime:idx  dew_point_2m:K  \\\n",
       "date_forecast                                                           \n",
       "2023-05-01 00:00:00            0.000              0.0      271.650024   \n",
       "2023-05-01 01:00:00            0.000              0.0      271.450012   \n",
       "2023-05-01 02:00:00            0.000              0.0      271.049988   \n",
       "2023-05-01 03:00:00           11.675              0.0      270.649994   \n",
       "2023-05-01 04:00:00           76.875              0.0      270.375000   \n",
       "\n",
       "                     diffuse_rad:W  diffuse_rad_1h:J  direct_rad:W  ...  \\\n",
       "date_forecast                                                       ...   \n",
       "2023-05-01 00:00:00       0.000000          0.000000      0.000000  ...   \n",
       "2023-05-01 01:00:00       0.000000          0.000000      0.000000  ...   \n",
       "2023-05-01 02:00:00       0.000000          0.000000      0.000000  ...   \n",
       "2023-05-01 03:00:00       9.375000      16845.226562      2.100000  ...   \n",
       "2023-05-01 04:00:00      47.400002     102209.703125     25.450001  ...   \n",
       "\n",
       "                     super_cooled_liquid_water:kgm2  t_1000hPa:K  \\\n",
       "date_forecast                                                      \n",
       "2023-05-01 00:00:00                             0.0   273.799988   \n",
       "2023-05-01 01:00:00                             0.0   273.799988   \n",
       "2023-05-01 02:00:00                             0.0   273.849976   \n",
       "2023-05-01 03:00:00                             0.0   273.899994   \n",
       "2023-05-01 04:00:00                             0.0   273.924988   \n",
       "\n",
       "                     total_cloud_cover:p  visibility:m  wind_speed_10m:ms  \\\n",
       "date_forecast                                                               \n",
       "2023-05-01 00:00:00            74.949997  29907.500000              3.950   \n",
       "2023-05-01 01:00:00            77.474998  29519.074219              3.825   \n",
       "2023-05-01 02:00:00            88.099998  31009.125000              3.650   \n",
       "2023-05-01 03:00:00            68.599998  34552.500000              3.500   \n",
       "2023-05-01 04:00:00            66.300003  35483.875000              3.325   \n",
       "\n",
       "                     wind_speed_u_10m:ms  wind_speed_v_10m:ms  \\\n",
       "date_forecast                                                   \n",
       "2023-05-01 00:00:00                2.100                 3.35   \n",
       "2023-05-01 01:00:00                1.925                 3.30   \n",
       "2023-05-01 02:00:00                1.750                 3.20   \n",
       "2023-05-01 03:00:00                1.450                 3.15   \n",
       "2023-05-01 04:00:00                1.300                 3.05   \n",
       "\n",
       "                     wind_speed_w_1000hPa:ms  ceiling_height_agl:m  \\\n",
       "date_forecast                                                        \n",
       "2023-05-01 00:00:00                      0.0            912.700012   \n",
       "2023-05-01 01:00:00                      0.0           1571.974976   \n",
       "2023-05-01 02:00:00                      0.0           1791.300049   \n",
       "2023-05-01 03:00:00                      0.0           2312.875000   \n",
       "2023-05-01 04:00:00                      0.0           2198.299805   \n",
       "\n",
       "                     cloud_base_agl:m  \n",
       "date_forecast                          \n",
       "2023-05-01 00:00:00       1061.550049  \n",
       "2023-05-01 01:00:00       1075.100098  \n",
       "2023-05-01 02:00:00       1200.400024  \n",
       "2023-05-01 03:00:00       1179.849976  \n",
       "2023-05-01 04:00:00        920.049988  \n",
       "\n",
       "[5 rows x 45 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}