Code/UDP_Consumption_Forecasting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "6cbeeac8-e656-4092-847c-16b84099ab92",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "%run \"./Imports\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "71ba8b35-d2bc-4872-8a7e-1d6512fdf0f4",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%run \"./Model_Tuning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "9217cbf4-b0d0-4349-b291-be0347c2574b",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%run \"./General_Functions\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "fd5e664a-e5c4-4a79-ae22-cd19d47318da",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Nerdearla 2021 - E2E Unified Demand Planning: Demand Forecasting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "db593455-d711-40a9-9fcc-5099b7146da9",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "This notebook contains the code of the Demand Forecasting Pipeline, which is the process used to tune (train +\n",
    "validation) and back-test the models for each product (SKU) according to the different defined experiments. In this\n",
    "context, an experiment is a scenario composed of:\n",
    "\n",
    "  * Algorithm, such as Prophet, SARIMAX, XGBoost.\n",
    "  * Set of input features (excluded from the scope of the workshop)\n",
    "\n",
    "For each product this pipeline will train, tune, yield a best model (best set of hyperparameters) and backtest it for\n",
    "each of the different experiments; which means that every product will have as many \"best models\" as the number of\n",
    "experiments. The decision about which of these models to use as the final model is done by selecting the better on in\n",
    "terms of the validation WAPE.\n",
    "\n",
    "The result of this process consists of logging for all products the best model of each experiment into the Mlflow\n",
    "tracking API and generating the forecast for the back-testing period with each one of these, after that only the\n",
    "forecast corresponding to the best model among all the experiments is kept.\n",
    "\n",
    "The functions included are:\n",
    "\n",
    "| Function | Description |\n",
    "| -------- | ----------- |\n",
    "| `obtain_models` | for each product, obtains the best forecasting model per experiment |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "289dd2be-b148-4ee4-a432-9f2d62f140c1",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "###### Initializing variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "83df40c6-0c9d-445c-927a-c12773b8f262",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Experiment variables\n",
    "algorithms = [\"prophet\", \"sarimax\"]\n",
    "holidays = False\n",
    "num_evals = 10\n",
    "\n",
    "# Dates for validation\n",
    "start_val = \"2019-01-01\"\n",
    "end_val = \"2019-06-30\"\n",
    "\n",
    "# Dates for testing\n",
    "start_test = \"2019-07-01\"\n",
    "end_test = \"2019-12-31\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "2447cbbd-fb2e-4060-b2d1-b5d13eda09e8",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### Defining search space of each algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "441a1f45-5305-4e0c-98f6-15f0035c07cd",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Defining search space for prophet\n",
    "params_prophet = {\n",
    "    \"changepoint_prior_scale\":  hp.loguniform(\"changepoint_prior_scale\", np.log(0.001), np.log(0.5)),\n",
    "    \"seasonality_prior_scale\": hp.loguniform(\"seasonality_prior_scale\", np.log(0.01), np.log(10)),\n",
    "    \"holidays_prior_scale\": hp.loguniform(\"holidays_prior_scale\", np.log(0.01), np.log(10)),\n",
    "}\n",
    "\n",
    "# Defining search space for sarimax\n",
    "params_sarimax = {\n",
    "    \"p\": hp.choice(\"p\", [0, 1, 2]),\n",
    "    \"d\": hp.choice(\"d\", [0, 1]),\n",
    "    \"q\": hp.choice(\"q\", [0, 1, 2]),\n",
    "    \"P\": hp.choice(\"P\", [0, 1, 2]),\n",
    "    \"D\": hp.choice(\"D\", [0, 1]),\n",
    "    \"Q\": hp.choice(\"Q\", [0, 1, 2]),\n",
    "    \"s\": 12\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "c4298942-32a4-401c-a4cc-fc33938c2570",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### Setting Mlflow experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a39f1db3-cdc1-46b2-98b5-6395b2e5292f",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Defining experiment path\n",
    "mlflow_exp = r\"/UDP_E2E_Forecasting/nerdearla_udp_consumption\"\n",
    "\n",
    "# Launching Mlflow client\n",
    "client = MlflowClient()\n",
    "\n",
    "# Creating experiment or re-using it if already exists\n",
    "experiment = client.get_experiment_by_name(mlflow_exp)\n",
    "if experiment is None:\n",
    "    exp_id = mlflow.create_experiment(mlflow_exp)\n",
    "else:\n",
    "    exp_id = experiment.experiment_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "ae9018f9-d202-45ea-a284-eefa0b025a17",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### Defining modeling function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a0c6c6b4-d1b5-4fe8-8913-b7ea06d0436a",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def obtain_models(data, df_frds):\n",
    "    \"\"\"\n",
    "    For each product, obtains the best forecasting model per experiment; where an experiment is defined by an algorithm\n",
    "    in the context of the workshop.\n",
    "\n",
    "    Obtaining the best model of each experiment is done by performing hyperparameter tuning, which involves training and\n",
    "    validating multiple sets of hyperparameters to then select the best performing set according to a specific metric on\n",
    "    the validation set. Finally, all the best models from the experiments are used to generate the forecast for the\n",
    "    back-testing set and their performances on that set are recorded.\n",
    "\n",
    "    Results for each experiment such as train WAPE, validation WAPE and test WAPE are logged into Mlflow.\n",
    "\n",
    "    Parameters\n",
    "    __________\n",
    "        data (pd.DataFrame): Dataset with the time series of the product.\n",
    "        df_frds (pd.DataFrame): Dataset with holidays.\n",
    "\n",
    "    Returns\n",
    "    _______\n",
    "        df_forecasts (pd.DataFrame): Table with the forecasts of the best models for the back-testing set.\n",
    "    \"\"\"\n",
    "    # Ensuring order of observations\n",
    "    data = data.sort_values(by=\"ds\", ascending=True).reset_index(drop=True)\n",
    "\n",
    "    # Obtaining product info\n",
    "    sku = data[\"n_sku\"][0]\n",
    "\n",
    "    # Splitting the series\n",
    "    df_trainval, df_test = split_series(data, start_test, end_test)\n",
    "\n",
    "    # Defining the output object\n",
    "    df_forecasts = pd.DataFrame()\n",
    "\n",
    "    # Looping over the algorithms\n",
    "    for algorithm in algorithms:\n",
    "        # Validating the algorithm to use\n",
    "        if algorithm == \"sarimax\":\n",
    "            search_space = params_sarimax\n",
    "        elif algorithm == \"prophet\":\n",
    "            search_space = params_prophet\n",
    "\n",
    "        # Tuning the model\n",
    "        results = tune_ts_model(\n",
    "            algorithm, search_space, num_evals, df_trainval, start_val, end_val, holidays=holidays, df_frds=None\n",
    "        )\n",
    "\n",
    "        # Re-fitting model and generating forecast\n",
    "        df_fcst_alg, test_wape = refit_generate_forecast(\n",
    "            algorithm, results[\"params\"], df_trainval, df_test, holidays, df_frds\n",
    "        )\n",
    "\n",
    "        # Appending forecast to the output object\n",
    "        df_forecasts = df_forecasts.append(df_fcst_alg)\n",
    "\n",
    "        # Creando figura con las curvas de actuals y forecasts\n",
    "        fig = plt.figure(figsize=(20, 5))\n",
    "        plt.plot(data[\"ds\"].values, data[\"y\"].values, label=\"actuals\")\n",
    "        plt.plot(df_fcst_alg[\"ds\"].values, df_fcst_alg[\"fcst\"].values, label=\"forecasts\", color=\"red\")\n",
    "        plt.legend()\n",
    "        plt.grid()\n",
    "\n",
    "        # Starting run and assigning tags\n",
    "        mlflow.start_run(experiment_id=exp_id, run_name=str(sku))\n",
    "        mlflow.set_tags(\n",
    "           {\n",
    "            \"experiment\": \"Nerdearla 2021\",\n",
    "            \"product\": sku,\n",
    "            \"algorithm\": algorithm\n",
    "           }\n",
    "        )\n",
    "\n",
    "        # Logging results in Mlflow\n",
    "        mlflow.log_metrics({\"train_wape\": results[\"train_wape\"], \"val_wape\": results[\"val_wape\"], \"test_wape\": test_wape})\n",
    "        mlflow.log_figure(fig, \"forecast_demand.png\")\n",
    "\n",
    "        # Ending run\n",
    "        mlflow.end_run()\n",
    "\n",
    "    # Adding identification column\n",
    "    df_forecasts[\"n_sku\"] = sku\n",
    "\n",
    "    return df_forecasts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "27e011dd-8a99-4ee6-bf67-016cb48fe12d",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "##### Training pipeline main code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "532cb83f-e20a-4953-a783-79991c85028f",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "###### 1. Loading the preprocessed data from Delta table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "800af28d-2da0-49c2-a6bc-8dafa94f6460",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "feriados = pd.DataFrame({\"ds\": [\"1949-03-01\", \"1954-03-01\", \"1959-03-01\", \"1964-03-01\"], \"holiday\": [\"a\", \"a\", \"a\", \"a\"]})\n",
    "feriados[\"ds\"] = pd.to_datetime(feriados[\"ds\"])\n",
    "\n",
    "df_data = spark.read.csv(r\"/FileStore/tables/raw_consumption_data_clean.csv\", sep=',', header=True, inferSchema=True)\n",
    "df_data = df_data.withColumn(\"ds\", to_date(df_data[\"ds\"], \"yyyy-MM-dd\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "3412dfad-38d2-4a83-aee8-481ac1107a08",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### 2. Performing modeling of SKUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "f1b76dcf-1dc3-4932-915a-92e728c5b045",
     "showTitle": false,
     "title": ""
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>algorithm</th><th>ds</th><th>fcst</th><th>n_sku</th><th>exp_key</th></tr></thead><tbody><tr><td>prophet</td><td>1958-09-01</td><td>420.96454</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1958-10-01</td><td>390.5454</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1958-11-01</td><td>364.838</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1958-12-01</td><td>391.77945</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-01-01</td><td>399.5302</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-02-01</td><td>396.8166</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-03-01</td><td>428.68533</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-04-01</td><td>422.648</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-05-01</td><td>426.91406</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-06-01</td><td>463.09787</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-07-01</td><td>496.06696</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-08-01</td><td>495.0767</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-09-01</td><td>458.8408</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-10-01</td><td>428.62274</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-11-01</td><td>403.22598</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1959-12-01</td><td>430.29724</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-01-01</td><td>437.43195</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-02-01</td><td>435.24017</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-03-01</td><td>469.1634</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-04-01</td><td>462.53842</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-05-01</td><td>464.00095</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-06-01</td><td>501.43695</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-07-01</td><td>531.9415</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-08-01</td><td>534.6722</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-09-01</td><td>497.63693</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-10-01</td><td>466.8577</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-11-01</td><td>440.55038</td><td>1</td><td>1_prophet</td></tr><tr><td>prophet</td><td>1960-12-01</td><td>467.1395</td><td>1</td><td>1_prophet</td></tr><tr><td>sarimax</td><td>1958-09-01</td><td>441.04758</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1958-10-01</td><td>383.4599</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1958-11-01</td><td>340.56912</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1958-12-01</td><td>371.62842</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-01-01</td><td>375.03467</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-02-01</td><td>352.30878</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-03-01</td><td>395.36584</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-04-01</td><td>380.65823</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-05-01</td><td>396.6769</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-06-01</td><td>468.63712</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-07-01</td><td>525.6049</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-08-01</td><td>540.5417</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-09-01</td><td>476.66022</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-10-01</td><td>419.11697</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-11-01</td><td>376.45737</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1959-12-01</td><td>407.3586</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-01-01</td><td>410.90823</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-02-01</td><td>388.29736</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-03-01</td><td>431.48444</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-04-01</td><td>416.9464</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-05-01</td><td>432.7006</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-06-01</td><td>504.84085</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-07-01</td><td>561.72986</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-08-01</td><td>576.5718</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-09-01</td><td>512.6236</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-10-01</td><td>455.0389</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-11-01</td><td>412.23282</td><td>1</td><td>1_sarimax</td></tr><tr><td>sarimax</td><td>1960-12-01</td><td>443.21664</td><td>1</td><td>1_sarimax</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         "prophet",
         "1958-09-01",
         420.96454,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1958-10-01",
         390.5454,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1958-11-01",
         364.838,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1958-12-01",
         391.77945,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-01-01",
         399.5302,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-02-01",
         396.8166,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-03-01",
         428.68533,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-04-01",
         422.648,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-05-01",
         426.91406,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-06-01",
         463.09787,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-07-01",
         496.06696,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-08-01",
         495.0767,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-09-01",
         458.8408,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-10-01",
         428.62274,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-11-01",
         403.22598,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1959-12-01",
         430.29724,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-01-01",
         437.43195,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-02-01",
         435.24017,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-03-01",
         469.1634,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-04-01",
         462.53842,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-05-01",
         464.00095,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-06-01",
         501.43695,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-07-01",
         531.9415,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-08-01",
         534.6722,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-09-01",
         497.63693,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-10-01",
         466.8577,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-11-01",
         440.55038,
         1,
         "1_prophet"
        ],
        [
         "prophet",
         "1960-12-01",
         467.1395,
         1,
         "1_prophet"
        ],
        [
         "sarimax",
         "1958-09-01",
         441.04758,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1958-10-01",
         383.4599,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1958-11-01",
         340.56912,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1958-12-01",
         371.62842,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-01-01",
         375.03467,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-02-01",
         352.30878,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-03-01",
         395.36584,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-04-01",
         380.65823,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-05-01",
         396.6769,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-06-01",
         468.63712,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-07-01",
         525.6049,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-08-01",
         540.5417,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-09-01",
         476.66022,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-10-01",
         419.11697,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-11-01",
         376.45737,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1959-12-01",
         407.3586,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-01-01",
         410.90823,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-02-01",
         388.29736,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-03-01",
         431.48444,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-04-01",
         416.9464,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-05-01",
         432.7006,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-06-01",
         504.84085,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-07-01",
         561.72986,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-08-01",
         576.5718,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-09-01",
         512.6236,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-10-01",
         455.0389,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-11-01",
         412.23282,
         1,
         "1_sarimax"
        ],
        [
         "sarimax",
         "1960-12-01",
         443.21664,
         1,
         "1_sarimax"
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": [],
        "xColumns": [],
        "yColumns": []
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "algorithm",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "ds",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "fcst",
         "type": "\"float\""
        },
        {
         "metadata": "{}",
         "name": "n_sku",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "exp_key",
         "type": "\"string\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Defining schema of the resulting dataframe:\n",
    "result_schema = StructType(\n",
    "    [\n",
    "     StructField(\"algorithm\", StringType(), False),\n",
    "     StructField(\"ds\", DateType(), False),\n",
    "     StructField(\"fcst\", FloatType(), False),\n",
    "     StructField(\"n_sku\", IntegerType(), False)\n",
    "    ]\n",
    ")\n",
    "\n",
    "# Performing modeling of the DFUs\n",
    "df_fcsts = df_data.groupBy(\"n_sku\") \\\n",
    "    .applyInPandas(\n",
    "        lambda df: obtain_models(df, feriados),\n",
    "        result_schema\n",
    "    ) \\\n",
    "    .persist(StorageLevel.MEMORY_ONLY)\n",
    "\n",
    "# Adding identification key of experiments\n",
    "df_fcsts = df_fcsts.withColumn(\"exp_key\", concat(df_fcsts[\"n_sku\"], lit(\"_\"), df_fcsts[\"algorithm\"]))\n",
    "rows = df_fcsts.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "153c5c22-3581-47f6-b3b0-8ffaefe2ef62",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### 3. Selecting best experiment per SKU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "eeecd477-1365-4754-9f4d-d68d4c1a69d7",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Loading experiment results as a Spark DataFrame\n",
    "df_exp = spark.read.format(\"mlflow-experiment\").load(exp_id)\n",
    "df_exp = df_exp.select(\"tags.product\", \"tags.algorithm\", \"metrics.train_wape\", \"metrics.val_wape\", \"metrics.test_wape\").toPandas()\n",
    "\n",
    "# Creating identification key of experiments\n",
    "df_exp[\"exp_key\"] = df_exp[\"product\"] + \"_\" + df_exp[\"algorithm\"]\n",
    "\n",
    "# Obtaining best result per SKU\n",
    "best_exps = df_exp.groupby(by=[\"product\"]).agg({\"val_wape\": \"min\"}).reset_index()\n",
    "best_exps[\"best\"] = 1\n",
    "\n",
    "# Filtering the best result\n",
    "df_exp = pd.merge(df_exp, best_exps, on=[\"product\", \"val_wape\"], how=\"left\")\n",
    "df_exp = df_exp[df_exp[\"best\"] == 1]\n",
    "list_best = list(df_exp[\"exp_key\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "7ac3ebcb-1e8e-471e-a796-c8ff6e5bc269",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "###### 4. Filtering forecasts of the best experiments and saving on Delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "8531749f-f664-474f-8611-5944ac2dc7b0",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p1\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "INFO:py4j.java_gateway:Received command c on object id p0\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">INFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p1\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\nINFO:py4j.java_gateway:Received command c on object id p0\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Filtering forecasts of the best experiments\n",
    "df_fcsts = df_fcsts.filter(df_fcsts[\"exp_key\"].isin(list_best))\n",
    "\n",
    "# Writing results to Delta\n",
    "df_fcsts.write.mode(\"overwrite\")\\\n",
    "    .format(\"delta\") \\\n",
    "    .option(\"overwriteSchema\", \"true\") \\\n",
    "    .save(\"dbfs:/FileStore/results/demand_forecasts\")"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "UDP_Consumption_Forecasting",
   "notebookOrigID": 154105517223501,
   "widgets": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}