main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = pandas.read_csv('data.csv', encoding='ISO-8859-1')\n",
    "\n",
    "dataframe_counts = np.copy(dataframe.values[:,1:])\n",
    "dataframe_counts[dataframe_counts == '..'] = 0 # Use zeros to indicate missing data\n",
    "dataframe_counts = dataframe_counts.astype(np.int)\n",
    "\n",
    "dataframe.values[:,1:] = np.copy(dataframe_counts) # Replace pesky string integers\n",
    "\n",
    "data_point_count = dataframe_counts.shape[0]\n",
    "median_count = np.median(dataframe_counts[dataframe_counts > 0])\n",
    "\n",
    "years = dataframe.columns[1:]\n",
    "years = [int(s) for s in years]\n",
    "\n",
    "names = dataframe.values[:,0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_counts_pp = dataframe_counts / median_count # Scaled version of the data\n",
    "\n",
    "train_data_point_count = int(0.8 * data_point_count)\n",
    "val_data_point_count = data_point_count - train_data_point_count\n",
    "\n",
    "train_dataframe = dataframe_counts_pp[:train_data_point_count,:]\n",
    "val_dataframe = dataframe_counts_pp[train_data_point_count:,:]\n",
    "\n",
    "train_names = names[:train_data_point_count]\n",
    "val_names = names[train_data_point_count:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, LSTM, Conv1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session() # In case you want to get rid of models in RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "DATA_DIMENSIONALITY = 1 # Per (lastname, year) there is one piece of data: the count\n",
    "input_shape = (None, DATA_DIMENSIONALITY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mean Absolute Error (MAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = Sequential([LSTM(64, return_sequences=True, input_shape=input_shape),\n",
    "                    LSTM(64, return_sequences=True),\n",
    "                    Dense(1)])\n",
    "\n",
    "model.compile(optimizer=keras.optimizers.Adam(),\n",
    "              loss=keras.losses.MeanAbsoluteError(),\n",
    "              metrics=['mae'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a custom data generator\n",
    "Long story short, we want to be able to split our data up into random windows, but with respect to the window's size and locations. This is not feasible to create as a dataset (keeping all possible subsets in RAM is not a good idea). We therefore need a custom data generator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference:\n",
    "# https://towardsdatascience.com/implementing-custom-data-generators-in-keras-de56f013581c\n",
    "\n",
    "class DataGenerator(tf.keras.utils.Sequence):\n",
    "    def __init__(self, data, min_seq_length=3, max_seq_length = None):\n",
    "        self.data = data\n",
    "        self.indices = list(range(data.shape[0]))\n",
    "        self.random_indices = None\n",
    "        \n",
    "        self.seq_lengths = data.shape[1]\n",
    "        self.min_seq_length = min_seq_length\n",
    "        self.max_seq_length = max_seq_length\n",
    "        \n",
    "        self.on_epoch_end()\n",
    "\n",
    "        if max_seq_length is None:\n",
    "            self.max_seq_length = self.seq_lengths - 1\n",
    "\n",
    "    def __len__(self):\n",
    "        res = len(self.indices) // BATCH_SIZE\n",
    "        if res == 0 and len(self.indices) > 0:\n",
    "            res = 1\n",
    "        return res\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        random_indices_batch = self.random_indices[index * BATCH_SIZE : (index + 1) * BATCH_SIZE]\n",
    "        window_size_batch = np.random.randint(self.min_seq_length, high=self.max_seq_length)\n",
    "        window_offsets_batch = np.random.randint(0, high=(self.seq_lengths - window_size_batch), size=BATCH_SIZE)\n",
    "        \n",
    "        X = np.zeros((BATCH_SIZE, window_size_batch, DATA_DIMENSIONALITY))\n",
    "        Y = np.zeros((BATCH_SIZE, window_size_batch, DATA_DIMENSIONALITY))\n",
    "\n",
    "        for i in range(BATCH_SIZE):\n",
    "            i_r = random_indices_batch[i]\n",
    "            w_o = window_offsets_batch[i]\n",
    "            \n",
    "            start_index = w_o\n",
    "            stop_index = w_o + window_size_batch\n",
    "            \n",
    "            X[i, :, :] = np.expand_dims(self.data[i_r, start_index:stop_index], axis=-1)\n",
    "            Y[i, :, :] = np.expand_dims(self.data[i_r, start_index+1:stop_index+1], axis=-1)\n",
    "        \n",
    "        return X, Y\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        random_indices_length = np.maximum(BATCH_SIZE, len(self.indices))\n",
    "        self.random_indices = np.random.randint(0, high=len(self.indices), size=random_indices_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fitting the model\n",
    "\n",
    "Possible improvements:\n",
    "* Implement checkpoints\n",
    "* Implement early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dg_train = DataGenerator(train_dataframe, min_seq_length=20)\n",
    "dg_val = DataGenerator(val_dataframe, min_seq_length=20)\n",
    "\n",
    "history = model.fit(dg_train, validation_data=dg_val, epochs=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.title(\"Mean Absolute Error (MAE)\")\n",
    "plt.plot(history.history['mae'], label='MAE')\n",
    "plt.plot(history.history['val_mae'], label='Validation MAE')\n",
    "plt.legend(loc='upper right')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction vs reality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Available names:\")\n",
    "print(\", \".join(list(val_names)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Note\n",
    "The model returns a sequence of values. Sometimes the magnitude of this sequence does not match the magnitude of the actual data. This can cause it to spuriously seem like the model is predicting an increase/decrease. We remedy this by looking at the difference between the last and the second last value in the output of the data, rather than just the last value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_n_steps(input_data, n):\n",
    "    predictions = np.zeros(n)\n",
    "    current_window = input_data\n",
    "\n",
    "    # NOTE: This portion predicts based on the last value of the network.\n",
    "    # for i in range(prediction_count):\n",
    "    #     current_window = model.predict(current_window)\n",
    "    #     predictions[i] = current_window[0, -1, 0]\n",
    "\n",
    "    # NOTE: This portion predicts based on the difference between\n",
    "    # the last and second last values of the network.\n",
    "    last_value = input_data[0, -1, 0]\n",
    "    for i in range(n):\n",
    "        current_window = model.predict(current_window)\n",
    "        predicted_diff = current_window[0, -1, 0] - current_window[0, -2, 0]\n",
    "        predictions[i] = last_value + predicted_diff\n",
    "        last_value = predictions[i]\n",
    "        \n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"Sjöberg\"\n",
    "svensson_index = np.argwhere(val_names == name)[0][0]\n",
    "\n",
    "svensson_data = np.copy(val_dataframe[svensson_index,0:])\n",
    "svensson_data = np.expand_dims(svensson_data, axis=0)\n",
    "svensson_data = np.expand_dims(svensson_data, axis=-1)\n",
    "\n",
    "prediction_count = 11\n",
    "\n",
    "predictions = predict_n_steps(svensson_data[:,:-prediction_count,:], prediction_count)\n",
    "\n",
    "svensson_data *= median_count\n",
    "predictions *= median_count\n",
    "\n",
    "svensson_data = np.squeeze(svensson_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_years = list(range(years[-1] + 1 - prediction_count, years[-1] + 1))\n",
    "\n",
    "plt.title(\"Number of people with the last name {}\".format(name))\n",
    "plt.plot(years, svensson_data, label='Actual')\n",
    "plt.plot(prediction_years, predictions, label='Predicted')\n",
    "plt.legend(loc='lower left')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make predictions for the future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name = \"Sjöberg\"\n",
    "svensson_index = np.argwhere(val_names == name)[0][0]\n",
    "\n",
    "svensson_data = np.copy(val_dataframe[svensson_index,:])\n",
    "svensson_data = np.expand_dims(svensson_data, axis=0)\n",
    "svensson_data = np.expand_dims(svensson_data, axis=-1)\n",
    "\n",
    "prediction_count = 11\n",
    "\n",
    "predictions = predict_n_steps(svensson_data, prediction_count)\n",
    "\n",
    "svensson_data *= median_count\n",
    "predictions *= median_count\n",
    "\n",
    "svensson_data = np.squeeze(svensson_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot predictions for the future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "prediction_years = list(range(years[-1] + 1, years[-1] + 1 + prediction_count))\n",
    "\n",
    "plt.title(\"Number of people with the last name {}\".format(name))\n",
    "plt.plot(years, svensson_data, label='Actual')\n",
    "plt.plot(prediction_years, predictions, label='Predicted')\n",
    "plt.legend(loc='lower left')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sidenote: finding a nice learning rate\n",
    "In short: results do not seem to be very dependent on learning rate in this case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential([LSTM(64, return_sequences=True, input_shape=input_shape),\n",
    "                    LSTM(64, return_sequences=True),\n",
    "                    Dense(1)])\n",
    "\n",
    "model.compile(optimizer=keras.optimizers.Adam(),\n",
    "              loss=keras.losses.MeanAbsoluteError(),\n",
    "              metrics=['mae'])\n",
    "\n",
    "lr_schedule = keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10**(epoch / 20))\n",
    "\n",
    "dg_train = DataGenerator(train_dataframe, min_seq_length=20)\n",
    "\n",
    "history = model.fit(dg_train, epochs=100, callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
    "plt.axis([1e-8, 1e-4, 0, 30])\n",
    "plt.xlabel(\"lr\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}