code/AutomatedTransformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BYQT5T1tcb02"
      },
      "source": [
        "# Pip Installs\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jtFg8BL7ckHE"
      },
      "outputs": [],
      "source": [
        "!pip install ipython-autotime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DjuWSEXHcn3Q"
      },
      "outputs": [],
      "source": [
        "!pip install pyyaml h5py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ULiWo8TpldAb"
      },
      "outputs": [],
      "source": [
        "!pip install tensorflow-privacy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install ann_visualizer"
      ],
      "metadata": {
        "id": "DaeQr4r2dE3E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install graphviz"
      ],
      "metadata": {
        "id": "PTb-tWZudHx1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqScMeNYLoBo"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KxnCDJgYrK9j"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pickle\n",
        "import scipy.signal\n",
        "from scipy import fft\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "\n",
        "import tensorflow as tf\n",
        "tf.__version__\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import LeaveOneGroupOut\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "# Neu\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow_privacy.privacy.optimizers import dp_optimizer_keras\n",
        "\n",
        "import tensorflow_privacy\n",
        "\n",
        "from ann_visualizer.visualize import ann_viz;\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XW3qCiFK1O8n"
      },
      "outputs": [],
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "\n",
        "tf.config.list_physical_devices('GPU')\n",
        "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1r0naEHz0wt6"
      },
      "outputs": [],
      "source": [
        "%load_ext autotime"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RiBUcvWLrK9k"
      },
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  import google.colab\n",
        "  IN_COLAB = True\n",
        "except:\n",
        "  IN_COLAB = False"
      ],
      "metadata": {
        "id": "wA5e2KWULS59"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fmfNdXnxrK9m"
      },
      "outputs": [],
      "source": [
        "if not IN_COLAB:\n",
        "  DATA_PATH = os.path.realpath(\"../data/WESAD\")\n",
        "\n",
        "  checkpoint_prepath = os.path.realpath(\"../models/\")\n",
        "else:\n",
        "  from google.colab import drive\n",
        "  drive.mount(\"/content/drive\")\n",
        "  !ls \"/content/drive/MyDrive/Masterarbeit/Transformer/data/WESAD\"\n",
        "  DATA_PATH = os.path.realpath(\"/content/drive/MyDrive/Masterarbeit/Transformer/data/WESAD\")\n",
        "  checkpoint_prepath = os.path.realpath(\"/content/drive/MyDrive/Masterarbeit/Transformer/code/models/\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oy-5GraPrK9n"
      },
      "outputs": [],
      "source": [
        "class Subject:\n",
        "    \"\"\"Subject of the WESAD dataset.\n",
        "    Subject Class inspired by: https://github.com/WJMatthew/WESAD\"\"\"\n",
        "\n",
        "    def __init__(self, main_path, subject_number):\n",
        "        self.name = f'S{subject_number}'\n",
        "        self.subject_keys = ['signal', 'label', 'subject']\n",
        "        self.signal_keys = ['chest', 'wrist']\n",
        "        self.chest_keys = ['ACC', 'ECG', 'EMG', 'EDA', 'Temp', 'Resp']\n",
        "        self.wrist_keys = ['ACC', 'BVP', 'EDA', 'TEMP']\n",
        "        with open(os.path.join(main_path, self.name) + '/' + self.name + '.pkl', 'rb') as file:\n",
        "            self.data = pickle.load(file, encoding='latin1')\n",
        "        self.labels = self.data['label']\n",
        "\n",
        "    def get_wrist_data(self):\n",
        "        \"\"\"Returns data measured by the E4 Empatica\"\"\"\n",
        "\n",
        "        data = self.data['signal']['wrist']\n",
        "        return data\n",
        "    \n",
        "    def get_subject_dataframe(self):\n",
        "        \"\"\"Returns a dataframe with the preprocessed data of the subject\"\"\"\n",
        "        wrist_data = self.get_wrist_data()\n",
        "        bvp_signal = wrist_data['BVP'][:,0]\n",
        "        eda_signal = wrist_data['EDA'][:,0]\n",
        "        acc_x_signal = wrist_data['ACC'][:,0]\n",
        "        acc_y_signal = wrist_data['ACC'][:,1]\n",
        "        acc_z_signal = wrist_data['ACC'][:,2]\n",
        "        temp_signal = wrist_data['TEMP'][:,0]\n",
        "        # Upsampling data to match BVP data sampling rate using fourier method as described in Paper/dataset\n",
        "        eda_upsampled = scipy.signal.resample(eda_signal, len(bvp_signal))\n",
        "        temp_upsampled = scipy.signal.resample(temp_signal, len(bvp_signal))\n",
        "        acc_x_upsampled = scipy.signal.resample(acc_x_signal, len(bvp_signal))\n",
        "        acc_y_upsampled = scipy.signal.resample(acc_y_signal, len(bvp_signal))\n",
        "        acc_z_upsampled = scipy.signal.resample(acc_z_signal, len(bvp_signal))\n",
        "        label_df = pd.DataFrame(self.labels, columns=['label'])\n",
        "        label_df.index = [(1 / 700) * i for i in range(len(label_df))] # 700 is the sampling rate of the label\n",
        "        label_df.index = pd.to_datetime(label_df.index, unit='s')\n",
        "        data_arrays = zip(bvp_signal, eda_upsampled, acc_x_upsampled, acc_y_upsampled, acc_z_upsampled, temp_upsampled)\n",
        "        df = pd.DataFrame(data=data_arrays, columns=['BVP', 'EDA', 'ACC_x', 'ACC_y', 'ACC_z', 'TEMP'])\n",
        "        df.index = [(1 / 64) * i for i in range(len(df))] # 64 = sampling rate of BVP\n",
        "        df.index = pd.to_datetime(df.index, unit='s')\n",
        "        df = df.join(label_df)\n",
        "        df['label'] = df['label'].fillna(method='ffill')\n",
        "        df.reset_index(drop=True, inplace=True)\n",
        "        df.drop(df[df['label'].isin([0.0, 4.0, 5.0, 6.0, 7.0])].index, inplace=True)\n",
        "        df['label'] = df['label'].replace([1.0, 2.0, 3.0], [0, 1, 0])\n",
        "        df.reset_index(drop=True, inplace=True)\n",
        "        df = (df-df.min())/(df.max()-df.min()) # Normalize data (no train test leakage since data frame per subject) \n",
        "        return df\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PKCCTAAurK9o"
      },
      "outputs": [],
      "source": [
        "s2 = Subject(DATA_PATH, 2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NnryI-rrrK9p"
      },
      "outputs": [],
      "source": [
        "s2.get_subject_dataframe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cwBcLAwZrK9q"
      },
      "outputs": [],
      "source": [
        "def create_subjects_data() -> dict:\n",
        "    # Create a dictionary with all the subjects and belonging dataframes\n",
        "    subjects = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17]\n",
        "    subjects_data = {}\n",
        "    for subject_num in subjects:\n",
        "        subject = Subject(DATA_PATH, subject_num)\n",
        "        subjects_data[subject.name] = subject.get_subject_dataframe()\n",
        "\n",
        "    return subjects_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V_vz0sK1rK9q"
      },
      "source": [
        "### Window"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F_8TFRBGrK9r"
      },
      "source": [
        "1. Creating the windows\n",
        "2. Create subwindows from the windows\n",
        "3. Calculate the fft of the subwindows\n",
        "4. Average the subwindows\n",
        "\n",
        "![fft](../images/fft.png)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9kdu8rgTrK9s"
      },
      "outputs": [],
      "source": [
        "# Subwindow length of the biosignals\n",
        "signal_subwindow_dict = {\n",
        "    'ACC_x': 7,\n",
        "    'ACC_y': 7,\n",
        "    'ACC_z': 7,\n",
        "    'BVP': 30,\n",
        "    'EDA': 30,\n",
        "    'TEMP': 35\n",
        "    }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4fFchRHzrK9t"
      },
      "outputs": [],
      "source": [
        "# most frequent element in list\n",
        "def most_common(lst):\n",
        "    return max(set(lst), key=lst.count)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4zWYpvW8rK9u"
      },
      "outputs": [],
      "source": [
        "def create_windows(df: pd.DataFrame) -> tuple([pd.DataFrame,list]):\n",
        "    \"\"\"Creates windows from the dataframe and returns the windows and the labels.\n",
        "    If the window is assigned to multiple labels, the most common label is chosen for that period.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): Subject DataFrame\n",
        "\n",
        "    Returns:\n",
        "        tuple[pd.DataFrame,list]: Windows representing the activity of the subject in one minute and the corresponding labels.\n",
        "    \"\"\"\n",
        "\n",
        "    window_len = 64 * 60 # fs = 64 and window length in seconds = 60\n",
        "    windows, labels = zip(*[(df[i:i+window_len], int(most_common(df['label'][i:i+window_len].to_list()))) for i in range(0,df.shape[0],window_len)])\n",
        "    return windows, labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FQ-6UGY7rK9u"
      },
      "outputs": [],
      "source": [
        "def create_subwindows(df: pd.DataFrame, signal_subwindow_len: int, signal_name: str) -> list:\n",
        "    \"\"\"The function creates subwindows from the windows.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): Windows representing the activity of the subject in one minute.\n",
        "        signal_subwindow_len (int): Length of the subwindows.\n",
        "        signal_name (str): Name of the signal.\n",
        "\n",
        "    Returns:\n",
        "        list: Subwindows of the signal in the window.\n",
        "    \"\"\"\n",
        "    subwindow_len = 64 * signal_subwindow_len # fs = 64 and sub-window length in seconds = 30\n",
        "    window_len = 64 * 60 # fs = 64 and window length in seconds = 60\n",
        "    window_shift = int(64 * 0.25) # fs = 64 and window shift in seconds = 0.25\n",
        "    subwindows = []\n",
        "\n",
        "    for i in range(0, window_len, window_shift):\n",
        "        if i + subwindow_len <= window_len:\n",
        "            subwindow = df[signal_name][i:i+subwindow_len]\n",
        "            subwindows.append(subwindow)\n",
        "    return subwindows\n",
        "            "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bXcKXPgQrK9v"
      },
      "outputs": [],
      "source": [
        "def fft_subwindows(subwindows: list, duration: int, f_s: int) -> list:\n",
        "    \"\"\"Calculates the fft of the subwindows.\n",
        "\n",
        "    Args:\n",
        "        subwindows (list): C\n",
        "        duration (int): _description_\n",
        "        f_s (int): _description_\n",
        "\n",
        "    Returns:\n",
        "        list: Fft coefficients of the subwindows.\n",
        "    \"\"\"\n",
        "    freqs= []\n",
        "    yfs = []\n",
        "    for subwindow in subwindows:\n",
        "        y = np.array(subwindow)\n",
        "        yf = scipy.fft.fft(y)\n",
        "        l = len(yf)\n",
        "        N = f_s * duration\n",
        "        freq = scipy.fft.fftfreq(N, 1/f_s)\n",
        "\n",
        "        l //= 2\n",
        "        amps = np.abs(yf[0:l])\n",
        "        freq = np.abs(freq[0:l])\n",
        "\n",
        "        # Sort descending amp   \n",
        "        p = amps.argsort()[::-1]\n",
        "        freq = freq[p]\n",
        "        amps = amps[p]\n",
        "\n",
        "        freqs.append(freq)\n",
        "        yfs.append(amps)\n",
        "    return np.asarray(freqs), np.asarray(yfs)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pXgM5w6GrK9v"
      },
      "outputs": [],
      "source": [
        "def average_window(subwindows_fft: list) -> list:\n",
        "    \"\"\"Calculates the average of the fft coefficients of the subwindows.\n",
        "\n",
        "    Args:\n",
        "        subwindows_fft (list): List of fft coefficients of the subwindows.\n",
        "\n",
        "    Returns:\n",
        "        list: Average of the fft coefficients of the subwindow for signals.\n",
        "    \"\"\"\n",
        "    len_yfs = len(subwindows_fft[0])\n",
        "    avg_yfs = []\n",
        "    for i in range(len_yfs):\n",
        "        i_yfs = []\n",
        "        for yf in subwindows_fft:\n",
        "            try:\n",
        "                i_yfs.append(yf[i])\n",
        "            except IndexError:\n",
        "                pass\n",
        "        avg_yfs.append(sum(i_yfs)/len(i_yfs))\n",
        "    return avg_yfs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eYdBIjAYrK9v"
      },
      "outputs": [],
      "source": [
        "def create_preprocessed_subjects_data(subjects_data: dict) -> dict:\n",
        "# Creates averaged windows for all subjects from dataframes\n",
        "\n",
        "    subjects_preprosessed_data = {}\n",
        "    for subject_name, subject_df in subjects_data.items():\n",
        "        subjects_preprosessed_data[subject_name] = {}\n",
        "        windows, labels = create_windows(subject_df)\n",
        "        yfs_per_min_for_signal = {}\n",
        "        X = []\n",
        "        for i in range(0,len(windows) - 1):\n",
        "            for signal in signal_subwindow_dict.keys():\n",
        "\n",
        "                duration_in_sec = signal_subwindow_dict[signal]\n",
        "\n",
        "                subwindows = create_subwindows(windows[i], signal_subwindow_len=duration_in_sec, signal_name=signal)\n",
        "                freqs, yfs = fft_subwindows(subwindows, duration_in_sec, 64)\n",
        "                yfs_average = average_window(yfs)[:210]\n",
        "                yfs_per_min_for_signal[signal] = yfs_average\n",
        "                \n",
        "            X.append(pd.DataFrame(yfs_per_min_for_signal).T)\n",
        "        y = list(labels[:len(windows)-1])\n",
        "        subjects_preprosessed_data[subject_name]['X'] = X\n",
        "        subjects_preprosessed_data[subject_name]['y'] = y\n",
        "    \n",
        "    return subjects_preprosessed_data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kpyf8pRLrK9w"
      },
      "outputs": [],
      "source": [
        "subjects_data = create_subjects_data()\n",
        "subjects_preprocessed_data = create_preprocessed_subjects_data(subjects_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C20NZIK8rK9w"
      },
      "outputs": [],
      "source": [
        "subjects_preprocessed_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aYWHD0VjrK9x"
      },
      "outputs": [],
      "source": [
        "def get_subject_window_data(subjects_preprosessed_data: dict) -> tuple([list, list]):\n",
        "    # Created train and test data for leave one out cross validation\n",
        "    all_subjects_X = []\n",
        "    all_subjects_y = []\n",
        "    for subject_name, subject_data in subjects_preprosessed_data.items():\n",
        "        all_subjects_X.append(subject_data['X'])\n",
        "        all_subjects_y.append(subject_data['y'])\n",
        "    \n",
        "    return (all_subjects_X, all_subjects_y)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YpvXypU-rK9x"
      },
      "outputs": [],
      "source": [
        "all_subjects_X, all_subjects_y = get_subject_window_data(subjects_preprocessed_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KWx9LwKIrK9x"
      },
      "source": [
        "![OS_Sensors](../images/os_sensors.png)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r3Nm9bNUrK9x"
      },
      "outputs": [],
      "source": [
        "SMARTWATCH_OS = {\n",
        "    'E4': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'EDA', 'BVP'],\n",
        "    #'Tizen': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'BVP'],\n",
        "    #'WearOS_watchOS': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP'],\n",
        "    #'Fitbit': ['ACC_x', 'ACC_y', 'ACC_z', 'TEMP', 'EDA'],\n",
        "    #'PiaOS': ['TEMP', 'EDA', 'BVP']\n",
        "    }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TrgOg67ErK9x"
      },
      "outputs": [],
      "source": [
        "def filter_for_smartwatch_os(smartwatch_os_name: str, all_subjects_X: list) -> list:\n",
        "    # Adjusts the data for the smartwatch os\n",
        "    all_subjects_X_adjusted_for_smartwatch_os = []\n",
        "    for subject_data in all_subjects_X:\n",
        "        subject_adjusted_for_smartwatch_os = []\n",
        "        for window in subject_data:\n",
        "            subject_adjusted_for_smartwatch_os.append(window.loc[SMARTWATCH_OS[smartwatch_os_name]])\n",
        "        all_subjects_X_adjusted_for_smartwatch_os.append(subject_adjusted_for_smartwatch_os)\n",
        "    return all_subjects_X_adjusted_for_smartwatch_os\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVz4lLUDMSmH"
      },
      "source": [
        "# Transformer Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77xcGqoBMcTz"
      },
      "source": [
        "## Build Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Qz6GiLRKDRo"
      },
      "source": [
        "### Transformer Model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rBzmvZAOKL9G"
      },
      "outputs": [],
      "source": [
        "def transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n",
        "    # Normalization and Attention\n",
        "    x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n",
        "    x = layers.MultiHeadAttention(\n",
        "        key_dim=head_size, num_heads=num_heads, dropout=dropout\n",
        "    )(x, x)\n",
        "    x = layers.Dropout(dropout)(x)\n",
        "    res = x + inputs\n",
        "\n",
        "    # Feed Forward Part\n",
        "    x = layers.LayerNormalization(epsilon=1e-6)(res)\n",
        "    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n",
        "    x = layers.Dropout(dropout)(x)\n",
        "    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n",
        "    return x + res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "STOR01UpKT0g"
      },
      "outputs": [],
      "source": [
        "def build_model(\n",
        "   input_shape,\n",
        "    head_size,\n",
        "    num_heads,\n",
        "    ff_dim,\n",
        "    num_transformer_blocks,\n",
        "    mlp_units,\n",
        "    dropout,\n",
        "    mlp_dropout\n",
        "):\n",
        "\n",
        "    inputs = keras.Input(input_shape)\n",
        "    x = inputs\n",
        "    #layers.LSTM(1)\n",
        "    for _ in range(num_transformer_blocks):\n",
        "        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n",
        "\n",
        "    x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n",
        "    for dim in mlp_units:\n",
        "        x = layers.Dense(dim, activation=\"relu\")(x)\n",
        "        x = layers.Dropout(mlp_dropout)(x)\n",
        "    #outputs = layers.Dense(num_output_class, activation=\"sigmoid\")(x)\n",
        "    outputs = layers.Dense(2, activation=\"sigmoid\")(x)\n",
        "\n",
        "    #outputs = layers.Dense(num_output_class, activation=\"softmax\")(x)\n",
        "    \n",
        "    return keras.Model(inputs, outputs)\n",
        "    #model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OYGJ-hXWvheP"
      },
      "outputs": [],
      "source": [
        "from tensorflow_privacy.privacy.analysis.compute_noise_from_budget_lib import compute_noise as tfp_computer_noise\n",
        "\n",
        "\"\"\"\n",
        "Calculate noise for given training hyperparameters\n",
        "\"\"\"\n",
        "def compute_noise(n, batch_size, target_epsilon, epochs, delta, min_noise=1e-5):\n",
        "  return tfp_computer_noise(n, batch_size, target_epsilon, epochs, delta, min_noise)\n",
        "\n",
        "\"\"\"\n",
        "Calculate Delta for given training dataset size n\n",
        "\"\"\"\n",
        "def compute_delta(n):\n",
        "# delta should be one magnitude lower than inverse of training set size: 1/n\n",
        "# e.g. 1e-5 for n=60.000\n",
        "# take 1e-x, were x is the magnitude of training set size\n",
        "  delta = np.power(10, - float(len(str(n)))) # remove all trailing decimals\n",
        "  return delta"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loop"
      ],
      "metadata": {
        "id": "QTgdGm2KAnMr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def setparams(epochs, num_transformer_blocks, mlp_dropout, dropout, num_heads, head_size, dp_selected,l2_norm_clip, noise_multiplier):\n",
        "\n",
        "  params = {'num_transformer_blocks':num_transformer_blocks, \n",
        "          'min_delta_loss':0.001, \n",
        "          'epochs':epochs,\n",
        "          'batch_size':50,\n",
        "          'lr':0.0001,\n",
        "          'mlp_dropout': mlp_dropout,\n",
        "          'dropout': dropout,\n",
        "          'num_heads':num_heads,\n",
        "          'ff_dim':4,\n",
        "          'head_size':head_size,\n",
        "          'dp_selected':dp_selected,\n",
        "          'l2_norm_clip':l2_norm_clip,\n",
        "          'noise_multiplier':noise_multiplier,\n",
        "          'num_microbatches':1\n",
        "          }\n",
        "\n",
        "  \n",
        "  return params"
      ],
      "metadata": {
        "id": "d-SbOQO0Asm0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def trainmodel(params):\n",
        "  # create an empty dictionary to store the number of epochs\n",
        "  epochs_dict = {}\n",
        "  max_epochs = 0;\n",
        "\n",
        "  for os, signals in SMARTWATCH_OS.items():\n",
        "    with tf.device('/device:GPU:0'):\n",
        "      print(f'\\n\\n\\nSmartwatchOS: {os} - Signals: {signals} - Num. Signals: {len(signals)}')\n",
        "      print(f'Number of signals: {len(signals)}')\n",
        "      all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    \n",
        "    groups_set = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\n",
        "    subject_ids = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17] # ids for subjects in WESAD dataset\n",
        "    \n",
        "    num_signals = len(signals) # Number of signals in the WESAD dataset measured by the empatica e4\n",
        "    num_output_class = 2 # Number of output classes (2 - non-stress vs stress)\n",
        "    num_epochs = params[\"epochs\"]\n",
        "\n",
        "    all_acc_histories = []\n",
        "    all_loss_histories = []\n",
        "\n",
        "    \n",
        "\n",
        "    for i in groups_set:\n",
        "        test_index = groups_set[i]\n",
        "        train_index = [x for x in groups_set if x != test_index]\n",
        "        print(train_index, test_index)\n",
        "        print(f'SmartwatchOS: {os} - Person: {test_index}')\n",
        "\n",
        "        X_train = np.concatenate(np.array([all_subjects_X_os[x] for x in train_index], dtype=object))\n",
        "        y_train = np.concatenate(np.array([all_subjects_y[y] for y in train_index], dtype=object))\n",
        "        X_test = all_subjects_X_os[test_index]\n",
        "        y_test = all_subjects_y[test_index]\n",
        "        \n",
        "        weight_balance = y_train.tolist().count(0)/y_train.tolist().count(1)\n",
        "\n",
        "        X_train = np.asarray(X_train)\n",
        "        #print(X_train)\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_train = np.asarray(y_train)\n",
        "        y_test = np.asarray(y_test)\n",
        "\n",
        "        y_train = tf.keras.utils.to_categorical(y_train, num_output_class)\n",
        "        y_test = tf.keras.utils.to_categorical(y_test, num_output_class)\n",
        "\n",
        "        tf.keras.backend.clear_session()\n",
        "\n",
        "        #input_shape = (6,210,)\n",
        "        input_shape = X_train.shape[1:]\n",
        "        #input_shape = [num_signals, 210, 1]\n",
        "        print(input_shape)\n",
        "        model = build_model(\n",
        "                input_shape,\n",
        "                head_size=params[\"head_size\"],\n",
        "                num_heads=params[\"num_heads\"],\n",
        "                ff_dim=params[\"ff_dim\"],\n",
        "                num_transformer_blocks=params[\"num_transformer_blocks\"],\n",
        "                mlp_units=[128],\n",
        "                mlp_dropout=params[\"mlp_dropout\"],\n",
        "                dropout=params[\"dropout\"]\n",
        "                )\n",
        "        \n",
        "        if params[\"dp_selected\"] == 0:\n",
        "          optimizer = keras.optimizers.Adam(learning_rate=params[\"lr\"])  \n",
        "\n",
        "        elif params[\"dp_selected\"] == 1:\n",
        "          # Select your differentially private optimizer\n",
        "          optimizer = tensorflow_privacy.VectorizedDPKerasAdamOptimizer(\n",
        "                        l2_norm_clip= params[\"l2_norm_clip\"], \n",
        "                        noise_multiplier= params[\"noise_multiplier\"], \n",
        "                        num_microbatches= params[\"num_microbatches\"], \n",
        "                        learning_rate=params[\"lr\"])\n",
        "\n",
        "\n",
        "\n",
        "        model.compile(\n",
        "        #loss=\"sparse_categorical_crossentropy\",\n",
        "        #loss=\"categorical_crossentropy\",\n",
        "        loss=\"binary_crossentropy\",\n",
        "        optimizer=optimizer,\n",
        "        #metrics=[\"sparse_categorical_accuracy\"],\n",
        "        #metrics=[\"categorical_accuracy\"],\n",
        "        metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.BinaryAccuracy()]\n",
        "        )\n",
        "        \n",
        "        \n",
        "        #build_model(num_signals, num_output_class)\n",
        "\n",
        "        checkpoint = tf.keras.callbacks.ModelCheckpoint(#filepath=checkpoint_path,  # Path to save the model file\n",
        "            checkpoint_prepath + f\"/{os}/wesad_{os}_binary_s{subject_ids[test_index]}_{num_epochs}.h5\",                                            \n",
        "            monitor=\"loss\", # The metric name to monitor\n",
        "            save_best_only=True # If True, it only saves the \"best\" model according to the quantity monitored \n",
        "        )\n",
        "\n",
        "\n",
        "        \n",
        "\n",
        "        early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "            monitor=\"loss\",     # Quantity to be monitored.\n",
        "            min_delta=params[\"min_delta_loss\"],     # Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.\n",
        "            patience=10,        # Number of epochs with no improvement after which training will be stopped.\n",
        "            restore_best_weights=True\n",
        "        )\n",
        "\n",
        "        \n",
        "        history = model.fit(\n",
        "        X_train,\n",
        "        y_train,\n",
        "        validation_split=0.2,\n",
        "        epochs=num_epochs,\n",
        "        batch_size=params[\"batch_size\"],\n",
        "        class_weight={0: 1, 1: weight_balance},\n",
        "        #callbacks=[neptune_callback]                                  # No Early Stopping, No Checkpoint, Neptune\n",
        "        callbacks=[checkpoint, early_stopping]                       # Checkpoint, Early Stopping, no Neptune\n",
        "        #callbacks=[checkpoint, early_stopping, neptune_callback]     # Checkpoint, Early Stopping, Neptune\n",
        "        #callbacks=[early_stopping, neptune_callback]                 # Early Stopping, Neptune\n",
        "        \n",
        "        )\n",
        "\n",
        "        #Getting actual number of epochs per Subject (due to early stopping it differs)\n",
        "        number_of_epochs_it_ran = len(history.history['loss'])\n",
        "        n_epochs_best = number_of_epochs_it_ran\n",
        "        # n_epochs_best = number_of_epochs_it_ran - 10 #subtracting patience when using early stopping\n",
        "        #Rewrite max_epochs if current Epoch is higher\n",
        "        if n_epochs_best > max_epochs:\n",
        "          max_epochs = n_epochs_best\n",
        "        # Add number of epochs to dictionary for each subject != the current used subject\n",
        "        for other_subject in train_index:\n",
        "          if other_subject not in epochs_dict:\n",
        "            epochs_dict[other_subject] = n_epochs_best\n",
        "          else:\n",
        "            epochs_dict[other_subject] += n_epochs_best\n",
        "        print(epochs_dict)\n",
        "      \n",
        "      \n",
        "\n",
        "\n",
        "    eval_metrics =  model.evaluate(X_test, y_test, verbose=0 )\n",
        "    loss = model.evaluate(X_test, y_test, verbose=0 )[0]\n",
        "    accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "    precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "    recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "    if (precision + recall) != 0:\n",
        "      f1 = 2 * precision * recall / (precision + recall)\n",
        "    else:\n",
        "      f1 = 0\n",
        "\n",
        "    \n",
        "    \n",
        "  return model"
      ],
      "metadata": {
        "id": "fJUSG2sbA_be"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def losoeval(model, params):\n",
        "  # Evaluating every models on the corresponding test dataset not seen during training.\n",
        "  os_scores_acc = {}\n",
        "  os_scores_f1 = {}\n",
        "\n",
        "  for os, signals in SMARTWATCH_OS.items():\n",
        "\n",
        "    all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    all_accuracies = []\n",
        "    all_precisions = []\n",
        "    all_recalls = []\n",
        "    all_f1s = []\n",
        "    subject_ids = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17] # ids for subjects in WESAD dataset\n",
        "    \n",
        "    for i, subject_id in enumerate(subject_ids):\n",
        "        X_test = all_subjects_X_os[i]\n",
        "        y_test = all_subjects_y[i]\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_test = np.asarray(y_test)\n",
        "        \n",
        "        y_test = tf.keras.utils.to_categorical(y_test, 2)\n",
        "        \n",
        "                                            \n",
        "          \n",
        "        accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "        precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "        recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "        if (precision + recall) != 0:\n",
        "          f1 = 2 * precision * recall / (precision + recall)\n",
        "        else:\n",
        "          f1 = 0\n",
        "        all_accuracies.append(accuracy)\n",
        "        all_precisions.append(precision)\n",
        "        all_recalls.append(recall)\n",
        "        all_f1s.append(f1)\n",
        "\n",
        "    print(f'Smartwatch OS: {os}')\n",
        "    print(f'Evaluation of Transformer model trained on {params[\"epochs\"]} epochs\\n')\n",
        "    print(f'Subject\\t\\t Accuracy\\tPrecision\\tRecall\\t\\tF1-Score')\n",
        "    print(\"************************************************************************\")\n",
        "    for i in range(len(all_accuracies)):\n",
        "        print(f'S{subject_ids[i]}\\t\\t {round(all_accuracies[i], 5):.5f}\\t{round(all_precisions[i], 5):.5f}\\t\\t{round(all_recalls[i], 5):.5f}\\t\\t{round(all_f1s[i], 5):.5f}')\n",
        "\n",
        "    print(\"************************************************************************\")\n",
        "    print(f'Average\\t\\t {round(np.mean(all_accuracies), 5):.5f}\\t{round(np.mean(all_precisions), 5):.5f}\\t\\t{round(np.mean(all_recalls), 5):.5f}\\t\\t{round(np.mean(all_f1s), 5):.5f}\\n\\n\\n')\n",
        "\n",
        "    os_scores_acc[os] = all_accuracies\n",
        "    os_scores_f1[os] = all_f1s\n",
        "  \n",
        "  \n",
        "\n",
        " \n"
      ],
      "metadata": {
        "id": "Xf1gQ1bYBNWL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loopstart"
      ],
      "metadata": {
        "id": "wI15JOHSB2xN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dic_numencblocks= [1,2,3,4,5,6,7]\n",
        "dic_mlpdropout= [0.6,0.6,0.6,0.6,0.6]\n",
        "dic_dropout= [0.4,0.4,0.4,0.4,0.4,0.5,0.5,0.5,0.5,0.5,0.6,0.6,0.6,0.6,0.6,0.7,0.7,0.7,0.7,0.7]\n",
        "dic_heads = [5,5,5]\n",
        "dic_headsize = [512]\n",
        "dic_epochs = [110]\n",
        "dic_noise = [68.61306896515201,68.61306896515201,68.61306896515201,68.61306896515201,68.61306896515201]\n",
        "\n",
        "index = 0;\n",
        "\n",
        "\n",
        "for noise in dic_noise:\n",
        "\n",
        "  params = setparams(110, 8, 0.25, 0.25, 4, 256, 1, 2.0, noise)\n",
        "  model = trainmodel(params)\n",
        "  losoeval(model, params)\n",
        "  \n"
      ],
      "metadata": {
        "id": "M0tTJoKGB5_z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Neptune AI"
      ],
      "metadata": {
        "id": "wfe-XokHPpJG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install neptune"
      ],
      "metadata": {
        "id": "4ygyC4yGXnDy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T5Eox1iqcmSG"
      },
      "outputs": [],
      "source": [
        "!pip install -U neptune-tensorflow-keras"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import neptune\n",
        "from neptune.integrations.tensorflow_keras import NeptuneCallback"
      ],
      "metadata": {
        "id": "bkOVrXG_P3A0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Loop with Neptune AI\n"
      ],
      "metadata": {
        "id": "632HUHIXIBJn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def setparams(epochs, num_transformer_blocks, mlp_dropout, dropout, num_heads, head_size, dp_selected,l2_norm_clip, noise_multiplier):\n",
        "\n",
        "  params = {'num_transformer_blocks':num_transformer_blocks, \n",
        "          'min_delta_loss':0.001, \n",
        "          'epochs':epochs,\n",
        "          'batch_size':50,\n",
        "          'lr':0.0001,\n",
        "          'mlp_dropout': mlp_dropout,\n",
        "          'dropout': dropout,\n",
        "          'num_heads':num_heads,\n",
        "          'ff_dim':4,\n",
        "          'head_size':head_size,\n",
        "          'dp_selected':dp_selected,\n",
        "          'l2_norm_clip':l2_norm_clip,\n",
        "          'noise_multiplier':noise_multiplier,\n",
        "          'num_microbatches':1\n",
        "          }\n",
        "\n",
        "  \n",
        "  return params\n",
        "  \n"
      ],
      "metadata": {
        "id": "0UROVAVT-4Mc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def trainmodel(run, params):\n",
        "  # create an empty dictionary to store the number of epochs\n",
        "  epochs_dict = {}\n",
        "  max_epochs = 0;\n",
        "  neptune_callback = NeptuneCallback(run=run, log_model_diagram=True)    #set if neptune ai is installed\n",
        "\n",
        "  for os, signals in SMARTWATCH_OS.items():\n",
        "    with tf.device('/device:GPU:0'):\n",
        "      print(f'\\n\\n\\nSmartwatchOS: {os} - Signals: {signals} - Num. Signals: {len(signals)}')\n",
        "      print(f'Number of signals: {len(signals)}')\n",
        "      all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    \n",
        "    groups_set = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\n",
        "    subject_ids = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17] # ids for subjects in WESAD dataset\n",
        "    \n",
        "    num_signals = len(signals) # Number of signals in the WESAD dataset measured by the empatica e4\n",
        "    num_output_class = 2 # Number of output classes (2 - non-stress vs stress)\n",
        "    num_epochs = params[\"epochs\"]\n",
        "\n",
        "    all_acc_histories = []\n",
        "    all_loss_histories = []\n",
        "\n",
        "    \n",
        "\n",
        "    for i in groups_set:\n",
        "        test_index = groups_set[i]\n",
        "        train_index = [x for x in groups_set if x != test_index]\n",
        "        print(train_index, test_index)\n",
        "        print(f'SmartwatchOS: {os} - Person: {test_index}')\n",
        "\n",
        "        X_train = np.concatenate(np.array([all_subjects_X_os[x] for x in train_index], dtype=object))\n",
        "        y_train = np.concatenate(np.array([all_subjects_y[y] for y in train_index], dtype=object))\n",
        "        X_test = all_subjects_X_os[test_index]\n",
        "        y_test = all_subjects_y[test_index]\n",
        "        \n",
        "        weight_balance = y_train.tolist().count(0)/y_train.tolist().count(1)\n",
        "\n",
        "        X_train = np.asarray(X_train)\n",
        "        #print(X_train)\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_train = np.asarray(y_train)\n",
        "        y_test = np.asarray(y_test)\n",
        "\n",
        "        y_train = tf.keras.utils.to_categorical(y_train, num_output_class)\n",
        "        y_test = tf.keras.utils.to_categorical(y_test, num_output_class)\n",
        "\n",
        "        tf.keras.backend.clear_session()\n",
        "\n",
        "        #input_shape = (6,210,)\n",
        "        input_shape = X_train.shape[1:]\n",
        "        #input_shape = [num_signals, 210, 1]\n",
        "        print(input_shape)\n",
        "        model = build_model(\n",
        "                input_shape,\n",
        "                head_size=params[\"head_size\"],\n",
        "                num_heads=params[\"num_heads\"],\n",
        "                ff_dim=params[\"ff_dim\"],\n",
        "                num_transformer_blocks=params[\"num_transformer_blocks\"],\n",
        "                mlp_units=[128],\n",
        "                mlp_dropout=params[\"mlp_dropout\"],\n",
        "                dropout=params[\"dropout\"]\n",
        "                )\n",
        "        \n",
        "        if params[\"dp_selected\"] == 0:\n",
        "          optimizer = keras.optimizers.Adam(learning_rate=params[\"lr\"])  \n",
        "\n",
        "        elif params[\"dp_selected\"] == 1:\n",
        "          # Select your differentially private optimizer\n",
        "          optimizer = tensorflow_privacy.VectorizedDPKerasAdamOptimizer(\n",
        "                        l2_norm_clip= params[\"l2_norm_clip\"], \n",
        "                        noise_multiplier= params[\"noise_multiplier\"], \n",
        "                        num_microbatches= params[\"num_microbatches\"], \n",
        "                        learning_rate=params[\"lr\"])\n",
        "\n",
        "\n",
        "\n",
        "        model.compile(\n",
        "        #loss=\"sparse_categorical_crossentropy\",\n",
        "        #loss=\"categorical_crossentropy\",\n",
        "        loss=\"binary_crossentropy\",\n",
        "        optimizer=optimizer,\n",
        "        #metrics=[\"sparse_categorical_accuracy\"],\n",
        "        #metrics=[\"categorical_accuracy\"],\n",
        "        metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.BinaryAccuracy()]\n",
        "        )\n",
        "        \n",
        "        \n",
        "        #build_model(num_signals, num_output_class)\n",
        "\n",
        "        checkpoint = tf.keras.callbacks.ModelCheckpoint(#filepath=checkpoint_path,  # Path to save the model file\n",
        "            checkpoint_prepath + f\"/{os}/wesad_{os}_binary_s{subject_ids[test_index]}_{num_epochs}.h5\",                                            \n",
        "            monitor=\"loss\", # The metric name to monitor\n",
        "            save_best_only=True # If True, it only saves the \"best\" model according to the quantity monitored \n",
        "        )\n",
        "\n",
        "\n",
        "        \n",
        "\n",
        "        early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "            monitor=\"loss\",     # Quantity to be monitored.\n",
        "            min_delta=params[\"min_delta_loss\"],     # Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.\n",
        "            patience=10,        # Number of epochs with no improvement after which training will be stopped.\n",
        "            restore_best_weights=True\n",
        "        )\n",
        "\n",
        "        \n",
        "        history = model.fit(\n",
        "        X_train,\n",
        "        y_train,\n",
        "        validation_split=0.2,\n",
        "        epochs=num_epochs,\n",
        "        batch_size=params[\"batch_size\"],\n",
        "        class_weight={0: 1, 1: weight_balance},\n",
        "        callbacks=[neptune_callback]                                  # No Early Stopping, No Checkpoint, Neptune\n",
        "        #callbacks=[checkpoint, early_stopping]                       # Checkpoint, Early Stopping, no Neptune\n",
        "        #callbacks=[checkpoint, early_stopping, neptune_callback]     # Checkpoint, Early Stopping, Neptune\n",
        "        #callbacks=[early_stopping, neptune_callback]                 # Early Stopping, Neptune\n",
        "        \n",
        "        )\n",
        "\n",
        "        #Getting actual number of epochs per Subject (due to early stopping it differs)\n",
        "        number_of_epochs_it_ran = len(history.history['loss'])\n",
        "        n_epochs_best = number_of_epochs_it_ran\n",
        "        # n_epochs_best = number_of_epochs_it_ran - 10 #subtracting patience when using early stopping\n",
        "        #Rewrite max_epochs if current Epoch is higher\n",
        "        if n_epochs_best > max_epochs:\n",
        "          max_epochs = n_epochs_best\n",
        "        # Add number of epochs to dictionary for each subject != the current used subject\n",
        "        for other_subject in train_index:\n",
        "          if other_subject not in epochs_dict:\n",
        "            epochs_dict[other_subject] = n_epochs_best\n",
        "          else:\n",
        "            epochs_dict[other_subject] += n_epochs_best\n",
        "        print(epochs_dict)\n",
        "      \n",
        "      \n",
        "\n",
        "\n",
        "    eval_metrics =  model.evaluate(X_test, y_test, verbose=0 )\n",
        "    loss = model.evaluate(X_test, y_test, verbose=0 )[0]\n",
        "    accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "    precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "    recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "    if (precision + recall) != 0:\n",
        "      f1 = 2 * precision * recall / (precision + recall)\n",
        "    else:\n",
        "      f1 = 0\n",
        "\n",
        "    \n",
        "    run[\"eval/loss\"] = loss\n",
        "    run[\"eval/accuracy\"] = accuracy\n",
        "    run[\"eval/precision\"] = precision\n",
        "    run[\"eval/recall\"] = recall\n",
        "    run[\"eval/f1\"] = f1\n",
        "    run[\"eval/maxepochs\"] = max_epochs\n",
        "  return model"
      ],
      "metadata": {
        "id": "0u4f4UAm9dwd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def losoeval(model, run, params):\n",
        "  # Evaluating every models on the corresponding test dataset not seen during training.\n",
        "  os_scores_acc = {}\n",
        "  os_scores_f1 = {}\n",
        "\n",
        "  for os, signals in SMARTWATCH_OS.items():\n",
        "\n",
        "    all_subjects_X_os = filter_for_smartwatch_os(os, all_subjects_X)\n",
        "\n",
        "    all_accuracies = []\n",
        "    all_precisions = []\n",
        "    all_recalls = []\n",
        "    all_f1s = []\n",
        "    subject_ids = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17] # ids for subjects in WESAD dataset\n",
        "    \n",
        "    for i, subject_id in enumerate(subject_ids):\n",
        "        X_test = all_subjects_X_os[i]\n",
        "        y_test = all_subjects_y[i]\n",
        "        X_test = np.asarray(X_test)\n",
        "        y_test = np.asarray(y_test)\n",
        "        \n",
        "        y_test = tf.keras.utils.to_categorical(y_test, 2)\n",
        "        \n",
        "                                            \n",
        "          \n",
        "        accuracy = model.evaluate(X_test, y_test, verbose=0 )[1]\n",
        "        precision = model.evaluate(X_test, y_test, verbose=0 )[2]\n",
        "        recall = model.evaluate(X_test, y_test, verbose=0 )[3]\n",
        "        if (precision + recall) != 0:\n",
        "          f1 = 2 * precision * recall / (precision + recall)\n",
        "        else:\n",
        "          f1 = 0\n",
        "        all_accuracies.append(accuracy)\n",
        "        all_precisions.append(precision)\n",
        "        all_recalls.append(recall)\n",
        "        all_f1s.append(f1)\n",
        "\n",
        "    print(f'Smartwatch OS: {os}')\n",
        "    print(f'Evaluation of Transformer model trained on {params[\"epochs\"]} epochs\\n')\n",
        "    print(f'Subject\\t\\t Accuracy\\tPrecision\\tRecall\\t\\tF1-Score')\n",
        "    print(\"************************************************************************\")\n",
        "    for i in range(len(all_accuracies)):\n",
        "        print(f'S{subject_ids[i]}\\t\\t {round(all_accuracies[i], 5):.5f}\\t{round(all_precisions[i], 5):.5f}\\t\\t{round(all_recalls[i], 5):.5f}\\t\\t{round(all_f1s[i], 5):.5f}')\n",
        "\n",
        "    print(\"************************************************************************\")\n",
        "    print(f'Average\\t\\t {round(np.mean(all_accuracies), 5):.5f}\\t{round(np.mean(all_precisions), 5):.5f}\\t\\t{round(np.mean(all_recalls), 5):.5f}\\t\\t{round(np.mean(all_f1s), 5):.5f}\\n\\n\\n')\n",
        "\n",
        "    os_scores_acc[os] = all_accuracies\n",
        "    os_scores_f1[os] = all_f1s\n",
        "  run[\"loso/accuracy\"] = round(np.mean(all_accuracies), 5)\n",
        "  run[\"loso/precision\"] = round(np.mean(all_precisions), 5)\n",
        "  run[\"loso/recall\"] = round(np.mean(all_recalls), 5)\n",
        "  run[\"loso/f1\"] = round(np.mean(all_f1s), 5)\n",
        "  \n",
        "\n",
        "  run.stop()\n"
      ],
      "metadata": {
        "id": "4M7qCe1QA57L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loopstart with Neptune AI"
      ],
      "metadata": {
        "id": "GJeA_b0HYdRZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dic_numencblocks= [1,2,3,4,5,6,7]\n",
        "dic_mlpdropout= [0.6,0.6,0.6,0.6,0.6]\n",
        "dic_dropout= [0.4,0.4,0.4,0.4,0.4,0.5,0.5,0.5,0.5,0.5,0.6,0.6,0.6,0.6,0.6,0.7,0.7,0.7,0.7,0.7]\n",
        "dic_heads = [5,5,5]\n",
        "dic_headsize = [512]\n",
        "dic_epochs = [110]\n",
        "dic_noise = [68.61306896515201,68.61306896515201,68.61306896515201,68.61306896515201,68.61306896515201]\n",
        "\n",
        "index = 0;\n",
        "run = neptune.init_run( api_token=\"add your api_token here\", project=\"add your project name here\")\n",
        "\n",
        "for noise in dic_noise:\n",
        "  if index != 0:\n",
        "    run = neptune.init_run( api_token=\"add your api_token here\", project=\"add your project name here\")\n",
        "\n",
        "  index = index +1\n",
        "  params = setparams(110, 8, 0.25, 0.25, 4, 256, 1, 2.0, noise)\n",
        "  run['hyper-parameters'] = params\n",
        "  model = trainmodel(run,params)\n",
        "  losoeval(model, run, params)\n",
        "  \n",
        "\n",
        "\n",
        "  \n",
        "  "
      ],
      "metadata": {
        "id": "p-Gv6VypB6Mu"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "cqScMeNYLoBo",
        "RiBUcvWLrK9k",
        "3Qz6GiLRKDRo",
        "wfe-XokHPpJG"
      ],
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3.9.13 ('pageblank')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "05320e546f369dcc8e105d9a3b35062d70587d63bda88e4cea376da0cda0edd6"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}