bytehub_featurestore.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    },
    "colab": {
      "name": "bytehub-featurestore.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bytehub-ai/code-examples/blob/main/bytehub_featurestore.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gn87fD_mfCHB",
        "outputId": "df435cb6-6688-4302-c463-0ebc46bd8587"
      },
      "source": [
        "!pip install bytehub>=0.2.1"
      ],
      "id": "Gn87fD_mfCHB",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[31mERROR: distributed 2021.2.0 has requirement cloudpickle>=1.5.0, but you'll have cloudpickle 1.3.0 which is incompatible.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "second-trouble"
      },
      "source": [
        "import pandas as pd\n",
        "import bytehub as bh\n",
        "import shutil\n",
        "import os"
      ],
      "id": "second-trouble",
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "protective-button"
      },
      "source": [
        "# Remove previously created Feature Store\n",
        "try:\n",
        "    os.remove('bytehub.db')\n",
        "except FileNotFoundError:\n",
        "    pass\n",
        "try:\n",
        "    shutil.rmtree('/tmp/test-data')\n",
        "except FileNotFoundError:\n",
        "    pass"
      ],
      "id": "protective-button",
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "greater-dimension"
      },
      "source": [
        "# Create a Feature Store\n",
        "fs = bh.FeatureStore()"
      ],
      "id": "greater-dimension",
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "perceived-concept"
      },
      "source": [
        "# Create a namespace to store some data\n",
        "fs.create_namespace(\n",
        "    'test', url='/tmp/test-data', description='Test data'\n",
        ")"
      ],
      "id": "perceived-concept",
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "mighty-tension",
        "outputId": "31455dc6-3024-4237-cc08-6be7f0f08e7c"
      },
      "source": [
        "fs.list_namespaces()"
      ],
      "id": "mighty-tension",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>version</th>\n",
              "      <th>description</th>\n",
              "      <th>meta</th>\n",
              "      <th>storage_options</th>\n",
              "      <th>url</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>test</td>\n",
              "      <td>1</td>\n",
              "      <td>Test data</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>/tmp/test-data</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   name  version description meta storage_options             url\n",
              "0  test        1   Test data   {}              {}  /tmp/test-data"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "superb-portfolio",
        "outputId": "22471eb2-e81a-47e2-f418-5faacdaa8cd4"
      },
      "source": [
        "# Get some Bitcoin Price data\n",
        "import requests\n",
        "\n",
        "response = requests.get('https://api.coindesk.com/v1/bpi/historical/close.json?start=2015-01-01&end=2021-01-01')\n",
        "response.raise_for_status()\n",
        "\n",
        "df_close = pd.DataFrame(\n",
        "    {\n",
        "        'time': pd.to_datetime(list(response.json().get('bpi').keys())),\n",
        "        'created_time': pd.to_datetime(list(response.json().get('bpi').keys())),\n",
        "        'value': response.json().get('bpi').values()\n",
        "    }\n",
        ").set_index('time')\n",
        "df_close.head()"
      ],
      "id": "superb-portfolio",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>created_time</th>\n",
              "      <th>value</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>time</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2015-01-01</th>\n",
              "      <td>2015-01-01</td>\n",
              "      <td>313.9247</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-01-02</th>\n",
              "      <td>2015-01-02</td>\n",
              "      <td>314.5916</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-01-03</th>\n",
              "      <td>2015-01-03</td>\n",
              "      <td>279.8507</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-01-04</th>\n",
              "      <td>2015-01-04</td>\n",
              "      <td>263.6343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-01-05</th>\n",
              "      <td>2015-01-05</td>\n",
              "      <td>272.9486</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           created_time     value\n",
              "time                             \n",
              "2015-01-01   2015-01-01  313.9247\n",
              "2015-01-02   2015-01-02  314.5916\n",
              "2015-01-03   2015-01-03  279.8507\n",
              "2015-01-04   2015-01-04  263.6343\n",
              "2015-01-05   2015-01-05  272.9486"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "likely-wilson"
      },
      "source": [
        "# Store it in features in our namespace\n",
        "fs.create_feature('test/bitcoin.close', partition='year')\n",
        "fs.save_dataframe(df_close, 'test/bitcoin.close')"
      ],
      "id": "likely-wilson",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "floral-berkeley",
        "outputId": "376e389d-675f-4142-8ca3-b894e9ae0719"
      },
      "source": [
        "# Compute some moving averages\n",
        "df_averages = pd.concat(\n",
        "    [\n",
        "        df_close.ewm(halflife=5).mean().rename(columns={'value': 'test/bitcoin.ewma-5'}),\n",
        "        df_close.ewm(halflife=20).mean().rename(columns={'value': 'test/bitcoin.ewma-20'})\n",
        "    ], axis=1\n",
        ")\n",
        "df_averages.tail()"
      ],
      "id": "floral-berkeley",
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>test/bitcoin.ewma-5</th>\n",
              "      <th>test/bitcoin.ewma-20</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>time</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2020-12-28</th>\n",
              "      <td>23791.513397</td>\n",
              "      <td>19433.880895</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-12-29</th>\n",
              "      <td>24252.672472</td>\n",
              "      <td>19703.668488</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-12-30</th>\n",
              "      <td>24852.459289</td>\n",
              "      <td>20016.453770</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-12-31</th>\n",
              "      <td>25383.694626</td>\n",
              "      <td>20320.976560</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2021-01-01</th>\n",
              "      <td>25902.538373</td>\n",
              "      <td>20629.961254</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            test/bitcoin.ewma-5  test/bitcoin.ewma-20\n",
              "time                                                 \n",
              "2020-12-28         23791.513397          19433.880895\n",
              "2020-12-29         24252.672472          19703.668488\n",
              "2020-12-30         24852.459289          20016.453770\n",
              "2020-12-31         25383.694626          20320.976560\n",
              "2021-01-01         25902.538373          20629.961254"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "assured-demand"
      },
      "source": [
        "# Store the averages in the feature store\n",
        "fs.create_feature('test/bitcoin.ewma-5', partition='year')\n",
        "fs.create_feature('test/bitcoin.ewma-20', partition='year')\n",
        "fs.save_dataframe(df_averages)"
      ],
      "id": "assured-demand",
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        },
        "id": "further-league",
        "outputId": "9e1afcb9-98ce-4fc2-98ee-4f6fead26f62"
      },
      "source": [
        "# List features\n",
        "fs.list_features(regex=r'.ewma.')"
      ],
      "id": "further-league",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>namespace</th>\n",
              "      <th>name</th>\n",
              "      <th>version</th>\n",
              "      <th>description</th>\n",
              "      <th>meta</th>\n",
              "      <th>partition</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>test</td>\n",
              "      <td>bitcoin.ewma-5</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>{}</td>\n",
              "      <td>year</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>test</td>\n",
              "      <td>bitcoin.ewma-20</td>\n",
              "      <td>1</td>\n",
              "      <td></td>\n",
              "      <td>{}</td>\n",
              "      <td>year</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  namespace             name  version description meta partition\n",
              "1      test   bitcoin.ewma-5        1               {}      year\n",
              "2      test  bitcoin.ewma-20        1               {}      year"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "included-quantity",
        "outputId": "ffa4f057-58d5-4ad3-95a8-0232869cbd32"
      },
      "source": [
        "# Load and filter data\n",
        "df = fs.load_dataframe(['test/bitcoin.close', 'test/bitcoin.ewma-5'], from_date='2020-01-01', to_date='2021-12-31')\n",
        "df.head()"
      ],
      "id": "included-quantity",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>test/bitcoin.close</th>\n",
              "      <th>test/bitcoin.ewma-5</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>time</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2020-01-01</th>\n",
              "      <td>7188.4633</td>\n",
              "      <td>7244.504693</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-01-02</th>\n",
              "      <td>6961.5683</td>\n",
              "      <td>7207.878736</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-01-03</th>\n",
              "      <td>7346.5800</td>\n",
              "      <td>7225.833537</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-01-04</th>\n",
              "      <td>7355.8550</td>\n",
              "      <td>7242.664742</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-01-05</th>\n",
              "      <td>7356.3117</td>\n",
              "      <td>7257.376277</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            test/bitcoin.close  test/bitcoin.ewma-5\n",
              "time                                               \n",
              "2020-01-01           7188.4633          7244.504693\n",
              "2020-01-02           6961.5683          7207.878736\n",
              "2020-01-03           7346.5800          7225.833537\n",
              "2020-01-04           7355.8550          7242.664742\n",
              "2020-01-05           7356.3117          7257.376277"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "dutch-technique",
        "outputId": "340269b0-0683-4ef3-a9fd-6621e1637eba"
      },
      "source": [
        "# Resample to monthly timeseries\n",
        "df = fs.load_dataframe(fs.list_features(regex=r'bitcoin\\..'), from_date='2020-01-01', to_date='2021-12-31', freq='1M')\n",
        "df.head()"
      ],
      "id": "dutch-technique",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>test/bitcoin.close</th>\n",
              "      <th>test/bitcoin.ewma-5</th>\n",
              "      <th>test/bitcoin.ewma-20</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2020-01-31</th>\n",
              "      <td>9346.1333</td>\n",
              "      <td>8887.704013</td>\n",
              "      <td>8238.519660</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-02-29</th>\n",
              "      <td>8538.3983</td>\n",
              "      <td>9291.578593</td>\n",
              "      <td>9086.141735</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-03-31</th>\n",
              "      <td>6449.9500</td>\n",
              "      <td>6423.066033</td>\n",
              "      <td>7490.873582</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-04-30</th>\n",
              "      <td>8740.7500</td>\n",
              "      <td>7632.546030</td>\n",
              "      <td>7383.347519</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2020-05-31</th>\n",
              "      <td>9545.1500</td>\n",
              "      <td>9279.717876</td>\n",
              "      <td>8623.681782</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            test/bitcoin.close  test/bitcoin.ewma-5  test/bitcoin.ewma-20\n",
              "2020-01-31           9346.1333          8887.704013           8238.519660\n",
              "2020-02-29           8538.3983          9291.578593           9086.141735\n",
              "2020-03-31           6449.9500          6423.066033           7490.873582\n",
              "2020-04-30           8740.7500          7632.546030           7383.347519\n",
              "2020-05-31           9545.1500          9279.717876           8623.681782"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "graduate-experiment",
        "outputId": "5f946f1d-54d0-4ee6-dd00-3ea9ab46e500"
      },
      "source": [
        "# Get last values\n",
        "bitcoin_features = fs.list_features(regex=r'bitcoin\\..')\n",
        "last_values = fs.last(bitcoin_features)\n",
        "last_values"
      ],
      "id": "graduate-experiment",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'test/bitcoin.close': 29391.775,\n",
              " 'test/bitcoin.ewma-20': 20629.961254095204,\n",
              " 'test/bitcoin.ewma-5': 25902.538372937153}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "passing-sunglasses"
      },
      "source": [
        "# Delete the features\n",
        "for feature in last_values.keys():\n",
        "    fs.delete_feature(feature)"
      ],
      "id": "passing-sunglasses",
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "premier-wright"
      },
      "source": [
        "# Delete the namespace\n",
        "fs.delete_namespace('test')"
      ],
      "id": "premier-wright",
      "execution_count": 20,
      "outputs": []
    }
  ]
}