pl_agg_markPrice.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime\n",
    "import urllib.request\n",
    "from tqdm import tqdm\n",
    "import zipfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_klines_data(trading_pair: str, from_date: datetime, number_of_days: int, download_dir: str):\n",
    "    trading_pair = trading_pair.upper()\n",
    "    date_list = [from_date - datetime.timedelta(days=x) for x in range(1, number_of_days + 1)]\n",
    "    os.makedirs(download_dir, exist_ok=True)\n",
    "    data = pl.DataFrame()\n",
    "    print(f\"DOWNLOADING {number_of_days} FILES TO {download_dir} AND GENERATING mark price klines\")\n",
    "    for current_date in tqdm(date_list):\n",
    "        try:\n",
    "            zip_file_path = f\"./{download_dir}/{trading_pair}_{current_date.strftime('%Y-%m-%d')}\"\n",
    "            urllib.request.urlretrieve(f\"https://data.binance.vision/data/futures/um/daily/markPriceKlines/{trading_pair}/1m/{trading_pair}-1m-{current_date.strftime('%Y-%m-%d')}.zip\", zip_file_path)\n",
    "            with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
    "                zip_ref.extractall(download_dir)\n",
    "            os.remove(zip_file_path)\n",
    "            file = f\"./{download_dir}/{trading_pair}-1m-{current_date.strftime('%Y-%m-%d')}.csv\"\n",
    "            downloaded = pl.read_csv(file)\n",
    "            os.remove(file)\n",
    "            downloaded = downloaded.select(pl.col([\"close_time\", \"open\", \"high\", \"low\", \"close\"]))\n",
    "            downloaded = downloaded.rename({\"close_time\": \"time\"})\n",
    "            downloaded = downloaded.lazy().with_columns(pl.from_epoch(\"time\", unit='ms'))\n",
    "            downloaded = downloaded.lazy().with_columns(pl.from_epoch(\"time\", unit='us')).collect()\n",
    "            data = pl.concat([data, downloaded])\n",
    "        except Exception as e:\n",
    "            print(f\"Something went wrong with {trading_pair} {current_date}: {e}\")\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DOWNLOADING 136 FILES TO SOLUSDT_trades AND GENERATING mark price klines\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 136/136 [03:02<00:00,  1.34s/it]\n"
     ]
    }
   ],
   "source": [
    "trading_pair = \"ETHUSDT\"\n",
    "trading_pair = trading_pair.upper()\n",
    "from_date = datetime.datetime.strptime('23032023', \"%d%m%Y\").date()\n",
    "download_dir = f\"./{trading_pair}_daily_trades_data_{from_date.strftime('%Y-%m-%d')}\"\n",
    "df = get_klines_data(trading_pair, from_date, 136, \"SOLUSDT_trades\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(195840, 5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (30, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>time</th><th>open</th><th>high</th><th>low</th><th>close</th></tr><tr><td>datetime[μs]</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>2023-03-22 00:00:59.999</td><td>1800.385763</td><td>1801.08</td><td>1799.695763</td><td>1800.52166</td></tr><tr><td>2023-03-22 00:01:59.999</td><td>1800.52166</td><td>1801.36</td><td>1799.94</td><td>1800.13</td></tr><tr><td>2023-03-22 00:02:59.999</td><td>1800.13</td><td>1800.99</td><td>1799.84</td><td>1800.06</td></tr><tr><td>2023-03-22 00:03:59.999</td><td>1800.06</td><td>1801.0</td><td>1800.05</td><td>1800.53</td></tr><tr><td>2023-03-22 00:04:59.999</td><td>1800.53</td><td>1800.77</td><td>1800.024079</td><td>1800.44</td></tr><tr><td>2023-03-22 00:05:59.999</td><td>1800.44</td><td>1800.61</td><td>1799.657776</td><td>1799.805028</td></tr><tr><td>2023-03-22 00:06:59.999</td><td>1799.805028</td><td>1799.92</td><td>1798.980513</td><td>1799.640564</td></tr><tr><td>2023-03-22 00:07:59.999</td><td>1799.630308</td><td>1799.73</td><td>1798.790436</td><td>1799.08</td></tr><tr><td>2023-03-22 00:08:59.999</td><td>1799.14</td><td>1800.45</td><td>1799.14</td><td>1799.535925</td></tr><tr><td>2023-03-22 00:09:59.999</td><td>1799.500891</td><td>1799.506019</td><td>1798.902733</td><td>1798.902733</td></tr><tr><td>2023-03-22 00:10:59.999</td><td>1798.896417</td><td>1799.25</td><td>1798.75279</td><td>1798.881058</td></tr><tr><td>2023-03-22 00:11:59.999</td><td>1798.881058</td><td>1799.63</td><td>1798.697595</td><td>1798.697595</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>2023-03-22 00:18:59.999</td><td>1798.54</td><td>1799.33</td><td>1798.397299</td><td>1798.704949</td></tr><tr><td>2023-03-22 00:19:59.999</td><td>1798.642385</td><td>1798.642385</td><td>1796.463257</td><td>1797.66</td></tr><tr><td>2023-03-22 00:20:59.999</td><td>1797.66</td><td>1798.82</td><td>1797.66</td><td>1798.136308</td></tr><tr><td>2023-03-22 00:21:59.999</td><td>1798.105675</td><td>1798.105675</td><td>1796.634953</td><td>1796.85</td></tr><tr><td>2023-03-22 00:22:59.999</td><td>1796.84</td><td>1796.93</td><td>1796.448752</td><td>1796.56891</td></tr><tr><td>2023-03-22 00:23:59.999</td><td>1796.566291</td><td>1797.33</td><td>1796.536731</td><td>1796.73</td></tr><tr><td>2023-03-22 00:24:59.999</td><td>1797.15</td><td>1797.200637</td><td>1796.896932</td><td>1796.903188</td></tr><tr><td>2023-03-22 00:25:59.999</td><td>1796.904726</td><td>1798.29</td><td>1796.904726</td><td>1798.29</td></tr><tr><td>2023-03-22 00:26:59.999</td><td>1798.29</td><td>1798.3</td><td>1797.2</td><td>1797.92</td></tr><tr><td>2023-03-22 00:27:59.999</td><td>1797.997573</td><td>1798.28</td><td>1797.109761</td><td>1797.109761</td></tr><tr><td>2023-03-22 00:28:59.999</td><td>1796.99335</td><td>1796.99335</td><td>1796.26683</td><td>1796.65</td></tr><tr><td>2023-03-22 00:29:59.999</td><td>1796.66</td><td>1797.64</td><td>1796.135286</td><td>1796.135368</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (30, 5)\n",
       "┌─────────────────────────┬─────────────┬────────────┬─────────────┬─────────────┐\n",
       "│ time                    ┆ open        ┆ high       ┆ low         ┆ close       │\n",
       "│ ---                     ┆ ---         ┆ ---        ┆ ---         ┆ ---         │\n",
       "│ datetime[μs]            ┆ f64         ┆ f64        ┆ f64         ┆ f64         │\n",
       "╞═════════════════════════╪═════════════╪════════════╪═════════════╪═════════════╡\n",
       "│ 2023-03-22 00:00:59.999 ┆ 1800.385763 ┆ 1801.08    ┆ 1799.695763 ┆ 1800.52166  │\n",
       "│ 2023-03-22 00:01:59.999 ┆ 1800.52166  ┆ 1801.36    ┆ 1799.94     ┆ 1800.13     │\n",
       "│ 2023-03-22 00:02:59.999 ┆ 1800.13     ┆ 1800.99    ┆ 1799.84     ┆ 1800.06     │\n",
       "│ 2023-03-22 00:03:59.999 ┆ 1800.06     ┆ 1801.0     ┆ 1800.05     ┆ 1800.53     │\n",
       "│ …                       ┆ …           ┆ …          ┆ …           ┆ …           │\n",
       "│ 2023-03-22 00:26:59.999 ┆ 1798.29     ┆ 1798.3     ┆ 1797.2      ┆ 1797.92     │\n",
       "│ 2023-03-22 00:27:59.999 ┆ 1797.997573 ┆ 1798.28    ┆ 1797.109761 ┆ 1797.109761 │\n",
       "│ 2023-03-22 00:28:59.999 ┆ 1796.99335  ┆ 1796.99335 ┆ 1796.26683  ┆ 1796.65     │\n",
       "│ 2023-03-22 00:29:59.999 ┆ 1796.66     ┆ 1797.64    ┆ 1796.135286 ┆ 1796.135368 │\n",
       "└─────────────────────────┴─────────────┴────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write_parquet(\"./ETHUSDT_MARK_PRICE_136_23032023.parquet\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}