time_cv-multi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report\n",
    "import lightgbm\n",
    "from sklearn.model_selection import StratifiedKFold, RandomizedSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.calibration import calibration_curve\n",
    "from sklearn.calibration import CalibratedClassifierCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "wl_odds_player_df = pd.read_csv(dir + \"wl_odds_player_df.csv\")\n",
    "\n",
    "wl_odds_player_df = wl_odds_player_df.drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['league_id', 'season', 'date', 'stage', 'match_id', 'team_id',\n",
       "       'm_rating', 'win', 'lose', 'rating_diff', 'home', 'cumulative_win',\n",
       "       'cumulative_lose', 'match_num', 'match_bin', 'draw',\n",
       "       'avg_overall_rating', 'avg_volleys_imp', 'avg_long_passing',\n",
       "       'avg_finishing', 'avg_ball_control', 'avg_dribbling',\n",
       "       'avg_short_passing', 'avg_reactions', 'avg_vision_imp', 'avg_penalties',\n",
       "       'avg_agility_imp', 'result', 'cumulative_result'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wl_odds_player_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "seasons = sorted(wl_odds_player_df[\"season\"].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 1 season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_01 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-1):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+2][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+2][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[train_df[\"home\"]==1]\n",
    "    calib_df = calib_df.loc[calib_df[\"home\"]==1]\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"home\"]==1]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"result\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"result\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_01.append(log_loss(test_df[\"result\"], lgb_calib_pred))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 2 season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_02 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-2):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+3][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+3][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[train_df[\"home\"]==1]\n",
    "    calib_df = calib_df.loc[calib_df[\"home\"]==1]\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"home\"]==1]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"result\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"result\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_02.append(log_loss(test_df[\"result\"], lgb_calib_pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 3 seasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_03 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[train_df[\"home\"]==1]\n",
    "    calib_df = calib_df.loc[calib_df[\"home\"]==1]\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"home\"]==1]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"result\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"result\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_03.append(log_loss(test_df[\"result\"], lgb_calib_pred))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0058695291211024, 0.9984708416646916, 0.9954775308959847)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(ll_01), np.mean(ll_02), np.mean(ll_03)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 3 seasons with current season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_04 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "    \n",
    "    test_dates = sorted(test_df[\"date\"].unique())\n",
    "    n = len(test_dates) // 2\n",
    "\n",
    "    train_df = pd.concat([train_df, \n",
    "                          test_df.loc[test_df[\"date\"].isin(test_dates[:n])]])\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"date\"].isin(test_dates[n:])]\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[train_df[\"home\"]==1]\n",
    "    calib_df = calib_df.loc[calib_df[\"home\"]==1]\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"home\"]==1]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"result\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"result\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_04.append(log_loss(test_df[\"result\"], lgb_calib_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0301665441044627"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(ll_04)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_05 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "    \n",
    "    test_dates = sorted(test_df[\"date\"].unique())\n",
    "    n = len(test_dates) // 2\n",
    "\n",
    "    #train_df = pd.concat([train_df, \n",
    "    #                      test_df.loc[test_df[\"date\"].isin(test_dates[:n])]])\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"date\"].isin(test_dates[n:])]\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[train_df[\"home\"]==1]\n",
    "    calib_df = calib_df.loc[calib_df[\"home\"]==1]\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"home\"]==1]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"result\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"result\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_05.append(log_loss(test_df[\"result\"], lgb_calib_pred))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.978746809994982"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(ll_05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.54      0.47      0.50       445\n",
      "           0       0.00      0.00      0.00       350\n",
      "           1       0.53      0.86      0.65       648\n",
      "\n",
      "    accuracy                           0.53      1443\n",
      "   macro avg       0.36      0.44      0.38      1443\n",
      "weighted avg       0.40      0.53      0.45      1443\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Watanatornpitakp/opt/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "y_true = test_df[\"result\"].values\n",
    "y_pred = lgb_calib.predict(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                         \"cumulative_lose\", \"cumulative_win\",\n",
    "                                         \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "\n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.46      0.64      0.54       445\n",
      "           0       0.50      0.01      0.02       350\n",
      "           1       0.57      0.72      0.64       648\n",
      "\n",
      "    accuracy                           0.52      1443\n",
      "   macro avg       0.51      0.46      0.40      1443\n",
      "weighted avg       0.52      0.52      0.46      1443\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_true = test_df[\"result\"].values\n",
    "y_pred = lgb_cv.predict(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                      \"cumulative_lose\", \"cumulative_win\",\n",
    "                                      \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "\n",
    "print(classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}