feat_sel_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "from ml_bets.constants import FEATURES_PATH\n",
    "from ml_bets.features.features import Features\n",
    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
    "from feature_selection.feature_selection import FeatureSelection\n",
    "\n",
    "from pycaret.utils import check_metric\n",
    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
    "    create_model,\n",
    "    finalize_model,\n",
    "    optimize_threshold,              \n",
    "    save_model,\n",
    "    compare_models, \n",
    "    evaluate_model,\n",
    "    get_config,\n",
    "    setup,\n",
    "    tune_model,\n",
    "    predict_model,\n",
    ")\n",
    "\n",
    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
    "from ml_bets.research.datasets import create_dataset\n",
    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = \"goals_2.5\"\n",
    "test_date = \"1-Dec-2021\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/guillem/ml_bets/data/future_matches/ESP1C.xls\n",
      "/home/guillem/ml_bets/data/future_matches/ING1C.xls\n",
      "/home/guillem/ml_bets/data/future_matches/ITA1C.xls\n",
      "Excel file /home/guillem/ml_bets/data/future_matches/MEX1C.xls is empty. Skipping\n",
      "/home/guillem/ml_bets/data/future_matches/FRA1C.xls\n",
      "Excel file /home/guillem/ml_bets/data/future_matches/MLS1C.xls is empty. Skipping\n",
      "/home/guillem/ml_bets/data/future_matches/ALE1C.xls\n"
     ]
    }
   ],
   "source": [
    "feats = Features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def patched_create_dataset(\n",
    "    test_date: str,\n",
    "    target: str,\n",
    "    columns=None,\n",
    "    odds_features: bool = True,\n",
    "    summary: bool = True,\n",
    "    odds_rankings: bool = True,\n",
    "    include_std: bool = True,\n",
    "    features: Features = None,\n",
    "    ignore_features=None,\n",
    "    drop_future_matches: bool = True,\n",
    "    test_weeks: int = 4,\n",
    "):\n",
    "    features = features or Features()\n",
    "    examples = features.create(\n",
    "        columns=columns,\n",
    "        odds_features=odds_features,\n",
    "        odds_rankings=odds_rankings,\n",
    "        referee_features=True,\n",
    "        include_std=include_std,\n",
    "        summary=summary,\n",
    "    )\n",
    "    examples = examples[[x for x in examples.columns if \"possession\" not in x]]\n",
    "    if ignore_features is not None:\n",
    "        examples.drop(columns=ignore_features, inplace=True)\n",
    "    pds = PipelineDatasets(\n",
    "        examples=examples,\n",
    "        features=features,\n",
    "        target=target,\n",
    "        drop_future_matches=drop_future_matches,\n",
    "        test_size=test_date,\n",
    "        test_weeks=test_weeks,\n",
    "    )\n",
    "    return pds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = patched_create_dataset(target=target,\n",
    "                    test_date=test_date,\n",
    "                    features=feats,\n",
    "                    odds_features=True,\n",
    "                    include_std=True,\n",
    "                    ignore_features=IGNORE_FEATURES+[\"referee\", \"hour_rank\", \"hour_before_16\", \"is_weekend\"],\n",
    "                    drop_future_matches=False,\n",
    "                    \n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = ds.train_data.copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "setup_kwargs = dict(\n",
    "        preprocess=True,\n",
    "        #custom_pipeline=loaded,\n",
    "        train_size=0.75,\n",
    "        session_id=123,\n",
    "        normalize=True,\n",
    "       # normalize_method=\"robust\",\n",
    "        transformation=True,\n",
    "        ignore_low_variance=True,\n",
    "        remove_multicollinearity=False,\n",
    "        multicollinearity_threshold=0.8,\n",
    "        n_jobs=-1,\n",
    "        use_gpu=False,\n",
    "        profile=False,\n",
    "        #ignore_features=ignore_features,\n",
    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
    "        remove_perfect_collinearity=True,\n",
    "        create_clusters=False,\n",
    "        fold=3,\n",
    "        feature_selection=False,\n",
    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
    "        feature_selection_threshold=0.5,\n",
    "        combine_rare_levels=False,\n",
    "        rare_level_threshold=0.02,\n",
    "        pca=False,\n",
    "        pca_method=\"kernel\",\n",
    "        pca_components=50,\n",
    "        polynomial_features=False,\n",
    "        polynomial_degree=2,\n",
    "        polynomial_threshold=0.05,\n",
    "        trigonometry_features=False,\n",
    "        remove_outliers=True,\n",
    "        outliers_threshold=0.01,\n",
    "        feature_ratio=False,\n",
    "        feature_interaction=False,\n",
    "        # Makes everything slow AF. use to find out possibly interesting features\n",
    "        interaction_threshold=0.05,\n",
    "        fix_imbalance=True,\n",
    "        log_experiment=False,\n",
    "        verbose=False,\n",
    "        silent=True,\n",
    "        experiment_name=\"lagstest\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature selection (without triplet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5158</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5292</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5257</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5111</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0323</td>\n",
       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0323</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb80daa7e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5103</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5309</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.4946</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5199</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5069</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0212</td>\n",
       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0212</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb80c0ebc10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5282</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5543</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5351</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5366</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5359</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0561</td>\n",
       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0561</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb8237074c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5296</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5570</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5297</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5385</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5341</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0591</td>\n",
       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0591</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb81b0d0070>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metric_param = {\n",
    "        \"Accuracy\": -0.1,\n",
    "        \"AUC\": -0.1,\n",
    "        \"Recall\": -0.1,\n",
    "        \"Precision\": -0.1,\n",
    "        \"F1\": -0.1,\n",
    "        \"Kappa\": -1.0,\n",
    "        \"MCC\": -1.0,\n",
    "    }\n",
    "feat_sel = FeatureSelection(target=target,\n",
    "                            dataset=train_data.dropna(),#[list(set(new_subset+new_feat+[target]))],\n",
    "                            target_features=500,\n",
    "                            filter_metrics=metric_param,\n",
    "                            include=[\"lr\"],\n",
    "                            setup_kwargs=setup_kwargs,\n",
    "                            optimize=True,\n",
    "                            opt_list=[\"AUC\"],\n",
    "                           )\n",
    "selected_features = feat_sel.repeat_pipeline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DEF_mean_a',\n",
       " 'MED_diff',\n",
       " 'accurate_cross_nocorner_std_diff',\n",
       " 'accurate_cross_std_h',\n",
       " 'accurate_fwd_zone_pass_mean_diff',\n",
       " 'accurate_fwd_zone_pass_std_a',\n",
       " 'accurate_goal_kicks_std_h',\n",
       " 'accurate_keeper_sweeper_mean_a',\n",
       " 'accurate_keeper_sweeper_mean_h',\n",
       " 'accurate_keeper_sweeper_std_a',\n",
       " 'accurate_launches_mean_a',\n",
       " 'accurate_layoffs_mean_a',\n",
       " 'accurate_layoffs_std_a',\n",
       " 'accurate_pass_std_diff',\n",
       " 'accurate_through_ball_mean_diff',\n",
       " 'accurate_throws_std_diff',\n",
       " 'accurate_throws_std_h',\n",
       " 'att_assist_openplay_mean_a',\n",
       " 'att_corner_ratio_mean_a',\n",
       " 'att_freekick_goal_std_diff',\n",
       " 'att_freekick_miss_mean_h',\n",
       " 'att_freekick_miss_std_a',\n",
       " 'att_freekick_total_mean_a',\n",
       " 'att_freekick_total_mean_h',\n",
       " 'att_goal_high_centre_mean_diff',\n",
       " 'att_goal_high_centre_std_diff',\n",
       " 'att_goal_high_left_mean_diff',\n",
       " 'att_goal_high_left_std_h',\n",
       " 'att_goal_high_right_std_a',\n",
       " 'att_goal_high_right_std_diff',\n",
       " 'att_goal_low_centre_mean_a',\n",
       " 'att_goal_low_centre_std_a',\n",
       " 'att_goal_low_centre_std_h',\n",
       " 'att_goal_low_right_mean_diff',\n",
       " 'att_hd_goal_mean_a',\n",
       " 'att_hd_target_mean_a',\n",
       " 'att_ibox_own_goal_std_h',\n",
       " 'att_lf_goal_mean_diff',\n",
       " 'att_lf_goal_mean_h',\n",
       " 'att_lf_goal_std_h',\n",
       " 'att_lg_centre_mean_a',\n",
       " 'att_lg_centre_mean_diff',\n",
       " 'att_lg_centre_mean_h',\n",
       " 'att_miss_high_left_std_a',\n",
       " 'att_miss_right_std_h',\n",
       " 'att_obox_blocked_mean_h',\n",
       " 'att_obox_goal_mean_h',\n",
       " 'att_obox_goal_std_a',\n",
       " 'att_obox_goal_std_diff',\n",
       " 'att_obox_miss_mean_a',\n",
       " 'att_obox_miss_mean_diff',\n",
       " 'att_obox_post_mean_diff',\n",
       " 'att_obx_right_mean_diff',\n",
       " 'att_obxd_right_mean_diff',\n",
       " 'att_obxd_right_std_diff',\n",
       " 'att_openplay_mean_h',\n",
       " 'att_pen_goal_mean_a',\n",
       " 'att_post_left_mean_h',\n",
       " 'att_post_left_std_diff',\n",
       " 'attempts_conceded_ibox_std_diff',\n",
       " 'attempts_conceded_obox_std_a',\n",
       " 'attempts_conceded_obox_std_diff',\n",
       " 'attempts_conceded_obox_std_h',\n",
       " 'attempts_obox_mean_h',\n",
       " 'bc_miss_div_scored_std_h',\n",
       " 'big_chance_created_mean_h',\n",
       " 'big_chance_created_std_diff',\n",
       " 'big_chance_missed_std_diff',\n",
       " 'big_chance_scored_mean_diff',\n",
       " 'blocked_cross_mean_a',\n",
       " 'contentious_decision_mean_a',\n",
       " 'contentious_decision_std_a',\n",
       " 'corners_ratio_mean_diff',\n",
       " 'diving_save_std_diff',\n",
       " 'effective_clearance_mean_diff',\n",
       " 'effective_head_clearance_mean_h',\n",
       " 'first_yellow_card_std_a',\n",
       " 'goal_assist_deadball_mean_diff',\n",
       " 'goal_assist_deadball_std_diff',\n",
       " 'goal_assist_deadball_std_h',\n",
       " 'goal_assist_intent_norm_mean_h',\n",
       " 'goal_assist_intentional_mean_h',\n",
       " 'goal_assist_intentional_std_h',\n",
       " 'goal_assist_mean_a',\n",
       " 'goal_assist_mean_diff',\n",
       " 'goal_assist_openplay_std_a',\n",
       " 'goal_assist_openplay_std_h',\n",
       " 'goal_assist_setplay_mean_a',\n",
       " 'goal_assist_std_diff',\n",
       " 'goal_assist_std_h',\n",
       " 'goal_fastbreak_mean_diff',\n",
       " 'goal_kicks_per_shot_mean_a',\n",
       " 'goals_2t_mean_h',\n",
       " 'goals_mean_diff',\n",
       " 'goals_openplay_mean_a',\n",
       " 'interceptions_in_box_std_a',\n",
       " 'interceptions_in_box_std_diff',\n",
       " 'interceptions_in_box_std_h',\n",
       " 'last_man_tackle_mean_diff',\n",
       " 'last_man_tackle_std_a',\n",
       " 'leftside_pass_mean_a',\n",
       " 'long_pass_own_to_opp_success_std_diff',\n",
       " 'lost_corners_mean_a',\n",
       " 'no_foot_goals_ratio_mean_h',\n",
       " 'no_foot_goals_ratio_std_a',\n",
       " 'no_foot_goals_ratio_std_h',\n",
       " 'odd_ratio_under_goals_2.5',\n",
       " 'odds_away_over_cards_4.5_h',\n",
       " 'odds_away_over_cards_5.5_a',\n",
       " 'odds_away_under_cards_3.5_h',\n",
       " 'odds_home_under_cards_5.5_a',\n",
       " 'odds_home_under_goals_0.5_diff',\n",
       " 'ontarget_att_assist_mean_a',\n",
       " 'own_goals_std_h',\n",
       " 'passes_left_mean_diff',\n",
       " 'passes_right_mean_a',\n",
       " 'passes_right_mean_diff',\n",
       " 'passes_right_std_a',\n",
       " 'passes_right_std_diff',\n",
       " 'passes_right_std_h',\n",
       " 'pen_goals_conceded_std_diff',\n",
       " 'penalty_conceded_mean_a',\n",
       " 'penalty_faced_std_diff',\n",
       " 'penalty_save_std_h',\n",
       " 'points_diff',\n",
       " 'poss_won_att_3rd_std_a',\n",
       " 'poss_won_att_3rd_std_diff',\n",
       " 'poss_won_att_3rd_std_h',\n",
       " 'prob_over_goals_2.5',\n",
       " 'prob_squared_over_goals_4.5',\n",
       " 'prob_squared_under_goals_4.5',\n",
       " 'prob_under_goals_2.5',\n",
       " 'ratio_over_corners_11.5_diff',\n",
       " 'ratio_over_corners_9.5_diff',\n",
       " 'ratio_under_corners_10.5_h',\n",
       " 'ratio_under_goals_4.5_h',\n",
       " 'raw_prob_over_goals_1.5',\n",
       " 'red_card_1t_std_a',\n",
       " 'red_card_2t_mean_a',\n",
       " 'red_card_mean_h',\n",
       " 'red_card_std_a',\n",
       " 'red_card_std_diff',\n",
       " 'right_to_left_goals_mean_diff',\n",
       " 'right_to_left_goals_std_diff',\n",
       " 'saves_std_h',\n",
       " 'second_yellow_mean_diff',\n",
       " 'second_yellow_std_a',\n",
       " 'shots_std_diff',\n",
       " 'successful_final_third_passes_std_h',\n",
       " 'total_bets_under_cards_4.5_a',\n",
       " 'total_bets_under_corners_9.5_a',\n",
       " 'total_clearance_mean_a',\n",
       " 'total_expulsions_ref',\n",
       " 'total_fastbreak_std_diff',\n",
       " 'total_fwd_zone_pass_std_a',\n",
       " 'total_high_claim_mean_diff',\n",
       " 'total_keeper_sweeper_mean_a',\n",
       " 'total_keeper_sweeper_mean_diff',\n",
       " 'total_keeper_sweeper_std_diff',\n",
       " 'total_launches_mean_a',\n",
       " 'total_launches_std_h',\n",
       " 'total_layoffs_mean_diff',\n",
       " 'total_layoffs_std_diff',\n",
       " 'total_red_card_match_ref',\n",
       " 'total_second_yel_card_ref',\n",
       " 'total_through_ball_mean_a',\n",
       " 'total_through_ball_std_a',\n",
       " 'total_throws_std_a',\n",
       " 'total_throws_std_diff',\n",
       " 'total_win_pct_over_corners_9.5_a',\n",
       " 'total_win_pct_over_goals_2.5_h',\n",
       " 'total_win_pct_under_goals_1.5_a',\n",
       " 'total_win_pct_under_goals_3.5_h',\n",
       " 'total_yel_card_std_a',\n",
       " 'win_pct_away_over_goals_2.5_diff',\n",
       " 'win_pct_home_over_cards_6.5_a',\n",
       " 'win_pct_home_over_goals_2.5_h',\n",
       " 'win_pct_home_under_cards_4.5_diff',\n",
       " 'won_corners_ratio_mean_a',\n",
       " 'won_corners_ratio_mean_h',\n",
       " 'won_corners_ratio_std_h']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(selected_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### new imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "from ml_bets.constants import FEATURES_PATH\n",
    "from ml_bets.features.features import Features\n",
    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
    "#from featsel.feature_selection import FeatureSelection\n",
    "\n",
    "from pycaret.utils import check_metric\n",
    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
    "    create_model,\n",
    "    finalize_model,\n",
    "    optimize_threshold,              \n",
    "    save_model,\n",
    "    compare_models, \n",
    "    evaluate_model,\n",
    "    get_config,\n",
    "    setup,\n",
    "    tune_model,\n",
    "    predict_model,\n",
    ")\n",
    "\n",
    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
    "from ml_bets.research.datasets import create_dataset\n",
    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
    "index = feats.matches[~ix].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_dataset(test_date):\n",
    "    ds = create_dataset(target=target,\n",
    "                        test_date=test_date,\n",
    "                        features=feats,\n",
    "                        odds_features=True,\n",
    "                        include_std=True,\n",
    "                        test_weeks=4,\n",
    "                        ignore_features=IGNORE_FEATURES,\n",
    "                        drop_future_matches=False,\n",
    "                       )\n",
    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
    "    train_data = train_data[train_data.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
    "    setup_kwargs = dict(\n",
    "        preprocess=True,\n",
    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
    "        #custom_pipeline=loaded,\n",
    "        #train_size=0.75,\n",
    "        session_id=123,\n",
    "        normalize=True,\n",
    "        normalize_method=\"robust\",\n",
    "        transformation=True,\n",
    "        ignore_low_variance=True,\n",
    "        remove_multicollinearity=False,\n",
    "        multicollinearity_threshold=0.8,\n",
    "        n_jobs=-1,\n",
    "        use_gpu=False,\n",
    "        profile=False,\n",
    "        #ignore_features=ignore_features,\n",
    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
    "        remove_perfect_collinearity=True,\n",
    "        create_clusters=False,\n",
    "        fold=4,\n",
    "        feature_selection=False,\n",
    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
    "        feature_selection_threshold=0.5,\n",
    "        combine_rare_levels=False,\n",
    "        rare_level_threshold=0.02,\n",
    "        pca=True,\n",
    "        pca_method=\"kernel\",\n",
    "        pca_components=50,\n",
    "        polynomial_features=False,\n",
    "        polynomial_degree=2,\n",
    "        polynomial_threshold=0.05,\n",
    "        trigonometry_features=False,\n",
    "        remove_outliers=True,\n",
    "        outliers_threshold=0.01,\n",
    "        feature_ratio=False,\n",
    "        feature_interaction=False,\n",
    "        # Makes everything slow AF. use to find out possibly interesting features\n",
    "        interaction_threshold=0.05,\n",
    "        fix_imbalance=True,\n",
    "        log_experiment=False,\n",
    "        verbose=False,\n",
    "        silent=True,\n",
    "        experiment_name=\"lagstest\",\n",
    "    )\n",
    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
    "    return train_data, test_data, val_data, ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
    "def train_ensemble():\n",
    "    top_models = compare_models(\n",
    "            n_select=8,\n",
    "            sort='MCC',\n",
    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"rf\", \"svm\"],\n",
    "            verbose=True,\n",
    "        )\n",
    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
    "    blend = blend_models(cali)\n",
    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
    "    return opti, cali, tuned_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_21ce2d37_829b_11ec_86da_31d766f3ddbd th {\n",
       "          text-align: left;\n",
       "    }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  lightgrey;\n",
       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "            background-color:  lightgrey;\n",
       "        }</style><table id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5831</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6161</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5937</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6030</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5971</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.1652</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.1658</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.6300</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >svm</th>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >SVM - Linear Kernel</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.5780</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.0000</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5975</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6037</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.5845</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1544</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.1656</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.2050</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.5825</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5937</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6018</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.5966</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.1639</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.1645</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >lda</th>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >Linear Discriminant Analysis</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5819</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6107</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5962</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6003</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5972</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1624</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1629</td>\n",
       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.2100</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff4940a2880>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "top_models = compare_models(\n",
    "            n_select=8,\n",
    "            sort='MCC',\n",
    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
    "            verbose=True,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate_model(top_models[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature selection (adding triplet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(181,\n",
       " 583,\n",
       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
       "  'DEF_mean_a',\n",
       "  'DEF_mean_diff',\n",
       "  'DEF_mean_h',\n",
       "  'MED_a',\n",
       "  'MED_diff',\n",
       "  'MED_h',\n",
       "  'MED_mean_a',\n",
       "  'MED_mean_diff',\n",
       "  'MED_mean_h',\n",
       "  'MED_std_a',\n",
       "  'MED_std_diff',\n",
       "  'MED_std_h',\n",
       "  'accurate_cross_nocorner_std_a',\n",
       "  'accurate_cross_nocorner_std_diff',\n",
       "  'accurate_cross_nocorner_std_h',\n",
       "  'accurate_cross_std_a',\n",
       "  'accurate_cross_std_diff',\n",
       "  'accurate_cross_std_h',\n",
       "  'accurate_fwd_zone_pass_mean_a',\n",
       "  'accurate_fwd_zone_pass_mean_diff',\n",
       "  'accurate_fwd_zone_pass_mean_h',\n",
       "  'accurate_fwd_zone_pass_std_a',\n",
       "  'accurate_fwd_zone_pass_std_diff',\n",
       "  'accurate_fwd_zone_pass_std_h',\n",
       "  'accurate_goal_kicks_std_a',\n",
       "  'accurate_goal_kicks_std_diff',\n",
       "  'accurate_goal_kicks_std_h',\n",
       "  'accurate_keeper_sweeper_mean_a',\n",
       "  'accurate_keeper_sweeper_mean_diff',\n",
       "  'accurate_keeper_sweeper_mean_h',\n",
       "  'accurate_keeper_sweeper_std_a',\n",
       "  'accurate_keeper_sweeper_std_diff',\n",
       "  'accurate_keeper_sweeper_std_h',\n",
       "  'accurate_launches_mean_a',\n",
       "  'accurate_launches_mean_diff',\n",
       "  'accurate_launches_mean_h',\n",
       "  'accurate_layoffs_mean_a',\n",
       "  'accurate_layoffs_mean_diff',\n",
       "  'accurate_layoffs_mean_h',\n",
       "  'accurate_layoffs_std_a',\n",
       "  'accurate_layoffs_std_diff',\n",
       "  'accurate_layoffs_std_h',\n",
       "  'accurate_pass_std_a',\n",
       "  'accurate_pass_std_diff',\n",
       "  'accurate_pass_std_h',\n",
       "  'accurate_through_ball_mean_a',\n",
       "  'accurate_through_ball_mean_diff',\n",
       "  'accurate_through_ball_mean_h',\n",
       "  'accurate_throws_std_a',\n",
       "  'accurate_throws_std_diff',\n",
       "  'accurate_throws_std_h',\n",
       "  'att_assist_openplay_mean_a',\n",
       "  'att_assist_openplay_mean_diff',\n",
       "  'att_assist_openplay_mean_h',\n",
       "  'att_corner_ratio_mean_a',\n",
       "  'att_corner_ratio_mean_diff',\n",
       "  'att_corner_ratio_mean_h',\n",
       "  'att_freekick_goal_std_a',\n",
       "  'att_freekick_goal_std_diff',\n",
       "  'att_freekick_goal_std_h',\n",
       "  'att_freekick_miss_mean_a',\n",
       "  'att_freekick_miss_mean_diff',\n",
       "  'att_freekick_miss_mean_h',\n",
       "  'att_freekick_miss_std_a',\n",
       "  'att_freekick_miss_std_diff',\n",
       "  'att_freekick_miss_std_h',\n",
       "  'att_freekick_total_mean_a',\n",
       "  'att_freekick_total_mean_diff',\n",
       "  'att_freekick_total_mean_h',\n",
       "  'att_goal_high_centre_mean_a',\n",
       "  'att_goal_high_centre_mean_diff',\n",
       "  'att_goal_high_centre_mean_h',\n",
       "  'att_goal_high_centre_std_a',\n",
       "  'att_goal_high_centre_std_diff',\n",
       "  'att_goal_high_centre_std_h',\n",
       "  'att_goal_high_left_mean_a',\n",
       "  'att_goal_high_left_mean_diff',\n",
       "  'att_goal_high_left_mean_h',\n",
       "  'att_goal_high_left_std_a',\n",
       "  'att_goal_high_left_std_diff',\n",
       "  'att_goal_high_left_std_h',\n",
       "  'att_goal_high_right_std_a',\n",
       "  'att_goal_high_right_std_diff',\n",
       "  'att_goal_high_right_std_h',\n",
       "  'att_goal_low_centre_mean_a',\n",
       "  'att_goal_low_centre_mean_diff',\n",
       "  'att_goal_low_centre_mean_h',\n",
       "  'att_goal_low_centre_std_a',\n",
       "  'att_goal_low_centre_std_diff',\n",
       "  'att_goal_low_centre_std_h',\n",
       "  'att_goal_low_right_mean_a',\n",
       "  'att_goal_low_right_mean_diff',\n",
       "  'att_goal_low_right_mean_h',\n",
       "  'att_hd_goal_mean_a',\n",
       "  'att_hd_goal_mean_diff',\n",
       "  'att_hd_goal_mean_h',\n",
       "  'att_hd_target_mean_a',\n",
       "  'att_hd_target_mean_diff',\n",
       "  'att_hd_target_mean_h',\n",
       "  'att_ibox_own_goal_std_a',\n",
       "  'att_ibox_own_goal_std_diff',\n",
       "  'att_ibox_own_goal_std_h',\n",
       "  'att_lf_goal_mean_a',\n",
       "  'att_lf_goal_mean_diff',\n",
       "  'att_lf_goal_mean_h',\n",
       "  'att_lf_goal_std_a',\n",
       "  'att_lf_goal_std_diff',\n",
       "  'att_lf_goal_std_h',\n",
       "  'att_lg_centre_mean_a',\n",
       "  'att_lg_centre_mean_diff',\n",
       "  'att_lg_centre_mean_h',\n",
       "  'att_miss_high_left_std_a',\n",
       "  'att_miss_high_left_std_diff',\n",
       "  'att_miss_high_left_std_h',\n",
       "  'att_miss_right_std_a',\n",
       "  'att_miss_right_std_diff',\n",
       "  'att_miss_right_std_h',\n",
       "  'att_obox_blocked_mean_a',\n",
       "  'att_obox_blocked_mean_diff',\n",
       "  'att_obox_blocked_mean_h',\n",
       "  'att_obox_goal_mean_a',\n",
       "  'att_obox_goal_mean_diff',\n",
       "  'att_obox_goal_mean_h',\n",
       "  'att_obox_goal_std_a',\n",
       "  'att_obox_goal_std_diff',\n",
       "  'att_obox_goal_std_h',\n",
       "  'att_obox_miss_mean_a',\n",
       "  'att_obox_miss_mean_diff',\n",
       "  'att_obox_miss_mean_h',\n",
       "  'att_obox_post_mean_a',\n",
       "  'att_obox_post_mean_diff',\n",
       "  'att_obox_post_mean_h',\n",
       "  'att_obx_right_mean_a',\n",
       "  'att_obx_right_mean_diff',\n",
       "  'att_obx_right_mean_h',\n",
       "  'att_obxd_right_mean_a',\n",
       "  'att_obxd_right_mean_diff',\n",
       "  'att_obxd_right_mean_h',\n",
       "  'att_obxd_right_std_a',\n",
       "  'att_obxd_right_std_diff',\n",
       "  'att_obxd_right_std_h',\n",
       "  'att_openplay_mean_a',\n",
       "  'att_openplay_mean_diff',\n",
       "  'att_openplay_mean_h',\n",
       "  'att_pen_goal_mean_a',\n",
       "  'att_pen_goal_mean_diff',\n",
       "  'att_pen_goal_mean_h',\n",
       "  'att_post_left_mean_a',\n",
       "  'att_post_left_mean_diff',\n",
       "  'att_post_left_mean_h',\n",
       "  'att_post_left_std_a',\n",
       "  'att_post_left_std_diff',\n",
       "  'att_post_left_std_h',\n",
       "  'attempts_conceded_ibox_std_a',\n",
       "  'attempts_conceded_ibox_std_diff',\n",
       "  'attempts_conceded_ibox_std_h',\n",
       "  'attempts_conceded_obox_std_a',\n",
       "  'attempts_conceded_obox_std_diff',\n",
       "  'attempts_conceded_obox_std_h',\n",
       "  'attempts_obox_mean_a',\n",
       "  'attempts_obox_mean_diff',\n",
       "  'attempts_obox_mean_h',\n",
       "  'bc_miss_div_scored_std_a',\n",
       "  'bc_miss_div_scored_std_diff',\n",
       "  'bc_miss_div_scored_std_h',\n",
       "  'big_chance_created_mean_a',\n",
       "  'big_chance_created_mean_diff',\n",
       "  'big_chance_created_mean_h',\n",
       "  'big_chance_created_std_a',\n",
       "  'big_chance_created_std_diff',\n",
       "  'big_chance_created_std_h',\n",
       "  'big_chance_missed_std_a',\n",
       "  'big_chance_missed_std_diff',\n",
       "  'big_chance_missed_std_h',\n",
       "  'big_chance_scored_mean_a',\n",
       "  'big_chance_scored_mean_diff',\n",
       "  'big_chance_scored_mean_diff_mul_total_fwd_zone_pass_mean_diff',\n",
       "  'big_chance_scored_mean_h',\n",
       "  'blocked_cross_mean_a',\n",
       "  'blocked_cross_mean_diff',\n",
       "  'blocked_cross_mean_h',\n",
       "  'contentious_decision_mean_a',\n",
       "  'contentious_decision_mean_diff',\n",
       "  'contentious_decision_mean_h',\n",
       "  'contentious_decision_std_a',\n",
       "  'contentious_decision_std_diff',\n",
       "  'contentious_decision_std_h',\n",
       "  'corner_taken_std_mul_att_corner_ratio_mean_diffs',\n",
       "  'corners_ratio_mean_a',\n",
       "  'corners_ratio_mean_diff',\n",
       "  'corners_ratio_mean_h',\n",
       "  'defender_goals_mean_a',\n",
       "  'defender_goals_mean_diff',\n",
       "  'defender_goals_mean_h',\n",
       "  'diving_save_std_a',\n",
       "  'diving_save_std_diff',\n",
       "  'diving_save_std_h',\n",
       "  'effective_clearance_mean_a',\n",
       "  'effective_clearance_mean_diff',\n",
       "  'effective_clearance_mean_h',\n",
       "  'effective_head_clearance_mean_a',\n",
       "  'effective_head_clearance_mean_diff',\n",
       "  'effective_head_clearance_mean_h',\n",
       "  'first_yellow_card_std_a',\n",
       "  'first_yellow_card_std_diff',\n",
       "  'first_yellow_card_std_h',\n",
       "  'forward_goals_mean_a',\n",
       "  'forward_goals_mean_diff',\n",
       "  'forward_goals_mean_h',\n",
       "  'fwd_pass_div_long_pass_own_to_opp_success_std_a',\n",
       "  'fwd_pass_div_long_pass_own_to_opp_success_std_diff',\n",
       "  'fwd_pass_div_long_pass_own_to_opp_success_std_h',\n",
       "  'goal_assist_deadball_mean_a',\n",
       "  'goal_assist_deadball_mean_diff',\n",
       "  'goal_assist_deadball_mean_h',\n",
       "  'goal_assist_deadball_std_a',\n",
       "  'goal_assist_deadball_std_diff',\n",
       "  'goal_assist_deadball_std_h',\n",
       "  'goal_assist_intent_norm_mean_a',\n",
       "  'goal_assist_intent_norm_mean_diff',\n",
       "  'goal_assist_intent_norm_mean_h',\n",
       "  'goal_assist_intentional_mean_a',\n",
       "  'goal_assist_intentional_mean_diff',\n",
       "  'goal_assist_intentional_mean_h',\n",
       "  'goal_assist_intentional_std_a',\n",
       "  'goal_assist_intentional_std_diff',\n",
       "  'goal_assist_intentional_std_h',\n",
       "  'goal_assist_mean_a',\n",
       "  'goal_assist_mean_diff',\n",
       "  'goal_assist_mean_h',\n",
       "  'goal_assist_openplay_std_a',\n",
       "  'goal_assist_openplay_std_diff',\n",
       "  'goal_assist_openplay_std_h',\n",
       "  'goal_assist_setplay_mean_a',\n",
       "  'goal_assist_setplay_mean_diff',\n",
       "  'goal_assist_setplay_mean_h',\n",
       "  'goal_assist_std_a',\n",
       "  'goal_assist_std_diff',\n",
       "  'goal_assist_std_h',\n",
       "  'goal_fastbreak_mean_a',\n",
       "  'goal_fastbreak_mean_diff',\n",
       "  'goal_fastbreak_mean_h',\n",
       "  'goal_kicks_per_shot_mean_a',\n",
       "  'goal_kicks_per_shot_mean_diff',\n",
       "  'goal_kicks_per_shot_mean_h',\n",
       "  'goals_2t_mean_a',\n",
       "  'goals_2t_mean_diff',\n",
       "  'goals_2t_mean_h',\n",
       "  'goals_mean_a',\n",
       "  'goals_mean_diff',\n",
       "  'goals_mean_h',\n",
       "  'goals_openplay_mean_a',\n",
       "  'goals_openplay_mean_diff',\n",
       "  'goals_openplay_mean_h',\n",
       "  'high_to_low_goals_mean_a',\n",
       "  'high_to_low_goals_mean_diff',\n",
       "  'high_to_low_goals_mean_h',\n",
       "  'imp_prob_over_goals_0.5_a',\n",
       "  'imp_prob_over_goals_0.5_diff',\n",
       "  'imp_prob_over_goals_0.5_h',\n",
       "  'imp_prob_over_goals_1.5_a',\n",
       "  'imp_prob_over_goals_1.5_diff',\n",
       "  'imp_prob_over_goals_1.5_h',\n",
       "  'imp_prob_over_goals_2.5_a',\n",
       "  'imp_prob_over_goals_2.5_diff',\n",
       "  'imp_prob_over_goals_2.5_h',\n",
       "  'imp_prob_over_goals_3.5_a',\n",
       "  'imp_prob_over_goals_3.5_diff',\n",
       "  'imp_prob_over_goals_3.5_h',\n",
       "  'imp_prob_over_goals_4.5_a',\n",
       "  'imp_prob_over_goals_4.5_diff',\n",
       "  'imp_prob_over_goals_4.5_h',\n",
       "  'imp_prob_under_goals_0.5_a',\n",
       "  'imp_prob_under_goals_0.5_diff',\n",
       "  'imp_prob_under_goals_0.5_h',\n",
       "  'imp_prob_under_goals_1.5_a',\n",
       "  'imp_prob_under_goals_1.5_diff',\n",
       "  'imp_prob_under_goals_1.5_h',\n",
       "  'imp_prob_under_goals_2.5_a',\n",
       "  'imp_prob_under_goals_2.5_diff',\n",
       "  'imp_prob_under_goals_2.5_h',\n",
       "  'imp_prob_under_goals_3.5_a',\n",
       "  'imp_prob_under_goals_3.5_diff',\n",
       "  'imp_prob_under_goals_3.5_h',\n",
       "  'imp_prob_under_goals_4.5_a',\n",
       "  'imp_prob_under_goals_4.5_diff',\n",
       "  'imp_prob_under_goals_4.5_h',\n",
       "  'interceptions_in_box_std_a',\n",
       "  'interceptions_in_box_std_diff',\n",
       "  'interceptions_in_box_std_h',\n",
       "  'last_man_tackle_mean_a',\n",
       "  'last_man_tackle_mean_diff',\n",
       "  'last_man_tackle_mean_h',\n",
       "  'last_man_tackle_std_a',\n",
       "  'last_man_tackle_std_diff',\n",
       "  'last_man_tackle_std_h',\n",
       "  'left_div_right_foot_goals_mean_a',\n",
       "  'left_div_right_foot_goals_mean_diff',\n",
       "  'left_div_right_foot_goals_mean_h',\n",
       "  'leftside_pass_div_accurate_pass_std_a',\n",
       "  'leftside_pass_div_accurate_pass_std_diff',\n",
       "  'leftside_pass_div_accurate_pass_std_h',\n",
       "  'leftside_pass_mean_a',\n",
       "  'leftside_pass_mean_diff',\n",
       "  'leftside_pass_mean_h',\n",
       "  'long_pass_own_to_opp_success_std_a',\n",
       "  'long_pass_own_to_opp_success_std_diff',\n",
       "  'long_pass_own_to_opp_success_std_h',\n",
       "  'lost_corners_mean_a',\n",
       "  'lost_corners_mean_diff',\n",
       "  'lost_corners_mean_h',\n",
       "  'midfielder_goals_mean_a',\n",
       "  'midfielder_goals_mean_diff',\n",
       "  'midfielder_goals_mean_h',\n",
       "  'no_foot_goals_ratio_mean_a',\n",
       "  'no_foot_goals_ratio_mean_diff',\n",
       "  'no_foot_goals_ratio_mean_h',\n",
       "  'no_foot_goals_ratio_std_a',\n",
       "  'no_foot_goals_ratio_std_diff',\n",
       "  'no_foot_goals_ratio_std_h',\n",
       "  'odd_ratio_over_corners_11.5',\n",
       "  'odd_ratio_over_corners_9.5',\n",
       "  'odd_ratio_under_corners_10.5',\n",
       "  'odd_ratio_under_goals_1.5',\n",
       "  'odd_ratio_under_goals_2.5',\n",
       "  'odd_ratio_under_goals_3.5',\n",
       "  'odd_ratio_under_goals_4.5',\n",
       "  'odd_ratio_under_goals_4.5',\n",
       "  'odds_away_over_cards_4.5_a',\n",
       "  'odds_away_over_cards_4.5_diff',\n",
       "  'odds_away_over_cards_4.5_h',\n",
       "  'odds_away_over_cards_5.5_a',\n",
       "  'odds_away_over_cards_5.5_diff',\n",
       "  'odds_away_over_cards_5.5_h',\n",
       "  'odds_away_under_cards_3.5_a',\n",
       "  'odds_away_under_cards_3.5_diff',\n",
       "  'odds_away_under_cards_3.5_h',\n",
       "  'odds_home_under_cards_5.5_a',\n",
       "  'odds_home_under_cards_5.5_diff',\n",
       "  'odds_home_under_cards_5.5_h',\n",
       "  'odds_home_under_goals_0.5_a',\n",
       "  'odds_home_under_goals_0.5_diff',\n",
       "  'odds_home_under_goals_0.5_h',\n",
       "  'ontarget_att_assist_mean_a',\n",
       "  'ontarget_att_assist_mean_diff',\n",
       "  'ontarget_att_assist_mean_h',\n",
       "  'own_goals_mean_a',\n",
       "  'own_goals_mean_diff',\n",
       "  'own_goals_mean_h',\n",
       "  'own_goals_std_a',\n",
       "  'own_goals_std_diff',\n",
       "  'own_goals_std_h',\n",
       "  'passes_left_mean_a',\n",
       "  'passes_left_mean_diff',\n",
       "  'passes_left_mean_h',\n",
       "  'passes_right_mean_a',\n",
       "  'passes_right_mean_diff',\n",
       "  'passes_right_mean_h',\n",
       "  'passes_right_std_a',\n",
       "  'passes_right_std_diff',\n",
       "  'passes_right_std_h',\n",
       "  'pen_goals_conceded_std_a',\n",
       "  'pen_goals_conceded_std_diff',\n",
       "  'pen_goals_conceded_std_h',\n",
       "  'penalty_conceded_mean_a',\n",
       "  'penalty_conceded_mean_diff',\n",
       "  'penalty_conceded_mean_h',\n",
       "  'penalty_faced_std_a',\n",
       "  'penalty_faced_std_diff',\n",
       "  'penalty_faced_std_h',\n",
       "  'penalty_save_std_a',\n",
       "  'penalty_save_std_diff',\n",
       "  'penalty_save_std_h',\n",
       "  'points_a',\n",
       "  'points_diff',\n",
       "  'points_h',\n",
       "  'points_mean_a',\n",
       "  'points_mean_diff',\n",
       "  'points_mean_h',\n",
       "  'points_std_a',\n",
       "  'points_std_diff',\n",
       "  'points_std_h',\n",
       "  'poss_won_att_3rd_std_a',\n",
       "  'poss_won_att_3rd_std_diff',\n",
       "  'poss_won_att_3rd_std_h',\n",
       "  'prob_over_goals_1.5',\n",
       "  'prob_over_goals_2.5',\n",
       "  'prob_over_goals_3.5',\n",
       "  'prob_over_goals_4.5',\n",
       "  'prob_squared_over_goals_1.5',\n",
       "  'prob_squared_over_goals_2.5',\n",
       "  'prob_squared_over_goals_3.5',\n",
       "  'prob_squared_over_goals_4.5',\n",
       "  'prob_squared_under_goals_1.5',\n",
       "  'prob_squared_under_goals_2.5',\n",
       "  'prob_squared_under_goals_3.5',\n",
       "  'prob_squared_under_goals_4.5',\n",
       "  'prob_under_goals_1.5',\n",
       "  'prob_under_goals_2.5',\n",
       "  'prob_under_goals_3.5',\n",
       "  'prob_under_goals_4.5',\n",
       "  'ratio_over_corners_11.5_a',\n",
       "  'ratio_over_corners_11.5_diff',\n",
       "  'ratio_over_corners_11.5_h',\n",
       "  'ratio_over_corners_9.5_a',\n",
       "  'ratio_over_corners_9.5_diff',\n",
       "  'ratio_over_corners_9.5_h',\n",
       "  'ratio_under_corners_10.5_a',\n",
       "  'ratio_under_corners_10.5_diff',\n",
       "  'ratio_under_corners_10.5_h',\n",
       "  'ratio_under_goals_4.5_a',\n",
       "  'ratio_under_goals_4.5_diff',\n",
       "  'ratio_under_goals_4.5_h',\n",
       "  'raw_prob_over_goals_1.5',\n",
       "  'raw_prob_over_goals_1.5',\n",
       "  'raw_prob_over_goals_2.5',\n",
       "  'raw_prob_over_goals_2.5',\n",
       "  'raw_prob_over_goals_3.5',\n",
       "  'raw_prob_over_goals_3.5',\n",
       "  'raw_prob_over_goals_4.5',\n",
       "  'raw_prob_over_goals_4.5',\n",
       "  'raw_prob_under_goals_1.5',\n",
       "  'raw_prob_under_goals_2.5',\n",
       "  'raw_prob_under_goals_3.5',\n",
       "  'raw_prob_under_goals_4.5',\n",
       "  'red_card_1t_std_a',\n",
       "  'red_card_1t_std_diff',\n",
       "  'red_card_1t_std_h',\n",
       "  'red_card_2t_mean_a',\n",
       "  'red_card_2t_mean_diff',\n",
       "  'red_card_2t_mean_h',\n",
       "  'red_card_mean_a',\n",
       "  'red_card_mean_diff',\n",
       "  'red_card_mean_h',\n",
       "  'red_card_std_a',\n",
       "  'red_card_std_diff',\n",
       "  'red_card_std_h',\n",
       "  'rescinded_red_card_mean_a',\n",
       "  'rescinded_red_card_mean_diff',\n",
       "  'rescinded_red_card_mean_h',\n",
       "  'rescinded_red_card_std_a',\n",
       "  'rescinded_red_card_std_diff',\n",
       "  'rescinded_red_card_std_h',\n",
       "  'right_to_left_goals_mean_a',\n",
       "  'right_to_left_goals_mean_a',\n",
       "  'right_to_left_goals_mean_diff',\n",
       "  'right_to_left_goals_mean_diff',\n",
       "  'right_to_left_goals_mean_h',\n",
       "  'right_to_left_goals_mean_h',\n",
       "  'right_to_left_goals_std_a',\n",
       "  'right_to_left_goals_std_diff',\n",
       "  'right_to_left_goals_std_h',\n",
       "  'rightside_pass_div_accurate_pass_std_a',\n",
       "  'rightside_pass_div_accurate_pass_std_diff',\n",
       "  'rightside_pass_div_accurate_pass_std_h',\n",
       "  'rightside_pass_div_leftside_pass_mean_a',\n",
       "  'rightside_pass_div_leftside_pass_mean_diff',\n",
       "  'rightside_pass_div_leftside_pass_mean_h',\n",
       "  'saves_std_a',\n",
       "  'saves_std_diff',\n",
       "  'saves_std_h',\n",
       "  'second_yellow_mean_a',\n",
       "  'second_yellow_mean_diff',\n",
       "  'second_yellow_mean_h',\n",
       "  'second_yellow_std_a',\n",
       "  'second_yellow_std_diff',\n",
       "  'second_yellow_std_h',\n",
       "  'shots_div_passes_left_mean_a',\n",
       "  'shots_div_passes_left_mean_diff',\n",
       "  'shots_div_passes_left_mean_h',\n",
       "  'shots_div_passes_right_mean_a',\n",
       "  'shots_div_passes_right_mean_diff',\n",
       "  'shots_div_passes_right_mean_h',\n",
       "  'shots_div_passes_right_std_a',\n",
       "  'shots_div_passes_right_std_diff',\n",
       "  'shots_div_passes_right_std_h',\n",
       "  'shots_mul_goals_mean_a',\n",
       "  'shots_mul_goals_mean_diff',\n",
       "  'shots_mul_goals_mean_h',\n",
       "  'shots_std_a',\n",
       "  'shots_std_diff',\n",
       "  'shots_std_h',\n",
       "  'successful_final_third_passes_std_a',\n",
       "  'successful_final_third_passes_std_diff',\n",
       "  'successful_final_third_passes_std_h',\n",
       "  'total_bets_under_cards_4.5_a',\n",
       "  'total_bets_under_cards_4.5_diff',\n",
       "  'total_bets_under_cards_4.5_h',\n",
       "  'total_bets_under_corners_9.5_a',\n",
       "  'total_bets_under_corners_9.5_diff',\n",
       "  'total_bets_under_corners_9.5_h',\n",
       "  'total_clearance_mean_a',\n",
       "  'total_clearance_mean_diff',\n",
       "  'total_clearance_mean_h',\n",
       "  'total_expulsions_ref',\n",
       "  'total_fastbreak_std_a',\n",
       "  'total_fastbreak_std_diff',\n",
       "  'total_fastbreak_std_h',\n",
       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_a',\n",
       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_diff',\n",
       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_h',\n",
       "  'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff',\n",
       "  'total_fwd_zone_pass_std_a',\n",
       "  'total_fwd_zone_pass_std_diff',\n",
       "  'total_fwd_zone_pass_std_h',\n",
       "  'total_high_claim_mean_a',\n",
       "  'total_high_claim_mean_diff',\n",
       "  'total_high_claim_mean_h',\n",
       "  'total_keeper_sweeper_mean_a',\n",
       "  'total_keeper_sweeper_mean_diff',\n",
       "  'total_keeper_sweeper_mean_h',\n",
       "  'total_keeper_sweeper_std_a',\n",
       "  'total_keeper_sweeper_std_diff',\n",
       "  'total_keeper_sweeper_std_h',\n",
       "  'total_launches_mean_a',\n",
       "  'total_launches_mean_diff',\n",
       "  'total_launches_mean_h',\n",
       "  'total_launches_std_a',\n",
       "  'total_launches_std_diff',\n",
       "  'total_launches_std_h',\n",
       "  'total_layoffs_mean_a',\n",
       "  'total_layoffs_mean_diff',\n",
       "  'total_layoffs_mean_h',\n",
       "  'total_layoffs_std_a',\n",
       "  'total_layoffs_std_diff',\n",
       "  'total_layoffs_std_h',\n",
       "  'total_red_card_match_ref',\n",
       "  'total_red_card_mean_a',\n",
       "  'total_red_card_mean_diff',\n",
       "  'total_red_card_mean_h',\n",
       "  'total_red_card_std_a',\n",
       "  'total_red_card_std_diff',\n",
       "  'total_red_card_std_h',\n",
       "  'total_second_yel_card_away_ref',\n",
       "  'total_second_yel_card_home_ref',\n",
       "  'total_second_yel_card_ref',\n",
       "  'total_through_ball_mean_a',\n",
       "  'total_through_ball_mean_diff',\n",
       "  'total_through_ball_mean_h',\n",
       "  'total_through_ball_std_a',\n",
       "  'total_through_ball_std_diff',\n",
       "  'total_through_ball_std_h',\n",
       "  'total_throws_std_a',\n",
       "  'total_throws_std_diff',\n",
       "  'total_throws_std_h',\n",
       "  'total_win_pct_over_corners_9.5_a',\n",
       "  'total_win_pct_over_corners_9.5_diff',\n",
       "  'total_win_pct_over_corners_9.5_h',\n",
       "  'total_win_pct_over_goals_2.5_a',\n",
       "  'total_win_pct_over_goals_2.5_diff',\n",
       "  'total_win_pct_over_goals_2.5_h',\n",
       "  'total_win_pct_under_goals_1.5_a',\n",
       "  'total_win_pct_under_goals_1.5_diff',\n",
       "  'total_win_pct_under_goals_1.5_h',\n",
       "  'total_win_pct_under_goals_3.5_a',\n",
       "  'total_win_pct_under_goals_3.5_diff',\n",
       "  'total_win_pct_under_goals_3.5_h',\n",
       "  'total_yel_card_std_a',\n",
       "  'total_yel_card_std_diff',\n",
       "  'total_yel_card_std_h',\n",
       "  'win_pct_away_over_goals_2.5_a',\n",
       "  'win_pct_away_over_goals_2.5_diff',\n",
       "  'win_pct_away_over_goals_2.5_h',\n",
       "  'win_pct_home_over_cards_6.5_a',\n",
       "  'win_pct_home_over_cards_6.5_diff',\n",
       "  'win_pct_home_over_cards_6.5_h',\n",
       "  'win_pct_home_over_goals_2.5_a',\n",
       "  'win_pct_home_over_goals_2.5_diff',\n",
       "  'win_pct_home_over_goals_2.5_h',\n",
       "  'win_pct_home_under_cards_4.5_a',\n",
       "  'win_pct_home_under_cards_4.5_diff',\n",
       "  'win_pct_home_under_cards_4.5_h',\n",
       "  'won_corners_ratio_mean_a',\n",
       "  'won_corners_ratio_mean_a',\n",
       "  'won_corners_ratio_mean_diff',\n",
       "  'won_corners_ratio_mean_diff',\n",
       "  'won_corners_ratio_mean_h',\n",
       "  'won_corners_ratio_mean_h',\n",
       "  'won_corners_ratio_std_a',\n",
       "  'won_corners_ratio_std_diff',\n",
       "  'won_corners_ratio_std_h'])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
    "\n",
    "len(selected_features), len(new_subset), sorted(new_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5199</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5415</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5514</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5271</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5390</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0388</td>\n",
       "                        <td id=\"T_527aac44_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0388</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb80d69f0d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5227</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5462</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5162</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5320</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5240</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0456</td>\n",
       "                        <td id=\"T_60334a94_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0456</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb820814340>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metric_param = {\n",
    "        \"Accuracy\": -0.1,\n",
    "        \"AUC\": -0.1,\n",
    "        \"Recall\": -0.1,\n",
    "        \"Precision\": -0.1,\n",
    "        \"F1\": -0.1,\n",
    "        \"Kappa\": -1.0,\n",
    "        \"MCC\": -1.0,\n",
    "    }\n",
    "feat_sel = FeatureSelection(target=target,\n",
    "                            dataset=train_data.dropna()[list(set(new_subset+[target]))],\n",
    "                            target_features=250,\n",
    "                            filter_metrics=metric_param,\n",
    "                            include=[\"lr\"],\n",
    "                            setup_kwargs=setup_kwargs,\n",
    "                            optimize=True,\n",
    "                            opt_list=[\"AUC\"],\n",
    "                           )\n",
    "selected_features = feat_sel.repeat_pipeline()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145,\n",
       " 276,\n",
       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
       "  'ATT_h_mul_shots_mul_goals_mean_diff',\n",
       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
       "  'accurate_cross_nocorner_std_a',\n",
       "  'accurate_cross_nocorner_std_diff',\n",
       "  'accurate_cross_nocorner_std_h',\n",
       "  'accurate_cross_std_a',\n",
       "  'accurate_cross_std_diff',\n",
       "  'accurate_cross_std_h',\n",
       "  'accurate_goal_kicks_std_a',\n",
       "  'accurate_goal_kicks_std_diff',\n",
       "  'accurate_goal_kicks_std_h',\n",
       "  'accurate_keeper_sweeper_mean_a',\n",
       "  'accurate_keeper_sweeper_mean_diff',\n",
       "  'accurate_keeper_sweeper_mean_h',\n",
       "  'accurate_keeper_sweeper_std_a',\n",
       "  'accurate_keeper_sweeper_std_diff',\n",
       "  'accurate_keeper_sweeper_std_h',\n",
       "  'accurate_launches_mean_a',\n",
       "  'accurate_launches_mean_diff',\n",
       "  'accurate_launches_mean_h',\n",
       "  'accurate_layoffs_mean_a',\n",
       "  'accurate_layoffs_mean_diff',\n",
       "  'accurate_layoffs_mean_h',\n",
       "  'accurate_layoffs_std_a',\n",
       "  'accurate_layoffs_std_diff',\n",
       "  'accurate_layoffs_std_h',\n",
       "  'accurate_through_ball_mean_a',\n",
       "  'accurate_through_ball_mean_diff',\n",
       "  'accurate_through_ball_mean_h',\n",
       "  'accurate_throws_std_a',\n",
       "  'accurate_throws_std_diff',\n",
       "  'accurate_throws_std_h',\n",
       "  'att_freekick_miss_mean_a',\n",
       "  'att_freekick_miss_mean_diff',\n",
       "  'att_freekick_miss_mean_h',\n",
       "  'att_freekick_miss_std_a',\n",
       "  'att_freekick_miss_std_diff',\n",
       "  'att_freekick_miss_std_h',\n",
       "  'att_goal_low_centre_mean_a',\n",
       "  'att_goal_low_centre_mean_diff',\n",
       "  'att_goal_low_centre_mean_h',\n",
       "  'att_goal_low_centre_std_a',\n",
       "  'att_goal_low_centre_std_diff',\n",
       "  'att_goal_low_centre_std_h',\n",
       "  'att_goal_low_right_mean_a',\n",
       "  'att_goal_low_right_mean_diff',\n",
       "  'att_goal_low_right_mean_h',\n",
       "  'att_hd_goal_mean_a',\n",
       "  'att_hd_goal_mean_diff',\n",
       "  'att_hd_goal_mean_h',\n",
       "  'att_hd_target_mean_a',\n",
       "  'att_hd_target_mean_diff',\n",
       "  'att_hd_target_mean_h',\n",
       "  'att_lf_goal_mean_a',\n",
       "  'att_lf_goal_mean_diff',\n",
       "  'att_lf_goal_mean_h',\n",
       "  'att_lf_goal_std_a',\n",
       "  'att_lf_goal_std_diff',\n",
       "  'att_lf_goal_std_h',\n",
       "  'att_miss_high_left_std_a',\n",
       "  'att_miss_high_left_std_diff',\n",
       "  'att_miss_high_left_std_h',\n",
       "  'att_obox_blocked_mean_a',\n",
       "  'att_obox_blocked_mean_diff',\n",
       "  'att_obox_blocked_mean_h',\n",
       "  'att_pen_goal_mean_a',\n",
       "  'att_pen_goal_mean_diff',\n",
       "  'att_pen_goal_mean_h',\n",
       "  'attempts_conceded_ibox_std_a',\n",
       "  'attempts_conceded_ibox_std_diff',\n",
       "  'attempts_conceded_ibox_std_h',\n",
       "  'attempts_conceded_obox_std_a',\n",
       "  'attempts_conceded_obox_std_diff',\n",
       "  'attempts_conceded_obox_std_h',\n",
       "  'attempts_obox_mean_a',\n",
       "  'attempts_obox_mean_diff',\n",
       "  'attempts_obox_mean_h',\n",
       "  'big_chance_created_mean_a',\n",
       "  'big_chance_created_mean_diff',\n",
       "  'big_chance_created_mean_h',\n",
       "  'big_chance_created_std_a',\n",
       "  'big_chance_created_std_diff',\n",
       "  'big_chance_created_std_h',\n",
       "  'big_chance_missed_std_a',\n",
       "  'big_chance_missed_std_diff',\n",
       "  'big_chance_missed_std_h',\n",
       "  'blocked_cross_mean_a',\n",
       "  'blocked_cross_mean_diff',\n",
       "  'blocked_cross_mean_h',\n",
       "  'contentious_decision_mean_a',\n",
       "  'contentious_decision_mean_diff',\n",
       "  'contentious_decision_mean_h',\n",
       "  'contentious_decision_std_a',\n",
       "  'contentious_decision_std_diff',\n",
       "  'contentious_decision_std_h',\n",
       "  'defender_goals_mean_a',\n",
       "  'defender_goals_mean_a',\n",
       "  'defender_goals_mean_diff',\n",
       "  'defender_goals_mean_diff',\n",
       "  'defender_goals_mean_h',\n",
       "  'defender_goals_mean_h',\n",
       "  'diving_save_std_a',\n",
       "  'diving_save_std_diff',\n",
       "  'diving_save_std_h',\n",
       "  'effective_head_clearance_mean_a',\n",
       "  'effective_head_clearance_mean_diff',\n",
       "  'effective_head_clearance_mean_h',\n",
       "  'forward_goals_mean_a',\n",
       "  'forward_goals_mean_diff',\n",
       "  'forward_goals_mean_h',\n",
       "  'goal_assist_deadball_std_a',\n",
       "  'goal_assist_deadball_std_diff',\n",
       "  'goal_assist_deadball_std_h',\n",
       "  'goal_assist_intent_norm_mean_a',\n",
       "  'goal_assist_intent_norm_mean_diff',\n",
       "  'goal_assist_intent_norm_mean_h',\n",
       "  'goal_assist_intentional_mean_a',\n",
       "  'goal_assist_intentional_mean_diff',\n",
       "  'goal_assist_intentional_mean_h',\n",
       "  'goal_assist_intentional_std_a',\n",
       "  'goal_assist_intentional_std_diff',\n",
       "  'goal_assist_intentional_std_h',\n",
       "  'goal_assist_mean_a',\n",
       "  'goal_assist_mean_diff',\n",
       "  'goal_assist_mean_h',\n",
       "  'goal_assist_openplay_std_a',\n",
       "  'goal_assist_openplay_std_diff',\n",
       "  'goal_assist_openplay_std_h',\n",
       "  'goal_assist_setplay_mean_a',\n",
       "  'goal_assist_setplay_mean_diff',\n",
       "  'goal_assist_setplay_mean_h',\n",
       "  'goals_2t_mean_a',\n",
       "  'goals_2t_mean_diff',\n",
       "  'goals_2t_mean_h',\n",
       "  'goals_mean_a',\n",
       "  'goals_mean_diff',\n",
       "  'goals_mean_h',\n",
       "  'goals_openplay_mean_a',\n",
       "  'goals_openplay_mean_diff',\n",
       "  'goals_openplay_mean_h',\n",
       "  'high_to_low_goals_mean_a',\n",
       "  'high_to_low_goals_mean_diff',\n",
       "  'high_to_low_goals_mean_h',\n",
       "  'interceptions_in_box_std_a',\n",
       "  'interceptions_in_box_std_diff',\n",
       "  'interceptions_in_box_std_h',\n",
       "  'left_div_right_foot_goals_mean_a',\n",
       "  'left_div_right_foot_goals_mean_a',\n",
       "  'left_div_right_foot_goals_mean_diff',\n",
       "  'left_div_right_foot_goals_mean_diff',\n",
       "  'left_div_right_foot_goals_mean_h',\n",
       "  'left_div_right_foot_goals_mean_h',\n",
       "  'lost_corners_mean_a',\n",
       "  'lost_corners_mean_diff',\n",
       "  'lost_corners_mean_h',\n",
       "  'midfielder_goals_mean_a',\n",
       "  'midfielder_goals_mean_diff',\n",
       "  'midfielder_goals_mean_h',\n",
       "  'no_foot_goals_ratio_mean_a',\n",
       "  'no_foot_goals_ratio_mean_diff',\n",
       "  'no_foot_goals_ratio_mean_h',\n",
       "  'no_foot_goals_ratio_std_a',\n",
       "  'no_foot_goals_ratio_std_diff',\n",
       "  'no_foot_goals_ratio_std_h',\n",
       "  'odd_ratio_over_corners_11.5',\n",
       "  'odd_ratio_under_corners_10.5',\n",
       "  'odds_away_over_cards_5.5_a',\n",
       "  'odds_away_over_cards_5.5_diff',\n",
       "  'odds_away_over_cards_5.5_h',\n",
       "  'odds_home_under_cards_5.5_a',\n",
       "  'odds_home_under_cards_5.5_diff',\n",
       "  'odds_home_under_cards_5.5_h',\n",
       "  'odds_home_under_goals_0.5_a',\n",
       "  'odds_home_under_goals_0.5_diff',\n",
       "  'odds_home_under_goals_0.5_h',\n",
       "  'ontarget_att_assist_mean_a',\n",
       "  'ontarget_att_assist_mean_diff',\n",
       "  'ontarget_att_assist_mean_h',\n",
       "  'own_goals_mean_a',\n",
       "  'own_goals_mean_diff',\n",
       "  'own_goals_mean_h',\n",
       "  'penalty_conceded_mean_a',\n",
       "  'penalty_conceded_mean_diff',\n",
       "  'penalty_conceded_mean_h',\n",
       "  'penalty_faced_std_a',\n",
       "  'penalty_faced_std_diff',\n",
       "  'penalty_faced_std_h',\n",
       "  'penalty_save_std_a',\n",
       "  'penalty_save_std_diff',\n",
       "  'penalty_save_std_h',\n",
       "  'poss_won_att_3rd_std_a',\n",
       "  'poss_won_att_3rd_std_diff',\n",
       "  'poss_won_att_3rd_std_h',\n",
       "  'ratio_over_corners_11.5_a',\n",
       "  'ratio_over_corners_11.5_diff',\n",
       "  'ratio_over_corners_11.5_h',\n",
       "  'ratio_under_corners_10.5_a',\n",
       "  'ratio_under_corners_10.5_diff',\n",
       "  'ratio_under_corners_10.5_h',\n",
       "  'red_card_std_a',\n",
       "  'red_card_std_diff',\n",
       "  'red_card_std_h',\n",
       "  'rescinded_red_card_std_a',\n",
       "  'rescinded_red_card_std_diff',\n",
       "  'rescinded_red_card_std_h',\n",
       "  'right_to_left_goals_mean_a',\n",
       "  'right_to_left_goals_mean_diff',\n",
       "  'right_to_left_goals_mean_h',\n",
       "  'saves_std_a',\n",
       "  'saves_std_diff',\n",
       "  'saves_std_h',\n",
       "  'shots_mul_goals_mean_a',\n",
       "  'shots_mul_goals_mean_a',\n",
       "  'shots_mul_goals_mean_diff',\n",
       "  'shots_mul_goals_mean_diff',\n",
       "  'shots_mul_goals_mean_h',\n",
       "  'shots_mul_goals_mean_h',\n",
       "  'shots_std_a',\n",
       "  'shots_std_diff',\n",
       "  'shots_std_h',\n",
       "  'total_fastbreak_std_a',\n",
       "  'total_fastbreak_std_diff',\n",
       "  'total_fastbreak_std_h',\n",
       "  'total_keeper_sweeper_mean_a',\n",
       "  'total_keeper_sweeper_mean_diff',\n",
       "  'total_keeper_sweeper_mean_h',\n",
       "  'total_keeper_sweeper_std_a',\n",
       "  'total_keeper_sweeper_std_diff',\n",
       "  'total_keeper_sweeper_std_h',\n",
       "  'total_launches_mean_a',\n",
       "  'total_launches_mean_diff',\n",
       "  'total_launches_mean_h',\n",
       "  'total_launches_std_a',\n",
       "  'total_launches_std_diff',\n",
       "  'total_launches_std_h',\n",
       "  'total_layoffs_std_a',\n",
       "  'total_layoffs_std_diff',\n",
       "  'total_layoffs_std_h',\n",
       "  'total_red_card_std_a',\n",
       "  'total_red_card_std_diff',\n",
       "  'total_red_card_std_h',\n",
       "  'total_through_ball_mean_a',\n",
       "  'total_through_ball_mean_diff',\n",
       "  'total_through_ball_mean_h',\n",
       "  'total_through_ball_std_a',\n",
       "  'total_through_ball_std_diff',\n",
       "  'total_through_ball_std_h',\n",
       "  'total_throws_std_a',\n",
       "  'total_throws_std_diff',\n",
       "  'total_throws_std_h',\n",
       "  'total_win_pct_over_goals_2.5_a',\n",
       "  'total_win_pct_over_goals_2.5_diff',\n",
       "  'total_win_pct_over_goals_2.5_h',\n",
       "  'total_win_pct_under_goals_1.5_a',\n",
       "  'total_win_pct_under_goals_1.5_diff',\n",
       "  'total_win_pct_under_goals_1.5_h',\n",
       "  'total_win_pct_under_goals_3.5_a',\n",
       "  'total_win_pct_under_goals_3.5_diff',\n",
       "  'total_win_pct_under_goals_3.5_h',\n",
       "  'win_pct_away_over_goals_2.5_a',\n",
       "  'win_pct_away_over_goals_2.5_diff',\n",
       "  'win_pct_away_over_goals_2.5_h',\n",
       "  'win_pct_home_over_cards_6.5_a',\n",
       "  'win_pct_home_over_cards_6.5_diff',\n",
       "  'win_pct_home_over_cards_6.5_h',\n",
       "  'win_pct_home_over_goals_2.5_a',\n",
       "  'win_pct_home_over_goals_2.5_diff',\n",
       "  'win_pct_home_over_goals_2.5_h',\n",
       "  'win_pct_home_under_cards_4.5_a',\n",
       "  'win_pct_home_under_cards_4.5_diff',\n",
       "  'win_pct_home_under_cards_4.5_h',\n",
       "  'won_corners_ratio_std_a',\n",
       "  'won_corners_ratio_std_diff',\n",
       "  'won_corners_ratio_std_h'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
    "\n",
    "len(selected_features), len(new_subset), sorted(new_subset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### new imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "from ml_bets.constants import FEATURES_PATH\n",
    "from ml_bets.features.features import Features\n",
    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
    "#from featsel.feature_selection import FeatureSelection\n",
    "\n",
    "from pycaret.utils import check_metric\n",
    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
    "    create_model,\n",
    "    finalize_model,\n",
    "    optimize_threshold,              \n",
    "    save_model,\n",
    "    compare_models, \n",
    "    evaluate_model,\n",
    "    get_config,\n",
    "    setup,\n",
    "    tune_model,\n",
    "    predict_model,\n",
    ")\n",
    "\n",
    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
    "from ml_bets.research.datasets import create_dataset\n",
    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
    "index = feats.matches[~ix].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_feats = ['acc_cross_nocorner_pct_std_a',\n",
    " 'acc_cross_nocorner_pct_std_diff',\n",
    " 'acc_cross_nocorner_pct_std_h',\n",
    " 'accurate_flick_on_mean_h',\n",
    " 'accurate_flick_on_mean_a',\n",
    " 'accurate_freekick_cross_mean_a',\n",
    " 'accurate_freekick_cross_mean_diff',\n",
    " 'accurate_freekick_cross_mean_h',\n",
    " 'accurate_goal_kicks_mean_diff',\n",
    " 'accurate_goal_kicks_std_a',\n",
    " 'accurate_goal_kicks_std_diff',\n",
    " 'accurate_goal_kicks_std_h',\n",
    " 'accurate_keeper_sweeper_mean_a',\n",
    " 'accurate_keeper_sweeper_std_diff',\n",
    " 'accurate_keeper_sweeper_std_h',\n",
    " 'accurate_pass_std_a',\n",
    " 'accurate_pass_std_h',\n",
    " 'accurate_through_ball_mean_diff',\n",
    " 'att_assist_openplay_std_a',\n",
    " 'att_assist_openplay_std_h',\n",
    " 'att_cmiss_left_mean_a',\n",
    " 'att_cmiss_left_mean_h',\n",
    " 'att_cmiss_left_std_a',\n",
    " 'att_cmiss_left_std_diff',\n",
    " 'att_cmiss_left_std_h',\n",
    " 'att_goal_high_centre_mean_diff',\n",
    " 'att_goal_high_centre_std_diff',\n",
    " 'att_goal_high_right_std_a',\n",
    " 'att_goal_low_centre_mean_diff',\n",
    " 'att_goal_low_centre_mean_h',\n",
    " 'att_goal_low_centre_std_a',\n",
    " 'att_goal_low_centre_std_h',\n",
    " 'att_hd_goal_mean_a',\n",
    " 'att_hd_goal_mean_h',\n",
    " 'att_hd_target_mean_a',\n",
    " 'att_hd_target_mean_h',\n",
    " 'att_ibox_own_goal_mean_diff',\n",
    " 'att_ibox_own_goal_std_h',\n",
    " 'att_ibox_own_goal_std_a',\n",
    " 'att_miss_high_mean_a',\n",
    " 'att_miss_high_mean_h',\n",
    " 'att_miss_high_right_mean_diff',\n",
    " 'att_obxd_right_mean_diff',\n",
    " 'att_obxd_right_std_diff',\n",
    " 'att_one_on_one_mean_h',\n",
    " 'att_one_on_one_mean_a',\n",
    " 'att_one_on_one_std_diff',\n",
    " 'att_one_on_one_std_h',\n",
    " 'att_post_high_std_a',\n",
    " 'att_post_high_std_h',\n",
    " 'att_post_right_mean_a',\n",
    " 'att_post_right_mean_h',\n",
    " 'attempts_ibox_std_a',\n",
    " 'attempts_ibox_std_h',\n",
    " 'backward_pass_mean_diff',\n",
    " 'backward_pass_mean_h',\n",
    " 'big_chance_created_mean_h',\n",
    " 'big_chance_created_std_diff',\n",
    " 'clean_sheet_std_diff',\n",
    " 'contentious_decision_mean_a',\n",
    " 'contentious_decision_mean_diff',\n",
    " 'contentious_decision_std_a',\n",
    " 'contentious_decision_std_h',\n",
    " 'duel_won_pct_mean_diff',\n",
    " 'effective_clearance_mean_diff',\n",
    " 'error_lead_to_goal_mean_diff',\n",
    " 'error_lead_to_goal_std_diff',\n",
    " 'first_yellow_card_1t_mean_h',\n",
    " 'first_yellow_card_1t_mean_a',\n",
    " 'foul_throw_in_std_diff',\n",
    " 'fouled_final_third_mean_diff',\n",
    " 'goal_assist_deadball_mean_diff',\n",
    " 'goal_assist_openplay_std_diff',\n",
    " 'goal_assist_openplay_std_h',\n",
    " 'goal_assist_setplay_std_a',\n",
    " 'goal_assist_setplay_std_diff',\n",
    " 'goal_assist_setplay_std_h',\n",
    " 'goal_assist_std_diff',\n",
    " 'goals_mean_diff',\n",
    " 'goals_openplay_std_h',\n",
    " 'goals_openplay_std_a',\n",
    " 'good_high_claim_mean_diff',\n",
    " 'high_to_low_goals_mean_a',\n",
    " 'high_to_low_goals_mean_diff',\n",
    " 'high_to_low_goals_std_a',\n",
    " 'high_to_low_goals_std_h',\n",
    " 'imp_prob_under_goals_0.5_h',\n",
    " 'imp_prob_under_goals_2.5_diff',\n",
    " 'interception_mean_a',\n",
    " 'interception_mean_diff',\n",
    " 'interception_mean_h',\n",
    " 'interceptions_in_box_std_a',\n",
    " 'interceptions_in_box_std_diff',\n",
    " 'interceptions_in_box_std_h',\n",
    " 'last_man_tackle_mean_a',\n",
    " 'last_man_tackle_mean_diff',\n",
    " 'left_div_right_foot_goals_std_diff',\n",
    " 'leftside_pass_mean_a',\n",
    " 'leftside_pass_mean_h',\n",
    " 'leftside_pass_std_a',\n",
    " 'long_pass_own_to_opp_mean_diff',\n",
    " 'long_pass_own_to_opp_mean_h',\n",
    " 'no_foot_goals_ratio_std_a',\n",
    " 'no_foot_goals_ratio_std_diff',\n",
    " 'no_foot_goals_ratio_std_h',\n",
    " 'odd_ratio_under_corners_10.5',\n",
    " 'odd_ratio_under_corners_8.5',\n",
    " 'odds_home_under_both_score_h',\n",
    " 'odds_away_under_both_score_a',\n",
    " 'odds_away_under_goals_4.5_a',\n",
    " 'odds_home_under_goals_4.5_h',\n",
    " 'odds_away_over_goals_4.5_a',\n",
    " 'odds_home_over_goals_4.5_h',\n",
    " 'odds_home_under_goals_0.5_diff',\n",
    " 'odds_home_under_goals_0.5_h',\n",
    " 'own_goals_std_h',\n",
    " 'own_goals_std_a',\n",
    " 'pen_goals_conceded_mean_diff',\n",
    " 'penalty_faced_std_a',\n",
    " 'penalty_won_std_a',\n",
    " 'penalty_faced_std_h',\n",
    " 'penalty_won_std_h',\n",
    " 'poss_won_att_3rd_std_a',\n",
    " 'poss_won_att_3rd_std_diff',\n",
    " 'poss_won_att_3rd_std_h',\n",
    " 'post_scoring_att_std_a',\n",
    " 'post_scoring_att_std_h',\n",
    " 'prob_squared_under_goals_2.5',\n",
    " 'pts_dropped_winning_pos_mean_a',\n",
    " 'pts_dropped_winning_pos_std_diff',\n",
    " 'pts_dropped_winning_pos_std_h',\n",
    " 'ratio_over_goals_2.5_a',\n",
    " 'ratio_over_goals_2.5_h',\n",
    " 'ratio_under_goals_2.5_a',\n",
    " 'ratio_under_goals_2.5_diff',\n",
    " 'raw_prob_over_goals_1.5',\n",
    " 'raw_prob_over_goals_2.5',\n",
    " 'raw_prob_over_goals_3.5',\n",
    " 'raw_prob_under_goals_1.5',\n",
    " 'raw_prob_under_goals_2.5',\n",
    " 'raw_prob_under_goals_3.5',\n",
    " 'raw_prob_under_goals_4.5',\n",
    " 'red_card_1t_mean_a',\n",
    " 'red_card_1t_mean_diff',\n",
    " 'red_card_1t_mean_h',\n",
    " 'red_card_2t_mean_a',\n",
    " 'red_card_2t_mean_diff',\n",
    " 'red_card_mean_h',\n",
    " 'red_card_mean_a',\n",
    " 'red_card_std_diff',\n",
    " 'right_to_left_goals_mean_diff',\n",
    " 'right_to_left_goals_std_diff',\n",
    " 'rightside_pass_div_leftside_pass_mean_a',\n",
    " 'rightside_pass_div_leftside_pass_mean_h',\n",
    " 'second_yellow_mean_h',\n",
    " 'shots_mul_goals_std_h',\n",
    " 'second_yellow_mean_a',\n",
    " 'shots_mul_goals_std_a',\n",
    " 'successful_final_third_passes_mean_a',\n",
    " 'successful_final_third_passes_mean_diff',\n",
    " 'successful_final_third_passes_std_a',\n",
    " 'successful_final_third_passes_std_diff',\n",
    " 'successful_put_through_std_a',\n",
    " 'successful_put_through_std_diff',\n",
    " 'successful_put_through_std_h',\n",
    " 'total_clearance_mean_a',\n",
    " 'total_clearance_mean_h',\n",
    " 'total_fastbreak_std_diff',\n",
    " 'total_high_claim_mean_a',\n",
    " 'total_high_claim_mean_diff',\n",
    " 'total_high_claim_mean_h',\n",
    " 'total_keeper_sweeper_mean_a',\n",
    " 'total_keeper_sweeper_mean_h',\n",
    " 'total_launches_std_h',\n",
    " 'total_launches_std_a',\n",
    " 'total_red_card_mean_h',\n",
    " 'total_red_card_mean_a',\n",
    " 'total_red_card_std_diff',\n",
    " 'total_throws_std_a',\n",
    " 'total_throws_std_diff',\n",
    " 'total_win_pct_over_goals_1.5_a',\n",
    " 'total_win_pct_over_goals_1.5_h',\n",
    " 'total_win_pct_over_goals_2.5_a',\n",
    " 'total_win_pct_over_goals_2.5_h',\n",
    " 'total_win_pct_over_goals_3.5_a',\n",
    " 'total_win_pct_over_goals_3.5_h',\n",
    " 'total_win_pct_under_goals_1.5_a',\n",
    " 'total_win_pct_under_goals_1.5_h',\n",
    " 'total_win_pct_under_goals_2.5_a',\n",
    " 'total_win_pct_under_goals_2.5_h',\n",
    " 'total_win_pct_under_goals_3.5_a',\n",
    " 'total_yel_card_std_a',\n",
    " 'total_yel_card_std_diff',\n",
    " 'total_yel_card_std_h',\n",
    " 'win_pct_away_over_goals_2.5_a',\n",
    " 'win_pct_away_over_goals_2.5_diff',\n",
    " 'win_pct_away_under_goals_2.5_a',\n",
    " 'win_pct_away_under_goals_2.5_diff',\n",
    " 'win_pct_home_over_goals_2.5_a',\n",
    " 'win_pct_home_over_goals_2.5_diff',\n",
    " 'win_pct_home_over_goals_2.5_h',\n",
    " 'win_pct_home_under_goals_2.5_diff',\n",
    " 'win_pct_home_under_goals_2.5_h',\n",
    " 'winner_mean_diff']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_dataset(test_date):\n",
    "    ds = create_dataset(target=target,\n",
    "                        test_date=test_date,\n",
    "                        features=feats,\n",
    "                        odds_features=True,\n",
    "                        include_std=True,\n",
    "                        test_weeks=6,\n",
    "                        ignore_features=IGNORE_FEATURES,\n",
    "                        drop_future_matches=False,\n",
    "                       )\n",
    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
    "    train_data = train_data[train_data.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
    "    setup_kwargs = dict(\n",
    "        preprocess=True,\n",
    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
    "        #custom_pipeline=loaded,\n",
    "        #train_size=0.75,\n",
    "        session_id=123,\n",
    "        normalize=True,\n",
    "        normalize_method=\"robust\",\n",
    "        transformation=True,\n",
    "        ignore_low_variance=True,\n",
    "        remove_multicollinearity=False,\n",
    "        multicollinearity_threshold=0.8,\n",
    "        n_jobs=-1,\n",
    "        use_gpu=False,\n",
    "        profile=False,\n",
    "        #ignore_features=ignore_features,\n",
    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
    "        remove_perfect_collinearity=True,\n",
    "        create_clusters=False,\n",
    "        fold=4,\n",
    "        feature_selection=False,\n",
    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
    "        feature_selection_threshold=0.5,\n",
    "        combine_rare_levels=False,\n",
    "        rare_level_threshold=0.02,\n",
    "        pca=False,\n",
    "        pca_method=\"linear\",\n",
    "        pca_components=30,\n",
    "        polynomial_features=False,\n",
    "        polynomial_degree=2,\n",
    "        polynomial_threshold=0.05,\n",
    "        trigonometry_features=False,\n",
    "        remove_outliers=True,\n",
    "        outliers_threshold=0.01,\n",
    "        feature_ratio=False,\n",
    "        feature_interaction=False,\n",
    "        # Makes everything slow AF. use to find out possibly interesting features\n",
    "        interaction_threshold=0.05,\n",
    "        fix_imbalance=True,\n",
    "        log_experiment=False,\n",
    "        verbose=False,\n",
    "        silent=True,\n",
    "        experiment_name=\"lagstest\",\n",
    "    )\n",
    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
    "    return train_data, test_data, val_data, ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
    "def train_ensemble():\n",
    "    top_models = compare_models(\n",
    "            n_select=8,\n",
    "            sort='MCC',\n",
    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"rf\", \"svm\"],\n",
    "            verbose=True,\n",
    "        )\n",
    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
    "    blend = blend_models(cali)\n",
    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
    "    return opti, cali, tuned_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
    "def train_linear_models():\n",
    "    top_models = compare_models(\n",
    "            n_select=8,\n",
    "            sort='MCC',\n",
    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
    "            verbose=True,\n",
    "        )\n",
    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
    "    return tuned_models, cali"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6{\n",
       "            background:  yellow;\n",
       "        }</style><table id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Accuracy</th>        <th class=\"col_heading level0 col1\" >AUC</th>        <th class=\"col_heading level0 col2\" >Recall</th>        <th class=\"col_heading level0 col3\" >Prec.</th>        <th class=\"col_heading level0 col4\" >F1</th>        <th class=\"col_heading level0 col5\" >Kappa</th>        <th class=\"col_heading level0 col6\" >MCC</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >0.6414</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6893</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6716</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.6507</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6610</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.2806</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2807</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >0.5997</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6586</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6321</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.6124</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6221</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.1969</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1970</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >0.6216</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6679</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.6123</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.6442</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6278</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.2435</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2439</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >0.6126</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.6475</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6015</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.6345</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6175</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.2257</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.2260</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row4\" class=\"row_heading level0 row4\" >Mean</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0\" class=\"data row4 col0\" >0.6188</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1\" class=\"data row4 col1\" >0.6658</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2\" class=\"data row4 col2\" >0.6294</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3\" class=\"data row4 col3\" >0.6354</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4\" class=\"data row4 col4\" >0.6321</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5\" class=\"data row4 col5\" >0.2367</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6\" class=\"data row4 col6\" >0.2369</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row5\" class=\"row_heading level0 row5\" >SD</th>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col0\" class=\"data row5 col0\" >0.0152</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col1\" class=\"data row5 col1\" >0.0154</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col2\" class=\"data row5 col2\" >0.0267</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col3\" class=\"data row5 col3\" >0.0145</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col4\" class=\"data row5 col4\" >0.0171</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col5\" class=\"data row5 col5\" >0.0303</td>\n",
       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col6\" class=\"data row5 col6\" >0.0303</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f44e43353d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tuned_linear, cali_linear = train_linear_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_08d08ce2_82b5_11ec_86da_31d766f3ddbd th {\n",
       "          text-align: left;\n",
       "    }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  lightgrey;\n",
       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "            background-color:  lightgrey;\n",
       "        }</style><table id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6166</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6734</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5836</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6500</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.6081</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2351</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.2400</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.3000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >lda</th>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >Linear Discriminant Analysis</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6166</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6705</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5892</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6475</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.6115</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.2347</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.2387</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.3025</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6166</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5892</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6476</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.6115</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2347</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.2387</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2850</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >svm</th>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >SVM - Linear Kernel</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5841</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.0000</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5441</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6157</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5734</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1708</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1739</td>\n",
       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.3050</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb80cbcb2e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "top_models = compare_models(\n",
    "            n_select=8,\n",
    "            sort='MCC',\n",
    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
    "            verbose=True,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.10126692364605047, average=False, class_weight=None,\n",
       "              early_stopping=False, epsilon=0.1, eta0=0.001022673152073415,\n",
       "              fit_intercept=False, l1_ratio=0.2203376725525369,\n",
       "              learning_rate='optimal', loss='hinge', max_iter=1000,\n",
       "              n_iter_no_change=5, n_jobs=-1, penalty='l1', power_t=0.5,\n",
       "              random_state=123, shuffle=True, tol=0.001,\n",
       "              validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned_linear[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6522b430b2a64128ac3789a8d93475f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate_model(tuned_linear[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9de03ccce3a4047b615b146297be69e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate_model(cali_linear[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "market=\"goals\"\n",
    "cutoff = 2.5\n",
    "tips = tips_from_model(model=cali_linear[0],\n",
    "                test_data=test_data,\n",
    "                features=feats,\n",
    "                market=market,\n",
    "                cutoff=cutoff,\n",
    "                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>win</th>\n",
       "      <th>profit</th>\n",
       "      <th>exp_payoff</th>\n",
       "      <th>exp_payoff_prec</th>\n",
       "      <th>odds</th>\n",
       "      <th>confidence</th>\n",
       "      <th>count</th>\n",
       "      <th>buenas</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bet_type</th>\n",
       "      <th>match_week</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">goals_over_2.5</th>\n",
       "      <th>2021_19</th>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.434545</td>\n",
       "      <td>1.193888</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.743636</td>\n",
       "      <td>0.683591</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_20</th>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.144000</td>\n",
       "      <td>1.087417</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.641000</td>\n",
       "      <td>0.664270</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_21</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.359000</td>\n",
       "      <td>1.130386</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.685000</td>\n",
       "      <td>0.671600</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_22</th>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>1.130597</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.708571</td>\n",
       "      <td>0.662800</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_23</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>-0.140000</td>\n",
       "      <td>1.216788</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.815000</td>\n",
       "      <td>0.667750</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_24</th>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.058750</td>\n",
       "      <td>1.168317</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.730000</td>\n",
       "      <td>0.674975</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_25</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>1.200240</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.800000</td>\n",
       "      <td>0.666800</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">goals_under_2.5</th>\n",
       "      <th>2021_19</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.255000</td>\n",
       "      <td>1.034973</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.630000</td>\n",
       "      <td>0.634150</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_20</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.372000</td>\n",
       "      <td>1.142580</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.706000</td>\n",
       "      <td>0.670480</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_21</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>-0.363333</td>\n",
       "      <td>1.200528</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.746667</td>\n",
       "      <td>0.686033</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_22</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.670000</td>\n",
       "      <td>1.025046</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.670000</td>\n",
       "      <td>0.613800</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_23</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>1.011686</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.670000</td>\n",
       "      <td>0.605800</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021_24</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.755000</td>\n",
       "      <td>1.155859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.755000</td>\n",
       "      <td>0.662375</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>model</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 win    profit  exp_payoff  exp_payoff_prec  \\\n",
       "bet_type        match_week                                                    \n",
       "goals_over_2.5  2021_19     0.818182  0.434545    1.193888              NaN   \n",
       "                2021_20     0.700000  0.144000    1.087417              NaN   \n",
       "                2021_21     0.800000  0.359000    1.130386              NaN   \n",
       "                2021_22     0.571429  0.040000    1.130597              NaN   \n",
       "                2021_23     0.500000 -0.140000    1.216788              NaN   \n",
       "                2021_24     0.625000  0.058750    1.168317              NaN   \n",
       "                2021_25     0.000000 -1.000000    1.200240              NaN   \n",
       "goals_under_2.5 2021_19     0.750000  0.255000    1.034973              NaN   \n",
       "                2021_20     0.800000  0.372000    1.142580              NaN   \n",
       "                2021_21     0.333333 -0.363333    1.200528              NaN   \n",
       "                2021_22     1.000000  0.670000    1.025046              NaN   \n",
       "                2021_23     0.000000 -1.000000    1.011686              NaN   \n",
       "                2021_24     1.000000  0.755000    1.155859              NaN   \n",
       "\n",
       "                                odds  confidence  count  buenas  model  \n",
       "bet_type        match_week                                              \n",
       "goals_over_2.5  2021_19     1.743636    0.683591     11       9  model  \n",
       "                2021_20     1.641000    0.664270     10       7  model  \n",
       "                2021_21     1.685000    0.671600     10       8  model  \n",
       "                2021_22     1.708571    0.662800      7       4  model  \n",
       "                2021_23     1.815000    0.667750      4       2  model  \n",
       "                2021_24     1.730000    0.674975      8       5  model  \n",
       "                2021_25     1.800000    0.666800      1       0  model  \n",
       "goals_under_2.5 2021_19     1.630000    0.634150      4       3  model  \n",
       "                2021_20     1.706000    0.670480      5       4  model  \n",
       "                2021_21     1.746667    0.686033      3       1  model  \n",
       "                2021_22     1.670000    0.613800      1       1  model  \n",
       "                2021_23     1.670000    0.605800      1       0  model  \n",
       "                2021_24     1.755000    0.662375      4       4  model  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def show_results(df, groupby=[\"bet_type\", \"match_week\"]):\n",
    "    ix = np.logical_and(df[\"win\"]>=0, True)# df[\"label\"]>=1)\n",
    "    #ix = np.logical_and(df[\"consensus\"] >=0.1, ix)\n",
    "    ix2 = np.logical_and(df[\"odds\"]>=1.5, df[\"odds\"]<2.)\n",
    "    ix = np.logical_and(ix, ix2)\n",
    "    #ix = np.logical_and(ix, ~df[\"validation\"])\n",
    "    #ix = np.logical_and(ix, df[\"exp_payoff_prec\"] >0.88)\n",
    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] <1.2)\n",
    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] >0.8)\n",
    "    ix = np.logical_and(ix, df[\"confidence\"] >=0.60)\n",
    "    #ix = np.logical_and(ix, df[\"confidence\"] <=0.75)\n",
    "    x = df[ix].groupby(groupby)[[\"win\", \"profit\", \"exp_payoff\", \"exp_payoff_prec\", \"odds\", \"confidence\"]].mean()\n",
    "    x[\"count\"] = df[ix].groupby(groupby)[[\"win\"]].count()\n",
    "    x[\"buenas\"] = df[ix].groupby(groupby)[[\"win\"]].sum().astype(int)\n",
    "    x[\"model\"] = df[\"model\"].iloc[0]\n",
    "    return x\n",
    "show_results(tips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}