housing-price-prediction-XGBoost-and-optuna.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "68b84162",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:47.597551Z",
     "iopub.status.busy": "2024-06-20T10:16:47.597189Z",
     "iopub.status.idle": "2024-06-20T10:16:48.361454Z",
     "shell.execute_reply": "2024-06-20T10:16:48.360556Z"
    },
    "papermill": {
     "duration": 0.772065,
     "end_time": "2024-06-20T10:16:48.363738",
     "exception": false,
     "start_time": "2024-06-20T10:16:47.591673",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/simple-housing-price-prediction/sample_submission.csv\n",
      "/kaggle/input/simple-housing-price-prediction/train.csv\n",
      "/kaggle/input/simple-housing-price-prediction/test.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "50a06b29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:48.373245Z",
     "iopub.status.busy": "2024-06-20T10:16:48.372877Z",
     "iopub.status.idle": "2024-06-20T10:16:49.868819Z",
     "shell.execute_reply": "2024-06-20T10:16:49.867851Z"
    },
    "papermill": {
     "duration": 1.503634,
     "end_time": "2024-06-20T10:16:49.871613",
     "exception": false,
     "start_time": "2024-06-20T10:16:48.367979",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f480b9c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:49.884370Z",
     "iopub.status.busy": "2024-06-20T10:16:49.883603Z",
     "iopub.status.idle": "2024-06-20T10:16:50.966561Z",
     "shell.execute_reply": "2024-06-20T10:16:50.965770Z"
    },
    "papermill": {
     "duration": 1.091464,
     "end_time": "2024-06-20T10:16:50.968771",
     "exception": false,
     "start_time": "2024-06-20T10:16:49.877307",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data_path = \"../input/simple-housing-price-prediction/train.csv\"\n",
    "test_data_path = \"../input/simple-housing-price-prediction/test.csv\"\n",
    "\n",
    "df = pd.read_csv(train_data_path)\n",
    "df_test = pd.read_csv(test_data_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "38e53081",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:50.977953Z",
     "iopub.status.busy": "2024-06-20T10:16:50.977669Z",
     "iopub.status.idle": "2024-06-20T10:16:51.015750Z",
     "shell.execute_reply": "2024-06-20T10:16:51.014787Z"
    },
    "papermill": {
     "duration": 0.045008,
     "end_time": "2024-06-20T10:16:51.017870",
     "exception": false,
     "start_time": "2024-06-20T10:16:50.972862",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>house_id</th>\n",
       "      <th>date</th>\n",
       "      <th>location</th>\n",
       "      <th>type</th>\n",
       "      <th>block</th>\n",
       "      <th>street</th>\n",
       "      <th>storey_range</th>\n",
       "      <th>area_sqm</th>\n",
       "      <th>flat_model</th>\n",
       "      <th>commence_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>199577</td>\n",
       "      <td>2006-09</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>5 ROOM</td>\n",
       "      <td>107D</td>\n",
       "      <td>Agawan Court</td>\n",
       "      <td>07 TO 09</td>\n",
       "      <td>110.0</td>\n",
       "      <td>D</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>217021</td>\n",
       "      <td>2007-06</td>\n",
       "      <td>Fresno</td>\n",
       "      <td>3 ROOM</td>\n",
       "      <td>678</td>\n",
       "      <td>Cleo St</td>\n",
       "      <td>07 TO 09</td>\n",
       "      <td>64.0</td>\n",
       "      <td>N</td>\n",
       "      <td>1988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>308062</td>\n",
       "      <td>2010-09</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>4 ROOM</td>\n",
       "      <td>5</td>\n",
       "      <td>E Pleasant View Way</td>\n",
       "      <td>10 TO 12</td>\n",
       "      <td>92.0</td>\n",
       "      <td>K</td>\n",
       "      <td>1976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>212465</td>\n",
       "      <td>2007-04</td>\n",
       "      <td>Austin</td>\n",
       "      <td>4 ROOM</td>\n",
       "      <td>326</td>\n",
       "      <td>Park Hollow Ln</td>\n",
       "      <td>10 TO 12</td>\n",
       "      <td>92.0</td>\n",
       "      <td>K</td>\n",
       "      <td>1977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60654</td>\n",
       "      <td>2001-10</td>\n",
       "      <td>Honolulu</td>\n",
       "      <td>4 ROOM</td>\n",
       "      <td>794</td>\n",
       "      <td>Ala Puawa Place</td>\n",
       "      <td>04 TO 06</td>\n",
       "      <td>102.0</td>\n",
       "      <td>G</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   house_id     date  location    type block               street  \\\n",
       "0    199577  2006-09   Raleigh  5 ROOM  107D       Agawan Court     \n",
       "1    217021  2007-06    Fresno  3 ROOM   678              Cleo St   \n",
       "2    308062  2010-09    Tucson  4 ROOM     5  E Pleasant View Way   \n",
       "3    212465  2007-04    Austin  4 ROOM   326       Park Hollow Ln   \n",
       "4     60654  2001-10  Honolulu  4 ROOM   794    Ala Puawa Place     \n",
       "\n",
       "  storey_range  area_sqm flat_model  commence_date  \n",
       "0     07 TO 09     110.0          D           2003  \n",
       "1     07 TO 09      64.0          N           1988  \n",
       "2     10 TO 12      92.0          K           1976  \n",
       "3     10 TO 12      92.0          K           1977  \n",
       "4     04 TO 06     102.0          G           1998  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df.copy()\n",
    "y = X.pop('price')\n",
    "X_test = df_test.copy()\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "865b4599",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:51.027366Z",
     "iopub.status.busy": "2024-06-20T10:16:51.027105Z",
     "iopub.status.idle": "2024-06-20T10:16:51.146892Z",
     "shell.execute_reply": "2024-06-20T10:16:51.146083Z"
    },
    "papermill": {
     "duration": 0.127052,
     "end_time": "2024-06-20T10:16:51.149173",
     "exception": false,
     "start_time": "2024-06-20T10:16:51.022121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y,train_size=0.8, test_size=0.2,\n",
    "                                                                random_state=0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd6d90c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:51.159070Z",
     "iopub.status.busy": "2024-06-20T10:16:51.158779Z",
     "iopub.status.idle": "2024-06-20T10:16:51.321617Z",
     "shell.execute_reply": "2024-06-20T10:16:51.320854Z"
    },
    "papermill": {
     "duration": 0.170314,
     "end_time": "2024-06-20T10:16:51.323922",
     "exception": false,
     "start_time": "2024-06-20T10:16:51.153608",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "categorical_cols = [cname for cname in X_train.columns if\n",
    "                    X_train[cname].nunique() < 10 and \n",
    "                    X_train[cname].dtype == \"object\"]\n",
    "\n",
    "numerical_cols = [cname for cname in X_train.columns if \n",
    "                X_train[cname].dtype in ['int64', 'float64']]\n",
    "\n",
    "my_cols = categorical_cols + numerical_cols\n",
    "X_train = X_train[my_cols].copy()\n",
    "X_valid = X_valid[my_cols].copy()\n",
    "X_test = X_test[my_cols].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "68d6a551",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:51.334399Z",
     "iopub.status.busy": "2024-06-20T10:16:51.333700Z",
     "iopub.status.idle": "2024-06-20T10:16:51.338995Z",
     "shell.execute_reply": "2024-06-20T10:16:51.338159Z"
    },
    "papermill": {
     "duration": 0.012414,
     "end_time": "2024-06-20T10:16:51.340790",
     "exception": false,
     "start_time": "2024-06-20T10:16:51.328376",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Preprocessing for numerical data\n",
    "numerical_transformer = SimpleImputer(strategy='median')\n",
    "\n",
    "# Preprocessing for categorical data\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='most_frequent')),\n",
    "    ('onehot', OneHotEncoder(handle_unknown='ignore'))\n",
    "])\n",
    "\n",
    "# Bundle preprocessing for numerical and categorical data\n",
    "preprocessor = ColumnTransformer(transformers=[\n",
    "    ('num', numerical_transformer, numerical_cols),\n",
    "    ('cat', categorical_transformer, categorical_cols)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f1e234de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:51.350126Z",
     "iopub.status.busy": "2024-06-20T10:16:51.349856Z",
     "iopub.status.idle": "2024-06-20T10:16:51.354803Z",
     "shell.execute_reply": "2024-06-20T10:16:51.354013Z"
    },
    "papermill": {
     "duration": 0.01184,
     "end_time": "2024-06-20T10:16:51.356816",
     "exception": false,
     "start_time": "2024-06-20T10:16:51.344976",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import optuna\n",
    "# X = X_train\n",
    "# def score_dataset(X, y, model=XGBRegressor()):\n",
    "#     # Label encoding for categoricals\n",
    "#     #\n",
    "#     # Label encoding is good for XGBoost and RandomForest, but one-hot\n",
    "#     # would be better for models like Lasso or Ridge. The `cat.codes`\n",
    "#     # attribute holds the category levels.\n",
    "#     for colname in X.select_dtypes([\"category\"]):\n",
    "#         X[colname] = X[colname].cat.codes\n",
    "#     # Metric for Housing competition is RMSLE (Root Mean Squared Log Error)\n",
    "#     log_y = np.log(y)\n",
    "#     score = cross_val_score(\n",
    "#         model, X, log_y, cv=5, scoring=\"neg_mean_squared_error\",\n",
    "#     )\n",
    "#     score = -1 * score.mean()\n",
    "#     score = np.sqrt(score)\n",
    "#     return score\n",
    "\n",
    "# def objective(trial):\n",
    "#     xgb_params = dict(\n",
    "#         max_depth=trial.suggest_int(\"max_depth\", 2, 10),\n",
    "#         learning_rate=trial.suggest_float(\"learning_rate\", 1e-4, 1e-1, log=True),\n",
    "#         n_estimators=trial.suggest_int(\"n_estimators\", 1000, 8000),\n",
    "#         min_child_weight=trial.suggest_int(\"min_child_weight\", 1, 10),\n",
    "#         colsample_bytree=trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "#         subsample=trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "#         reg_alpha=trial.suggest_float(\"reg_alpha\", 1e-4, 1e2, log=True),\n",
    "#         reg_lambda=trial.suggest_float(\"reg_lambda\", 1e-4, 1e2, log=True),\n",
    "#     )\n",
    "#     xgb = XGBRegressor(**xgb_params)\n",
    "#     return score_dataset(X_train, y_train, xgb)\n",
    "\n",
    "# study = optuna.create_study(direction=\"minimize\")\n",
    "# study.optimize(objective, n_trials=20)\n",
    "# xgb_params = study.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a713f37e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:16:51.366145Z",
     "iopub.status.busy": "2024-06-20T10:16:51.365704Z",
     "iopub.status.idle": "2024-06-20T10:18:23.930958Z",
     "shell.execute_reply": "2024-06-20T10:18:23.929949Z"
    },
    "papermill": {
     "duration": 92.572237,
     "end_time": "2024-06-20T10:18:23.933107",
     "exception": false,
     "start_time": "2024-06-20T10:16:51.360870",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:889: UserWarning: `early_stopping_rounds` in `fit` method is deprecated for better compatibility with scikit-learn, use `early_stopping_rounds` in constructor or`set_params` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 29120.773242935753\n"
     ]
    }
   ],
   "source": [
    "# Define model\n",
    "model = XGBRegressor(n_estimators=4222,\n",
    "                     learning_rate=0.001779955848977826,\n",
    "                     max_depth=9,\n",
    "                     min_child_weight=2,\n",
    "                     random_state=0,\n",
    "                     colsample_bytree=0.8068642787690588,\n",
    "                     subsample=0.8493678320968241,\n",
    "                     reg_alpha=0.031161768449367588,\n",
    "                     reg_lambda=0.2657100756884015\n",
    "                    )\n",
    "# Bundle preprocessing and modeling code in a pipeline\n",
    "my_pipeline = Pipeline(steps=[('preprocessor', preprocessor),\n",
    "                              ('model', model)\n",
    "                             ])\n",
    "\n",
    "preprocessor.fit_transform(X_train)\n",
    "X_valid_transformed = preprocessor.transform(X_valid)\n",
    "fit_params = {\"model__eval_set\": [(X_valid_transformed, y_valid)], \n",
    "              \"model__early_stopping_rounds\": 5,\n",
    "              \"model__verbose\": False}\n",
    "\n",
    "# Preprocessing of training data, fit model \n",
    "my_pipeline.fit(X_train, y_train, **fit_params)\n",
    "\n",
    "# Preprocessing of validation data, get predictions\n",
    "preds = my_pipeline.predict(X_valid)\n",
    "\n",
    "# Evaluate the model\n",
    "print('MAE:', mean_absolute_error(y_valid, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b06f277b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:18:23.947395Z",
     "iopub.status.busy": "2024-06-20T10:18:23.947015Z",
     "iopub.status.idle": "2024-06-20T10:18:29.424207Z",
     "shell.execute_reply": "2024-06-20T10:18:29.423437Z"
    },
    "papermill": {
     "duration": 5.487236,
     "end_time": "2024-06-20T10:18:29.426611",
     "exception": false,
     "start_time": "2024-06-20T10:18:23.939375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Preprocessing of test data, fit model\n",
    "preds_test = my_pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7575409c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-20T10:18:29.438732Z",
     "iopub.status.busy": "2024-06-20T10:18:29.438434Z",
     "iopub.status.idle": "2024-06-20T10:18:29.582258Z",
     "shell.execute_reply": "2024-06-20T10:18:29.581418Z"
    },
    "papermill": {
     "duration": 0.152314,
     "end_time": "2024-06-20T10:18:29.584802",
     "exception": false,
     "start_time": "2024-06-20T10:18:29.432488",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "output = pd.DataFrame({'house_id': X_test.house_id,\n",
    "                       'price': preds_test})\n",
    "output.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 6856479,
     "sourceId": 62928,
     "sourceType": "competition"
    }
   ],
   "dockerImageVersionId": 30698,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 105.288148,
   "end_time": "2024-06-20T10:18:30.108562",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-06-20T10:16:44.820414",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}