robust.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个文件专门用来测试异常值下已训练好的模型的稳定性。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "from IPython.display import display #方便对数据进行可视化\n",
    "from sklearn.model_selection import train_test_split\n",
    "#从头开始处理数据\n",
    "\n",
    "# 导入训练数据\n",
    "train_data = pd.read_csv(\n",
    "    \"train.csv\", \n",
    "    low_memory=False)\n",
    "\n",
    "store_features = pd.read_csv(\"store.csv\")\n",
    "# 导入测试数据\n",
    "test_data = pd.read_csv(\"test.csv\")\n",
    "\n",
    "#融合\n",
    "train_data = pd.merge(train_data, store_features, on = 'Store')\n",
    "test_data = pd.merge(test_data, store_features, on = 'Store')\n",
    "\n",
    "train_data = train_data.drop('Customers', axis = 1)\n",
    "sales = train_data['Sales']\n",
    "features = train_data.drop('Sales', axis = 1)\n",
    "\n",
    "# 对字符串特征进行独热编码\n",
    "category = ['StoreType', 'Assortment', 'PromoInterval', 'StateHoliday']\n",
    "\n",
    "\n",
    "#首先将训练集和测试集合并，然后统一独热编码，然后再拆分为训练集和测试集\n",
    "features_test = test_data.drop('Id', axis = 1)\n",
    "data_merge = pd.concat([features, features_test], ignore_index = True)\n",
    "data_merge_pd = pd.get_dummies(data_merge, columns = category)\n",
    "\n",
    "# 需要把NA/NAN的数据转化成0\n",
    "data_merge_pd.fillna(0, inplace = True)\n",
    "\n",
    "#增加一步，增加‘Year', 'month', 'day'三个有序分类数据\n",
    "\n",
    "data_merge_pd['year'] = data_merge_pd['Date'].apply(lambda x: float(x.split('-')[0]))\n",
    "data_merge_pd['month'] = data_merge_pd['Date'].apply(lambda x: float(x.split('-')[1]))\n",
    "data_merge_pd['day'] = data_merge_pd['Date'].apply(lambda x: float(x.split('-')[2]))\n",
    "data_merge_pd = data_merge_pd.drop('Date', axis = 1)\n",
    "\n",
    "#拆分\n",
    "\n",
    "features_pd = data_merge_pd[0:features.shape[0]]\n",
    "features_test_pd = data_merge_pd[features.shape[0]:data_merge_pd.shape[0]]\n",
    "\n",
    "#下面将sales做log处理\n",
    "\n",
    "sales_log = np.log(train_data['Sales'] + 1) #+1是为了避免log（0）的情况同时使得数据从0开始\n",
    "\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    features_pd, sales_log, test_size = 0.2, random_state = 99)#重新划分一下\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "dval = xgb.DMatrix(X_val, label = y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 9.092271e+00  8.675215e+00 -2.974093e-03 ...  9.222454e+00  8.534106e+00\n",
      "  9.355307e-03]\n",
      "0.07235972307593926\n",
      "The rmspe of the model on validation data set is 0.072360\n"
     ]
    }
   ],
   "source": [
    "#载入模型\n",
    "\n",
    "model  = xgb.Booster()\n",
    "model.load_model('rossmann.model')\n",
    "\n",
    "def rmspe(y, y_hat):\n",
    "    #由于sales取了对数，这里要还原\n",
    "    y = np.exp(y) - 1\n",
    "    y_hat = np.exp(y_hat) - 1\n",
    "    \n",
    "    #由于有yi=0的情况，直接计算会得出无穷大的结果，所以需要处理一下\n",
    "    #定义一个y_rev来表示y的倒数，y为0时y_rev也赋值为0（因为在项目中提到“ Any day and store with 0 sales is ignored in scoring.“）\n",
    "    y_rev = np.zeros(y.shape, dtype = float)\n",
    "    n_zero = y != 0\n",
    "    y_rev[n_zero] = 1./y[n_zero]\n",
    "\n",
    "    rmspe = np.sqrt(np.mean(((y - y_hat)*y_rev)**2))\n",
    "    return rmspe\n",
    "\n",
    "#定义一个打分函数\n",
    "def score(bst, dval):\n",
    "    y_pred = bst.predict(dval)\n",
    "    print(y_pred)\n",
    "    y_val = dval.get_label()\n",
    "    score = rmspe(y_val, y_pred)\n",
    "    print(score)\n",
    "    print('The rmspe of the model on validation data set is {:.6f}'.format(score))\n",
    "    pass\n",
    "\n",
    "score(model, dval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "#我们把验证集中1%的数据的‘Day’这一项，改成一个随机的异常值\n",
    "one_percent = int(0.01 * X_val.shape[0])\n",
    "temp = np.array(X_val['day'])\n",
    "for i in range(0, one_percent):\n",
    "    index = randint(0, X_val.shape[0] - 1)\n",
    "    temp[index] = randint(1, 10) * 100\n",
    "    \n",
    "X_val = X_val.drop('day', axis = 1)\n",
    "X_val['day'] = temp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "dval = xgb.DMatrix(X_val, label = y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 9.092271e+00  8.675215e+00 -2.974093e-03 ...  9.222454e+00  8.534106e+00\n",
      "  9.355307e-03]\n",
      "0.07610700502446097\n",
      "The rmspe of the model on validation data set is 0.076107\n"
     ]
    }
   ],
   "source": [
    "score(model, dval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 9.092271e+00  8.675215e+00 -2.974093e-03 ...  9.222454e+00  8.534106e+00\n",
      "  9.355307e-03]\n",
      "0.07998797411602782\n",
      "The rmspe of the model on validation data set is 0.079988\n"
     ]
    }
   ],
   "source": [
    "#我们把验证集中10%的数据的‘Day’这一项，改成一个随机的异常值\n",
    "one_percent = int(0.01 * X_val.shape[0])\n",
    "temp = np.array(X_val['day'])\n",
    "for i in range(0, one_percent):\n",
    "    index = randint(0, X_val.shape[0] - 1)\n",
    "    temp[index] = randint(1, 10) * 100\n",
    "    \n",
    "X_val = X_val.drop('day', axis = 1)\n",
    "X_val['day'] = temp\n",
    "dtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "dval = xgb.DMatrix(X_val, label = y_val)\n",
    "score(model, dval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}