test_data_cleaning .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dependencies\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from sklearn.metrics import confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Customer_ID</th>\n",
       "      <th>Month</th>\n",
       "      <th>Name</th>\n",
       "      <th>Age</th>\n",
       "      <th>SSN</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Monthly_Inhand_Salary</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>...</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x160a</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>September</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>23</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.030402</td>\n",
       "      <td>22 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x160b</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>October</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>24</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.053114</td>\n",
       "      <td>22 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x160c</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>November</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>24</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.811894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>148.23393788500925</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>264.67544623342997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x160d</td>\n",
       "      <td>CUS_0xd40</td>\n",
       "      <td>December</td>\n",
       "      <td>Aaron Maashoh</td>\n",
       "      <td>24_</td>\n",
       "      <td>821-00-0265</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.430559</td>\n",
       "      <td>23 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x1616</td>\n",
       "      <td>CUS_0x21b1</td>\n",
       "      <td>September</td>\n",
       "      <td>Rick Rothackerj</td>\n",
       "      <td>28</td>\n",
       "      <td>004-07-5839</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>3037.986667</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.926822</td>\n",
       "      <td>27 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>39.684018417945296</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.2984336755923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>0x25fe5</td>\n",
       "      <td>CUS_0x8600</td>\n",
       "      <td>December</td>\n",
       "      <td>Sarah McBridec</td>\n",
       "      <td>4975</td>\n",
       "      <td>031-35-0942</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>1929.906667</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>34.780553</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>146.48632477751087</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>275.53956951573343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>0x25fee</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>September</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>27.758522</td>\n",
       "      <td>31 Years and 11 Months</td>\n",
       "      <td>NM</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>181.44299902757518</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>409.39456169535066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>0x25fef</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>October</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.858542</td>\n",
       "      <td>32 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>__10000__</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.7263321025098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>0x25ff0</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>November</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.139840</td>\n",
       "      <td>32 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>97.59857973344877</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.23898098947717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>0x25ff1</td>\n",
       "      <td>CUS_0x942c</td>\n",
       "      <td>December</td>\n",
       "      <td>Nicks</td>\n",
       "      <td>25</td>\n",
       "      <td>078-73-5990</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.108530</td>\n",
       "      <td>32 Years and 2 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>220.45787812168732</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.37968260123847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID Customer_ID      Month             Name   Age          SSN  \\\n",
       "0       0x160a   CUS_0xd40  September    Aaron Maashoh    23  821-00-0265   \n",
       "1       0x160b   CUS_0xd40    October    Aaron Maashoh    24  821-00-0265   \n",
       "2       0x160c   CUS_0xd40   November    Aaron Maashoh    24  821-00-0265   \n",
       "3       0x160d   CUS_0xd40   December    Aaron Maashoh   24_  821-00-0265   \n",
       "4       0x1616  CUS_0x21b1  September  Rick Rothackerj    28  004-07-5839   \n",
       "...        ...         ...        ...              ...   ...          ...   \n",
       "49995  0x25fe5  CUS_0x8600   December   Sarah McBridec  4975  031-35-0942   \n",
       "49996  0x25fee  CUS_0x942c  September            Nicks    25  078-73-5990   \n",
       "49997  0x25fef  CUS_0x942c    October            Nicks    25  078-73-5990   \n",
       "49998  0x25ff0  CUS_0x942c   November            Nicks    25  078-73-5990   \n",
       "49999  0x25ff1  CUS_0x942c   December            Nicks    25  078-73-5990   \n",
       "\n",
       "      Occupation Annual_Income  Monthly_Inhand_Salary  Num_Bank_Accounts  ...  \\\n",
       "0      Scientist      19114.12            1824.843333                  3  ...   \n",
       "1      Scientist      19114.12            1824.843333                  3  ...   \n",
       "2      Scientist      19114.12            1824.843333                  3  ...   \n",
       "3      Scientist      19114.12                    NaN                  3  ...   \n",
       "4        _______      34847.84            3037.986667                  2  ...   \n",
       "...          ...           ...                    ...                ...  ...   \n",
       "49995  Architect      20002.88            1929.906667                 10  ...   \n",
       "49996   Mechanic      39628.99                    NaN                  4  ...   \n",
       "49997   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "49998   Mechanic      39628.99                    NaN                  4  ...   \n",
       "49999   Mechanic      39628.99            3359.415833                  4  ...   \n",
       "\n",
       "       Num_Credit_Inquiries  Credit_Mix Outstanding_Debt  \\\n",
       "0                    2022.0        Good           809.98   \n",
       "1                       4.0        Good           809.98   \n",
       "2                       4.0        Good           809.98   \n",
       "3                       4.0        Good           809.98   \n",
       "4                       5.0        Good           605.03   \n",
       "...                     ...         ...              ...   \n",
       "49995                  12.0           _           3571.7   \n",
       "49996                   7.0        Good           502.38   \n",
       "49997                   7.0        Good           502.38   \n",
       "49998                   7.0        Good           502.38   \n",
       "49999                   7.0           _           502.38   \n",
       "\n",
       "      Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                    35.030402   22 Years and 9 Months                    No   \n",
       "1                    33.053114  22 Years and 10 Months                    No   \n",
       "2                    33.811894                     NaN                    No   \n",
       "3                    32.430559   23 Years and 0 Months                    No   \n",
       "4                    25.926822   27 Years and 3 Months                    No   \n",
       "...                        ...                     ...                   ...   \n",
       "49995                34.780553                     NaN                   Yes   \n",
       "49996                27.758522  31 Years and 11 Months                    NM   \n",
       "49997                36.858542   32 Years and 0 Months                    No   \n",
       "49998                39.139840   32 Years and 1 Months                    No   \n",
       "49999                34.108530   32 Years and 2 Months                    No   \n",
       "\n",
       "      Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0               49.574949       236.64268203272135   \n",
       "1               49.574949       21.465380264657146   \n",
       "2               49.574949       148.23393788500925   \n",
       "3               49.574949        39.08251089460281   \n",
       "4               18.816215       39.684018417945296   \n",
       "...                   ...                      ...   \n",
       "49995           60.964772       146.48632477751087   \n",
       "49996           35.104023       181.44299902757518   \n",
       "49997           35.104023                __10000__   \n",
       "49998           35.104023        97.59857973344877   \n",
       "49999           35.104023       220.45787812168732   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance  \n",
       "0        Low_spent_Small_value_payments  186.26670208571772  \n",
       "1      High_spent_Medium_value_payments  361.44400385378196  \n",
       "2       Low_spent_Medium_value_payments  264.67544623342997  \n",
       "3      High_spent_Medium_value_payments  343.82687322383634  \n",
       "4       High_spent_Large_value_payments   485.2984336755923  \n",
       "...                                 ...                 ...  \n",
       "49995    Low_spent_Small_value_payments  275.53956951573343  \n",
       "49996    Low_spent_Small_value_payments  409.39456169535066  \n",
       "49997    Low_spent_Large_value_payments   349.7263321025098  \n",
       "49998   High_spent_Small_value_payments  463.23898098947717  \n",
       "49999   Low_spent_Medium_value_payments  360.37968260123847  \n",
       "\n",
       "[50000 rows x 27 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in CSV to df\n",
    "\n",
    "file_path = Path(\"test.csv\")\n",
    "df_test = pd.read_csv(file_path)\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "General Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                             0\n",
       "Customer_ID                    0\n",
       "Month                          0\n",
       "Name                        5015\n",
       "Age                            0\n",
       "SSN                            0\n",
       "Occupation                     0\n",
       "Annual_Income                  0\n",
       "Monthly_Inhand_Salary       7498\n",
       "Num_Bank_Accounts              0\n",
       "Num_Credit_Card                0\n",
       "Interest_Rate                  0\n",
       "Num_of_Loan                    0\n",
       "Type_of_Loan                5704\n",
       "Delay_from_due_date            0\n",
       "Num_of_Delayed_Payment      3498\n",
       "Changed_Credit_Limit           0\n",
       "Num_Credit_Inquiries        1035\n",
       "Credit_Mix                     0\n",
       "Outstanding_Debt               0\n",
       "Credit_Utilization_Ratio       0\n",
       "Credit_History_Age          4470\n",
       "Payment_of_Min_Amount          0\n",
       "Total_EMI_per_month            0\n",
       "Amount_invested_monthly     2271\n",
       "Payment_Behaviour              0\n",
       "Monthly_Balance              562\n",
       "dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dtm # of NaN values in each column\n",
    "list_of_columns = ['ID', 'Customer_ID', 'Month', 'Name', 'Age', 'SSN', 'Occupation', 'Annual_Income', 'Monthly_Inhand_Salary',\n",
    "                   'Num_Bank_Accounts', 'Num_Credit_Card', 'Interest_Rate', 'Num_of_Loan', 'Type_of_Loan', 'Delay_from_due_date',\n",
    "                   'Num_of_Delayed_Payment', 'Changed_Credit_Limit', 'Num_Credit_Inquiries', 'Credit_Mix', 'Outstanding_Debt',\n",
    "                   'Credit_Utilization_Ratio', 'Credit_History_Age', 'Payment_of_Min_Amount', 'Total_EMI_per_month', 'Amount_invested_monthly',\n",
    "                   'Payment_Behaviour', 'Monthly_Balance']\n",
    "\n",
    "\n",
    "nan_counts = df_test[list_of_columns].isna().sum()\n",
    "nan_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.030402</td>\n",
       "      <td>22 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.053114</td>\n",
       "      <td>22 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>4</td>\n",
       "      <td>12.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.811894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>148.23393788500925</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>264.67544623342997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24_</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.430559</td>\n",
       "      <td>23 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.926822</td>\n",
       "      <td>27 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>39.684018417945296</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.2984336755923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>4975</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>_</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>34.780553</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>146.48632477751087</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>275.53956951573343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2_</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>27.758522</td>\n",
       "      <td>31 Years and 11 Months</td>\n",
       "      <td>NM</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>181.44299902757518</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>409.39456169535066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>13.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.858542</td>\n",
       "      <td>32 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>__10000__</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.7263321025098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2_</td>\n",
       "      <td>21</td>\n",
       "      <td>6_</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.139840</td>\n",
       "      <td>32 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>97.59857973344877</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.23898098947717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.108530</td>\n",
       "      <td>32 Years and 2 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>220.45787812168732</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.37968260123847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0        23  Scientist      19114.12                  3                4   \n",
       "1        24  Scientist      19114.12                  3                4   \n",
       "2        24  Scientist      19114.12                  3                4   \n",
       "3       24_  Scientist      19114.12                  3                4   \n",
       "4        28    _______      34847.84                  2                4   \n",
       "...     ...        ...           ...                ...              ...   \n",
       "49995  4975  Architect      20002.88                 10                8   \n",
       "49996    25   Mechanic      39628.99                  4                6   \n",
       "49997    25   Mechanic      39628.99                  4                6   \n",
       "49998    25   Mechanic      39628.99                  4                6   \n",
       "49999    25   Mechanic      39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "1                  3           4                    3                      9   \n",
       "2                  3           4                   -1                      4   \n",
       "3                  3           4                    4                      5   \n",
       "4                  6           1                    3                      1   \n",
       "...              ...         ...                  ...                    ...   \n",
       "49995             29           5                   33                     25   \n",
       "49996              7          2_                   20                    NaN   \n",
       "49997              7           2                   23                      5   \n",
       "49998              7          2_                   21                     6_   \n",
       "49999              7           2                   22                      5   \n",
       "\n",
       "      Changed_Credit_Limit  Num_Credit_Inquiries Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27                2022.0       Good           809.98   \n",
       "1                    13.27                   4.0       Good           809.98   \n",
       "2                    12.27                   4.0       Good           809.98   \n",
       "3                    11.27                   4.0       Good           809.98   \n",
       "4                     5.42                   5.0       Good           605.03   \n",
       "...                    ...                   ...        ...              ...   \n",
       "49995                18.31                  12.0          _           3571.7   \n",
       "49996                 11.5                   7.0       Good           502.38   \n",
       "49997                 13.5                   7.0       Good           502.38   \n",
       "49998                 11.5                   7.0       Good           502.38   \n",
       "49999                 11.5                   7.0          _           502.38   \n",
       "\n",
       "       Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                     35.030402   22 Years and 9 Months                    No   \n",
       "1                     33.053114  22 Years and 10 Months                    No   \n",
       "2                     33.811894                     NaN                    No   \n",
       "3                     32.430559   23 Years and 0 Months                    No   \n",
       "4                     25.926822   27 Years and 3 Months                    No   \n",
       "...                         ...                     ...                   ...   \n",
       "49995                 34.780553                     NaN                   Yes   \n",
       "49996                 27.758522  31 Years and 11 Months                    NM   \n",
       "49997                 36.858542   32 Years and 0 Months                    No   \n",
       "49998                 39.139840   32 Years and 1 Months                    No   \n",
       "49999                 34.108530   32 Years and 2 Months                    No   \n",
       "\n",
       "       Total_EMI_per_month Amount_invested_monthly  \\\n",
       "0                49.574949      236.64268203272135   \n",
       "1                49.574949      21.465380264657146   \n",
       "2                49.574949      148.23393788500925   \n",
       "3                49.574949       39.08251089460281   \n",
       "4                18.816215      39.684018417945296   \n",
       "...                    ...                     ...   \n",
       "49995            60.964772      146.48632477751087   \n",
       "49996            35.104023      181.44299902757518   \n",
       "49997            35.104023               __10000__   \n",
       "49998            35.104023       97.59857973344877   \n",
       "49999            35.104023      220.45787812168732   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance  \n",
       "0        Low_spent_Small_value_payments  186.26670208571772  \n",
       "1      High_spent_Medium_value_payments  361.44400385378196  \n",
       "2       Low_spent_Medium_value_payments  264.67544623342997  \n",
       "3      High_spent_Medium_value_payments  343.82687322383634  \n",
       "4       High_spent_Large_value_payments   485.2984336755923  \n",
       "...                                 ...                 ...  \n",
       "49995    Low_spent_Small_value_payments  275.53956951573343  \n",
       "49996    Low_spent_Small_value_payments  409.39456169535066  \n",
       "49997    Low_spent_Large_value_payments   349.7263321025098  \n",
       "49998   High_spent_Small_value_payments  463.23898098947717  \n",
       "49999   Low_spent_Medium_value_payments  360.37968260123847  \n",
       "\n",
       "[50000 rows x 20 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop duplicate or unnecessary info for classsification model\n",
    "# Name and SSN are duplicate due to Customer_ID\n",
    "columns_to_drop = ['Month', 'Name', 'SSN', 'Monthly_Inhand_Salary', 'Type_of_Loan', 'Customer_ID', 'ID']\n",
    "df_test.drop(columns=columns_to_drop, inplace=True)\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.030402</td>\n",
       "      <td>22 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.053114</td>\n",
       "      <td>22 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24_</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.430559</td>\n",
       "      <td>23 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.926822</td>\n",
       "      <td>27 Years and 3 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>39.684018417945296</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.2984336755923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.116600</td>\n",
       "      <td>27 Years and 4 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>251.62736875017606</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.3550833433617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>37.528511</td>\n",
       "      <td>6 Years and 5 Months</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>71.79442082882734</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23147346441687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>27.027812</td>\n",
       "      <td>6 Years and 6 Months</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>50.84684680498023</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.179047488264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>13.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.858542</td>\n",
       "      <td>32 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>__10000__</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.7263321025098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2_</td>\n",
       "      <td>21</td>\n",
       "      <td>6_</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.139840</td>\n",
       "      <td>32 Years and 1 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>97.59857973344877</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.23898098947717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.108530</td>\n",
       "      <td>32 Years and 2 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>220.45787812168732</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.37968260123847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39156 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist      19114.12                  3                4   \n",
       "1       24  Scientist      19114.12                  3                4   \n",
       "3      24_  Scientist      19114.12                  3                4   \n",
       "4       28    _______      34847.84                  2                4   \n",
       "5       28    Teacher      34847.84                  2                4   \n",
       "...    ...        ...           ...                ...              ...   \n",
       "49993   29  Architect      20002.88                 10                8   \n",
       "49994   29    _______      20002.88                 10                8   \n",
       "49997   25   Mechanic      39628.99                  4                6   \n",
       "49998   25   Mechanic      39628.99                  4                6   \n",
       "49999   25   Mechanic      39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "1                  3           4                    3                      9   \n",
       "3                  3           4                    4                      5   \n",
       "4                  6           1                    3                      1   \n",
       "5                  6           1                    3                      3   \n",
       "...              ...         ...                  ...                    ...   \n",
       "49993             29           5                   33                     25   \n",
       "49994             29           5                   33                     22   \n",
       "49997              7           2                   23                      5   \n",
       "49998              7          2_                   21                     6_   \n",
       "49999              7           2                   22                      5   \n",
       "\n",
       "      Changed_Credit_Limit  Num_Credit_Inquiries Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27                2022.0       Good           809.98   \n",
       "1                    13.27                   4.0       Good           809.98   \n",
       "3                    11.27                   4.0       Good           809.98   \n",
       "4                     5.42                   5.0       Good           605.03   \n",
       "5                     5.42                   5.0       Good           605.03   \n",
       "...                    ...                   ...        ...              ...   \n",
       "49993                18.31                  12.0        Bad           3571.7   \n",
       "49994                18.31                  12.0        Bad           3571.7   \n",
       "49997                 13.5                   7.0       Good           502.38   \n",
       "49998                 11.5                   7.0       Good           502.38   \n",
       "49999                 11.5                   7.0          _           502.38   \n",
       "\n",
       "       Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                     35.030402   22 Years and 9 Months                    No   \n",
       "1                     33.053114  22 Years and 10 Months                    No   \n",
       "3                     32.430559   23 Years and 0 Months                    No   \n",
       "4                     25.926822   27 Years and 3 Months                    No   \n",
       "5                     30.116600   27 Years and 4 Months                    No   \n",
       "...                         ...                     ...                   ...   \n",
       "49993                 37.528511    6 Years and 5 Months                   Yes   \n",
       "49994                 27.027812    6 Years and 6 Months                   Yes   \n",
       "49997                 36.858542   32 Years and 0 Months                    No   \n",
       "49998                 39.139840   32 Years and 1 Months                    No   \n",
       "49999                 34.108530   32 Years and 2 Months                    No   \n",
       "\n",
       "       Total_EMI_per_month Amount_invested_monthly  \\\n",
       "0                49.574949      236.64268203272135   \n",
       "1                49.574949      21.465380264657146   \n",
       "3                49.574949       39.08251089460281   \n",
       "4                18.816215      39.684018417945296   \n",
       "5                18.816215      251.62736875017606   \n",
       "...                    ...                     ...   \n",
       "49993            60.964772       71.79442082882734   \n",
       "49994            60.964772       50.84684680498023   \n",
       "49997            35.104023               __10000__   \n",
       "49998            35.104023       97.59857973344877   \n",
       "49999            35.104023      220.45787812168732   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance  \n",
       "0        Low_spent_Small_value_payments  186.26670208571772  \n",
       "1      High_spent_Medium_value_payments  361.44400385378196  \n",
       "3      High_spent_Medium_value_payments  343.82687322383634  \n",
       "4       High_spent_Large_value_payments   485.2984336755923  \n",
       "5        Low_spent_Large_value_payments   303.3550833433617  \n",
       "...                                 ...                 ...  \n",
       "49993    Low_spent_Small_value_payments  350.23147346441687  \n",
       "49994   High_spent_Small_value_payments    341.179047488264  \n",
       "49997    Low_spent_Large_value_payments   349.7263321025098  \n",
       "49998   High_spent_Small_value_payments  463.23898098947717  \n",
       "49999   Low_spent_Medium_value_payments  360.37968260123847  \n",
       "\n",
       "[39156 rows x 20 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop any row with null value NaN\n",
    "df_test.dropna(inplace=True)\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22 Years and 9 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22 Years and 10 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24_</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23 Years and 0 Months</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0   23  Scientist      19114.12                  3                4   \n",
       "1   24  Scientist      19114.12                  3                4   \n",
       "3  24_  Scientist      19114.12                  3                4   \n",
       "\n",
       "   Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0              3           4                    3                      7   \n",
       "1              3           4                    3                      9   \n",
       "3              3           4                    4                      5   \n",
       "\n",
       "  Changed_Credit_Limit  Num_Credit_Inquiries Credit_Mix Outstanding_Debt  \\\n",
       "0                11.27                2022.0       Good           809.98   \n",
       "1                13.27                   4.0       Good           809.98   \n",
       "3                11.27                   4.0       Good           809.98   \n",
       "\n",
       "   Credit_Utilization_Ratio      Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                     35.03   22 Years and 9 Months                    No   \n",
       "1                     33.05  22 Years and 10 Months                    No   \n",
       "3                     32.43   23 Years and 0 Months                    No   \n",
       "\n",
       "   Total_EMI_per_month Amount_invested_monthly  \\\n",
       "0            49.574949      236.64268203272135   \n",
       "1            49.574949      21.465380264657146   \n",
       "3            49.574949       39.08251089460281   \n",
       "\n",
       "                  Payment_Behaviour     Monthly_Balance  \n",
       "0    Low_spent_Small_value_payments  186.26670208571772  \n",
       "1  High_spent_Medium_value_payments  361.44400385378196  \n",
       "3  High_spent_Medium_value_payments  343.82687322383634  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clean Credit_Utilization_Ratio\n",
    "\n",
    "# cast as int & round\n",
    "df_test[\"Credit_Utilization_Ratio\"] = df_test[\"Credit_Utilization_Ratio\"].astype(float)\n",
    "df_test[\"Credit_Utilization_Ratio\"] = df_test[\"Credit_Utilization_Ratio\"].round(2)\n",
    "\n",
    "df_test.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24_</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0   23  Scientist      19114.12                  3                4   \n",
       "1   24  Scientist      19114.12                  3                4   \n",
       "3  24_  Scientist      19114.12                  3                4   \n",
       "\n",
       "   Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0              3           4                    3                      7   \n",
       "1              3           4                    3                      9   \n",
       "3              3           4                    4                      5   \n",
       "\n",
       "  Changed_Credit_Limit  Num_Credit_Inquiries Credit_Mix Outstanding_Debt  \\\n",
       "0                11.27                2022.0       Good           809.98   \n",
       "1                13.27                   4.0       Good           809.98   \n",
       "3                11.27                   4.0       Good           809.98   \n",
       "\n",
       "   Credit_Utilization_Ratio  Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                     35.03                22.8                    No   \n",
       "1                     33.05                22.8                    No   \n",
       "3                     32.43                23.0                    No   \n",
       "\n",
       "   Total_EMI_per_month Amount_invested_monthly  \\\n",
       "0            49.574949      236.64268203272135   \n",
       "1            49.574949      21.465380264657146   \n",
       "3            49.574949       39.08251089460281   \n",
       "\n",
       "                  Payment_Behaviour     Monthly_Balance  \n",
       "0    Low_spent_Small_value_payments  186.26670208571772  \n",
       "1  High_spent_Medium_value_payments  361.44400385378196  \n",
       "3  High_spent_Medium_value_payments  343.82687322383634  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert credit history to numerical\n",
    "\n",
    "# Function to convert age string to total years\n",
    "def convert_age_to_years(age_str):\n",
    "    years, months = map(str.strip, age_str.split('Years and'))\n",
    "    total_months = int(years) * 12 + int(months.split()[0])\n",
    "    total_years = total_months / 12  # Convert total months to years\n",
    "    return round(total_years, 1)  # Round to 1 decimal place\n",
    "\n",
    "# Convert age strings to total years\n",
    "df_test['Credit_History_Age'] = df_test['Credit_History_Age'].apply(convert_age_to_years).astype(float)\n",
    "\n",
    "df_test.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.64268203272135</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.26670208571772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380264657146</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44400385378196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24_</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.08251089460281</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.82687322383634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>39.684018417945296</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.2984336755923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>251.62736875017606</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.3550833433617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>71.79442082882734</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23147346441687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.7</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>50.84684680498023</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.179047488264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>13.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.86</td>\n",
       "      <td>32.0</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>__10000__</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.7263321025098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2_</td>\n",
       "      <td>21</td>\n",
       "      <td>6_</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>97.59857973344877</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.23898098947717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>220.45787812168732</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.37968260123847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39156 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist      19114.12                  3                4   \n",
       "1       24  Scientist      19114.12                  3                4   \n",
       "3      24_  Scientist      19114.12                  3                4   \n",
       "4       28    _______      34847.84                  2                4   \n",
       "5       28    Teacher      34847.84                  2                4   \n",
       "...    ...        ...           ...                ...              ...   \n",
       "49993   29  Architect      20002.88                 10                8   \n",
       "49994   29    _______      20002.88                 10                8   \n",
       "49997   25   Mechanic      39628.99                  4                6   \n",
       "49998   25   Mechanic      39628.99                  4                6   \n",
       "49999   25   Mechanic      39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate Num_of_Loan  Delay_from_due_date Num_of_Delayed_Payment  \\\n",
       "0                  3           4                    3                      7   \n",
       "1                  3           4                    3                      9   \n",
       "3                  3           4                    4                      5   \n",
       "4                  6           1                    3                      1   \n",
       "5                  6           1                    3                      3   \n",
       "...              ...         ...                  ...                    ...   \n",
       "49993             29           5                   33                     25   \n",
       "49994             29           5                   33                     22   \n",
       "49997              7           2                   23                      5   \n",
       "49998              7          2_                   21                     6_   \n",
       "49999              7           2                   22                      5   \n",
       "\n",
       "      Changed_Credit_Limit  Num_Credit_Inquiries Credit_Mix Outstanding_Debt  \\\n",
       "0                    11.27                2022.0       Good           809.98   \n",
       "1                    13.27                   4.0       Good           809.98   \n",
       "3                    11.27                   4.0       Good           809.98   \n",
       "4                     5.42                   5.0       Good           605.03   \n",
       "5                     5.42                   5.0       Good           605.03   \n",
       "...                    ...                   ...        ...              ...   \n",
       "49993                18.31                  12.0        Bad           3571.7   \n",
       "49994                18.31                  12.0        Bad           3571.7   \n",
       "49997                 13.5                   7.0       Good           502.38   \n",
       "49998                 11.5                   7.0       Good           502.38   \n",
       "49999                 11.5                   7.0          _           502.38   \n",
       "\n",
       "       Credit_Utilization_Ratio  Credit_History_Age Payment_of_Min_Amount  \\\n",
       "0                         35.03                22.8                    No   \n",
       "1                         33.05                22.8                    No   \n",
       "3                         32.43                23.0                    No   \n",
       "4                         25.93                27.2                    No   \n",
       "5                         30.12                27.3                    No   \n",
       "...                         ...                 ...                   ...   \n",
       "49993                     37.53                 6.4                   Yes   \n",
       "49994                     27.03                 6.5                   Yes   \n",
       "49997                     36.86                32.0                    No   \n",
       "49998                     39.14                32.1                    No   \n",
       "49999                     34.11                32.2                    No   \n",
       "\n",
       "       Total_EMI_per_month Amount_invested_monthly  \\\n",
       "0                49.574949      236.64268203272135   \n",
       "1                49.574949      21.465380264657146   \n",
       "3                49.574949       39.08251089460281   \n",
       "4                18.816215      39.684018417945296   \n",
       "5                18.816215      251.62736875017606   \n",
       "...                    ...                     ...   \n",
       "49993            60.964772       71.79442082882734   \n",
       "49994            60.964772       50.84684680498023   \n",
       "49997            35.104023               __10000__   \n",
       "49998            35.104023       97.59857973344877   \n",
       "49999            35.104023      220.45787812168732   \n",
       "\n",
       "                      Payment_Behaviour     Monthly_Balance  \n",
       "0        Low_spent_Small_value_payments  186.26670208571772  \n",
       "1      High_spent_Medium_value_payments  361.44400385378196  \n",
       "3      High_spent_Medium_value_payments  343.82687322383634  \n",
       "4       High_spent_Large_value_payments   485.2984336755923  \n",
       "5        Low_spent_Large_value_payments   303.3550833433617  \n",
       "...                                 ...                 ...  \n",
       "49993    Low_spent_Small_value_payments  350.23147346441687  \n",
       "49994   High_spent_Small_value_payments    341.179047488264  \n",
       "49997    Low_spent_Large_value_payments   349.7263321025098  \n",
       "49998   High_spent_Small_value_payments  463.23898098947717  \n",
       "49999   Low_spent_Medium_value_payments  360.37968260123847  \n",
       "\n",
       "[39156 rows x 20 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use Regular Expressions to Remove Typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>236.642682</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.266702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>21.465380</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.444004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>39.082511</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.826873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>39.684018</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.298434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.816215</td>\n",
       "      <td>251.627369</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.355083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>71.794421</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.231473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.964772</td>\n",
       "      <td>50.846847</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.179047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.86</td>\n",
       "      <td>32.0</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>10000.000000</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.726332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>97.598580</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.238981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>220.457878</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.379683</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39156 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49997   25   Mechanic       39628.99                  4                6   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49997              7          2.0                 23.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                11.27                2022.0   \n",
       "1                         9.0                13.27                   4.0   \n",
       "3                         5.0                11.27                   4.0   \n",
       "4                         1.0                 5.42                   5.0   \n",
       "5                         3.0                 5.42                   5.0   \n",
       "...                       ...                  ...                   ...   \n",
       "49993                    25.0                18.31                  12.0   \n",
       "49994                    22.0                18.31                  12.0   \n",
       "49997                     5.0                 13.5                   7.0   \n",
       "49998                     6.0                 11.5                   7.0   \n",
       "49999                     5.0                 11.5                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49997       Good            502.38                     36.86   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No            49.574949   \n",
       "1                    22.8                    No            49.574949   \n",
       "3                    23.0                    No            49.574949   \n",
       "4                    27.2                    No            18.816215   \n",
       "5                    27.3                    No            18.816215   \n",
       "...                   ...                   ...                  ...   \n",
       "49993                 6.4                   Yes            60.964772   \n",
       "49994                 6.5                   Yes            60.964772   \n",
       "49997                32.0                    No            35.104023   \n",
       "49998                32.1                    No            35.104023   \n",
       "49999                32.2                    No            35.104023   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                   236.642682    Low_spent_Small_value_payments   \n",
       "1                    21.465380  High_spent_Medium_value_payments   \n",
       "3                    39.082511  High_spent_Medium_value_payments   \n",
       "4                    39.684018   High_spent_Large_value_payments   \n",
       "5                   251.627369    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49993                71.794421    Low_spent_Small_value_payments   \n",
       "49994                50.846847   High_spent_Small_value_payments   \n",
       "49997             10000.000000    Low_spent_Large_value_payments   \n",
       "49998                97.598580   High_spent_Small_value_payments   \n",
       "49999               220.457878   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0           186.266702  \n",
       "1           361.444004  \n",
       "3           343.826873  \n",
       "4           485.298434  \n",
       "5           303.355083  \n",
       "...                ...  \n",
       "49993       350.231473  \n",
       "49994       341.179047  \n",
       "49997       349.726332  \n",
       "49998       463.238981  \n",
       "49999       360.379683  \n",
       "\n",
       "[39156 rows x 20 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns_to_regex = ['Annual_Income', 'Age', 'Delay_from_due_date', 'Num_of_Loan','Amount_invested_monthly', 'Monthly_Balance', 'Num_of_Delayed_Payment', 'Outstanding_Debt']\n",
    "\n",
    "for column in columns_to_regex:\n",
    "    df_test[column] = df_test[column].astype(str).str.replace('_', '')\n",
    "    df_test[column] = df_test[column].astype(float)  # Convert the column back to float\n",
    "\n",
    "df_test['Age'] = df_test['Age'].astype(int)\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 39156 entries, 0 to 49999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   Age                       39156 non-null  int64  \n",
      " 1   Occupation                39156 non-null  object \n",
      " 2   Annual_Income             39156 non-null  float64\n",
      " 3   Num_Bank_Accounts         39156 non-null  int64  \n",
      " 4   Num_Credit_Card           39156 non-null  int64  \n",
      " 5   Interest_Rate             39156 non-null  int64  \n",
      " 6   Num_of_Loan               39156 non-null  float64\n",
      " 7   Delay_from_due_date       39156 non-null  float64\n",
      " 8   Num_of_Delayed_Payment    39156 non-null  float64\n",
      " 9   Changed_Credit_Limit      39156 non-null  object \n",
      " 10  Num_Credit_Inquiries      39156 non-null  float64\n",
      " 11  Credit_Mix                39156 non-null  object \n",
      " 12  Outstanding_Debt          39156 non-null  float64\n",
      " 13  Credit_Utilization_Ratio  39156 non-null  float64\n",
      " 14  Credit_History_Age        39156 non-null  float64\n",
      " 15  Payment_of_Min_Amount     39156 non-null  object \n",
      " 16  Total_EMI_per_month       39156 non-null  float64\n",
      " 17  Amount_invested_monthly   39156 non-null  float64\n",
      " 18  Payment_Behaviour         39156 non-null  object \n",
      " 19  Monthly_Balance           39156 non-null  float64\n",
      "dtypes: float64(11), int64(4), object(5)\n",
      "memory usage: 6.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace nonnumeric values with numeric values (to allow casting of variables as float in next block)\n",
    "df_test['Changed_Credit_Limit'] = df_test['Changed_Credit_Limit'].replace('-', '')\n",
    "df_test['Changed_Credit_Limit'] = pd.to_numeric(df_test['Changed_Credit_Limit'], errors='coerce')\n",
    "df_test['Changed_Credit_Limit'] = df_test['Changed_Credit_Limit'].fillna(0) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.86</td>\n",
       "      <td>32.0</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>10000.00</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39156 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49997   25   Mechanic       39628.99                  4                6   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49997              7          2.0                 23.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49997                     5.0                 13.50                   7.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49997       Good            502.38                     36.86   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49997                32.0                    No                35.10   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49997                 10000.00    Low_spent_Large_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49997           349.73  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[39156 rows x 20 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of columns to round and cast as floats\n",
    "columns_to_round = ['Num_of_Loan', 'Delay_from_due_date', 'Amount_invested_monthly', 'Monthly_Balance', 'Changed_Credit_Limit', 'Total_EMI_per_month']\n",
    "\n",
    "# Round the columns to 2 decimal places and cast them as floats\n",
    "df_test[columns_to_round] = df_test[columns_to_round].round(2).astype(float)\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 39156 entries, 0 to 49999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   Age                       39156 non-null  int64  \n",
      " 1   Occupation                39156 non-null  object \n",
      " 2   Annual_Income             39156 non-null  float64\n",
      " 3   Num_Bank_Accounts         39156 non-null  int64  \n",
      " 4   Num_Credit_Card           39156 non-null  int64  \n",
      " 5   Interest_Rate             39156 non-null  int64  \n",
      " 6   Num_of_Loan               39156 non-null  float64\n",
      " 7   Delay_from_due_date       39156 non-null  float64\n",
      " 8   Num_of_Delayed_Payment    39156 non-null  float64\n",
      " 9   Changed_Credit_Limit      39156 non-null  float64\n",
      " 10  Num_Credit_Inquiries      39156 non-null  float64\n",
      " 11  Credit_Mix                39156 non-null  object \n",
      " 12  Outstanding_Debt          39156 non-null  float64\n",
      " 13  Credit_Utilization_Ratio  39156 non-null  float64\n",
      " 14  Credit_History_Age        39156 non-null  float64\n",
      " 15  Payment_of_Min_Amount     39156 non-null  object \n",
      " 16  Total_EMI_per_month       39156 non-null  float64\n",
      " 17  Amount_invested_monthly   39156 non-null  float64\n",
      " 18  Payment_Behaviour         39156 non-null  object \n",
      " 19  Monthly_Balance           39156 non-null  float64\n",
      "dtypes: float64(12), int64(4), object(4)\n",
      "memory usage: 6.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        11.27\n",
       "1        13.27\n",
       "3        11.27\n",
       "4         5.42\n",
       "5         5.42\n",
       "         ...  \n",
       "49993    18.31\n",
       "49994    18.31\n",
       "49997    13.50\n",
       "49998    11.50\n",
       "49999    11.50\n",
       "Name: Changed_Credit_Limit, Length: 39156, dtype: float64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['Changed_Credit_Limit']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Bank_Accounts\n",
       " 6       5062\n",
       " 8       5007\n",
       " 7       4982\n",
       " 5       4792\n",
       " 4       4789\n",
       " 3       4631\n",
       " 9       2157\n",
       " 10      2015\n",
       " 1       1760\n",
       " 2       1737\n",
       " 0       1691\n",
       " 11        16\n",
       "-1         11\n",
       " 1542       4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of bank acct outliers\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test['Num_Bank_Accounts'].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find and remove outlier, typo rows for # of bank accts\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 15\n",
    "\n",
    "column = 'Num_Bank_Accounts'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and see where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Credit_Card\n",
       "5       7106\n",
       "7       6403\n",
       "6       6368\n",
       "4       5451\n",
       "3       5079\n",
       "8       1943\n",
       "10      1830\n",
       "9       1815\n",
       "2        880\n",
       "1        843\n",
       "11        34\n",
       "0         10\n",
       "537        4\n",
       "338        4\n",
       "715        4\n",
       "910        4\n",
       "1203       4\n",
       "1250       4\n",
       "41         4\n",
       "571        4\n",
       "591        4\n",
       "211        4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of credit card outliers\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test['Num_Credit_Card'].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find and remove outlier, typo rows by # of credit cards\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 16\n",
    "\n",
    "column = 'Num_Credit_Card'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and see where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Interest_Rate\n",
       "5     1905\n",
       "8     1862\n",
       "6     1786\n",
       "12    1741\n",
       "10    1721\n",
       "9     1711\n",
       "7     1700\n",
       "11    1635\n",
       "18    1553\n",
       "15    1534\n",
       "20    1472\n",
       "17    1446\n",
       "16    1419\n",
       "19    1363\n",
       "3     1057\n",
       "1     1020\n",
       "4      954\n",
       "2      947\n",
       "13     885\n",
       "14     866\n",
       "32     666\n",
       "22     655\n",
       "28     633\n",
       "24     626\n",
       "30     621\n",
       "23     619\n",
       "29     611\n",
       "21     611\n",
       "25     601\n",
       "27     590\n",
       "26     567\n",
       "34     550\n",
       "31     545\n",
       "33     538\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of interest rate\n",
    "\n",
    "column = 'Interest_Rate'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.86</td>\n",
       "      <td>32.0</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>10000.00</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>37010 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49997   25   Mechanic       39628.99                  4                6   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49997              7          2.0                 23.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49997                     5.0                 13.50                   7.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49997       Good            502.38                     36.86   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49997                32.0                    No                35.10   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49997                 10000.00    Low_spent_Large_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49997           349.73  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[37010 rows x 20 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows by interest rate\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 40\n",
    "\n",
    "column = 'Interest_Rate'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age\n",
       " 39     1151\n",
       " 35     1114\n",
       " 32     1113\n",
       " 44     1101\n",
       " 22     1089\n",
       " 27     1081\n",
       " 29     1076\n",
       " 36     1074\n",
       " 20     1071\n",
       " 37     1069\n",
       " 41     1064\n",
       " 28     1046\n",
       " 25     1045\n",
       " 26     1040\n",
       " 30     1038\n",
       " 24     1034\n",
       " 19      997\n",
       " 42      997\n",
       " 33      985\n",
       " 31      984\n",
       " 21      979\n",
       " 38      966\n",
       " 23      948\n",
       " 34      940\n",
       " 45      936\n",
       " 40      920\n",
       " 43      882\n",
       " 46      877\n",
       " 18      607\n",
       " 17      567\n",
       " 15      567\n",
       " 16      559\n",
       " 49      521\n",
       " 51      511\n",
       " 47      509\n",
       " 55      506\n",
       " 53      503\n",
       " 50      502\n",
       " 54      502\n",
       " 48      461\n",
       " 52      457\n",
       " 56      425\n",
       "-500     357\n",
       " 14      139\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of age\n",
    "\n",
    "column = 'Age'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>36.86</td>\n",
       "      <td>32.0</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>10000.00</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>349.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>34121 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49997   25   Mechanic       39628.99                  4                6   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49997              7          2.0                 23.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49997                     5.0                 13.50                   7.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49997       Good            502.38                     36.86   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49997                32.0                    No                35.10   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49997                 10000.00    Low_spent_Large_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49997           349.73  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[34121 rows x 20 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Age\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 18\n",
    "upper_bound = 90\n",
    "\n",
    "column = 'Age'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Payment_Behaviour\n",
       "Low_spent_Small_value_payments      8388\n",
       "High_spent_Medium_value_payments    6120\n",
       "High_spent_Large_value_payments     4808\n",
       "Low_spent_Medium_value_payments     4656\n",
       "High_spent_Small_value_payments     3941\n",
       "Low_spent_Large_value_payments      3635\n",
       "!@9#%8                              2573\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Payment_Behaviour\n",
    "\n",
    "column = 'Payment_Behaviour'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_test[df_test['Payment_Behaviour'] != '!@9#%8']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Amount_invested_monthly\n",
       "10000.00    1456\n",
       "0.00          63\n",
       "98.41          7\n",
       "92.56          6\n",
       "86.25          6\n",
       "            ... \n",
       "61.18          4\n",
       "91.32          4\n",
       "101.80         4\n",
       "22.38          4\n",
       "74.81          4\n",
       "Name: count, Length: 620, dtype: int64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Amount_invested_monthly\n",
    "\n",
    "column = 'Amount_invested_monthly'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30092 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49992   29  Architect       20002.88                 10                8   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49992             29          5.0                 33.0   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                    25.0                 18.31                   9.0   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[30092 rows x 20 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Amount_invested_monthly\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 9000\n",
    "\n",
    "column = 'Amount_invested_monthly'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_of_Loan\n",
       " 2.0      4617\n",
       " 3.0      4603\n",
       " 4.0      4467\n",
       " 0.0      3489\n",
       " 1.0      3342\n",
       " 6.0      2188\n",
       " 5.0      2093\n",
       " 7.0      2047\n",
       "-100.0    1169\n",
       " 9.0      1014\n",
       " 8.0       912\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Num_of_Loan\n",
    "\n",
    "column = 'Num_of_Loan'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28772 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49992   29  Architect       20002.88                 10                8   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49992             29          5.0                 33.0   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                    25.0                 18.31                   9.0   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[28772 rows x 20 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Num_of_Loan\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 10\n",
    "\n",
    "column = 'Num_of_Loan'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_of_Delayed_Payment\n",
       " 19.0    1706\n",
       " 12.0    1624\n",
       " 10.0    1618\n",
       " 15.0    1606\n",
       " 18.0    1606\n",
       " 16.0    1587\n",
       " 11.0    1575\n",
       " 17.0    1567\n",
       " 9.0     1555\n",
       " 8.0     1549\n",
       " 20.0    1514\n",
       " 14.0    1268\n",
       " 13.0    1240\n",
       " 21.0     816\n",
       " 7.0      778\n",
       " 6.0      737\n",
       " 22.0     689\n",
       " 5.0      651\n",
       " 3.0      616\n",
       " 23.0     608\n",
       " 4.0      601\n",
       " 2.0      589\n",
       " 1.0      556\n",
       " 0.0      556\n",
       " 24.0     497\n",
       " 25.0     469\n",
       " 26.0      84\n",
       "-1.0       82\n",
       "-2.0       66\n",
       " 27.0      63\n",
       " 28.0      35\n",
       "-3.0       31\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Num_of_Delayed_Payment\n",
    "\n",
    "column = 'Num_of_Delayed_Payment'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>35.03</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>236.64</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>186.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>33.05</td>\n",
       "      <td>22.8</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>21.47</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>361.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>Scientist</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>809.98</td>\n",
       "      <td>32.43</td>\n",
       "      <td>23.0</td>\n",
       "      <td>No</td>\n",
       "      <td>49.57</td>\n",
       "      <td>39.08</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>343.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7.0</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28360 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "0       23  Scientist       19114.12                  3                4   \n",
       "1       24  Scientist       19114.12                  3                4   \n",
       "3       24  Scientist       19114.12                  3                4   \n",
       "4       28    _______       34847.84                  2                4   \n",
       "5       28    Teacher       34847.84                  2                4   \n",
       "...    ...        ...            ...                ...              ...   \n",
       "49992   29  Architect       20002.88                 10                8   \n",
       "49993   29  Architect       20002.88                 10                8   \n",
       "49994   29    _______       20002.88                 10                8   \n",
       "49998   25   Mechanic       39628.99                  4                6   \n",
       "49999   25   Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  3          4.0                  3.0   \n",
       "1                  3          4.0                  3.0   \n",
       "3                  3          4.0                  4.0   \n",
       "4                  6          1.0                  3.0   \n",
       "5                  6          1.0                  3.0   \n",
       "...              ...          ...                  ...   \n",
       "49992             29          5.0                 33.0   \n",
       "49993             29          5.0                 33.0   \n",
       "49994             29          5.0                 33.0   \n",
       "49998              7          2.0                 21.0   \n",
       "49999              7          2.0                 22.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                2022.0   \n",
       "1                         9.0                 13.27                   4.0   \n",
       "3                         5.0                 11.27                   4.0   \n",
       "4                         1.0                  5.42                   5.0   \n",
       "5                         3.0                  5.42                   5.0   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                    25.0                 18.31                   9.0   \n",
       "49993                    25.0                 18.31                  12.0   \n",
       "49994                    22.0                 18.31                  12.0   \n",
       "49998                     6.0                 11.50                   7.0   \n",
       "49999                     5.0                 11.50                   7.0   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0           Good            809.98                     35.03   \n",
       "1           Good            809.98                     33.05   \n",
       "3           Good            809.98                     32.43   \n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "0                    22.8                    No                49.57   \n",
       "1                    22.8                    No                49.57   \n",
       "3                    23.0                    No                49.57   \n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "0                       236.64    Low_spent_Small_value_payments   \n",
       "1                        21.47  High_spent_Medium_value_payments   \n",
       "3                        39.08  High_spent_Medium_value_payments   \n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "0               186.27  \n",
       "1               361.44  \n",
       "3               343.83  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[28360 rows x 20 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Num_of_Delayed_Payment\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 30\n",
    "column = 'Num_of_Delayed_Payment'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Payment_of_Min_Amount\n",
       "Yes    14335\n",
       "No     10591\n",
       "NM      3434\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Payment_of_Min_Amount\n",
    "\n",
    "column = 'Payment_of_Min_Amount'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop NM from Payment_of_Min_Amount\n",
    "df_test = df_test[df_test['Payment_of_Min_Amount'] != 'NM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Outstanding_Debt\n",
       "573.77     8\n",
       "1360.45    8\n",
       "413.08     7\n",
       "2530.46    7\n",
       "2646.00    7\n",
       "          ..\n",
       "1264.51    4\n",
       "413.34     4\n",
       "1082.13    4\n",
       "1219.91    4\n",
       "1163.09    4\n",
       "Name: count, Length: 1068, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Outstanding_Debt\n",
    "\n",
    "column = 'Outstanding_Debt'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_7382/171639993.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_test['Num_of_Delayed_Payment'] = df_test['Num_of_Delayed_Payment'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_7382/171639993.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_test['Num_Credit_Inquiries'] = df_test['Num_Credit_Inquiries'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_7382/171639993.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_test['Num_of_Loan'] = df_test['Num_of_Loan'].astype(int)\n",
      "/var/folders/zt/x756b70j3v75355fl48s3dzc0000gn/T/ipykernel_7382/171639993.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_test['Delay_from_due_date'] = df_test['Delay_from_due_date'].astype(int)\n"
     ]
    }
   ],
   "source": [
    "# cast some columns as int when appropriate\n",
    "\n",
    "df_test['Num_of_Delayed_Payment'] = df_test['Num_of_Delayed_Payment'].astype(int)\n",
    "df_test['Num_Credit_Inquiries'] = df_test['Num_Credit_Inquiries'].astype(int)\n",
    "df_test['Num_of_Loan'] = df_test['Num_of_Loan'].astype(int)\n",
    "df_test['Delay_from_due_date'] = df_test['Delay_from_due_date'].astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Delay_from_due_date\n",
       " 13    895\n",
       " 15    893\n",
       " 11    854\n",
       " 10    845\n",
       " 8     843\n",
       "      ... \n",
       " 64     14\n",
       " 63     12\n",
       " 65     12\n",
       "-5       7\n",
       " 67      6\n",
       "Name: count, Length: 72, dtype: int64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Delay_from_due_date\n",
    "\n",
    "column = 'Delay_from_due_date'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2.492600e+04\n",
       "mean     1.606978e+05\n",
       "std      1.314917e+06\n",
       "min      7.006035e+03\n",
       "25%      1.946292e+04\n",
       "50%      3.748569e+04\n",
       "75%      7.304456e+04\n",
       "max      2.413726e+07\n",
       "Name: Annual_Income, dtype: float64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Annual_Income\n",
    "\n",
    "column = 'Annual_Income'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].describe()\n",
    "values_counted\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>35</td>\n",
       "      <td>Engineer</td>\n",
       "      <td>143162.64</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2.10</td>\n",
       "      <td>3</td>\n",
       "      <td>Good</td>\n",
       "      <td>1303.01</td>\n",
       "      <td>35.69</td>\n",
       "      <td>18.5</td>\n",
       "      <td>No</td>\n",
       "      <td>246.99</td>\n",
       "      <td>453.62</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>788.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>39.38</td>\n",
       "      <td>17.9</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>69.11</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>425.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>36.82</td>\n",
       "      <td>18.1</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>191.89</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>332.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18030 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age    Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "4       28       _______       34847.84                  2                4   \n",
       "5       28       Teacher       34847.84                  2                4   \n",
       "9       35      Engineer      143162.64                  1                5   \n",
       "12      55  Entrepreneur       30689.89                  2                5   \n",
       "14      55  Entrepreneur       30689.89                  2                5   \n",
       "...    ...           ...            ...                ...              ...   \n",
       "49992   29     Architect       20002.88                 10                8   \n",
       "49993   29     Architect       20002.88                 10                8   \n",
       "49994   29       _______       20002.88                 10                8   \n",
       "49998   25      Mechanic       39628.99                  4                6   \n",
       "49999   25      Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "4                  6            1                    3   \n",
       "5                  6            1                    3   \n",
       "9                  8            3                    6   \n",
       "12                 4            1                    5   \n",
       "14                 4            1                    5   \n",
       "...              ...          ...                  ...   \n",
       "49992             29            5                   33   \n",
       "49993             29            5                   33   \n",
       "49994             29            5                   33   \n",
       "49998              7            2                   21   \n",
       "49999              7            2                   22   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "4                           1                  5.42                     5   \n",
       "5                           3                  5.42                     5   \n",
       "9                           3                  2.10                     3   \n",
       "12                          6                  1.99                     4   \n",
       "14                          6                  1.99                     4   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                      25                 18.31                     9   \n",
       "49993                      25                 18.31                    12   \n",
       "49994                      22                 18.31                    12   \n",
       "49998                       6                 11.50                     7   \n",
       "49999                       5                 11.50                     7   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "9           Good           1303.01                     35.69   \n",
       "12          Good            632.46                     39.38   \n",
       "14          Good            632.46                     36.82   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "9                    18.5                    No               246.99   \n",
       "12                   17.9                    No                16.42   \n",
       "14                   18.1                    No                16.42   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "9                       453.62    Low_spent_Large_value_payments   \n",
       "12                       69.11  High_spent_Medium_value_payments   \n",
       "14                      191.89   Low_spent_Medium_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "9               788.11  \n",
       "12              425.73  \n",
       "14              332.95  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[18030 rows x 20 columns]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Annual_Income\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 20000\n",
    "upper_bound = 1000000\n",
    "\n",
    "column = 'Annual_Income'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Num_Credit_Inquiries\n",
       "5     2065\n",
       "4     1962\n",
       "6     1751\n",
       "3     1576\n",
       "7     1572\n",
       "8     1414\n",
       "2     1153\n",
       "9     1120\n",
       "10     868\n",
       "11     830\n",
       "1      795\n",
       "12     713\n",
       "13     518\n",
       "0      463\n",
       "14     346\n",
       "15     271\n",
       "16     207\n",
       "17      92\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a picture of Delay_from_due_date\n",
    "\n",
    "column = 'Num_Credit_Inquiries'\n",
    "\n",
    "# value counts\n",
    "values_counted = df_test[column].value_counts()\n",
    "\n",
    "# filter out values with counts >1\n",
    "filtered_counts = values_counted[values_counted > 3]\n",
    "\n",
    "filtered_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>35</td>\n",
       "      <td>Engineer</td>\n",
       "      <td>143162.64</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2.10</td>\n",
       "      <td>3</td>\n",
       "      <td>Good</td>\n",
       "      <td>1303.01</td>\n",
       "      <td>35.69</td>\n",
       "      <td>18.5</td>\n",
       "      <td>No</td>\n",
       "      <td>246.99</td>\n",
       "      <td>453.62</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>788.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>39.38</td>\n",
       "      <td>17.9</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>69.11</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>425.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>36.82</td>\n",
       "      <td>18.1</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>191.89</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>332.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17716 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age    Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "4       28       _______       34847.84                  2                4   \n",
       "5       28       Teacher       34847.84                  2                4   \n",
       "9       35      Engineer      143162.64                  1                5   \n",
       "12      55  Entrepreneur       30689.89                  2                5   \n",
       "14      55  Entrepreneur       30689.89                  2                5   \n",
       "...    ...           ...            ...                ...              ...   \n",
       "49992   29     Architect       20002.88                 10                8   \n",
       "49993   29     Architect       20002.88                 10                8   \n",
       "49994   29       _______       20002.88                 10                8   \n",
       "49998   25      Mechanic       39628.99                  4                6   \n",
       "49999   25      Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "4                  6            1                    3   \n",
       "5                  6            1                    3   \n",
       "9                  8            3                    6   \n",
       "12                 4            1                    5   \n",
       "14                 4            1                    5   \n",
       "...              ...          ...                  ...   \n",
       "49992             29            5                   33   \n",
       "49993             29            5                   33   \n",
       "49994             29            5                   33   \n",
       "49998              7            2                   21   \n",
       "49999              7            2                   22   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "4                           1                  5.42                     5   \n",
       "5                           3                  5.42                     5   \n",
       "9                           3                  2.10                     3   \n",
       "12                          6                  1.99                     4   \n",
       "14                          6                  1.99                     4   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                      25                 18.31                     9   \n",
       "49993                      25                 18.31                    12   \n",
       "49994                      22                 18.31                    12   \n",
       "49998                       6                 11.50                     7   \n",
       "49999                       5                 11.50                     7   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "9           Good           1303.01                     35.69   \n",
       "12          Good            632.46                     39.38   \n",
       "14          Good            632.46                     36.82   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "9                    18.5                    No               246.99   \n",
       "12                   17.9                    No                16.42   \n",
       "14                   18.1                    No                16.42   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "9                       453.62    Low_spent_Large_value_payments   \n",
       "12                       69.11  High_spent_Medium_value_payments   \n",
       "14                      191.89   Low_spent_Medium_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "9               788.11  \n",
       "12              425.73  \n",
       "14              332.95  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[17716 rows x 20 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Annual_Income\n",
    "\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 20\n",
    "\n",
    "column = 'Num_Credit_Inquiries'\n",
    "\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "\n",
    "df_test = cleaned_df\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 17716 entries, 4 to 49999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   Age                       17716 non-null  int64  \n",
      " 1   Occupation                17716 non-null  object \n",
      " 2   Annual_Income             17716 non-null  float64\n",
      " 3   Num_Bank_Accounts         17716 non-null  int64  \n",
      " 4   Num_Credit_Card           17716 non-null  int64  \n",
      " 5   Interest_Rate             17716 non-null  int64  \n",
      " 6   Num_of_Loan               17716 non-null  int64  \n",
      " 7   Delay_from_due_date       17716 non-null  int64  \n",
      " 8   Num_of_Delayed_Payment    17716 non-null  int64  \n",
      " 9   Changed_Credit_Limit      17716 non-null  float64\n",
      " 10  Num_Credit_Inquiries      17716 non-null  int64  \n",
      " 11  Credit_Mix                17716 non-null  object \n",
      " 12  Outstanding_Debt          17716 non-null  float64\n",
      " 13  Credit_Utilization_Ratio  17716 non-null  float64\n",
      " 14  Credit_History_Age        17716 non-null  float64\n",
      " 15  Payment_of_Min_Amount     17716 non-null  object \n",
      " 16  Total_EMI_per_month       17716 non-null  float64\n",
      " 17  Amount_invested_monthly   17716 non-null  float64\n",
      " 18  Payment_Behaviour         17716 non-null  object \n",
      " 19  Monthly_Balance           17716 non-null  float64\n",
      "dtypes: float64(8), int64(8), object(4)\n",
      "memory usage: 2.8+ MB\n"
     ]
    }
   ],
   "source": [
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Payment_of_Min_Amount</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Payment_Behaviour</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>_______</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>25.93</td>\n",
       "      <td>27.2</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>39.68</td>\n",
       "      <td>High_spent_Large_value_payments</td>\n",
       "      <td>485.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>28</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>34847.84</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>Good</td>\n",
       "      <td>605.03</td>\n",
       "      <td>30.12</td>\n",
       "      <td>27.3</td>\n",
       "      <td>No</td>\n",
       "      <td>18.82</td>\n",
       "      <td>251.63</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>303.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>35</td>\n",
       "      <td>Engineer</td>\n",
       "      <td>143162.64</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2.10</td>\n",
       "      <td>3</td>\n",
       "      <td>Good</td>\n",
       "      <td>1303.01</td>\n",
       "      <td>35.69</td>\n",
       "      <td>18.5</td>\n",
       "      <td>No</td>\n",
       "      <td>246.99</td>\n",
       "      <td>453.62</td>\n",
       "      <td>Low_spent_Large_value_payments</td>\n",
       "      <td>788.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>39.38</td>\n",
       "      <td>17.9</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>69.11</td>\n",
       "      <td>High_spent_Medium_value_payments</td>\n",
       "      <td>425.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>55</td>\n",
       "      <td>Entrepreneur</td>\n",
       "      <td>30689.89</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1.99</td>\n",
       "      <td>4</td>\n",
       "      <td>Good</td>\n",
       "      <td>632.46</td>\n",
       "      <td>36.82</td>\n",
       "      <td>18.1</td>\n",
       "      <td>No</td>\n",
       "      <td>16.42</td>\n",
       "      <td>191.89</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>332.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49992</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>9</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>32.39</td>\n",
       "      <td>6.3</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>107.21</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>314.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49993</th>\n",
       "      <td>29</td>\n",
       "      <td>Architect</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>25</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>37.53</td>\n",
       "      <td>6.4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>71.79</td>\n",
       "      <td>Low_spent_Small_value_payments</td>\n",
       "      <td>350.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49994</th>\n",
       "      <td>29</td>\n",
       "      <td>_______</td>\n",
       "      <td>20002.88</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>18.31</td>\n",
       "      <td>12</td>\n",
       "      <td>Bad</td>\n",
       "      <td>3571.70</td>\n",
       "      <td>27.03</td>\n",
       "      <td>6.5</td>\n",
       "      <td>Yes</td>\n",
       "      <td>60.96</td>\n",
       "      <td>50.85</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>341.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>Good</td>\n",
       "      <td>502.38</td>\n",
       "      <td>39.14</td>\n",
       "      <td>32.1</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>97.60</td>\n",
       "      <td>High_spent_Small_value_payments</td>\n",
       "      <td>463.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>25</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>11.50</td>\n",
       "      <td>7</td>\n",
       "      <td>_</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.11</td>\n",
       "      <td>32.2</td>\n",
       "      <td>No</td>\n",
       "      <td>35.10</td>\n",
       "      <td>220.46</td>\n",
       "      <td>Low_spent_Medium_value_payments</td>\n",
       "      <td>360.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17711 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age    Occupation  Annual_Income  Num_Bank_Accounts  Num_Credit_Card  \\\n",
       "4       28       _______       34847.84                  2                4   \n",
       "5       28       Teacher       34847.84                  2                4   \n",
       "9       35      Engineer      143162.64                  1                5   \n",
       "12      55  Entrepreneur       30689.89                  2                5   \n",
       "14      55  Entrepreneur       30689.89                  2                5   \n",
       "...    ...           ...            ...                ...              ...   \n",
       "49992   29     Architect       20002.88                 10                8   \n",
       "49993   29     Architect       20002.88                 10                8   \n",
       "49994   29       _______       20002.88                 10                8   \n",
       "49998   25      Mechanic       39628.99                  4                6   \n",
       "49999   25      Mechanic       39628.99                  4                6   \n",
       "\n",
       "       Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "4                  6            1                    3   \n",
       "5                  6            1                    3   \n",
       "9                  8            3                    6   \n",
       "12                 4            1                    5   \n",
       "14                 4            1                    5   \n",
       "...              ...          ...                  ...   \n",
       "49992             29            5                   33   \n",
       "49993             29            5                   33   \n",
       "49994             29            5                   33   \n",
       "49998              7            2                   21   \n",
       "49999              7            2                   22   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "4                           1                  5.42                     5   \n",
       "5                           3                  5.42                     5   \n",
       "9                           3                  2.10                     3   \n",
       "12                          6                  1.99                     4   \n",
       "14                          6                  1.99                     4   \n",
       "...                       ...                   ...                   ...   \n",
       "49992                      25                 18.31                     9   \n",
       "49993                      25                 18.31                    12   \n",
       "49994                      22                 18.31                    12   \n",
       "49998                       6                 11.50                     7   \n",
       "49999                       5                 11.50                     7   \n",
       "\n",
       "      Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "4           Good            605.03                     25.93   \n",
       "5           Good            605.03                     30.12   \n",
       "9           Good           1303.01                     35.69   \n",
       "12          Good            632.46                     39.38   \n",
       "14          Good            632.46                     36.82   \n",
       "...          ...               ...                       ...   \n",
       "49992        Bad           3571.70                     32.39   \n",
       "49993        Bad           3571.70                     37.53   \n",
       "49994        Bad           3571.70                     27.03   \n",
       "49998       Good            502.38                     39.14   \n",
       "49999          _            502.38                     34.11   \n",
       "\n",
       "       Credit_History_Age Payment_of_Min_Amount  Total_EMI_per_month  \\\n",
       "4                    27.2                    No                18.82   \n",
       "5                    27.3                    No                18.82   \n",
       "9                    18.5                    No               246.99   \n",
       "12                   17.9                    No                16.42   \n",
       "14                   18.1                    No                16.42   \n",
       "...                   ...                   ...                  ...   \n",
       "49992                 6.3                   Yes                60.96   \n",
       "49993                 6.4                   Yes                60.96   \n",
       "49994                 6.5                   Yes                60.96   \n",
       "49998                32.1                    No                35.10   \n",
       "49999                32.2                    No                35.10   \n",
       "\n",
       "       Amount_invested_monthly                 Payment_Behaviour  \\\n",
       "4                        39.68   High_spent_Large_value_payments   \n",
       "5                       251.63    Low_spent_Large_value_payments   \n",
       "9                       453.62    Low_spent_Large_value_payments   \n",
       "12                       69.11  High_spent_Medium_value_payments   \n",
       "14                      191.89   Low_spent_Medium_value_payments   \n",
       "...                        ...                               ...   \n",
       "49992                   107.21    Low_spent_Small_value_payments   \n",
       "49993                    71.79    Low_spent_Small_value_payments   \n",
       "49994                    50.85   High_spent_Small_value_payments   \n",
       "49998                    97.60   High_spent_Small_value_payments   \n",
       "49999                   220.46   Low_spent_Medium_value_payments   \n",
       "\n",
       "       Monthly_Balance  \n",
       "4               485.30  \n",
       "5               303.36  \n",
       "9               788.11  \n",
       "12              425.73  \n",
       "14              332.95  \n",
       "...                ...  \n",
       "49992           314.82  \n",
       "49993           350.23  \n",
       "49994           341.18  \n",
       "49998           463.24  \n",
       "49999           360.38  \n",
       "\n",
       "[17711 rows x 20 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  find and remove outlier, typo rows - Monthly_Balance\n",
    "# Define the range of normal values\n",
    "lower_bound = 0\n",
    "upper_bound = 2000\n",
    "column = 'Monthly_Balance'\n",
    "# testing variable\n",
    "within_range_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "# used this to play with range and dtm where cutoff for normal values were\n",
    "outliers_df = df_test[(df_test[column] < lower_bound) | (df_test[column] > upper_bound)]\n",
    "# Filter the DataFrame to keep only rows with values within the specified range\n",
    "cleaned_df = df_test[(df_test[column] >= lower_bound) & (df_test[column] <= upper_bound)]\n",
    "df_test = cleaned_df\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export the DataFrame to a CSV file\n",
    "df_test.to_csv('cleaned_data_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}