data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d98ad80e-a6bf-45da-9e57-9170d74a2ba6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import csv\n",
    "import os\n",
    "import sys\n",
    "from contextlib import contextmanager\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76098049-1b35-4ff7-99c1-e7a3d2f1266e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matti\\MAIniacs\n"
     ]
    }
   ],
   "source": [
    "script_path = os.getcwd()\n",
    "file_path = r\".\\Data\"\n",
    "extension = \"csv\"\n",
    "\n",
    "categorical_features = [\"sex\", \"restecg\", \"slope\", \"thal\", \"cp\"]\n",
    "integer_features = [\"age\", \"trestbps\", \"chol\", \"thalach\", \"ca\"]\n",
    "float_features = [\"oldpeak\"]\n",
    "boolean_features = [\"fbs\", \"exang\"]\n",
    "\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3b0c2df7-b2b6-4c74-8fdb-bfec6fa6c678",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting up a context manager in order to change to the data folder, create a dictionary of dataframes from all the csv files in the folder\n",
    "# and then returning to the original folder.\n",
    "\n",
    "@contextmanager\n",
    "def gen_list(file_path, extension):\n",
    "    f = os.chdir(file_path)\n",
    "    yield f\n",
    "    os.chdir(script_path)\n",
    "    \n",
    "with gen_list(file_path, extension) as f:\n",
    "    csv_list = glob.glob(r\"*{}\".format(extension))\n",
    "    dataframe_names = []\n",
    "    df_dict = {}\n",
    "    iterator = 0\n",
    "    for i in csv_list:\n",
    "        dataframe_names.append(i[:-4].replace(\".\", \"_\")) # Saves the names of the csv files minus .csv and also changes the periods in the name to underscores.\n",
    "    for j in dataframe_names:\n",
    "        df_dict[j] = pd.read_csv(csv_list[iterator], sep=None, engine=\"python\") # Key is the name of the csv file formatted as above.\n",
    "        iterator += 1\n",
    "\n",
    "# Saves the column names of processed_cleveland and sets the other datasets column names to the same.     \n",
    "\n",
    "column_names = list(df_dict[\"processed_cleveland\"].columns)\n",
    "\n",
    "for df, names in df_dict.items():\n",
    "    names.columns = column_names\n",
    "    \n",
    "# Function to set the dtypes as specified in the earlier lists. The function increases the amount of null values for some reason,\n",
    "# most probably cause by the to_numeric function's errors parameter. TODO: Find out which values is changing to nulls.\n",
    "\n",
    "def set_dtypes(df_dict):\n",
    "    for df, name in df_dict.items():\n",
    "        for column in name.columns:\n",
    "            if column in categorical_features:\n",
    "                name[column] = name[column].astype(\"category\", copy=False)\n",
    "            elif column in boolean_features:\n",
    "                name[column] = name[column].astype(\"bool\", copy=False)\n",
    "            elif column in float_features:\n",
    "                name[column] = pd.to_numeric(name[column], downcast=\"float\", errors=\"coerce\")\n",
    "            elif column in integer_features:\n",
    "                name[column] = pd.to_numeric(name[column], downcast=\"float\", errors=\"coerce\")\n",
    "                name[column] = pd.to_numeric(name[column], downcast=\"integer\")\n",
    "    return df_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c1d185bb-e283-49bc-83e0-786be11e372e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['processed_cleveland', 'processed_hungarian', 'processed_switzerland', 'processed_va'])\n"
     ]
    }
   ],
   "source": [
    "print(df_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "450be428-d6fd-452b-82a7-6fcec11c4c56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed_cleveland\n",
      "0\n",
      "processed_hungarian\n",
      "0\n",
      "processed_switzerland\n",
      "0\n",
      "processed_va\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# Imputing nulls instead of \"?\" for all dataframes in the dictionary.\n",
    "# TODO: print name of the dataframe before isnull list.\n",
    "\n",
    "for name in df_dict.keys():\n",
    "    print(name)\n",
    "    print(df_dict[name].isnull().sum().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07a0ddcf-45e3-4f9c-a872-e94101a583fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed_cleveland\n",
      "6\n",
      "processed_hungarian\n",
      "782\n",
      "processed_switzerland\n",
      "273\n",
      "processed_va\n",
      "698\n"
     ]
    }
   ],
   "source": [
    "for name in df_dict:\n",
    "    df_dict[name].replace(to_replace=\"?\", value=np.nan, inplace=True)\n",
    "\n",
    "for name in df_dict:\n",
    "    print(name)\n",
    "    print(df_dict[name].isnull().sum().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b280ee9-470a-414e-bebd-6ece0fbbaf23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed_cleveland\n",
      "age\n",
      "[63. 67. 37. 41. 56. 62. 57. 53. 44. 52. 48. 54. 49. 64. 58. 60. 50. 66.\n",
      " 43. 40. 69. 59. 42. 55. 61. 65. 71. 51. 46. 45. 39. 68. 47. 34. 35. 29.\n",
      " 70. 77. 38. 74. 76.]\n",
      "sex\n",
      "[1. 0.]\n",
      "cp\n",
      "[1. 4. 3. 2.]\n",
      "trestbps\n",
      "[145. 160. 120. 130. 140. 172. 150. 110. 132. 117. 135. 112. 105. 124.\n",
      " 125. 142. 128. 170. 155. 104. 180. 138. 108. 134. 122. 115. 118. 100.\n",
      " 200.  94. 165. 102. 152. 101. 126. 174. 148. 178. 158. 192. 129. 144.\n",
      " 123. 136. 146. 106. 156. 154. 114. 164.]\n",
      "chol\n",
      "[233. 286. 229. 250. 204. 236. 268. 354. 254. 203. 192. 294. 256. 263.\n",
      " 199. 168. 239. 275. 266. 211. 283. 284. 224. 206. 219. 340. 226. 247.\n",
      " 167. 230. 335. 234. 177. 276. 353. 243. 225. 302. 212. 330. 175. 417.\n",
      " 197. 198. 290. 253. 172. 273. 213. 305. 216. 304. 188. 282. 185. 232.\n",
      " 326. 231. 269. 267. 248. 360. 258. 308. 245. 270. 208. 264. 321. 274.\n",
      " 325. 235. 257. 164. 141. 252. 255. 201. 222. 260. 182. 303. 265. 309.\n",
      " 307. 249. 186. 341. 183. 407. 217. 288. 220. 209. 227. 261. 174. 281.\n",
      " 221. 205. 240. 289. 318. 298. 564. 246. 322. 299. 300. 293. 277. 214.\n",
      " 207. 223. 160. 394. 184. 315. 409. 244. 195. 196. 126. 313. 259. 200.\n",
      " 262. 215. 228. 193. 271. 210. 327. 149. 295. 306. 178. 237. 218. 242.\n",
      " 319. 166. 180. 311. 278. 342. 169. 187. 157. 176. 241. 131.]\n",
      "fbs\n",
      "[1. 0.]\n",
      "restecg\n",
      "[2. 0. 1.]\n",
      "thalach\n",
      "[150. 108. 129. 187. 172. 178. 160. 163. 147. 155. 148. 153. 142. 173.\n",
      " 162. 174. 168. 139. 171. 144. 132. 158. 114. 151. 161. 179. 120. 112.\n",
      " 137. 157. 169. 165. 123. 128. 152. 140. 188. 109. 125. 131. 170. 113.\n",
      "  99. 177. 141. 180. 111. 143. 182. 156. 115. 149. 145. 146. 175. 186.\n",
      " 185. 159. 130. 190. 136.  97. 127. 154. 133. 126. 202. 103. 166. 164.\n",
      " 184. 124. 122.  96. 138.  88. 105. 194. 195. 106. 167.  95. 192. 117.\n",
      " 121. 116.  71. 118. 181. 134.  90.]\n",
      "exang\n",
      "[0. 1.]\n",
      "oldpeak\n",
      "[2.3 1.5 2.6 3.5 1.4 0.8 3.6 0.6 3.1 0.4 1.3 0.  0.5 1.6 1.  1.2 0.2 1.8\n",
      " 3.2 2.4 2.  2.5 2.2 2.8 3.  3.4 6.2 4.  5.6 2.9 0.1 2.1 1.9 4.2 0.9 1.1\n",
      " 3.8 0.7 0.3 4.4]\n",
      "slope\n",
      "[3. 2. 1.]\n",
      "ca\n",
      "['0.0' '3.0' '2.0' '1.0' nan]\n",
      "thal\n",
      "['6.0' '3.0' '7.0' nan]\n",
      "num\n",
      "[0 2 1 3 4]\n",
      "processed_hungarian\n",
      "age\n",
      "[28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51\n",
      " 52 53 54 55 56 57 58 59 60 61 62 63 65 66]\n",
      "sex\n",
      "[1 0]\n",
      "cp\n",
      "[2 1 3 4]\n",
      "trestbps\n",
      "['130' '120' '140' '170' '100' '105' '110' '125' '150' '98' '112' '145'\n",
      " '190' '160' '115' '142' '180' '132' '135' nan '108' '124' '113' '122'\n",
      " '92' '118' '106' '200' '138' '136' '128' '155']\n",
      "chol\n",
      "['132' '243' nan '237' '219' '198' '225' '254' '298' '161' '214' '220'\n",
      " '160' '167' '308' '264' '166' '340' '209' '260' '211' '173' '283' '194'\n",
      " '223' '315' '275' '297' '292' '182' '200' '204' '241' '339' '147' '273'\n",
      " '307' '289' '215' '281' '250' '184' '245' '291' '295' '269' '196' '268'\n",
      " '228' '358' '201' '249' '266' '186' '207' '218' '412' '224' '238' '230'\n",
      " '163' '240' '280' '257' '263' '276' '284' '195' '227' '253' '187' '202'\n",
      " '328' '168' '216' '129' '190' '188' '179' '210' '272' '180' '100' '259'\n",
      " '468' '274' '320' '221' '309' '312' '171' '208' '246' '305' '217' '365'\n",
      " '344' '394' '256' '326' '277' '270' '229' '85' '347' '251' '222' '287'\n",
      " '318' '213' '294' '193' '271' '156' '267' '282' '117' '466' '247' '226'\n",
      " '265' '206' '288' '303' '338' '248' '306' '529' '392' '231' '329' '355'\n",
      " '233' '242' '603' '255' '172' '175' '290' '341' '234' '342' '404' '518'\n",
      " '285' '279' '388' '164' '336' '491' '205' '212' '331' '393']\n",
      "fbs\n",
      "['0' nan '1']\n",
      "restecg\n",
      "['2' '0' '1' nan]\n",
      "thalach\n",
      "['185' '160' '170' '150' '165' '184' '155' '190' '168' '180' '178' '172'\n",
      " '130' '142' '98' '158' '129' '146' '145' '120' '106' '132' '140' '138'\n",
      " '167' '188' '144' '137' '136' '152' '175' '176' '118' '154' '115' '135'\n",
      " '122' '110' '90' '116' '174' '125' nan '148' '100' '164' '139' '127'\n",
      " '162' '112' '134' '114' '128' '126' '124' '153' '166' '103' '156' '87'\n",
      " '102' '92' '99' '121' '91' '108' '96' '82' '105' '143' '119' '94']\n",
      "exang\n",
      "['0' '1' nan]\n",
      "oldpeak\n",
      "[0.  1.  2.  1.5 0.5 3.  0.8 2.5 4.  5. ]\n",
      "slope\n",
      "[nan '2' '1' '3']\n",
      "ca\n",
      "[nan '0']\n",
      "thal\n",
      "[nan '6' '3' '7']\n",
      "num\n",
      "[0 1]\n",
      "processed_switzerland\n",
      "age\n",
      "[32 34 35 36 38 40 41 42 43 45 46 47 48 50 51 52 53 54 55 56 57 58 59 60\n",
      " 61 62 63 64 65 66 67 68 69 70 72 73 74]\n",
      "sex\n",
      "[1 0]\n",
      "cp\n",
      "[1 4 3 2]\n",
      "trestbps\n",
      "['95' '115' nan '110' '105' '100' '135' '150' '125' '145' '140' '155'\n",
      " '160' '120' '130' '165' '80' '180' '170' '200' '185']\n",
      "chol\n",
      "[0]\n",
      "fbs\n",
      "[nan '0' '1']\n",
      "restecg\n",
      "['0' nan '1' '2']\n",
      "thalach\n",
      "['127' '154' '130' '125' '166' '156' '179' '128' '150' '120' '144' '176'\n",
      " '99' '122' '145' '140' '138' '133' '113' '118' '149' '124' '110' '139'\n",
      " '92' '104' '170' '163' '60' '126' '82' '95' '115' '135' '141' '155' '83'\n",
      " '97' '98' '100' '148' '103' '121' '131' '182' '105' '175' '94' '119'\n",
      " '143' '63' '70' '77' '117' '123' '134' '72' '78' '109' '86' '114' '93'\n",
      " '67' '90' '108' '136' nan '157']\n",
      "exang\n",
      "['0' '1' nan]\n",
      "oldpeak\n",
      "['.7' '.2' nan '1' '2.8' '0' '-1.1' '1.6' '-1.5' '1.5' '2' '.5' '-.1'\n",
      " '-2.6' '2.1' '-.7' '2.2' '3' '.1' '.3' '-2' '-1' '1.8' '1.4' '2.6' '.9'\n",
      " '2.4' '1.1' '.4' '2.5' '1.7' '-.8' '-.5' '-.9' '3.7' '1.3']\n",
      "slope\n",
      "['1' nan '2' '3']\n",
      "ca\n",
      "[nan '1' '2']\n",
      "thal\n",
      "[nan '7' '6' '3']\n",
      "num\n",
      "[1 3 2 0 4]\n",
      "processed_va\n",
      "age\n",
      "[63 44 60 55 66 65 56 59 62 57 46 58 64 74 52 69 51 54 77 61 40 41 42 53\n",
      " 68 67 72 75 49 35 43 48 50 45 76 70 71 38 37]\n",
      "sex\n",
      "[1 0]\n",
      "cp\n",
      "[4 3 2 1]\n",
      "trestbps\n",
      "['140' '130' '132' '142' '110' '120' '150' '180' '160' '126' nan '128'\n",
      " '170' '152' '116' '124' '0' '122' '144' '154' '125' '104' '136' '134'\n",
      " '138' '178' '146' '135' '158' '106' '112' '102' '96' '172' '155' '156'\n",
      " '118' '100' '190' '114' '127']\n",
      "chol\n",
      "['260' '209' '218' '228' '213' '0' '236' '267' '166' '220' '177' '186'\n",
      " '100' '171' '230' '281' '203' '277' '233' '240' '153' '224' '316' '311'\n",
      " '270' '217' '214' '252' '339' '216' '276' '458' '241' '384' '297' '248'\n",
      " '308' '208' '227' '210' '245' '225' '198' '195' '161' '258' '235' '305'\n",
      " '223' '282' '349' nan '160' '312' '283' '142' '211' '306' '222' '202'\n",
      " '197' '204' '274' '192' '298' '272' '200' '261' '181' '221' '175' '219'\n",
      " '310' '232' '273' '182' '292' '289' '193' '170' '369' '173' '271' '244'\n",
      " '285' '243' '237' '165' '287' '256' '264' '226' '207' '284' '337' '254'\n",
      " '300' '333' '139' '385']\n",
      "fbs\n",
      "['0' '1' nan]\n",
      "restecg\n",
      "[1 2 0]\n",
      "thalach\n",
      "['112' '127' '140' '149' '99' '120' '105' '141' '157' '117' nan '148' '86'\n",
      " '84' '125' '118' '124' '106' '111' '180' '129' '110' '155' '122' '133'\n",
      " '131' '80' '165' '107' '128' '160' '97' '161' '130' '108' '123' '144'\n",
      " '102' '145' '69' '138' '150' '88' '132' '121' '135' '100' '162' '73'\n",
      " '154' '115' '119' '159' '94' '113' '98' '96' '151' '126' '93']\n",
      "exang\n",
      "['1' '0' nan]\n",
      "oldpeak\n",
      "['3' '0' '1.5' '2.5' '1.3' '-0.5' '2' '0.5' '1' nan '1.6' '4' '3.5' '0.8'\n",
      " '1.7']\n",
      "slope\n",
      "['2' nan '3' '1']\n",
      "ca\n",
      "[nan '0']\n",
      "thal\n",
      "[nan '3' '7' '6']\n",
      "num\n",
      "[2 0 1 3 4]\n"
     ]
    }
   ],
   "source": [
    "for df, name in df_dict.items():\n",
    "    print(df)\n",
    "    for i in name.columns:\n",
    "        print(i)\n",
    "        print(name[i].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b6fef2b0-f507-4f8a-938b-68af4e6a44ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dict = set_dtypes(df_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c50f4f1f-9b8e-479f-9336-54c3ba97c9d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}