Dealing with missing values.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "efaa138b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#How to deal with the missing values of data?\n",
    "#1. Keep it\n",
    "#2 Remove it\n",
    "#3 Replace it\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.nan\n",
    "\n",
    "#Lets firsst see the keep values command\n",
    "#But first its important to review about how NaN values work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "62763a4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NaT"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.NA\n",
    "pd.NaT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "27f532ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.nan = np.nan\n",
    "#(this will be false)\n",
    "#is different from\n",
    "np.nan is np.nan\n",
    "#(this will be true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "96a26c87",
   "metadata": {},
   "outputs": [],
   "source": [
    "mv_scores = pd.read_csv('movie_scores.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "60772b8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age  sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0\n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN\n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0\n",
       "4       Emma     Stone  31.0    f              7.0               9.0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7347dec4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.isnull of   first_name last_name   age  sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0\n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN\n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0\n",
       "4       Emma     Stone  31.0    f              7.0               9.0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores.notnull\n",
    "mv_scores.isnull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "097727d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0   m              8.0              10.0\n",
       "3      Oprah   Winfrey  66.0   f              6.0               8.0\n",
       "4       Emma     Stone  31.0   f              7.0               9.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we can select columns where we have not null values or where we have null values\n",
    "mv_scores[mv_scores['pre_movie_score'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1489670b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age  sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0\n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN\n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0\n",
       "4       Emma     Stone  31.0    f              7.0               9.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores[(mv_scores['pre_movie_score'].notnull()) & (mv_scores['first_name']=='Emma')]\n",
    "mv_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e9f6c5fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0   m              8.0              10.0\n",
       "3      Oprah   Winfrey  66.0   f              6.0               8.0\n",
       "4       Emma     Stone  31.0   f              7.0               9.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores\n",
    "#dropna with some criteria:\n",
    "#thresh means that if a row has more than N non-NaN values, keep it\n",
    "#Example:\n",
    "mv_scores.dropna(thresh=1)\n",
    "#All keep because all of them have at least one non-nan value.If I put thresh=5, Hugh Jackman will probably be erased\n",
    "mv_scores.dropna(thresh=5)\n",
    "#voilá!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "de4a2a10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age  sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0\n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN\n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0\n",
       "4       Emma     Stone  31.0    f              7.0               9.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now lets talk about the axis argument\n",
    "mv_scores\n",
    "#If we use axis=1 only, it will erase all columns that have NaN values.\n",
    "#We do not need to use it now\n",
    "#mv_scores.dropna(axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "722b6318",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0   m              8.0              10.0\n",
       "3      Oprah   Winfrey  66.0   f              6.0               8.0\n",
       "4       Emma     Stone  31.0   f              7.0               9.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Next argument is subset argument\n",
    "mv_scores.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4036ea64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age  sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0\n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN\n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0\n",
       "4       Emma     Stone  31.0    f              7.0               9.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f321e710",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age sex  pre_movie_score  post_movie_score\n",
       "0        Tom     Hanks  63.0   m              8.0              10.0\n",
       "2       Hugh   Jackman  51.0   m              NaN               NaN\n",
       "3      Oprah   Winfrey  66.0   f              6.0               8.0\n",
       "4       Emma     Stone  31.0   f              7.0               9.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#But if I ask a subset, lets see what happened:\n",
    "mv_scores.dropna(subset=['last_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "74640baa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    8.0\n",
       "1    7.0\n",
       "2    7.0\n",
       "3    6.0\n",
       "4    7.0\n",
       "Name: pre_movie_score, dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now lets see the Filling in the data commands\n",
    "mv_scores.fillna('New value!')\n",
    "#Not good...We want to add criteria!\n",
    "mv_scores['pre_movie_scores']=mv_scores['pre_movie_score'].fillna(0)\n",
    "#ok, it works!\n",
    "#we can also add using a function!\n",
    "mv_scores['pre_movie_score'].fillna(mv_scores['pre_movie_score'].mean())\n",
    "mv_scores.finllna(mv_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7f2ee17d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>pre_movie_score</th>\n",
       "      <th>post_movie_score</th>\n",
       "      <th>pre_movie_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tom</td>\n",
       "      <td>Hanks</td>\n",
       "      <td>63.0</td>\n",
       "      <td>m</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hugh</td>\n",
       "      <td>Jackman</td>\n",
       "      <td>51.0</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oprah</td>\n",
       "      <td>Winfrey</td>\n",
       "      <td>66.0</td>\n",
       "      <td>f</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Emma</td>\n",
       "      <td>Stone</td>\n",
       "      <td>31.0</td>\n",
       "      <td>f</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name last_name   age  sex  pre_movie_score  post_movie_score  \\\n",
       "0        Tom     Hanks  63.0    m              8.0              10.0   \n",
       "1        NaN       NaN   NaN  NaN              NaN               NaN   \n",
       "2       Hugh   Jackman  51.0    m              NaN               NaN   \n",
       "3      Oprah   Winfrey  66.0    f              6.0               8.0   \n",
       "4       Emma     Stone  31.0    f              7.0               9.0   \n",
       "\n",
       "   pre_movie_scores  \n",
       "0               8.0  \n",
       "1               0.0  \n",
       "2               0.0  \n",
       "3               6.0  \n",
       "4               7.0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a76c3299",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Next filling is Linear Regression!\n",
    "airline_tix ={'first':100,'business':np.nan,'economy-plus':50,'economy':30}\n",
    "ser=pd.Series(airline_tix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9ef1077c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "first           100.0\n",
       "business          NaN\n",
       "economy-plus     50.0\n",
       "economy          30.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3402e58b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "first           100.0\n",
       "business         75.0\n",
       "economy-plus     50.0\n",
       "economy          30.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Lets do a linear interpolation between the upper and lower values\n",
    "ser.interpolate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4decbf95",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}