Lec 32 - Combining DataFrames.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:b82eaf936d673802b3e8255d702cbaf5faa956bb4d22f50257b9aa9576ec3c6c"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "from pandas import Series,DataFrame"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lets make some Series to work with\n",
      "\n",
      "#First Series\n",
      "ser1 = Series([2,np.nan,4,np.nan,6,np.nan],\n",
      "           index=['Q','R','S','T','U','V'])\n",
      "\n",
      "#Second Series (based off length of ser1)\n",
      "ser2 = Series(np.arange(len(ser1), dtype=np.float64),\n",
      "           index=['Q','R','S','T','U','V'])\n",
      "\n",
      "ser2[-1] = np.nan"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ser1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "Q     2\n",
        "R   NaN\n",
        "S     4\n",
        "T   NaN\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ser2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "Q     0\n",
        "R     1\n",
        "S     2\n",
        "T     3\n",
        "U     4\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's get a series where the value of ser1 is chosen if ser2 is NAN,otherwise let the value be ser1\n",
      "Series(np.where(pd.isnull(ser1),ser2,ser1),index=ser1.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "Q     2\n",
        "R     1\n",
        "S     4\n",
        "T     3\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Take a moment to really understand how the above worked"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now we can do the same thing simply by using combine_first with pandas\n",
      "ser1.combine_first(ser2)\n",
      "\n",
      "#This combines the Series values, choosing the values of the calling Series first, unless its a NAN"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "Q     2\n",
        "R     1\n",
        "S     4\n",
        "T     3\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets how this works on a DataFrame!"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lets make some \n",
      "dframe_odds = DataFrame({'X': [1., np.nan, 3., np.nan],\n",
      "                     'Y': [np.nan, 5., np.nan, 7.],\n",
      "                     'Z': [np.nan, 9., np.nan, 11.]})\n",
      "dframe_evens = DataFrame({'X': [2., 4., np.nan, 6., 8.],\n",
      "                     'Y': [np.nan, 10., 12., 14., 16.]})\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Show\n",
      "dframe_odds"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  1</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  5</td>\n",
        "      <td>  9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  3</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  7</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 35,
       "text": [
        "    X   Y   Z\n",
        "0   1 NaN NaN\n",
        "1 NaN   5   9\n",
        "2   3 NaN NaN\n",
        "3 NaN   7  11"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Show\n",
      "dframe_evens"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  4</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>NaN</td>\n",
        "      <td> 12</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  6</td>\n",
        "      <td> 14</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  8</td>\n",
        "      <td> 16</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "    X   Y\n",
        "0   2 NaN\n",
        "1   4  10\n",
        "2 NaN  12\n",
        "3   6  14\n",
        "4   8  16"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets combine using odds values first, unless theres a NAN, then put the evens values\n",
      "dframe_odds.combine_first(dframe_evens)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 4</td>\n",
        "      <td>  5</td>\n",
        "      <td>  9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 3</td>\n",
        "      <td> 12</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 6</td>\n",
        "      <td>  7</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 8</td>\n",
        "      <td> 16</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 38,
       "text": [
        "   X   Y   Z\n",
        "0  1 NaN NaN\n",
        "1  4   5   9\n",
        "2  3  12 NaN\n",
        "3  6   7  11\n",
        "4  8  16 NaN"
       ]
      }
     ],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Next up: Reshaping DataFrames!"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}