campaign_finance_eda_aggregate_pac.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Subject:__ Topics of Tweets of US Congress through the Lens of Campaign Finance\n",
    "\n",
    "__Date:__ 09/13/2018\n",
    "\n",
    "__Author:__ Edmund D. Chitwood\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Summary:__<br>\n",
    "<br>The following Notebook\n",
    "- perform EDA on aggreate camapaign contribution data for current Congress, and\n",
    "- aggregate PAC contribution totals for each relevant member of Congress\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributions = pd.read_pickle('contributions_congress.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6477, 6)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributions.total = contributions.total.apply(lambda x: int(x))\n",
    "contributions.indivs = contributions.indivs.apply(lambda x: int(x))\n",
    "contributions.pacs = contributions.pacs.apply(lambda x: int(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>indivs</th>\n",
       "      <th>pacs</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6.477000e+03</td>\n",
       "      <td>6.477000e+03</td>\n",
       "      <td>6.477000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.387461e+05</td>\n",
       "      <td>7.820888e+04</td>\n",
       "      <td>2.169550e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.205258e+05</td>\n",
       "      <td>1.299483e+05</td>\n",
       "      <td>4.918567e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.937200e+04</td>\n",
       "      <td>-7.500000e+03</td>\n",
       "      <td>-7.848000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.525000e+03</td>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>2.650000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.144700e+04</td>\n",
       "      <td>3.100000e+04</td>\n",
       "      <td>6.690000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.993000e+04</td>\n",
       "      <td>8.655000e+04</td>\n",
       "      <td>1.727500e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.380421e+06</td>\n",
       "      <td>1.683410e+06</td>\n",
       "      <td>7.412821e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             indivs          pacs         total\n",
       "count  6.477000e+03  6.477000e+03  6.477000e+03\n",
       "mean   1.387461e+05  7.820888e+04  2.169550e+05\n",
       "std    4.205258e+05  1.299483e+05  4.918567e+05\n",
       "min   -1.937200e+04 -7.500000e+03 -7.848000e+03\n",
       "25%    3.525000e+03  1.000000e+04  2.650000e+04\n",
       "50%    2.144700e+04  3.100000e+04  6.690000e+04\n",
       "75%    7.993000e+04  8.655000e+04  1.727500e+05\n",
       "max    6.380421e+06  1.683410e+06  7.412821e+06"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1405217325"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions.total.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Agribusiness', 'Communic/Electronics', 'Construction', 'Defense',\n",
       "       'Energy/Nat Resource', 'Finance/Insur/RealEst', 'Health',\n",
       "       'Lawyers & Lobbyists', 'Transportation', 'Misc Business', 'Labor',\n",
       "       'Ideology/Single-Issue', 'Other', ''], dtype=object)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions.sector_name.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(499, 6)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions[contributions.sector_name=='Construction'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Finance/Insur/RealEst', 98677365),\n",
       " ('Ideology/Single-Issue', 74720701),\n",
       " ('Health', 61478600),\n",
       " ('Misc Business', 46388597),\n",
       " ('Labor', 40430030),\n",
       " ('Energy/Nat Resource', 37687841),\n",
       " ('Communic/Electronics', 33333296),\n",
       " ('Agribusiness', 30404282),\n",
       " ('Transportation', 30124702),\n",
       " ('Defense', 19931592),\n",
       " ('Lawyers & Lobbyists', 16792870),\n",
       " ('Construction', 14957881),\n",
       " ('Other', 1631189),\n",
       " ('', 0)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pac_money_sector_totals = dict()\n",
    "\n",
    "for i in contributions.sector_name.unique():\n",
    "    pac_money_sector_totals[i] = contributions[contributions.sector_name==i].pacs.sum()\n",
    "\n",
    "sorted(pac_money_sector_totals.items(), key=lambda kv: kv[1],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Finance/Insur/RealEst', 297512353),\n",
       " ('Ideology/Single-Issue', 182747274),\n",
       " ('Other', 157386366),\n",
       " ('Lawyers & Lobbyists', 147443049),\n",
       " ('Misc Business', 137980958),\n",
       " ('Health', 118239165),\n",
       " ('Communic/Electronics', 79936741),\n",
       " ('Energy/Nat Resource', 68402712),\n",
       " ('Agribusiness', 55699720),\n",
       " ('Transportation', 50373705),\n",
       " ('Construction', 43947135),\n",
       " ('Labor', 40946010),\n",
       " ('Defense', 24602137),\n",
       " ('', 0)]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "money_sector_totals = dict()\n",
    "\n",
    "for i in contributions.sector_name.unique():\n",
    "    money_sector_totals[i] = contributions[contributions.sector_name==i].total.sum()\n",
    "\n",
    "sorted(money_sector_totals.items(), key=lambda kv: kv[1],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Finance/Insur/RealEst', (507, 6)),\n",
       " ('Health', (506, 6)),\n",
       " ('Misc Business', (506, 6)),\n",
       " ('Lawyers & Lobbyists', (505, 6)),\n",
       " ('Ideology/Single-Issue', (505, 6)),\n",
       " ('Communic/Electronics', (503, 6)),\n",
       " ('Transportation', (503, 6)),\n",
       " ('Energy/Nat Resource', (502, 6)),\n",
       " ('Construction', (499, 6)),\n",
       " ('Other', (499, 6)),\n",
       " ('Agribusiness', (495, 6)),\n",
       " ('Labor', (475, 6)),\n",
       " ('Defense', (471, 6)),\n",
       " ('', (1, 6))]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sector_by_number_congressmen = dict()\n",
    "\n",
    "for i in contributions.sector_name.unique():\n",
    "    sector_by_number_congressmen[i] = contributions[contributions.sector_name==i].shape\n",
    "\n",
    "sorted(sector_by_number_congressmen.items(), key=lambda kv: kv[1],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3290891"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions[contributions['os_id']=='N00003535'].pacs.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6477, 6)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = contributions.groupby('os_id')\n",
    "\n",
    "os_id = []\n",
    "pacs = []\n",
    "\n",
    "for name, group in grouped:\n",
    "    os_id.append(name)\n",
    "    pacs.append(group[group['os_id']==name].pacs.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributions_totals = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributions_totals['os_id'] = os_id\n",
    "contributions_totals['pacs'] = pacs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>os_id</th>\n",
       "      <th>pacs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N00000078</td>\n",
       "      <td>744634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N00000153</td>\n",
       "      <td>2019609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N00000179</td>\n",
       "      <td>479044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N00000270</td>\n",
       "      <td>2513089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>N00000362</td>\n",
       "      <td>1894962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       os_id     pacs\n",
       "0  N00000078   744634\n",
       "1  N00000153  2019609\n",
       "2  N00000179   479044\n",
       "3  N00000270  2513089\n",
       "4  N00000362  1894962"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions_totals.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pacs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.080000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.971633e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.944337e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.050000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.689995e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.380310e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.237790e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.879797e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               pacs\n",
       "count  5.080000e+02\n",
       "mean   9.971633e+05\n",
       "std    9.944337e+05\n",
       "min   -1.050000e+04\n",
       "25%    3.689995e+05\n",
       "50%    6.380310e+05\n",
       "75%    1.237790e+06\n",
       "max    5.879797e+06"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions_totals.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "638031.0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions_totals.pacs.median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "997163.2795275591"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contributions_totals.pacs.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributions_totals.to_pickle('contributions_pacs.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}