BigQueryGithubWQuery.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BigQueryGithub.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/qwiksilva/cs224w-github-rec/blob/master/BigQueryGithubWQuery.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8Iwwgh0maHE",
        "colab_type": "code",
        "outputId": "45e91de4-f922-4aff-c796-7333b287b8b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "print('Authenticated')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Authenticated\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJYZ5pDtQ8hP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext google.colab.data_table\n",
        "# %unload_ext google.colab.data_table"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cmXado7xTScp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Fetch the largest 20 projects by the number of prs they have.\n",
        "# Save output in a variable `df`\n",
        "%%bigquery --project cs224w-257119 df\n",
        "SELECT prs.num_prs, prs.base_repo_id, projects.name, projects.deleted\n",
        "FROM (\n",
        "  SELECT COUNT(base_repo_id) as num_prs, base_repo_id\n",
        "  FROM `ghtorrent-bq.ght_2018_04_01.pull_requests`\n",
        "  GROUP BY base_repo_id\n",
        ") prs\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.projects` projects ON prs.base_repo_id = projects.id\n",
        "ORDER BY prs.num_prs DESC\n",
        "LIMIT 20\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p-eZpWdvR8xR",
        "colab_type": "code",
        "outputId": "8f120d4c-3004-4816-e150-1dd70d83570f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 508
        }
      },
      "source": [
        "df"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.module+javascript": "\n      import \"https://ssl.gstatic.com/colaboratory/data_table/9e554b27bdd509f3/data_table.js\";\n\n      window.createDataTable({\n        data: [[{\n            'v': 0,\n            'f': \"0\",\n        },\n{\n            'v': 82846,\n            'f': \"82846\",\n        },\n{\n            'v': 20170979,\n            'f': \"20170979\",\n        },\n\"everypolitician-data\",\nfalse],\n [{\n            'v': 1,\n            'f': \"1\",\n        },\n{\n            'v': 69340,\n            'f': \"69340\",\n        },\n{\n            'v': 70604300,\n            'f': \"70604300\",\n        },\n\"signcla-probe-repo\",\nfalse],\n [{\n            'v': 2,\n            'f': \"2\",\n        },\n{\n            'v': 55760,\n            'f': \"55760\",\n        },\n{\n            'v': 789,\n            'f': \"789\",\n        },\n\"liferay-portal\",\nfalse],\n [{\n            'v': 3,\n            'f': \"3\",\n        },\n{\n            'v': 54805,\n            'f': \"54805\",\n        },\n{\n            'v': 33253535,\n            'f': \"33253535\",\n        },\n\"sauron\",\nfalse],\n [{\n            'v': 4,\n            'f': \"4\",\n        },\n{\n            'v': 40082,\n            'f': \"40082\",\n        },\n{\n            'v': 9570147,\n            'f': \"9570147\",\n        },\n\"homebrew-cask\",\nfalse],\n [{\n            'v': 5,\n            'f': \"5\",\n        },\n{\n            'v': 33504,\n            'f': \"33504\",\n        },\n{\n            'v': 6866209,\n            'f': \"6866209\",\n        },\n\"homebrew\",\nfalse],\n [{\n            'v': 6,\n            'f': \"6\",\n        },\n{\n            'v': 31212,\n            'f': \"31212\",\n        },\n{\n            'v': 202,\n            'f': \"202\",\n        },\n\"gaia\",\nfalse],\n [{\n            'v': 7,\n            'f': \"7\",\n        },\n{\n            'v': 30909,\n            'f': \"30909\",\n        },\n{\n            'v': 1803391,\n            'f': \"1803391\",\n        },\n\"snoopys-sandbox\",\nfalse],\n [{\n            'v': 8,\n            'f': \"8\",\n        },\n{\n            'v': 28820,\n            'f': \"28820\",\n        },\n{\n            'v': 3777,\n            'f': \"3777\",\n        },\n\"nixpkgs\",\nfalse],\n [{\n            'v': 9,\n            'f': \"9\",\n        },\n{\n            'v': 28289,\n            'f': \"28289\",\n        },\n{\n            'v': 22091370,\n            'f': \"22091370\",\n        },\n\"kubernetes\",\nfalse],\n [{\n            'v': 10,\n            'f': \"10\",\n        },\n{\n            'v': 28273,\n            'f': \"28273\",\n        },\n{\n            'v': 1142,\n            'f': \"1142\",\n        },\n\"salt\",\nfalse],\n [{\n            'v': 11,\n            'f': \"11\",\n        },\n{\n            'v': 23473,\n            'f': \"23473\",\n        },\n{\n            'v': 32902832,\n            'f': \"32902832\",\n        },\n\"homebrew-core\",\nfalse],\n [{\n            'v': 12,\n            'f': \"12\",\n        },\n{\n            'v': 22997,\n            'f': \"22997\",\n        },\n{\n            'v': 7301975,\n            'f': \"7301975\",\n        },\n\"patchwork\",\nfalse],\n [{\n            'v': 13,\n            'f': \"13\",\n        },\n{\n            'v': 21373,\n            'f': \"21373\",\n        },\n{\n            'v': 30513020,\n            'f': \"30513020\",\n        },\n\"sauron-demo\",\nfalse],\n [{\n            'v': 14,\n            'f': \"14\",\n        },\n{\n            'v': 20960,\n            'f': \"20960\",\n        },\n{\n            'v': 3149844,\n            'f': \"3149844\",\n        },\n\"cmssw\",\nfalse],\n [{\n            'v': 15,\n            'f': \"15\",\n        },\n{\n            'v': 20836,\n            'f': \"20836\",\n        },\n{\n            'v': 1334,\n            'f': \"1334\",\n        },\n\"rails\",\nfalse],\n [{\n            'v': 16,\n            'f': \"16\",\n        },\n{\n            'v': 20806,\n            'f': \"20806\",\n        },\n{\n            'v': 8196280,\n            'f': \"8196280\",\n        },\n\"spark\",\nfalse],\n [{\n            'v': 17,\n            'f': \"17\",\n        },\n{\n            'v': 20747,\n            'f': \"20747\",\n        },\n{\n            'v': 1486,\n            'f': \"1486\",\n        },\n\"ansible\",\nfalse],\n [{\n            'v': 18,\n            'f': \"18\",\n        },\n{\n            'v': 20554,\n            'f': \"20554\",\n        },\n{\n            'v': 17694,\n            'f': \"17694\",\n        },\n\"ceph\",\nfalse],\n [{\n            'v': 19,\n            'f': \"19\",\n        },\n{\n            'v': 20537,\n            'f': \"20537\",\n        },\n{\n            'v': 11703554,\n            'f': \"11703554\",\n        },\n\"code-dot-org\",\nfalse]],\n        columns: [[\"number\", \"index\"], [\"number\", \"num_prs\"], [\"number\", \"base_repo_id\"], [\"string\", \"name\"], [\"string\", \"deleted\"]],\n        columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n        rowsPerPage: 25,\n        helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n      });\n    ",
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>num_prs</th>\n",
              "      <th>base_repo_id</th>\n",
              "      <th>name</th>\n",
              "      <th>deleted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>82846</td>\n",
              "      <td>20170979</td>\n",
              "      <td>everypolitician-data</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>69340</td>\n",
              "      <td>70604300</td>\n",
              "      <td>signcla-probe-repo</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>55760</td>\n",
              "      <td>789</td>\n",
              "      <td>liferay-portal</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>54805</td>\n",
              "      <td>33253535</td>\n",
              "      <td>sauron</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>40082</td>\n",
              "      <td>9570147</td>\n",
              "      <td>homebrew-cask</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>33504</td>\n",
              "      <td>6866209</td>\n",
              "      <td>homebrew</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>31212</td>\n",
              "      <td>202</td>\n",
              "      <td>gaia</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>30909</td>\n",
              "      <td>1803391</td>\n",
              "      <td>snoopys-sandbox</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>28820</td>\n",
              "      <td>3777</td>\n",
              "      <td>nixpkgs</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>28289</td>\n",
              "      <td>22091370</td>\n",
              "      <td>kubernetes</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>28273</td>\n",
              "      <td>1142</td>\n",
              "      <td>salt</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>23473</td>\n",
              "      <td>32902832</td>\n",
              "      <td>homebrew-core</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>22997</td>\n",
              "      <td>7301975</td>\n",
              "      <td>patchwork</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>21373</td>\n",
              "      <td>30513020</td>\n",
              "      <td>sauron-demo</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>20960</td>\n",
              "      <td>3149844</td>\n",
              "      <td>cmssw</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>20836</td>\n",
              "      <td>1334</td>\n",
              "      <td>rails</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>20806</td>\n",
              "      <td>8196280</td>\n",
              "      <td>spark</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>20747</td>\n",
              "      <td>1486</td>\n",
              "      <td>ansible</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>20554</td>\n",
              "      <td>17694</td>\n",
              "      <td>ceph</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>20537</td>\n",
              "      <td>11703554</td>\n",
              "      <td>code-dot-org</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    num_prs  base_repo_id                  name  deleted\n",
              "0     82846      20170979  everypolitician-data    False\n",
              "1     69340      70604300    signcla-probe-repo    False\n",
              "2     55760           789        liferay-portal    False\n",
              "3     54805      33253535                sauron    False\n",
              "4     40082       9570147         homebrew-cask    False\n",
              "5     33504       6866209              homebrew    False\n",
              "6     31212           202                  gaia    False\n",
              "7     30909       1803391       snoopys-sandbox    False\n",
              "8     28820          3777               nixpkgs    False\n",
              "9     28289      22091370            kubernetes    False\n",
              "10    28273          1142                  salt    False\n",
              "11    23473      32902832         homebrew-core    False\n",
              "12    22997       7301975             patchwork    False\n",
              "13    21373      30513020           sauron-demo    False\n",
              "14    20960       3149844                 cmssw    False\n",
              "15    20836          1334                 rails    False\n",
              "16    20806       8196280                 spark    False\n",
              "17    20747          1486               ansible    False\n",
              "18    20554         17694                  ceph    False\n",
              "19    20537      11703554          code-dot-org    False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "To8FochaWglF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Fetch the largest 20 projects by number of unqiue commenters\n",
        "%%bigquery --project cs224w-257119 df\n",
        "SELECT COUNT(comments.comment_id) as num_comments, COUNT(DISTINCT comments.user_id) as num_commenters, prs.base_repo_id\n",
        "FROM `ghtorrent-bq.ght_2018_04_01.pull_request_comments` comments\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.pull_requests` prs ON prs.id = comments.pull_request_id\n",
        "GROUP BY prs.base_repo_id\n",
        "ORDER BY num_commenters DESC\n",
        "LIMIT 20"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbkTWpepuxfC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Pulling the comment data; I threw in everything I thought may be useful\n",
        "\n",
        "%%bigquery --project cs224w-257119 pr_comments_spark\n",
        "SELECT pr_comments.pull_request_id AS pr_id, \n",
        "pr.pullreq_id as github_pr_id,\n",
        "pr_comments.user_id as commenter_id, \n",
        "head_commits.author_id as head_commit_author_id,\n",
        "head_commits.committer_id as head_commit_committer_id, \n",
        "base_commits.author_id as base_commit_author_id,\n",
        "base_commits.committer_id as base_commit_committer_id,\n",
        "pr.head_repo_id,\n",
        "pr.base_repo_id,\n",
        "base_project.owner_id as base_repo_owner,\n",
        "head_project.owner_id as head_repo_owner,\n",
        "pr.head_commit_id,\n",
        "pr.base_commit_id,\n",
        "pr.pullreq_id,\n",
        "pr_comments.created_at as comment_created_at,\n",
        "pr_comments.position as comment_position,\n",
        "pr_comments.comment_id,\n",
        "pr_comments.body as comment_body\n",
        "FROM \n",
        "`ghtorrent-bq.ght_2018_04_01.pull_request_comments` pr_comments\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.pull_requests` pr\n",
        "ON pr_comments.pull_request_id = pr.id\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.commits` head_commits\n",
        "ON pr.head_commit_id = head_commits.id\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.commits` base_commits\n",
        "ON pr.base_commit_id = base_commits.id\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.projects` base_project\n",
        "ON base_project.id = pr.base_repo_id\n",
        "INNER JOIN `ghtorrent-bq.ght_2018_04_01.projects` head_project\n",
        "ON head_project.id = pr.head_repo_id\n",
        "#Spark\n",
        "WHERE pr.base_repo_id = 8196280"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gW8BgJ0z3qEy",
        "colab_type": "text"
      },
      "source": [
        "The head committer and author tends to be the same. The base committer and author can vary.\n",
        "I checked, and the head commit ids match the user who submitted the pull request.\n",
        "I'm still not sure how exactly to interpertet the base_commit_author and base_committer_ids. \n",
        "For rereference, this is the pullrequest corresponding to row 0 (github_pr_id 18476)\n",
        "\n",
        "https://github.com/apache/spark/pull/18476\n",
        "\n",
        "The usernames of the ids are as follows:\n",
        "\n",
        "head_commit_author_id / head_commit_committer_id: **sadikovi**\n",
        "\n",
        "base_commit_author_id: **liufengdb**\n",
        "\n",
        "base_commit_committer_id: **zsxwing**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LD3M9M71JzX_",
        "colab_type": "code",
        "outputId": "b072c05a-b71b-4f4d-a09e-d5dd2cba0aaa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 762
        }
      },
      "source": [
        "pr_comments_spark"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pr_id</th>\n",
              "      <th>github_pr_id</th>\n",
              "      <th>commenter_id</th>\n",
              "      <th>head_commit_author_id</th>\n",
              "      <th>head_commit_committer_id</th>\n",
              "      <th>base_commit_author_id</th>\n",
              "      <th>base_commit_committer_id</th>\n",
              "      <th>head_repo_id</th>\n",
              "      <th>base_repo_id</th>\n",
              "      <th>base_repo_owner</th>\n",
              "      <th>head_repo_owner</th>\n",
              "      <th>head_commit_id</th>\n",
              "      <th>base_commit_id</th>\n",
              "      <th>pullreq_id</th>\n",
              "      <th>comment_created_at</th>\n",
              "      <th>comment_position</th>\n",
              "      <th>comment_id</th>\n",
              "      <th>comment_body</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>31859265</td>\n",
              "      <td>19854</td>\n",
              "      <td>231317</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>3394061</td>\n",
              "      <td>3394061</td>\n",
              "      <td>15912312</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>6186517</td>\n",
              "      <td>851427448</td>\n",
              "      <td>851176969</td>\n",
              "      <td>19854</td>\n",
              "      <td>2017-11-30 18:00:48+00:00</td>\n",
              "      <td>17</td>\n",
              "      <td>154154076</td>\n",
              "      <td>This change isn't valid and isn't related</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>31859265</td>\n",
              "      <td>19854</td>\n",
              "      <td>40383</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>3394061</td>\n",
              "      <td>3394061</td>\n",
              "      <td>15912312</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>6186517</td>\n",
              "      <td>851427448</td>\n",
              "      <td>851176969</td>\n",
              "      <td>19854</td>\n",
              "      <td>2017-12-01 04:33:54+00:00</td>\n",
              "      <td>17</td>\n",
              "      <td>154266377</td>\n",
              "      <td>ditto. I think it is better to remove commente...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>31859265</td>\n",
              "      <td>19854</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>3394061</td>\n",
              "      <td>3394061</td>\n",
              "      <td>15912312</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>6186517</td>\n",
              "      <td>851427448</td>\n",
              "      <td>851176969</td>\n",
              "      <td>19854</td>\n",
              "      <td>2017-12-06 08:16:22+00:00</td>\n",
              "      <td>17</td>\n",
              "      <td>155168946</td>\n",
              "      <td>@srowen ： have deleted changes about Platform....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>31859265</td>\n",
              "      <td>19854</td>\n",
              "      <td>40383</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>3394061</td>\n",
              "      <td>3394061</td>\n",
              "      <td>15912312</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>6186517</td>\n",
              "      <td>851427448</td>\n",
              "      <td>851176969</td>\n",
              "      <td>19854</td>\n",
              "      <td>2017-12-01 04:32:05+00:00</td>\n",
              "      <td>7</td>\n",
              "      <td>154266242</td>\n",
              "      <td>I think the direct reference should be https:/...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>31859265</td>\n",
              "      <td>19854</td>\n",
              "      <td>40383</td>\n",
              "      <td>6186517</td>\n",
              "      <td>6186517</td>\n",
              "      <td>3394061</td>\n",
              "      <td>3394061</td>\n",
              "      <td>15912312</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>6186517</td>\n",
              "      <td>851427448</td>\n",
              "      <td>851176969</td>\n",
              "      <td>19854</td>\n",
              "      <td>2017-12-01 04:33:03+00:00</td>\n",
              "      <td>7</td>\n",
              "      <td>154266308</td>\n",
              "      <td>And we should not leave commented code.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130629</th>\n",
              "      <td>23245670</td>\n",
              "      <td>17575</td>\n",
              "      <td>231317</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>28831</td>\n",
              "      <td>9760153</td>\n",
              "      <td>47401183</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>8172967</td>\n",
              "      <td>665447478</td>\n",
              "      <td>665273982</td>\n",
              "      <td>17575</td>\n",
              "      <td>2017-04-10 14:31:24+00:00</td>\n",
              "      <td>106</td>\n",
              "      <td>110671100</td>\n",
              "      <td>OK no problem, leave it. Just riffing while we...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130630</th>\n",
              "      <td>23245670</td>\n",
              "      <td>17575</td>\n",
              "      <td>231317</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>28831</td>\n",
              "      <td>9760153</td>\n",
              "      <td>47401183</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>8172967</td>\n",
              "      <td>665447478</td>\n",
              "      <td>665273982</td>\n",
              "      <td>17575</td>\n",
              "      <td>2017-04-10 13:59:38+00:00</td>\n",
              "      <td>64</td>\n",
              "      <td>110662125</td>\n",
              "      <td>or does `itemsets.foreach(set =&gt; uniqItems ++=...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130631</th>\n",
              "      <td>23245670</td>\n",
              "      <td>17575</td>\n",
              "      <td>231317</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>28831</td>\n",
              "      <td>9760153</td>\n",
              "      <td>47401183</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>8172967</td>\n",
              "      <td>665447478</td>\n",
              "      <td>665273982</td>\n",
              "      <td>17575</td>\n",
              "      <td>2017-04-11 07:07:26+00:00</td>\n",
              "      <td>70</td>\n",
              "      <td>110827932</td>\n",
              "      <td>Nit, but this should unindent one unit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130632</th>\n",
              "      <td>23245670</td>\n",
              "      <td>17575</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>28831</td>\n",
              "      <td>9760153</td>\n",
              "      <td>47401183</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>8172967</td>\n",
              "      <td>665447478</td>\n",
              "      <td>665273982</td>\n",
              "      <td>17575</td>\n",
              "      <td>2017-04-10 14:07:20+00:00</td>\n",
              "      <td>48</td>\n",
              "      <td>110664282</td>\n",
              "      <td>Yep, it can. It even avoids a useless cast.  I...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130633</th>\n",
              "      <td>23245670</td>\n",
              "      <td>17575</td>\n",
              "      <td>231317</td>\n",
              "      <td>8172967</td>\n",
              "      <td>8172967</td>\n",
              "      <td>28831</td>\n",
              "      <td>9760153</td>\n",
              "      <td>47401183</td>\n",
              "      <td>8196280</td>\n",
              "      <td>13369</td>\n",
              "      <td>8172967</td>\n",
              "      <td>665447478</td>\n",
              "      <td>665273982</td>\n",
              "      <td>17575</td>\n",
              "      <td>2017-04-10 08:36:13+00:00</td>\n",
              "      <td>84</td>\n",
              "      <td>110599784</td>\n",
              "      <td>We generally start wrapping the args to 4-spac...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>130634 rows × 18 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           pr_id  ...                                       comment_body\n",
              "0       31859265  ...          This change isn't valid and isn't related\n",
              "1       31859265  ...  ditto. I think it is better to remove commente...\n",
              "2       31859265  ...  @srowen ： have deleted changes about Platform....\n",
              "3       31859265  ...  I think the direct reference should be https:/...\n",
              "4       31859265  ...            And we should not leave commented code.\n",
              "...          ...  ...                                                ...\n",
              "130629  23245670  ...  OK no problem, leave it. Just riffing while we...\n",
              "130630  23245670  ...  or does `itemsets.foreach(set => uniqItems ++=...\n",
              "130631  23245670  ...             Nit, but this should unindent one unit\n",
              "130632  23245670  ...  Yep, it can. It even avoids a useless cast.  I...\n",
              "130633  23245670  ...  We generally start wrapping the args to 4-spac...\n",
              "\n",
              "[130634 rows x 18 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4GxrgaqqxYmP",
        "colab_type": "text"
      },
      "source": [
        "These next cells are so that we don't have to acess BigQeury everytime we want to work with the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsGmJw0OeSNF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Download to csv\n",
        "from google.colab import files\n",
        "\n",
        "with open('pr_comments_spark.csv', 'w') as f:\n",
        "  pr_comments_spark.to_csv(f)\n",
        "\n",
        "files.download('pr_comments_spark.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6K8MvtUxubKL",
        "colab_type": "text"
      },
      "source": [
        "I downloaded a copy of the csv and put in under data/pr_comments_spark.csv in out github repo. Pull the repo to get the data locally on your computer. Then, when you want to use it again, you can upload your local copy by running the cell below.\n",
        "\n",
        "I also discovered that collabs are saved as jupyter notebooks in the github repo, so I imagine you could also run them locally and get the data just by calling something like \n",
        "\n",
        "```\n",
        "df = pd.read_csv('data/pr_comments_spark.csv')\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pWs8KaIFwScn",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "outputId": "450d8600-d841-4d12-941d-a44e715064bb"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "csv_file = None\n",
        "for fn in uploaded.keys():\n",
        "  csv_file = uploaded[fn]\n",
        "\n",
        "if csv_file:\n",
        "  spark_data = pd.read_csv(csv_file)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-28dc2b34-7d5d-4cdf-9f1e-b34abdcfc97e\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-28dc2b34-7d5d-4cdf-9f1e-b34abdcfc97e\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}