preliminary_data_analysis.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"trueaskreddit_2020.csv\"\n",
    "output = \"new_data/trueaskreddit_2020.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     Post ID                                              Title  \\\n",
       "0     eicd1g  What video game is so good you can't believe i...   \n",
       "1     eidgb1       What is a girl version of Timothée Chalamet?   \n",
       "2     eiiklq  If the US just kinda of the left the middle Ea...   \n",
       "3     eik3j7  How do clubs etc, with a drink minimum handle ...   \n",
       "4     eipcan    Travel to the Future is possible with cryonics?   \n",
       "...      ...                                                ...   \n",
       "2222  k767tv  Are you sick of leaving items behind / losing ...   \n",
       "2223  k7fots  People of Reddit, what is it like growing up i...   \n",
       "2224  k7h019  People of Reddit who engage in the \"pls unfrie...   \n",
       "2225  k7p97p                                     I feel so numb   \n",
       "2226  k7ubyv                     Why are athletes paid so much?   \n",
       "\n",
       "                                                    Url              Author  \\\n",
       "0     https://www.reddit.com/r/TrueAskReddit/comment...         doorbellguy   \n",
       "1     /r/movies/comments/eidcbf/what_is_a_girl_versi...        dasisteinwug   \n",
       "2     https://www.reddit.com/r/TrueAskReddit/comment...        mojimbo54321   \n",
       "3     https://www.reddit.com/r/TrueAskReddit/comment...          Dolamite02   \n",
       "4     https://www.reddit.com/r/TrueAskReddit/comment...            twig4567   \n",
       "...                                                 ...                 ...   \n",
       "2222  https://www.reddit.com/r/TrueAskReddit/comment...            Revo0x0c   \n",
       "2223  https://www.reddit.com/r/TrueAskReddit/comment...     my_throw_awayyy   \n",
       "2224  https://www.reddit.com/r/TrueAskReddit/comment...      alexthagreat98   \n",
       "2225  https://www.reddit.com/r/TrueAskReddit/comment...      Ray_ofsunshine   \n",
       "2226  https://www.reddit.com/r/TrueAskReddit/comment...  Cwasaunt_Jimmy_Kun   \n",
       "\n",
       "      Score         Publish Date  Total No. of Comments  \\\n",
       "0         1  2020-01-01 09:55:10                      2   \n",
       "1         1  2020-01-01 11:40:07                      2   \n",
       "2         1  2020-01-01 21:57:03                     71   \n",
       "3         1  2020-01-02 00:25:11                      5   \n",
       "4         1  2020-01-02 07:07:36                      2   \n",
       "...     ...                  ...                    ...   \n",
       "2222      0  2020-12-05 19:53:49                      5   \n",
       "2223      1  2020-12-06 05:02:30                     31   \n",
       "2224      1  2020-12-06 06:16:15                      2   \n",
       "2225      1  2020-12-06 15:24:14                      5   \n",
       "2226      1  2020-12-06 22:06:34                    107   \n",
       "\n",
       "                                              Permalink  Flair  \\\n",
       "0     /r/TrueAskReddit/comments/eicd1g/what_video_ga...    NaN   \n",
       "1     /r/TrueAskReddit/comments/eidgb1/what_is_a_gir...    NaN   \n",
       "2     /r/TrueAskReddit/comments/eiiklq/if_the_us_jus...    NaN   \n",
       "3     /r/TrueAskReddit/comments/eik3j7/how_do_clubs_...    NaN   \n",
       "4     /r/TrueAskReddit/comments/eipcan/travel_to_the...    NaN   \n",
       "...                                                 ...    ...   \n",
       "2222  /r/TrueAskReddit/comments/k767tv/are_you_sick_...    NaN   \n",
       "2223  /r/TrueAskReddit/comments/k7fots/people_of_red...    NaN   \n",
       "2224  /r/TrueAskReddit/comments/k7h019/people_of_red...    NaN   \n",
       "2225   /r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/    NaN   \n",
       "2226  /r/TrueAskReddit/comments/k7ubyv/why_are_athle...    NaN   \n",
       "\n",
       "                                               SelfText  \n",
       "0                                                   NaN  \n",
       "1                                                   NaN  \n",
       "2     After hearing about the protestors at the Bagh...  \n",
       "3     I assume there's different policies, but surel...  \n",
       "4     You know how when you go to sleep, you have no...  \n",
       "...                                                 ...  \n",
       "2222   \\n\\nSuch as your phone, passport, wallet, mas...  \n",
       "2223                                                NaN  \n",
       "2224  This isn't an opinion post about whether you t...  \n",
       "2225  I moved out across the country by myself almos...  \n",
       "2226  I'm a huge sports fan myself but I have no ide...  \n",
       "\n",
       "[2227 rows x 10 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Post ID</th>\n      <th>Title</th>\n      <th>Url</th>\n      <th>Author</th>\n      <th>Score</th>\n      <th>Publish Date</th>\n      <th>Total No. of Comments</th>\n      <th>Permalink</th>\n      <th>Flair</th>\n      <th>SelfText</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>eicd1g</td>\n      <td>What video game is so good you can't believe i...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>doorbellguy</td>\n      <td>1</td>\n      <td>2020-01-01 09:55:10</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eicd1g/what_video_ga...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>eidgb1</td>\n      <td>What is a girl version of Timothée Chalamet?</td>\n      <td>/r/movies/comments/eidcbf/what_is_a_girl_versi...</td>\n      <td>dasisteinwug</td>\n      <td>1</td>\n      <td>2020-01-01 11:40:07</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eidgb1/what_is_a_gir...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>eiiklq</td>\n      <td>If the US just kinda of the left the middle Ea...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>mojimbo54321</td>\n      <td>1</td>\n      <td>2020-01-01 21:57:03</td>\n      <td>71</td>\n      <td>/r/TrueAskReddit/comments/eiiklq/if_the_us_jus...</td>\n      <td>NaN</td>\n      <td>After hearing about the protestors at the Bagh...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>eik3j7</td>\n      <td>How do clubs etc, with a drink minimum handle ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Dolamite02</td>\n      <td>1</td>\n      <td>2020-01-02 00:25:11</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/eik3j7/how_do_clubs_...</td>\n      <td>NaN</td>\n      <td>I assume there's different policies, but surel...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>eipcan</td>\n      <td>Travel to the Future is possible with cryonics?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>twig4567</td>\n      <td>1</td>\n      <td>2020-01-02 07:07:36</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eipcan/travel_to_the...</td>\n      <td>NaN</td>\n      <td>You know how when you go to sleep, you have no...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2222</th>\n      <td>k767tv</td>\n      <td>Are you sick of leaving items behind / losing ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Revo0x0c</td>\n      <td>0</td>\n      <td>2020-12-05 19:53:49</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k767tv/are_you_sick_...</td>\n      <td>NaN</td>\n      <td>\\n\\nSuch as your phone, passport, wallet, mas...</td>\n    </tr>\n    <tr>\n      <th>2223</th>\n      <td>k7fots</td>\n      <td>People of Reddit, what is it like growing up i...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>my_throw_awayyy</td>\n      <td>1</td>\n      <td>2020-12-06 05:02:30</td>\n      <td>31</td>\n      <td>/r/TrueAskReddit/comments/k7fots/people_of_red...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2224</th>\n      <td>k7h019</td>\n      <td>People of Reddit who engage in the \"pls unfrie...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>alexthagreat98</td>\n      <td>1</td>\n      <td>2020-12-06 06:16:15</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/k7h019/people_of_red...</td>\n      <td>NaN</td>\n      <td>This isn't an opinion post about whether you t...</td>\n    </tr>\n    <tr>\n      <th>2225</th>\n      <td>k7p97p</td>\n      <td>I feel so numb</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Ray_ofsunshine</td>\n      <td>1</td>\n      <td>2020-12-06 15:24:14</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/</td>\n      <td>NaN</td>\n      <td>I moved out across the country by myself almos...</td>\n    </tr>\n    <tr>\n      <th>2226</th>\n      <td>k7ubyv</td>\n      <td>Why are athletes paid so much?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Cwasaunt_Jimmy_Kun</td>\n      <td>1</td>\n      <td>2020-12-06 22:06:34</td>\n      <td>107</td>\n      <td>/r/TrueAskReddit/comments/k7ubyv/why_are_athle...</td>\n      <td>NaN</td>\n      <td>I'm a huge sports fan myself but I have no ide...</td>\n    </tr>\n  </tbody>\n</table>\n<p>2227 rows × 10 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     Post ID                                              Title  \\\n",
       "0     eicd1g  What video game is so good you can't believe i...   \n",
       "1     eidgb1       What is a girl version of Timothée Chalamet?   \n",
       "2     eiiklq  If the US just kinda of the left the middle Ea...   \n",
       "3     eik3j7  How do clubs etc, with a drink minimum handle ...   \n",
       "4     eipcan    Travel to the Future is possible with cryonics?   \n",
       "...      ...                                                ...   \n",
       "2222  k767tv  Are you sick of leaving items behind / losing ...   \n",
       "2223  k7fots  People of Reddit, what is it like growing up i...   \n",
       "2224  k7h019  People of Reddit who engage in the \"pls unfrie...   \n",
       "2225  k7p97p                                     I feel so numb   \n",
       "2226  k7ubyv                     Why are athletes paid so much?   \n",
       "\n",
       "                                                    Url              Author  \\\n",
       "0     https://www.reddit.com/r/TrueAskReddit/comment...         doorbellguy   \n",
       "1     /r/movies/comments/eidcbf/what_is_a_girl_versi...        dasisteinwug   \n",
       "2     https://www.reddit.com/r/TrueAskReddit/comment...        mojimbo54321   \n",
       "3     https://www.reddit.com/r/TrueAskReddit/comment...          Dolamite02   \n",
       "4     https://www.reddit.com/r/TrueAskReddit/comment...            twig4567   \n",
       "...                                                 ...                 ...   \n",
       "2222  https://www.reddit.com/r/TrueAskReddit/comment...            Revo0x0c   \n",
       "2223  https://www.reddit.com/r/TrueAskReddit/comment...     my_throw_awayyy   \n",
       "2224  https://www.reddit.com/r/TrueAskReddit/comment...      alexthagreat98   \n",
       "2225  https://www.reddit.com/r/TrueAskReddit/comment...      Ray_ofsunshine   \n",
       "2226  https://www.reddit.com/r/TrueAskReddit/comment...  Cwasaunt_Jimmy_Kun   \n",
       "\n",
       "      Score         Publish Date  Total No. of Comments  \\\n",
       "0         1  2020-01-01 09:55:10                      2   \n",
       "1         1  2020-01-01 11:40:07                      2   \n",
       "2         1  2020-01-01 21:57:03                     71   \n",
       "3         1  2020-01-02 00:25:11                      5   \n",
       "4         1  2020-01-02 07:07:36                      2   \n",
       "...     ...                  ...                    ...   \n",
       "2222      0  2020-12-05 19:53:49                      5   \n",
       "2223      1  2020-12-06 05:02:30                     31   \n",
       "2224      1  2020-12-06 06:16:15                      2   \n",
       "2225      1  2020-12-06 15:24:14                      5   \n",
       "2226      1  2020-12-06 22:06:34                    107   \n",
       "\n",
       "                                              Permalink  Flair  \\\n",
       "0     /r/TrueAskReddit/comments/eicd1g/what_video_ga...    NaN   \n",
       "1     /r/TrueAskReddit/comments/eidgb1/what_is_a_gir...    NaN   \n",
       "2     /r/TrueAskReddit/comments/eiiklq/if_the_us_jus...    NaN   \n",
       "3     /r/TrueAskReddit/comments/eik3j7/how_do_clubs_...    NaN   \n",
       "4     /r/TrueAskReddit/comments/eipcan/travel_to_the...    NaN   \n",
       "...                                                 ...    ...   \n",
       "2222  /r/TrueAskReddit/comments/k767tv/are_you_sick_...    NaN   \n",
       "2223  /r/TrueAskReddit/comments/k7fots/people_of_red...    NaN   \n",
       "2224  /r/TrueAskReddit/comments/k7h019/people_of_red...    NaN   \n",
       "2225   /r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/    NaN   \n",
       "2226  /r/TrueAskReddit/comments/k7ubyv/why_are_athle...    NaN   \n",
       "\n",
       "                                               SelfText  \n",
       "0                                                   NaN  \n",
       "1                                                   NaN  \n",
       "2     After hearing about the protestors at the Bagh...  \n",
       "3     I assume there's different policies, but surel...  \n",
       "4     You know how when you go to sleep, you have no...  \n",
       "...                                                 ...  \n",
       "2222   \\n\\nSuch as your phone, passport, wallet, mas...  \n",
       "2223                                                NaN  \n",
       "2224  This isn't an opinion post about whether you t...  \n",
       "2225  I moved out across the country by myself almos...  \n",
       "2226  I'm a huge sports fan myself but I have no ide...  \n",
       "\n",
       "[2227 rows x 10 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Post ID</th>\n      <th>Title</th>\n      <th>Url</th>\n      <th>Author</th>\n      <th>Score</th>\n      <th>Publish Date</th>\n      <th>Total No. of Comments</th>\n      <th>Permalink</th>\n      <th>Flair</th>\n      <th>SelfText</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>eicd1g</td>\n      <td>What video game is so good you can't believe i...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>doorbellguy</td>\n      <td>1</td>\n      <td>2020-01-01 09:55:10</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eicd1g/what_video_ga...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>eidgb1</td>\n      <td>What is a girl version of Timothée Chalamet?</td>\n      <td>/r/movies/comments/eidcbf/what_is_a_girl_versi...</td>\n      <td>dasisteinwug</td>\n      <td>1</td>\n      <td>2020-01-01 11:40:07</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eidgb1/what_is_a_gir...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>eiiklq</td>\n      <td>If the US just kinda of the left the middle Ea...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>mojimbo54321</td>\n      <td>1</td>\n      <td>2020-01-01 21:57:03</td>\n      <td>71</td>\n      <td>/r/TrueAskReddit/comments/eiiklq/if_the_us_jus...</td>\n      <td>NaN</td>\n      <td>After hearing about the protestors at the Bagh...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>eik3j7</td>\n      <td>How do clubs etc, with a drink minimum handle ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Dolamite02</td>\n      <td>1</td>\n      <td>2020-01-02 00:25:11</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/eik3j7/how_do_clubs_...</td>\n      <td>NaN</td>\n      <td>I assume there's different policies, but surel...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>eipcan</td>\n      <td>Travel to the Future is possible with cryonics?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>twig4567</td>\n      <td>1</td>\n      <td>2020-01-02 07:07:36</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eipcan/travel_to_the...</td>\n      <td>NaN</td>\n      <td>You know how when you go to sleep, you have no...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2222</th>\n      <td>k767tv</td>\n      <td>Are you sick of leaving items behind / losing ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Revo0x0c</td>\n      <td>0</td>\n      <td>2020-12-05 19:53:49</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k767tv/are_you_sick_...</td>\n      <td>NaN</td>\n      <td>\\n\\nSuch as your phone, passport, wallet, mas...</td>\n    </tr>\n    <tr>\n      <th>2223</th>\n      <td>k7fots</td>\n      <td>People of Reddit, what is it like growing up i...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>my_throw_awayyy</td>\n      <td>1</td>\n      <td>2020-12-06 05:02:30</td>\n      <td>31</td>\n      <td>/r/TrueAskReddit/comments/k7fots/people_of_red...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2224</th>\n      <td>k7h019</td>\n      <td>People of Reddit who engage in the \"pls unfrie...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>alexthagreat98</td>\n      <td>1</td>\n      <td>2020-12-06 06:16:15</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/k7h019/people_of_red...</td>\n      <td>NaN</td>\n      <td>This isn't an opinion post about whether you t...</td>\n    </tr>\n    <tr>\n      <th>2225</th>\n      <td>k7p97p</td>\n      <td>I feel so numb</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Ray_ofsunshine</td>\n      <td>1</td>\n      <td>2020-12-06 15:24:14</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/</td>\n      <td>NaN</td>\n      <td>I moved out across the country by myself almos...</td>\n    </tr>\n    <tr>\n      <th>2226</th>\n      <td>k7ubyv</td>\n      <td>Why are athletes paid so much?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Cwasaunt_Jimmy_Kun</td>\n      <td>1</td>\n      <td>2020-12-06 22:06:34</td>\n      <td>107</td>\n      <td>/r/TrueAskReddit/comments/k7ubyv/why_are_athle...</td>\n      <td>NaN</td>\n      <td>I'm a huge sports fan myself but I have no ide...</td>\n    </tr>\n  </tbody>\n</table>\n<p>2227 rows × 10 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 33
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 2227 entries, 0 to 2226\nData columns (total 10 columns):\n #   Column                 Non-Null Count  Dtype  \n---  ------                 --------------  -----  \n 0   Post ID                2227 non-null   object \n 1   Title                  2227 non-null   object \n 2   Url                    2227 non-null   object \n 3   Author                 2227 non-null   object \n 4   Score                  2227 non-null   int64  \n 5   Publish Date           2227 non-null   object \n 6   Total No. of Comments  2227 non-null   int64  \n 7   Permalink              2227 non-null   object \n 8   Flair                  0 non-null      float64\n 9   SelfText               1747 non-null   object \ndtypes: float64(1), int64(2), object(7)\nmemory usage: 174.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nInt64Index: 1729 entries, 2 to 2226\nData columns (total 10 columns):\n #   Column                 Non-Null Count  Dtype  \n---  ------                 --------------  -----  \n 0   Post ID                1729 non-null   object \n 1   Title                  1729 non-null   object \n 2   Url                    1729 non-null   object \n 3   Author                 1729 non-null   object \n 4   Score                  1729 non-null   int64  \n 5   Publish Date           1729 non-null   object \n 6   Total No. of Comments  1729 non-null   int64  \n 7   Permalink              1729 non-null   object \n 8   Flair                  0 non-null      float64\n 9   SelfText               1729 non-null   object \ndtypes: float64(1), int64(2), object(7)\nmemory usage: 148.6+ KB\n"
     ]
    }
   ],
   "source": [
    "def dataclean(data):\n",
    "    d = data.drop_duplicates()\n",
    "    d.dropna(subset=['SelfText'], inplace = True)\n",
    "    d = d[d.SelfText != \"[deleted]\"]\n",
    "    return d\n",
    "\n",
    "d_2 = dataclean(df)\n",
    "d_2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     Post ID                                              Title  \\\n",
       "2     eiiklq  If the US just kinda of the left the middle Ea...   \n",
       "3     eik3j7  How do clubs etc, with a drink minimum handle ...   \n",
       "4     eipcan    Travel to the Future is possible with cryonics?   \n",
       "5     eir251               How does one overcome fear of death?   \n",
       "6     eir6rr  How practically valuable are academic research...   \n",
       "...      ...                                                ...   \n",
       "2221  k6ydrq  So can synthesized meat replaced traditional m...   \n",
       "2222  k767tv  Are you sick of leaving items behind / losing ...   \n",
       "2224  k7h019  People of Reddit who engage in the \"pls unfrie...   \n",
       "2225  k7p97p                                     I feel so numb   \n",
       "2226  k7ubyv                     Why are athletes paid so much?   \n",
       "\n",
       "                                                    Url              Author  \\\n",
       "2     https://www.reddit.com/r/TrueAskReddit/comment...        mojimbo54321   \n",
       "3     https://www.reddit.com/r/TrueAskReddit/comment...          Dolamite02   \n",
       "4     https://www.reddit.com/r/TrueAskReddit/comment...            twig4567   \n",
       "5     https://www.reddit.com/r/TrueAskReddit/comment...           Dharmsara   \n",
       "6     https://www.reddit.com/r/TrueAskReddit/comment...       ForbidReality   \n",
       "...                                                 ...                 ...   \n",
       "2221  https://www.reddit.com/r/TrueAskReddit/comment...             bivox01   \n",
       "2222  https://www.reddit.com/r/TrueAskReddit/comment...            Revo0x0c   \n",
       "2224  https://www.reddit.com/r/TrueAskReddit/comment...      alexthagreat98   \n",
       "2225  https://www.reddit.com/r/TrueAskReddit/comment...      Ray_ofsunshine   \n",
       "2226  https://www.reddit.com/r/TrueAskReddit/comment...  Cwasaunt_Jimmy_Kun   \n",
       "\n",
       "      Score         Publish Date  Total No. of Comments  \\\n",
       "2         1  2020-01-01 21:57:03                     71   \n",
       "3         1  2020-01-02 00:25:11                      5   \n",
       "4         1  2020-01-02 07:07:36                      2   \n",
       "5         1  2020-01-02 09:23:37                     51   \n",
       "6         1  2020-01-02 09:33:43                      2   \n",
       "...     ...                  ...                    ...   \n",
       "2221     27  2020-12-05 09:33:53                     38   \n",
       "2222      0  2020-12-05 19:53:49                      5   \n",
       "2224      1  2020-12-06 06:16:15                      2   \n",
       "2225      1  2020-12-06 15:24:14                      5   \n",
       "2226      1  2020-12-06 22:06:34                    107   \n",
       "\n",
       "                                              Permalink  Flair  \\\n",
       "2     /r/TrueAskReddit/comments/eiiklq/if_the_us_jus...    NaN   \n",
       "3     /r/TrueAskReddit/comments/eik3j7/how_do_clubs_...    NaN   \n",
       "4     /r/TrueAskReddit/comments/eipcan/travel_to_the...    NaN   \n",
       "5     /r/TrueAskReddit/comments/eir251/how_does_one_...    NaN   \n",
       "6     /r/TrueAskReddit/comments/eir6rr/how_practical...    NaN   \n",
       "...                                                 ...    ...   \n",
       "2221  /r/TrueAskReddit/comments/k6ydrq/so_can_synthe...    NaN   \n",
       "2222  /r/TrueAskReddit/comments/k767tv/are_you_sick_...    NaN   \n",
       "2224  /r/TrueAskReddit/comments/k7h019/people_of_red...    NaN   \n",
       "2225   /r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/    NaN   \n",
       "2226  /r/TrueAskReddit/comments/k7ubyv/why_are_athle...    NaN   \n",
       "\n",
       "                                               SelfText  \n",
       "2     After hearing about the protestors at the Bagh...  \n",
       "3     I assume there's different policies, but surel...  \n",
       "4     You know how when you go to sleep, you have no...  \n",
       "5     Hi Reddit. I’m asking for a friend, I hope you...  \n",
       "6     The relationship between scientific publishing...  \n",
       "...                                                 ...  \n",
       "2221  Seen an interesting article about it . And lon...  \n",
       "2222   \\n\\nSuch as your phone, passport, wallet, mas...  \n",
       "2224  This isn't an opinion post about whether you t...  \n",
       "2225  I moved out across the country by myself almos...  \n",
       "2226  I'm a huge sports fan myself but I have no ide...  \n",
       "\n",
       "[1729 rows x 10 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Post ID</th>\n      <th>Title</th>\n      <th>Url</th>\n      <th>Author</th>\n      <th>Score</th>\n      <th>Publish Date</th>\n      <th>Total No. of Comments</th>\n      <th>Permalink</th>\n      <th>Flair</th>\n      <th>SelfText</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>eiiklq</td>\n      <td>If the US just kinda of the left the middle Ea...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>mojimbo54321</td>\n      <td>1</td>\n      <td>2020-01-01 21:57:03</td>\n      <td>71</td>\n      <td>/r/TrueAskReddit/comments/eiiklq/if_the_us_jus...</td>\n      <td>NaN</td>\n      <td>After hearing about the protestors at the Bagh...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>eik3j7</td>\n      <td>How do clubs etc, with a drink minimum handle ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Dolamite02</td>\n      <td>1</td>\n      <td>2020-01-02 00:25:11</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/eik3j7/how_do_clubs_...</td>\n      <td>NaN</td>\n      <td>I assume there's different policies, but surel...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>eipcan</td>\n      <td>Travel to the Future is possible with cryonics?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>twig4567</td>\n      <td>1</td>\n      <td>2020-01-02 07:07:36</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eipcan/travel_to_the...</td>\n      <td>NaN</td>\n      <td>You know how when you go to sleep, you have no...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>eir251</td>\n      <td>How does one overcome fear of death?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Dharmsara</td>\n      <td>1</td>\n      <td>2020-01-02 09:23:37</td>\n      <td>51</td>\n      <td>/r/TrueAskReddit/comments/eir251/how_does_one_...</td>\n      <td>NaN</td>\n      <td>Hi Reddit. I’m asking for a friend, I hope you...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>eir6rr</td>\n      <td>How practically valuable are academic research...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>ForbidReality</td>\n      <td>1</td>\n      <td>2020-01-02 09:33:43</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/eir6rr/how_practical...</td>\n      <td>NaN</td>\n      <td>The relationship between scientific publishing...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2221</th>\n      <td>k6ydrq</td>\n      <td>So can synthesized meat replaced traditional m...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>bivox01</td>\n      <td>27</td>\n      <td>2020-12-05 09:33:53</td>\n      <td>38</td>\n      <td>/r/TrueAskReddit/comments/k6ydrq/so_can_synthe...</td>\n      <td>NaN</td>\n      <td>Seen an interesting article about it . And lon...</td>\n    </tr>\n    <tr>\n      <th>2222</th>\n      <td>k767tv</td>\n      <td>Are you sick of leaving items behind / losing ...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Revo0x0c</td>\n      <td>0</td>\n      <td>2020-12-05 19:53:49</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k767tv/are_you_sick_...</td>\n      <td>NaN</td>\n      <td>\\n\\nSuch as your phone, passport, wallet, mas...</td>\n    </tr>\n    <tr>\n      <th>2224</th>\n      <td>k7h019</td>\n      <td>People of Reddit who engage in the \"pls unfrie...</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>alexthagreat98</td>\n      <td>1</td>\n      <td>2020-12-06 06:16:15</td>\n      <td>2</td>\n      <td>/r/TrueAskReddit/comments/k7h019/people_of_red...</td>\n      <td>NaN</td>\n      <td>This isn't an opinion post about whether you t...</td>\n    </tr>\n    <tr>\n      <th>2225</th>\n      <td>k7p97p</td>\n      <td>I feel so numb</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Ray_ofsunshine</td>\n      <td>1</td>\n      <td>2020-12-06 15:24:14</td>\n      <td>5</td>\n      <td>/r/TrueAskReddit/comments/k7p97p/i_feel_so_numb/</td>\n      <td>NaN</td>\n      <td>I moved out across the country by myself almos...</td>\n    </tr>\n    <tr>\n      <th>2226</th>\n      <td>k7ubyv</td>\n      <td>Why are athletes paid so much?</td>\n      <td>https://www.reddit.com/r/TrueAskReddit/comment...</td>\n      <td>Cwasaunt_Jimmy_Kun</td>\n      <td>1</td>\n      <td>2020-12-06 22:06:34</td>\n      <td>107</td>\n      <td>/r/TrueAskReddit/comments/k7ubyv/why_are_athle...</td>\n      <td>NaN</td>\n      <td>I'm a huge sports fan myself but I have no ide...</td>\n    </tr>\n  </tbody>\n</table>\n<p>1729 rows × 10 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "d_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "metadata": {},
     "execution_count": 37
    }
   ],
   "source": [
    "type(d_2.SelfText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_2.SelfText.to_csv(output, index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "## BATCH MODE\n",
    "# post_counts = []\n",
    "# for i in range(2010, 2021):\n",
    "#     filename = \"suicidewatch_\" + str(i) + \".csv\"\n",
    "#     output = \"new_data/sw_\"+ str(i) +\".txt\"\n",
    "#     df = pd.read_csv(filename)\n",
    "#     df = dataclean(df)\n",
    "#     post_counts.append(df.shape[0])\n",
    "#     df.SelfText.to_csv(output, index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(post_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}