Test Database Builder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Routine stuff\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from string import punctuation\n",
    "import os\n",
    "import operator\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import vocab\n",
    "data = pd.read_csv(\"vocab.csv\",delimiter=\",\")\n",
    "data = data.drop('Unnamed: 0',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['apr',\n",
       " 'gmt',\n",
       " '1993',\n",
       " 'writes',\n",
       " 'references',\n",
       " 'article',\n",
       " 'sender',\n",
       " 'people',\n",
       " 'university',\n",
       " '1',\n",
       " 'nntp-posting-host',\n",
       " '93',\n",
       " '2',\n",
       " 'xref',\n",
       " 'cantaloupe.srv.cs.cmu.edu',\n",
       " 'news',\n",
       " 'distribution',\n",
       " '3',\n",
       " 'god',\n",
       " '0',\n",
       " \"max>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax\",\n",
       " '20',\n",
       " '5',\n",
       " '6',\n",
       " '4',\n",
       " '16',\n",
       " '15',\n",
       " 'government',\n",
       " '21',\n",
       " 'question',\n",
       " 'reply-to',\n",
       " 'read',\n",
       " 'usenet',\n",
       " 'usa',\n",
       " 'windows',\n",
       " 'david',\n",
       " 'software',\n",
       " 'data',\n",
       " 'program',\n",
       " 'file',\n",
       " '23',\n",
       " '10',\n",
       " 'space',\n",
       " 'version',\n",
       " 'science',\n",
       " 'power',\n",
       " 'john',\n",
       " 'drive',\n",
       " '22',\n",
       " 'fri',\n",
       " 'day',\n",
       " 'true',\n",
       " 'law',\n",
       " 'public',\n",
       " 'tue',\n",
       " 'game',\n",
       " '19',\n",
       " '8',\n",
       " '7',\n",
       " 'lot',\n",
       " 'car',\n",
       " 'set',\n",
       " 'real',\n",
       " '17',\n",
       " 'systems',\n",
       " 'support',\n",
       " 'life',\n",
       " 'free',\n",
       " 'list',\n",
       " 'jesus',\n",
       " 'image',\n",
       " 'sun',\n",
       " '18',\n",
       " 'card',\n",
       " '24',\n",
       " 'reason',\n",
       " '25',\n",
       " 'wrong',\n",
       " 'message',\n",
       " 'thu',\n",
       " 'hard',\n",
       " 'post',\n",
       " 'key',\n",
       " 'mon',\n",
       " 'center',\n",
       " 'internet',\n",
       " 'technology',\n",
       " 'bit',\n",
       " 'email',\n",
       " 'bad',\n",
       " 'send',\n",
       " 'files',\n",
       " 'heard',\n",
       " 'team',\n",
       " '26',\n",
       " 'person',\n",
       " 'evidence',\n",
       " 'christian',\n",
       " 'access',\n",
       " 'mark',\n",
       " 'wrote',\n",
       " 'jews',\n",
       " 'national',\n",
       " 'children',\n",
       " '14',\n",
       " 'idea',\n",
       " 'human',\n",
       " 'e-mail',\n",
       " 'opinions',\n",
       " '12',\n",
       " 'mail',\n",
       " 'times',\n",
       " 'control',\n",
       " 'keywords',\n",
       " 'questions',\n",
       " 'book',\n",
       " 'change',\n",
       " 'start',\n",
       " 'note',\n",
       " 'left',\n",
       " 'chip',\n",
       " 'remember',\n",
       " 'phone',\n",
       " '30',\n",
       " 'window',\n",
       " 'israel',\n",
       " 'based',\n",
       " 'word',\n",
       " '9',\n",
       " 'code',\n",
       " 'michael',\n",
       " 'gun',\n",
       " 'standard',\n",
       " 'institute',\n",
       " 'followup-to',\n",
       " 'source',\n",
       " '11',\n",
       " 'stuff',\n",
       " 'rights',\n",
       " 'net',\n",
       " 'info',\n",
       " '13',\n",
       " 'money',\n",
       " 'department',\n",
       " 'games',\n",
       " 'president',\n",
       " 'answer',\n",
       " '27',\n",
       " 'price',\n",
       " 'running',\n",
       " 'graphics',\n",
       " 'agree',\n",
       " 'dos',\n",
       " 'local',\n",
       " 'address',\n",
       " 'steve',\n",
       " 'buy',\n",
       " 'matter',\n",
       " 'american',\n",
       " 'paul',\n",
       " 'days',\n",
       " 'mac',\n",
       " 'bible',\n",
       " 'mike',\n",
       " 'war',\n",
       " 'hope',\n",
       " 'told',\n",
       " 'pretty',\n",
       " 'live',\n",
       " 'approved',\n",
       " 'care',\n",
       " 'disk',\n",
       " 'current',\n",
       " 'play',\n",
       " 'issue',\n",
       " 'original',\n",
       " 'feel',\n",
       " 'claim',\n",
       " 'sale',\n",
       " 'network',\n",
       " 'love',\n",
       " 'jim',\n",
       " 'understand',\n",
       " 'fax',\n",
       " 'religion',\n",
       " 'truth',\n",
       " 'box',\n",
       " 'programs',\n",
       " 'simply',\n",
       " 'college',\n",
       " 'mind',\n",
       " 'history',\n",
       " 'april',\n",
       " 'machine',\n",
       " 'speed',\n",
       " 'engineering',\n",
       " 'started',\n",
       " 'guess',\n",
       " 'armenian',\n",
       " 'video',\n",
       " 'canada',\n",
       " 'church',\n",
       " 'fbi',\n",
       " 'dept',\n",
       " 'memory',\n",
       " 'posting',\n",
       " 'unix',\n",
       " 'hand',\n",
       " 'robert',\n",
       " 'health',\n",
       " 'color',\n",
       " 'country',\n",
       " 'pc',\n",
       " 'encryption',\n",
       " 'service',\n",
       " 'pay',\n",
       " 'turkish',\n",
       " 'jewish',\n",
       " 'private',\n",
       " 'u.s',\n",
       " 'earth',\n",
       " 'house',\n",
       " 'x-newsreader',\n",
       " 'black',\n",
       " 'rec.sport.hockey',\n",
       " '28',\n",
       " 'christians',\n",
       " 'death',\n",
       " 'difference',\n",
       " 'display',\n",
       " 'user',\n",
       " 'school',\n",
       " 'ftp',\n",
       " 'sat',\n",
       " 'win',\n",
       " 'type',\n",
       " 'copy',\n",
       " 'server',\n",
       " 'california',\n",
       " 'hell',\n",
       " 'armenians',\n",
       " 'white',\n",
       " 'view',\n",
       " 'opinion',\n",
       " 'sense',\n",
       " 'faith',\n",
       " 'company',\n",
       " '1.1',\n",
       " 'clipper',\n",
       " 'soc.religion.christian',\n",
       " 'sort',\n",
       " 'package',\n",
       " 'deal',\n",
       " 'police',\n",
       " 'rec.sport.baseball',\n",
       " 'political',\n",
       " 'summary',\n",
       " 'major',\n",
       " 'rec.motorcycles',\n",
       " 'morality',\n",
       " 'moral',\n",
       " 'city',\n",
       " 'services',\n",
       " 'contact',\n",
       " 'talking',\n",
       " 'christian@aramis.rutgers.edu',\n",
       " 'check',\n",
       " 'couple',\n",
       " 'christ',\n",
       " 'single',\n",
       " 'killed',\n",
       " 'nice',\n",
       " 'cost',\n",
       " 'frank',\n",
       " 'write',\n",
       " 'body',\n",
       " 'security',\n",
       " 'james',\n",
       " '29',\n",
       " 'light',\n",
       " 'press',\n",
       " 'religious',\n",
       " 'argument',\n",
       " 'objective',\n",
       " 'happened',\n",
       " 'ibm',\n",
       " 'simple',\n",
       " 'san',\n",
       " 'dead',\n",
       " 'images',\n",
       " 'night',\n",
       " 'correct',\n",
       " 'hockey',\n",
       " 'monitor',\n",
       " 'provide',\n",
       " 'comp.sys.mac.hardware',\n",
       " '100',\n",
       " 'board',\n",
       " 'tin',\n",
       " 'fine',\n",
       " 'society',\n",
       " '50',\n",
       " 'fast',\n",
       " 'statement',\n",
       " 'position',\n",
       " 'form',\n",
       " 'experience',\n",
       " 'hardware',\n",
       " 'dave',\n",
       " 'western',\n",
       " 'format',\n",
       " 'situation',\n",
       " 'force',\n",
       " 'rest',\n",
       " 'york',\n",
       " 'rec.autos',\n",
       " 'women',\n",
       " 'washington',\n",
       " 'reading',\n",
       " 'common',\n",
       " 'laws',\n",
       " 'clinton',\n",
       " 'text',\n",
       " 'israeli',\n",
       " 'division',\n",
       " 'books',\n",
       " 'brian',\n",
       " 'sound',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'personal',\n",
       " 'discussion',\n",
       " 'guy',\n",
       " 'period',\n",
       " 'players',\n",
       " 'users',\n",
       " 'lost',\n",
       " 'scsi',\n",
       " 'jpeg',\n",
       " 'study',\n",
       " 'hear',\n",
       " 'business',\n",
       " 'alt.atheism',\n",
       " 'size',\n",
       " 'bob',\n",
       " 'test',\n",
       " 'exist',\n",
       " 'advance',\n",
       " 'bike',\n",
       " 'happen',\n",
       " 'theory',\n",
       " 'account',\n",
       " 'hit',\n",
       " 'red',\n",
       " 'radio',\n",
       " 'screen',\n",
       " 'future',\n",
       " 'peter',\n",
       " 'corporation',\n",
       " 'head',\n",
       " 'office',\n",
       " 'series',\n",
       " 'apple',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'application',\n",
       " 'talk',\n",
       " 'accept',\n",
       " 'easy',\n",
       " 'medical',\n",
       " 'level',\n",
       " 'faq',\n",
       " 'special',\n",
       " 'add',\n",
       " 'months',\n",
       " 'considered',\n",
       " 'article-i.d',\n",
       " 'driver',\n",
       " 'insurance',\n",
       " 'week',\n",
       " 'guns',\n",
       " 'drivers',\n",
       " 'posted',\n",
       " 'ms',\n",
       " 'season',\n",
       " 'kill',\n",
       " 'worth',\n",
       " 'job',\n",
       " 'koresh',\n",
       " 'sell',\n",
       " 'peace',\n",
       " 'runs',\n",
       " 'expect',\n",
       " 'ma',\n",
       " 'short',\n",
       " 'project',\n",
       " '32',\n",
       " 'manager',\n",
       " 'db',\n",
       " 'comp.windows.x',\n",
       " 'pittsburgh',\n",
       " 'thinking',\n",
       " 'texas',\n",
       " 'international',\n",
       " 'model',\n",
       " 'assume',\n",
       " 'ground',\n",
       " 'coming',\n",
       " 'ii',\n",
       " 'values',\n",
       " 'communications',\n",
       " 'doubt',\n",
       " 'quality',\n",
       " 'mode',\n",
       " 'military',\n",
       " 'land',\n",
       " 'united',\n",
       " 'federal',\n",
       " 'st',\n",
       " '40',\n",
       " 'reference',\n",
       " 'includes',\n",
       " 'offer',\n",
       " 'comp.graphics',\n",
       " 'speak',\n",
       " 'process',\n",
       " 'bbs',\n",
       " 'tom',\n",
       " 'policy',\n",
       " 'response',\n",
       " 'i.e',\n",
       " 'hold',\n",
       " 'strong',\n",
       " 'sci.space',\n",
       " 'involved',\n",
       " 'thomas',\n",
       " 'eric',\n",
       " 'story',\n",
       " 'baseball',\n",
       " 'explain',\n",
       " 'previous',\n",
       " 'cars',\n",
       " 'bus',\n",
       " 'gas',\n",
       " 'toronto',\n",
       " 'specific',\n",
       " 'performance',\n",
       " 'talk.politics.guns',\n",
       " 'computing',\n",
       " 'issues',\n",
       " 'road',\n",
       " 'weapons',\n",
       " 'andrew',\n",
       " 'error',\n",
       " 'dr',\n",
       " 'freedom',\n",
       " 'building',\n",
       " 'result',\n",
       " 'total',\n",
       " 'legal',\n",
       " 'design',\n",
       " 'dod',\n",
       " 'water',\n",
       " 'population',\n",
       " '0400',\n",
       " 'league',\n",
       " 'ram',\n",
       " 'leave',\n",
       " 'development',\n",
       " 'field',\n",
       " 'save',\n",
       " 'christianity',\n",
       " 'party',\n",
       " 'deleted',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'report',\n",
       " 'action',\n",
       " 'close',\n",
       " 'rate',\n",
       " 'appreciated',\n",
       " 'tape',\n",
       " 'basis',\n",
       " 'player',\n",
       " 'friend',\n",
       " 'child',\n",
       " 'richard',\n",
       " 'claims',\n",
       " 'return',\n",
       " 'smith',\n",
       " 'nasa',\n",
       " 'tv',\n",
       " 'knowledge',\n",
       " 'choice',\n",
       " 'voice',\n",
       " 'media',\n",
       " 'cut',\n",
       " 'technical',\n",
       " 'sci.crypt',\n",
       " 'average',\n",
       " 'ideas',\n",
       " '1992',\n",
       " 'build',\n",
       " 'boston',\n",
       " 'reasons',\n",
       " 'mentioned',\n",
       " 'reply',\n",
       " 'jon',\n",
       " 'scott',\n",
       " 'armenia',\n",
       " 'drug',\n",
       " 'applications',\n",
       " 'complete',\n",
       " 'sin',\n",
       " 'takes',\n",
       " 'administration',\n",
       " 'univ',\n",
       " 'exists',\n",
       " 'chicago',\n",
       " '31',\n",
       " 'die',\n",
       " 'drives',\n",
       " 'crime',\n",
       " 'anonymous',\n",
       " 'completely',\n",
       " 'disclaimer',\n",
       " 'appears',\n",
       " 'sounds',\n",
       " 'required',\n",
       " 'batf',\n",
       " 'comments',\n",
       " 'shot',\n",
       " 'court',\n",
       " 'output',\n",
       " 'modem',\n",
       " 'chris',\n",
       " 'weeks',\n",
       " 'bought',\n",
       " 'germany',\n",
       " 'scientific',\n",
       " 'details',\n",
       " 'basic',\n",
       " 'lord',\n",
       " 'newsgroup',\n",
       " 'waco',\n",
       " 'al',\n",
       " 'nature',\n",
       " 'created',\n",
       " 'des',\n",
       " 'keith',\n",
       " 'illinois',\n",
       " 'sources',\n",
       " \"god's\",\n",
       " 'existence',\n",
       " 'muslim',\n",
       " 'living',\n",
       " 'cover',\n",
       " 'looked',\n",
       " 'universe',\n",
       " 'chance',\n",
       " 'create',\n",
       " 'food',\n",
       " 'air',\n",
       " 'individual',\n",
       " 'supposed',\n",
       " 'cards',\n",
       " 'controller',\n",
       " 'george',\n",
       " 'finally',\n",
       " 'meaning',\n",
       " 'belief',\n",
       " 'terms',\n",
       " '33',\n",
       " 'algorithm',\n",
       " 'main',\n",
       " 'faster',\n",
       " 'purpose',\n",
       " 'digital',\n",
       " 'machines',\n",
       " 'arab',\n",
       " 'views',\n",
       " 'folks',\n",
       " 'russian',\n",
       " 'arms',\n",
       " 'follow',\n",
       " 'muslims',\n",
       " 'site',\n",
       " 'manual',\n",
       " 'mouse',\n",
       " 'mass',\n",
       " 'blue',\n",
       " 'in-reply-to',\n",
       " 'lead',\n",
       " 'final',\n",
       " 'attack',\n",
       " 'north',\n",
       " 'prove',\n",
       " 'plan',\n",
       " 'family',\n",
       " 'term',\n",
       " 'turkey',\n",
       " 'language',\n",
       " 'noise',\n",
       " 'atheists',\n",
       " 'community',\n",
       " 'learn',\n",
       " 'base',\n",
       " 'events',\n",
       " 'figure',\n",
       " 'genocide',\n",
       " 'rule',\n",
       " 'goal',\n",
       " 'built',\n",
       " 'directory',\n",
       " 'secret',\n",
       " 'entire',\n",
       " 'interface',\n",
       " 'command',\n",
       " 'ny',\n",
       " 'cheap',\n",
       " 'lots',\n",
       " 'america',\n",
       " 'reasonable',\n",
       " 'motif',\n",
       " 'designed',\n",
       " 'object',\n",
       " 'tim',\n",
       " '2000',\n",
       " 'condition',\n",
       " 'record',\n",
       " 'dan',\n",
       " 'wait',\n",
       " 'commercial',\n",
       " 'joseph',\n",
       " 'advice',\n",
       " 'engine',\n",
       " 'normal',\n",
       " 'gif',\n",
       " 'allowed',\n",
       " 'paper',\n",
       " 'sex',\n",
       " 'library',\n",
       " 'drugs',\n",
       " 'defense',\n",
       " 'rules',\n",
       " 'stated',\n",
       " 'hours',\n",
       " 'device',\n",
       " 'release',\n",
       " 'uk',\n",
       " 'south',\n",
       " 'miles',\n",
       " 'carry',\n",
       " 'event',\n",
       " 'edt',\n",
       " 'lab',\n",
       " 'product',\n",
       " 'serial',\n",
       " 'laboratory',\n",
       " 'easily',\n",
       " 'port',\n",
       " 'privacy',\n",
       " 'reality',\n",
       " 'request',\n",
       " 'wondering',\n",
       " 'difficult',\n",
       " 'tax',\n",
       " 'pat',\n",
       " 'son',\n",
       " 'happy',\n",
       " 'la',\n",
       " 'fan',\n",
       " 'responsible',\n",
       " 'break',\n",
       " 'bring',\n",
       " 'poster',\n",
       " 'decided',\n",
       " 'stupid',\n",
       " 'joe',\n",
       " '3.1',\n",
       " 'studies',\n",
       " 'lives',\n",
       " 'hp',\n",
       " 'solution',\n",
       " 'market',\n",
       " 'physical',\n",
       " 'definition',\n",
       " 'products',\n",
       " 'bits',\n",
       " 'turks',\n",
       " 'method',\n",
       " '35',\n",
       " 'suppose',\n",
       " 'west',\n",
       " 'citizens',\n",
       " 'changed',\n",
       " 'illegal',\n",
       " 'trouble',\n",
       " 'analysis',\n",
       " 'station',\n",
       " 'owner',\n",
       " 'mention',\n",
       " 'guys',\n",
       " 'equipment',\n",
       " 'companies',\n",
       " 'attempt',\n",
       " 'actions',\n",
       " 'originator',\n",
       " 'oil',\n",
       " 'abortion',\n",
       " 'father',\n",
       " 'conference',\n",
       " 'disease',\n",
       " 'green',\n",
       " 'imagine',\n",
       " 'reported',\n",
       " 'class',\n",
       " 'fans',\n",
       " 'civil',\n",
       " 'died',\n",
       " 'corp',\n",
       " 'brought',\n",
       " 'forget',\n",
       " 'pick',\n",
       " 'age',\n",
       " 'limited',\n",
       " 'official',\n",
       " 'ray',\n",
       " 'devices',\n",
       " 'printer',\n",
       " 'hedrick@athos.rutgers.edu',\n",
       " 'wife',\n",
       " 'launch',\n",
       " 'apply',\n",
       " 'letter',\n",
       " 'murder',\n",
       " 'extra',\n",
       " 'street',\n",
       " 'east',\n",
       " 'lack',\n",
       " 'education',\n",
       " 'chips',\n",
       " 'gm',\n",
       " 'door',\n",
       " 'decision',\n",
       " 'environment',\n",
       " 'greek',\n",
       " 'watch',\n",
       " 'americans',\n",
       " 'hate',\n",
       " 'misc.forsale',\n",
       " 'avoid',\n",
       " 'stand',\n",
       " 'georgia',\n",
       " 'e.g',\n",
       " 'congress',\n",
       " 'heart',\n",
       " 'king',\n",
       " 'ide',\n",
       " 'homosexuality',\n",
       " 'btw',\n",
       " 'search',\n",
       " 'mission',\n",
       " 'cable',\n",
       " 'require',\n",
       " 'reports',\n",
       " 'trade',\n",
       " 'function',\n",
       " 'coverage',\n",
       " 'caused',\n",
       " 'worse',\n",
       " 'safety',\n",
       " 'fall',\n",
       " 'teams',\n",
       " 'evil',\n",
       " 'gary',\n",
       " 'online',\n",
       " 'compound',\n",
       " 'nhl',\n",
       " 'quote',\n",
       " 'volume',\n",
       " '1st',\n",
       " 'internal',\n",
       " 'continue',\n",
       " 'authority',\n",
       " 'europe',\n",
       " 'ken',\n",
       " 'excellent',\n",
       " 'choose',\n",
       " 'ability',\n",
       " 'blood',\n",
       " 'starting',\n",
       " 'serdar',\n",
       " 'input',\n",
       " 'natural',\n",
       " 'material',\n",
       " 'addition',\n",
       " 'practice',\n",
       " 'microsoft',\n",
       " 'pass',\n",
       " 'tools',\n",
       " 'hedrick@geneva.rutgers.edu',\n",
       " 'protect',\n",
       " 'friends',\n",
       " 'cpu',\n",
       " 'killing',\n",
       " 'suspect',\n",
       " 'supply',\n",
       " 'proof',\n",
       " 'countries',\n",
       " 'context',\n",
       " 'jack',\n",
       " 'obvious',\n",
       " 'approach',\n",
       " 'jeff',\n",
       " 'won',\n",
       " 'gordon',\n",
       " 'requires',\n",
       " 'installed',\n",
       " 'floppy',\n",
       " 'stay',\n",
       " 'yeah',\n",
       " 'disks',\n",
       " 'talk.politics.mideast',\n",
       " 'basically',\n",
       " 'poor',\n",
       " 'colorado',\n",
       " 'sites',\n",
       " 'lee',\n",
       " 'stephen',\n",
       " 'author',\n",
       " 'orbit',\n",
       " 'month',\n",
       " 'expressed',\n",
       " 'unit',\n",
       " 'respect',\n",
       " 'doug',\n",
       " 'soldiers',\n",
       " 'argic',\n",
       " 'effective',\n",
       " 'picture',\n",
       " 'committee',\n",
       " '44',\n",
       " 'range',\n",
       " 'protection',\n",
       " 'cambridge',\n",
       " 'logic',\n",
       " 'earlier',\n",
       " 'moon',\n",
       " 'matthew',\n",
       " 'prevent',\n",
       " 'vote',\n",
       " '80',\n",
       " 'hands',\n",
       " 'reserve',\n",
       " 'costs',\n",
       " 'shuttle',\n",
       " 'social',\n",
       " 'handle',\n",
       " 'electronic',\n",
       " 'homosexual',\n",
       " 'independent',\n",
       " 'at&t',\n",
       " 'atf',\n",
       " 'administrator',\n",
       " 'clock',\n",
       " 'define',\n",
       " 'islam',\n",
       " 'played',\n",
       " 'shipping',\n",
       " 'ron',\n",
       " 'false',\n",
       " 'minutes',\n",
       " 'mormons',\n",
       " 'posts',\n",
       " 'los',\n",
       " 'increase',\n",
       " '3d',\n",
       " 'published',\n",
       " 'majority',\n",
       " '60',\n",
       " 'supports',\n",
       " 'heaven',\n",
       " 'nt',\n",
       " 'middle',\n",
       " 'and/or',\n",
       " 'beliefs',\n",
       " 'programming',\n",
       " '34',\n",
       " 'trust',\n",
       " 'amendment',\n",
       " 'understanding',\n",
       " 'realize',\n",
       " 'print',\n",
       " 'standards',\n",
       " 'modern',\n",
       " 'army',\n",
       " 'mother',\n",
       " 'playing',\n",
       " 'treatment',\n",
       " 'mary',\n",
       " 'agents',\n",
       " 'mailing',\n",
       " 'enforcement',\n",
       " 'os/2',\n",
       " 'names',\n",
       " 'writing',\n",
       " 'comment',\n",
       " 'articles',\n",
       " 'ohio',\n",
       " 'austin',\n",
       " 'vga',\n",
       " 'soviet',\n",
       " 'march',\n",
       " 'sold',\n",
       " 'gay',\n",
       " 'charles',\n",
       " '1991',\n",
       " 'suggestions',\n",
       " 'acts',\n",
       " 'larry',\n",
       " 'art',\n",
       " 'morning',\n",
       " 'calls',\n",
       " 'personally',\n",
       " 'driving',\n",
       " 'hey',\n",
       " 'serve',\n",
       " 'student',\n",
       " '200',\n",
       " 'effort',\n",
       " 'nsa',\n",
       " 'features',\n",
       " 'arguments',\n",
       " 'tel',\n",
       " 'regular',\n",
       " 'recall',\n",
       " 'dangerous',\n",
       " 'convert',\n",
       " 'rob',\n",
       " 'hole',\n",
       " 'straight',\n",
       " 'constitution',\n",
       " 'myers',\n",
       " 'statements',\n",
       " 'pain',\n",
       " 'gateway',\n",
       " 'secure',\n",
       " 'draw',\n",
       " 'hot',\n",
       " 'energy',\n",
       " 'perfect',\n",
       " 'developed',\n",
       " 'intended',\n",
       " 'behavior',\n",
       " 'responsibility',\n",
       " 'virginia',\n",
       " 'aware',\n",
       " 'widget',\n",
       " 'kids',\n",
       " 'risk',\n",
       " 'paid',\n",
       " 'physics',\n",
       " 'cd',\n",
       " 'fair',\n",
       " 'direct',\n",
       " 'sexual',\n",
       " 'multiple',\n",
       " 'buying',\n",
       " 'medicine',\n",
       " 'atheism',\n",
       " 'notice',\n",
       " 'justice',\n",
       " '90',\n",
       " 'greatly',\n",
       " 'resources',\n",
       " 'block',\n",
       " 'fit',\n",
       " '500',\n",
       " 'assuming',\n",
       " 'roger',\n",
       " 'title',\n",
       " '1990',\n",
       " 'surface',\n",
       " '37',\n",
       " 'loss',\n",
       " 'colors',\n",
       " 'supported',\n",
       " 'switch',\n",
       " '36',\n",
       " 'judge',\n",
       " 'british',\n",
       " '38',\n",
       " 'angeles',\n",
       " 'effects',\n",
       " 'remote',\n",
       " '2nd',\n",
       " 'popular',\n",
       " 'option',\n",
       " 'william',\n",
       " \"o'dwyer\",\n",
       " 'slow',\n",
       " ...]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K = 2000 #limit of vocab considered\n",
    "vocab = list(data[:K]['0'])\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['apr',\n",
       " 'gmt',\n",
       " '1993',\n",
       " 'writes',\n",
       " 'references',\n",
       " 'article',\n",
       " 'sender',\n",
       " 'people',\n",
       " 'university',\n",
       " '1',\n",
       " 'nntp-posting-host',\n",
       " '93',\n",
       " '2',\n",
       " 'xref',\n",
       " 'cantaloupe.srv.cs.cmu.edu',\n",
       " 'news',\n",
       " 'distribution',\n",
       " '3',\n",
       " 'god',\n",
       " '0',\n",
       " \"max>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax\",\n",
       " '20',\n",
       " '5',\n",
       " '6',\n",
       " '4',\n",
       " '16',\n",
       " '15',\n",
       " 'government',\n",
       " '21',\n",
       " 'question',\n",
       " 'reply-to',\n",
       " 'read',\n",
       " 'usenet',\n",
       " 'usa',\n",
       " 'windows',\n",
       " 'david',\n",
       " 'software',\n",
       " 'data',\n",
       " 'program',\n",
       " 'file',\n",
       " '23',\n",
       " '10',\n",
       " 'space',\n",
       " 'version',\n",
       " 'science',\n",
       " 'power',\n",
       " 'john',\n",
       " 'drive',\n",
       " '22',\n",
       " 'fri',\n",
       " 'day',\n",
       " 'true',\n",
       " 'law',\n",
       " 'public',\n",
       " 'tue',\n",
       " 'game',\n",
       " '19',\n",
       " '8',\n",
       " '7',\n",
       " 'lot',\n",
       " 'car',\n",
       " 'set',\n",
       " 'real',\n",
       " '17',\n",
       " 'systems',\n",
       " 'support',\n",
       " 'life',\n",
       " 'free',\n",
       " 'list',\n",
       " 'jesus',\n",
       " 'image',\n",
       " 'sun',\n",
       " '18',\n",
       " 'card',\n",
       " '24',\n",
       " 'reason',\n",
       " '25',\n",
       " 'wrong',\n",
       " 'message',\n",
       " 'thu',\n",
       " 'hard',\n",
       " 'post',\n",
       " 'key',\n",
       " 'mon',\n",
       " 'center',\n",
       " 'internet',\n",
       " 'technology',\n",
       " 'bit',\n",
       " 'email',\n",
       " 'bad',\n",
       " 'send',\n",
       " 'files',\n",
       " 'heard',\n",
       " 'team',\n",
       " '26',\n",
       " 'person',\n",
       " 'evidence',\n",
       " 'christian',\n",
       " 'access',\n",
       " 'mark',\n",
       " 'wrote',\n",
       " 'jews',\n",
       " 'national',\n",
       " 'children',\n",
       " '14',\n",
       " 'idea',\n",
       " 'human',\n",
       " 'e-mail',\n",
       " 'opinions',\n",
       " '12',\n",
       " 'mail',\n",
       " 'times',\n",
       " 'control',\n",
       " 'keywords',\n",
       " 'questions',\n",
       " 'book',\n",
       " 'change',\n",
       " 'start',\n",
       " 'note',\n",
       " 'left',\n",
       " 'chip',\n",
       " 'remember',\n",
       " 'phone',\n",
       " '30',\n",
       " 'window',\n",
       " 'israel',\n",
       " 'based',\n",
       " 'word',\n",
       " '9',\n",
       " 'code',\n",
       " 'michael',\n",
       " 'gun',\n",
       " 'standard',\n",
       " 'institute',\n",
       " 'followup-to',\n",
       " 'source',\n",
       " '11',\n",
       " 'stuff',\n",
       " 'rights',\n",
       " 'net',\n",
       " 'info',\n",
       " '13',\n",
       " 'money',\n",
       " 'department',\n",
       " 'games',\n",
       " 'president',\n",
       " 'answer',\n",
       " '27',\n",
       " 'price',\n",
       " 'running',\n",
       " 'graphics',\n",
       " 'agree',\n",
       " 'dos',\n",
       " 'local',\n",
       " 'address',\n",
       " 'steve',\n",
       " 'buy',\n",
       " 'matter',\n",
       " 'american',\n",
       " 'paul',\n",
       " 'days',\n",
       " 'mac',\n",
       " 'bible',\n",
       " 'mike',\n",
       " 'war',\n",
       " 'hope',\n",
       " 'told',\n",
       " 'pretty',\n",
       " 'live',\n",
       " 'approved',\n",
       " 'care',\n",
       " 'disk',\n",
       " 'current',\n",
       " 'play',\n",
       " 'issue',\n",
       " 'original',\n",
       " 'feel',\n",
       " 'claim',\n",
       " 'sale',\n",
       " 'network',\n",
       " 'love',\n",
       " 'jim',\n",
       " 'understand',\n",
       " 'fax',\n",
       " 'religion',\n",
       " 'truth',\n",
       " 'box',\n",
       " 'programs',\n",
       " 'simply',\n",
       " 'college',\n",
       " 'mind',\n",
       " 'history',\n",
       " 'april',\n",
       " 'machine',\n",
       " 'speed',\n",
       " 'engineering',\n",
       " 'started',\n",
       " 'guess',\n",
       " 'armenian',\n",
       " 'video',\n",
       " 'canada',\n",
       " 'church',\n",
       " 'fbi',\n",
       " 'dept',\n",
       " 'memory',\n",
       " 'posting',\n",
       " 'unix',\n",
       " 'hand',\n",
       " 'robert',\n",
       " 'health',\n",
       " 'color',\n",
       " 'country',\n",
       " 'pc',\n",
       " 'encryption',\n",
       " 'service',\n",
       " 'pay',\n",
       " 'turkish',\n",
       " 'jewish',\n",
       " 'private',\n",
       " 'u.s',\n",
       " 'earth',\n",
       " 'house',\n",
       " 'x-newsreader',\n",
       " 'black',\n",
       " 'rec.sport.hockey',\n",
       " '28',\n",
       " 'christians',\n",
       " 'death',\n",
       " 'difference',\n",
       " 'display',\n",
       " 'user',\n",
       " 'school',\n",
       " 'ftp',\n",
       " 'sat',\n",
       " 'win',\n",
       " 'type',\n",
       " 'copy',\n",
       " 'server',\n",
       " 'california',\n",
       " 'hell',\n",
       " 'armenians',\n",
       " 'white',\n",
       " 'view',\n",
       " 'opinion',\n",
       " 'sense',\n",
       " 'faith',\n",
       " 'company',\n",
       " '1.1',\n",
       " 'clipper',\n",
       " 'soc.religion.christian',\n",
       " 'sort',\n",
       " 'package',\n",
       " 'deal',\n",
       " 'police',\n",
       " 'rec.sport.baseball',\n",
       " 'political',\n",
       " 'summary',\n",
       " 'major',\n",
       " 'rec.motorcycles',\n",
       " 'morality',\n",
       " 'moral',\n",
       " 'city',\n",
       " 'services',\n",
       " 'contact',\n",
       " 'talking',\n",
       " 'christian@aramis.rutgers.edu',\n",
       " 'check',\n",
       " 'couple',\n",
       " 'christ',\n",
       " 'single',\n",
       " 'killed',\n",
       " 'nice',\n",
       " 'cost',\n",
       " 'frank',\n",
       " 'write',\n",
       " 'body',\n",
       " 'security',\n",
       " 'james',\n",
       " '29',\n",
       " 'light',\n",
       " 'press',\n",
       " 'religious',\n",
       " 'argument',\n",
       " 'objective',\n",
       " 'happened',\n",
       " 'ibm',\n",
       " 'simple',\n",
       " 'san',\n",
       " 'dead',\n",
       " 'images',\n",
       " 'night',\n",
       " 'correct',\n",
       " 'hockey',\n",
       " 'monitor',\n",
       " 'provide',\n",
       " 'comp.sys.mac.hardware',\n",
       " '100',\n",
       " 'board',\n",
       " 'tin',\n",
       " 'fine',\n",
       " 'society',\n",
       " '50',\n",
       " 'fast',\n",
       " 'statement',\n",
       " 'position',\n",
       " 'form',\n",
       " 'experience',\n",
       " 'hardware',\n",
       " 'dave',\n",
       " 'western',\n",
       " 'format',\n",
       " 'situation',\n",
       " 'force',\n",
       " 'rest',\n",
       " 'york',\n",
       " 'rec.autos',\n",
       " 'women',\n",
       " 'washington',\n",
       " 'reading',\n",
       " 'common',\n",
       " 'laws',\n",
       " 'clinton',\n",
       " 'text',\n",
       " 'israeli',\n",
       " 'division',\n",
       " 'books',\n",
       " 'brian',\n",
       " 'sound',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'personal',\n",
       " 'discussion',\n",
       " 'guy',\n",
       " 'period',\n",
       " 'players',\n",
       " 'users',\n",
       " 'lost',\n",
       " 'scsi',\n",
       " 'jpeg',\n",
       " 'study',\n",
       " 'hear',\n",
       " 'business',\n",
       " 'alt.atheism',\n",
       " 'size',\n",
       " 'bob',\n",
       " 'test',\n",
       " 'exist',\n",
       " 'advance',\n",
       " 'bike',\n",
       " 'happen',\n",
       " 'theory',\n",
       " 'account',\n",
       " 'hit',\n",
       " 'red',\n",
       " 'radio',\n",
       " 'screen',\n",
       " 'future',\n",
       " 'peter',\n",
       " 'corporation',\n",
       " 'head',\n",
       " 'office',\n",
       " 'series',\n",
       " 'apple',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'application',\n",
       " 'talk',\n",
       " 'accept',\n",
       " 'easy',\n",
       " 'medical',\n",
       " 'level',\n",
       " 'faq',\n",
       " 'special',\n",
       " 'add',\n",
       " 'months',\n",
       " 'considered',\n",
       " 'article-i.d',\n",
       " 'driver',\n",
       " 'insurance',\n",
       " 'week',\n",
       " 'guns',\n",
       " 'drivers',\n",
       " 'posted',\n",
       " 'ms',\n",
       " 'season',\n",
       " 'kill',\n",
       " 'worth',\n",
       " 'job',\n",
       " 'koresh',\n",
       " 'sell',\n",
       " 'peace',\n",
       " 'runs',\n",
       " 'expect',\n",
       " 'ma',\n",
       " 'short',\n",
       " 'project',\n",
       " '32',\n",
       " 'manager',\n",
       " 'db',\n",
       " 'comp.windows.x',\n",
       " 'pittsburgh',\n",
       " 'thinking',\n",
       " 'texas',\n",
       " 'international',\n",
       " 'model',\n",
       " 'assume',\n",
       " 'ground',\n",
       " 'coming',\n",
       " 'ii',\n",
       " 'values',\n",
       " 'communications',\n",
       " 'doubt',\n",
       " 'quality',\n",
       " 'mode',\n",
       " 'military',\n",
       " 'land',\n",
       " 'united',\n",
       " 'federal',\n",
       " 'st',\n",
       " '40',\n",
       " 'reference',\n",
       " 'includes',\n",
       " 'offer',\n",
       " 'comp.graphics',\n",
       " 'speak',\n",
       " 'process',\n",
       " 'bbs',\n",
       " 'tom',\n",
       " 'policy',\n",
       " 'response',\n",
       " 'i.e',\n",
       " 'hold',\n",
       " 'strong',\n",
       " 'sci.space',\n",
       " 'involved',\n",
       " 'thomas',\n",
       " 'eric',\n",
       " 'story',\n",
       " 'baseball',\n",
       " 'explain',\n",
       " 'previous',\n",
       " 'cars',\n",
       " 'bus',\n",
       " 'gas',\n",
       " 'toronto',\n",
       " 'specific',\n",
       " 'performance',\n",
       " 'talk.politics.guns',\n",
       " 'computing',\n",
       " 'issues',\n",
       " 'road',\n",
       " 'weapons',\n",
       " 'andrew',\n",
       " 'error',\n",
       " 'dr',\n",
       " 'freedom',\n",
       " 'building',\n",
       " 'result',\n",
       " 'total',\n",
       " 'legal',\n",
       " 'design',\n",
       " 'dod',\n",
       " 'water',\n",
       " 'population',\n",
       " '0400',\n",
       " 'league',\n",
       " 'ram',\n",
       " 'leave',\n",
       " 'development',\n",
       " 'field',\n",
       " 'save',\n",
       " 'christianity',\n",
       " 'party',\n",
       " 'deleted',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'report',\n",
       " 'action',\n",
       " 'close',\n",
       " 'rate',\n",
       " 'appreciated',\n",
       " 'tape',\n",
       " 'basis',\n",
       " 'player',\n",
       " 'friend',\n",
       " 'child',\n",
       " 'richard',\n",
       " 'claims',\n",
       " 'return',\n",
       " 'smith',\n",
       " 'nasa',\n",
       " 'tv',\n",
       " 'knowledge',\n",
       " 'choice',\n",
       " 'voice',\n",
       " 'media',\n",
       " 'cut',\n",
       " 'technical',\n",
       " 'sci.crypt',\n",
       " 'average',\n",
       " 'ideas',\n",
       " '1992',\n",
       " 'build',\n",
       " 'boston',\n",
       " 'reasons',\n",
       " 'mentioned',\n",
       " 'reply',\n",
       " 'jon',\n",
       " 'scott',\n",
       " 'armenia',\n",
       " 'drug',\n",
       " 'applications',\n",
       " 'complete',\n",
       " 'sin',\n",
       " 'takes',\n",
       " 'administration',\n",
       " 'univ',\n",
       " 'exists',\n",
       " 'chicago',\n",
       " '31',\n",
       " 'die',\n",
       " 'drives',\n",
       " 'crime',\n",
       " 'anonymous',\n",
       " 'completely',\n",
       " 'disclaimer',\n",
       " 'appears',\n",
       " 'sounds',\n",
       " 'required',\n",
       " 'batf',\n",
       " 'comments',\n",
       " 'shot',\n",
       " 'court',\n",
       " 'output',\n",
       " 'modem',\n",
       " 'chris',\n",
       " 'weeks',\n",
       " 'bought',\n",
       " 'germany',\n",
       " 'scientific',\n",
       " 'details',\n",
       " 'basic',\n",
       " 'lord',\n",
       " 'newsgroup',\n",
       " 'waco',\n",
       " 'al',\n",
       " 'nature',\n",
       " 'created',\n",
       " 'des',\n",
       " 'keith',\n",
       " 'illinois',\n",
       " 'sources',\n",
       " \"god's\",\n",
       " 'existence',\n",
       " 'muslim',\n",
       " 'living',\n",
       " 'cover',\n",
       " 'looked',\n",
       " 'universe',\n",
       " 'chance',\n",
       " 'create',\n",
       " 'food',\n",
       " 'air',\n",
       " 'individual',\n",
       " 'supposed',\n",
       " 'cards',\n",
       " 'controller',\n",
       " 'george',\n",
       " 'finally',\n",
       " 'meaning',\n",
       " 'belief',\n",
       " 'terms',\n",
       " '33',\n",
       " 'algorithm',\n",
       " 'main',\n",
       " 'faster',\n",
       " 'purpose',\n",
       " 'digital',\n",
       " 'machines',\n",
       " 'arab',\n",
       " 'views',\n",
       " 'folks',\n",
       " 'russian',\n",
       " 'arms',\n",
       " 'follow',\n",
       " 'muslims',\n",
       " 'site',\n",
       " 'manual',\n",
       " 'mouse',\n",
       " 'mass',\n",
       " 'blue',\n",
       " 'in-reply-to',\n",
       " 'lead',\n",
       " 'final',\n",
       " 'attack',\n",
       " 'north',\n",
       " 'prove',\n",
       " 'plan',\n",
       " 'family',\n",
       " 'term',\n",
       " 'turkey',\n",
       " 'language',\n",
       " 'noise',\n",
       " 'atheists',\n",
       " 'community',\n",
       " 'learn',\n",
       " 'base',\n",
       " 'events',\n",
       " 'figure',\n",
       " 'genocide',\n",
       " 'rule',\n",
       " 'goal',\n",
       " 'built',\n",
       " 'directory',\n",
       " 'secret',\n",
       " 'entire',\n",
       " 'interface',\n",
       " 'command',\n",
       " 'ny',\n",
       " 'cheap',\n",
       " 'lots',\n",
       " 'america',\n",
       " 'reasonable',\n",
       " 'motif',\n",
       " 'designed',\n",
       " 'object',\n",
       " 'tim',\n",
       " '2000',\n",
       " 'condition',\n",
       " 'record',\n",
       " 'dan',\n",
       " 'wait',\n",
       " 'commercial',\n",
       " 'joseph',\n",
       " 'advice',\n",
       " 'engine',\n",
       " 'normal',\n",
       " 'gif',\n",
       " 'allowed',\n",
       " 'paper',\n",
       " 'sex',\n",
       " 'library',\n",
       " 'drugs',\n",
       " 'defense',\n",
       " 'rules',\n",
       " 'stated',\n",
       " 'hours',\n",
       " 'device',\n",
       " 'release',\n",
       " 'uk',\n",
       " 'south',\n",
       " 'miles',\n",
       " 'carry',\n",
       " 'event',\n",
       " 'edt',\n",
       " 'lab',\n",
       " 'product',\n",
       " 'serial',\n",
       " 'laboratory',\n",
       " 'easily',\n",
       " 'port',\n",
       " 'privacy',\n",
       " 'reality',\n",
       " 'request',\n",
       " 'wondering',\n",
       " 'difficult',\n",
       " 'tax',\n",
       " 'pat',\n",
       " 'son',\n",
       " 'happy',\n",
       " 'la',\n",
       " 'fan',\n",
       " 'responsible',\n",
       " 'break',\n",
       " 'bring',\n",
       " 'poster',\n",
       " 'decided',\n",
       " 'stupid',\n",
       " 'joe',\n",
       " '3.1',\n",
       " 'studies',\n",
       " 'lives',\n",
       " 'hp',\n",
       " 'solution',\n",
       " 'market',\n",
       " 'physical',\n",
       " 'definition',\n",
       " 'products',\n",
       " 'bits',\n",
       " 'turks',\n",
       " 'method',\n",
       " '35',\n",
       " 'suppose',\n",
       " 'west',\n",
       " 'citizens',\n",
       " 'changed',\n",
       " 'illegal',\n",
       " 'trouble',\n",
       " 'analysis',\n",
       " 'station',\n",
       " 'owner',\n",
       " 'mention',\n",
       " 'guys',\n",
       " 'equipment',\n",
       " 'companies',\n",
       " 'attempt',\n",
       " 'actions',\n",
       " 'originator',\n",
       " 'oil',\n",
       " 'abortion',\n",
       " 'father',\n",
       " 'conference',\n",
       " 'disease',\n",
       " 'green',\n",
       " 'imagine',\n",
       " 'reported',\n",
       " 'class',\n",
       " 'fans',\n",
       " 'civil',\n",
       " 'died',\n",
       " 'corp',\n",
       " 'brought',\n",
       " 'forget',\n",
       " 'pick',\n",
       " 'age',\n",
       " 'limited',\n",
       " 'official',\n",
       " 'ray',\n",
       " 'devices',\n",
       " 'printer',\n",
       " 'hedrick@athos.rutgers.edu',\n",
       " 'wife',\n",
       " 'launch',\n",
       " 'apply',\n",
       " 'letter',\n",
       " 'murder',\n",
       " 'extra',\n",
       " 'street',\n",
       " 'east',\n",
       " 'lack',\n",
       " 'education',\n",
       " 'chips',\n",
       " 'gm',\n",
       " 'door',\n",
       " 'decision',\n",
       " 'environment',\n",
       " 'greek',\n",
       " 'watch',\n",
       " 'americans',\n",
       " 'hate',\n",
       " 'misc.forsale',\n",
       " 'avoid',\n",
       " 'stand',\n",
       " 'georgia',\n",
       " 'e.g',\n",
       " 'congress',\n",
       " 'heart',\n",
       " 'king',\n",
       " 'ide',\n",
       " 'homosexuality',\n",
       " 'btw',\n",
       " 'search',\n",
       " 'mission',\n",
       " 'cable',\n",
       " 'require',\n",
       " 'reports',\n",
       " 'trade',\n",
       " 'function',\n",
       " 'coverage',\n",
       " 'caused',\n",
       " 'worse',\n",
       " 'safety',\n",
       " 'fall',\n",
       " 'teams',\n",
       " 'evil',\n",
       " 'gary',\n",
       " 'online',\n",
       " 'compound',\n",
       " 'nhl',\n",
       " 'quote',\n",
       " 'volume',\n",
       " '1st',\n",
       " 'internal',\n",
       " 'continue',\n",
       " 'authority',\n",
       " 'europe',\n",
       " 'ken',\n",
       " 'excellent',\n",
       " 'choose',\n",
       " 'ability',\n",
       " 'blood',\n",
       " 'starting',\n",
       " 'serdar',\n",
       " 'input',\n",
       " 'natural',\n",
       " 'material',\n",
       " 'addition',\n",
       " 'practice',\n",
       " 'microsoft',\n",
       " 'pass',\n",
       " 'tools',\n",
       " 'hedrick@geneva.rutgers.edu',\n",
       " 'protect',\n",
       " 'friends',\n",
       " 'cpu',\n",
       " 'killing',\n",
       " 'suspect',\n",
       " 'supply',\n",
       " 'proof',\n",
       " 'countries',\n",
       " 'context',\n",
       " 'jack',\n",
       " 'obvious',\n",
       " 'approach',\n",
       " 'jeff',\n",
       " 'won',\n",
       " 'gordon',\n",
       " 'requires',\n",
       " 'installed',\n",
       " 'floppy',\n",
       " 'stay',\n",
       " 'yeah',\n",
       " 'disks',\n",
       " 'talk.politics.mideast',\n",
       " 'basically',\n",
       " 'poor',\n",
       " 'colorado',\n",
       " 'sites',\n",
       " 'lee',\n",
       " 'stephen',\n",
       " 'author',\n",
       " 'orbit',\n",
       " 'month',\n",
       " 'expressed',\n",
       " 'unit',\n",
       " 'respect',\n",
       " 'doug',\n",
       " 'soldiers',\n",
       " 'argic',\n",
       " 'effective',\n",
       " 'picture',\n",
       " 'committee',\n",
       " '44',\n",
       " 'range',\n",
       " 'protection',\n",
       " 'cambridge',\n",
       " 'logic',\n",
       " 'earlier',\n",
       " 'moon',\n",
       " 'matthew',\n",
       " 'prevent',\n",
       " 'vote',\n",
       " '80',\n",
       " 'hands',\n",
       " 'reserve',\n",
       " 'costs',\n",
       " 'shuttle',\n",
       " 'social',\n",
       " 'handle',\n",
       " 'electronic',\n",
       " 'homosexual',\n",
       " 'independent',\n",
       " 'at&t',\n",
       " 'atf',\n",
       " 'administrator',\n",
       " 'clock',\n",
       " 'define',\n",
       " 'islam',\n",
       " 'played',\n",
       " 'shipping',\n",
       " 'ron',\n",
       " 'false',\n",
       " 'minutes',\n",
       " 'mormons',\n",
       " 'posts',\n",
       " 'los',\n",
       " 'increase',\n",
       " '3d',\n",
       " 'published',\n",
       " 'majority',\n",
       " '60',\n",
       " 'supports',\n",
       " 'heaven',\n",
       " 'nt',\n",
       " 'middle',\n",
       " 'and/or',\n",
       " 'beliefs',\n",
       " 'programming',\n",
       " '34',\n",
       " 'trust',\n",
       " 'amendment',\n",
       " 'understanding',\n",
       " 'realize',\n",
       " 'print',\n",
       " 'standards',\n",
       " 'modern',\n",
       " 'army',\n",
       " 'mother',\n",
       " 'playing',\n",
       " 'treatment',\n",
       " 'mary',\n",
       " 'agents',\n",
       " 'mailing',\n",
       " 'enforcement',\n",
       " 'os/2',\n",
       " 'names',\n",
       " 'writing',\n",
       " 'comment',\n",
       " 'articles',\n",
       " 'ohio',\n",
       " 'austin',\n",
       " 'vga',\n",
       " 'soviet',\n",
       " 'march',\n",
       " 'sold',\n",
       " 'gay',\n",
       " 'charles',\n",
       " '1991',\n",
       " 'suggestions',\n",
       " 'acts',\n",
       " 'larry',\n",
       " 'art',\n",
       " 'morning',\n",
       " 'calls',\n",
       " 'personally',\n",
       " 'driving',\n",
       " 'hey',\n",
       " 'serve',\n",
       " 'student',\n",
       " '200',\n",
       " 'effort',\n",
       " 'nsa',\n",
       " 'features',\n",
       " 'arguments',\n",
       " 'tel',\n",
       " 'regular',\n",
       " 'recall',\n",
       " 'dangerous',\n",
       " 'convert',\n",
       " 'rob',\n",
       " 'hole',\n",
       " 'straight',\n",
       " 'constitution',\n",
       " 'myers',\n",
       " 'statements',\n",
       " 'pain',\n",
       " 'gateway',\n",
       " 'secure',\n",
       " 'draw',\n",
       " 'hot',\n",
       " 'energy',\n",
       " 'perfect',\n",
       " 'developed',\n",
       " 'intended',\n",
       " 'behavior',\n",
       " 'responsibility',\n",
       " 'virginia',\n",
       " 'aware',\n",
       " 'widget',\n",
       " 'kids',\n",
       " 'risk',\n",
       " 'paid',\n",
       " 'physics',\n",
       " 'cd',\n",
       " 'fair',\n",
       " 'direct',\n",
       " 'sexual',\n",
       " 'multiple',\n",
       " 'buying',\n",
       " 'medicine',\n",
       " 'atheism',\n",
       " 'notice',\n",
       " 'justice',\n",
       " '90',\n",
       " 'greatly',\n",
       " 'resources',\n",
       " 'block',\n",
       " 'fit',\n",
       " '500',\n",
       " 'assuming',\n",
       " 'roger',\n",
       " 'title',\n",
       " '1990',\n",
       " 'surface',\n",
       " '37',\n",
       " 'loss',\n",
       " 'colors',\n",
       " 'supported',\n",
       " 'switch',\n",
       " '36',\n",
       " 'judge',\n",
       " 'british',\n",
       " '38',\n",
       " 'angeles',\n",
       " 'effects',\n",
       " 'remote',\n",
       " '2nd',\n",
       " 'popular',\n",
       " 'option',\n",
       " 'william',\n",
       " \"o'dwyer\",\n",
       " 'slow',\n",
       " ...]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = vocab\n",
    "columns.append('CLASSIFICATION')\n",
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>apr</th>\n",
       "      <th>gmt</th>\n",
       "      <th>1993</th>\n",
       "      <th>writes</th>\n",
       "      <th>references</th>\n",
       "      <th>article</th>\n",
       "      <th>sender</th>\n",
       "      <th>people</th>\n",
       "      <th>university</th>\n",
       "      <th>1</th>\n",
       "      <th>...</th>\n",
       "      <th>finland</th>\n",
       "      <th>bikes</th>\n",
       "      <th>livesey@solntze.wpd.sgi.com</th>\n",
       "      <th>shit</th>\n",
       "      <th>spot</th>\n",
       "      <th>justify</th>\n",
       "      <th>explained</th>\n",
       "      <th>52</th>\n",
       "      <th>learning</th>\n",
       "      <th>CLASSIFICATION</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [apr, gmt, 1993, writes, references, article, sender, people, university, 1, nntp-posting-host, 93, 2, xref, cantaloupe.srv.cs.cmu.edu, news, distribution, 3, god, 0, max>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax>'ax, 20, 5, 6, 4, 16, 15, government, 21, question, reply-to, read, usenet, usa, windows, david, software, data, program, file, 23, 10, space, version, science, power, john, drive, 22, fri, day, true, law, public, tue, game, 19, 8, 7, lot, car, set, real, 17, systems, support, life, free, list, jesus, image, sun, 18, card, 24, reason, 25, wrong, message, thu, hard, post, key, mon, center, internet, technology, bit, email, bad, send, files, heard, team, 26, person, evidence, christian, access, mark, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 2001 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns=columns) #make empty dataframe with headings as vocab words & classification\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] The system cannot find the path specified: '/Users/jo/Documents/CN_ML/Text Classification - Project/20_newsgroups'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-f99b22b7a2bb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#refer there for the comments\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mtempPath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/Users/jo/Documents/CN_ML/Text Classification - Project/20_newsgroups\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtempPath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mcountDir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mtotalDir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetcwd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: '/Users/jo/Documents/CN_ML/Text Classification - Project/20_newsgroups'"
     ]
    }
   ],
   "source": [
    "#iterating just like done in vocabulary builder\n",
    "#refer there for the comments\n",
    "tempPath =r\"C:\\Users\\Himanshu gupta\\Desktop\\ML\\best_submissions\\best_submission\\himgupta- Text Classification Project\\20_newsgroups\"\n",
    "os.chdir(tempPath)\n",
    "countDir = 0\n",
    "totalDir = len(os.listdir(os.getcwd()))\n",
    "for directory in os.listdir(os.getcwd()):\n",
    "    if directory[0]=='.':\n",
    "        continue\n",
    "    path = tempPath + \"/\" + directory\n",
    "    os.chdir(path)\n",
    "    files = os.listdir(os.getcwd())\n",
    "    for file in files:\n",
    "        if file[0]=='.':\n",
    "            continue\n",
    "    n = len(files)\n",
    "    n_train = int(n*0.8) #number of training data\n",
    "    #n_test = n-n_train\n",
    "    #files = files[:n_train]\n",
    "    files = files[n_train:] #selecting only testing files (opposite of training files)\n",
    "    count = 0\n",
    "    print(n-n_train, \"files to process in\", directory)\n",
    "    dirData = [[0]*len(columns)]\n",
    "    for file in files:\n",
    "        row_data = [0]*len(columns)\n",
    "        f = open(file,\"r\",errors=\"ignore\")\n",
    "        for word in f.read().split():\n",
    "            word = word.strip(punctuation).lower()\n",
    "            #if word not in stop_words and word not in ignore_words and word.isalpha() and len(word)!=1:\n",
    "            if word in vocab:\n",
    "                row_data[columns.index(word)] = row_data[columns.index(word)] + 1\n",
    "        row_data[columns.index('CLASSIFICATION')] = directory\n",
    "        dirData.append(row_data)\n",
    "        count += 1\n",
    "        if count%100 == 0:\n",
    "            print(\"Done with files: \", count)\n",
    "    tempDF = pd.DataFrame(dirData[1:], columns=columns)\n",
    "    df = df.append(tempDF, ignore_index=True)\n",
    "    countDir += 1\n",
    "    print(\"Done with directory: \", directory)\n",
    "    print(\"Number: \", countDir, \"of\", totalDir)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bak = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>apr</th>\n",
       "      <th>gmt</th>\n",
       "      <th>1993</th>\n",
       "      <th>writes</th>\n",
       "      <th>references</th>\n",
       "      <th>article</th>\n",
       "      <th>sender</th>\n",
       "      <th>people</th>\n",
       "      <th>university</th>\n",
       "      <th>1</th>\n",
       "      <th>...</th>\n",
       "      <th>finland</th>\n",
       "      <th>bikes</th>\n",
       "      <th>livesey@solntze.wpd.sgi.com</th>\n",
       "      <th>shit</th>\n",
       "      <th>spot</th>\n",
       "      <th>justify</th>\n",
       "      <th>explained</th>\n",
       "      <th>52</th>\n",
       "      <th>learning</th>\n",
       "      <th>CLASSIFICATION</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3971</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3972</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3973</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3974</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3975</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3976</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3977</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3978</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3979</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3980</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3981</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3982</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3983</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3985</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3986</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3988</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3989</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3990</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3992</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3993</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3994</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3995</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3996</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3997</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4000</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4001 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     apr gmt 1993 writes references article sender people university  1  \\\n",
       "0      1   1    1      0          0       0      0      1          0  0   \n",
       "1      1   1    0      0          1       1      0      2          0  0   \n",
       "2      0   1    1      2          1       2      0      3          0  0   \n",
       "3      0   1    0      2          1       2      0      0          1  0   \n",
       "4      1   1    0      1          1       0      1      0          0  0   \n",
       "5      1   0    1      0          0       0      1      0          0  0   \n",
       "6      1   1    1      1          1       0      1      3          2  0   \n",
       "7      1   1    0      1          1       1      1      0          1  0   \n",
       "8      1   1    0      1          1       0      1      7          1  0   \n",
       "9      1   0    0      0          0       0      1      9          0  2   \n",
       "10     1   0    0      1          1       1      1      0          0  1   \n",
       "11     2   1    1      1          1       1      1      1          0  0   \n",
       "12     0   0    1      0          1       0      0      0          0  0   \n",
       "13     1   1    0      1          1       0      1      1          1  0   \n",
       "14     0   2    2      3          1       2      1      0          0  0   \n",
       "15     1   1    1      2          1       2      1      1          1  0   \n",
       "16     1   1    1      0          0       0      1      1          3  1   \n",
       "17     1   1    1      1          1       1      0      0          0  0   \n",
       "18     1   1    1      0          1       0      1      1          1  0   \n",
       "19     1   1    0      3          1       3      0      0          1  0   \n",
       "20     1   1    0      1          1       1      1      0          0  0   \n",
       "21     2   1    0      4          1       4      1      0          0  0   \n",
       "22     1   1    1      0          0       0      1      0          0  0   \n",
       "23     1   1    1      1          1       1      1      0          0  0   \n",
       "24     0   0    1      0          1       4      0      1          5  4   \n",
       "25     1   1    0      1          1       1      1      3          0  0   \n",
       "26     1   1    0      1          1       0      1      0          0  0   \n",
       "27     0   1    8      0          0       3      1      3          0  1   \n",
       "28     1   1    1      1          1       1      0      1          1  2   \n",
       "29     1   1    1      1          1       1      0      0          0  0   \n",
       "...   ..  ..  ...    ...        ...     ...    ...    ...        ... ..   \n",
       "3971   1   1    1      1          1       1      1      0          0  1   \n",
       "3972   1   1    1      1          1       0      0      1          0  1   \n",
       "3973   1   1    1      2          1       2      1      0          0  0   \n",
       "3974   1   1    0      1          1       1      0      1          0  1   \n",
       "3975   1   1    0      1          1       1      0      0          0  0   \n",
       "3976   1   1    1      1          1       1      1      0          1  0   \n",
       "3977   1   0    0      1          1       1      0      0          1  0   \n",
       "3978   1   1    0      1          1       1      1      1          0  0   \n",
       "3979   1   1    1      2          1       2      0      1          0  1   \n",
       "3980   1   1    0      2          1       0      1      0          0  0   \n",
       "3981   1   1    0      0          1       1      0      6          2  0   \n",
       "3982   1   1    0      3          1       3      0      0          0  1   \n",
       "3983   1   0    1      2          1       1      0      2          1  0   \n",
       "3984   1   1    1      2          1       2      0      0          1  0   \n",
       "3985   1   0    1      2          1       2      0      0          0  1   \n",
       "3986   1   1    1      2          1       2      0      0          0  0   \n",
       "3987   1   1    1      0          0       0      1      0          0  0   \n",
       "3988   1   1    1      0          1       0      1      0          1  0   \n",
       "3989   2   1    1      2          1       3      1      0          1  2   \n",
       "3990   1   1    0      1          1       1      1      1          0  1   \n",
       "3991   1   1    1      2          1       1      0      0          0  0   \n",
       "3992   1   1    1      2          1       1      1      0          2  1   \n",
       "3993   1   0    1      3          1       3      0      1          0  1   \n",
       "3994   1   0    1      1          1       0      0      0          0  0   \n",
       "3995   1   1    1      2          1       3      0      0          1  0   \n",
       "3996   1   1    1      1          0       1      1      1          0  0   \n",
       "3997   1   1    1      1          1       1      0      0          0  0   \n",
       "3998   1   1    1      1          1       1      0      1          0  0   \n",
       "3999   1   1    1      2          1       2      0      0          0  0   \n",
       "4000   1   0    1      3          1       3      0      1          0  0   \n",
       "\n",
       "              ...           finland bikes livesey@solntze.wpd.sgi.com shit  \\\n",
       "0             ...                 0     0                           0    0   \n",
       "1             ...                 0     0                           0    0   \n",
       "2             ...                 0     0                           0    0   \n",
       "3             ...                 0     0                           0    0   \n",
       "4             ...                 0     0                           0    0   \n",
       "5             ...                 0     0                           0    0   \n",
       "6             ...                 0     0                           0    0   \n",
       "7             ...                 0     0                           0    0   \n",
       "8             ...                 0     0                           0    0   \n",
       "9             ...                 0     0                           0    0   \n",
       "10            ...                 0     0                           0    0   \n",
       "11            ...                 0     0                           0    0   \n",
       "12            ...                 0     0                           0    0   \n",
       "13            ...                 0     0                           0    0   \n",
       "14            ...                 0     0                           0    0   \n",
       "15            ...                 0     0                           0    0   \n",
       "16            ...                 0     0                           0    0   \n",
       "17            ...                 0     0                           0    0   \n",
       "18            ...                 0     0                           0    0   \n",
       "19            ...                 0     0                           0    0   \n",
       "20            ...                 0     0                           0    0   \n",
       "21            ...                 0     0                           0    0   \n",
       "22            ...                 0     0                           0    0   \n",
       "23            ...                 0     0                           0    0   \n",
       "24            ...                 0     0                           0    0   \n",
       "25            ...                 0     0                           0    0   \n",
       "26            ...                 0     0                           0    0   \n",
       "27            ...                 0     0                           0    0   \n",
       "28            ...                 0     0                           0    0   \n",
       "29            ...                 0     0                           0    0   \n",
       "...           ...               ...   ...                         ...  ...   \n",
       "3971          ...                 0     0                           0    0   \n",
       "3972          ...                 0     0                           0    0   \n",
       "3973          ...                 0     0                           0    0   \n",
       "3974          ...                 0     0                           0    0   \n",
       "3975          ...                 0     0                           0    0   \n",
       "3976          ...                 0     0                           0    0   \n",
       "3977          ...                 0     0                           0    0   \n",
       "3978          ...                 0     0                           0    0   \n",
       "3979          ...                 0     0                           1    0   \n",
       "3980          ...                 0     0                           0    0   \n",
       "3981          ...                 0     0                           0    0   \n",
       "3982          ...                 0     0                           0    0   \n",
       "3983          ...                 0     0                           0    0   \n",
       "3984          ...                 0     0                           0    0   \n",
       "3985          ...                 0     0                           0    0   \n",
       "3986          ...                 0     0                           0    0   \n",
       "3987          ...                 0     0                           0    0   \n",
       "3988          ...                 0     0                           0    0   \n",
       "3989          ...                 0     0                           0    0   \n",
       "3990          ...                 0     0                           0    0   \n",
       "3991          ...                 0     0                           0    0   \n",
       "3992          ...                 0     0                           0    0   \n",
       "3993          ...                 0     0                           0    0   \n",
       "3994          ...                 0     0                           0    0   \n",
       "3995          ...                 0     0                           0    0   \n",
       "3996          ...                 0     0                           0    0   \n",
       "3997          ...                 0     0                           0    0   \n",
       "3998          ...                 0     0                           0    0   \n",
       "3999          ...                 0     0                           0    0   \n",
       "4000          ...                 0     0                           0    1   \n",
       "\n",
       "     spot justify explained 52 learning         CLASSIFICATION  \n",
       "0       1       0         0  0        0  talk.politics.mideast  \n",
       "1       0       0         0  0        0  talk.politics.mideast  \n",
       "2       0       0         0  0        0  talk.politics.mideast  \n",
       "3       0       0         0  0        0  talk.politics.mideast  \n",
       "4       0       0         0  0        0  talk.politics.mideast  \n",
       "5       0       0         0  0        0  talk.politics.mideast  \n",
       "6       0       0         0  0        0  talk.politics.mideast  \n",
       "7       0       0         0  0        0  talk.politics.mideast  \n",
       "8       0       0         0  0        0  talk.politics.mideast  \n",
       "9       0       0         0  0        0  talk.politics.mideast  \n",
       "10      0       0         0  1        0  talk.politics.mideast  \n",
       "11      0       0         0  0        0  talk.politics.mideast  \n",
       "12      0       0         0  0        0  talk.politics.mideast  \n",
       "13      0       0         0  0        0  talk.politics.mideast  \n",
       "14      0       0         0  0        0  talk.politics.mideast  \n",
       "15      0       0         0  0        0  talk.politics.mideast  \n",
       "16      0       0         0  0        0  talk.politics.mideast  \n",
       "17      0       0         0  0        0  talk.politics.mideast  \n",
       "18      0       0         0  0        0  talk.politics.mideast  \n",
       "19      0       0         0  0        0  talk.politics.mideast  \n",
       "20      0       0         0  0        0  talk.politics.mideast  \n",
       "21      0       0         0  0        0  talk.politics.mideast  \n",
       "22      0       0         0  0        0  talk.politics.mideast  \n",
       "23      0       0         0  1        0  talk.politics.mideast  \n",
       "24      0       0         2  0        0  talk.politics.mideast  \n",
       "25      0       0         0  0        0  talk.politics.mideast  \n",
       "26      0       0         0  0        0  talk.politics.mideast  \n",
       "27      0       0         0  0        0  talk.politics.mideast  \n",
       "28      0       0         0  0        0  talk.politics.mideast  \n",
       "29      0       0         0  0        0  talk.politics.mideast  \n",
       "...   ...     ...       ... ..      ...                    ...  \n",
       "3971    0       0         0  0        0     talk.religion.misc  \n",
       "3972    0       0         0  0        0     talk.religion.misc  \n",
       "3973    0       0         0  0        0     talk.religion.misc  \n",
       "3974    0       0         0  0        0     talk.religion.misc  \n",
       "3975    0       0         0  0        0     talk.religion.misc  \n",
       "3976    0       0         0  0        0     talk.religion.misc  \n",
       "3977    0       0         0  0        0     talk.religion.misc  \n",
       "3978    0       0         0  0        0     talk.religion.misc  \n",
       "3979    0       0         0  0        0     talk.religion.misc  \n",
       "3980    0       1         0  0        0     talk.religion.misc  \n",
       "3981    0       0         0  0        0     talk.religion.misc  \n",
       "3982    0       0         0  0        0     talk.religion.misc  \n",
       "3983    0       0         0  0        0     talk.religion.misc  \n",
       "3984    0       0         0  0        0     talk.religion.misc  \n",
       "3985    0       0         0  0        0     talk.religion.misc  \n",
       "3986    0       0         0  0        0     talk.religion.misc  \n",
       "3987    0       0         0  0        0     talk.religion.misc  \n",
       "3988    0       0         0  0        0     talk.religion.misc  \n",
       "3989    0       0         0  0        0     talk.religion.misc  \n",
       "3990    0       0         0  0        0     talk.religion.misc  \n",
       "3991    0       0         0  0        0     talk.religion.misc  \n",
       "3992    0       0         0  0        0     talk.religion.misc  \n",
       "3993    0       0         0  0        0     talk.religion.misc  \n",
       "3994    0       0         0  0        0     talk.religion.misc  \n",
       "3995    0       0         0  0        0     talk.religion.misc  \n",
       "3996    0       0         0  0        0     talk.religion.misc  \n",
       "3997    0       0         0  0        0     talk.religion.misc  \n",
       "3998    0       0         0  0        0     talk.religion.misc  \n",
       "3999    0       0         0  0        0     talk.religion.misc  \n",
       "4000    0       0         0  0        0     talk.religion.misc  \n",
       "\n",
       "[4001 rows x 2001 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#saving testing dataset\n",
    "os.chdir(r\"C:\\Users\\Himanshu gupta\\Desktop\\ML\\best_submissions\\best_submission\\himgupta- Text Classification Project\\20_newsgroups\")\n",
    "df.to_csv(\"test.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}