w2v_and_RQ1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /Users/ozilman/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to /Users/ozilman/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# General stuff\n",
    "import tqdm \n",
    "import pathlib\n",
    "import re\n",
    "import importlib\n",
    "import sys\n",
    "import pprint\n",
    "\n",
    "# DS stuff\n",
    "import math\n",
    "import numpy.linalg as linalg\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "# NLP stuff\n",
    "import nltk\n",
    "from   nltk import word_tokenize\n",
    "from   nltk.probability import FreqDist\n",
    "from   nltk.stem import WordNetLemmatizer\n",
    "import gensim\n",
    "import gensim.downloader\n",
    "import spacy\n",
    "\n",
    "# Finanace stuff\n",
    "import yfinance as yf\n",
    "\n",
    "# Download NLP models\n",
    "spacy_model = spacy.load('en_core_web_sm')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "w2v_pre = gensim.downloader.load('word2vec-google-news-300')\n",
    "\n",
    "# My codes\n",
    "import data_read\n",
    "import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'preprocessing' from '/Users/ozilman/NLP/finance_sentiment_proj/git_repo/nlp_financial_sentiment/preprocessing.py'>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reload My codes\n",
    "importlib.reload(data_read)\n",
    "importlib.reload(preprocessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Load Reuters data corpus. Parse headers and dates from the financial news articles '''\n",
    "headers = data_read.parse_news_data()\n",
    "#relevant_news = get_relevant_news(traded_symbols, tickers,  headers)\n",
    "print(len(headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Create text corpus, of processed, tokenized headers '''\n",
    "\n",
    "def generate_tokenized_corpus(headers):\n",
    "    WNLemmatizer = WordNetLemmatizer()\n",
    "    ticker_re = preprocessing.get_ticker_re()\n",
    "    remove_list = preprocessing.get_stop_words()\n",
    "\n",
    "    '''Process and tokenize.'''\n",
    "    tokenized_headers = []\n",
    "    for header in tqdm.tqdm(headers):\n",
    "        header = preprocessing.remove_tickers(ticker_re, header)\n",
    "        header = preprocessing.NER_processing(header)\n",
    "        #print(f\"({ticker}) {header}\")\n",
    "        header_tokens = [word.lower() for word in word_tokenize(header)\n",
    "                if word not in remove_list  \n",
    "                ]\n",
    "        header_tokens =  preprocessing.lemmatize(header_tokens, WNLemmatizer)\n",
    "        tokenized_headers.append(header_tokens)\n",
    "        \n",
    "    corpus = [token for header in tokenized_headers for token in header]\n",
    "    freqdist = FreqDist(corpus)\n",
    "    return tokenized_headers, corpus, freqdist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 104406/104406 [17:08<00:00, 101.49it/s]\n"
     ]
    }
   ],
   "source": [
    "''' Train Word2Vec embeddings '''\n",
    "tokenized_headers, corpus, freqdist = generate_tokenized_corpus(list(zip(*headers))[1])\n",
    "vector_size = 300\n",
    "w2v = gensim.models.Word2Vec(sentences=tokenized_headers, vector_size=vector_size, min_count=5, epochs=300, workers=8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' W2V Generate Paragraph embedding '''\n",
    "####\n",
    "# Functions to get a document (phrase/header) embedding by averaging the individual \n",
    "# word embeddings in the document. \n",
    "####\n",
    "def get_doc_embedding(doc, weighting, wv_model, word_freq_dict, a = 0.001):\n",
    "    ''' Convert a document to an embedding by averaging its word embeddings.\n",
    "        Three possible averaging schemes, SIF, logaritmic, and regular unweighted average.\n",
    "    '''\n",
    "    size = wv_model.vector_size\n",
    "    q_vec = np.zeros(size)\n",
    "    total_weight = 0\n",
    "    q_size = 0\n",
    "    weight = 1\n",
    "    for word in doc:\n",
    "        if word in wv_model:\n",
    "            if weighting == \"SIF\":\n",
    "                word_freq = word_freq_dict[word]\n",
    "                weight = a / (a + word_freq)\n",
    "                q_vec += weight * wv_model[word]\n",
    "                total_weight += weight\n",
    "                q_size += 1\n",
    "            elif weighting == \"LOG\":\n",
    "                word_freq = word_freq_dict[word]\n",
    "                weight = math.log(1 / word_freq)\n",
    "                q_vec += weight * wv_model[word]\n",
    "                total_weight += weight\n",
    "                q_size += 1\n",
    "            elif weighting == \"AVG\":\n",
    "                q_vec += weight * wv_model[word]\n",
    "                total_weight += 1\n",
    "                q_size += 1\n",
    "\n",
    "    # If there are two many words not in wocab that got removed,\n",
    "    # then a short sentence will not have a lot of signal. \n",
    "    if q_size >= 3:\n",
    "        q_vec = q_vec / total_weight\n",
    "        return q_vec\n",
    "    else:\n",
    "        return None\n",
    "        \n",
    "def get_dataset_embeddings(docs, weighting, wv_model, word_freq_dict):\n",
    "    ''' Get embbedding for each document, then normalize all \n",
    "        of them and return a 2 dim np array with doc embeddings as rows.\n",
    "    '''\n",
    "    doc_vecs = []\n",
    "    valid_indices_list = []\n",
    "    for i, doc in enumerate(docs):\n",
    "        embedding = get_doc_embedding(doc, weighting, wv_model, word_freq_dict)\n",
    "        if embedding is not None:\n",
    "            doc_vecs.append(embedding)\n",
    "            valid_indices_list.append(i) # Mask to algin y with the X\n",
    "                                        # that had docs removed from it\n",
    "                                        # because they are too short.\n",
    "            \n",
    "        \n",
    "    doc_vecs = np.vstack(doc_vecs)\n",
    "    #row_norm = linalg.norm(doc_vecs, axis=1)\n",
    "    #row_norm = row_norm[:, np.newaxis]\n",
    "    #doc_vecs = doc_vecs / row_norm\n",
    "    return doc_vecs, valid_indices_list\n",
    "    #return doc_vecs\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1682/1682 [00:15<00:00, 108.83it/s]\n",
      "/var/folders/dw/vzm8v7wn111ccjzl7139x7xc0000gq/T/ipykernel_659/923715438.py:21: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  docs_test_PhraseBank = np.array(docs_test)[mask_test]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.76      0.82       110\n",
      "           1       0.89      0.95      0.92       226\n",
      "\n",
      "    accuracy                           0.89       336\n",
      "   macro avg       0.88      0.86      0.87       336\n",
      "weighted avg       0.89      0.89      0.88       336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "''' \n",
    "Evaluate Model on PhraseBank Dataset.\n",
    "-- Word2Vec - Reuters -- \n",
    "'''\n",
    "# Split to train and test. \n",
    "WNLemmatizer = WordNetLemmatizer()\n",
    "remove_list = preprocessing.get_stop_words()\n",
    "# Split to train and test data. For the FinancialPhraseBank dataset\n",
    "phrases, labels = data_read.load_fin_pharsebank()\n",
    "phrases, _, _ = generate_tokenized_corpus(phrases)\n",
    "docs_train, docs_test, y_train, y_test = train_test_split(phrases, labels, test_size = 0.2, random_state = 3)\n",
    "                                             \n",
    "# Vectorize :\n",
    "# 1. Word2Vec\n",
    "\n",
    "freqdist = None\n",
    "X_train, mask_train = get_dataset_embeddings(docs_train, \"AVG\", w2v.wv, freqdist)\n",
    "X_test_PhraseBank, mask_test = get_dataset_embeddings(docs_test, \"AVG\", w2v.wv, freqdist)\n",
    "y_train = np.array(y_train)[mask_train]\n",
    "y_test_PhraseBank = np.array(y_test)[mask_test]\n",
    "docs_test_PhraseBank = np.array(docs_test)[mask_test]\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.001, random_state=3, max_iter=400, activation='relu')\n",
    "mlp_model.fit(X_train,y_train)\n",
    "y_hat = mlp_model.predict(X_test_PhraseBank)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test_PhraseBank, y_hat))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.73      0.78       110\n",
      "           1       0.88      0.94      0.91       227\n",
      "\n",
      "    accuracy                           0.87       337\n",
      "   macro avg       0.86      0.83      0.85       337\n",
      "weighted avg       0.87      0.87      0.87       337\n",
      "\n"
     ]
    }
   ],
   "source": [
    "''' \n",
    "Evaluate Model on PhraseBank Dataset.\n",
    " -- CountVectorizer -- \n",
    "'''\n",
    "# Split to train and test data. For the Financial PhraseBank dataset\n",
    "phrases, labels = data_read.load_fin_pharsebank()\n",
    "#phrases = tokenize_docs(phrases, WNLemmatizer, ticker_re)\n",
    "docs_train, docs_test, y_train, y_test = train_test_split(phrases, labels, test_size = 0.2, random_state = 3)\n",
    "                                             \n",
    "# 2. BOW CountVectorizer  \n",
    "vectorizer = CountVectorizer(preprocessor=preprocessing.preprocess_doc,\n",
    "                             min_df = 4)\n",
    "\n",
    "X_train = vectorizer.fit_transform(docs_train)\n",
    "X_test = vectorizer.transform(docs_test)\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.0001, random_state=3, max_iter=1000, activation='relu')\n",
    "mlp_model.fit(X_train,y_train)\n",
    "y_hat = mlp_model.predict(X_test)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test,y_hat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  767 of 767 completed\n",
      "\n",
      "187 Failed downloads:\n",
      "- TEG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- LEH: No data found for this date range, symbol may be delisted\n",
      "- VNT: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- TWC: No data found for this date range, symbol may be delisted\n",
      "- MON: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- HSP: No data found for this date range, symbol may be delisted\n",
      "- ACAS: No data found for this date range, symbol may be delisted\n",
      "- FDC: No data found, symbol may be delisted\n",
      "- DOW: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CMCSK: No data found for this date range, symbol may be delisted\n",
      "- QRVO: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- JDSU: No data found for this date range, symbol may be delisted\n",
      "- RHT: No data found, symbol may be delisted\n",
      "- PAYC: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- TIF: No data found, symbol may be delisted\n",
      "- HPE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- JOY: No data found for this date range, symbol may be delisted\n",
      "- PCS: No data found for this date range, symbol may be delisted\n",
      "- MRNA: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CEPH: No data found for this date range, symbol may be delisted\n",
      "- GENZ: No data found for this date range, symbol may be delisted\n",
      "- WIN: No data found, symbol may be delisted\n",
      "- AGN: No data found, symbol may be delisted\n",
      "- XTO: No data found for this date range, symbol may be delisted\n",
      "- CHK: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- SHLD: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- BXLT: No data found for this date range, symbol may be delisted\n",
      "- WCG: No data found, symbol may be delisted\n",
      "- AKS: No data found, symbol may be delisted\n",
      "- ALXN: No data found, symbol may be delisted\n",
      "- WPX: No data found, symbol may be delisted\n",
      "- SWY: No data found for this date range, symbol may be delisted\n",
      "- MWW: No data found for this date range, symbol may be delisted\n",
      "- CFG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- ALTR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FTR: No data found, symbol may be delisted\n",
      "- AVP: No data found, symbol may be delisted\n",
      "- SCG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- ARG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- Q: No data found for this date range, symbol may be delisted\n",
      "- CZR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DWDP: No data found, symbol may be delisted\n",
      "- SE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- HLT: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CDAY: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FRX: No data found, symbol may be delisted\n",
      "- DPS: No data found for this date range, symbol may be delisted\n",
      "- SNDK: No data found for this date range, symbol may be delisted\n",
      "- DELL: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- WB: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- PYPL: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- SIAL: No data found for this date range, symbol may be delisted\n",
      "- ESV: No data found, symbol may be delisted\n",
      "- BHF: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- VIAB: No data found, symbol may be delisted\n",
      "- LO: No data found for this date range, symbol may be delisted\n",
      "- XEC: No data found, symbol may be delisted\n",
      "- FOXA: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FTV: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DTV: No data found, symbol may be delisted\n",
      "- TYC: No data found for this date range, symbol may be delisted\n",
      "- ANET: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CPGX: No data found for this date range, symbol may be delisted\n",
      "- OTIS: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- LW: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- XL: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- BEAM: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- QEP: No data found, symbol may be delisted\n",
      "- CCE: No data found for this date range, symbol may be delisted\n",
      "- BJS: No data found for this date range, symbol may be delisted\n",
      "- PCL: No data found for this date range, symbol may be delisted\n",
      "- YHOO: No data found for this date range, symbol may be delisted\n",
      "- ETFC: No data found, symbol may be delisted\n",
      "- SPLS: No data found for this date range, symbol may be delisted\n",
      "- CXO: No data found, symbol may be delisted\n",
      "- MOLX: No data found for this date range, symbol may be delisted\n",
      "- RAI: No data found for this date range, symbol may be delisted\n",
      "- STJ: No data found for this date range, symbol may be delisted\n",
      "- CTVA: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FLIR: No data found, symbol may be delisted\n",
      "- AYE: No data found for this date range, symbol may be delisted\n",
      "- BF.B: No data found for this date range, symbol may be delisted\n",
      "- FII: No data found, symbol may be delisted\n",
      "- RTN: No data found, symbol may be delisted\n",
      "- NOVL: No data found for this date range, symbol may be delisted\n",
      "- ACE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FOX: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- WYN: No data found for this date range, symbol may be delisted\n",
      "- S: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- LIFE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- NVLS: No data found for this date range, symbol may be delisted\n",
      "- APOL: No data found for this date range, symbol may be delisted\n",
      "- WRK: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DNR: No data found, symbol may be delisted\n",
      "- LLTC: No data found for this date range, symbol may be delisted\n",
      "- ADT: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CELG: No data found, symbol may be delisted\n",
      "- JNS: No data found for this date range, symbol may be delisted\n",
      "- SAI: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- LVLT: No data found for this date range, symbol may be delisted\n",
      "- BRCM: No data found for this date range, symbol may be delisted\n",
      "- SLE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- LXK: No data found for this date range, symbol may be delisted\n",
      "- BTU: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- KRFT: No data found for this date range, symbol may be delisted\n",
      "- TLAB: No data found for this date range, symbol may be delisted\n",
      "- KSU: No data found, symbol may be delisted\n",
      "- MEE: No data found for this date range, symbol may be delisted\n",
      "- BRK.B: No data found, symbol may be delisted\n",
      "- DYN: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CTLT: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CPWR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CSRA: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- WFM: No data found for this date range, symbol may be delisted\n",
      "- KHC: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- ANR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- TSG: No data found, symbol may be delisted\n",
      "- DJ: No data found for this date range, symbol may be delisted\n",
      "- NYX: No data found for this date range, symbol may be delisted\n",
      "- GAS: No data found for this date range, symbol may be delisted\n",
      "- CVC: No data found for this date range, symbol may be delisted\n",
      "- CTX: No data found for this date range, symbol may be delisted\n",
      "- CAM: No data found for this date range, symbol may be delisted\n",
      "- PGN: No data found for this date range, symbol may be delisted\n",
      "- MHS: No data found for this date range, symbol may be delisted\n",
      "- DV: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- TE: No data found for this date range, symbol may be delisted\n",
      "- ETSY: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CFN: No data found for this date range, symbol may be delisted\n",
      "- MFE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DNB: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- FRE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DO: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- SYF: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- SEDG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- MXIM: No data found, symbol may be delisted\n",
      "- MNK: No data found, symbol may be delisted\n",
      "- KSE: No data found for this date range, symbol may be delisted\n",
      "- HCBK: No data found for this date range, symbol may be delisted\n",
      "- NBL: No data found, symbol may be delisted\n",
      "- AV: No data found for this date range, symbol may be delisted\n",
      "- EK: No data found for this date range, symbol may be delisted\n",
      "- ABK: No data found for this date range, symbol may be delisted\n",
      "- GGP: No data found for this date range, symbol may be delisted\n",
      "- GMCR: No data found for this date range, symbol may be delisted\n",
      "- TSS: No data found, symbol may be delisted\n",
      "- FNM: No data found for this date range, symbol may be delisted\n",
      "- JCP: No data found, symbol may be delisted\n",
      "- IR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- MI: No data found for this date range, symbol may be delisted\n",
      "- INFO: No data found, symbol may be delisted\n",
      "- NAVI: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- DF: No data found, symbol may be delisted\n",
      "- SGP: No data found for this date range, symbol may be delisted\n",
      "- FDO: No data found for this date range, symbol may be delisted\n",
      "- LM: No data found, symbol may be delisted\n",
      "- CEG: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- PETM: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- MJN: No data found for this date range, symbol may be delisted\n",
      "- UA: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- TRB: No data found for this date range, symbol may be delisted\n",
      "- OMX: No data found for this date range, symbol may be delisted\n",
      "- BCR: No data found for this date range, symbol may be delisted\n",
      "- HOT: No data found for this date range, symbol may be delisted\n",
      "- KEYS: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CSC: No data found for this date range, symbol may be delisted\n",
      "- VAR: No data found, symbol may be delisted\n",
      "- HAR: No data found for this date range, symbol may be delisted\n",
      "- NE: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- RRD: No data found, symbol may be delisted\n",
      "- CBE: No data found for this date range, symbol may be delisted\n",
      "- MIL: No data found for this date range, symbol may be delisted\n",
      "- RX: No data found for this date range, symbol may be delisted\n",
      "- CARR: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- STI: No data found, symbol may be delisted\n",
      "- HWM: No data found for this date range, symbol may be delisted\n",
      "- PCP: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- CVH: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- ARNC: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- PLL: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- XLNX: No data found, symbol may be delisted\n",
      "- SLR: No data found for this date range, symbol may be delisted\n",
      "- JNY: No data found for this date range, symbol may be delisted\n",
      "- RDC: No data found, symbol may be delisted\n",
      "- APC: No data found, symbol may be delisted\n",
      "- OGN: Data doesn't exist for startDate = 1161320400, endDate = 1384927200\n",
      "- POM: No data found for this date range, symbol may be delisted\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/dw/vzm8v7wn111ccjzl7139x7xc0000gq/T/ipykernel_659/4237973528.py:22: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "  market_data_cleaned[(\"Change\", sym)] = price_ratio\n"
     ]
    }
   ],
   "source": [
    "''' \n",
    "-- Generate the proposed Autolabel Dataset --\n",
    "\n",
    "This will download all SP500 stock prices as a dataframe, and process the dataframe to \n",
    "have a daily change % for each stock (EOD price)/(Start of day price). \n",
    "\n",
    "'''\n",
    "# Create my dataset: get tickers and price history from YahooFinance\n",
    "tickers = data_read.get_sp500_ticker_names()\n",
    "symbols = yf.Tickers(\" \".join(tickers.keys()))\n",
    "market_data = symbols.history(interval = \"1d\", start=\"2006-10-20\", end=\"2013-11-20\", actions= False)\n",
    "\n",
    "market_data_cleaned = market_data.dropna(axis=0, how = \"all\")\n",
    "market_data_cleaned = market_data_cleaned.dropna(axis=1, how = \"all\")\n",
    "\n",
    "cols, symbols = zip(*market_data_cleaned.columns)\n",
    "\n",
    "traded_symbols = set(symbols)\n",
    "for sym in traded_symbols:\n",
    "    price_ratio  = market_data_cleaned[('Close',sym)] / market_data_cleaned[('Open',sym)]\n",
    "    price_ratio = 100 * (price_ratio -1)\n",
    "    market_data_cleaned[(\"Change\", sym)] = price_ratio \n",
    "\n",
    "market_data_cleaned.index = pd.to_datetime(market_data_cleaned.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' -- Generate the proposed Autolabel Dataset -- Part 2\n",
    "\n",
    "Build labeled news dataset:\n",
    "Reterieve only the news that mention companies from S&P500. For each news header keep it only\n",
    "if the price of the company it talks about changes by more then +-2%. Label the obes than rose\n",
    "more then two percent with 1 (positive), and the others with 0 (negative)\n",
    "\n",
    "'''\n",
    "relevant_news = data_read.get_relevant_news(traded_symbols, tickers, headers)\n",
    "out_of_trade_days_cnt = 0\n",
    "nan_ticker_change_cnt = 0\n",
    "for news_item in relevant_news:\n",
    "    date, symbol, _, _ = news_item\n",
    "    if pd.to_datetime(date) not in market_data_cleaned.index:\n",
    "        news_item[2] = None \n",
    "        out_of_trade_days_cnt += 1\n",
    "    else:\n",
    "        print(\"date {} sym {} \".format(date, symbol))\n",
    "        ticker_day_change = market_data_cleaned.loc[pd.to_datetime(date) ,(\"Change\",symbol)]\n",
    "        if not isinstance(ticker_day_change, float):\n",
    "            nan_ticker_change_cnt += 1\n",
    "        #print(ticker_day_change)\n",
    "        if  pd.notna(ticker_day_change) and (ticker_day_change > 2 or ticker_day_change < -2):\n",
    "            news_item[2] = ticker_day_change\n",
    "\n",
    "labeled_dataset = pd.DataFrame(relevant_news, columns = [\"date\", \"symbol\", \"day_change\", \"header\"])\n",
    "labeled_dataset.set_index(\"date\", inplace=True)\n",
    "''' Final step: label changes > +2% as 1 and changes  < -2% as 0. '''\n",
    "reuters_labeled = labeled_dataset.dropna(axis=0, how = \"any\")\n",
    "reuters_labeled = reuters_labeled[ (reuters_labeled[\"day_change\"] > 0) | (reuters_labeled[\"day_change\"] < 0)] \n",
    "reuters_labeled.loc[reuters_labeled[\"day_change\"] > 0, 'day_change'] = 1\n",
    "reuters_labeled.loc[reuters_labeled[\"day_change\"] < 0, 'day_change'] = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "''' \n",
    "-- Generate the proposed Autolabel Dataset only assignind a pos/neg label if there is\n",
    "   no significant market movement that day ( -1.2% < x < 1.2%) --\n",
    "\n",
    "* Run this cell or the previous cell to generate the automatically-labelled dataset\n",
    "  depends if you want to account for market ups/ and down or not.\n",
    "\n",
    "This will download all SP500 stock prices as a dataframe, and process the dataframe to \n",
    "have a daily change % for each stock (EOD price)/(Start of day price). \n",
    "\n",
    "'''\n",
    "\n",
    "# Get S&P500 tickers and price history from YahooFinance\n",
    "tickers = data_read.get_sp500_ticker_names()\n",
    "symbols = yf.Tickers(\" \".join(tickers.keys()))\n",
    "market_data = symbols.history(interval = \"1d\", start=\"2006-10-20\", end=\"2013-11-20\", actions= False)\n",
    "market_data.index = pd.to_datetime(market_data.index)\n",
    "\n",
    "# Drop all days without trading, and all NaN columns from dataframe retrieved by yahoofinance.\n",
    "# Get the \n",
    "market_data = market_data.dropna(axis=0, how = \"all\")\n",
    "market_data= market_data.dropna(axis=1, how = \"all\")\n",
    "_, symbols = zip(*market_data.columns)\n",
    "traded_symbols = set(symbols)\n",
    "\n",
    "# Add daily change % for each stock (EOD price)/(Start of day price) column to the dataframe. \n",
    "for sym in traded_symbols:\n",
    "    price_ratio  = market_data[('Close',sym)] / market_data[('Open',sym)]\n",
    "    price_ratio = 100 * (price_ratio -1)\n",
    "    market_data[(\"Change\", sym)] = price_ratio \n",
    "\n",
    "\n",
    "# Calculate SP500 unweighted index (it is the average price of all the stocks in the S&P500) '''\n",
    "snp_index = market_data[\"Change\"].sum(axis=1) / market_data[\"Change\"].notna().sum(axis=1)\n",
    "market_data[(\"Change\",\"SNP_INDX\")] = snp_index\n",
    "\n",
    "'''\n",
    "Build labeled news dataset:\n",
    "Reterieve only the reuters news that mention companies from S&P500. For each news header keep it only\n",
    "if the price of the company it talks about changes by more then +-2%. Label the obes than rose\n",
    "more then two percent with 1 (positive), and the others with 0 (negative)\n",
    "'''\n",
    "relevant_news = data_read.get_relevant_news(traded_symbols, tickers, headers)\n",
    "for news_item in relevant_news:\n",
    "    date, symbol, _, _ = news_item\n",
    "    if pd.to_datetime(date) not in market_data.index:\n",
    "        news_item[2] = None \n",
    "    else:\n",
    "        #print(\"date {} sym {} \".format(date, symbol))\n",
    "        ticker_day_change = market_data.loc[pd.to_datetime(date) ,(\"Change\",symbol)]\n",
    "        sp_index_change = market_data.loc[pd.to_datetime(date) ,(\"Change\",\"SNP_INDX\")]\n",
    "        if  pd.notna(ticker_day_change) and (\n",
    "                (ticker_day_change > 2 and sp_index_change < 1.2 ) \n",
    "                or \n",
    "                (ticker_day_change < -2 and sp_index_change > -1.2) \n",
    "            ) :\n",
    "            news_item[2] = ticker_day_change\n",
    "\n",
    "reuters_labeled = pd.DataFrame(relevant_news, columns = [\"date\", \"symbol\", \"day_change\", \"header\"])\n",
    "reuters_labeled.set_index(\"date\", inplace=True)\n",
    "reuters_labeled = reuters_labeled.dropna(axis=0, how = \"any\")\n",
    "reuters_labeled = reuters_labeled[ (reuters_labeled[\"day_change\"] > 0) | (reuters_labeled[\"day_change\"] < 0)] \n",
    "reuters_labeled.loc[reuters_labeled[\"day_change\"] > 0, 'day_change'] = 1\n",
    "reuters_labeled.loc[reuters_labeled[\"day_change\"] < 0, 'day_change'] = 0\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4880/4880 [00:45<00:00, 106.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.57      0.58      0.58       499\n",
      "         1.0       0.55      0.54      0.55       473\n",
      "\n",
      "    accuracy                           0.56       972\n",
      "   macro avg       0.56      0.56      0.56       972\n",
      "weighted avg       0.56      0.56      0.56       972\n",
      "\n"
     ]
    }
   ],
   "source": [
    "''' \n",
    " -- Evaluate W2V Model on ReutersAuto Dataset --\n",
    "'''\n",
    "# Split to train and test data. For the our automatically-labelled Reuters dataset.\n",
    "labeled_headers = reuters_labeled['header']\n",
    "labels = reuters_labeled['day_change']\n",
    "\n",
    "labeled_headers_tokens, _, _ = generate_tokenized_corpus(labeled_headers)\n",
    "docs_train, docs_test, y_train, y_test = train_test_split(labeled_headers_tokens, labels, test_size = 0.2, random_state = 3)\n",
    "\n",
    "#1. Word2Vec embeddings\n",
    "X_train, mask_train = get_dataset_embeddings(docs_train, \"AVG\", w2v.wv, freqdist)\n",
    "X_test, mask_test = get_dataset_embeddings(docs_test, \"AVG\", w2v.wv, freqdist)\n",
    "y_train = np.array(y_train)[mask_train]\n",
    "y_test = np.array(y_test)[mask_test]\n",
    "\n",
    "#print(X_train[:20])\n",
    "#print(y_train[:20])\n",
    "#print(f\"train {X_train.shape} test {X_test.shape} train y {y_train.shape} test {y_test.shape}\")\n",
    "\n",
    "# 2. BOW CountVectorizer  \n",
    "'''vectorizer = CountVectorizer(preprocessor=preprocess_header,\n",
    "                             min_df = 4)\n",
    "\n",
    "X_train = vectorizer.fit_transform(docs_train)\n",
    "X_test = vectorizer.transform(docs_test)\n",
    "\n",
    "print(f\"train {X_train.shape} test {X_test.shape} train y {y_train.shape} test {y_test.shape}\")'''\n",
    "\n",
    "'''lr_model = LogisticRegression(solver=\"liblinear\", random_state=3)\n",
    "lr_model.fit(X_train, y_train)\n",
    "y_hat = lr_model.predict(X_test)\n",
    "'''\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.00005, hidden_layer_sizes=(100,), random_state=3, max_iter=3000, activation='relu')\n",
    "mlp_model.fit(X_train, y_train)\n",
    "y_hat = mlp_model.predict(X_test)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test, y_hat))\n",
    "                                             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' -- RQ1 -- \n",
    "    Evaluate sentiment trained model on the PhraseBank Dataset \n",
    "''' \n",
    "\n",
    "from sklearn.datasets import load_files\n",
    "\n",
    "dataset = load_files(\"./movie-reviews/\")\n",
    "data = [data.decode('UTF-8') for data in dataset.data]\n",
    "#data = tokenize_docs(data, WNLemmatizer, ticker_re)\n",
    "\n",
    "docs_train, docs_test, y_train, _ = train_test_split(dataset.data, dataset.target, test_size = 0.001, random_state = 3)\n",
    "#docs_train, docs_test, y_train, y_test = train_test_split(data, dataset.target, test_size = 0.2, random_state = 3)\n",
    "\n",
    "X_train, mask_train = get_dataset_embeddings(docs_train, \"AVG\", w2v_pre, None)\n",
    "#X_test, mask_test = get_dataset_embeddings(docs_test, \"LOG\", w2v_pre, freqdist)\n",
    "y_train = np.array(y_train)[mask_train]\n",
    "#y_test = np.array(y_test)[mask_test]\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.0001, random_state=3, max_iter=1000, activation='relu')\n",
    "mlp_model.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_hat = mlp_model.predict(X_test_PhraseBank)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test_PhraseBank, y_hat))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' -- RQ1 --\n",
    "    Sentiment trained model, evaluate on sentiment data  \n",
    "    -- Countvectorizer -- \n",
    "'''\n",
    "dataset = load_files(\"./movie-reviews/\")\n",
    "data = [data.decode('UTF-8') for data in dataset.data]\n",
    "\n",
    "docs_train, docs_test , y_train, y_test = train_test_split(data, dataset.target, test_size = 0.2, random_state = 3)\n",
    "\n",
    "# 2. BOW CountVectorizer  \n",
    "vectorizer = CountVectorizer(preprocessor=preprocessing.preprocess_doc,\n",
    "                             min_df = 4)\n",
    "\n",
    "X_train = vectorizer.fit_transform(docs_train)\n",
    "X_test = vectorizer.transform(docs_test)\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.00005, random_state=3, max_iter=400, activation='relu')\n",
    "mlp_model.fit(X_train,y_train)\n",
    "\n",
    "\n",
    "y_hat = mlp_model.predict(X_test)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test,y_hat))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.34      0.37      0.35       110\n",
      "           1       0.68      0.64      0.66       226\n",
      "\n",
      "    accuracy                           0.55       336\n",
      "   macro avg       0.51      0.51      0.51       336\n",
      "weighted avg       0.57      0.55      0.56       336\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "''' -- RQ1 -- \n",
    "    Evaluate sentiment trained model on the PhraseBank Dataset \n",
    "    * CountVectorizer *\n",
    "'''\n",
    "\n",
    "dataset = load_files(\"./movie-reviews/\")\n",
    "data = [data.decode('UTF-8') for data in dataset.data]\n",
    "\n",
    "docs_train, _ , y_train, _ = train_test_split(data, dataset.target, test_size = 0.2, random_state = 3)\n",
    "\n",
    "\n",
    "# 2. BOW CountVectorizer  \n",
    "vectorizer = CountVectorizer(preprocessor=preprocessing.preprocess_doc,\n",
    "                             min_df = 4)\n",
    "\n",
    "X_train = vectorizer.fit_transform(docs_train)\n",
    "X_test = vectorizer.transform([\" \".join(doc) for doc in docs_test_PhraseBank])\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.0001, random_state=3, max_iter=1000, activation='relu')\n",
    "mlp_model.fit(X_train, y_train)\n",
    "\n",
    "y_hat = mlp_model.predict(X_test)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test_PhraseBank, y_hat))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [01:12<00:00, 137.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.73      0.72       908\n",
      "           1       0.73      0.71      0.72       920\n",
      "\n",
      "    accuracy                           0.72      1828\n",
      "   macro avg       0.72      0.72      0.72      1828\n",
      "weighted avg       0.72      0.72      0.72      1828\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ozilman/NLP/venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "''' -- Sentiment140 Data Trial (not used for PhraseBank) -- '''\n",
    "\n",
    "sentiment_data = pd.read_csv(\"./sentiment140_sample1.csv\", header = None, encoding = \"ISO-8859-1\")\n",
    "#sentiment_data.head()\n",
    "sentiment_data_pos_neg = sentiment_data.loc[(sentiment_data[0] == 0) | (sentiment_data[0] == 4)]\n",
    "len(sentiment_data)\n",
    "\n",
    "sentiment_data_pos_neg = sentiment_data_pos_neg[[0,5]]\n",
    "sentiment_data_pos_neg.iloc[sentiment_data_pos_neg[0] == 4, 0] = 1\n",
    "\n",
    "''' \n",
    "Evaluate Model on PhraseBank Dataset.\n",
    "'''\n",
    "# Split to train and test. \n",
    "# Split to train and test data. For the FiQa dataset\n",
    "labeled_headers = sentiment_data_pos_neg.iloc[:,1]\n",
    "labels = sentiment_data_pos_neg.iloc[:,0]\n",
    "\n",
    "labeled_headers_tokens, _, _ = generate_tokenized_corpus(labeled_headers)\n",
    "docs_train, docs_test, y_train, y_test = train_test_split(\n",
    "                                                labeled_headers_tokens, \n",
    "                                                labels, \n",
    "                                                test_size = 0.2, \n",
    "                                                random_state = 3\n",
    "                                             )\n",
    "\n",
    "X_train, mask_train = get_dataset_embeddings(docs_train, \"AVG\", w2v_pre, None)\n",
    "X_test, mask_test = get_dataset_embeddings(docs_test, \"AVG\", w2v_pre, None)\n",
    "y_train = np.array(y_train)[mask_train]\n",
    "y_test = np.array(y_test)[mask_test]\n",
    "\n",
    "'''_, _, y_train, y_test = train_test_split(\n",
    "            labeled_headers_tokens, \n",
    "            labels, \n",
    "            test_size = 0.2, \n",
    "            random_state = 3\n",
    ")'''\n",
    "\n",
    "mlp_model = MLPClassifier(learning_rate_init=0.0001, random_state=3, max_iter=1000, activation='relu')\n",
    "mlp_model.fit(X_train,y_train)\n",
    "y_hat = mlp_model.predict(X_test)\n",
    "print(\"classification report:\\n\", metrics.classification_report(y_test,y_hat))\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "76555c72c80d8d4a69c4ec0bb5b1922ba2358a3f5fae3d9701a735d0328ce790"
  },
  "kernelspec": {
   "display_name": "Python 3.9.10 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}