chris_stuff.Rmd

---
title: "Chris Stuff"
author: "Altamash Rafiq"
date: "4/2/2020"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE, message = FALSE, warning = FALSE, comment = NA)
library(tidyverse)
library(text2vec)
library(tidytext)
library(widyr)
library(irlba)
```

```{r}
data = read_csv('data/ted_expanded.csv')
transcripts = data %>% select(url, transcript)
transcripts = transcripts[1:100,]
```

```{r}
tidy_skipgrams = transcripts %>%
    unnest_tokens(ngram, transcript, token = "ngrams", n = 8) %>%
    mutate(ngramID = row_number()) %>% 
    tidyr::unite(skipgramID, url, ngramID) %>%
    unnest_tokens(word, ngram)

unigram_probs = transcripts %>%
    unnest_tokens(word, transcript) %>%
    count(word, sort = TRUE) %>%
    mutate(p = n / sum(n))

skipgram_probs = tidy_skipgrams %>%
    pairwise_count(word, skipgramID, diag = TRUE, sort = TRUE) %>%
    mutate(p = n / sum(n))

normalized_prob = skipgram_probs %>%
    filter(n > 20) %>%
    rename(word1 = item1, word2 = item2) %>%
    left_join(unigram_probs %>%
                  select(word1 = word, p1 = p),
              by = "word1") %>%
    left_join(unigram_probs %>%
                  select(word2 = word, p2 = p),
              by = "word2") %>%
    mutate(p_together = p / p1 / p2)

pmi_matrix = normalized_prob %>%
    mutate(pmi = log10(p_together)) %>%
    cast_sparse(word1, word2, pmi)

#remove missing data
pmi_matrix@x[is.na(pmi_matrix@x)] = 0
#run SVD
pmi_svd = irlba(pmi_matrix, 256, maxit = 500)
#next we output the word vectors:
word_vectors = pmi_svd$u
rownames(word_vectors) = rownames(pmi_matrix)
```