Label Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading libraries\n",
    "import pandas as pd\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "stemmer = SnowballStemmer(\"english\")\n",
    "import nltk\n",
    "import re\n",
    "from sklearn.cluster import SpectralClustering\n",
    "from sklearn.metrics import silhouette_score\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from collections import Counter\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing data\n",
    "ted_main = pd.read_csv('ted_main.csv')\n",
    "ted_main['tags'] = ted_main['tags'].apply(lambda x: ast.literal_eval(x))\n",
    "transcripts = pd.read_csv('transcripts.csv')\n",
    "ted_merged = pd.merge(left=transcripts,\n",
    "                      right=ted_main,\n",
    "                      left_on='url',\n",
    "                      right_on='url')\n",
    "transcript = ted_merged.transcript"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        if re.search('[a-zA-Z]', token):\n",
    "            filtered_tokens.append(token)\n",
    "#    stems = [stemmer.stem(t) for t in filtered_tokens]\n",
    "    return filtered_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 29s\n",
      "(2467, 364)\n"
     ]
    }
   ],
   "source": [
    "doc = transcript.tolist()\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,\n",
    "                                 min_df=0.2, stop_words='english',\n",
    "                                 use_idf=True, tokenizer=tokenize, ngram_range=(1,3))\n",
    "\n",
    "%time tfidf_matrix = tfidf_vectorizer.fit_transform(doc) #fit the vectorizer to synopses\n",
    "\n",
    "print(tfidf_matrix.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Spectural Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8]\n"
     ]
    }
   ],
   "source": [
    "n_cluster = range(2,11)\n",
    "best_param = []\n",
    "list_score = []\n",
    "for n in n_cluster:\n",
    "    model = SpectralClustering(n_clusters=n)\n",
    "    model.fit(tfidf_matrix)\n",
    "    label = model.labels_\n",
    "    list_score.append(silhouette_score(tfidf_matrix, label))\n",
    "list_score = np.array(list_score)\n",
    "best_param.append(n_cluster[list_score.argmax()])\n",
    "print(best_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 266, 2: 166, 1: 361, 6: 578, 5: 770, 7: 82, 4: 156, 3: 88})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = SpectralClustering(n_clusters=8)\n",
    "model.fit(tfidf_matrix)\n",
    "label = model.labels_\n",
    "clusters = label.tolist()\n",
    "Counter(clusters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### KMeans Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuned Kmeans Parameter: {'n_clusters': 10}\n"
     ]
    }
   ],
   "source": [
    "n_cluster = list(range(2,11))\n",
    "param_grid = {'n_clusters': n_cluster}\n",
    "kmeans = KMeans()\n",
    "kmeans_cv = GridSearchCV(kmeans, param_grid)\n",
    "kmeans_cv.fit(tfidf_matrix)\n",
    "print(\"Tuned Kmeans Parameter: {}\".format(kmeans_cv.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({5: 248, 4: 451, 6: 368, 2: 450, 1: 544, 7: 104, 0: 182, 3: 120})"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km_model = KMeans(n_clusters=8)\n",
    "km_model.fit(tfidf_matrix)\n",
    "km_label = km_model.labels_\n",
    "km_clusters = km_label.tolist()\n",
    "Counter(km_clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>transcript</th>\n",
       "      <th>tags</th>\n",
       "      <th>cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Why we love, why we cheat</td>\n",
       "      <td>I'd like to talk today about the two biggest s...</td>\n",
       "      <td>[cognitive science, culture, evolution, gender...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Happiness in body and soul</td>\n",
       "      <td>I bet you're worried.(Laughter)I was worried. ...</td>\n",
       "      <td>[culture, entertainment, gender, global issues...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>\"Black Men Ski\"</td>\n",
       "      <td>Has anyone ever been to Aspen, Colorado? It's ...</td>\n",
       "      <td>[culture, entertainment, live music, music, pe...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>Tales of passion</td>\n",
       "      <td>Thank you so much. It's really scary to be her...</td>\n",
       "      <td>[South America, entertainment, global issues, ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>What security means to me</td>\n",
       "      <td>I think it'll be a relief to some people and a...</td>\n",
       "      <td>[activism, culture, global issues, spoken word...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>A passionate, personal case for education</td>\n",
       "      <td>This is my first trip, my first foreign trip a...</td>\n",
       "      <td>[culture, education, global issues, politics, ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>The surprising spread of Idol TV</td>\n",
       "      <td>I'd like to ask you, what do these three peopl...</td>\n",
       "      <td>[culture, entertainment, global issues, poetry...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>Photographing the hidden story</td>\n",
       "      <td>My name is Ryan Lobo, and I've been involved i...</td>\n",
       "      <td>[Asia, art, humanity, photography, storytellin...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643</th>\n",
       "      <td>Radical women, embracing tradition</td>\n",
       "      <td>Salaam. Namaskar. Good morning. Given my TED p...</td>\n",
       "      <td>[culture, feminism, social change, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>Women, wartime and the dream of peace</td>\n",
       "      <td>I woke up in the middle of the night with the ...</td>\n",
       "      <td>[drones, global issues, iraq, war, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>A feminine response to Iceland's financial crash</td>\n",
       "      <td>It sure used to be a lot easier to be from Ice...</td>\n",
       "      <td>[business, feminism, politics, women, women in...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>A call to men</td>\n",
       "      <td>I grew up in New York City, between Harlem and...</td>\n",
       "      <td>[Gender equality, bullying, culture, feminism,...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>New data on the rise of women</td>\n",
       "      <td>We are now going through an amazing and unprec...</td>\n",
       "      <td>[Gender equality, culture, economics, educatio...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>Why we have too few women leaders</td>\n",
       "      <td>So for any of us in this room today, let's sta...</td>\n",
       "      <td>[Gender equality, business, feminism, goal-set...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>A test that finds 3x more breast tumors, and w...</td>\n",
       "      <td>There are two groups of women when it comes to...</td>\n",
       "      <td>[business, cancer, medicine, science, technology]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>Drawing on humor for change</td>\n",
       "      <td>(Laughter)I was afraid of womanhood. Not that ...</td>\n",
       "      <td>[art, comedy, culture, feminism, humor, media,...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>824</th>\n",
       "      <td>Social media and the end of gender</td>\n",
       "      <td>I'm going to make an argument today that may s...</td>\n",
       "      <td>[advertising, culture, entertainment, media, s...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829</th>\n",
       "      <td>Mother and daughter doctor-heroes</td>\n",
       "      <td>Hawa Abdi: Many people — 20 years for Somalia ...</td>\n",
       "      <td>[community, global issues, health, medicine, s...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>834</th>\n",
       "      <td>Inspiring a life of immersion</td>\n",
       "      <td>I've been spending a lot of time traveling aro...</td>\n",
       "      <td>[culture, economics, global issues, investment...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>On being a woman and a diplomat</td>\n",
       "      <td>Pat Mitchell: What is the story of this pin?Ma...</td>\n",
       "      <td>[Foreign Policy, Gender equality, business, iraq]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>This isn't her mother's feminism</td>\n",
       "      <td>So I was born on the last day of the last year...</td>\n",
       "      <td>[culture, feminism, history, identity, women, ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>906</th>\n",
       "      <td>Art in exile</td>\n",
       "      <td>The story I wanted to share with you today is ...</td>\n",
       "      <td>[art, film, global issues, photography, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>942</th>\n",
       "      <td>See Yemen through my eyes</td>\n",
       "      <td>Pat Mitchell: You have brought us images from ...</td>\n",
       "      <td>[communication, global issues, journalism, war...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967</th>\n",
       "      <td>Compassion and the true meaning of empathy</td>\n",
       "      <td>I want to address the issue of compassion. Com...</td>\n",
       "      <td>[Buddhism, compassion, culture, death, global ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>What we learn before we're born</td>\n",
       "      <td>My subject today is learning. And in that spir...</td>\n",
       "      <td>[brain, children, culture, education, evolutio...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1057</th>\n",
       "      <td>\"Women of Hope\"</td>\n",
       "      <td>(Music)♫ They stood together ♫ ♫ under a tree ...</td>\n",
       "      <td>[TEDx, activism, live music, music, performanc...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1068</th>\n",
       "      <td>Women entrepreneurs, example not exception</td>\n",
       "      <td>We do not invest in victims, we invest in surv...</td>\n",
       "      <td>[TEDx, entrepreneur, global issues, poverty, w...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>The single biggest health threat women face</td>\n",
       "      <td>One out of two of you women will be impacted b...</td>\n",
       "      <td>[TEDx, activism, biology, disease, health, hea...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>Let's put birth control back on the agenda</td>\n",
       "      <td>Today, I'd like to talk with you about somethi...</td>\n",
       "      <td>[TEDx, culture, global issues, health care, po...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1150</th>\n",
       "      <td>A teen just trying to figure it out</td>\n",
       "      <td>Four years ago today, exactly, actually, I sta...</td>\n",
       "      <td>[TEDx, entertainment, fashion, feminism, ident...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1171</th>\n",
       "      <td>The attitudes that sparked Arab Spring</td>\n",
       "      <td>My talk today is about something maybe a coupl...</td>\n",
       "      <td>[Egypt, TEDx, data, economics, protests, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188</th>\n",
       "      <td>Women should represent women in media</td>\n",
       "      <td>Like most journalists, I'm an idealist. I love...</td>\n",
       "      <td>[TEDx, global issues, journalism, media, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1340</th>\n",
       "      <td>How movies teach manhood</td>\n",
       "      <td>You know, my favorite part of being a dad is t...</td>\n",
       "      <td>[TEDx, culture, film, identity, men, movies, p...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349</th>\n",
       "      <td>Why Libya's revolution didn't work -- and what...</td>\n",
       "      <td>I have never, ever forgotten the words of my g...</td>\n",
       "      <td>[compassion, democracy, global issues, politic...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1433</th>\n",
       "      <td>Violence against women -- it's a men's issue</td>\n",
       "      <td>I'm going to share with you a paradigm-shiftin...</td>\n",
       "      <td>[Gender equality, TEDx, feminism, men, sexual ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1445</th>\n",
       "      <td>A Saudi woman who dared to drive</td>\n",
       "      <td>Allow me to start this talk with a question to...</td>\n",
       "      <td>[Islam, Middle East, cars, culture, feminism, ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1533</th>\n",
       "      <td>Why stay in Chernobyl? Because it's home.</td>\n",
       "      <td>Three years ago, I was standing about a hundre...</td>\n",
       "      <td>[adventure, disaster relief, nuclear energy]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1578</th>\n",
       "      <td>So we leaned in ... now what?</td>\n",
       "      <td>Pat Mitchell: Your first time back on the TEDW...</td>\n",
       "      <td>[Gender equality, feminism, leadership, women,...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1581</th>\n",
       "      <td>A little-told tale of sex and sensuality</td>\n",
       "      <td>So when I was in Morocco, in Casablanca, not s...</td>\n",
       "      <td>[Egypt, Middle East, TED Fellows, culture, rel...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1582</th>\n",
       "      <td>His and hers … healthcare</td>\n",
       "      <td>Some of my most wonderful memories of childhoo...</td>\n",
       "      <td>[depression, health, heart health, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>How we turned the tide on domestic violence (H...</td>\n",
       "      <td>I want you to imagine what a breakthrough this...</td>\n",
       "      <td>[gender, violence, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1615</th>\n",
       "      <td>Can we all \"have it all\"?</td>\n",
       "      <td>So my moment of truth did not come all at once...</td>\n",
       "      <td>[Gender equality, inequality, men, policy, wom...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1728</th>\n",
       "      <td>Find your voice against gender violence</td>\n",
       "      <td>Talking about empowerment is odd, because when...</td>\n",
       "      <td>[TEDx, activism, gender, india, media, sexual ...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1746</th>\n",
       "      <td>The courage to tell a hidden story</td>\n",
       "      <td>When I turned 19, I started my career as the f...</td>\n",
       "      <td>[Middle East, TED Fellows, gender, global issu...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1750</th>\n",
       "      <td>The career advice you probably didn’t get</td>\n",
       "      <td>Women represent 50 percent of middle managemen...</td>\n",
       "      <td>[TEDx, business, gender, leadership, men, wome...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1798</th>\n",
       "      <td>Fighters and mourners of the Ukrainian revolution</td>\n",
       "      <td>When I arrived in Kiev, on February 1 this yea...</td>\n",
       "      <td>[TED Fellows, photography, war]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1828</th>\n",
       "      <td>10 myths about psychology, debunked</td>\n",
       "      <td>You've heard of your I.Q., your general intell...</td>\n",
       "      <td>[TEDYouth, TEDx, brain, personality, psychology]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>For these women, reading is a daring act</td>\n",
       "      <td>As an Arab female photographer, I have always ...</td>\n",
       "      <td>[Islam, Middle East, TED Fellows, education, f...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>How I work to protect women from honor killings</td>\n",
       "      <td>While preparing for my talk I was reflecting o...</td>\n",
       "      <td>[Middle East, activism, feminism, violence, wo...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1856</th>\n",
       "      <td>The good news about PMS</td>\n",
       "      <td>How many people here have heard of PMS? Everyb...</td>\n",
       "      <td>[TEDx, depression, psychology, women]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "23                            Why we love, why we cheat   \n",
       "24                           Happiness in body and soul   \n",
       "101                                     \"Black Men Ski\"   \n",
       "169                                    Tales of passion   \n",
       "289                           What security means to me   \n",
       "449           A passionate, personal case for education   \n",
       "541                    The surprising spread of Idol TV   \n",
       "562                      Photographing the hidden story   \n",
       "643                  Radical women, embracing tradition   \n",
       "782               Women, wartime and the dream of peace   \n",
       "790    A feminine response to Iceland's financial crash   \n",
       "791                                       A call to men   \n",
       "793                       New data on the rise of women   \n",
       "798                   Why we have too few women leaders   \n",
       "805   A test that finds 3x more breast tumors, and w...   \n",
       "819                         Drawing on humor for change   \n",
       "824                  Social media and the end of gender   \n",
       "829                   Mother and daughter doctor-heroes   \n",
       "834                       Inspiring a life of immersion   \n",
       "836                     On being a woman and a diplomat   \n",
       "847                    This isn't her mother's feminism   \n",
       "906                                        Art in exile   \n",
       "942                           See Yemen through my eyes   \n",
       "967          Compassion and the true meaning of empathy   \n",
       "1028                    What we learn before we're born   \n",
       "1057                                    \"Women of Hope\"   \n",
       "1068         Women entrepreneurs, example not exception   \n",
       "1114        The single biggest health threat women face   \n",
       "1131         Let's put birth control back on the agenda   \n",
       "1150                A teen just trying to figure it out   \n",
       "1171             The attitudes that sparked Arab Spring   \n",
       "1188              Women should represent women in media   \n",
       "1340                           How movies teach manhood   \n",
       "1349  Why Libya's revolution didn't work -- and what...   \n",
       "1433       Violence against women -- it's a men's issue   \n",
       "1445                   A Saudi woman who dared to drive   \n",
       "1533          Why stay in Chernobyl? Because it's home.   \n",
       "1578                      So we leaned in ... now what?   \n",
       "1581           A little-told tale of sex and sensuality   \n",
       "1582                          His and hers … healthcare   \n",
       "1590  How we turned the tide on domestic violence (H...   \n",
       "1615                          Can we all \"have it all\"?   \n",
       "1728            Find your voice against gender violence   \n",
       "1746                 The courage to tell a hidden story   \n",
       "1750          The career advice you probably didn’t get   \n",
       "1798  Fighters and mourners of the Ukrainian revolution   \n",
       "1828                10 myths about psychology, debunked   \n",
       "1837           For these women, reading is a daring act   \n",
       "1841    How I work to protect women from honor killings   \n",
       "1856                            The good news about PMS   \n",
       "\n",
       "                                             transcript  \\\n",
       "23    I'd like to talk today about the two biggest s...   \n",
       "24    I bet you're worried.(Laughter)I was worried. ...   \n",
       "101   Has anyone ever been to Aspen, Colorado? It's ...   \n",
       "169   Thank you so much. It's really scary to be her...   \n",
       "289   I think it'll be a relief to some people and a...   \n",
       "449   This is my first trip, my first foreign trip a...   \n",
       "541   I'd like to ask you, what do these three peopl...   \n",
       "562   My name is Ryan Lobo, and I've been involved i...   \n",
       "643   Salaam. Namaskar. Good morning. Given my TED p...   \n",
       "782   I woke up in the middle of the night with the ...   \n",
       "790   It sure used to be a lot easier to be from Ice...   \n",
       "791   I grew up in New York City, between Harlem and...   \n",
       "793   We are now going through an amazing and unprec...   \n",
       "798   So for any of us in this room today, let's sta...   \n",
       "805   There are two groups of women when it comes to...   \n",
       "819   (Laughter)I was afraid of womanhood. Not that ...   \n",
       "824   I'm going to make an argument today that may s...   \n",
       "829   Hawa Abdi: Many people — 20 years for Somalia ...   \n",
       "834   I've been spending a lot of time traveling aro...   \n",
       "836   Pat Mitchell: What is the story of this pin?Ma...   \n",
       "847   So I was born on the last day of the last year...   \n",
       "906   The story I wanted to share with you today is ...   \n",
       "942   Pat Mitchell: You have brought us images from ...   \n",
       "967   I want to address the issue of compassion. Com...   \n",
       "1028  My subject today is learning. And in that spir...   \n",
       "1057  (Music)♫ They stood together ♫ ♫ under a tree ...   \n",
       "1068  We do not invest in victims, we invest in surv...   \n",
       "1114  One out of two of you women will be impacted b...   \n",
       "1131  Today, I'd like to talk with you about somethi...   \n",
       "1150  Four years ago today, exactly, actually, I sta...   \n",
       "1171  My talk today is about something maybe a coupl...   \n",
       "1188  Like most journalists, I'm an idealist. I love...   \n",
       "1340  You know, my favorite part of being a dad is t...   \n",
       "1349  I have never, ever forgotten the words of my g...   \n",
       "1433  I'm going to share with you a paradigm-shiftin...   \n",
       "1445  Allow me to start this talk with a question to...   \n",
       "1533  Three years ago, I was standing about a hundre...   \n",
       "1578  Pat Mitchell: Your first time back on the TEDW...   \n",
       "1581  So when I was in Morocco, in Casablanca, not s...   \n",
       "1582  Some of my most wonderful memories of childhoo...   \n",
       "1590  I want you to imagine what a breakthrough this...   \n",
       "1615  So my moment of truth did not come all at once...   \n",
       "1728  Talking about empowerment is odd, because when...   \n",
       "1746  When I turned 19, I started my career as the f...   \n",
       "1750  Women represent 50 percent of middle managemen...   \n",
       "1798  When I arrived in Kiev, on February 1 this yea...   \n",
       "1828  You've heard of your I.Q., your general intell...   \n",
       "1837  As an Arab female photographer, I have always ...   \n",
       "1841  While preparing for my talk I was reflecting o...   \n",
       "1856  How many people here have heard of PMS? Everyb...   \n",
       "\n",
       "                                                   tags  cluster  \n",
       "23    [cognitive science, culture, evolution, gender...        7  \n",
       "24    [culture, entertainment, gender, global issues...        7  \n",
       "101   [culture, entertainment, live music, music, pe...        7  \n",
       "169   [South America, entertainment, global issues, ...        7  \n",
       "289   [activism, culture, global issues, spoken word...        7  \n",
       "449   [culture, education, global issues, politics, ...        7  \n",
       "541   [culture, entertainment, global issues, poetry...        7  \n",
       "562   [Asia, art, humanity, photography, storytellin...        7  \n",
       "643           [culture, feminism, social change, women]        7  \n",
       "782           [drones, global issues, iraq, war, women]        7  \n",
       "790   [business, feminism, politics, women, women in...        7  \n",
       "791   [Gender equality, bullying, culture, feminism,...        7  \n",
       "793   [Gender equality, culture, economics, educatio...        7  \n",
       "798   [Gender equality, business, feminism, goal-set...        7  \n",
       "805   [business, cancer, medicine, science, technology]        7  \n",
       "819   [art, comedy, culture, feminism, humor, media,...        7  \n",
       "824   [advertising, culture, entertainment, media, s...        7  \n",
       "829   [community, global issues, health, medicine, s...        7  \n",
       "834   [culture, economics, global issues, investment...        7  \n",
       "836   [Foreign Policy, Gender equality, business, iraq]        7  \n",
       "847   [culture, feminism, history, identity, women, ...        7  \n",
       "906      [art, film, global issues, photography, women]        7  \n",
       "942   [communication, global issues, journalism, war...        7  \n",
       "967   [Buddhism, compassion, culture, death, global ...        7  \n",
       "1028  [brain, children, culture, education, evolutio...        7  \n",
       "1057  [TEDx, activism, live music, music, performanc...        7  \n",
       "1068  [TEDx, entrepreneur, global issues, poverty, w...        7  \n",
       "1114  [TEDx, activism, biology, disease, health, hea...        7  \n",
       "1131  [TEDx, culture, global issues, health care, po...        7  \n",
       "1150  [TEDx, entertainment, fashion, feminism, ident...        7  \n",
       "1171    [Egypt, TEDx, data, economics, protests, women]        7  \n",
       "1188    [TEDx, global issues, journalism, media, women]        7  \n",
       "1340  [TEDx, culture, film, identity, men, movies, p...        7  \n",
       "1349  [compassion, democracy, global issues, politic...        7  \n",
       "1433  [Gender equality, TEDx, feminism, men, sexual ...        7  \n",
       "1445  [Islam, Middle East, cars, culture, feminism, ...        7  \n",
       "1533       [adventure, disaster relief, nuclear energy]        7  \n",
       "1578  [Gender equality, feminism, leadership, women,...        7  \n",
       "1581  [Egypt, Middle East, TED Fellows, culture, rel...        7  \n",
       "1582          [depression, health, heart health, women]        7  \n",
       "1590                          [gender, violence, women]        7  \n",
       "1615  [Gender equality, inequality, men, policy, wom...        7  \n",
       "1728  [TEDx, activism, gender, india, media, sexual ...        7  \n",
       "1746  [Middle East, TED Fellows, gender, global issu...        7  \n",
       "1750  [TEDx, business, gender, leadership, men, wome...        7  \n",
       "1798                    [TED Fellows, photography, war]        7  \n",
       "1828   [TEDYouth, TEDx, brain, personality, psychology]        7  \n",
       "1837  [Islam, Middle East, TED Fellows, education, f...        7  \n",
       "1841  [Middle East, activism, feminism, violence, wo...        7  \n",
       "1856              [TEDx, depression, psychology, women]        7  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "ted_merged['cluster'] = clusters\n",
    "ted_w_cluster = ted_merged[['title','transcript','tags','cluster']]\n",
    "ted_w_cluster[ted_w_cluster['cluster']==7][:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>transcript</th>\n",
       "      <th>tags</th>\n",
       "      <th>cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "      <td>Good morning. How are you?(Laughter)It's been ...</td>\n",
       "      <td>[children, creativity, culture, dance, educati...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Averting the climate crisis</td>\n",
       "      <td>Thank you so much, Chris. And it's truly a gre...</td>\n",
       "      <td>[alternative energy, cars, climate change, cul...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Simplicity sells</td>\n",
       "      <td>(Music: \"The Sound of Silence,\" Simon &amp; Garfun...</td>\n",
       "      <td>[computers, entertainment, interface design, m...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Greening the ghetto</td>\n",
       "      <td>If you're here today — and I'm very happy that...</td>\n",
       "      <td>[MacArthur grant, activism, business, cities, ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The best stats you've ever seen</td>\n",
       "      <td>About 10 years ago, I took on the task to teac...</td>\n",
       "      <td>[Africa, Asia, Google, demo, economics, global...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Why we do what we do</td>\n",
       "      <td>Thank you. I have to tell you I'm both challen...</td>\n",
       "      <td>[business, culture, entertainment, goal-settin...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Letting go of God</td>\n",
       "      <td>On September 10, the morning of my seventh bir...</td>\n",
       "      <td>[Christianity, God, atheism, comedy, culture, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Behind the design of Seattle's library</td>\n",
       "      <td>I'm going to present three projects in rapid f...</td>\n",
       "      <td>[architecture, collaboration, culture, design,...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Let's teach religion -- all religion -- in sch...</td>\n",
       "      <td>It's wonderful to be back. I love this wonderf...</td>\n",
       "      <td>[God, TED Brain Trust, atheism, brain, cogniti...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>A life of purpose</td>\n",
       "      <td>I'm often asked, \"What surprised you about the...</td>\n",
       "      <td>[Christianity, God, culture, happiness, leader...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>My wish: A call for open-source architecture</td>\n",
       "      <td>I'm going to take you on a journey very quickl...</td>\n",
       "      <td>[activism, architecture, collaboration, cultur...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>My wish: A global day of film</td>\n",
       "      <td>I can't help but this wish: to think about whe...</td>\n",
       "      <td>[TED Prize, art, culture, entertainment, film,...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>My wish: Help me stop pandemics</td>\n",
       "      <td>I'm the luckiest guy in the world. I got to se...</td>\n",
       "      <td>[TED Prize, collaboration, disease, ebola, glo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>The radical promise of the multi-touch interface</td>\n",
       "      <td>I'm really excited to be here today. I'll show...</td>\n",
       "      <td>[demo, design, interface design, technology]</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>One Laptop per Child</td>\n",
       "      <td>I've been at MIT for 44 years. I went to TED I...</td>\n",
       "      <td>[children, design, education, entrepreneur, gl...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>An 11-year-old's magical violin</td>\n",
       "      <td>(Music)(Music ends)(Applause)(Applause ends)Hi...</td>\n",
       "      <td>[entertainment, music, performance, violin, wu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Improvising on piano, aged 14</td>\n",
       "      <td>(Music)(Music ends)(Applause)Thank you!(Applau...</td>\n",
       "      <td>[creativity, entertainment, music, performance...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Simple designs to save a life</td>\n",
       "      <td>In terms of invention, I'd like to tell you th...</td>\n",
       "      <td>[MacArthur grant, alternative energy, design, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Organic design, inspired by nature</td>\n",
       "      <td>My name is Lovegrove. I only know nine Lovegro...</td>\n",
       "      <td>[DNA, biology, creativity, design, industrial ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>The birth of Wikipedia</td>\n",
       "      <td>Charles Van Doren, who was later a senior edit...</td>\n",
       "      <td>[business, collaboration, culture, invention, ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>The birth of the open-source learning revolution</td>\n",
       "      <td>I'm Rich Baraniuk and what I'd like to talk a ...</td>\n",
       "      <td>[business, collaboration, culture, global issu...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Nerdcore comedy</td>\n",
       "      <td>You know, when Chris first approached me to sp...</td>\n",
       "      <td>[collaboration, comedy, community, culture, da...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Meet the founder of the blog revolution</td>\n",
       "      <td>Over the past couple of days, as I've been pre...</td>\n",
       "      <td>[business, communication, community, culture, ...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Why we love, why we cheat</td>\n",
       "      <td>I'd like to talk today about the two biggest s...</td>\n",
       "      <td>[cognitive science, culture, evolution, gender...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Happiness in body and soul</td>\n",
       "      <td>I bet you're worried.(Laughter)I was worried. ...</td>\n",
       "      <td>[culture, entertainment, gender, global issues...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Chemical scum that dream of distant quasars</td>\n",
       "      <td>We've been told to go out on a limb and say so...</td>\n",
       "      <td>[climate change, cosmos, culture, environment,...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Why the universe seems so strange</td>\n",
       "      <td>My title: \"Queerer than we can suppose: the st...</td>\n",
       "      <td>[astronomy, biology, cognitive science, cosmos...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>The freakonomics of crack dealing</td>\n",
       "      <td>You'll be happy to know that I'll be talking n...</td>\n",
       "      <td>[business, cities, culture, economics, narcoti...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Choice, happiness and spaghetti sauce</td>\n",
       "      <td>I think I was supposed to talk about my new bo...</td>\n",
       "      <td>[business, choice, consumerism, culture, econo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>The surprising science of happiness</td>\n",
       "      <td>When you have 21 minutes to speak, two million...</td>\n",
       "      <td>[TED Brain Trust, brain, choice, culture, evol...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2437</th>\n",
       "      <td>What would happen if we upload our brains to c...</td>\n",
       "      <td>Someday, we may have robots as smart as people...</td>\n",
       "      <td>[AI, computers, economics, future, humanity, i...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2438</th>\n",
       "      <td>What the sugar coating on your cells is trying...</td>\n",
       "      <td>This is a talk about sugar and cancer. I becam...</td>\n",
       "      <td>[biology, biomechanics, cancer, chemistry, dis...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2439</th>\n",
       "      <td>\"The Sacred Art of the Ori\"</td>\n",
       "      <td>[Yoruba: Freeborn Ijebu-Ode son,] [of true Ogb...</td>\n",
       "      <td>[Africa, art, beauty, creativity, performance ...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2440</th>\n",
       "      <td>When workers own companies, the economy is mor...</td>\n",
       "      <td>Are you tired of your boss?(Laughter)Are you t...</td>\n",
       "      <td>[TEDx, business, capitalism, community, econom...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2441</th>\n",
       "      <td>Fun, fierce and fantastical African art</td>\n",
       "      <td>So, my mother's a pediatrician, and when I was...</td>\n",
       "      <td>[Africa, TED Fellows, art, creativity, storyte...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2442</th>\n",
       "      <td>Why we need to end the era of orphanages</td>\n",
       "      <td>These are some photos of me volunteering in a ...</td>\n",
       "      <td>[TEDx, activism, children, family, poverty, so...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2443</th>\n",
       "      <td>Can a robot pass a university entrance exam?</td>\n",
       "      <td>Today, I'm going to talk about AI and us. AI r...</td>\n",
       "      <td>[AI, education, intelligence, robots, teaching...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2444</th>\n",
       "      <td>How the US government spies on people who prot...</td>\n",
       "      <td>We are all activists now.(Applause)Thank you.I...</td>\n",
       "      <td>[TEDx, United States, activism, democracy, gov...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2445</th>\n",
       "      <td>How your pictures can help reclaim lost history</td>\n",
       "      <td>Why do people deliberately destroy cultural he...</td>\n",
       "      <td>[TEDx, ancient world, archaeology, conservatio...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2446</th>\n",
       "      <td>Dare to refuse the origin myths that claim who...</td>\n",
       "      <td>I'm Chetan Bhatt and when I give my name, I'm ...</td>\n",
       "      <td>[TEDx, community, humanity, identity, inequali...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2447</th>\n",
       "      <td>A smog vacuum cleaner and other magical city d...</td>\n",
       "      <td>Do you remember these glow-in-the-dark little ...</td>\n",
       "      <td>[architecture, art, cities, creativity, design...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2448</th>\n",
       "      <td>How young people join violent extremist groups...</td>\n",
       "      <td>So in 2011, I altered my name so that I could ...</td>\n",
       "      <td>[TEDx, security, social media, terrorism, viol...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2449</th>\n",
       "      <td>How to build a company where the best ideas win</td>\n",
       "      <td>Whether you like it or not, radical transparen...</td>\n",
       "      <td>[algorithm, business, collaboration, communica...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2450</th>\n",
       "      <td>Would you live in a floating city in the sky?</td>\n",
       "      <td>This is my favorite place on earth, Salar de U...</td>\n",
       "      <td>[TED en Español, art, creativity, design, envi...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2451</th>\n",
       "      <td>What it feels like to see Earth from space</td>\n",
       "      <td>It's Christmas Eve, 1968. The Apollo 8 spacecr...</td>\n",
       "      <td>[TEDx, art, climate change, environment, photo...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2452</th>\n",
       "      <td>Who belongs in a city?</td>\n",
       "      <td>Cities are like siblings in a large polygamous...</td>\n",
       "      <td>[Africa, activism, cities, government, poverty...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2453</th>\n",
       "      <td>How our friendship survives our opposing politics</td>\n",
       "      <td>Caitlin Quattromani: The election of 2016 felt...</td>\n",
       "      <td>[TEDx, communication, friendship, politics, so...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2454</th>\n",
       "      <td>There's more to life than being happy</td>\n",
       "      <td>I used to think the whole purpose of life was ...</td>\n",
       "      <td>[happiness, humanity, personal growth, psychol...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2455</th>\n",
       "      <td>What really motivates people to be honest in b...</td>\n",
       "      <td>How many companies have you interacted with to...</td>\n",
       "      <td>[TEDx, business, corruption, economics, financ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2456</th>\n",
       "      <td>A forgotten ancient grain that could help Afri...</td>\n",
       "      <td>I was born and raised in Dakar, Senegal, and t...</td>\n",
       "      <td>[Africa, agriculture, farming, food, innovation]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2457</th>\n",
       "      <td>The real reason manufacturing jobs are disappe...</td>\n",
       "      <td>When someone mentions Cuba, what do you think ...</td>\n",
       "      <td>[business, capitalism, collaboration, economic...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2458</th>\n",
       "      <td>The fascinating physics of everyday life</td>\n",
       "      <td>As you heard, I'm a physicist. And I think the...</td>\n",
       "      <td>[TEDx, physics, play, science]</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2459</th>\n",
       "      <td>Living sculptures that stand for history's truths</td>\n",
       "      <td>I'd like for you to take a moment to imagine t...</td>\n",
       "      <td>[Africa, activism, art, community, history, id...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2460</th>\n",
       "      <td>How digital DNA could help you make better hea...</td>\n",
       "      <td>Today I'm here, actually, to pose you a questi...</td>\n",
       "      <td>[DNA, Human body, biology, data, entrepreneur,...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2461</th>\n",
       "      <td>Why Africa must become a center of knowledge a...</td>\n",
       "      <td>What stands between Africa's current prostrate...</td>\n",
       "      <td>[Africa, agriculture, history, leadership, lib...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2462</th>\n",
       "      <td>What we're missing in the debate about immigra...</td>\n",
       "      <td>So, Ma was trying to explain something to me a...</td>\n",
       "      <td>[TED Residency, United States, community, fami...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2463</th>\n",
       "      <td>The most Martian place on Earth</td>\n",
       "      <td>This is a picture of a sunset on Mars taken by...</td>\n",
       "      <td>[Mars, South America, TED Fellows, astrobiolog...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>What intelligent machines can learn from a sch...</td>\n",
       "      <td>In my early days as a graduate student, I went...</td>\n",
       "      <td>[AI, ants, fish, future, innovation, insects, ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>A black man goes undercover in the alt-right</td>\n",
       "      <td>I took a cell phone and accidentally made myse...</td>\n",
       "      <td>[Internet, TEDx, United States, community, com...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>How a video game might help us build better ci...</td>\n",
       "      <td>We humans are becoming an urban species, so ci...</td>\n",
       "      <td>[cities, design, future, infrastructure, play,...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2467 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0                           Do schools kill creativity?   \n",
       "1                           Averting the climate crisis   \n",
       "2                                      Simplicity sells   \n",
       "3                                   Greening the ghetto   \n",
       "4                       The best stats you've ever seen   \n",
       "5                                  Why we do what we do   \n",
       "6                                     Letting go of God   \n",
       "7                Behind the design of Seattle's library   \n",
       "8     Let's teach religion -- all religion -- in sch...   \n",
       "9                                     A life of purpose   \n",
       "10         My wish: A call for open-source architecture   \n",
       "11                        My wish: A global day of film   \n",
       "12                      My wish: Help me stop pandemics   \n",
       "13     The radical promise of the multi-touch interface   \n",
       "14                                 One Laptop per Child   \n",
       "15                      An 11-year-old's magical violin   \n",
       "16                        Improvising on piano, aged 14   \n",
       "17                        Simple designs to save a life   \n",
       "18                   Organic design, inspired by nature   \n",
       "19                               The birth of Wikipedia   \n",
       "20     The birth of the open-source learning revolution   \n",
       "21                                      Nerdcore comedy   \n",
       "22              Meet the founder of the blog revolution   \n",
       "23                            Why we love, why we cheat   \n",
       "24                           Happiness in body and soul   \n",
       "25          Chemical scum that dream of distant quasars   \n",
       "26                    Why the universe seems so strange   \n",
       "27                    The freakonomics of crack dealing   \n",
       "28                Choice, happiness and spaghetti sauce   \n",
       "29                  The surprising science of happiness   \n",
       "...                                                 ...   \n",
       "2437  What would happen if we upload our brains to c...   \n",
       "2438  What the sugar coating on your cells is trying...   \n",
       "2439                        \"The Sacred Art of the Ori\"   \n",
       "2440  When workers own companies, the economy is mor...   \n",
       "2441            Fun, fierce and fantastical African art   \n",
       "2442           Why we need to end the era of orphanages   \n",
       "2443       Can a robot pass a university entrance exam?   \n",
       "2444  How the US government spies on people who prot...   \n",
       "2445    How your pictures can help reclaim lost history   \n",
       "2446  Dare to refuse the origin myths that claim who...   \n",
       "2447  A smog vacuum cleaner and other magical city d...   \n",
       "2448  How young people join violent extremist groups...   \n",
       "2449    How to build a company where the best ideas win   \n",
       "2450      Would you live in a floating city in the sky?   \n",
       "2451         What it feels like to see Earth from space   \n",
       "2452                             Who belongs in a city?   \n",
       "2453  How our friendship survives our opposing politics   \n",
       "2454              There's more to life than being happy   \n",
       "2455  What really motivates people to be honest in b...   \n",
       "2456  A forgotten ancient grain that could help Afri...   \n",
       "2457  The real reason manufacturing jobs are disappe...   \n",
       "2458           The fascinating physics of everyday life   \n",
       "2459  Living sculptures that stand for history's truths   \n",
       "2460  How digital DNA could help you make better hea...   \n",
       "2461  Why Africa must become a center of knowledge a...   \n",
       "2462  What we're missing in the debate about immigra...   \n",
       "2463                    The most Martian place on Earth   \n",
       "2464  What intelligent machines can learn from a sch...   \n",
       "2465       A black man goes undercover in the alt-right   \n",
       "2466  How a video game might help us build better ci...   \n",
       "\n",
       "                                             transcript  \\\n",
       "0     Good morning. How are you?(Laughter)It's been ...   \n",
       "1     Thank you so much, Chris. And it's truly a gre...   \n",
       "2     (Music: \"The Sound of Silence,\" Simon & Garfun...   \n",
       "3     If you're here today — and I'm very happy that...   \n",
       "4     About 10 years ago, I took on the task to teac...   \n",
       "5     Thank you. I have to tell you I'm both challen...   \n",
       "6     On September 10, the morning of my seventh bir...   \n",
       "7     I'm going to present three projects in rapid f...   \n",
       "8     It's wonderful to be back. I love this wonderf...   \n",
       "9     I'm often asked, \"What surprised you about the...   \n",
       "10    I'm going to take you on a journey very quickl...   \n",
       "11    I can't help but this wish: to think about whe...   \n",
       "12    I'm the luckiest guy in the world. I got to se...   \n",
       "13    I'm really excited to be here today. I'll show...   \n",
       "14    I've been at MIT for 44 years. I went to TED I...   \n",
       "15    (Music)(Music ends)(Applause)(Applause ends)Hi...   \n",
       "16    (Music)(Music ends)(Applause)Thank you!(Applau...   \n",
       "17    In terms of invention, I'd like to tell you th...   \n",
       "18    My name is Lovegrove. I only know nine Lovegro...   \n",
       "19    Charles Van Doren, who was later a senior edit...   \n",
       "20    I'm Rich Baraniuk and what I'd like to talk a ...   \n",
       "21    You know, when Chris first approached me to sp...   \n",
       "22    Over the past couple of days, as I've been pre...   \n",
       "23    I'd like to talk today about the two biggest s...   \n",
       "24    I bet you're worried.(Laughter)I was worried. ...   \n",
       "25    We've been told to go out on a limb and say so...   \n",
       "26    My title: \"Queerer than we can suppose: the st...   \n",
       "27    You'll be happy to know that I'll be talking n...   \n",
       "28    I think I was supposed to talk about my new bo...   \n",
       "29    When you have 21 minutes to speak, two million...   \n",
       "...                                                 ...   \n",
       "2437  Someday, we may have robots as smart as people...   \n",
       "2438  This is a talk about sugar and cancer. I becam...   \n",
       "2439  [Yoruba: Freeborn Ijebu-Ode son,] [of true Ogb...   \n",
       "2440  Are you tired of your boss?(Laughter)Are you t...   \n",
       "2441  So, my mother's a pediatrician, and when I was...   \n",
       "2442  These are some photos of me volunteering in a ...   \n",
       "2443  Today, I'm going to talk about AI and us. AI r...   \n",
       "2444  We are all activists now.(Applause)Thank you.I...   \n",
       "2445  Why do people deliberately destroy cultural he...   \n",
       "2446  I'm Chetan Bhatt and when I give my name, I'm ...   \n",
       "2447  Do you remember these glow-in-the-dark little ...   \n",
       "2448  So in 2011, I altered my name so that I could ...   \n",
       "2449  Whether you like it or not, radical transparen...   \n",
       "2450  This is my favorite place on earth, Salar de U...   \n",
       "2451  It's Christmas Eve, 1968. The Apollo 8 spacecr...   \n",
       "2452  Cities are like siblings in a large polygamous...   \n",
       "2453  Caitlin Quattromani: The election of 2016 felt...   \n",
       "2454  I used to think the whole purpose of life was ...   \n",
       "2455  How many companies have you interacted with to...   \n",
       "2456  I was born and raised in Dakar, Senegal, and t...   \n",
       "2457  When someone mentions Cuba, what do you think ...   \n",
       "2458  As you heard, I'm a physicist. And I think the...   \n",
       "2459  I'd like for you to take a moment to imagine t...   \n",
       "2460  Today I'm here, actually, to pose you a questi...   \n",
       "2461  What stands between Africa's current prostrate...   \n",
       "2462  So, Ma was trying to explain something to me a...   \n",
       "2463  This is a picture of a sunset on Mars taken by...   \n",
       "2464  In my early days as a graduate student, I went...   \n",
       "2465  I took a cell phone and accidentally made myse...   \n",
       "2466  We humans are becoming an urban species, so ci...   \n",
       "\n",
       "                                                   tags  cluster  \n",
       "0     [children, creativity, culture, dance, educati...        0  \n",
       "1     [alternative energy, cars, climate change, cul...        0  \n",
       "2     [computers, entertainment, interface design, m...        0  \n",
       "3     [MacArthur grant, activism, business, cities, ...        2  \n",
       "4     [Africa, Asia, Google, demo, economics, global...        1  \n",
       "5     [business, culture, entertainment, goal-settin...        6  \n",
       "6     [Christianity, God, atheism, comedy, culture, ...        0  \n",
       "7     [architecture, collaboration, culture, design,...        2  \n",
       "8     [God, TED Brain Trust, atheism, brain, cogniti...        5  \n",
       "9     [Christianity, God, culture, happiness, leader...        6  \n",
       "10    [activism, architecture, collaboration, cultur...        2  \n",
       "11    [TED Prize, art, culture, entertainment, film,...        6  \n",
       "12    [TED Prize, collaboration, disease, ebola, glo...        1  \n",
       "13         [demo, design, interface design, technology]        5  \n",
       "14    [children, design, education, entrepreneur, gl...        5  \n",
       "15    [entertainment, music, performance, violin, wu...        0  \n",
       "16    [creativity, entertainment, music, performance...        0  \n",
       "17    [MacArthur grant, alternative energy, design, ...        1  \n",
       "18    [DNA, biology, creativity, design, industrial ...        5  \n",
       "19    [business, collaboration, culture, invention, ...        5  \n",
       "20    [business, collaboration, culture, global issu...        5  \n",
       "21    [collaboration, comedy, community, culture, da...        0  \n",
       "22    [business, communication, community, culture, ...        6  \n",
       "23    [cognitive science, culture, evolution, gender...        7  \n",
       "24    [culture, entertainment, gender, global issues...        7  \n",
       "25    [climate change, cosmos, culture, environment,...        5  \n",
       "26    [astronomy, biology, cognitive science, cosmos...        4  \n",
       "27    [business, cities, culture, economics, narcoti...        0  \n",
       "28    [business, choice, consumerism, culture, econo...        0  \n",
       "29    [TED Brain Trust, brain, choice, culture, evol...        5  \n",
       "...                                                 ...      ...  \n",
       "2437  [AI, computers, economics, future, humanity, i...        5  \n",
       "2438  [biology, biomechanics, cancer, chemistry, dis...        5  \n",
       "2439  [Africa, art, beauty, creativity, performance ...        6  \n",
       "2440  [TEDx, business, capitalism, community, econom...        1  \n",
       "2441  [Africa, TED Fellows, art, creativity, storyte...        6  \n",
       "2442  [TEDx, activism, children, family, poverty, so...        6  \n",
       "2443  [AI, education, intelligence, robots, teaching...        6  \n",
       "2444  [TEDx, United States, activism, democracy, gov...        3  \n",
       "2445  [TEDx, ancient world, archaeology, conservatio...        5  \n",
       "2446  [TEDx, community, humanity, identity, inequali...        6  \n",
       "2447  [architecture, art, cities, creativity, design...        0  \n",
       "2448  [TEDx, security, social media, terrorism, viol...        5  \n",
       "2449  [algorithm, business, collaboration, communica...        5  \n",
       "2450  [TED en Español, art, creativity, design, envi...        4  \n",
       "2451  [TEDx, art, climate change, environment, photo...        4  \n",
       "2452  [Africa, activism, cities, government, poverty...        2  \n",
       "2453  [TEDx, communication, friendship, politics, so...        0  \n",
       "2454  [happiness, humanity, personal growth, psychol...        6  \n",
       "2455  [TEDx, business, corruption, economics, financ...        5  \n",
       "2456   [Africa, agriculture, farming, food, innovation]        1  \n",
       "2457  [business, capitalism, collaboration, economic...        1  \n",
       "2458                     [TEDx, physics, play, science]        5  \n",
       "2459  [Africa, activism, art, community, history, id...        7  \n",
       "2460  [DNA, Human body, biology, data, entrepreneur,...        5  \n",
       "2461  [Africa, agriculture, history, leadership, lib...        4  \n",
       "2462  [TED Residency, United States, community, fami...        1  \n",
       "2463  [Mars, South America, TED Fellows, astrobiolog...        4  \n",
       "2464  [AI, ants, fish, future, innovation, insects, ...        5  \n",
       "2465  [Internet, TEDx, United States, community, com...        0  \n",
       "2466  [cities, design, future, infrastructure, play,...        2  \n",
       "\n",
       "[2467 rows x 4 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ted_w_cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "c0_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 0]['tags'].tolist() for item in sub_list]\n",
    "c1_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 1]['tags'].tolist() for item in sub_list]\n",
    "c2_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 2]['tags'].tolist() for item in sub_list]\n",
    "c3_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 3]['tags'].tolist() for item in sub_list]\n",
    "c4_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 4]['tags'].tolist() for item in sub_list]\n",
    "c5_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 5]['tags'].tolist() for item in sub_list]\n",
    "c6_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 6]['tags'].tolist() for item in sub_list]\n",
    "c7_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 7]['tags'].tolist() for item in sub_list]\n",
    "# c8_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 8]['tags'].tolist() for item in sub_list]\n",
    "# c9_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 9]['tags'].tolist() for item in sub_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "c0_tag_stat = pd.Series(Counter(c0_tag))\n",
    "c1_tag_stat = pd.Series(Counter(c1_tag))\n",
    "c2_tag_stat = pd.Series(Counter(c2_tag))\n",
    "c3_tag_stat = pd.Series(Counter(c3_tag))\n",
    "c4_tag_stat = pd.Series(Counter(c4_tag))\n",
    "c5_tag_stat = pd.Series(Counter(c5_tag))\n",
    "c6_tag_stat = pd.Series(Counter(c6_tag))\n",
    "c7_tag_stat = pd.Series(Counter(c7_tag))\n",
    "# c8_tag_stat = pd.Series(Counter(c8_tag))\n",
    "# c9_tag_stat = pd.Series(Counter(c9_tag))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "entertainment    87\n",
      "culture          75\n",
      "humor            64\n",
      "technology       51\n",
      "TEDx             49\n",
      "science          39\n",
      "performance      39\n",
      "music            36\n",
      "comedy           36\n",
      "design           36\n",
      "dtype: int64\n",
      "\n",
      "global issues    198\n",
      "business         115\n",
      "economics         91\n",
      "technology        75\n",
      "politics          59\n",
      "TEDx              54\n",
      "culture           52\n",
      "social change     52\n",
      "health            49\n",
      "society           49\n",
      "dtype: int64\n",
      "\n",
      "design           106\n",
      "cities            65\n",
      "architecture      55\n",
      "technology        42\n",
      "culture           32\n",
      "art               32\n",
      "collaboration     24\n",
      "creativity        23\n",
      "business          22\n",
      "innovation        22\n",
      "dtype: int64\n",
      "\n",
      "technology       53\n",
      "data             38\n",
      "science          25\n",
      "health           17\n",
      "TEDx             16\n",
      "communication    13\n",
      "business         12\n",
      "computers        12\n",
      "global issues    12\n",
      "medicine         12\n",
      "dtype: int64\n",
      "\n",
      "science          86\n",
      "technology       57\n",
      "environment      50\n",
      "exploration      36\n",
      "nature           30\n",
      "TEDx             30\n",
      "design           28\n",
      "water            27\n",
      "global issues    26\n",
      "biology          25\n",
      "dtype: int64\n",
      "\n",
      "technology       355\n",
      "science          277\n",
      "design           163\n",
      "culture          133\n",
      "TEDx             133\n",
      "biology          112\n",
      "innovation        99\n",
      "business          93\n",
      "brain             88\n",
      "global issues     87\n",
      "dtype: int64\n",
      "\n",
      "culture          149\n",
      "global issues    108\n",
      "TEDx             100\n",
      "social change     88\n",
      "entertainment     85\n",
      "children          77\n",
      "technology        75\n",
      "society           70\n",
      "humanity          69\n",
      "storytelling      66\n",
      "dtype: int64\n",
      "\n",
      "women              64\n",
      "global issues      24\n",
      "feminism           21\n",
      "culture            20\n",
      "Gender equality    20\n",
      "activism           18\n",
      "TEDx               18\n",
      "inequality         17\n",
      "social change      16\n",
      "society            15\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(c0_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print (c1_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print (c2_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print (c3_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print (c4_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print (c5_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print(c6_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "print(c7_tag_stat.nlargest(10))\n",
    "print (\"\")\n",
    "# print(c8_tag_stat.nlargest(10))\n",
    "# print (\"\")\n",
    "# print(c9_tag_stat.nlargest(10))\n",
    "# print (\"\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}