20news_HLTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "22ae8605-b97e-432a-8952-cd7d1d26f18c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m Activating\u001b[22m\u001b[39m environment at `~/Documents/Study/Git_Personal/julia_bayes/TopicModels/Project.toml`\n"
     ]
    }
   ],
   "source": [
    "using Pkg\n",
    "using Distances\n",
    "using StatsBase\n",
    "using DataStructures\n",
    "using ProgressMeter\n",
    "using JSON\n",
    "Pkg.activate(\"TopicModels\")\n",
    "import TopicModels\n",
    "using PyCall\n",
    "using PyPlot\n",
    "\n",
    "using LSHFunctions, LinearAlgebra, BenchmarkTools\n",
    "#corpora = pyimport(\"gensim.corpora\")\n",
    "#ch = pyimport(\"gensim.models.coherencemodel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d6806b6a-9203-4630-8b03-2e4d1af93122",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject <module 'sklearn.feature_selection' from '/Users/khan/.julia/conda/3/lib/python3.8/site-packages/sklearn/feature_selection/__init__.py'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#nn = pyimport(\"torch.nn\")\n",
    "np = pyimport(\"numpy\")\n",
    "torch = pyimport(\"torch\")\n",
    "#plt = pyimport(\"matplotlib.pyplot\")\n",
    "f1_score = pyimport(\"sklearn.metrics\")\n",
    "feature_selection = pyimport(\"sklearn.feature_selection\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b108a02-0b59-4885-b0e0-358225247287",
   "metadata": {},
   "outputs": [],
   "source": [
    "#using 10,000 docs is too much becasue kp_similarity file exceeds 65GB, 5,000 is high so now using 2,000\n",
    "corpus, no_lemma_docs = TopicModels.preprocess(\"20news/20newsgroup_long_only.csv\", 2000); #Took around 1:25 mimnutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5b30b41b-60c3-4542-954d-2e50f885db34",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#load lda object from saved json\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\"); #only takes few seconds \n",
    "\n",
    "#or train new lda model from below command, uncomment to run.\n",
    "#lda = TopicModels.train(corpus, 20); #training took around 15 mints\n",
    "#TopicModels.saveLDA(lda, \"20news/lda_obj_2000_long.json\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "37880011-532f-40bb-b2ca-48eea8f8a250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Any[\"god\", \"jesus\", \"bible\", \"church\", \"christ\", \"christian\", \"christians\", \"faith\", \"lord\", \"believe\"]\n",
      "----------------------\n",
      "Any[\"said\", \"people\", \"one\", \"know\", \"would\", \"going\", \"came\", \"children\", \"went\", \"like\"]\n",
      "----------------------\n",
      "Any[\"evidence\", \"one\", \"book\", \"world\", \"two\", \"many\", \"history\", \"life\", \"years\", \"etc\"]\n",
      "----------------------\n",
      "Any[\"said\", \"one\", \"police\", \"city\", \"times\", \"years\", \"days\", \"kill\", \"soldiers\", \"two\"]\n",
      "----------------------\n",
      "Any[\"armenian\", \"turkish\", \"armenians\", \"jews\", \"war\", \"armenia\", \"russian\", \"azerbaijan\", \"genocide\", \"turkey\"]\n",
      "----------------------\n",
      "Any[\"game\", \"year\", \"good\", \"car\", \"get\", \"last\", \"got\", \"games\", \"first\", \"players\"]\n",
      "----------------------\n",
      "Any[\"image\", \"jpeg\", \"file\", \"gif\", \"quality\", \"color\", \"use\", \"format\", \"version\", \"may\"]\n",
      "----------------------\n",
      "Any[\"would\", \"one\", \"like\", \"people\", \"think\", \"know\", \"even\", \"make\", \"way\", \"good\"]\n",
      "----------------------\n",
      "Any[\"health\", \"medical\", \"cancer\", \"disease\", \"patients\", \"april\", \"drug\", \"det\", \"pit\", \"hiv\"]\n",
      "----------------------\n",
      "Any[\"file\", \"entry\", \"program\", \"use\", \"output\", \"int\", \"bits\", \"line\", \"entries\", \"section\"]\n",
      "----------------------\n",
      "Any[\"drive\", \"disk\", \"card\", \"hard\", \"new\", \"drives\", \"scsi\", \"system\", \"art\", \"appears\"]\n",
      "----------------------\n",
      "Any[\"key\", \"encryption\", \"use\", \"security\", \"privacy\", \"government\", \"chip\", \"keys\", \"information\", \"system\"]\n",
      "----------------------\n",
      "Any[\"list\", \"may\", \"university\", \"service\", \"research\", \"send\", \"news\", \"new\", \"email\", \"information\"]\n",
      "----------------------\n",
      "Any[\"president\", \"gun\", \"think\", \"going\", \"stephanopoulos\", \"myers\", \"know\", \"said\", \"house\", \"guns\"]\n",
      "----------------------\n",
      "Any[\"software\", \"available\", \"ftp\", \"windows\", \"system\", \"files\", \"version\", \"use\", \"server\", \"file\"]\n",
      "----------------------\n",
      "Any[\"use\", \"used\", \"one\", \"may\", \"problems\", \"two\", \"cause\", \"problem\", \"likely\", \"done\"]\n",
      "----------------------\n",
      "Any[\"people\", \"law\", \"government\", \"state\", \"right\", \"would\", \"rights\", \"israel\", \"states\", \"israeli\"]\n",
      "----------------------\n",
      "Any[\"space\", \"launch\", \"nasa\", \"earth\", \"system\", \"satellite\", \"first\", \"mission\", \"surface\", \"solar\"]\n",
      "----------------------\n",
      "Any[\"period\", \"pts\", \"team\", \"new\", \"play\", \"hockey\", \"league\", \"san\", \"chicago\", \"lost\"]\n",
      "----------------------\n",
      "Any[\"data\", \"available\", \"information\", \"subject\", \"one\", \"set\", \"used\", \"called\", \"source\", \"various\"]\n",
      "----------------------\n"
     ]
    }
   ],
   "source": [
    "topics, proportions = TopicModels.show_topics(lda, corpus, 10);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "36769cf7-d61a-4b93-9a23-d1ef47817062",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_distributions = TopicModels.sortedTopDocsForTopics(lda, corpus);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d7ed4ba-d8dd-4f2d-bb93-16e94fa03546",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = TopicModels.topic_of_each_doc(lda, corpus); # function implementation in #add_kp section\n",
    "#println(top_topic_for_each_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0a102a04-4f3c-4bd3-b576-8fe657c46625",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(225,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_number = 1\n",
    "topic_x_top_docs = [idx for (idx, val) in enumerate(top_topic_for_each_doc) if val==topic_number];\n",
    "size(topic_x_top_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "15170d87-c2e5-42e6-9f4f-5c62561a9975",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8, 18, 19, 23, 27, 43, 44, 67, 69, 84, 86, 87, 88, 101, 113, 116, 130, 145, 190, 222, 223, 226, 232, 235, 249, 256, 267, 292, 293, 294, 321, 332, 338, 339, 341, 346, 371, 393, 413, 417, 425, 433, 434, 435, 469, 478, 494, 495, 499, 519, 521, 529, 535, 548, 559, 560, 570, 606, 622, 625, 631, 639, 648, 661, 662, 671, 679, 680, 715, 717, 727, 731, 740, 753, 759, 772, 774, 776, 779, 799, 804, 806, 828, 833, 848, 867, 882, 900, 914, 921, 960, 966, 970, 972, 980, 982, 1013, 1035, 1036, 1059, 1062, 1063, 1067, 1074, 1080, 1106, 1120, 1130, 1136, 1137, 1143, 1147, 1155, 1157, 1161, 1169, 1187, 1188, 1194, 1196, 1208, 1216, 1222, 1233, 1240, 1260, 1264, 1274, 1276, 1278, 1282, 1285, 1297, 1298, 1310, 1326, 1329, 1333, 1350, 1352, 1359, 1377, 1379, 1380, 1393, 1394, 1398, 1400, 1401, 1412, 1414, 1427, 1430, 1432, 1434, 1435, 1442, 1448, 1472, 1475, 1480, 1487, 1517, 1522, 1527, 1538, 1550, 1552, 1561, 1565, 1567, 1572, 1584, 1625, 1638, 1642, 1662, 1674, 1677, 1678, 1692, 1693, 1705, 1709, 1714, 1716, 1739, 1749, 1753, 1763, 1767, 1774, 1779, 1787, 1788, 1792, 1824, 1833, 1836, 1839, 1840, 1844, 1845, 1854, 1863, 1877, 1879, 1890, 1891, 1896, 1902, 1931, 1936, 1954, 1955, 1960, 1961, 1963, 1977, 1983, 1987, 1990, 1993, 1996, 1997]\n"
     ]
    }
   ],
   "source": [
    "println(topic_x_top_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af7adcb9-401d-419d-83ec-3d23cacd991d",
   "metadata": {},
   "outputs": [],
   "source": [
    "fileKPDoc = \"20news/kp/20news_doc_keyphrases_long_only_clean_last_2000.json\"\n",
    "fileEMB = \"20news/kp/20news_keyphrase_long_only_2000_embedding.json\"\n",
    "fileSim = \"20news/kp/20news_keyphrase_long_only_2000_similarity.json\"\n",
    "fileKP = \"20news/kp/20news_keyphrases_long_only_2000.json\"\n",
    "kp = TopicModels.load_keyphrase(fileKPDoc, fileEMB, fileSim, fileKP);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ac9daaac-da91-4a25-a89b-d33c65b908e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, topic_distributions, 10, 1, 0.75);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d8f9c628-5f32-4a15-b06a-9d202fbc6e9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict{Any,Any}(\"posit\" => Any[\"posit\", \"decrypt\"])\n",
      "Any[249, 717, 753, 970, 1062, 1074, 1169, 1240]"
     ]
    }
   ],
   "source": [
    "x = 3\n",
    "println(sort(cluster_kp[x]))\n",
    "print(sort(docs_have[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "c92c8647-7bdb-4a4d-98fa-ba87618a82ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "TopicModels.removeDoc(lda, corpus, docs_have[3], 1) #remove posit kp from topic 1\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "b9467aa8-9bf7-43fd-98a2-69359594d1df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207,)"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_number = 1\n",
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)\n",
    "topic_x_top_docs = [idx for (idx, val) in enumerate(top_topic_for_each_doc) if val==topic_number];\n",
    "size(topic_x_top_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "1e85378f-ade3-435d-b8a6-441e3d6f79ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8, 18, 19, 23, 27, 43, 44, 67, 84, 86, 87, 88, 101, 108, 113, 116, 130, 145, 190, 222, 223, 226, 232, 235, 256, 267, 292, 293, 294, 321, 332, 338, 339, 341, 346, 368, 371, 393, 413, 417, 433, 434, 435, 469, 478, 494, 495, 519, 521, 529, 535, 548, 559, 560, 606, 622, 625, 631, 639, 648, 662, 671, 679, 680, 708, 715, 727, 731, 740, 759, 772, 774, 776, 779, 799, 806, 828, 833, 848, 866, 867, 882, 890, 900, 914, 921, 960, 966, 972, 980, 982, 1013, 1036, 1059, 1067, 1080, 1106, 1120, 1130, 1136, 1137, 1147, 1155, 1157, 1161, 1187, 1188, 1194, 1196, 1208, 1216, 1222, 1233, 1260, 1274, 1276, 1282, 1285, 1293, 1297, 1298, 1310, 1326, 1329, 1333, 1350, 1352, 1359, 1377, 1379, 1380, 1393, 1394, 1398, 1400, 1401, 1412, 1414, 1427, 1430, 1434, 1435, 1442, 1446, 1448, 1472, 1475, 1480, 1487, 1517, 1522, 1527, 1538, 1550, 1561, 1565, 1567, 1584, 1625, 1638, 1642, 1662, 1674, 1677, 1678, 1692, 1693, 1705, 1709, 1714, 1716, 1739, 1749, 1753, 1763, 1767, 1774, 1779, 1788, 1792, 1824, 1836, 1839, 1840, 1844, 1854, 1863, 1877, 1879, 1890, 1891, 1896, 1902, 1931, 1936, 1954, 1955, 1960, 1961, 1963, 1977, 1983, 1987, 1990, 1993, 1996, 1997]"
     ]
    }
   ],
   "source": [
    "print(topic_x_top_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dcfec8e9-dab2-43ca-8889-619a2e041c51",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#This kind of implementation is not a good idea(conclusion after checking the association numbers)\n",
    "cluster_kp_all_docs, docs_have_from_all_docs = TopicModels.top_x_kp_of_topic_m_for_all_docs(kp, topic_distributions, 10, 1, 0.8);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34fdbf29-5dea-453b-a800-34f8208fc45b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Remove_word refinement after stable marriage algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e0c3d839-631c-47fe-abc2-f4fa73247863",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cat_original = [17, 7, 18, 17, 10, 19, 16, 19, 18, 8, 13, 7, 12, 9, 18, 18, 17, 17, 15, 17, 7, 1, 19, 5, 4, 8, 19, 17, 15, 15, 17, 0, 19, 11, 17, 18, 9, 13, 19, 17, 13, 17, 19, 15, 7, 8, 11, 9, 1, 8, 0, 18, 17, 14, 14, 10, 15, 3, 8, 10, 10, 11, 17, 6, 17, 17, 15, 8, 15, 11, 2, 17, 9, 18, 17, 17, 10, 2, 9, 1, 10, 4, 9, 15, 3, 15, 19, 15, 16, 13, 17, 11, 0, 18, 10, 14, 6, 17, 15, 17, 19, 14, 8, 14, 5, 5, 11, 15, 6, 10, 5, 9, 15, 11, 9, 15, 7, 1, 14, 16, 15, 13, 13, 17, 0, 11, 16, 3, 16, 15, 14, 14, 19, 9, 18, 14, 17, 17, 4, 17, 15, 1, 16, 11, 15, 6, 2, 14, 15, 18, 8, 17, 18, 4, 7, 11, 17, 18, 9, 16, 18, 7, 11, 2, 15, 19, 1, 11, 14, 17, 1, 16, 6, 5, 19, 7, 11, 16, 8, 16, 10, 15, 17, 16, 16, 1, 0, 7, 15, 0, 6, 18, 10, 9, 11, 16, 19, 14, 6, 0, 0, 3, 17, 17, 12, 16, 8, 6, 19, 10, 17, 16, 14, 4, 16, 11, 1, 14, 0, 2, 16, 15, 15, 6, 15, 0, 14, 10, 15, 13, 2, 19, 15, 17, 15, 1, 9, 16, 18, 17, 6, 17, 8, 14, 11, 5, 15, 7, 15, 15, 12, 11, 14, 14, 3, 15, 3, 17, 3, 14, 11, 19, 10, 18, 15, 16, 15, 15, 9, 16, 11, 4, 19, 13, 3, 19, 9, 17, 18, 7, 7, 15, 16, 14, 15, 10, 5, 9, 14, 14, 17, 15, 15, 19, 14, 15, 15, 0, 4, 11, 11, 15, 12, 6, 3, 15, 14, 5, 14, 18, 11, 11, 14, 17, 17, 10, 9, 5, 7, 10, 19, 18, 17, 15, 18, 17, 11, 11, 13, 3, 2, 15, 10, 17, 2, 14, 13, 15, 15, 16, 15, 13, 14, 7, 16, 19, 3, 17, 17, 17, 14, 16, 9, 8, 14, 14, 16, 13, 12, 11, 17, 11, 1, 4, 11, 18, 16, 15, 11, 17, 15, 6, 17, 17, 13, 9, 10, 18, 17, 17, 13, 6, 1, 9, 14, 19, 3, 19, 13, 14, 10, 16, 19, 14, 9, 17, 7, 7, 11, 17, 18, 11, 17, 11, 18, 17, 16, 14, 1, 11, 16, 11, 18, 6, 1, 14, 15, 6, 17, 18, 16, 6, 13, 11, 0, 0, 16, 8, 17, 17, 10, 4, 15, 15, 15, 1, 13, 15, 14, 5, 15, 9, 1, 0, 13, 11, 1, 14, 16, 7, 2, 6, 10, 8, 14, 12, 19, 2, 7, 15, 17, 13, 17, 13, 17, 17, 19, 16, 15, 0, 12, 19, 19, 18, 15, 16, 9, 15, 16, 16, 12, 9, 9, 18, 4, 18, 11, 17, 15, 12, 12, 17, 17, 15, 19, 15, 19, 17, 15, 10, 14, 19, 16, 17, 10, 10, 14, 16, 5, 13, 9, 15, 17, 15, 19, 0, 18, 14, 15, 1, 15, 5, 17, 12, 18, 10, 5, 16, 0, 15, 1, 5, 7, 11, 19, 13, 11, 17, 9, 18, 9, 4, 10, 11, 5, 15, 12, 15, 1, 18, 6, 13, 18, 9, 1, 17, 18, 17, 15, 15, 19, 3, 17, 15, 9, 11, 14, 17, 14, 15, 1, 15, 15, 10, 3, 19, 17, 14, 6, 11, 12, 15, 17, 15, 13, 3, 0, 10, 4, 11, 0, 14, 7, 0, 15, 0, 2, 17, 17, 6, 19, 14, 17, 14, 6, 15, 12, 13, 4, 1, 17, 16, 17, 13, 18, 0, 11, 10, 3, 0, 13, 15, 13, 15, 15, 13, 5, 18, 1, 11, 19, 15, 13, 13, 10, 17, 13, 14, 19, 14, 0, 5, 7, 15, 11, 1, 15, 15, 14, 15, 13, 0, 15, 18, 7, 8, 13, 11, 16, 14, 19, 15, 16, 0, 4, 17, 11, 10, 11, 5, 19, 10, 17, 7, 10, 17, 9, 11, 15, 19, 17, 11, 11, 11, 10, 16, 14, 17, 18, 13, 16, 19, 9, 11, 18, 16, 1, 13, 16, 18, 16, 11, 16, 8, 14, 14, 16, 15, 13, 15, 11, 11, 16, 2, 15, 16, 19, 8, 17, 19, 0, 16, 15, 14, 17, 9, 19, 6, 10, 16, 15, 0, 17, 10, 6, 16, 16, 17, 11, 15, 14, 10, 2, 18, 0, 18, 16, 10, 8, 3, 17, 17, 15, 11, 17, 13, 9, 18, 19, 11, 17, 11, 11, 5, 18, 17, 13, 11, 14, 13, 17, 15, 2, 0, 12, 0, 0, 14, 15, 8, 18, 9, 17, 10, 15, 13, 13, 3, 4, 16, 10, 5, 2, 2, 16, 17, 15, 7, 15, 17, 16, 12, 17, 15, 16, 15, 17, 17, 17, 18, 19, 17, 14, 18, 11, 12, 18, 6, 14, 11, 0, 15, 10, 17, 3, 11, 9, 15, 11, 9, 13, 16, 15, 13, 17, 18, 10, 4, 18, 19, 5, 19, 17, 11, 10, 19, 17, 19, 18, 10, 3, 13, 10, 16, 11, 5, 0, 3, 11, 17, 13, 14, 6, 16, 4, 19, 19, 14, 1, 14, 16, 11, 6, 18, 19, 11, 10, 11, 19, 11, 9, 19, 10, 17, 10, 18, 14, 11, 9, 0, 9, 1, 15, 11, 8, 14, 18, 11, 16, 15, 16, 6, 18, 13, 18, 2, 17, 15, 16, 15, 11, 16, 5, 15, 11, 14, 1, 16, 15, 11, 19, 16, 11, 15, 17, 15, 2, 16, 11, 7, 13, 17, 16, 7, 10, 1, 16, 1, 18, 17, 11, 17, 11, 5, 17, 1, 18, 0, 18, 16, 17, 12, 17, 11, 11, 15, 15, 12, 0, 19, 10, 17, 14, 12, 0, 19, 11, 2, 16, 19, 17, 15, 10, 5, 16, 5, 5, 13, 11, 19, 4, 15, 14, 11, 17, 15, 14, 8, 17, 3, 13, 0, 17, 0, 10, 12, 14, 6, 13, 13, 11, 10, 13, 10, 14, 0, 19, 11, 7, 10, 14, 3, 15, 3, 14, 10, 17, 4, 5, 11, 0, 19, 10, 10, 4, 1, 13, 17, 7, 5, 19, 13, 11, 14, 19, 19, 18, 2, 13, 6, 9, 0, 11, 11, 6, 11, 18, 16, 14, 10, 2, 8, 16, 5, 17, 16, 13, 18, 17, 13, 18, 15, 19, 6, 18, 17, 19, 12, 18, 9, 13, 6, 0, 15, 11, 18, 11, 0, 18, 15, 10, 10, 6, 5, 17, 11, 1, 9, 1, 14, 12, 14, 10, 15, 11, 11, 18, 2, 10, 3, 17, 18, 14, 19, 17, 15, 9, 5, 1, 9, 13, 15, 13, 10, 17, 19, 4, 17, 17, 15, 10, 17, 13, 18, 17, 11, 18, 18, 15, 15, 17, 11, 10, 8, 0, 15, 15, 14, 11, 15, 11, 8, 15, 3, 12, 8, 15, 14, 15, 10, 10, 11, 11, 3, 15, 9, 19, 17, 4, 7, 15, 4, 9, 9, 1, 6, 13, 10, 19, 9, 11, 16, 19, 17, 16, 14, 10, 9, 15, 15, 9, 5, 0, 2, 18, 9, 15, 19, 19, 17, 5, 10, 14, 15, 17, 15, 18, 17, 15, 4, 8, 13, 8, 11, 1, 15, 13, 15, 9, 9, 17, 14, 10, 14, 0, 15, 0, 17, 10, 0, 17, 0, 10, 14, 0, 12, 16, 17, 13, 13, 19, 3, 15, 0, 16, 19, 11, 14, 17, 19, 0, 14, 2, 18, 17, 15, 2, 17, 3, 11, 14, 9, 4, 3, 9, 0, 11, 13, 5, 15, 10, 14, 10, 15, 0, 9, 17, 13, 0, 5, 17, 16, 7, 0, 19, 15, 18, 15, 17, 0, 1, 15, 11, 17, 15, 9, 18, 6, 11, 17, 12, 13, 15, 18, 7, 17, 15, 15, 6, 13, 11, 18, 4, 5, 13, 7, 17, 18, 17, 15, 18, 0, 6, 5, 18, 13, 14, 2, 19, 8, 8, 10, 8, 17, 11, 15, 10, 17, 0, 19, 8, 6, 15, 12, 15, 16, 12, 19, 6, 6, 9, 14, 14, 17, 16, 16, 13, 6, 13, 15, 6, 15, 0, 10, 10, 15, 13, 14, 15, 17, 10, 15, 17, 11, 5, 9, 12, 16, 10, 13, 16, 2, 17, 6, 5, 0, 15, 9, 15, 19, 0, 11, 18, 1, 3, 18, 18, 18, 17, 0, 4, 17, 15, 15, 17, 1, 1, 15, 1, 15, 15, 16, 19, 17, 15, 2, 17, 15, 11, 16, 11, 15, 12, 15, 6, 13, 16, 19, 3, 5, 5, 7, 6, 1, 1, 16, 15, 18, 0, 15, 0, 15, 17, 19, 15, 0, 16, 14, 17, 6, 7, 15, 11, 12, 15, 15, 17, 15, 12, 17, 1, 1, 14, 3, 6, 17, 17, 18, 15, 1, 6, 10, 18, 10, 2, 11, 13, 17, 16, 3, 17, 19, 19, 18, 15, 8, 5, 16, 11, 0, 9, 13, 0, 9, 10, 1, 15, 1, 10, 17, 1, 14, 16, 1, 14, 15, 11, 14, 6, 8, 15, 9, 19, 16, 7, 17, 14, 17, 15, 18, 5, 11, 15, 2, 16, 5, 15, 9, 12, 17, 10, 15, 17, 19, 9, 1, 15, 17, 18, 3, 16, 18, 1, 13, 3, 18, 18, 15, 17, 17, 9, 19, 3, 11, 16, 17, 10, 1, 7, 19, 9, 19, 1, 17, 10, 18, 17, 8, 10, 17, 19, 13, 5, 18, 15, 11, 19, 0, 14, 11, 15, 15, 18, 18, 16, 13, 2, 16, 12, 5, 6, 0, 5, 15, 3, 16, 8, 8, 1, 17, 13, 18, 17, 17, 14, 6, 16, 14, 4, 3, 3, 3, 17, 7, 4, 14, 15, 18, 11, 16, 17, 15, 17, 6, 9, 13, 17, 6, 17, 18, 3, 17, 15, 3, 15, 18, 10, 17, 2, 18, 11, 13, 11, 15, 1, 10, 18, 19, 14, 11, 18, 15, 0, 1, 15, 17, 15, 8, 11, 18, 1, 7, 19, 10, 0, 9, 17, 9, 17, 11, 1, 19, 7, 15, 13, 17, 17, 0, 9, 16, 18, 18, 13, 15, 5, 7, 15, 15, 4, 15, 5, 5, 8, 9, 11, 13, 12, 16, 17, 17, 1, 19, 15, 14, 13, 11, 13, 13, 13, 14, 2, 9, 15, 7, 15, 2, 8, 11, 15, 11, 0, 4, 11, 15, 0, 15, 1, 5, 17, 5, 11, 5, 4, 8, 17, 3, 5, 14, 5, 13, 10, 16, 17, 9, 7, 6, 10, 10, 19, 3, 14, 17, 13, 1, 16, 0, 7, 17, 15, 0, 11, 15, 19, 16, 18, 0, 10, 17, 1, 17, 1, 15, 15, 9, 12, 14, 15, 11, 10, 11, 16, 13, 17, 15, 2, 13, 19, 19, 15, 16, 10, 19, 6, 14, 17, 13, 19, 15, 6, 17, 16, 15, 17, 18, 7, 12, 14, 17, 13, 1, 5, 9, 0, 6, 4, 17, 4, 13, 17, 16, 19, 1, 6, 14, 11, 11, 9, 14, 10, 16, 3, 12, 17, 15, 17, 1, 3, 13, 14, 10, 5, 14, 15, 6, 10, 15, 16, 14, 15, 15, 9, 1, 17, 15, 19, 17, 2, 9, 9, 1, 15, 17, 17, 15, 16, 16, 14, 9, 17, 17, 10, 4, 15, 17, 16, 18, 18, 19, 17, 3, 12, 7, 11, 3, 13, 12, 15, 1, 19, 17, 12, 16, 9, 8, 14, 0, 16, 5, 10, 15, 15, 15, 12, 17, 18, 15, 14, 2, 15, 18, 0, 15, 4, 1, 1, 1, 13, 13, 3, 16, 9, 19, 17, 12, 10, 18, 0, 0, 14, 7, 13, 14, 17, 13, 17, 1, 6, 9, 15, 4, 15, 8, 17, 18, 7, 19, 16, 9, 6, 16, 10, 14, 6, 18, 18, 11, 17, 7, 18, 13, 3, 17, 17, 15, 19, 19, 11, 0, 17, 15, 19, 18, 15, 5, 10, 0, 0, 0, 1, 13, 0, 12, 18, 8, 0, 2, 15, 17, 18, 6, 17, 13, 15, 0, 17, 18, 15, 18, 13, 19, 10, 8, 15, 2, 1, 15, 17, 15, 17, 17]\n",
    "docs_cat = [i+1 for i in true_cat_original];\n",
    "docs_ls = [[] for i in 1:20]\n",
    "for (idx, val) in enumerate(docs_cat)\n",
    "    push!(docs_ls[val], idx)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a31f0f1e-3134-47cf-a99d-c831c82f188f",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_marriage_assigment = Dict(1=>16, 2=>9, 3=>20, 4=>5, 5=>18, 6=>10, 7=>2, 8=>1, 9=>13, 10=>4, \n",
    "                            11=>7, 12=>12, 13=>8, 14=>17, 15=>6, 16=>14, 17=>19, 18=>15, 19=>11, 20=>3);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "62d96380-f0ef-4f54-bb1d-6a51f65b09e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "already_reomved_words = [];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "6cee1fbc-8418-478d-b92c-79c8e954c754",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'convinces' in topic 8 === Remove word 'kingston' in topic 7 === Remove word 'batffbi' in topic 20 === Remove word 'int' in topic 10 === Remove word 'informant' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'controller' in topic 11 === Remove word 'address' in topic 13 === Remove word 'went' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'streak' in topic 19 === Remove word 'flags' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'legislative' in topic 16 === Remove word 'provable' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'bumped' in topic 3 === "
     ]
    }
   ],
   "source": [
    "#First loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "    if false_docs_for_cat != []\n",
    "        X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_reomved_words)\n",
    "                word_idx = i\n",
    "                push!(already_reomved_words, i)\n",
    "                break\n",
    "            end\n",
    "        end        \n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "aaeae42e-2f80-4b2c-a930-a16c3320c4bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.38732646450023844\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "7973dd5a-ca94-4a63-ae32-527a882d853c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'mansfield' in topic 7 === Remove word 'functions' in topic 20 === Remove word 'char' in topic 10 === Remove word 'gun' in topic 4 === Remove word 'file' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'arrghhh' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'hansson' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'zeresan' in topic 16 === Remove word 'openwinhomesharemanusrman' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheism' in topic 3 === "
     ]
    }
   ],
   "source": [
    "#Second loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "    if false_docs_for_cat != []\n",
    "        X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_reomved_words)\n",
    "                word_idx = i\n",
    "                push!(already_reomved_words, i)\n",
    "                break\n",
    "            end\n",
    "        end        \n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "b8686829-e519-4fbc-abc9-c4d11c6a3825",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37546502834324186\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "5f5221ed-9674-443c-a56a-5009a2b8984c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'supremacists' in topic 8 === Remove word 'many' in topic 7 === Remove word 'widget' in topic 20 === Remove word 'guidance' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'ftp' in topic 15 === Remove word 'card' in topic 11 === Remove word 'steven' in topic 13 === Remove word 'said' in topic 2 === Remove word 'team' in topic 6 === Remove word 'standings' in topic 19 === Remove word 'place' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'honda' in topic 16 === Remove word 'israli' in topic 18 === Remove word 'god' in topic 1 === Remove word 'stimulus' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'belief' in topic 3 === "
     ]
    }
   ],
   "source": [
    "#Third loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "    if false_docs_for_cat != []\n",
    "        X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_reomved_words)\n",
    "                word_idx = i\n",
    "                push!(already_reomved_words, i)\n",
    "                break\n",
    "            end\n",
    "        end        \n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "cf64a07d-9958-41df-b9d0-62d4212d8063",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3679746228106866\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "c533640f-3241-4d1f-8b9a-756a552b3cda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'oatesjuneau' in topic 8 === Remove word 'inefficient' in topic 7 === Remove word 'xterm' in topic 20 === Remove word 'include' in topic 10 === Remove word 'firearms' in topic 4 === Remove word 'available' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'came' in topic 2 === Remove word 'sharks' in topic 6 === Remove word 'san' in topic 19 === Remove word 'contains' in topic 12 === Remove word 'diet' in topic 9 === Remove word 'relevation' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'jews' in topic 17 === Remove word 'exists' in topic 3 === "
     ]
    }
   ],
   "source": [
    "#Fourth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "    if false_docs_for_cat != []\n",
    "        X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_reomved_words)\n",
    "                word_idx = i\n",
    "                push!(already_reomved_words, i)\n",
    "                break\n",
    "            end\n",
    "        end        \n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "1f6789cb-ceaa-4e7b-ac3a-bad461648207",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3754477304513362\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "0ee265f4-29c6-4390-a85c-14b4c02ae7c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'bludgeon' in topic 8 === Remove word 'crumenam' in topic 7 === Remove word 'motif' in topic 20 === Remove word 'function' in topic 10 === Remove word 'assault' in topic 4 === Remove word 'software' in topic 15 === Remove word 'multitasking' in topic 11 === Remove word 'gumulcine' in topic 13 === Remove word 'told' in topic 2 === Remove word 'games' in topic 6 === Remove word 'sermons' in topic 19 === Remove word 'squirt' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'lord' in topic 1 === Remove word 'reform' in topic 14 === Remove word 'country' in topic 17 === Remove word 'atheist' in topic 3 === "
     ]
    }
   ],
   "source": [
    "#Fifth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "    if false_docs_for_cat != []\n",
    "        X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_reomved_words)\n",
    "                word_idx = i\n",
    "                push!(already_reomved_words, i)\n",
    "                break\n",
    "            end\n",
    "        end        \n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "87b5fd81-8785-4979-94a0-da5d7ec83485",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.37433223674432287\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "78262b78-0bbc-4178-aac9-e04492d89e9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'vestments' in topic 7 === Remove word 'typex' in topic 20 === Remove word 'int' in topic 10 === Remove word 'archiveservergazoochengsuncom' in topic 4 === Remove word 'windows' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'attendees' in topic 13 === Remove word 'went' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'milwaukee' in topic 19 === Remove word 'course' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'yournameyourhost' in topic 16 === Remove word 'imputed' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'theists' in topic 3 === 0.3957691935979762\n",
      "Remove word 'exemption' in topic 8 === Remove word 'affair' in topic 7 === Remove word 'end' in topic 20 === Remove word 'char' in topic 10 === Remove word 'police' in topic 4 === Remove word 'file' in topic 15 === Remove word 'drives' in topic 11 === Remove word 'told' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'relatedness' in topic 6 === Remove word 'rockies' in topic 19 === Remove word 'since' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'tires' in topic 16 === Remove word 'stanley' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'senate' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'science' in topic 3 === 0.3848843768650462\n",
      "Remove word 'systems' in topic 8 === Remove word 'newsscicompanswers' in topic 7 === Remove word 'routines' in topic 20 === Remove word 'null' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'landlord' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'address' in topic 13 === Remove word 'came' in topic 2 === Remove word 'play' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'lot' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'nothing' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'time' in topic 5 === Remove word 'gun' in topic 17 === Remove word 'god' in topic 3 === 0.38897489105371497\n",
      "Remove word 'simpleminded' in topic 8 === Remove word 'nahh' in topic 7 === Remove word 'processing' in topic 20 === Remove word 'stdioh' in topic 10 === Remove word 'arab' in topic 4 === Remove word 'decade' in topic 15 === Remove word 'card' in topic 11 === Remove word 'engima' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'team' in topic 6 === Remove word 'sure' in topic 19 === Remove word 'enforcement' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'car' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'matthew' in topic 1 === Remove word 'release' in topic 14 === Remove word 'amendment' in topic 17 === Remove word 'belief' in topic 3 === 0.39274567654108755\n",
      "Remove word 'get' in topic 8 === Remove word 'sgml' in topic 7 === Remove word 'fortran' in topic 20 === Remove word 'include' in topic 10 === Remove word 'jews' in topic 4 === Remove word 'symbolize' in topic 15 === Remove word 'apple' in topic 11 === Remove word 'say' in topic 13 === Remove word 'floor' in topic 2 === Remove word 'season' in topic 6 === Remove word 'know' in topic 19 === Remove word 'yup' in topic 12 === Remove word 'nutrition' in topic 9 === Remove word 'parent' in topic 16 === Remove word 'explorerdgptorontoedupubhalfqwerty' in topic 18 === Remove word 'testament' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'firearms' in topic 17 === Remove word 'believe' in topic 3 === 0.3876512031584553\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "e853fd9f-a628-4038-9290-15ab58056a92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'rubinstein' in topic 8 === Remove word 'many' in topic 7 === Remove word 'loooooooooong' in topic 20 === Remove word 'int' in topic 10 === Remove word 'lady' in topic 4 === Remove word 'file' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'went' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'kansas' in topic 19 === Remove word 'several' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'utilsrc' in topic 16 === Remove word 'unfavorable' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheism' in topic 3 === 0.39458517618829964\n",
      "Remove word 'think' in topic 8 === Remove word 'decompressed' in topic 7 === Remove word 'graphics' in topic 20 === Remove word 'char' in topic 10 === Remove word 'attorney' in topic 4 === Remove word 'ftp' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'hotel' in topic 13 === Remove word 'came' in topic 2 === Remove word 'observing' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'one' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'quantized' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'belief' in topic 3 === 0.38452529794433643\n",
      "Remove word 'breadthtaking' in topic 8 === Remove word 'dos' in topic 7 === Remove word 'unix' in topic 20 === Remove word 'function' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'application' in topic 15 === Remove word 'card' in topic 11 === Remove word 'dackpermanetorg' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'team' in topic 6 === Remove word 'pct' in topic 19 === Remove word 'around' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'car' in topic 16 === Remove word 'good' in topic 18 === Remove word 'god' in topic 1 === Remove word 'senate' in topic 14 === Remove word 'rest' in topic 5 === Remove word 'gun' in topic 17 === Remove word 'faith' in topic 3 === 0.3853746261677411\n",
      "Remove word 'briannoonan' in topic 8 === Remove word 'compressed' in topic 7 === Remove word 'processing' in topic 20 === Remove word 'null' in topic 10 === Remove word 'compound' in topic 4 === Remove word 'windows' in topic 15 === Remove word 'antieducation' in topic 11 === Remove word 'new' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'play' in topic 6 === Remove word 'raiders' in topic 19 === Remove word 'know' in topic 12 === Remove word 'health' in topic 9 === Remove word 'tires' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'amendment' in topic 17 === Remove word 'beliefs' in topic 3 === 0.3894649974928275\n",
      "Remove word 'freezes' in topic 8 === Remove word 'ballistic' in topic 7 === Remove word 'fortran' in topic 20 === Remove word 'stdioh' in topic 10 === Remove word 'arab' in topic 4 === Remove word 'renbergstefan' in topic 15 === Remove word 'asynchronous' in topic 11 === Remove word 'mail' in topic 13 === Remove word 'lying' in topic 2 === Remove word 'gulu' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'involved' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'people' in topic 16 === Remove word 'treble' in topic 18 === Remove word 'lord' in topic 1 === Remove word 'immediate' in topic 14 === Remove word 'firearms' in topic 17 === Remove word 'atheist' in topic 3 === 0.3931218841865876\n",
      "248908.805244 seconds (168.95 G allocations: 2.930 TiB, 73.54% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "257786.4024040699"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "id": "97f57dcd-2915-4538-98a5-a32ccea3ba09",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'fractures' in topic 8 === Remove word 'humphries' in topic 7 === Remove word 'noninterlaced' in topic 20 === Remove word 'int' in topic 10 === Remove word 'exaclty' in topic 4 === Remove word 'ftp' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'mail' in topic 13 === Remove word 'yelena' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'standings' in topic 19 === Remove word 'one' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'commemoration' in topic 16 === Remove word 'jfifformat' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheism' in topic 3 === 0.38252950876103353\n",
      "Remove word 'think' in topic 8 === Remove word 'many' in topic 7 === Remove word 'motif' in topic 20 === Remove word 'char' in topic 10 === Remove word 'compound' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'drives' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'came' in topic 2 === Remove word 'car' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'give' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'afilipov' in topic 16 === Remove word 'baptizes' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'atheist' in topic 3 === 0.374286543129507\n",
      "Remove word 'exactly' in topic 8 === Remove word 'kigali' in topic 7 === Remove word 'xterm' in topic 20 === Remove word 'darken' in topic 10 === Remove word 'gun' in topic 4 === Remove word 'unix' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'valuemask' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'canada' in topic 6 === Remove word 'san' in topic 19 === Remove word 'well' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'screwdriver' in topic 16 === Remove word 'constantcolor' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'science' in topic 3 === 0.3738185500136463\n",
      "Remove word 'loach' in topic 8 === Remove word 'bisector' in topic 7 === Remove word 'window' in topic 20 === Remove word 'section' in topic 10 === Remove word 'firearms' in topic 4 === Remove word 'software' in topic 15 === Remove word 'dos' in topic 11 === Remove word 'ahmeteecgtorontoedu' in topic 13 === Remove word 'said' in topic 2 === Remove word 'games' in topic 6 === Remove word 'jtcentsodaberkeleyedu' in topic 19 === Remove word 'wrong' in topic 12 === Remove word 'treatment' in topic 9 === Remove word 'houston' in topic 16 === Remove word 'pseudox' in topic 18 === Remove word 'god' in topic 1 === Remove word 'jobs' in topic 14 === Remove word 'jews' in topic 17 === Remove word 'religion' in topic 3 === 0.3818672474628419\n",
      "Remove word 'serdar' in topic 8 === Remove word 'dreaded' in topic 7 === Remove word 'sunos' in topic 20 === Remove word 'benzion' in topic 10 === Remove word 'assault' in topic 4 === Remove word 'processing' in topic 15 === Remove word 'card' in topic 11 === Remove word 'hotel' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'charalambidi' in topic 6 === Remove word 'mean' in topic 19 === Remove word 'intent' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'clutch' in topic 16 === Remove word 'good' in topic 18 === Remove word 'matthew' in topic 1 === Remove word 'today' in topic 14 === Remove word 'arabs' in topic 17 === Remove word 'belief' in topic 3 === 0.38317258900021745\n",
      "236602.423804 seconds (167.18 G allocations: 2.899 TiB, 74.05% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "239452.51244187355"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "d7658358-11fd-4b67-bb5d-5bed1448c787",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'kraatveldstraat' in topic 8 === Remove word 'punditseditors' in topic 7 === Remove word 'headtohead' in topic 20 === Remove word 'int' in topic 10 === Remove word 'noonan' in topic 4 === Remove word 'file' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'mail' in topic 13 === Remove word 'went' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'cubs' in topic 19 === Remove word 'disease' in topic 9 === Remove word 'tensioners' in topic 16 === Remove word 'rabbi' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'luggers' in topic 3 === 0.39498399806502404\n",
      "Remove word 'think' in topic 8 === Remove word 'enligthen' in topic 7 === Remove word 'programming' in topic 20 === Remove word 'null' in topic 10 === Remove word 'establish' in topic 4 === Remove word 'software' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'ricks' in topic 13 === Remove word 'door' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'visvisual' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'identifier' in topic 16 === Remove word 'steadier' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheist' in topic 3 === 0.379056772931215\n",
      "Remove word 'jdishawhmcvaxclaremontedu' in topic 8 === Remove word 'windows' in topic 7 === Remove word 'widget' in topic 20 === Remove word 'code' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'send' in topic 15 === Remove word 'card' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'esdial' in topic 6 === Remove word 'san' in topic 19 === Remove word 'govermentturkish' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'rear' in topic 16 === Remove word 'amx' in topic 18 === Remove word 'god' in topic 1 === Remove word 'jobs' in topic 14 === Remove word 'gun' in topic 17 === Remove word 'belief' in topic 3 === 0.378659259610309\n",
      "Remove word 'exocode' in topic 8 === Remove word 'many' in topic 7 === Remove word 'window' in topic 20 === Remove word 'char' in topic 10 === Remove word 'agents' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'controller' in topic 11 === Remove word 'address' in topic 13 === Remove word 'ran' in topic 2 === Remove word 'team' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'employees' in topic 12 === Remove word 'treatment' in topic 9 === Remove word 'car' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'amendment' in topic 17 === Remove word 'believe' in topic 3 === 0.38802134272449457\n",
      "Remove word 'azt' in topic 8 === Remove word 'hamrlik' in topic 7 === Remove word 'sunos' in topic 20 === Remove word 'echo' in topic 10 === Remove word 'waco' in topic 4 === Remove word 'unix' in topic 15 === Remove word 'mac' in topic 11 === Remove word 'fax' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'courtesy' in topic 6 === Remove word 'meant' in topic 19 === Remove word 'among' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'tires' in topic 16 === Remove word 'good' in topic 18 === Remove word 'holy' in topic 1 === Remove word 'stimulus' in topic 14 === Remove word 'see' in topic 5 === Remove word 'firearms' in topic 17 === Remove word 'christianity' in topic 3 === 0.38883150445391274\n",
      "239229.634041 seconds (167.18 G allocations: 2.899 TiB, 73.75% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "239235.47860503197"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "0ca66a6d-f9bc-49c4-8924-a042765892d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'juggle' in topic 7 === Remove word 'farah' in topic 20 === Remove word 'int' in topic 10 === Remove word 'informant' in topic 4 === Remove word 'windows' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'went' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'cubs' in topic 19 === Remove word 'end' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'decongestant' in topic 16 === Remove word 'unresurrected' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'religious' in topic 3 === 0.39719774715467226\n",
      "Remove word 'commissioner' in topic 8 === Remove word 'many' in topic 7 === Remove word 'pixel' in topic 20 === Remove word 'char' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'file' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'address' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'inour' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'would' in topic 12 === Remove word 'nutrition' in topic 9 === Remove word 'liquidation' in topic 16 === Remove word 'arrangement' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'gun' in topic 17 === Remove word 'christianity' in topic 3 === 0.39358126923282777\n",
      "Remove word 'know' in topic 8 === Remove word 'xtappmainloop' in topic 7 === Remove word 'graphics' in topic 20 === Remove word 'function' in topic 10 === Remove word 'compound' in topic 4 === Remove word 'trianglehead' in topic 15 === Remove word 'card' in topic 11 === Remove word 'hotel' in topic 13 === Remove word 'came' in topic 2 === Remove word 'team' in topic 6 === Remove word 'streak' in topic 19 === Remove word 'always' in topic 12 === Remove word 'diet' in topic 9 === Remove word 'car' in topic 16 === Remove word 'constant' in topic 18 === Remove word 'god' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'belief' in topic 3 === 0.3951604158283063\n",
      "Remove word 'katella' in topic 8 === Remove word 'ferran' in topic 7 === Remove word 'animation' in topic 20 === Remove word 'stdioh' in topic 10 === Remove word 'baklavah' in topic 4 === Remove word 'iconised' in topic 15 === Remove word 'apple' in topic 11 === Remove word 'currentlyrunning' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'games' in topic 6 === Remove word 'jtcentsodaberkeleyedu' in topic 19 === Remove word 'someone' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'bike' in topic 16 === Remove word 'unhurt' in topic 18 === Remove word 'matthew' in topic 1 === Remove word 'stimulus' in topic 14 === Remove word 'arabs' in topic 17 === Remove word 'assertion' in topic 3 === 0.3955115194987468\n",
      "Remove word 'weaponssuprised' in topic 8 === Remove word 'arrestees' in topic 7 === Remove word 'processing' in topic 20 === Remove word 'include' in topic 10 === Remove word 'guns' in topic 4 === Remove word 'altered' in topic 15 === Remove word 'adapter' in topic 11 === Remove word 'students' in topic 13 === Remove word 'said' in topic 2 === Remove word 'game' in topic 6 === Remove word 'seattle' in topic 19 === Remove word 'art' in topic 12 === Remove word 'health' in topic 9 === Remove word 'people' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'release' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'atheists' in topic 3 === 0.3960773685328136\n",
      "204181.004667 seconds (167.15 G allocations: 2.899 TiB, 70.90% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "204185.39231991768"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1b7411de-5de8-432f-960a-60b1a289e9a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'sharon' in topic 7 === Remove word 'xmnwidth' in topic 20 === Remove word 'int' in topic 10 === Remove word 'prison' in topic 4 === Remove word 'software' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'information' in topic 13 === Remove word 'went' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'cincinnati' in topic 19 === Remove word 'things' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'coombes' in topic 16 === Remove word 'vettesrequestcompaqcom' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'theism' in topic 3 === 0.3918033516862313\n",
      "Remove word 'sanction' in topic 8 === Remove word 'many' in topic 7 === Remove word 'widget' in topic 20 === Remove word 'arg' in topic 10 === Remove word 'gun' in topic 4 === Remove word 'workstation' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'canadiensroy' in topic 13 === Remove word 'came' in topic 2 === Remove word 'honda' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'blows' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'gudas' in topic 16 === Remove word 'inconclusive' in topic 18 === Remove word 'god' in topic 1 === Remove word 'today' in topic 14 === Remove word 'like' in topic 5 === Remove word 'israel' in topic 17 === Remove word 'existence' in topic 3 === 0.38143412499033397\n",
      "Remove word 'thompsoncenter' in topic 8 === Remove word 'email' in topic 7 === Remove word 'xterm' in topic 20 === Remove word 'char' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'replier' in topic 13 === Remove word 'door' in topic 2 === Remove word 'games' in topic 6 === Remove word 'start' in topic 19 === Remove word 'immediate' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'xrules' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'health' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'belief' in topic 3 === 0.3830951721775533\n",
      "Remove word 'ena' in topic 8 === Remove word 'reincarnate' in topic 7 === Remove word 'motif' in topic 20 === Remove word 'function' in topic 10 === Remove word 'firearms' in topic 4 === Remove word 'unix' in topic 15 === Remove word 'card' in topic 11 === Remove word 'much' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'team' in topic 6 === Remove word 'best' in topic 19 === Remove word 'aspects' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'torque' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'budget' in topic 14 === Remove word 'palestinians' in topic 17 === Remove word 'christianity' in topic 3 === 0.3859691478507334\n",
      "Remove word 'get' in topic 8 === Remove word 'pellet' in topic 7 === Remove word 'sunos' in topic 20 === Remove word 'schoolcompanyorganization' in topic 10 === Remove word 'criminal' in topic 4 === Remove word 'processing' in topic 15 === Remove word 'unearthed' in topic 11 === Remove word 'hotel' in topic 13 === Remove word 'said' in topic 2 === Remove word 'unattacked' in topic 6 === Remove word 'right' in topic 19 === Remove word 'machineguns' in topic 12 === Remove word 'treatment' in topic 9 === Remove word 'car' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'lord' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'check' in topic 5 === Remove word 'state' in topic 17 === Remove word 'believe' in topic 3 === 0.3821738049348516\n",
      "56796.862377 seconds (170.73 G allocations: 2.960 TiB, 1.23% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "56799.0968477726"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "word_idx = -1\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e6788f17-0f63-43d9-b597-60292037b636",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'upsurge' in topic 7 === Remove word 'offices' in topic 20 === Remove word 'int' in topic 10 === Remove word 'koblitz' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'mail' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'oakland' in topic 19 === Remove word 'living' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'mamograma' in topic 16 === Remove word 'alarms' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'religious' in topic 3 === 0.40022796010062034\n",
      "Remove word 'crumbles' in topic 8 === Remove word 'saturnian' in topic 7 === Remove word 'triangles' in topic 20 === Remove word 'char' in topic 10 === Remove word 'investigator' in topic 4 === Remove word 'sites' in topic 15 === Remove word 'card' in topic 11 === Remove word 'picture' in topic 13 === Remove word 'went' in topic 2 === Remove word 'ride' in topic 6 === Remove word 'streak' in topic 19 === Remove word 'across' in topic 12 === Remove word 'clinical' in topic 9 === Remove word 'dcpengruarkedu' in topic 16 === Remove word 'encircled' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'gun' in topic 17 === Remove word 'belief' in topic 3 === 0.3913366913672137\n",
      "Remove word 'officers' in topic 8 === Remove word 'man' in topic 7 === Remove word 'unix' in topic 20 === Remove word 'mkentry' in topic 10 === Remove word 'agents' in topic 4 === Remove word 'windows' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'came' in topic 2 === Remove word 'game' in topic 6 === Remove word 'raiders' in topic 19 === Remove word 'well' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'people' in topic 16 === Remove word 'infectious' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'billion' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'god' in topic 3 === 0.3984703821847491\n",
      "Remove word 'democratclinton' in topic 8 === Remove word 'ohmite' in topic 7 === Remove word 'processing' in topic 20 === Remove word 'include' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'candiotti' in topic 15 === Remove word 'controller' in topic 11 === Remove word 'cto' in topic 13 === Remove word 'yelena' in topic 2 === Remove word 'team' in topic 6 === Remove word 'jtchernocfberkeleyedu' in topic 19 === Remove word 'ever' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'lord' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'state' in topic 17 === Remove word 'assertion' in topic 3 === 0.39558502578925714\n",
      "Remove word 'sideeffects' in topic 8 === Remove word 'file' in topic 7 === Remove word 'fortran' in topic 20 === Remove word 'extracting' in topic 10 === Remove word 'guns' in topic 4 === Remove word 'albania' in topic 15 === Remove word 'mac' in topic 11 === Remove word 'attendees' in topic 13 === Remove word 'door' in topic 2 === Remove word 'legislator' in topic 6 === Remove word 'phillies' in topic 19 === Remove word 'common' in topic 12 === Remove word 'health' in topic 9 === Remove word 'faintly' in topic 16 === Remove word 'good' in topic 18 === Remove word 'bible' in topic 1 === Remove word 'stimulus' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'christians' in topic 3 === 0.40246228191431865\n",
      "57830.124012 seconds (167.15 G allocations: 2.899 TiB, 1.26% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "57831.24010515213"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "word_idx = -1\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e7087eff-4c88-4e4a-acb8-360c2199c3de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'reinnoculation' in topic 8 === Remove word 'stretch' in topic 7 === Remove word 'kidnapping' in topic 20 === Remove word 'int' in topic 10 === Remove word 'prison' in topic 4 === Remove word 'software' in topic 15 === Remove word 'scsi' in topic 11 === Remove word 'xtappprocessevent' in topic 13 === Remove word 'went' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'montreal' in topic 19 === Remove word 'treat' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'berthiaume' in topic 16 === Remove word 'valueless' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheism' in topic 3 === 0.389791414139785\n",
      "Remove word 'think' in topic 8 === Remove word 'remind' in topic 7 === Remove word 'widgets' in topic 20 === Remove word 'char' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'graphics' in topic 15 === Remove word 'controller' in topic 11 === Remove word 'topics' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'wcsnbarc' in topic 6 === Remove word 'jtchernocfberkeleyedu' in topic 19 === Remove word 'earlier' in topic 12 === Remove word 'nutrition' in topic 9 === Remove word 'amp' in topic 16 === Remove word 'getactualinitptractual' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'existence' in topic 3 === 0.37373998495398136\n",
      "Remove word 'say' in topic 8 === Remove word 'plays' in topic 7 === Remove word 'motif' in topic 20 === Remove word 'noll' in topic 10 === Remove word 'handlewmmessagesw' in topic 4 === Remove word 'unix' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'address' in topic 13 === Remove word 'said' in topic 2 === Remove word 'team' in topic 6 === Remove word 'raiders' in topic 19 === Remove word 'common' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'car' in topic 16 === Remove word 'kewait' in topic 18 === Remove word 'god' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'amendment' in topic 17 === Remove word 'belief' in topic 3 === 0.38141099171490045\n",
      "Remove word 'gmeds' in topic 8 === Remove word 'unslung' in topic 7 === Remove word 'xterm' in topic 20 === Remove word 'chongo' in topic 10 === Remove word 'fire' in topic 4 === Remove word 'processing' in topic 15 === Remove word 'card' in topic 11 === Remove word 'christopher' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'game' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'one' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'tires' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'stimulus' in topic 14 === Remove word 'gun' in topic 17 === Remove word 'christianity' in topic 3 === 0.389385688729531\n",
      "Remove word 'erect' in topic 8 === Remove word 'guiding' in topic 7 === Remove word 'window' in topic 20 === Remove word 'stdioh' in topic 10 === Remove word 'arab' in topic 4 === Remove word 'directory' in topic 15 === Remove word 'multitasking' in topic 11 === Remove word 'fax' in topic 13 === Remove word 'door' in topic 2 === Remove word 'sharks' in topic 6 === Remove word 'checkup' in topic 19 === Remove word 'clinical' in topic 9 === Remove word 'angering' in topic 16 === Remove word 'good' in topic 18 === Remove word 'bible' in topic 1 === Remove word 'today' in topic 14 === Remove word 'firearms' in topic 17 === Remove word 'absolute' in topic 3 === 0.3874026342212453\n",
      "60225.133083 seconds (165.43 G allocations: 2.869 TiB, 1.36% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "60226.438605070114"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "word_idx = -1\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0c9173ba-7774-4480-9ada-88e1f21bf38d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove word 'think' in topic 8 === Remove word 'many' in topic 7 === Remove word 'irulesrcsitedef' in topic 20 === Remove word 'int' in topic 10 === Remove word 'told' in topic 4 === Remove word 'windows' in topic 15 === Remove word 'drive' in topic 11 === Remove word 'please' in topic 13 === Remove word 'went' in topic 2 === Remove word 'bike' in topic 6 === Remove word 'seattle' in topic 19 === Remove word 'everything' in topic 12 === Remove word 'medical' in topic 9 === Remove word 'raises' in topic 16 === Remove word 'instructs' in topic 18 === Remove word 'jesus' in topic 1 === Remove word 'president' in topic 14 === Remove word 'israeli' in topic 17 === Remove word 'programmmers' in topic 3 === 0.3927015509705446\n",
      "Remove word 'sorehandvmucsfedu' in topic 8 === Remove word 'pouches' in topic 7 === Remove word 'polyline' in topic 20 === Remove word 'char' in topic 10 === Remove word 'fbi' in topic 4 === Remove word 'file' in topic 15 === Remove word 'ide' in topic 11 === Remove word 'mantaining' in topic 13 === Remove word 'sumgait' in topic 2 === Remove word 'team' in topic 6 === Remove word 'astros' in topic 19 === Remove word 'get' in topic 12 === Remove word 'patients' in topic 9 === Remove word 'car' in topic 16 === Remove word 'acetylene' in topic 18 === Remove word 'christ' in topic 1 === Remove word 'secretary' in topic 14 === Remove word 'israel' in topic 17 === Remove word 'atheist' in topic 3 === 0.3955955274557391\n",
      "Remove word 'gored' in topic 8 === Remove word 'roguish' in topic 7 === Remove word 'graphics' in topic 20 === Remove word 'null' in topic 10 === Remove word 'gun' in topic 4 === Remove word 'software' in topic 15 === Remove word 'ram' in topic 11 === Remove word 'contact' in topic 13 === Remove word 'saw' in topic 2 === Remove word 'game' in topic 6 === Remove word 'kansas' in topic 19 === Remove word 'directed' in topic 12 === Remove word 'disease' in topic 9 === Remove word 'engine' in topic 16 === Remove word 'kind' in topic 18 === Remove word 'heaven' in topic 1 === Remove word 'today' in topic 14 === Remove word 'arab' in topic 17 === Remove word 'belief' in topic 3 === 0.3925017143976017\n",
      "Remove word 'three' in topic 8 === Remove word 'irving' in topic 7 === Remove word 'processing' in topic 20 === Remove word 'stdioh' in topic 10 === Remove word 'firearms' in topic 4 === Remove word 'rosenberger' in topic 15 === Remove word 'shortstories' in topic 11 === Remove word 'hotel' in topic 13 === Remove word 'said' in topic 2 === Remove word 'games' in topic 6 === Remove word 'jtchernocfberkeleyedu' in topic 19 === Remove word 'diet' in topic 9 === Remove word 'cars' in topic 16 === Remove word 'questions' in topic 18 === Remove word 'god' in topic 1 === Remove word 'edt' in topic 14 === Remove word 'jews' in topic 17 === Remove word 'claim' in topic 3 === 0.39208355068680173\n",
      "Remove word 'tribute' in topic 8 === Remove word 'xmgraph' in topic 7 === Remove word 'fortran' in topic 20 === Remove word 'include' in topic 10 === Remove word 'assault' in topic 4 === Remove word 'lunary' in topic 15 === Remove word 'adapter' in topic 11 === Remove word 'mattelautotrolcommattelautotrolcom' in topic 13 === Remove word 'came' in topic 2 === Remove word 'oof' in topic 6 === Remove word 'lost' in topic 19 === Remove word 'treatment' in topic 9 === Remove word 'confirmatory' in topic 16 === Remove word 'good' in topic 18 === Remove word 'ancient' in topic 1 === Remove word 'senate' in topic 14 === Remove word 'war' in topic 17 === Remove word 'beliefs' in topic 3 === 0.394477418349292\n",
      "56738.744620 seconds (163.70 G allocations: 2.839 TiB, 1.26% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "56739.99833512306"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_reomved_words = [];\n",
    "t0 = time();\n",
    "word_idx = -1\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        remaining = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat]) && !(i in false_docs_for_cat)]\n",
    "        if false_docs_for_cat != []\n",
    "            X, Y = make_PMI_dataset(false_docs_for_cat, remaining)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_reomved_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_reomved_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end        \n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Remove word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.removeWord(lda, corpus, word_idx, current_topic);\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);        \n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85d5c19b-3075-44e0-a728-e25bae242a65",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79b497fa-c2d0-4dd7-945f-3e4bd8585491",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf9995dd-3700-43d8-bc0e-8328031c1981",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "efc54129-b806-432e-870f-95a4b8a5b93d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "make_PMI_dataset (generic function with 1 method)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function make_PMI_dataset(this_y_1, this_y_0)\n",
    "    X = []; Y = []\n",
    "    for i in this_y_1\n",
    "        push!(X, position_to_feature(i))\n",
    "        push!(Y, 1)\n",
    "    end\n",
    "    for i in this_y_0\n",
    "        push!(X, position_to_feature(i))\n",
    "        push!(Y, 0)\n",
    "    end\n",
    "    return X, Y\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f64a680f-5294-4900-a16c-ba20fd02d58e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "position_to_feature (generic function with 1 method)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function position_to_feature(doc_idx)\n",
    "    position_vector = corpus.documents[doc_idx]\n",
    "    feature_vector = [0 for i in 1:corpus.vocab_count]\n",
    "    for (idx, val) in enumerate(position_vector)\n",
    "        feature_vector[val] +=1\n",
    "    end\n",
    "    return feature_vector\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e86abc7c-b2f3-458c-9a04-4d02c8c2a297",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false_docs_in_cat (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function false_docs_in_cat(lda, corpus, docs_ls, cat_idx)\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    topic_x_top_docs = [idx for (idx, val) in enumerate(predicted_with_marriage_assigment) if val==cat_idx];\n",
    "    docs_in_topic_notin_cat = []\n",
    "    for i in topic_x_top_docs \n",
    "        if !(i in docs_ls[cat_idx])\n",
    "            push!(docs_in_topic_notin_cat, i)\n",
    "        end\n",
    "    end\n",
    "    return docs_in_topic_notin_cat\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff09e01a-2ba1-4a77-b2a4-51840fc1e400",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic_of_each_doc (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function topic_of_each_doc(lda, corpus)\n",
    "    top_topic_for_each_doc = []\n",
    "    for i in 1:corpus.document_size\n",
    "        top_val = 0\n",
    "        top_topic = 0\n",
    "        for j in 1:lda.M\n",
    "            v = TopicModels.topicPredict(lda, i, j)\n",
    "            if v>=top_val\n",
    "                top_val = v\n",
    "                top_topic = j\n",
    "            end\n",
    "        end\n",
    "        push!(top_topic_for_each_doc, top_topic)\n",
    "    end\n",
    "    return top_topic_for_each_doc\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f31df117-ca7d-4178-9fc4-c52badbaa46e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Add_word refinement after stable marriage algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "191ebd48-558c-4a72-9dc3-25689d72886a",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cat_original = [17, 7, 18, 17, 10, 19, 16, 19, 18, 8, 13, 7, 12, 9, 18, 18, 17, 17, 15, 17, 7, 1, 19, 5, 4, 8, 19, 17, 15, 15, 17, 0, 19, 11, 17, 18, 9, 13, 19, 17, 13, 17, 19, 15, 7, 8, 11, 9, 1, 8, 0, 18, 17, 14, 14, 10, 15, 3, 8, 10, 10, 11, 17, 6, 17, 17, 15, 8, 15, 11, 2, 17, 9, 18, 17, 17, 10, 2, 9, 1, 10, 4, 9, 15, 3, 15, 19, 15, 16, 13, 17, 11, 0, 18, 10, 14, 6, 17, 15, 17, 19, 14, 8, 14, 5, 5, 11, 15, 6, 10, 5, 9, 15, 11, 9, 15, 7, 1, 14, 16, 15, 13, 13, 17, 0, 11, 16, 3, 16, 15, 14, 14, 19, 9, 18, 14, 17, 17, 4, 17, 15, 1, 16, 11, 15, 6, 2, 14, 15, 18, 8, 17, 18, 4, 7, 11, 17, 18, 9, 16, 18, 7, 11, 2, 15, 19, 1, 11, 14, 17, 1, 16, 6, 5, 19, 7, 11, 16, 8, 16, 10, 15, 17, 16, 16, 1, 0, 7, 15, 0, 6, 18, 10, 9, 11, 16, 19, 14, 6, 0, 0, 3, 17, 17, 12, 16, 8, 6, 19, 10, 17, 16, 14, 4, 16, 11, 1, 14, 0, 2, 16, 15, 15, 6, 15, 0, 14, 10, 15, 13, 2, 19, 15, 17, 15, 1, 9, 16, 18, 17, 6, 17, 8, 14, 11, 5, 15, 7, 15, 15, 12, 11, 14, 14, 3, 15, 3, 17, 3, 14, 11, 19, 10, 18, 15, 16, 15, 15, 9, 16, 11, 4, 19, 13, 3, 19, 9, 17, 18, 7, 7, 15, 16, 14, 15, 10, 5, 9, 14, 14, 17, 15, 15, 19, 14, 15, 15, 0, 4, 11, 11, 15, 12, 6, 3, 15, 14, 5, 14, 18, 11, 11, 14, 17, 17, 10, 9, 5, 7, 10, 19, 18, 17, 15, 18, 17, 11, 11, 13, 3, 2, 15, 10, 17, 2, 14, 13, 15, 15, 16, 15, 13, 14, 7, 16, 19, 3, 17, 17, 17, 14, 16, 9, 8, 14, 14, 16, 13, 12, 11, 17, 11, 1, 4, 11, 18, 16, 15, 11, 17, 15, 6, 17, 17, 13, 9, 10, 18, 17, 17, 13, 6, 1, 9, 14, 19, 3, 19, 13, 14, 10, 16, 19, 14, 9, 17, 7, 7, 11, 17, 18, 11, 17, 11, 18, 17, 16, 14, 1, 11, 16, 11, 18, 6, 1, 14, 15, 6, 17, 18, 16, 6, 13, 11, 0, 0, 16, 8, 17, 17, 10, 4, 15, 15, 15, 1, 13, 15, 14, 5, 15, 9, 1, 0, 13, 11, 1, 14, 16, 7, 2, 6, 10, 8, 14, 12, 19, 2, 7, 15, 17, 13, 17, 13, 17, 17, 19, 16, 15, 0, 12, 19, 19, 18, 15, 16, 9, 15, 16, 16, 12, 9, 9, 18, 4, 18, 11, 17, 15, 12, 12, 17, 17, 15, 19, 15, 19, 17, 15, 10, 14, 19, 16, 17, 10, 10, 14, 16, 5, 13, 9, 15, 17, 15, 19, 0, 18, 14, 15, 1, 15, 5, 17, 12, 18, 10, 5, 16, 0, 15, 1, 5, 7, 11, 19, 13, 11, 17, 9, 18, 9, 4, 10, 11, 5, 15, 12, 15, 1, 18, 6, 13, 18, 9, 1, 17, 18, 17, 15, 15, 19, 3, 17, 15, 9, 11, 14, 17, 14, 15, 1, 15, 15, 10, 3, 19, 17, 14, 6, 11, 12, 15, 17, 15, 13, 3, 0, 10, 4, 11, 0, 14, 7, 0, 15, 0, 2, 17, 17, 6, 19, 14, 17, 14, 6, 15, 12, 13, 4, 1, 17, 16, 17, 13, 18, 0, 11, 10, 3, 0, 13, 15, 13, 15, 15, 13, 5, 18, 1, 11, 19, 15, 13, 13, 10, 17, 13, 14, 19, 14, 0, 5, 7, 15, 11, 1, 15, 15, 14, 15, 13, 0, 15, 18, 7, 8, 13, 11, 16, 14, 19, 15, 16, 0, 4, 17, 11, 10, 11, 5, 19, 10, 17, 7, 10, 17, 9, 11, 15, 19, 17, 11, 11, 11, 10, 16, 14, 17, 18, 13, 16, 19, 9, 11, 18, 16, 1, 13, 16, 18, 16, 11, 16, 8, 14, 14, 16, 15, 13, 15, 11, 11, 16, 2, 15, 16, 19, 8, 17, 19, 0, 16, 15, 14, 17, 9, 19, 6, 10, 16, 15, 0, 17, 10, 6, 16, 16, 17, 11, 15, 14, 10, 2, 18, 0, 18, 16, 10, 8, 3, 17, 17, 15, 11, 17, 13, 9, 18, 19, 11, 17, 11, 11, 5, 18, 17, 13, 11, 14, 13, 17, 15, 2, 0, 12, 0, 0, 14, 15, 8, 18, 9, 17, 10, 15, 13, 13, 3, 4, 16, 10, 5, 2, 2, 16, 17, 15, 7, 15, 17, 16, 12, 17, 15, 16, 15, 17, 17, 17, 18, 19, 17, 14, 18, 11, 12, 18, 6, 14, 11, 0, 15, 10, 17, 3, 11, 9, 15, 11, 9, 13, 16, 15, 13, 17, 18, 10, 4, 18, 19, 5, 19, 17, 11, 10, 19, 17, 19, 18, 10, 3, 13, 10, 16, 11, 5, 0, 3, 11, 17, 13, 14, 6, 16, 4, 19, 19, 14, 1, 14, 16, 11, 6, 18, 19, 11, 10, 11, 19, 11, 9, 19, 10, 17, 10, 18, 14, 11, 9, 0, 9, 1, 15, 11, 8, 14, 18, 11, 16, 15, 16, 6, 18, 13, 18, 2, 17, 15, 16, 15, 11, 16, 5, 15, 11, 14, 1, 16, 15, 11, 19, 16, 11, 15, 17, 15, 2, 16, 11, 7, 13, 17, 16, 7, 10, 1, 16, 1, 18, 17, 11, 17, 11, 5, 17, 1, 18, 0, 18, 16, 17, 12, 17, 11, 11, 15, 15, 12, 0, 19, 10, 17, 14, 12, 0, 19, 11, 2, 16, 19, 17, 15, 10, 5, 16, 5, 5, 13, 11, 19, 4, 15, 14, 11, 17, 15, 14, 8, 17, 3, 13, 0, 17, 0, 10, 12, 14, 6, 13, 13, 11, 10, 13, 10, 14, 0, 19, 11, 7, 10, 14, 3, 15, 3, 14, 10, 17, 4, 5, 11, 0, 19, 10, 10, 4, 1, 13, 17, 7, 5, 19, 13, 11, 14, 19, 19, 18, 2, 13, 6, 9, 0, 11, 11, 6, 11, 18, 16, 14, 10, 2, 8, 16, 5, 17, 16, 13, 18, 17, 13, 18, 15, 19, 6, 18, 17, 19, 12, 18, 9, 13, 6, 0, 15, 11, 18, 11, 0, 18, 15, 10, 10, 6, 5, 17, 11, 1, 9, 1, 14, 12, 14, 10, 15, 11, 11, 18, 2, 10, 3, 17, 18, 14, 19, 17, 15, 9, 5, 1, 9, 13, 15, 13, 10, 17, 19, 4, 17, 17, 15, 10, 17, 13, 18, 17, 11, 18, 18, 15, 15, 17, 11, 10, 8, 0, 15, 15, 14, 11, 15, 11, 8, 15, 3, 12, 8, 15, 14, 15, 10, 10, 11, 11, 3, 15, 9, 19, 17, 4, 7, 15, 4, 9, 9, 1, 6, 13, 10, 19, 9, 11, 16, 19, 17, 16, 14, 10, 9, 15, 15, 9, 5, 0, 2, 18, 9, 15, 19, 19, 17, 5, 10, 14, 15, 17, 15, 18, 17, 15, 4, 8, 13, 8, 11, 1, 15, 13, 15, 9, 9, 17, 14, 10, 14, 0, 15, 0, 17, 10, 0, 17, 0, 10, 14, 0, 12, 16, 17, 13, 13, 19, 3, 15, 0, 16, 19, 11, 14, 17, 19, 0, 14, 2, 18, 17, 15, 2, 17, 3, 11, 14, 9, 4, 3, 9, 0, 11, 13, 5, 15, 10, 14, 10, 15, 0, 9, 17, 13, 0, 5, 17, 16, 7, 0, 19, 15, 18, 15, 17, 0, 1, 15, 11, 17, 15, 9, 18, 6, 11, 17, 12, 13, 15, 18, 7, 17, 15, 15, 6, 13, 11, 18, 4, 5, 13, 7, 17, 18, 17, 15, 18, 0, 6, 5, 18, 13, 14, 2, 19, 8, 8, 10, 8, 17, 11, 15, 10, 17, 0, 19, 8, 6, 15, 12, 15, 16, 12, 19, 6, 6, 9, 14, 14, 17, 16, 16, 13, 6, 13, 15, 6, 15, 0, 10, 10, 15, 13, 14, 15, 17, 10, 15, 17, 11, 5, 9, 12, 16, 10, 13, 16, 2, 17, 6, 5, 0, 15, 9, 15, 19, 0, 11, 18, 1, 3, 18, 18, 18, 17, 0, 4, 17, 15, 15, 17, 1, 1, 15, 1, 15, 15, 16, 19, 17, 15, 2, 17, 15, 11, 16, 11, 15, 12, 15, 6, 13, 16, 19, 3, 5, 5, 7, 6, 1, 1, 16, 15, 18, 0, 15, 0, 15, 17, 19, 15, 0, 16, 14, 17, 6, 7, 15, 11, 12, 15, 15, 17, 15, 12, 17, 1, 1, 14, 3, 6, 17, 17, 18, 15, 1, 6, 10, 18, 10, 2, 11, 13, 17, 16, 3, 17, 19, 19, 18, 15, 8, 5, 16, 11, 0, 9, 13, 0, 9, 10, 1, 15, 1, 10, 17, 1, 14, 16, 1, 14, 15, 11, 14, 6, 8, 15, 9, 19, 16, 7, 17, 14, 17, 15, 18, 5, 11, 15, 2, 16, 5, 15, 9, 12, 17, 10, 15, 17, 19, 9, 1, 15, 17, 18, 3, 16, 18, 1, 13, 3, 18, 18, 15, 17, 17, 9, 19, 3, 11, 16, 17, 10, 1, 7, 19, 9, 19, 1, 17, 10, 18, 17, 8, 10, 17, 19, 13, 5, 18, 15, 11, 19, 0, 14, 11, 15, 15, 18, 18, 16, 13, 2, 16, 12, 5, 6, 0, 5, 15, 3, 16, 8, 8, 1, 17, 13, 18, 17, 17, 14, 6, 16, 14, 4, 3, 3, 3, 17, 7, 4, 14, 15, 18, 11, 16, 17, 15, 17, 6, 9, 13, 17, 6, 17, 18, 3, 17, 15, 3, 15, 18, 10, 17, 2, 18, 11, 13, 11, 15, 1, 10, 18, 19, 14, 11, 18, 15, 0, 1, 15, 17, 15, 8, 11, 18, 1, 7, 19, 10, 0, 9, 17, 9, 17, 11, 1, 19, 7, 15, 13, 17, 17, 0, 9, 16, 18, 18, 13, 15, 5, 7, 15, 15, 4, 15, 5, 5, 8, 9, 11, 13, 12, 16, 17, 17, 1, 19, 15, 14, 13, 11, 13, 13, 13, 14, 2, 9, 15, 7, 15, 2, 8, 11, 15, 11, 0, 4, 11, 15, 0, 15, 1, 5, 17, 5, 11, 5, 4, 8, 17, 3, 5, 14, 5, 13, 10, 16, 17, 9, 7, 6, 10, 10, 19, 3, 14, 17, 13, 1, 16, 0, 7, 17, 15, 0, 11, 15, 19, 16, 18, 0, 10, 17, 1, 17, 1, 15, 15, 9, 12, 14, 15, 11, 10, 11, 16, 13, 17, 15, 2, 13, 19, 19, 15, 16, 10, 19, 6, 14, 17, 13, 19, 15, 6, 17, 16, 15, 17, 18, 7, 12, 14, 17, 13, 1, 5, 9, 0, 6, 4, 17, 4, 13, 17, 16, 19, 1, 6, 14, 11, 11, 9, 14, 10, 16, 3, 12, 17, 15, 17, 1, 3, 13, 14, 10, 5, 14, 15, 6, 10, 15, 16, 14, 15, 15, 9, 1, 17, 15, 19, 17, 2, 9, 9, 1, 15, 17, 17, 15, 16, 16, 14, 9, 17, 17, 10, 4, 15, 17, 16, 18, 18, 19, 17, 3, 12, 7, 11, 3, 13, 12, 15, 1, 19, 17, 12, 16, 9, 8, 14, 0, 16, 5, 10, 15, 15, 15, 12, 17, 18, 15, 14, 2, 15, 18, 0, 15, 4, 1, 1, 1, 13, 13, 3, 16, 9, 19, 17, 12, 10, 18, 0, 0, 14, 7, 13, 14, 17, 13, 17, 1, 6, 9, 15, 4, 15, 8, 17, 18, 7, 19, 16, 9, 6, 16, 10, 14, 6, 18, 18, 11, 17, 7, 18, 13, 3, 17, 17, 15, 19, 19, 11, 0, 17, 15, 19, 18, 15, 5, 10, 0, 0, 0, 1, 13, 0, 12, 18, 8, 0, 2, 15, 17, 18, 6, 17, 13, 15, 0, 17, 18, 15, 18, 13, 19, 10, 8, 15, 2, 1, 15, 17, 15, 17, 17]\n",
    "docs_cat = [i+1 for i in true_cat_original];\n",
    "docs_ls = [[] for i in 1:20]\n",
    "for (idx, val) in enumerate(docs_cat)\n",
    "    push!(docs_ls[val], idx)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "9a717f57-50ac-4cd2-a307-34f26e42dc32",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_marriage_assigment = Dict(1=>16, 2=>9, 3=>20, 4=>5, 5=>18, 6=>10, 7=>2, 8=>1, 9=>13, 10=>4, \n",
    "                            11=>7, 12=>12, 13=>8, 14=>17, 15=>6, 16=>14, 17=>19, 18=>15, 19=>11, 20=>3);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "e0ccbf22-ead4-4ea6-93ef-fbb298abef27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Any[]"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "already_done_words = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "146247c5-7ca9-44f6-8083-a08879c490d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add this word 'perpetrators' in topic 8 === Add this word 'graphics' in topic 7 === Add this word 'windows' in topic 20 === Add this word 'drive' in topic 10 === Add this word 'apple' in topic 4 === Add this word 'int' in topic 15 === Add this word 'shipping' in topic 11 === Add this word 'car' in topic 13 === Add this word 'bike' in topic 2 === Add this word 'braves' in topic 6 === Add this word 'team' in topic 19 === Add this word 'encryption' in topic 12 === Add this word 'asente' in topic 9 === Add this word 'medical' in topic 16 === Add this word 'space' in topic 18 === Add this word 'god' in topic 1 === Add this word 'gun' in topic 14 === Add this word 'israeli' in topic 5 === Add this word 'president' in topic 17 === Add this word 'jesus' in topic 3 === "
     ]
    }
   ],
   "source": [
    "# First loop of add_word refinement\n",
    "for current_cat in 1:20\n",
    "    true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "    all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "    if absent_docs != []\n",
    "        X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_done_words)\n",
    "                word_idx = i\n",
    "                push!(already_done_words, i)\n",
    "                break\n",
    "            end\n",
    "        end\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "21b2eb79-e4f3-4baa-8db9-23e584668f2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.403893761674023\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "d83bb6ca-e784-4beb-be3d-743b979a32b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'atheism' in topic 8 === Add word 'computer' in topic 7 === Add word 'dos' in topic 20 === Add word 'drives' in topic 10 === Add word 'mac' in topic 4 === Add word 'char' in topic 15 === Add word 'sale' in topic 11 === Add word 'cars' in topic 13 === Add word 'bikes' in topic 2 === Add word 'jays' in topic 6 === Add word 'hockey' in topic 19 === Add word 'clipper' in topic 12 === Add word 'atarist' in topic 9 === Add word 'diet' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'wannabe' in topic 17 === Add word 'bitmaps' in topic 3 === "
     ]
    }
   ],
   "source": [
    "# Second loop of add_word refinement\n",
    "for current_cat in 1:20\n",
    "    true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "    all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "    if absent_docs != []\n",
    "        X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_done_words)\n",
    "                word_idx = i\n",
    "                push!(already_done_words, i)\n",
    "                break\n",
    "            end\n",
    "        end\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "01b907c0-9fc1-43df-933e-e2a9da647120",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4056407508502718\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "1e2412a5-84db-45cf-aec5-1fc4014f451c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'atheist' in topic 8 === Add word 'image' in topic 7 === Add word 'applications' in topic 20 === Add word 'ide' in topic 10 === Add word 'iisi' in topic 4 === Add word 'function' in topic 15 === Add word 'secretaries' in topic 11 === Add word 'engine' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'marlins' in topic 6 === Add word 'nhl' in topic 19 === Add word 'secure' in topic 12 === Add word 'gslv' in topic 9 === Add word 'patients' in topic 16 === Add word 'sky' in topic 18 === Add word 'christianity' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'care' in topic 17 === Add word 'brings' in topic 3 === "
     ]
    }
   ],
   "source": [
    "# Third loop of add_word refinement\n",
    "for current_cat in 1:20\n",
    "    true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "    all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "    if absent_docs != []\n",
    "        X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_done_words)\n",
    "                word_idx = i\n",
    "                push!(already_done_words, i)\n",
    "                break\n",
    "            end\n",
    "        end\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "113a47db-7c85-4d9f-bca7-74900de523c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4153138619598633\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "f5467791-eeee-4c0e-af23-3cec110c8666",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'belief' in topic 8 === Add word 'manipulation' in topic 7 === Add word 'file' in topic 20 === Add word 'controller' in topic 10 === Add word 'macs' in topic 4 === Add word 'binsh' in topic 15 === Add word 'flmurdps' in topic 11 === Add word 'ford' in topic 13 === Add word 'riding' in topic 2 === Add word 'phillies' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'crypto' in topic 12 === Add word 'kinetic' in topic 9 === Add word 'medicine' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'christians' in topic 1 === Add word 'weapons' in topic 14 === Add word 'palestinians' in topic 5 === Add word 'economy' in topic 17 === Add word 'christ' in topic 3 === "
     ]
    }
   ],
   "source": [
    "# Fourth loop of add_word refinement\n",
    "for current_cat in 1:20\n",
    "    true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "    all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "    if absent_docs != []\n",
    "        X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_done_words)\n",
    "                word_idx = i\n",
    "                push!(already_done_words, i)\n",
    "                break\n",
    "            end\n",
    "        end\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "83c0e4e3-4891-44a6-bb46-40fa16967041",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.41403009246617767\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "cfce4e43-c9a9-4a60-b09b-51061b46ef8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'atheists' in topic 8 === Add word 'processing' in topic 7 === Add word 'compmailmh' in topic 20 === Add word 'card' in topic 10 === Add word 'shading' in topic 4 === Add word 'ped' in topic 15 === Add word 'cod' in topic 11 === Add word 'glass' in topic 13 === Add word 'ride' in topic 2 === Add word 'royals' in topic 6 === Add word 'drugresistant' in topic 19 === Add word 'privacy' in topic 12 === Add word 'snf' in topic 9 === Add word 'disease' in topic 16 === Add word 'orbit' in topic 18 === Add word 'christian' in topic 1 === Add word 'handgun' in topic 14 === Add word 'jews' in topic 5 === Add word 'taxes' in topic 17 === Add word 'abiding' in topic 3 === "
     ]
    }
   ],
   "source": [
    "# Fifth loop of add_word refinement\n",
    "for current_cat in 1:20\n",
    "    true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "    all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "    if absent_docs != []\n",
    "        X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "        word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "        for i in word_idxs\n",
    "            if !(i in already_done_words)\n",
    "                word_idx = i\n",
    "                push!(already_done_words, i)\n",
    "                break\n",
    "            end\n",
    "        end\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        wrd = corpus.reverse_vocabulary[word_idx]\n",
    "        print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "        TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "3c24f4e4-eefb-4eba-9a57-259b873e7659",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3940091516828013\n"
     ]
    }
   ],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "9aa228c5-b2fa-4855-870d-d0c1cc30fa11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'triggered' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'controller' in topic 10 === Add word 'apple' in topic 4 === Add word 'char' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'reds' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'understandablemost' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'president' in topic 17 === Add word 'jesus' in topic 3 === 0.3932848847864422\n",
      "Add word 'atheism' in topic 8 === Add word 'computer' in topic 7 === Add word 'unixxenix' in topic 20 === Add word 'drives' in topic 10 === Add word 'mac' in topic 4 === Add word 'window' in topic 15 === Add word 'waythey' in topic 11 === Add word 'cars' in topic 13 === Add word 'bikes' in topic 2 === Add word 'sox' in topic 6 === Add word 'team' in topic 19 === Add word 'clipper' in topic 12 === Add word 'electronics' in topic 9 === Add word 'medicine' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'senate' in topic 17 === Add word 'brucepartington' in topic 3 === 0.40092333977032685\n",
      "Add word 'belief' in topic 8 === Add word 'processing' in topic 7 === Add word 'wod' in topic 20 === Add word 'ide' in topic 10 === Add word 'iisi' in topic 4 === Add word 'widget' in topic 15 === Add word 'tompkins' in topic 11 === Add word 'engine' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'jays' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'privacy' in topic 12 === Add word 'carrie' in topic 9 === Add word 'diet' in topic 16 === Add word 'sky' in topic 18 === Add word 'christianity' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'health' in topic 17 === Add word 'life' in topic 3 === 0.4003559472398823\n",
      "Add word 'atheist' in topic 8 === Add word 'animation' in topic 7 === Add word 'nestorian' in topic 20 === Add word 'drive' in topic 10 === Add word 'macs' in topic 4 === Add word 'motif' in topic 15 === Add word 'mimirsteinuwashingtonedu' in topic 11 === Add word 'ford' in topic 13 === Add word 'riding' in topic 2 === Add word 'phillies' in topic 6 === Add word 'player' in topic 19 === Add word 'crypto' in topic 12 === Add word 'voltage' in topic 9 === Add word 'patients' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'christians' in topic 1 === Add word 'weapons' in topic 14 === Add word 'jews' in topic 5 === Add word 'xfedor' in topic 17 === Add word 'christian' in topic 3 === 0.4072625924296216\n",
      "Add word 'atheists' in topic 8 === Add word 'geometric' in topic 7 === Add word 'kierski' in topic 20 === Add word 'card' in topic 10 === Add word 'citizenry' in topic 4 === Add word 'set' in topic 15 === Add word 'sale' in topic 11 === Add word 'miles' in topic 13 === Add word 'ride' in topic 2 === Add word 'atlanta' in topic 6 === Add word 'nhl' in topic 19 === Add word 'nsa' in topic 12 === Add word 'tomlin' in topic 9 === Add word 'treatment' in topic 16 === Add word 'waynor' in topic 18 === Add word 'christ' in topic 1 === Add word 'weapon' in topic 14 === Add word 'palestinian' in topic 5 === Add word 'jobs' in topic 17 === Add word 'federally' in topic 3 === 0.38704709217417765\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "id": "98d62e19-e05d-46dc-9270-599ef9ac914d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'oceania' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'apple' in topic 4 === Add word 'window' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'cubs' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'alone' in topic 9 === Add word 'diet' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'president' in topic 17 === Add word 'christian' in topic 3 === 0.39749908619344326\n",
      "Add word 'atheism' in topic 8 === Add word 'computer' in topic 7 === Add word 'apps' in topic 20 === Add word 'controller' in topic 10 === Add word 'mac' in topic 4 === Add word 'widget' in topic 15 === Add word 'sale' in topic 11 === Add word 'cars' in topic 13 === Add word 'bikes' in topic 2 === Add word 'cincinnati' in topic 6 === Add word 'team' in topic 19 === Add word 'clipper' in topic 12 === Add word 'jancene' in topic 9 === Add word 'medicine' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'sandia' in topic 17 === Add word 'pwdstatsstino' in topic 3 === 0.3985306041868614\n",
      "Add word 'atheists' in topic 8 === Add word 'animation' in topic 7 === Add word 'nonpps' in topic 20 === Add word 'drives' in topic 10 === Add word 'internal' in topic 4 === Add word 'motif' in topic 15 === Add word 'barometric' in topic 11 === Add word 'engine' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'braves' in topic 6 === Add word 'nhl' in topic 19 === Add word 'secure' in topic 12 === Add word 'exegesis' in topic 9 === Add word 'medical' in topic 16 === Add word 'sky' in topic 18 === Add word 'christians' in topic 1 === Add word 'firearms' in topic 14 === Add word 'jews' in topic 5 === Add word 'compressive' in topic 17 === Add word 'bzztwrong' in topic 3 === 0.3991400658298733\n",
      "Add word 'atheist' in topic 8 === Add word 'jpeg' in topic 7 === Add word 'bulldozerlet' in topic 20 === Add word 'drive' in topic 10 === Add word 'macs' in topic 4 === Add word 'xterm' in topic 15 === Add word 'comercial' in topic 11 === Add word 'ford' in topic 13 === Add word 'riding' in topic 2 === Add word 'marlins' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'encrypted' in topic 12 === Add word 'voltage' in topic 9 === Add word 'patients' in topic 16 === Add word 'orbit' in topic 18 === Add word 'christ' in topic 1 === Add word 'weapons' in topic 14 === Add word 'arab' in topic 5 === Add word 'health' in topic 17 === Add word 'wiklund' in topic 3 === 0.3979569071905407\n",
      "Add word 'detriot' in topic 8 === Add word 'sgi' in topic 7 === Add word 'joemullen' in topic 20 === Add word 'scsi' in topic 10 === Add word 'overproliferation' in topic 4 === Add word 'mit' in topic 15 === Add word 'offer' in topic 11 === Add word 'bach' in topic 13 === Add word 'ride' in topic 2 === Add word 'milwaukee' in topic 6 === Add word 'season' in topic 19 === Add word 'privacy' in topic 12 === Add word 'incllined' in topic 9 === Add word 'clinical' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'sin' in topic 1 === Add word 'assault' in topic 14 === Add word 'jewish' in topic 5 === Add word 'clientmessage' in topic 17 === Add word 'maxson' in topic 3 === 0.38035218358692124\n",
      "140814.613781 seconds (175.86 G allocations: 3.053 TiB, 54.12% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "142725.4709019661"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "51c8af87-f7b2-46c5-bead-306918171a19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'altatheismmoderated' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'drives' in topic 10 === Add word 'apple' in topic 4 === Add word 'window' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'jays' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'sexmen' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'care' in topic 17 === Add word 'jesus' in topic 3 === 0.40015648990000696\n",
      "Add word 'atheism' in topic 8 === Add word 'computer' in topic 7 === Add word 'applications' in topic 20 === Add word 'ide' in topic 10 === Add word 'mac' in topic 4 === Add word 'widget' in topic 15 === Add word 'jective' in topic 11 === Add word 'cars' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'rams' in topic 6 === Add word 'team' in topic 19 === Add word 'clipper' in topic 12 === Add word 'hypnotist' in topic 9 === Add word 'patients' in topic 16 === Add word 'nasa' in topic 18 === Add word 'christ' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'non' in topic 17 === Add word 'escort' in topic 3 === 0.3967575923015562\n",
      "Add word 'entail' in topic 8 === Add word 'geometric' in topic 7 === Add word 'falsifies' in topic 20 === Add word 'drive' in topic 10 === Add word 'iisi' in topic 4 === Add word 'widgets' in topic 15 === Add word 'sale' in topic 11 === Add word 'engine' in topic 13 === Add word 'riding' in topic 2 === Add word 'braves' in topic 6 === Add word 'nhl' in topic 19 === Add word 'cryptography' in topic 12 === Add word 'marginaly' in topic 9 === Add word 'diet' in topic 16 === Add word 'sky' in topic 18 === Add word 'christians' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'health' in topic 17 === Add word 'lefttitlebutton' in topic 3 === 0.4127741950674541\n",
      "Add word 'atheists' in topic 8 === Add word 'image' in topic 7 === Add word 'sewell' in topic 20 === Add word 'card' in topic 10 === Add word 'tossed' in topic 4 === Add word 'default' in topic 15 === Add word 'sedsl' in topic 11 === Add word 'driving' in topic 13 === Add word 'bikes' in topic 2 === Add word 'jtchernocfberkeleyedu' in topic 6 === Add word 'play' in topic 19 === Add word 'crypto' in topic 12 === Add word 'marmet' in topic 9 === Add word 'medicine' in topic 16 === Add word 'responsibledeceiver' in topic 18 === Add word 'christianity' in topic 1 === Add word 'weapons' in topic 14 === Add word 'palestinian' in topic 5 === Add word 'president' in topic 17 === Add word 'hysteria' in topic 3 === 0.4181579601507143\n",
      "Add word 'atheist' in topic 8 === Add word 'processing' in topic 7 === Add word 'patrik' in topic 20 === Add word 'controller' in topic 10 === Add word 'dkims' in topic 4 === Add word 'motif' in topic 15 === Add word 'unitdisk' in topic 11 === Add word 'miles' in topic 13 === Add word 'ride' in topic 2 === Add word 'streak' in topic 6 === Add word 'games' in topic 19 === Add word 'privacy' in topic 12 === Add word 'willmanbell' in topic 9 === Add word 'physician' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'bible' in topic 1 === Add word 'handguns' in topic 14 === Add word 'palestinians' in topic 5 === Add word 'mover' in topic 17 === Add word 'glycogen' in topic 3 === 0.3950121012073577\n",
      "131973.229970 seconds (175.90 G allocations: 3.053 TiB, 52.30% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "134361.76042699814"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "65a207dc-ab8d-48f1-8149-114d38a6438d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'hypothesis' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'apple' in topic 4 === Add word 'function' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'reds' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'washington' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'taxes' in topic 17 === Add word 'christian' in topic 3 === 0.39750901038836106\n",
      "Add word 'atheism' in topic 8 === Add word 'animation' in topic 7 === Add word 'dos' in topic 20 === Add word 'controller' in topic 10 === Add word 'mac' in topic 4 === Add word 'window' in topic 15 === Add word 'cod' in topic 11 === Add word 'engine' in topic 13 === Add word 'bikes' in topic 2 === Add word 'giants' in topic 6 === Add word 'nhl' in topic 19 === Add word 'clipper' in topic 12 === Add word 'faceplants' in topic 9 === Add word 'medicine' in topic 16 === Add word 'nasa' in topic 18 === Add word 'christ' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'president' in topic 17 === Add word 'pros' in topic 3 === 0.40034954210373075\n",
      "Add word 'atheist' in topic 8 === Add word 'geometric' in topic 7 === Add word 'xlispbased' in topic 20 === Add word 'drives' in topic 10 === Add word 'ends' in topic 4 === Add word 'widget' in topic 15 === Add word 'distributors' in topic 11 === Add word 'cars' in topic 13 === Add word 'riding' in topic 2 === Add word 'phillies' in topic 6 === Add word 'team' in topic 19 === Add word 'crypto' in topic 12 === Add word 'petur' in topic 9 === Add word 'patients' in topic 16 === Add word 'sky' in topic 18 === Add word 'christianity' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'diagram' in topic 17 === Add word 'overlappipping' in topic 3 === 0.4183918647728715\n",
      "Add word 'assertion' in topic 8 === Add word 'computer' in topic 7 === Add word 'embarrassed' in topic 20 === Add word 'drive' in topic 10 === Add word 'kakou' in topic 4 === Add word 'widgets' in topic 15 === Add word 'price' in topic 11 === Add word 'ford' in topic 13 === Add word 'ride' in topic 2 === Add word 'braves' in topic 6 === Add word 'blainegbixcom' in topic 19 === Add word 'cryptography' in topic 12 === Add word 'glassware' in topic 9 === Add word 'diet' in topic 16 === Add word 'solar' in topic 18 === Add word 'bible' in topic 1 === Add word 'weapons' in topic 14 === Add word 'armenian' in topic 5 === Add word 'jobs' in topic 17 === Add word 'faith' in topic 3 === 0.42064128447105126\n",
      "Add word 'atheists' in topic 8 === Add word 'processing' in topic 7 === Add word 'motion' in topic 20 === Add word 'software' in topic 10 === Add word 'commandmentsdiscourses' in topic 4 === Add word 'code' in topic 15 === Add word 'sale' in topic 11 === Add word 'driving' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'jays' in topic 6 === Add word 'secotro' in topic 19 === Add word 'privacy' in topic 12 === Add word 'doctutorials' in topic 9 === Add word 'clinical' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'christians' in topic 1 === Add word 'crime' in topic 14 === Add word 'palestinian' in topic 5 === Add word 'health' in topic 17 === Add word 'yvon' in topic 3 === 0.4149658317057459\n",
      "144028.201743 seconds (175.89 G allocations: 3.053 TiB, 55.83% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "144033.13406205177"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "id": "f58c20e0-fb8c-40b8-baf2-d2e165dcfc89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'resumes' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'controller' in topic 10 === Add word 'apple' in topic 4 === Add word 'char' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'braves' in topic 6 === Add word 'team' in topic 19 === Add word 'encryption' in topic 12 === Add word 'circuit' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'house' in topic 17 === Add word 'paidophthoreseis' in topic 3 === 0.4065116551879326\n",
      "Add word 'atheism' in topic 8 === Add word 'image' in topic 7 === Add word 'dos' in topic 20 === Add word 'ide' in topic 10 === Add word 'mac' in topic 4 === Add word 'exit' in topic 15 === Add word 'instantiated' in topic 11 === Add word 'cars' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'streak' in topic 6 === Add word 'hockey' in topic 19 === Add word 'crypto' in topic 12 === Add word 'faddishness' in topic 9 === Add word 'patients' in topic 16 === Add word 'nasa' in topic 18 === Add word 'christians' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'president' in topic 17 === Add word 'jesus' in topic 3 === 0.4076083639296397\n",
      "Add word 'atheist' in topic 8 === Add word 'animation' in topic 7 === Add word 'laz' in topic 20 === Add word 'drive' in topic 10 === Add word 'sidewalls' in topic 4 === Add word 'int' in topic 15 === Add word 'cod' in topic 11 === Add word 'engine' in topic 13 === Add word 'bikes' in topic 2 === Add word 'jays' in topic 6 === Add word 'nhl' in topic 19 === Add word 'clipper' in topic 12 === Add word 'exercises' in topic 9 === Add word 'diet' in topic 16 === Add word 'sky' in topic 18 === Add word 'bible' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'health' in topic 17 === Add word 'tracy' in topic 3 === 0.42030407736286896\n",
      "Add word 'morals' in topic 8 === Add word 'computer' in topic 7 === Add word 'applications' in topic 20 === Add word 'drives' in topic 10 === Add word 'spelling' in topic 4 === Add word 'receive' in topic 15 === Add word 'gmbh' in topic 11 === Add word 'ford' in topic 13 === Add word 'ride' in topic 2 === Add word 'expos' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'nsa' in topic 12 === Add word 'used' in topic 9 === Add word 'medicine' in topic 16 === Add word 'solar' in topic 18 === Add word 'christianity' in topic 1 === Add word 'weapons' in topic 14 === Add word 'palestinian' in topic 5 === Add word 'senate' in topic 17 === Add word 'rclocal' in topic 3 === 0.4124661336098827\n",
      "Add word 'theism' in topic 8 === Add word 'visualization' in topic 7 === Add word 'file' in topic 20 === Add word 'meyers' in topic 10 === Add word 'iisi' in topic 4 === Add word 'darken' in topic 15 === Add word 'csync' in topic 11 === Add word 'driving' in topic 13 === Add word 'riding' in topic 2 === Add word 'milwaukee' in topic 6 === Add word 'wingbunderdaleunisaeduau' in topic 19 === Add word 'cryptography' in topic 12 === Add word 'exsoviet' in topic 9 === Add word 'clinical' in topic 16 === Add word 'orbit' in topic 18 === Add word 'faith' in topic 1 === Add word 'weapon' in topic 14 === Add word 'jewish' in topic 5 === Add word 'taxes' in topic 17 === Add word 'clarisportshockey' in topic 3 === 0.39084271477014443\n",
      "150040.703445 seconds (175.85 G allocations: 3.052 TiB, 56.97% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "156098.35910701752"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "id": "ef67f739-8efb-4dc1-aaef-e2d29f549923",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'book' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'mac' in topic 4 === Add word 'char' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'marlins' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'lovely' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'economic' in topic 17 === Add word 'jesus' in topic 3 === 0.3926946200601344\n",
      "Add word 'atheism' in topic 8 === Add word 'fax' in topic 7 === Add word 'applications' in topic 20 === Add word 'drive' in topic 10 === Add word 'apple' in topic 4 === Add word 'window' in topic 15 === Add word 'sale' in topic 11 === Add word 'cars' in topic 13 === Add word 'bikes' in topic 2 === Add word 'jays' in topic 6 === Add word 'nhl' in topic 19 === Add word 'clipper' in topic 12 === Add word 'creditors' in topic 9 === Add word 'patients' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'president' in topic 17 === Add word 'bits' in topic 3 === 0.39517955525469517\n",
      "Add word 'atheists' in topic 8 === Add word 'supercomputing' in topic 7 === Add word 'keysyms' in topic 20 === Add word 'drives' in topic 10 === Add word 'iisi' in topic 4 === Add word 'widget' in topic 15 === Add word 'hardwick' in topic 11 === Add word 'engine' in topic 13 === Add word 'ride' in topic 2 === Add word 'royals' in topic 6 === Add word 'team' in topic 19 === Add word 'crypto' in topic 12 === Add word 'circuits' in topic 9 === Add word 'diet' in topic 16 === Add word 'sky' in topic 18 === Add word 'christ' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'health' in topic 17 === Add word 'inhabiting' in topic 3 === 0.4143439587658801\n",
      "Add word 'theism' in topic 8 === Add word 'jpeg' in topic 7 === Add word 'undermine' in topic 20 === Add word 'controller' in topic 10 === Add word 'cpu' in topic 4 === Add word 'motif' in topic 15 === Add word 'reclassified' in topic 11 === Add word 'ford' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'phillies' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'privacy' in topic 12 === Add word 'voltage' in topic 9 === Add word 'medicine' in topic 16 === Add word 'orbit' in topic 18 === Add word 'christians' in topic 1 === Add word 'weapons' in topic 14 === Add word 'armenians' in topic 5 === Add word 'edt' in topic 17 === Add word 'prejudice' in topic 3 === 0.4320516300634598\n",
      "Add word 'atheist' in topic 8 === Add word 'geometric' in topic 7 === Add word 'nearlycriminal' in topic 20 === Add word 'card' in topic 10 === Add word 'pasquariello' in topic 4 === Add word 'default' in topic 15 === Add word 'offer' in topic 11 === Add word 'brake' in topic 13 === Add word 'riding' in topic 2 === Add word 'reds' in topic 6 === Add word 'storming' in topic 19 === Add word 'nsa' in topic 12 === Add word 'amp' in topic 9 === Add word 'clinical' in topic 16 === Add word 'shuttle' in topic 18 === Add word 'church' in topic 1 === Add word 'weapon' in topic 14 === Add word 'jews' in topic 5 === Add word 'jobs' in topic 17 === Add word 'fractal' in topic 3 === 0.4160004522226376\n",
      "144357.488307 seconds (175.86 G allocations: 3.053 TiB, 55.75% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "144362.32349085808"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "id": "b9b5559f-f27c-4a6f-9acd-87c71dd61a85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'mapssolar' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'apple' in topic 4 === Add word 'judgestoadcom' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'cubs' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'insult' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'economic' in topic 17 === Add word 'bible' in topic 3 === 0.40247930225341677\n",
      "Add word 'atheism' in topic 8 === Add word 'computer' in topic 7 === Add word 'dos' in topic 20 === Add word 'drive' in topic 10 === Add word 'mac' in topic 4 === Add word 'window' in topic 15 === Add word 'cod' in topic 11 === Add word 'cars' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'marlins' in topic 6 === Add word 'nhl' in topic 19 === Add word 'clipper' in topic 12 === Add word 'rossi' in topic 9 === Add word 'diet' in topic 16 === Add word 'nasa' in topic 18 === Add word 'christianity' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'reform' in topic 17 === Add word 'seqence' in topic 3 === 0.40476296884771096\n",
      "Add word 'atheist' in topic 8 === Add word 'jpeg' in topic 7 === Add word 'kid' in topic 20 === Add word 'drives' in topic 10 === Add word 'ram' in topic 4 === Add word 'widget' in topic 15 === Add word 'price' in topic 11 === Add word 'engine' in topic 13 === Add word 'bikes' in topic 2 === Add word 'phillies' in topic 6 === Add word 'team' in topic 19 === Add word 'crypto' in topic 12 === Add word 'circuit' in topic 9 === Add word 'medicine' in topic 16 === Add word 'sky' in topic 18 === Add word 'christians' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'gabe' in topic 17 === Add word 'jesus' in topic 3 === 0.4143749523910169\n",
      "Add word 'religion' in topic 8 === Add word 'applications' in topic 7 === Add word 'inspection' in topic 20 === Add word 'controller' in topic 10 === Add word 'iisi' in topic 4 === Add word 'code' in topic 15 === Add word 'leader' in topic 11 === Add word 'wheel' in topic 13 === Add word 'riding' in topic 2 === Add word 'rockies' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'privacy' in topic 12 === Add word 'hondal' in topic 9 === Add word 'patients' in topic 16 === Add word 'document' in topic 18 === Add word 'gigabit' in topic 1 === Add word 'weapons' in topic 14 === Add word 'palestinian' in topic 5 === Add word 'health' in topic 17 === Add word 'keum' in topic 3 === 0.42727136329091736\n",
      "Add word 'belief' in topic 8 === Add word 'processing' in topic 7 === Add word 'sodomboth' in topic 20 === Add word 'scsi' in topic 10 === Add word 'gunners' in topic 4 === Add word 'motif' in topic 15 === Add word 'sale' in topic 11 === Add word 'tires' in topic 13 === Add word 'ride' in topic 2 === Add word 'reds' in topic 6 === Add word 'shrontz' in topic 19 === Add word 'cryptography' in topic 12 === Add word 'lengthprefix' in topic 9 === Add word 'physicians' in topic 16 === Add word 'thoughtless' in topic 18 === Add word 'christ' in topic 1 === Add word 'handguns' in topic 14 === Add word 'armenians' in topic 5 === Add word 'president' in topic 17 === Add word 'since' in topic 3 === 0.4091887450895283\n",
      "171086.335821 seconds (175.89 G allocations: 3.053 TiB, 61.84% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "179257.42425704002"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "id": "d5e93b90-c0b2-4a3b-be2f-39a2d661b335",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'injectionstyle' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'apple' in topic 4 === Add word 'char' in topic 15 === Add word 'cod' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'reds' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'calsnow' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'president' in topic 17 === Add word 'jesus' in topic 3 === 0.3908968075803419\n",
      "Add word 'atheism' in topic 8 === Add word 'computational' in topic 7 === Add word 'dodging' in topic 20 === Add word 'drives' in topic 10 === Add word 'mac' in topic 4 === Add word 'widgets' in topic 15 === Add word 'shipping' in topic 11 === Add word 'cars' in topic 13 === Add word 'bikes' in topic 2 === Add word 'sox' in topic 6 === Add word 'team' in topic 19 === Add word 'clipper' in topic 12 === Add word 'machel' in topic 9 === Add word 'diet' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'reform' in topic 17 === Add word 'christian' in topic 3 === 0.40664819912120187\n",
      "Add word 'claim' in topic 8 === Add word 'jpeg' in topic 7 === Add word 'splitdesign' in topic 20 === Add word 'drive' in topic 10 === Add word 'iisi' in topic 4 === Add word 'window' in topic 15 === Add word 'melero' in topic 11 === Add word 'engine' in topic 13 === Add word 'riding' in topic 2 === Add word 'jays' in topic 6 === Add word 'nhl' in topic 19 === Add word 'crypto' in topic 12 === Add word 'alterius' in topic 9 === Add word 'medicine' in topic 16 === Add word 'nestorius' in topic 18 === Add word 'christians' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'jobs' in topic 17 === Add word 'populationthe' in topic 3 === 0.4051047487067903\n",
      "Add word 'atheist' in topic 8 === Add word 'processing' in topic 7 === Add word 'suspicions' in topic 20 === Add word 'controller' in topic 10 === Add word 'quadras' in topic 4 === Add word 'motif' in topic 15 === Add word 'sale' in topic 11 === Add word 'ford' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'braves' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'keys' in topic 12 === Add word 'roundofferror' in topic 9 === Add word 'patients' in topic 16 === Add word 'tatars' in topic 18 === Add word 'sulfur' in topic 1 === Add word 'weapons' in topic 14 === Add word 'jews' in topic 5 === Add word 'health' in topic 17 === Add word 'promulgator' in topic 3 === 0.4080623888576493\n",
      "Add word 'argument' in topic 8 === Add word 'computer' in topic 7 === Add word 'spred' in topic 20 === Add word 'card' in topic 10 === Add word 'ansiccoptions' in topic 4 === Add word 'widget' in topic 15 === Add word 'statue' in topic 11 === Add word 'driving' in topic 13 === Add word 'ride' in topic 2 === Add word 'giants' in topic 6 === Add word 'player' in topic 19 === Add word 'encrypted' in topic 12 === Add word 'toes' in topic 9 === Add word 'clinical' in topic 16 === Add word 'sky' in topic 18 === Add word 'christ' in topic 1 === Add word 'fbi' in topic 14 === Add word 'arabs' in topic 5 === Add word 'lim' in topic 17 === Add word 'neighbor' in topic 3 === 0.38313927144910387\n",
      "209538.381378 seconds (175.86 G allocations: 3.053 TiB, 68.44% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "210512.29003691673"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "7b300dcc-4652-4176-a05d-bb2851911c28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add word 'jbig' in topic 8 === Add word 'graphics' in topic 7 === Add word 'windows' in topic 20 === Add word 'ide' in topic 10 === Add word 'mac' in topic 4 === Add word 'window' in topic 15 === Add word 'shipping' in topic 11 === Add word 'car' in topic 13 === Add word 'bike' in topic 2 === Add word 'jays' in topic 6 === Add word 'hockey' in topic 19 === Add word 'encryption' in topic 12 === Add word 'circuit' in topic 9 === Add word 'medical' in topic 16 === Add word 'space' in topic 18 === Add word 'god' in topic 1 === Add word 'gun' in topic 14 === Add word 'israeli' in topic 5 === Add word 'reform' in topic 17 === Add word 'jesus' in topic 3 === 0.3953775497555705\n",
      "Add word 'atheism' in topic 8 === Add word 'animation' in topic 7 === Add word 'dos' in topic 20 === Add word 'drives' in topic 10 === Add word 'apple' in topic 4 === Add word 'widget' in topic 15 === Add word 'beads' in topic 11 === Add word 'engine' in topic 13 === Add word 'ride' in topic 2 === Add word 'marlins' in topic 6 === Add word 'team' in topic 19 === Add word 'cryptography' in topic 12 === Add word 'watt' in topic 9 === Add word 'medicine' in topic 16 === Add word 'nasa' in topic 18 === Add word 'bible' in topic 1 === Add word 'guns' in topic 14 === Add word 'israel' in topic 5 === Add word 'president' in topic 17 === Add word 'ahhh' in topic 3 === 0.39830371508844964\n",
      "Add word 'atheist' in topic 8 === Add word 'image' in topic 7 === Add word 'congresscritters' in topic 20 === Add word 'controller' in topic 10 === Add word 'quadra' in topic 4 === Add word 'code' in topic 15 === Add word 'cod' in topic 11 === Add word 'cars' in topic 13 === Add word 'motorcycle' in topic 2 === Add word 'phillies' in topic 6 === Add word 'nhl' in topic 19 === Add word 'clipper' in topic 12 === Add word 'unthankful' in topic 9 === Add word 'disease' in topic 16 === Add word 'sky' in topic 18 === Add word 'christians' in topic 1 === Add word 'firearms' in topic 14 === Add word 'arab' in topic 5 === Add word 'care' in topic 17 === Add word 'writings' in topic 3 === 0.3979474599685351\n",
      "Add word 'atheists' in topic 8 === Add word 'computer' in topic 7 === Add word 'cul' in topic 20 === Add word 'drive' in topic 10 === Add word 'macs' in topic 4 === Add word 'motif' in topic 15 === Add word 'things' in topic 11 === Add word 'ford' in topic 13 === Add word 'bikes' in topic 2 === Add word 'cardinals' in topic 6 === Add word 'playoffs' in topic 19 === Add word 'crypto' in topic 12 === Add word 'peaceably' in topic 9 === Add word 'diet' in topic 16 === Add word 'onevolume' in topic 18 === Add word 'christianity' in topic 1 === Add word 'weapons' in topic 14 === Add word 'arabs' in topic 5 === Add word 'laughter' in topic 17 === Add word 'unnerstand' in topic 3 === 0.4017784767948921\n",
      "Add word 'belief' in topic 8 === Add word 'jpeg' in topic 7 === Add word 'exeter' in topic 20 === Add word 'hopes' in topic 10 === Add word 'misrepresents' in topic 4 === Add word 'default' in topic 15 === Add word 'price' in topic 11 === Add word 'seats' in topic 13 === Add word 'riding' in topic 2 === Add word 'reds' in topic 6 === Add word 'ice' in topic 19 === Add word 'privacy' in topic 12 === Add word 'comm' in topic 9 === Add word 'health' in topic 16 === Add word 'solar' in topic 18 === Add word 'church' in topic 1 === Add word 'handguns' in topic 14 === Add word 'jews' in topic 5 === Add word 'taxes' in topic 17 === Add word 'bumbry' in topic 3 === 0.3754129288652029\n",
      "260584.847378 seconds (175.88 G allocations: 3.053 TiB, 73.56% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "260593.42877984047"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_words = [];\n",
    "t0 = time();\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        true_docs, absent_docs = docs_in_and_notin_cat(lda, corpus, current_cat)\n",
    "        all_false_negatives = [i for i in 1:corpus.document_size if !(i in docs_ls[current_cat])]\n",
    "        if absent_docs != []\n",
    "            X, Y = make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "            word_idxs = sortperm(feature_selection.mutual_info_classif(X, Y), rev=true)[1:50]\n",
    "            for i in word_idxs\n",
    "                if !(i in already_done_words)\n",
    "                    word_idx = i\n",
    "                    push!(already_done_words, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            wrd = corpus.reverse_vocabulary[word_idx]\n",
    "            print(\"Add word '$(wrd)' in topic $(current_topic) === \")\n",
    "            TopicModels.addWord(lda, corpus, word_idx, current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end\n",
    "time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5080272e-6c6c-4a7a-84e8-59d79a3197e1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0ba962b-3329-4d71-be90-73dcb48421b4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "20460d3e-6de6-43ec-935b-3c8a64e85ad2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "make_PMI_dataset (generic function with 1 method)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function make_PMI_dataset(absent_docs, all_false_negatives)\n",
    "    X = []; Y = []\n",
    "    for i in absent_docs\n",
    "        push!(X, position_to_feature(i))\n",
    "        push!(Y, 1)\n",
    "    end\n",
    "    for i in all_false_negatives\n",
    "        push!(X, position_to_feature(i))\n",
    "        push!(Y, 0)\n",
    "    end\n",
    "    return X, Y\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "114b2e0e-3a4b-40bc-96e1-e7d874341a37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "position_to_feature (generic function with 1 method)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function position_to_feature(doc_idx)\n",
    "    position_vector = corpus.documents[doc_idx]\n",
    "    feature_vector = [0 for i in 1:corpus.vocab_count]\n",
    "    for (idx, val) in enumerate(position_vector)\n",
    "        feature_vector[val] +=1\n",
    "    end\n",
    "    return feature_vector\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "db15307c-6fbb-442c-b0c7-7c7ad5a2c5f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "docs_in_and_notin_cat (generic function with 1 method)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function docs_in_and_notin_cat(lda, corpus, cat_idx)\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    topic_x_top_docs = [idx for (idx, val) in enumerate(predicted_with_marriage_assigment) if val==cat_idx];\n",
    "    docs_in_cat_notin_topic = []\n",
    "    for i in docs_ls[cat_idx]\n",
    "        if !(i in topic_x_top_docs)\n",
    "            push!(docs_in_cat_notin_topic, i)\n",
    "        end\n",
    "    end\n",
    "    docs_in_cat_in_topic = [i for i in docs_ls[cat_idx] if !(i in docs_in_cat_notin_topic)]\n",
    "    return docs_in_cat_in_topic, docs_in_cat_notin_topic\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c8e0f1f5-8db2-4479-8ac9-eff48517c36c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic_of_each_doc (generic function with 1 method)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function topic_of_each_doc(lda, corpus)\n",
    "    top_topic_for_each_doc = []\n",
    "    for i in 1:corpus.document_size\n",
    "        top_val = 0\n",
    "        top_topic = 0\n",
    "        for j in 1:lda.M\n",
    "            v = TopicModels.topicPredict(lda, i, j)\n",
    "            if v>=top_val\n",
    "                top_val = v\n",
    "                top_topic = j\n",
    "            end\n",
    "        end\n",
    "        push!(top_topic_for_each_doc, top_topic)\n",
    "    end\n",
    "    return top_topic_for_each_doc\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21bc3102-565d-42d6-b768-c8c4917eb705",
   "metadata": {},
   "source": [
    "## Remove_document refinement after stable marriage algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "aba20086-d828-4792-b0a3-a56310f64a97",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cat_original = [17, 7, 18, 17, 10, 19, 16, 19, 18, 8, 13, 7, 12, 9, 18, 18, 17, 17, 15, 17, 7, 1, 19, 5, 4, 8, 19, 17, 15, 15, 17, 0, 19, 11, 17, 18, 9, 13, 19, 17, 13, 17, 19, 15, 7, 8, 11, 9, 1, 8, 0, 18, 17, 14, 14, 10, 15, 3, 8, 10, 10, 11, 17, 6, 17, 17, 15, 8, 15, 11, 2, 17, 9, 18, 17, 17, 10, 2, 9, 1, 10, 4, 9, 15, 3, 15, 19, 15, 16, 13, 17, 11, 0, 18, 10, 14, 6, 17, 15, 17, 19, 14, 8, 14, 5, 5, 11, 15, 6, 10, 5, 9, 15, 11, 9, 15, 7, 1, 14, 16, 15, 13, 13, 17, 0, 11, 16, 3, 16, 15, 14, 14, 19, 9, 18, 14, 17, 17, 4, 17, 15, 1, 16, 11, 15, 6, 2, 14, 15, 18, 8, 17, 18, 4, 7, 11, 17, 18, 9, 16, 18, 7, 11, 2, 15, 19, 1, 11, 14, 17, 1, 16, 6, 5, 19, 7, 11, 16, 8, 16, 10, 15, 17, 16, 16, 1, 0, 7, 15, 0, 6, 18, 10, 9, 11, 16, 19, 14, 6, 0, 0, 3, 17, 17, 12, 16, 8, 6, 19, 10, 17, 16, 14, 4, 16, 11, 1, 14, 0, 2, 16, 15, 15, 6, 15, 0, 14, 10, 15, 13, 2, 19, 15, 17, 15, 1, 9, 16, 18, 17, 6, 17, 8, 14, 11, 5, 15, 7, 15, 15, 12, 11, 14, 14, 3, 15, 3, 17, 3, 14, 11, 19, 10, 18, 15, 16, 15, 15, 9, 16, 11, 4, 19, 13, 3, 19, 9, 17, 18, 7, 7, 15, 16, 14, 15, 10, 5, 9, 14, 14, 17, 15, 15, 19, 14, 15, 15, 0, 4, 11, 11, 15, 12, 6, 3, 15, 14, 5, 14, 18, 11, 11, 14, 17, 17, 10, 9, 5, 7, 10, 19, 18, 17, 15, 18, 17, 11, 11, 13, 3, 2, 15, 10, 17, 2, 14, 13, 15, 15, 16, 15, 13, 14, 7, 16, 19, 3, 17, 17, 17, 14, 16, 9, 8, 14, 14, 16, 13, 12, 11, 17, 11, 1, 4, 11, 18, 16, 15, 11, 17, 15, 6, 17, 17, 13, 9, 10, 18, 17, 17, 13, 6, 1, 9, 14, 19, 3, 19, 13, 14, 10, 16, 19, 14, 9, 17, 7, 7, 11, 17, 18, 11, 17, 11, 18, 17, 16, 14, 1, 11, 16, 11, 18, 6, 1, 14, 15, 6, 17, 18, 16, 6, 13, 11, 0, 0, 16, 8, 17, 17, 10, 4, 15, 15, 15, 1, 13, 15, 14, 5, 15, 9, 1, 0, 13, 11, 1, 14, 16, 7, 2, 6, 10, 8, 14, 12, 19, 2, 7, 15, 17, 13, 17, 13, 17, 17, 19, 16, 15, 0, 12, 19, 19, 18, 15, 16, 9, 15, 16, 16, 12, 9, 9, 18, 4, 18, 11, 17, 15, 12, 12, 17, 17, 15, 19, 15, 19, 17, 15, 10, 14, 19, 16, 17, 10, 10, 14, 16, 5, 13, 9, 15, 17, 15, 19, 0, 18, 14, 15, 1, 15, 5, 17, 12, 18, 10, 5, 16, 0, 15, 1, 5, 7, 11, 19, 13, 11, 17, 9, 18, 9, 4, 10, 11, 5, 15, 12, 15, 1, 18, 6, 13, 18, 9, 1, 17, 18, 17, 15, 15, 19, 3, 17, 15, 9, 11, 14, 17, 14, 15, 1, 15, 15, 10, 3, 19, 17, 14, 6, 11, 12, 15, 17, 15, 13, 3, 0, 10, 4, 11, 0, 14, 7, 0, 15, 0, 2, 17, 17, 6, 19, 14, 17, 14, 6, 15, 12, 13, 4, 1, 17, 16, 17, 13, 18, 0, 11, 10, 3, 0, 13, 15, 13, 15, 15, 13, 5, 18, 1, 11, 19, 15, 13, 13, 10, 17, 13, 14, 19, 14, 0, 5, 7, 15, 11, 1, 15, 15, 14, 15, 13, 0, 15, 18, 7, 8, 13, 11, 16, 14, 19, 15, 16, 0, 4, 17, 11, 10, 11, 5, 19, 10, 17, 7, 10, 17, 9, 11, 15, 19, 17, 11, 11, 11, 10, 16, 14, 17, 18, 13, 16, 19, 9, 11, 18, 16, 1, 13, 16, 18, 16, 11, 16, 8, 14, 14, 16, 15, 13, 15, 11, 11, 16, 2, 15, 16, 19, 8, 17, 19, 0, 16, 15, 14, 17, 9, 19, 6, 10, 16, 15, 0, 17, 10, 6, 16, 16, 17, 11, 15, 14, 10, 2, 18, 0, 18, 16, 10, 8, 3, 17, 17, 15, 11, 17, 13, 9, 18, 19, 11, 17, 11, 11, 5, 18, 17, 13, 11, 14, 13, 17, 15, 2, 0, 12, 0, 0, 14, 15, 8, 18, 9, 17, 10, 15, 13, 13, 3, 4, 16, 10, 5, 2, 2, 16, 17, 15, 7, 15, 17, 16, 12, 17, 15, 16, 15, 17, 17, 17, 18, 19, 17, 14, 18, 11, 12, 18, 6, 14, 11, 0, 15, 10, 17, 3, 11, 9, 15, 11, 9, 13, 16, 15, 13, 17, 18, 10, 4, 18, 19, 5, 19, 17, 11, 10, 19, 17, 19, 18, 10, 3, 13, 10, 16, 11, 5, 0, 3, 11, 17, 13, 14, 6, 16, 4, 19, 19, 14, 1, 14, 16, 11, 6, 18, 19, 11, 10, 11, 19, 11, 9, 19, 10, 17, 10, 18, 14, 11, 9, 0, 9, 1, 15, 11, 8, 14, 18, 11, 16, 15, 16, 6, 18, 13, 18, 2, 17, 15, 16, 15, 11, 16, 5, 15, 11, 14, 1, 16, 15, 11, 19, 16, 11, 15, 17, 15, 2, 16, 11, 7, 13, 17, 16, 7, 10, 1, 16, 1, 18, 17, 11, 17, 11, 5, 17, 1, 18, 0, 18, 16, 17, 12, 17, 11, 11, 15, 15, 12, 0, 19, 10, 17, 14, 12, 0, 19, 11, 2, 16, 19, 17, 15, 10, 5, 16, 5, 5, 13, 11, 19, 4, 15, 14, 11, 17, 15, 14, 8, 17, 3, 13, 0, 17, 0, 10, 12, 14, 6, 13, 13, 11, 10, 13, 10, 14, 0, 19, 11, 7, 10, 14, 3, 15, 3, 14, 10, 17, 4, 5, 11, 0, 19, 10, 10, 4, 1, 13, 17, 7, 5, 19, 13, 11, 14, 19, 19, 18, 2, 13, 6, 9, 0, 11, 11, 6, 11, 18, 16, 14, 10, 2, 8, 16, 5, 17, 16, 13, 18, 17, 13, 18, 15, 19, 6, 18, 17, 19, 12, 18, 9, 13, 6, 0, 15, 11, 18, 11, 0, 18, 15, 10, 10, 6, 5, 17, 11, 1, 9, 1, 14, 12, 14, 10, 15, 11, 11, 18, 2, 10, 3, 17, 18, 14, 19, 17, 15, 9, 5, 1, 9, 13, 15, 13, 10, 17, 19, 4, 17, 17, 15, 10, 17, 13, 18, 17, 11, 18, 18, 15, 15, 17, 11, 10, 8, 0, 15, 15, 14, 11, 15, 11, 8, 15, 3, 12, 8, 15, 14, 15, 10, 10, 11, 11, 3, 15, 9, 19, 17, 4, 7, 15, 4, 9, 9, 1, 6, 13, 10, 19, 9, 11, 16, 19, 17, 16, 14, 10, 9, 15, 15, 9, 5, 0, 2, 18, 9, 15, 19, 19, 17, 5, 10, 14, 15, 17, 15, 18, 17, 15, 4, 8, 13, 8, 11, 1, 15, 13, 15, 9, 9, 17, 14, 10, 14, 0, 15, 0, 17, 10, 0, 17, 0, 10, 14, 0, 12, 16, 17, 13, 13, 19, 3, 15, 0, 16, 19, 11, 14, 17, 19, 0, 14, 2, 18, 17, 15, 2, 17, 3, 11, 14, 9, 4, 3, 9, 0, 11, 13, 5, 15, 10, 14, 10, 15, 0, 9, 17, 13, 0, 5, 17, 16, 7, 0, 19, 15, 18, 15, 17, 0, 1, 15, 11, 17, 15, 9, 18, 6, 11, 17, 12, 13, 15, 18, 7, 17, 15, 15, 6, 13, 11, 18, 4, 5, 13, 7, 17, 18, 17, 15, 18, 0, 6, 5, 18, 13, 14, 2, 19, 8, 8, 10, 8, 17, 11, 15, 10, 17, 0, 19, 8, 6, 15, 12, 15, 16, 12, 19, 6, 6, 9, 14, 14, 17, 16, 16, 13, 6, 13, 15, 6, 15, 0, 10, 10, 15, 13, 14, 15, 17, 10, 15, 17, 11, 5, 9, 12, 16, 10, 13, 16, 2, 17, 6, 5, 0, 15, 9, 15, 19, 0, 11, 18, 1, 3, 18, 18, 18, 17, 0, 4, 17, 15, 15, 17, 1, 1, 15, 1, 15, 15, 16, 19, 17, 15, 2, 17, 15, 11, 16, 11, 15, 12, 15, 6, 13, 16, 19, 3, 5, 5, 7, 6, 1, 1, 16, 15, 18, 0, 15, 0, 15, 17, 19, 15, 0, 16, 14, 17, 6, 7, 15, 11, 12, 15, 15, 17, 15, 12, 17, 1, 1, 14, 3, 6, 17, 17, 18, 15, 1, 6, 10, 18, 10, 2, 11, 13, 17, 16, 3, 17, 19, 19, 18, 15, 8, 5, 16, 11, 0, 9, 13, 0, 9, 10, 1, 15, 1, 10, 17, 1, 14, 16, 1, 14, 15, 11, 14, 6, 8, 15, 9, 19, 16, 7, 17, 14, 17, 15, 18, 5, 11, 15, 2, 16, 5, 15, 9, 12, 17, 10, 15, 17, 19, 9, 1, 15, 17, 18, 3, 16, 18, 1, 13, 3, 18, 18, 15, 17, 17, 9, 19, 3, 11, 16, 17, 10, 1, 7, 19, 9, 19, 1, 17, 10, 18, 17, 8, 10, 17, 19, 13, 5, 18, 15, 11, 19, 0, 14, 11, 15, 15, 18, 18, 16, 13, 2, 16, 12, 5, 6, 0, 5, 15, 3, 16, 8, 8, 1, 17, 13, 18, 17, 17, 14, 6, 16, 14, 4, 3, 3, 3, 17, 7, 4, 14, 15, 18, 11, 16, 17, 15, 17, 6, 9, 13, 17, 6, 17, 18, 3, 17, 15, 3, 15, 18, 10, 17, 2, 18, 11, 13, 11, 15, 1, 10, 18, 19, 14, 11, 18, 15, 0, 1, 15, 17, 15, 8, 11, 18, 1, 7, 19, 10, 0, 9, 17, 9, 17, 11, 1, 19, 7, 15, 13, 17, 17, 0, 9, 16, 18, 18, 13, 15, 5, 7, 15, 15, 4, 15, 5, 5, 8, 9, 11, 13, 12, 16, 17, 17, 1, 19, 15, 14, 13, 11, 13, 13, 13, 14, 2, 9, 15, 7, 15, 2, 8, 11, 15, 11, 0, 4, 11, 15, 0, 15, 1, 5, 17, 5, 11, 5, 4, 8, 17, 3, 5, 14, 5, 13, 10, 16, 17, 9, 7, 6, 10, 10, 19, 3, 14, 17, 13, 1, 16, 0, 7, 17, 15, 0, 11, 15, 19, 16, 18, 0, 10, 17, 1, 17, 1, 15, 15, 9, 12, 14, 15, 11, 10, 11, 16, 13, 17, 15, 2, 13, 19, 19, 15, 16, 10, 19, 6, 14, 17, 13, 19, 15, 6, 17, 16, 15, 17, 18, 7, 12, 14, 17, 13, 1, 5, 9, 0, 6, 4, 17, 4, 13, 17, 16, 19, 1, 6, 14, 11, 11, 9, 14, 10, 16, 3, 12, 17, 15, 17, 1, 3, 13, 14, 10, 5, 14, 15, 6, 10, 15, 16, 14, 15, 15, 9, 1, 17, 15, 19, 17, 2, 9, 9, 1, 15, 17, 17, 15, 16, 16, 14, 9, 17, 17, 10, 4, 15, 17, 16, 18, 18, 19, 17, 3, 12, 7, 11, 3, 13, 12, 15, 1, 19, 17, 12, 16, 9, 8, 14, 0, 16, 5, 10, 15, 15, 15, 12, 17, 18, 15, 14, 2, 15, 18, 0, 15, 4, 1, 1, 1, 13, 13, 3, 16, 9, 19, 17, 12, 10, 18, 0, 0, 14, 7, 13, 14, 17, 13, 17, 1, 6, 9, 15, 4, 15, 8, 17, 18, 7, 19, 16, 9, 6, 16, 10, 14, 6, 18, 18, 11, 17, 7, 18, 13, 3, 17, 17, 15, 19, 19, 11, 0, 17, 15, 19, 18, 15, 5, 10, 0, 0, 0, 1, 13, 0, 12, 18, 8, 0, 2, 15, 17, 18, 6, 17, 13, 15, 0, 17, 18, 15, 18, 13, 19, 10, 8, 15, 2, 1, 15, 17, 15, 17, 17]\n",
    "docs_cat = [i+1 for i in true_cat_original];\n",
    "docs_ls = [[] for i in 1:20]\n",
    "for (idx, val) in enumerate(docs_cat)\n",
    "    push!(docs_ls[val], idx)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "846db3ae-3940-42d4-838a-e9fc2d2811b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_marriage_assigment = Dict(1=>16, 2=>9, 3=>20, 4=>5, 5=>18, 6=>10, 7=>2, 8=>1, 9=>13, 10=>4, \n",
    "                            11=>7, 12=>12, 13=>8, 14=>17, 15=>6, 16=>14, 17=>19, 18=>15, 19=>11, 20=>3);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "248b9272-e6e7-48ec-828c-7282bb9ee177",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "60afddc1-38e9-4eba-957c-f4b57e68cce7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Any[]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "already_done_docs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0e4f897e-41d9-4a2a-8226-28fc8b777cba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_cat = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e191bdfa-7f6d-4ebf-934c-af7144fb3ff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Array{Any,1}:\n",
       "    6\n",
       "   64\n",
       "  666\n",
       " 1091\n",
       " 1108\n",
       " 1847"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2322a983-06aa-460d-b0cc-abdc322991d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f61af591-ba18-48a6-9536-948bc72d2b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "TopicModels.removeDoc(lda, corpus, 6, current_topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "16ff63dc-705d-4c32-b0c2-63a8ab4e038d",
   "metadata": {},
   "outputs": [],
   "source": [
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "82fb639c-93b3-4a04-9320-0fce58baa217",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a948c829-1b33-482d-801e-1cc5a231fabc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9, 10, 19, 1, 11, 1, 19, 16, 19, 8, 14, 1, 1, 10, 14, 17, 18, 16, 16, 18, 8, 1, 16, 6, 7, 1, 16, 5, 1, 1, 18, 1, 1, 12, 1, 19, 10, 14, 1, 11, 13, 19, 16, 16, 1, 10, 4, 10, 6, 1, 1, 17, 18, 15, 15, 1, 20, 7, 10, 11, 11, 12, 9, 2, 18, 19, 16, 1, 1, 12, 6, 18, 10, 1, 1, 18, 13, 6, 1, 2, 10, 8, 11, 16, 1, 16, 16, 16, 5, 13, 1, 12, 1, 1, 1, 15, 6, 19, 1, 18, 16, 15, 9, 15, 4, 6, 1, 1, 7, 11, 4, 11, 16, 1, 10, 16, 10, 6, 15, 17, 1, 1, 1, 18, 1, 8, 1, 7, 17, 16, 8, 15, 1, 10, 5, 15, 18, 18, 7, 18, 1, 6, 17, 12, 16, 7, 7, 8, 1, 17, 10, 1, 1, 7, 1, 1, 1, 1, 10, 1, 1, 8, 12, 6, 1, 1, 6, 12, 15, 18, 1, 17, 6, 4, 20, 10, 12, 19, 1, 9, 10, 16, 18, 1, 5, 6, 1, 8, 1, 16, 7, 8, 10, 10, 12, 17, 5, 15, 8, 19, 1, 7, 18, 18, 14, 1, 10, 8, 1, 11, 19, 1, 15, 7, 19, 12, 3, 15, 1, 6, 10, 16, 16, 7, 1, 1, 1, 1, 16, 5, 6, 16, 1, 18, 16, 6, 10, 19, 19, 18, 7, 18, 10, 15, 12, 4, 1, 1, 16, 1, 4, 12, 8, 15, 1, 16, 7, 18, 7, 1, 1, 1, 11, 14, 1, 17, 16, 1, 10, 1, 12, 7, 1, 1, 6, 9, 10, 19, 1, 1, 1, 9, 19, 15, 16, 11, 8, 10, 15, 17, 18, 16, 16, 16, 15, 1, 9, 1, 7, 12, 12, 1, 14, 7, 7, 1, 17, 4, 15, 13, 12, 12, 15, 18, 18, 11, 10, 6, 9, 10, 16, 19, 18, 1, 17, 18, 12, 3, 1, 7, 6, 16, 11, 9, 3, 15, 6, 16, 16, 19, 16, 14, 15, 8, 9, 16, 4, 18, 18, 19, 15, 1, 10, 10, 15, 11, 17, 1, 1, 6, 9, 12, 8, 1, 12, 1, 1, 19, 1, 18, 16, 4, 19, 9, 14, 10, 1, 1, 18, 18, 1, 7, 6, 10, 15, 1, 7, 1, 14, 20, 11, 17, 16, 8, 10, 18, 1, 1, 1, 18, 1, 12, 19, 12, 1, 18, 17, 15, 6, 12, 5, 12, 16, 7, 6, 1, 16, 5, 9, 14, 1, 1, 1, 12, 1, 1, 1, 10, 18, 18, 1, 7, 16, 16, 16, 8, 1, 1, 15, 6, 9, 10, 4, 1, 9, 19, 8, 1, 19, 8, 1, 7, 10, 10, 6, 7, 1, 1, 10, 9, 19, 1, 5, 14, 18, 19, 1, 5, 16, 1, 14, 20, 20, 17, 1, 19, 11, 16, 19, 1, 7, 10, 10, 13, 7, 1, 1, 19, 8, 11, 1, 18, 18, 16, 16, 19, 1, 18, 1, 11, 15, 9, 1, 19, 10, 13, 1, 17, 1, 14, 11, 1, 18, 1, 1, 3, 1, 15, 16, 6, 16, 6, 19, 14, 19, 11, 3, 17, 16, 1, 4, 4, 1, 6, 16, 1, 1, 18, 11, 17, 10, 7, 10, 12, 4, 16, 1, 16, 2, 17, 7, 14, 1, 10, 1, 19, 1, 19, 16, 16, 20, 1, 19, 1, 10, 1, 8, 5, 15, 16, 3, 1, 1, 10, 7, 9, 18, 15, 7, 1, 7, 1, 18, 1, 2, 4, 1, 13, 7, 19, 1, 15, 8, 20, 1, 1, 1, 18, 9, 7, 1, 15, 1, 15, 8, 16, 8, 13, 7, 1, 9, 1, 18, 1, 19, 9, 1, 10, 1, 1, 11, 16, 13, 1, 16, 13, 4, 1, 6, 12, 16, 1, 14, 13, 1, 18, 13, 15, 16, 1, 1, 6, 1, 1, 12, 6, 1, 16, 1, 1, 20, 1, 1, 8, 1, 14, 14, 1, 1, 1, 20, 16, 1, 1, 1, 2, 19, 11, 1, 6, 16, 11, 9, 10, 11, 18, 11, 1, 16, 1, 18, 1, 1, 12, 10, 19, 15, 19, 1, 14, 14, 1, 10, 12, 1, 17, 3, 9, 1, 8, 17, 12, 19, 10, 15, 15, 1, 16, 14, 1, 1, 12, 19, 3, 16, 14, 16, 10, 19, 1, 1, 17, 1, 1, 18, 10, 16, 8, 10, 17, 16, 1, 18, 11, 6, 19, 19, 18, 12, 16, 15, 1, 3, 1, 1, 14, 12, 11, 8, 7, 18, 1, 16, 12, 18, 14, 10, 17, 16, 12, 18, 1, 12, 6, 5, 18, 14, 1, 15, 8, 18, 16, 6, 16, 14, 16, 1, 1, 16, 10, 17, 10, 19, 11, 16, 13, 1, 7, 7, 19, 1, 4, 1, 2, 17, 5, 9, 8, 16, 1, 11, 1, 18, 16, 19, 16, 19, 19, 19, 1, 1, 18, 15, 17, 12, 10, 5, 7, 17, 12, 1, 1, 10, 18, 7, 1, 10, 16, 12, 1, 1, 19, 16, 14, 18, 1, 10, 7, 19, 1, 6, 1, 18, 12, 10, 1, 18, 16, 1, 11, 7, 1, 11, 17, 12, 6, 1, 7, 8, 17, 14, 8, 7, 1, 7, 1, 16, 15, 2, 15, 19, 12, 8, 1, 1, 12, 10, 1, 1, 8, 10, 16, 10, 9, 13, 17, 15, 1, 1, 1, 10, 1, 1, 12, 10, 15, 1, 1, 1, 16, 19, 11, 19, 14, 1, 3, 18, 1, 17, 1, 12, 1, 4, 16, 1, 15, 1, 19, 1, 12, 16, 17, 12, 1, 9, 3, 1, 1, 12, 14, 1, 18, 1, 8, 10, 6, 17, 6, 17, 18, 12, 18, 12, 8, 5, 20, 19, 1, 17, 5, 18, 14, 15, 12, 1, 1, 19, 14, 1, 16, 13, 19, 15, 14, 1, 16, 12, 7, 10, 16, 1, 16, 11, 6, 1, 4, 6, 11, 12, 16, 9, 16, 15, 1, 18, 1, 15, 1, 18, 7, 14, 1, 20, 19, 11, 1, 6, 7, 1, 1, 12, 10, 1, 11, 15, 1, 1, 1, 10, 10, 15, 7, 16, 7, 15, 10, 18, 7, 4, 12, 1, 1, 10, 10, 7, 11, 8, 18, 8, 4, 16, 1, 1, 15, 16, 16, 1, 7, 14, 7, 10, 1, 12, 12, 7, 12, 17, 1, 15, 10, 6, 10, 5, 6, 18, 14, 1, 19, 16, 6, 19, 16, 1, 7, 1, 18, 16, 15, 1, 11, 1, 7, 1, 16, 12, 1, 12, 1, 17, 16, 11, 11, 7, 4, 18, 19, 8, 11, 3, 1, 4, 15, 17, 1, 1, 1, 17, 7, 10, 11, 18, 1, 17, 1, 5, 16, 10, 2, 1, 1, 8, 1, 13, 10, 1, 1, 7, 19, 1, 16, 10, 18, 1, 19, 18, 12, 17, 17, 1, 16, 1, 12, 10, 1, 3, 16, 16, 15, 12, 1, 1, 8, 1, 7, 1, 10, 16, 15, 1, 11, 11, 12, 12, 7, 16, 10, 16, 18, 7, 1, 16, 7, 10, 10, 6, 7, 1, 10, 16, 10, 19, 1, 1, 1, 1, 15, 11, 10, 1, 1, 10, 4, 20, 6, 1, 10, 16, 16, 17, 19, 6, 1, 1, 16, 19, 16, 17, 19, 1, 7, 10, 14, 10, 12, 6, 1, 14, 16, 10, 1, 19, 20, 11, 20, 19, 16, 1, 1, 10, 1, 18, 16, 11, 15, 1, 7, 1, 18, 1, 14, 1, 7, 16, 1, 19, 1, 12, 15, 18, 16, 1, 15, 7, 1, 18, 5, 6, 18, 1, 12, 3, 10, 8, 7, 10, 1, 1, 1, 4, 16, 11, 1, 10, 16, 1, 1, 18, 14, 1, 1, 19, 1, 1, 16, 9, 16, 17, 1, 18, 1, 12, 16, 12, 18, 16, 10, 15, 8, 1, 18, 8, 1, 1, 19, 10, 19, 16, 16, 8, 14, 1, 1, 1, 6, 14, 10, 18, 17, 9, 16, 17, 1, 7, 6, 1, 14, 6, 7, 17, 1, 9, 10, 10, 18, 12, 16, 7, 18, 16, 6, 9, 3, 16, 14, 1, 17, 1, 9, 7, 7, 10, 15, 15, 19, 1, 17, 14, 7, 20, 16, 7, 16, 1, 10, 11, 16, 11, 20, 16, 18, 11, 16, 18, 12, 6, 10, 1, 19, 11, 1, 1, 3, 19, 7, 6, 1, 16, 10, 16, 16, 1, 12, 15, 8, 7, 20, 14, 9, 19, 1, 1, 18, 16, 16, 18, 6, 6, 16, 6, 16, 16, 19, 20, 19, 1, 6, 18, 3, 4, 1, 1, 16, 1, 16, 7, 1, 19, 1, 4, 1, 6, 1, 7, 6, 2, 5, 16, 17, 1, 16, 1, 1, 19, 16, 16, 1, 1, 20, 18, 8, 14, 16, 1, 1, 1, 1, 18, 16, 14, 19, 1, 1, 1, 7, 6, 9, 1, 1, 1, 4, 7, 11, 19, 11, 1, 12, 1, 18, 5, 7, 5, 16, 1, 17, 16, 10, 4, 19, 12, 16, 10, 13, 19, 10, 10, 6, 16, 8, 11, 9, 3, 19, 1, 8, 1, 1, 12, 15, 7, 1, 9, 10, 1, 14, 10, 1, 20, 18, 1, 9, 6, 12, 1, 1, 1, 6, 16, 1, 9, 18, 11, 16, 18, 1, 10, 3, 16, 18, 19, 7, 17, 1, 6, 14, 7, 19, 1, 16, 19, 20, 11, 1, 7, 12, 1, 5, 11, 8, 10, 16, 10, 16, 1, 18, 1, 1, 18, 9, 10, 1, 16, 14, 6, 9, 16, 1, 16, 1, 1, 1, 1, 1, 1, 19, 1, 14, 1, 1, 14, 1, 7, 19, 6, 16, 6, 9, 1, 1, 14, 18, 13, 19, 18, 17, 15, 7, 17, 15, 11, 7, 1, 6, 9, 10, 7, 1, 1, 19, 19, 14, 18, 1, 19, 7, 10, 1, 1, 7, 19, 1, 7, 18, 1, 7, 16, 9, 11, 18, 6, 1, 1, 1, 12, 1, 6, 10, 1, 16, 15, 12, 13, 16, 1, 6, 19, 17, 1, 10, 1, 1, 6, 10, 1, 13, 1, 10, 5, 1, 18, 12, 8, 16, 10, 3, 13, 19, 18, 1, 10, 17, 17, 19, 9, 16, 1, 10, 16, 16, 7, 5, 4, 6, 10, 10, 1, 13, 14, 1, 18, 17, 1, 16, 16, 1, 11, 1, 13, 9, 1, 20, 1, 1, 5, 10, 16, 6, 1, 1, 16, 12, 20, 6, 12, 16, 3, 16, 1, 6, 18, 3, 1, 6, 7, 20, 17, 1, 4, 1, 4, 1, 1, 17, 18, 10, 10, 1, 1, 10, 16, 7, 1, 18, 13, 2, 17, 3, 10, 18, 5, 1, 1, 1, 16, 1, 17, 1, 13, 18, 8, 18, 1, 1, 1, 1, 14, 15, 16, 12, 11, 1, 9, 13, 19, 20, 6, 14, 1, 1, 16, 19, 7, 1, 8, 1, 19, 14, 1, 16, 7, 9, 1, 16, 1, 1, 10, 4, 15, 19, 13, 2, 4, 10, 1, 7, 7, 1, 1, 13, 19, 1, 20, 8, 7, 15, 12, 1, 10, 15, 10, 17, 14, 7, 18, 16, 9, 6, 7, 13, 15, 11, 6, 15, 16, 10, 1, 16, 1, 1, 16, 16, 1, 1, 18, 16, 20, 18, 2, 11, 11, 6, 8, 18, 18, 16, 17, 1, 1, 10, 18, 18, 10, 7, 16, 18, 1, 1, 1, 1, 19, 1, 1, 10, 12, 7, 14, 14, 16, 2, 16, 18, 14, 9, 10, 10, 15, 1, 17, 6, 13, 16, 16, 1, 1, 18, 17, 16, 15, 6, 5, 19, 1, 16, 7, 6, 2, 8, 14, 1, 6, 1, 1, 1, 1, 1, 11, 17, 1, 1, 1, 10, 1, 19, 18, 19, 19, 6, 8, 10, 1, 1, 16, 10, 18, 19, 17, 16, 1, 10, 7, 1, 11, 15, 7, 1, 17, 12, 18, 1, 1, 14, 1, 18, 19, 16, 16, 20, 1, 1, 18, 16, 16, 17, 16, 6, 13, 1, 1, 20, 6, 13, 1, 14, 1, 8, 19, 6, 16, 18, 17, 7, 1, 1, 16, 1, 18, 17, 16, 1, 9, 16, 11, 14, 16, 6, 2, 16, 16, 9, 1, 19]"
     ]
    }
   ],
   "source": [
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "print(predicted_with_marriage_assigment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1a045bb-0d2f-4bd8-8059-aaed07363a13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6b8ea088-374e-40af-a848-79bcc0e808e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39971314050362267, 0.3811414733709356, 0.38871513138412755, 0.4074351733065137, 0.4085268669490589, 11535.291230 seconds (160.65 G allocations: 2.692 TiB, 8.69% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9f5fe8d6-3e31-4182-ad14-fef0b9bb029c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.38352985972247033, 0.38206773523990567, 0.3767813265594643, 0.3860958960217968, 0.3892384042053968, 10493.513903 seconds (162.41 G allocations: 2.721 TiB, 8.37% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b0a8da28-b847-4320-bc10-b97f040c3f32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3926436033005589, 0.37625135549017746, 0.3785444706186156, 0.3718701565424993, 0.3759927339991509, 10216.785187 seconds (158.93 G allocations: 2.663 TiB, 9.00% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "672ca935-869a-452b-957c-27cfd3d571ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3832850816953736, 0.3652210312637861, 0.37220127093713723, 0.37853255863213975, 0.390190906453969, 11384.071323 seconds (164.10 G allocations: 2.750 TiB, 8.00% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "176e7434-2940-4a2d-9e9a-ce23aec931de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3910496282633884, 0.38841475060779806, 0.3928616722019845, 0.3934339934643961, 0.399411773064334, 11093.017515 seconds (162.38 G allocations: 2.721 TiB, 8.95% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "23751da7-6fe3-43c7-b90a-fd3bad98a40d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4011463228979381, 0.39096569020994654, 0.3938450727833862, 0.4085129069940024, 0.4158062754067186, 10393.455148 seconds (158.91 G allocations: 2.663 TiB, 8.24% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "2f6037d7-9378-4af2-965b-47635282d6ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3909799004389126, 0.37566707499716767, 0.3839828677274133, 0.39190921026851766, 0.4035818346037484, 10673.713715 seconds (162.38 G allocations: 2.721 TiB, 9.01% gc time)\n"
     ]
    }
   ],
   "source": [
    "#7th iteration\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "40167890-d9c2-4ec3-a47d-e9542b4f1034",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39178711525039284, 0.3730444630265938, 0.3797254324703822, 0.38723200513414835, 0.39000128734636574, 10932.916948 seconds (160.65 G allocations: 2.692 TiB, 8.61% gc time)\n"
     ]
    }
   ],
   "source": [
    "# 8th iteration\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "bbe55815-0cc1-42e3-b5a1-e2fd2595b1c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3889407200505525, 0.3756896025923708, 0.37995306226815645, 0.38817729262408635, 0.39861900850553117, 10742.241071 seconds (160.67 G allocations: 2.692 TiB, 9.10% gc time)\n"
     ]
    }
   ],
   "source": [
    "# 9th iteration\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "dd2cc36b-8e13-46cc-9ae0-5b6bdbe46c54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3947284835022867, 0.3942943598402001, 0.40388718244051025, 0.4099690066204232, 0.40881690603327653, 10182.828959 seconds (158.86 G allocations: 2.662 TiB, 8.44% gc time)\n"
     ]
    }
   ],
   "source": [
    "# 10th iteration\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "already_done_docs = []\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            for i in false_docs_for_cat\n",
    "                if !(i in already_done_docs)\n",
    "                    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "                    TopicModels.removeDoc(lda, corpus, i, current_topic)\n",
    "                    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "                    push!(already_done_docs, i)\n",
    "                    break\n",
    "                end\n",
    "            end\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f92ee5bd-0074-448e-8d83-2e3ba8d54ae1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d74f8268-f3ba-445e-8f98-dd1a8de2cf22",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ae3904cf-1845-485d-b844-8a8a652f94bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false_docs_in_cat (generic function with 1 method)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function false_docs_in_cat(lda, corpus, docs_ls, cat_idx)\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    topic_x_top_docs = [idx for (idx, val) in enumerate(predicted_with_marriage_assigment) if val==cat_idx];\n",
    "    docs_in_topic_notin_cat = []\n",
    "    for i in topic_x_top_docs \n",
    "        if !(i in docs_ls[cat_idx])\n",
    "            push!(docs_in_topic_notin_cat, i)\n",
    "        end\n",
    "    end\n",
    "    return docs_in_topic_notin_cat\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "570a76ff-4178-4630-985a-08c68f6ea8f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic_of_each_doc (generic function with 1 method)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function topic_of_each_doc(lda, corpus)\n",
    "    top_topic_for_each_doc = []\n",
    "    for i in 1:corpus.document_size\n",
    "        top_val = 0\n",
    "        top_topic = 0\n",
    "        for j in 1:lda.M\n",
    "            v = TopicModels.topicPredict(lda, i, j)\n",
    "            if v>=top_val\n",
    "                top_val = v\n",
    "                top_topic = j\n",
    "            end\n",
    "        end\n",
    "        push!(top_topic_for_each_doc, top_topic)\n",
    "    end\n",
    "    return top_topic_for_each_doc\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46158821-d4d4-46e2-919f-7cd74123763e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Remove_kp refinement after stable marriage algorithm (5 times)(repeat 10 times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "060a44e8-0c0a-44cb-afd8-54ea52c54f9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cat_original = [17, 7, 18, 17, 10, 19, 16, 19, 18, 8, 13, 7, 12, 9, 18, 18, 17, 17, 15, 17, 7, 1, 19, 5, 4, 8, 19, 17, 15, 15, 17, 0, 19, 11, 17, 18, 9, 13, 19, 17, 13, 17, 19, 15, 7, 8, 11, 9, 1, 8, 0, 18, 17, 14, 14, 10, 15, 3, 8, 10, 10, 11, 17, 6, 17, 17, 15, 8, 15, 11, 2, 17, 9, 18, 17, 17, 10, 2, 9, 1, 10, 4, 9, 15, 3, 15, 19, 15, 16, 13, 17, 11, 0, 18, 10, 14, 6, 17, 15, 17, 19, 14, 8, 14, 5, 5, 11, 15, 6, 10, 5, 9, 15, 11, 9, 15, 7, 1, 14, 16, 15, 13, 13, 17, 0, 11, 16, 3, 16, 15, 14, 14, 19, 9, 18, 14, 17, 17, 4, 17, 15, 1, 16, 11, 15, 6, 2, 14, 15, 18, 8, 17, 18, 4, 7, 11, 17, 18, 9, 16, 18, 7, 11, 2, 15, 19, 1, 11, 14, 17, 1, 16, 6, 5, 19, 7, 11, 16, 8, 16, 10, 15, 17, 16, 16, 1, 0, 7, 15, 0, 6, 18, 10, 9, 11, 16, 19, 14, 6, 0, 0, 3, 17, 17, 12, 16, 8, 6, 19, 10, 17, 16, 14, 4, 16, 11, 1, 14, 0, 2, 16, 15, 15, 6, 15, 0, 14, 10, 15, 13, 2, 19, 15, 17, 15, 1, 9, 16, 18, 17, 6, 17, 8, 14, 11, 5, 15, 7, 15, 15, 12, 11, 14, 14, 3, 15, 3, 17, 3, 14, 11, 19, 10, 18, 15, 16, 15, 15, 9, 16, 11, 4, 19, 13, 3, 19, 9, 17, 18, 7, 7, 15, 16, 14, 15, 10, 5, 9, 14, 14, 17, 15, 15, 19, 14, 15, 15, 0, 4, 11, 11, 15, 12, 6, 3, 15, 14, 5, 14, 18, 11, 11, 14, 17, 17, 10, 9, 5, 7, 10, 19, 18, 17, 15, 18, 17, 11, 11, 13, 3, 2, 15, 10, 17, 2, 14, 13, 15, 15, 16, 15, 13, 14, 7, 16, 19, 3, 17, 17, 17, 14, 16, 9, 8, 14, 14, 16, 13, 12, 11, 17, 11, 1, 4, 11, 18, 16, 15, 11, 17, 15, 6, 17, 17, 13, 9, 10, 18, 17, 17, 13, 6, 1, 9, 14, 19, 3, 19, 13, 14, 10, 16, 19, 14, 9, 17, 7, 7, 11, 17, 18, 11, 17, 11, 18, 17, 16, 14, 1, 11, 16, 11, 18, 6, 1, 14, 15, 6, 17, 18, 16, 6, 13, 11, 0, 0, 16, 8, 17, 17, 10, 4, 15, 15, 15, 1, 13, 15, 14, 5, 15, 9, 1, 0, 13, 11, 1, 14, 16, 7, 2, 6, 10, 8, 14, 12, 19, 2, 7, 15, 17, 13, 17, 13, 17, 17, 19, 16, 15, 0, 12, 19, 19, 18, 15, 16, 9, 15, 16, 16, 12, 9, 9, 18, 4, 18, 11, 17, 15, 12, 12, 17, 17, 15, 19, 15, 19, 17, 15, 10, 14, 19, 16, 17, 10, 10, 14, 16, 5, 13, 9, 15, 17, 15, 19, 0, 18, 14, 15, 1, 15, 5, 17, 12, 18, 10, 5, 16, 0, 15, 1, 5, 7, 11, 19, 13, 11, 17, 9, 18, 9, 4, 10, 11, 5, 15, 12, 15, 1, 18, 6, 13, 18, 9, 1, 17, 18, 17, 15, 15, 19, 3, 17, 15, 9, 11, 14, 17, 14, 15, 1, 15, 15, 10, 3, 19, 17, 14, 6, 11, 12, 15, 17, 15, 13, 3, 0, 10, 4, 11, 0, 14, 7, 0, 15, 0, 2, 17, 17, 6, 19, 14, 17, 14, 6, 15, 12, 13, 4, 1, 17, 16, 17, 13, 18, 0, 11, 10, 3, 0, 13, 15, 13, 15, 15, 13, 5, 18, 1, 11, 19, 15, 13, 13, 10, 17, 13, 14, 19, 14, 0, 5, 7, 15, 11, 1, 15, 15, 14, 15, 13, 0, 15, 18, 7, 8, 13, 11, 16, 14, 19, 15, 16, 0, 4, 17, 11, 10, 11, 5, 19, 10, 17, 7, 10, 17, 9, 11, 15, 19, 17, 11, 11, 11, 10, 16, 14, 17, 18, 13, 16, 19, 9, 11, 18, 16, 1, 13, 16, 18, 16, 11, 16, 8, 14, 14, 16, 15, 13, 15, 11, 11, 16, 2, 15, 16, 19, 8, 17, 19, 0, 16, 15, 14, 17, 9, 19, 6, 10, 16, 15, 0, 17, 10, 6, 16, 16, 17, 11, 15, 14, 10, 2, 18, 0, 18, 16, 10, 8, 3, 17, 17, 15, 11, 17, 13, 9, 18, 19, 11, 17, 11, 11, 5, 18, 17, 13, 11, 14, 13, 17, 15, 2, 0, 12, 0, 0, 14, 15, 8, 18, 9, 17, 10, 15, 13, 13, 3, 4, 16, 10, 5, 2, 2, 16, 17, 15, 7, 15, 17, 16, 12, 17, 15, 16, 15, 17, 17, 17, 18, 19, 17, 14, 18, 11, 12, 18, 6, 14, 11, 0, 15, 10, 17, 3, 11, 9, 15, 11, 9, 13, 16, 15, 13, 17, 18, 10, 4, 18, 19, 5, 19, 17, 11, 10, 19, 17, 19, 18, 10, 3, 13, 10, 16, 11, 5, 0, 3, 11, 17, 13, 14, 6, 16, 4, 19, 19, 14, 1, 14, 16, 11, 6, 18, 19, 11, 10, 11, 19, 11, 9, 19, 10, 17, 10, 18, 14, 11, 9, 0, 9, 1, 15, 11, 8, 14, 18, 11, 16, 15, 16, 6, 18, 13, 18, 2, 17, 15, 16, 15, 11, 16, 5, 15, 11, 14, 1, 16, 15, 11, 19, 16, 11, 15, 17, 15, 2, 16, 11, 7, 13, 17, 16, 7, 10, 1, 16, 1, 18, 17, 11, 17, 11, 5, 17, 1, 18, 0, 18, 16, 17, 12, 17, 11, 11, 15, 15, 12, 0, 19, 10, 17, 14, 12, 0, 19, 11, 2, 16, 19, 17, 15, 10, 5, 16, 5, 5, 13, 11, 19, 4, 15, 14, 11, 17, 15, 14, 8, 17, 3, 13, 0, 17, 0, 10, 12, 14, 6, 13, 13, 11, 10, 13, 10, 14, 0, 19, 11, 7, 10, 14, 3, 15, 3, 14, 10, 17, 4, 5, 11, 0, 19, 10, 10, 4, 1, 13, 17, 7, 5, 19, 13, 11, 14, 19, 19, 18, 2, 13, 6, 9, 0, 11, 11, 6, 11, 18, 16, 14, 10, 2, 8, 16, 5, 17, 16, 13, 18, 17, 13, 18, 15, 19, 6, 18, 17, 19, 12, 18, 9, 13, 6, 0, 15, 11, 18, 11, 0, 18, 15, 10, 10, 6, 5, 17, 11, 1, 9, 1, 14, 12, 14, 10, 15, 11, 11, 18, 2, 10, 3, 17, 18, 14, 19, 17, 15, 9, 5, 1, 9, 13, 15, 13, 10, 17, 19, 4, 17, 17, 15, 10, 17, 13, 18, 17, 11, 18, 18, 15, 15, 17, 11, 10, 8, 0, 15, 15, 14, 11, 15, 11, 8, 15, 3, 12, 8, 15, 14, 15, 10, 10, 11, 11, 3, 15, 9, 19, 17, 4, 7, 15, 4, 9, 9, 1, 6, 13, 10, 19, 9, 11, 16, 19, 17, 16, 14, 10, 9, 15, 15, 9, 5, 0, 2, 18, 9, 15, 19, 19, 17, 5, 10, 14, 15, 17, 15, 18, 17, 15, 4, 8, 13, 8, 11, 1, 15, 13, 15, 9, 9, 17, 14, 10, 14, 0, 15, 0, 17, 10, 0, 17, 0, 10, 14, 0, 12, 16, 17, 13, 13, 19, 3, 15, 0, 16, 19, 11, 14, 17, 19, 0, 14, 2, 18, 17, 15, 2, 17, 3, 11, 14, 9, 4, 3, 9, 0, 11, 13, 5, 15, 10, 14, 10, 15, 0, 9, 17, 13, 0, 5, 17, 16, 7, 0, 19, 15, 18, 15, 17, 0, 1, 15, 11, 17, 15, 9, 18, 6, 11, 17, 12, 13, 15, 18, 7, 17, 15, 15, 6, 13, 11, 18, 4, 5, 13, 7, 17, 18, 17, 15, 18, 0, 6, 5, 18, 13, 14, 2, 19, 8, 8, 10, 8, 17, 11, 15, 10, 17, 0, 19, 8, 6, 15, 12, 15, 16, 12, 19, 6, 6, 9, 14, 14, 17, 16, 16, 13, 6, 13, 15, 6, 15, 0, 10, 10, 15, 13, 14, 15, 17, 10, 15, 17, 11, 5, 9, 12, 16, 10, 13, 16, 2, 17, 6, 5, 0, 15, 9, 15, 19, 0, 11, 18, 1, 3, 18, 18, 18, 17, 0, 4, 17, 15, 15, 17, 1, 1, 15, 1, 15, 15, 16, 19, 17, 15, 2, 17, 15, 11, 16, 11, 15, 12, 15, 6, 13, 16, 19, 3, 5, 5, 7, 6, 1, 1, 16, 15, 18, 0, 15, 0, 15, 17, 19, 15, 0, 16, 14, 17, 6, 7, 15, 11, 12, 15, 15, 17, 15, 12, 17, 1, 1, 14, 3, 6, 17, 17, 18, 15, 1, 6, 10, 18, 10, 2, 11, 13, 17, 16, 3, 17, 19, 19, 18, 15, 8, 5, 16, 11, 0, 9, 13, 0, 9, 10, 1, 15, 1, 10, 17, 1, 14, 16, 1, 14, 15, 11, 14, 6, 8, 15, 9, 19, 16, 7, 17, 14, 17, 15, 18, 5, 11, 15, 2, 16, 5, 15, 9, 12, 17, 10, 15, 17, 19, 9, 1, 15, 17, 18, 3, 16, 18, 1, 13, 3, 18, 18, 15, 17, 17, 9, 19, 3, 11, 16, 17, 10, 1, 7, 19, 9, 19, 1, 17, 10, 18, 17, 8, 10, 17, 19, 13, 5, 18, 15, 11, 19, 0, 14, 11, 15, 15, 18, 18, 16, 13, 2, 16, 12, 5, 6, 0, 5, 15, 3, 16, 8, 8, 1, 17, 13, 18, 17, 17, 14, 6, 16, 14, 4, 3, 3, 3, 17, 7, 4, 14, 15, 18, 11, 16, 17, 15, 17, 6, 9, 13, 17, 6, 17, 18, 3, 17, 15, 3, 15, 18, 10, 17, 2, 18, 11, 13, 11, 15, 1, 10, 18, 19, 14, 11, 18, 15, 0, 1, 15, 17, 15, 8, 11, 18, 1, 7, 19, 10, 0, 9, 17, 9, 17, 11, 1, 19, 7, 15, 13, 17, 17, 0, 9, 16, 18, 18, 13, 15, 5, 7, 15, 15, 4, 15, 5, 5, 8, 9, 11, 13, 12, 16, 17, 17, 1, 19, 15, 14, 13, 11, 13, 13, 13, 14, 2, 9, 15, 7, 15, 2, 8, 11, 15, 11, 0, 4, 11, 15, 0, 15, 1, 5, 17, 5, 11, 5, 4, 8, 17, 3, 5, 14, 5, 13, 10, 16, 17, 9, 7, 6, 10, 10, 19, 3, 14, 17, 13, 1, 16, 0, 7, 17, 15, 0, 11, 15, 19, 16, 18, 0, 10, 17, 1, 17, 1, 15, 15, 9, 12, 14, 15, 11, 10, 11, 16, 13, 17, 15, 2, 13, 19, 19, 15, 16, 10, 19, 6, 14, 17, 13, 19, 15, 6, 17, 16, 15, 17, 18, 7, 12, 14, 17, 13, 1, 5, 9, 0, 6, 4, 17, 4, 13, 17, 16, 19, 1, 6, 14, 11, 11, 9, 14, 10, 16, 3, 12, 17, 15, 17, 1, 3, 13, 14, 10, 5, 14, 15, 6, 10, 15, 16, 14, 15, 15, 9, 1, 17, 15, 19, 17, 2, 9, 9, 1, 15, 17, 17, 15, 16, 16, 14, 9, 17, 17, 10, 4, 15, 17, 16, 18, 18, 19, 17, 3, 12, 7, 11, 3, 13, 12, 15, 1, 19, 17, 12, 16, 9, 8, 14, 0, 16, 5, 10, 15, 15, 15, 12, 17, 18, 15, 14, 2, 15, 18, 0, 15, 4, 1, 1, 1, 13, 13, 3, 16, 9, 19, 17, 12, 10, 18, 0, 0, 14, 7, 13, 14, 17, 13, 17, 1, 6, 9, 15, 4, 15, 8, 17, 18, 7, 19, 16, 9, 6, 16, 10, 14, 6, 18, 18, 11, 17, 7, 18, 13, 3, 17, 17, 15, 19, 19, 11, 0, 17, 15, 19, 18, 15, 5, 10, 0, 0, 0, 1, 13, 0, 12, 18, 8, 0, 2, 15, 17, 18, 6, 17, 13, 15, 0, 17, 18, 15, 18, 13, 19, 10, 8, 15, 2, 1, 15, 17, 15, 17, 17]\n",
    "docs_cat = [i+1 for i in true_cat_original];\n",
    "docs_ls = [[] for i in 1:20]\n",
    "for (idx, val) in enumerate(docs_cat)\n",
    "    push!(docs_ls[val], idx)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0b66ad56-4b68-43ff-8095-0b0f82ad14b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_marriage_assigment = Dict(1=>16, 2=>9, 3=>20, 4=>5, 5=>18, 6=>10, 7=>2, 8=>1, 9=>13, 10=>4, \n",
    "                            11=>7, 12=>12, 13=>8, 14=>17, 15=>6, 16=>14, 17=>19, 18=>15, 19=>11, 20=>3);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "25283198-3f47-48ee-b196-89de679cc200",
   "metadata": {},
   "outputs": [],
   "source": [
    "#First loop refinement for all topics  #run 10 times and average it. \n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    if false_docs_for_cat != []\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "040955c7-5425-415c-b6b4-44f8f8387aa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Second loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    if false_docs_for_cat != []\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "200e5716-6bc5-4c8d-8786-79972ee3abc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Third loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    if false_docs_for_cat != []\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "7dccf6c2-2413-4a06-a0be-256cdc29f092",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fourth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    if false_docs_for_cat != []\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "546e6928-39fa-4b39-878c-5cc54e584c5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fifth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "    if false_docs_for_cat != []\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bca64a5a-04d4-40cf-b9cc-1cd0cff88b1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "print(predicted_with_marriage_assigment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "45056a20-94cb-430c-a0f8-cd8211de7d98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For average F1 score graph repeat each loop 10 times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "3b75b8aa-d30a-452d-ad77-d0e276cd334a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40715968308965084\n",
      "0.44761922907736856\n",
      "0.4884414456072598\n",
      "0.513950478869657\n",
      "0.49671982726923436\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "33729cb3-215b-4809-89c9-94fe6e205ecc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4263987160163777\n",
      "0.4493100241246682\n",
      "0.511025028923098\n",
      "0.5313629421477779\n",
      "0.5555996235548682\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b3d44746-86d2-4970-8bfa-8e9067e250e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4231224578434103\n",
      "0.47888412754799475\n",
      "0.5137788456242917\n",
      "0.5452544082318326\n",
      "0.5924623614018195\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b4d60f61-aacc-4c0d-bc36-d2967290c0cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.43852465426672205\n",
      "0.4509907859664557\n",
      "0.41843268932906386\n",
      "0.4891594007459804\n",
      "0.522901230649405\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "2e6f3884-877d-487f-a046-c1025ca6ce19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4207942190758324\n",
      "0.48224284937465395\n",
      "0.4809345988828039\n",
      "0.5269625042356878\n",
      "0.5826378599987546\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "98a2dbd4-9ab9-45ae-8e8c-7c76f9ca56fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.43032779843499996\n",
      "0.46493379000033375\n",
      "0.5088458481016407\n",
      "0.5056242544510859\n",
      "0.5457140649364104\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "fd1cfe9d-6a7c-4bca-9ab1-dc61909fa849",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.43610743953734993\n",
      "0.4950801821633983\n",
      "0.5108369742202062\n",
      "0.5480885525274964\n",
      "0.5313184766082483\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d9653411-cb14-4ac6-ab6d-1b5ba0748ddf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4259438702719369\n",
      "0.4194072658924883\n",
      "0.4780083201683894\n",
      "0.4965140624739341\n",
      "0.5242814320080812\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "59b99347-3dde-4525-b577-b67e2f5ea05e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4244259322138922\n",
      "0.46173970289814315\n",
      "0.42952228301086526\n",
      "0.47161589643870994\n",
      "0.5122862190155076\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4f815bdc-54fc-4998-8930-f9d374a4dd23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4195130305336595\n",
      "0.43938520975167084\n",
      "0.4763695276997123\n",
      "0.546709274049811\n",
      "0.5513024446892776\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        false_docs_for_cat = false_docs_in_cat(lda, corpus, docs_ls, current_cat)\n",
    "        if false_docs_for_cat != []\n",
    "            cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [false_docs_for_cat], 1, 1, 0.75)\n",
    "            current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "            TopicModels.removeDoc(lda, corpus, docs_have[1], current_topic)\n",
    "            TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "        end\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    println(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79d23063-c838-4759-bc8a-86b71b157e4e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d989124d-0b64-4e51-b28f-9dd7b4604370",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false_docs_in_cat (generic function with 1 method)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function false_docs_in_cat(lda, corpus, docs_ls, cat_idx)\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    topic_x_top_docs = [idx for (idx, val) in enumerate(predicted_with_marriage_assigment) if val==cat_idx];\n",
    "    docs_in_topic_notin_cat = []\n",
    "    for i in topic_x_top_docs \n",
    "        if !(i in docs_ls[cat_idx])\n",
    "            push!(docs_in_topic_notin_cat, i)\n",
    "        end\n",
    "    end\n",
    "    return docs_in_topic_notin_cat\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b1a7e120-4577-4d5d-aa12-d8287406ed61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic_of_each_doc (generic function with 1 method)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function topic_of_each_doc(lda, corpus)\n",
    "    top_topic_for_each_doc = []\n",
    "    for i in 1:corpus.document_size\n",
    "        top_val = 0\n",
    "        top_topic = 0\n",
    "        for j in 1:lda.M\n",
    "            v = TopicModels.topicPredict(lda, i, j)\n",
    "            if v>=top_val\n",
    "                top_val = v\n",
    "                top_topic = j\n",
    "            end\n",
    "        end\n",
    "        push!(top_topic_for_each_doc, top_topic)\n",
    "    end\n",
    "    return top_topic_for_each_doc\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1c98910-8933-4c31-bce5-c01753964a8a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Add_kp refinement after stable marriage algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "461a35c8-7fd1-4889-94dd-7a9dae75ce6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cat_original = [17, 7, 18, 17, 10, 19, 16, 19, 18, 8, 13, 7, 12, 9, 18, 18, 17, 17, 15, 17, 7, 1, 19, 5, 4, 8, 19, 17, 15, 15, 17, 0, 19, 11, 17, 18, 9, 13, 19, 17, 13, 17, 19, 15, 7, 8, 11, 9, 1, 8, 0, 18, 17, 14, 14, 10, 15, 3, 8, 10, 10, 11, 17, 6, 17, 17, 15, 8, 15, 11, 2, 17, 9, 18, 17, 17, 10, 2, 9, 1, 10, 4, 9, 15, 3, 15, 19, 15, 16, 13, 17, 11, 0, 18, 10, 14, 6, 17, 15, 17, 19, 14, 8, 14, 5, 5, 11, 15, 6, 10, 5, 9, 15, 11, 9, 15, 7, 1, 14, 16, 15, 13, 13, 17, 0, 11, 16, 3, 16, 15, 14, 14, 19, 9, 18, 14, 17, 17, 4, 17, 15, 1, 16, 11, 15, 6, 2, 14, 15, 18, 8, 17, 18, 4, 7, 11, 17, 18, 9, 16, 18, 7, 11, 2, 15, 19, 1, 11, 14, 17, 1, 16, 6, 5, 19, 7, 11, 16, 8, 16, 10, 15, 17, 16, 16, 1, 0, 7, 15, 0, 6, 18, 10, 9, 11, 16, 19, 14, 6, 0, 0, 3, 17, 17, 12, 16, 8, 6, 19, 10, 17, 16, 14, 4, 16, 11, 1, 14, 0, 2, 16, 15, 15, 6, 15, 0, 14, 10, 15, 13, 2, 19, 15, 17, 15, 1, 9, 16, 18, 17, 6, 17, 8, 14, 11, 5, 15, 7, 15, 15, 12, 11, 14, 14, 3, 15, 3, 17, 3, 14, 11, 19, 10, 18, 15, 16, 15, 15, 9, 16, 11, 4, 19, 13, 3, 19, 9, 17, 18, 7, 7, 15, 16, 14, 15, 10, 5, 9, 14, 14, 17, 15, 15, 19, 14, 15, 15, 0, 4, 11, 11, 15, 12, 6, 3, 15, 14, 5, 14, 18, 11, 11, 14, 17, 17, 10, 9, 5, 7, 10, 19, 18, 17, 15, 18, 17, 11, 11, 13, 3, 2, 15, 10, 17, 2, 14, 13, 15, 15, 16, 15, 13, 14, 7, 16, 19, 3, 17, 17, 17, 14, 16, 9, 8, 14, 14, 16, 13, 12, 11, 17, 11, 1, 4, 11, 18, 16, 15, 11, 17, 15, 6, 17, 17, 13, 9, 10, 18, 17, 17, 13, 6, 1, 9, 14, 19, 3, 19, 13, 14, 10, 16, 19, 14, 9, 17, 7, 7, 11, 17, 18, 11, 17, 11, 18, 17, 16, 14, 1, 11, 16, 11, 18, 6, 1, 14, 15, 6, 17, 18, 16, 6, 13, 11, 0, 0, 16, 8, 17, 17, 10, 4, 15, 15, 15, 1, 13, 15, 14, 5, 15, 9, 1, 0, 13, 11, 1, 14, 16, 7, 2, 6, 10, 8, 14, 12, 19, 2, 7, 15, 17, 13, 17, 13, 17, 17, 19, 16, 15, 0, 12, 19, 19, 18, 15, 16, 9, 15, 16, 16, 12, 9, 9, 18, 4, 18, 11, 17, 15, 12, 12, 17, 17, 15, 19, 15, 19, 17, 15, 10, 14, 19, 16, 17, 10, 10, 14, 16, 5, 13, 9, 15, 17, 15, 19, 0, 18, 14, 15, 1, 15, 5, 17, 12, 18, 10, 5, 16, 0, 15, 1, 5, 7, 11, 19, 13, 11, 17, 9, 18, 9, 4, 10, 11, 5, 15, 12, 15, 1, 18, 6, 13, 18, 9, 1, 17, 18, 17, 15, 15, 19, 3, 17, 15, 9, 11, 14, 17, 14, 15, 1, 15, 15, 10, 3, 19, 17, 14, 6, 11, 12, 15, 17, 15, 13, 3, 0, 10, 4, 11, 0, 14, 7, 0, 15, 0, 2, 17, 17, 6, 19, 14, 17, 14, 6, 15, 12, 13, 4, 1, 17, 16, 17, 13, 18, 0, 11, 10, 3, 0, 13, 15, 13, 15, 15, 13, 5, 18, 1, 11, 19, 15, 13, 13, 10, 17, 13, 14, 19, 14, 0, 5, 7, 15, 11, 1, 15, 15, 14, 15, 13, 0, 15, 18, 7, 8, 13, 11, 16, 14, 19, 15, 16, 0, 4, 17, 11, 10, 11, 5, 19, 10, 17, 7, 10, 17, 9, 11, 15, 19, 17, 11, 11, 11, 10, 16, 14, 17, 18, 13, 16, 19, 9, 11, 18, 16, 1, 13, 16, 18, 16, 11, 16, 8, 14, 14, 16, 15, 13, 15, 11, 11, 16, 2, 15, 16, 19, 8, 17, 19, 0, 16, 15, 14, 17, 9, 19, 6, 10, 16, 15, 0, 17, 10, 6, 16, 16, 17, 11, 15, 14, 10, 2, 18, 0, 18, 16, 10, 8, 3, 17, 17, 15, 11, 17, 13, 9, 18, 19, 11, 17, 11, 11, 5, 18, 17, 13, 11, 14, 13, 17, 15, 2, 0, 12, 0, 0, 14, 15, 8, 18, 9, 17, 10, 15, 13, 13, 3, 4, 16, 10, 5, 2, 2, 16, 17, 15, 7, 15, 17, 16, 12, 17, 15, 16, 15, 17, 17, 17, 18, 19, 17, 14, 18, 11, 12, 18, 6, 14, 11, 0, 15, 10, 17, 3, 11, 9, 15, 11, 9, 13, 16, 15, 13, 17, 18, 10, 4, 18, 19, 5, 19, 17, 11, 10, 19, 17, 19, 18, 10, 3, 13, 10, 16, 11, 5, 0, 3, 11, 17, 13, 14, 6, 16, 4, 19, 19, 14, 1, 14, 16, 11, 6, 18, 19, 11, 10, 11, 19, 11, 9, 19, 10, 17, 10, 18, 14, 11, 9, 0, 9, 1, 15, 11, 8, 14, 18, 11, 16, 15, 16, 6, 18, 13, 18, 2, 17, 15, 16, 15, 11, 16, 5, 15, 11, 14, 1, 16, 15, 11, 19, 16, 11, 15, 17, 15, 2, 16, 11, 7, 13, 17, 16, 7, 10, 1, 16, 1, 18, 17, 11, 17, 11, 5, 17, 1, 18, 0, 18, 16, 17, 12, 17, 11, 11, 15, 15, 12, 0, 19, 10, 17, 14, 12, 0, 19, 11, 2, 16, 19, 17, 15, 10, 5, 16, 5, 5, 13, 11, 19, 4, 15, 14, 11, 17, 15, 14, 8, 17, 3, 13, 0, 17, 0, 10, 12, 14, 6, 13, 13, 11, 10, 13, 10, 14, 0, 19, 11, 7, 10, 14, 3, 15, 3, 14, 10, 17, 4, 5, 11, 0, 19, 10, 10, 4, 1, 13, 17, 7, 5, 19, 13, 11, 14, 19, 19, 18, 2, 13, 6, 9, 0, 11, 11, 6, 11, 18, 16, 14, 10, 2, 8, 16, 5, 17, 16, 13, 18, 17, 13, 18, 15, 19, 6, 18, 17, 19, 12, 18, 9, 13, 6, 0, 15, 11, 18, 11, 0, 18, 15, 10, 10, 6, 5, 17, 11, 1, 9, 1, 14, 12, 14, 10, 15, 11, 11, 18, 2, 10, 3, 17, 18, 14, 19, 17, 15, 9, 5, 1, 9, 13, 15, 13, 10, 17, 19, 4, 17, 17, 15, 10, 17, 13, 18, 17, 11, 18, 18, 15, 15, 17, 11, 10, 8, 0, 15, 15, 14, 11, 15, 11, 8, 15, 3, 12, 8, 15, 14, 15, 10, 10, 11, 11, 3, 15, 9, 19, 17, 4, 7, 15, 4, 9, 9, 1, 6, 13, 10, 19, 9, 11, 16, 19, 17, 16, 14, 10, 9, 15, 15, 9, 5, 0, 2, 18, 9, 15, 19, 19, 17, 5, 10, 14, 15, 17, 15, 18, 17, 15, 4, 8, 13, 8, 11, 1, 15, 13, 15, 9, 9, 17, 14, 10, 14, 0, 15, 0, 17, 10, 0, 17, 0, 10, 14, 0, 12, 16, 17, 13, 13, 19, 3, 15, 0, 16, 19, 11, 14, 17, 19, 0, 14, 2, 18, 17, 15, 2, 17, 3, 11, 14, 9, 4, 3, 9, 0, 11, 13, 5, 15, 10, 14, 10, 15, 0, 9, 17, 13, 0, 5, 17, 16, 7, 0, 19, 15, 18, 15, 17, 0, 1, 15, 11, 17, 15, 9, 18, 6, 11, 17, 12, 13, 15, 18, 7, 17, 15, 15, 6, 13, 11, 18, 4, 5, 13, 7, 17, 18, 17, 15, 18, 0, 6, 5, 18, 13, 14, 2, 19, 8, 8, 10, 8, 17, 11, 15, 10, 17, 0, 19, 8, 6, 15, 12, 15, 16, 12, 19, 6, 6, 9, 14, 14, 17, 16, 16, 13, 6, 13, 15, 6, 15, 0, 10, 10, 15, 13, 14, 15, 17, 10, 15, 17, 11, 5, 9, 12, 16, 10, 13, 16, 2, 17, 6, 5, 0, 15, 9, 15, 19, 0, 11, 18, 1, 3, 18, 18, 18, 17, 0, 4, 17, 15, 15, 17, 1, 1, 15, 1, 15, 15, 16, 19, 17, 15, 2, 17, 15, 11, 16, 11, 15, 12, 15, 6, 13, 16, 19, 3, 5, 5, 7, 6, 1, 1, 16, 15, 18, 0, 15, 0, 15, 17, 19, 15, 0, 16, 14, 17, 6, 7, 15, 11, 12, 15, 15, 17, 15, 12, 17, 1, 1, 14, 3, 6, 17, 17, 18, 15, 1, 6, 10, 18, 10, 2, 11, 13, 17, 16, 3, 17, 19, 19, 18, 15, 8, 5, 16, 11, 0, 9, 13, 0, 9, 10, 1, 15, 1, 10, 17, 1, 14, 16, 1, 14, 15, 11, 14, 6, 8, 15, 9, 19, 16, 7, 17, 14, 17, 15, 18, 5, 11, 15, 2, 16, 5, 15, 9, 12, 17, 10, 15, 17, 19, 9, 1, 15, 17, 18, 3, 16, 18, 1, 13, 3, 18, 18, 15, 17, 17, 9, 19, 3, 11, 16, 17, 10, 1, 7, 19, 9, 19, 1, 17, 10, 18, 17, 8, 10, 17, 19, 13, 5, 18, 15, 11, 19, 0, 14, 11, 15, 15, 18, 18, 16, 13, 2, 16, 12, 5, 6, 0, 5, 15, 3, 16, 8, 8, 1, 17, 13, 18, 17, 17, 14, 6, 16, 14, 4, 3, 3, 3, 17, 7, 4, 14, 15, 18, 11, 16, 17, 15, 17, 6, 9, 13, 17, 6, 17, 18, 3, 17, 15, 3, 15, 18, 10, 17, 2, 18, 11, 13, 11, 15, 1, 10, 18, 19, 14, 11, 18, 15, 0, 1, 15, 17, 15, 8, 11, 18, 1, 7, 19, 10, 0, 9, 17, 9, 17, 11, 1, 19, 7, 15, 13, 17, 17, 0, 9, 16, 18, 18, 13, 15, 5, 7, 15, 15, 4, 15, 5, 5, 8, 9, 11, 13, 12, 16, 17, 17, 1, 19, 15, 14, 13, 11, 13, 13, 13, 14, 2, 9, 15, 7, 15, 2, 8, 11, 15, 11, 0, 4, 11, 15, 0, 15, 1, 5, 17, 5, 11, 5, 4, 8, 17, 3, 5, 14, 5, 13, 10, 16, 17, 9, 7, 6, 10, 10, 19, 3, 14, 17, 13, 1, 16, 0, 7, 17, 15, 0, 11, 15, 19, 16, 18, 0, 10, 17, 1, 17, 1, 15, 15, 9, 12, 14, 15, 11, 10, 11, 16, 13, 17, 15, 2, 13, 19, 19, 15, 16, 10, 19, 6, 14, 17, 13, 19, 15, 6, 17, 16, 15, 17, 18, 7, 12, 14, 17, 13, 1, 5, 9, 0, 6, 4, 17, 4, 13, 17, 16, 19, 1, 6, 14, 11, 11, 9, 14, 10, 16, 3, 12, 17, 15, 17, 1, 3, 13, 14, 10, 5, 14, 15, 6, 10, 15, 16, 14, 15, 15, 9, 1, 17, 15, 19, 17, 2, 9, 9, 1, 15, 17, 17, 15, 16, 16, 14, 9, 17, 17, 10, 4, 15, 17, 16, 18, 18, 19, 17, 3, 12, 7, 11, 3, 13, 12, 15, 1, 19, 17, 12, 16, 9, 8, 14, 0, 16, 5, 10, 15, 15, 15, 12, 17, 18, 15, 14, 2, 15, 18, 0, 15, 4, 1, 1, 1, 13, 13, 3, 16, 9, 19, 17, 12, 10, 18, 0, 0, 14, 7, 13, 14, 17, 13, 17, 1, 6, 9, 15, 4, 15, 8, 17, 18, 7, 19, 16, 9, 6, 16, 10, 14, 6, 18, 18, 11, 17, 7, 18, 13, 3, 17, 17, 15, 19, 19, 11, 0, 17, 15, 19, 18, 15, 5, 10, 0, 0, 0, 1, 13, 0, 12, 18, 8, 0, 2, 15, 17, 18, 6, 17, 13, 15, 0, 17, 18, 15, 18, 13, 19, 10, 8, 15, 2, 1, 15, 17, 15, 17, 17]\n",
    "docs_cat = [i+1 for i in true_cat_original];\n",
    "docs_ls = [[] for i in 1:20]\n",
    "for (idx, val) in enumerate(docs_cat)\n",
    "    push!(docs_ls[val], idx)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4a7fb5fb-0336-4a93-a37e-226fc827df52",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_marriage_assigment = Dict(1=>16, 2=>9, 3=>20, 4=>5, 5=>18, 6=>10, 7=>2, 8=>1, 9=>13, 10=>4, \n",
    "                            11=>7, 12=>12, 13=>8, 14=>17, 15=>6, 16=>14, 17=>19, 18=>15, 19=>11, 20=>3);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "2475abce-9e7b-45fe-a684-bca388f2e4ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dict{Any,Any}(\"archive\" => Any[\"archive\", \"archive -\"])\n",
      "Any[516, 1135, 1715, 1746]\n"
     ]
    }
   ],
   "source": [
    "current_cat_topic = 1\n",
    "absent_docs = docs_not_in_cat(lda, corpus, current_cat_topic);\n",
    "cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "println(cluster_kp[1])\n",
    "println(docs_have[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "07a92cb3-08c1-49a5-8833-9eacdcec1182",
   "metadata": {},
   "outputs": [],
   "source": [
    "#First loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "    cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "    TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "24b9749b-cc88-48ca-a25c-8be1af7b9193",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Second loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "    cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "    TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "98bb5c8a-c58c-4b47-8f80-d220d49942ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Third loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "    cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "    TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "1cbec404-7de9-437f-a01c-519685930f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Forth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "    cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "    TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "e8064400-6116-4f7a-b11e-de371e5a92a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fifth loop refinement for all topics\n",
    "for current_cat in 1:20\n",
    "    absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "    cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "    current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "    TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "    TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa70880-2306-49fe-8cf4-0fcc1accb296",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "print(predicted_with_marriage_assigment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "384f679d-9085-49a1-acf2-d97704f0d4fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9136472"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TopicModels.saveLDA(lda, \"20news/lda_obj_2000_long_after_5_add_kp_refinement_loops.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fe1f678f-38ea-49fb-8a15-aabbf2fabbf8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4981237570161637, 0.5536430887830217, 0.5985042331563979, 0.6301116369046398, 0.6571118709406457, 12337.312139 seconds (174.58 G allocations: 2.925 TiB, 7.83% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b3b74b7f-696d-4bcb-a41b-97757f2bd614",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47834983352410504, 0.5414053178970297, 0.5975092423837479, 0.6375182183336289, 0.6590115982688469, 11541.739819 seconds (174.56 G allocations: 2.925 TiB, 7.25% gc time)\n"
     ]
    }
   ],
   "source": [
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "51d854e0-cfd8-4e7b-90ba-4fa44c35d90b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4922246480815575, 0.5599986099243948, 0.6055545087666807, 0.6358811797052127, 0.6551872718799856, 11240.933803 seconds (174.55 G allocations: 2.925 TiB, 9.22% gc time)\n"
     ]
    }
   ],
   "source": [
    "#4th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "247a7edf-babd-4d0e-8cc1-5c0eec32013f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48090449130559193, 0.5467769322172225, 0.5924679455294509, 0.6355353801219835, 0.6606495302383468, 11088.489347 seconds (174.54 G allocations: 2.925 TiB, 7.35% gc time)\n"
     ]
    }
   ],
   "source": [
    "#5th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1fd75dcc-fe1b-49c2-851c-8e67e45cf604",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48852304566623933, 0.5391349963006075, 0.5908350123495628, 0.634459113680114, 0.6574371860681041, 11396.217523 seconds (174.56 G allocations: 2.925 TiB, 8.98% gc time)\n"
     ]
    }
   ],
   "source": [
    "#6th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "318e374e-e4d1-4e60-b049-919ef1e5ff81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4922783803339358, 0.5500721087433691, 0.5977003916121185, 0.6325027349419882, 0.6667342170447722, 11181.287427 seconds (174.56 G allocations: 2.925 TiB, 7.97% gc time)\n"
     ]
    }
   ],
   "source": [
    "#7th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7d15b27b-fe0c-4c11-b922-8477d93fb1cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.488235607118574, 0.547561140374842, 0.5898806841442585, 0.6188663372609233, 0.6568470427448289, 11228.159918 seconds (174.55 G allocations: 2.925 TiB, 9.00% gc time)\n"
     ]
    }
   ],
   "source": [
    "#8th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d7f70216-5625-429d-9933-6c54523d8b17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48149048948335227, 0.5511192665250314, 0.592911515492085, 0.6352784321886389, 0.6601727293092041, 11204.808255 seconds (174.55 G allocations: 2.925 TiB, 8.12% gc time)\n"
     ]
    }
   ],
   "source": [
    "#9th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0d12de4e-febc-4a68-aaca-bfd4c0f2984e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.486556244110898, 0.555697978115983, 0.6093576275552786, 0.6224150907475438, 0.6558080317371272, 11282.479334 seconds (174.54 G allocations: 2.925 TiB, 9.05% gc time)\n"
     ]
    }
   ],
   "source": [
    "#10th time experiment\n",
    "lda = TopicModels.loadLDA(\"20news/lda_obj_2000_long.json\");\n",
    "@time for loop in 1:5\n",
    "    for current_cat in 1:20\n",
    "        absent_docs = docs_not_in_cat(lda, corpus, current_cat)\n",
    "        cluster_kp, docs_have = TopicModels.top_x_kp_of_topic_m(kp, [absent_docs], 1, 1, 0.85)\n",
    "        current_topic = [k for (k,v) in topic_marriage_assigment if v==current_cat][1]\n",
    "        TopicModels.addDoc(lda, corpus, docs_have[1], current_topic)\n",
    "        TopicModels.gibbsSampling(lda, corpus.documents, 20);\n",
    "    end\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    print(f1_score.f1_score(docs_cat, predicted_with_marriage_assigment, average=\"weighted\"))\n",
    "    print(\", \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5adf2819-4aed-4a3f-9bb5-f100c9a797b6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "027d1036-d6e9-473e-9a2f-5c15afe63320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "docs_not_in_cat (generic function with 1 method)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function docs_not_in_cat(lda, corpus, cat_idx)\n",
    "    top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "    predicted_with_marriage_assigment = [topic_marriage_assigment[i] for i in top_topic_for_each_doc]\n",
    "    topic_x_top_docs = [idx for (idx, val) in enumerate(predicted_with_marriage_assigment) if val==cat_idx];\n",
    "    docs_in_cat_notin_topic = []\n",
    "    for i in docs_ls[cat_idx]\n",
    "        if !(i in topic_x_top_docs)\n",
    "            push!(docs_in_cat_notin_topic, i)\n",
    "        end\n",
    "    end\n",
    "    return docs_in_cat_notin_topic\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a0e5d017-914e-4f56-bab2-c997340f5738",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "top_kp_from_these_docs (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function top_kp_from_these_docs(these_docs)\n",
    "    return TopicModels.keyphrases_of_topic(kp, [these_docs], 1)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e7b35f9f-0ba1-4761-aab6-d86f0e1cf7f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "topic_of_each_doc (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function topic_of_each_doc(lda, corpus)\n",
    "    top_topic_for_each_doc = []\n",
    "    for i in 1:corpus.document_size\n",
    "        top_val = 0\n",
    "        top_topic = 0\n",
    "        for j in 1:lda.M\n",
    "            v = TopicModels.topicPredict(lda, i, j)\n",
    "            if v>=top_val\n",
    "                top_val = v\n",
    "                top_topic = j\n",
    "            end\n",
    "        end\n",
    "        push!(top_topic_for_each_doc, top_topic)\n",
    "    end\n",
    "    return top_topic_for_each_doc\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34de9e42-ac0c-4617-a65c-87a790997847",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 3 refinement to improve cat christian with topic 1 association"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5e0ee81f-f92b-46d1-a95a-803b2c441fbf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cluster_kp_8, docs_have_8 = TopicModels.top_x_kp_of_topic_m(kp, topic_distributions, 10, 8, 0.75);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "53861239-9aca-4af4-85a8-b4e604a7afde",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict{Any,Any}(\"god\" => Any[\"bible\", \"god\", \"christ\", \"jesus\", \"scripture\", \"no god\\\" and experiences\\\"\", \"goodness\"])\n",
      "Any[32, 444, 584, 644, 986, 992, 1129, 1335, 1513, 1542, 1607, 1803]"
     ]
    }
   ],
   "source": [
    "x = 8\n",
    "println(sort(cluster_kp_8[x]))\n",
    "print(sort(docs_have_8[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "884d9020-cb6f-4f47-9adc-c7980e7ef0b7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Choose which one is best (for now subjective analysis)(choosing kp_8)\n",
    "topic_8_kp_1 -> 8 docs out of 29 of cat \"christ\"\n",
    "topic_8_kp_2 -> 4 docs out of 9 of cat \"christ\"\n",
    "topic_8_kp_3 -> 0 docs out of 7 of cat \"christ\"\n",
    "topic_8_kp_4 -> 7 docs out of 22 of cat \"christ\"\n",
    "topic_8_kp_5 -> 3 docs out of 10 of cat \"christ\"\n",
    "topic_8_kp_6 -> 3 docs out of 6 of cat \"christ\"\n",
    "topic_8_kp_7 -> 3 docs out of 7 of cat \"christ\"\n",
    "topic_8_kp_8 -> 7 docs out of 12 of cat \"christ\"\n",
    "topic_8_kp_9 -> 6 docs out of 13 of cat \"christ\"\n",
    "topic_8_kp_10 -> 3 docs out of 4 of cat \"christ\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ebd9c1b9-61ba-453c-b9cb-7e4c17b2da95",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Apply refinement of add topic8_kp_8 to topic1\n",
    "TopicModels.addDoc(lda, corpus, docs_have_8[8], 1)\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e2a61eab-aeea-4e16-bda2-f503eb6ba7e4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f31b898f-e268-4c20-9f3c-46e9a783a1ed",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "topic_distributions = TopicModels.sortedTopDocsForTopics(lda, corpus);\n",
    "cluster_kp_8, docs_have_8 = TopicModels.top_x_kp_of_topic_m(kp, topic_distributions, 10, 8, 0.75);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "78e06b32-e7cf-4d46-a5b3-82943e88beb2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict{Any,Any}(\"religion\" => Any[\"bible\", \"morality\", \"moral\", \"religion\", \"atheism\", \"sin\", \"atheism?\", \"spiritual\", \"culture\", \"atheist\", \"christianity\", \"theism\", \"\\\"judaism\"])\n",
      "Any[233, 262, 457, 587, 620, 745, 908, 959, 965, 1021, 1042, 1094, 1217, 1256, 1405, 1436, 1446, 1571, 1762, 1778, 1782, 1793, 1901, 1918, 1958, 1966, 1984, 1999]"
     ]
    }
   ],
   "source": [
    "x = 1\n",
    "println(sort(cluster_kp_8[x]))\n",
    "print(sort(docs_have_8[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "383634e3-52e1-41ac-9418-10914c1e0b8c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Choose which one is best (for now subjective analysis)(choosing kp_1)\n",
    "topic_8_kp_1 -> 7 docs out of 28 of cat \"christ\"\n",
    "topic_8_kp_2 -> 4 docs out of 11 of cat \"christ\"\n",
    "topic_8_kp_3 -> 5 docs out of 21 of cat \"christ\"\n",
    "topic_8_kp_4 -> 1 docs out of 7 of cat \"christ\"\n",
    "topic_8_kp_5 -> 3 docs out of 12 of cat \"christ\"\n",
    "topic_8_kp_6 -> 3 docs out of 8 of cat \"christ\"\n",
    "topic_8_kp_7 -> 2 docs out of 5 of cat \"christ\"\n",
    "topic_8_kp_8 -> 10 docs out of 37 of cat \"christ\"\n",
    "topic_8_kp_9 -> 2 docs out of 4 of cat \"christ\"\n",
    "topic_8_kp_10 -> 0 docs out of 15 of cat \"christ\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e9ef45a7-3383-403f-8b59-2162eb60c7c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Apply refinement of add topic8_kp_1 to topic1\n",
    "TopicModels.addDoc(lda, corpus, docs_have_8[1], 1)\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d5101c53-51c7-4a4c-b759-6da3b984cf1b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d760d750-b431-4896-bff3-82c8f3aacb98",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "topic_distributions = TopicModels.sortedTopDocsForTopics(lda, corpus);\n",
    "cluster_kp_8, docs_have_8 = TopicModels.top_x_kp_of_topic_m(kp, topic_distributions, 10, 8, 0.75);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "a35faed7-f863-477e-a85e-319bd2cb6826",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict{Any,Any}(\"argument\" => Any[\"reasoning\", \"argument\", \"interpretation\", \"contention\", \"rationalism\", \"discussion\"])\n",
      "Any[530, 573, 582, 595, 669, 742, 1003, 1073, 1179, 1459, 1708]"
     ]
    }
   ],
   "source": [
    "x = 10\n",
    "println(sort(cluster_kp_8[x]))\n",
    "print(sort(docs_have_8[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "262ac4de-4f7c-43ee-be04-71161e878f2f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Choose which one is best (for now subjective analysis)(choosing kp_1)\n",
    "topic_8_kp_1 -> 6 docs out of 15 of cat \"christ\"\n",
    "topic_8_kp_2 -> 4 docs out of 11 of cat \"christ\"\n",
    "topic_8_kp_3 -> 0 docs out of 9 of cat \"christ\"\n",
    "topic_8_kp_4 -> 5 docs out of 9 of cat \"christ\"\n",
    "topic_8_kp_5 -> 5 docs out of 13 of cat \"christ\"\n",
    "topic_8_kp_6 -> 3 docs out of 6 of cat \"christ\"\n",
    "topic_8_kp_7 -> 2 docs out of 6 of cat \"christ\"\n",
    "topic_8_kp_8 -> 1 docs out of 7 of cat \"christ\"\n",
    "topic_8_kp_9 -> 1 docs out of 5 of cat \"christ\"\n",
    "topic_8_kp_10 -> 6 docs out of 11 of cat \"christ\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "1899830b-f01a-4a89-9764-208faf9100d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Apply refinement of add topic8_kp_10 to topic1\n",
    "TopicModels.addDoc(lda, corpus, docs_have_8[10], 1)\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "8e5958d1-5036-456c-b4ab-e0d0b8fb97b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c011da5-5ead-437e-9f9f-b2f27f9ca756",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "topic_distributions = TopicModels.sortedTopDocsForTopics(lda, corpus);\n",
    "cluster_kp_8, docs_have_8 = TopicModels.top_x_kp_of_topic_m(kp, topic_distributions, 10, 8, 0.75);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "99807779-74f9-454a-b385-5235fd0b068d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Apply refinement of add topic8_kp_4 to topic1\n",
    "TopicModels.addDoc(lda, corpus, docs_have_8[4], 1)\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "e7d9b4d2-3732-410e-86cd-c1b6e7b379bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "b8d2fbea-e5f8-47e7-92ac-741d077f692a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Apply refinement of add topic8_kp_8 to topic1\n",
    "TopicModels.addDoc(lda, corpus, docs_have_8[8], 1)\n",
    "TopicModels.gibbsSampling(lda, corpus.documents, 20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4792d9cb-345d-472e-84f0-74826fb06e34",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "top_topic_for_each_doc = topic_of_each_doc(lda, corpus);\n",
    "#println(top_topic_for_each_doc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.5.2",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}