NLP Tools - Part 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "727025e7-223f-4cd6-906d-a0d2a1bf1756",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import CountVectorizer, HashingTF, IDF, RegexTokenizer, Word2Vec\n",
    "from pyspark.sql.functions import format_number, col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "69e1b793-0966-41ed-9e5b-e6a139539c8c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - hive</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"/?o=3373316291219255#setting/sparkui/0609-054349-x6tsw2v0/driver-3389005562028568110\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.5.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*, 4]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Databricks Shell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f63f6537340>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName('nlp_part2').getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a8b8b057-87bd-4d6e-a4be-a9ef4e86c38a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n|label|            sentence|\n+-----+--------------------+\n|  0.0|Hi I just heard a...|\n|  0.0|I wish Java could...|\n|  1.0|Logistic regressi...|\n+-----+--------------------+\n\n"
     ]
    }
   ],
   "source": [
    "sent_df = spark.createDataFrame([\n",
    "    (0.0,'Hi I just heard about Spark'),\n",
    "    (0.0,'I wish Java could use case classes'),\n",
    "    (1.0,'Logistic regression models are neat')\n",
    "],['label','sentence'])\n",
    "sent_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "9011fb0a-3172-42e5-a0fb-315e31edcdae",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------------------------+------------------------------------------+\n|label|sentence                           |tokens                                    |\n+-----+-----------------------------------+------------------------------------------+\n|0.0  |Hi I just heard about Spark        |[hi, i, just, heard, about, spark]        |\n|0.0  |I wish Java could use case classes |[i, wish, java, could, use, case, classes]|\n|1.0  |Logistic regression models are neat|[logistic, regression, models, are, neat] |\n+-----+-----------------------------------+------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "tokenizer = RegexTokenizer(inputCol='sentence',outputCol='tokens',pattern='\\\\W')\n",
    "tokenized_sent_df = tokenizer.transform(sent_df)\n",
    "tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "847994f2-9257-4e85-b644-3cf718e32335",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+\n|label|sentence                           |tokens                                    |rawFeatures                                                                         |\n+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+\n|0.0  |Hi I just heard about Spark        |[hi, i, just, heard, about, spark]        |(262144,[18700,19036,33808,66273,97171,173558],[1.0,1.0,1.0,1.0,1.0,1.0])           |\n|0.0  |I wish Java could use case classes |[i, wish, java, could, use, case, classes]|(262144,[19036,20719,55551,58672,98717,109547,192310],[1.0,1.0,1.0,1.0,1.0,1.0,1.0])|\n|1.0  |Logistic regression models are neat|[logistic, regression, models, are, neat] |(262144,[46243,58267,91006,160975,190884],[1.0,1.0,1.0,1.0,1.0])                    |\n+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "tf = HashingTF(inputCol='tokens',outputCol='rawFeatures')\n",
    "tokenized_sent_df = tf.transform(tokenized_sent_df)\n",
    "tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "22d60ccb-6208-4b0e-9084-1dc972d3194b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+\n|label|sentence                           |tokens                                    |rawFeatures                                                                         |features                                                                                            |\n+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+\n|0.0  |Hi I just heard about Spark        |[hi, i, just, heard, about, spark]        |(262144,[18700,19036,33808,66273,97171,173558],[1.0,1.0,1.0,1.0,1.0,1.0])           |(262144,[18700,19036,33808,66273,97171,173558],[0.0,0.28768207245178085,0.0,0.0,0.0,0.0])           |\n|0.0  |I wish Java could use case classes |[i, wish, java, could, use, case, classes]|(262144,[19036,20719,55551,58672,98717,109547,192310],[1.0,1.0,1.0,1.0,1.0,1.0,1.0])|(262144,[19036,20719,55551,58672,98717,109547,192310],[0.28768207245178085,0.0,0.0,0.0,0.0,0.0,0.0])|\n|1.0  |Logistic regression models are neat|[logistic, regression, models, are, neat] |(262144,[46243,58267,91006,160975,190884],[1.0,1.0,1.0,1.0,1.0])                    |(262144,[46243,58267,91006,160975,190884],[0.0,0.0,0.0,0.0,0.0])                                    |\n+-----+-----------------------------------+------------------------------------------+------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "idf = IDF(inputCol='rawFeatures',outputCol='features',minDocFreq=2)\n",
    "idf_model = idf.fit(tokenized_sent_df)\n",
    "tokenized_sent_df = idf_model.transform(tokenized_sent_df)\n",
    "tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "842eb622-5f37-4b23-93b5-7c5464954fac",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------------------------------------------------------------------------------------------+\n|label|features                                                                                            |\n+-----+----------------------------------------------------------------------------------------------------+\n|0.0  |(262144,[18700,19036,33808,66273,97171,173558],[0.0,0.28768207245178085,0.0,0.0,0.0,0.0])           |\n|0.0  |(262144,[19036,20719,55551,58672,98717,109547,192310],[0.28768207245178085,0.0,0.0,0.0,0.0,0.0,0.0])|\n|1.0  |(262144,[46243,58267,91006,160975,190884],[0.0,0.0,0.0,0.0,0.0])                                    |\n+-----+----------------------------------------------------------------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "tokenized_sent_df.select('label','features').show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "59e86de3-4cbb-46b3-9b76-8f1af79d0c33",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------+\n| id|          words|\n+---+---------------+\n|  0|      [a, b, c]|\n|  1|   [a, b, c, a]|\n|  2|[a, c, b, a, b]|\n+---+---------------+\n\n"
     ]
    }
   ],
   "source": [
    "words_df = spark.createDataFrame([\n",
    "    (0,\"a b c\".split()),\n",
    "    (1,\"a b c a\".split()),\n",
    "    (2,\"a c b a b\".split())\n",
    "],['id','words'])\n",
    "words_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "e1cf048f-a161-4a94-bb38-0fb09376c841",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "CountVectorizerModel: uid=CountVectorizer_54378f8705a6, vocabularySize=3"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(inputCol='words',outputCol='features',vocabSize=3,minDF=2)\n",
    "cv_model = vectorizer.fit(words_df)\n",
    "cv_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b44ad110-e89a-49cb-a27f-7708bf3259e5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------+-------------------------+\n|id |words          |features                 |\n+---+---------------+-------------------------+\n|0  |[a, b, c]      |(3,[0,1,2],[1.0,1.0,1.0])|\n|1  |[a, b, c, a]   |(3,[0,1,2],[2.0,1.0,1.0])|\n|2  |[a, c, b, a, b]|(3,[0,1,2],[2.0,2.0,1.0])|\n+---+---------------+-------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "vectorized_words_df = cv_model.transform(words_df)\n",
    "vectorized_words_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "f825f128-c636-4f80-a944-36abec761262",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n|            sentence|\n+--------------------+\n|[b, a, b, a, b, a...|\n|[b, a, b, a, b, a...|\n+--------------------+\n\n"
     ]
    }
   ],
   "source": [
    "sent = (\"b a \" * 100 + \"a c b \" * 10).split(\" \")\n",
    "\n",
    "words_df = spark.createDataFrame([\n",
    "    (sent,), \n",
    "    (sent,)\n",
    "], [\"sentence\"])\n",
    "words_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8334d301-8cce-4e05-9f7b-5a33d443cdbe",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Word2VecModel: uid=Word2Vec_0f743b1716f7, numWords=4, vectorSize=5"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv = Word2Vec(inputCol='sentence',vectorSize=5,seed=48,windowSize=10,maxSentenceLength=500,minCount=1,outputCol='embeddings')\n",
    "wv_model = wv.fit(words_df)\n",
    "wv_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "0e066eb0-6567-42ca-8e36-fb06e06e4d6f",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------------------------------------------------------------------------------------------------------+\n|word|vector                                                                                                    |\n+----+----------------------------------------------------------------------------------------------------------+\n|a   |[0.43426597118377686,-0.12470865249633789,-0.5958183407783508,-0.19327032566070557,0.7723482847213745]    |\n|b   |[0.6657069325447083,-0.1493173986673355,-0.27567753195762634,0.29769062995910645,0.42605358362197876]     |\n|c   |[0.35067981481552124,-0.030499424785375595,-0.35829445719718933,-0.025705864652991295,0.40814152359962463]|\n|    |[0.07161520421504974,0.06222974881529808,-0.14337986707687378,-0.04092687740921974,0.1599714756011963]    |\n+----+----------------------------------------------------------------------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "wv_model.getVectors().show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "770e45b8-71a7-4206-9164-681dc48e3ff1",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+\n|word|similarity|\n+----+----------+\n|   c|   0.86205|\n|   a|   0.75559|\n+----+----------+\n\n"
     ]
    }
   ],
   "source": [
    "wv_model.findSynonyms(\"b\",2).select(['word',format_number(col('similarity'),5).alias('similarity')]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "dbb31450-26f7-4eb8-9f43-9b9c7d632481",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+\n|word|similarity|\n+----+----------+\n|   a|   0.97499|\n|    |   0.91277|\n+----+----------+\n\n"
     ]
    }
   ],
   "source": [
    "wv_model.findSynonyms(\"c\",2).select(['word',format_number(col('similarity'),5).alias('similarity')]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "0479722f-28a0-49e7-8641-3bc6cf672a01",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+\n|word|similarity|\n+----+----------+\n|   c|   0.97499|\n|    |   0.92265|\n+----+----------+\n\n"
     ]
    }
   ],
   "source": [
    "wv_model.findSynonyms(\"a\",2).select(['word',format_number(col('similarity'),5).alias('similarity')]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b1e8134a-3c37-4211-a7cb-814524621477",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+\n|sentence                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |embeddings                                                                                            |\n+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+\n|[b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, ]|[0.5392875877164659,-0.13153952435949148,-0.43112930926409637,0.048433973804696814,0.5890285377894645]|\n|[b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, b, a, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, a, c, b, ]|[0.5392875877164659,-0.13153952435949148,-0.43112930926409637,0.048433973804696814,0.5890285377894645]|\n+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+\n\n"
     ]
    }
   ],
   "source": [
    "words_df = wv_model.transform(words_df)\n",
    "words_df.show(truncate=False)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "NLP Tools - Part 2",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}