NLP Tools - Part 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "48c44e0f-90de-4195-ae26-8e60d57a9af2",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import Tokenizer, RegexTokenizer, StopWordsRemover, NGram\n",
    "from pyspark.sql.functions import col, udf\n",
    "from pyspark.sql.types import IntegerType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "4ceb9d18-45b9-4377-83d0-247e29870a80",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - hive</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"/?o=3373316291219255#setting/sparkui/0609-054349-x6tsw2v0/driver-5044825885341433446\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.5.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*, 4]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Databricks Shell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f17e0420b50>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName('nlp_part1').getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8de09352-efee-453d-8269-03f34e18c333",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+\n",
      "|id |sentence                               |\n",
      "+---+---------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |\n",
      "|1  |I wish that Java could use case classes|\n",
      "|2  |Logistic,regression,models,are,neat    |\n",
      "+---+---------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sent_df = spark.createDataFrame([\n",
    "    (0,'Hi I just heard about Spark'),\n",
    "    (1,'I wish that Java could use case classes'),\n",
    "    (2,'Logistic,regression,models,are,neat')\n",
    "],['id','sentence'])\n",
    "sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "22806093-9449-41f6-b683-97b14401c7f3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+\n",
      "|id |sentence                               |tokens                                          |\n",
      "+---+---------------------------------------+------------------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic,regression,models,are,neat]           |\n",
      "+---+---------------------------------------+------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(inputCol='sentence',outputCol='tokens')\n",
    "tokenized_sent_df = tokenizer.transform(sent_df)\n",
    "tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "cbd7da1b-95d0-4a26-88b5-3f180d9c2e3c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function __main__.<lambda>(words)>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_tokens = udf(lambda words: len(words), IntegerType())\n",
    "count_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "493c2445-691c-41c2-81cc-9df0547ad8c7",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "|id |sentence                               |tokens                                          |num_tokens|\n",
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |6         |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|8         |\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic,regression,models,are,neat]           |1         |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tokenized_sent_df = tokenized_sent_df.withColumn('num_tokens',count_tokens(col('tokens')))\n",
    "tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8d9fc870-3e90-46df-8240-1f4df35c0cef",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+\n",
      "|id |sentence                               |tokens                                          |\n",
      "+---+---------------------------------------+------------------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic, regression, models, are, neat]       |\n",
      "+---+---------------------------------------+------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "regex_tokenizer = RegexTokenizer(inputCol='sentence',outputCol='tokens',pattern='\\\\W')\n",
    "regex_tokenized_sent_df = regex_tokenizer.transform(sent_df)\n",
    "regex_tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "fd6df9c1-7496-434c-bc65-d6b05752dd94",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "|id |sentence                               |tokens                                          |num_tokens|\n",
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |6         |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|8         |\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic, regression, models, are, neat]       |5         |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "regex_tokenized_sent_df = regex_tokenized_sent_df.withColumn('num_tokens',count_tokens(col('tokens')))\n",
    "regex_tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "06e829ae-172e-4335-bf35-d06c05e82d11",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+\n",
      "|id |sentence                               |tokens                                          |num_tokens|filtered_tokens                     |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |6         |[hi, heard, spark]                  |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|8         |[wish, java, use, case, classes]    |\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic, regression, models, are, neat]       |5         |[logistic, regression, models, neat]|\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "remover = StopWordsRemover(inputCol='tokens',outputCol='filtered_tokens')\n",
    "regex_tokenized_sent_df = remover.transform(regex_tokenized_sent_df)\n",
    "regex_tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "c107cbc8-862a-41ee-9466-b4a4bb0953ae",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+\n",
      "|id |sentence                               |tokens                                          |num_tokens|filtered_tokens                     |bigram_tokens                                                                |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |6         |[hi, heard, spark]                  |[hi i, i just, just heard, heard about, about spark]                         |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|8         |[wish, java, use, case, classes]    |[i wish, wish that, that java, java could, could use, use case, case classes]|\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic, regression, models, are, neat]       |5         |[logistic, regression, models, neat]|[logistic regression, regression models, models are, are neat]               |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ngram = NGram(n=2,inputCol='tokens',outputCol='bigram_tokens')\n",
    "regex_tokenized_sent_df = ngram.transform(regex_tokenized_sent_df)\n",
    "regex_tokenized_sent_df.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "83a7aec5-0ab4-483e-adef-dd1554c4f7d6",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------------------------+\n",
      "|id |sentence                               |tokens                                          |num_tokens|filtered_tokens                     |bigram_tokens                                                                |trigram_tokens                                                                                  |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------------------------+\n",
      "|0  |Hi I just heard about Spark            |[hi, i, just, heard, about, spark]              |6         |[hi, heard, spark]                  |[hi i, i just, just heard, heard about, about spark]                         |[hi i just, i just heard, just heard about, heard about spark]                                  |\n",
      "|1  |I wish that Java could use case classes|[i, wish, that, java, could, use, case, classes]|8         |[wish, java, use, case, classes]    |[i wish, wish that, that java, java could, could use, use case, case classes]|[i wish that, wish that java, that java could, java could use, could use case, use case classes]|\n",
      "|2  |Logistic,regression,models,are,neat    |[logistic, regression, models, are, neat]       |5         |[logistic, regression, models, neat]|[logistic regression, regression models, models are, are neat]               |[logistic regression models, regression models are, models are neat]                            |\n",
      "+---+---------------------------------------+------------------------------------------------+----------+------------------------------------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ngram = NGram(n=3,inputCol='tokens',outputCol='trigram_tokens')\n",
    "regex_tokenized_sent_df = ngram.transform(regex_tokenized_sent_df)\n",
    "regex_tokenized_sent_df.show(truncate=False)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "NLP Tools - Part 1",
   "widgets": {}
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}