NLP_tokenize_lemma_stemming_stopwords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 词干提取   stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "porter_stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'step'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "porter_stemmer.stem('stepping')#词干提取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multipli'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "porter_stemmer.stem('multiply')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#词性归一 lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordnet_lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dog'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordnet_lemmatizer.lemmatize('dogs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标注词性 pos_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'be'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordnet_lemmatizer.lemmatize('are', pos='v')#pos_tag默认是nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = nltk.word_tokenize('what does the fox say')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['what', 'does', 'the', 'fox', 'say']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('what', 'WDT'),\n",
       " ('does', 'VBZ'),\n",
       " ('the', 'DT'),\n",
       " ('fox', 'NNS'),\n",
       " ('say', 'VBP')]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.pos_tag(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 简单的ML情感分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos\n"
     ]
    }
   ],
   "source": [
    "from nltk.classify import NaiveBayesClassifier\n",
    "# 随⼿造点训练集\n",
    "s1 = 'this is a good book'\n",
    "s2 = 'this is a awesome book'\n",
    "s3 = 'this is a bad book'\n",
    "s4 = 'this is a terrible book'\n",
    "def preprocess(s):\n",
    "# Func: 句⼦处理\n",
    "# 这⾥简单的⽤了split(), 把句⼦中每个单词分开\n",
    "# 显然 还有更多的processing method可以⽤\n",
    "    return {word: True for word in s.lower().split()}\n",
    "    #字典生成式\n",
    "    # {'this': True, 'is':True, 'a':True, 'good':True, 'book':True}\n",
    "\n",
    "# 把训练集给做成标准形式\n",
    "training_data = [[preprocess(s1), 'pos'],\n",
    "                 [preprocess(s2), 'pos'],\n",
    "                 [preprocess(s3), 'neg'],\n",
    "                 [preprocess(s4), 'neg']\n",
    "                ]\n",
    "# 喂给model吃\n",
    "model = NaiveBayesClassifier.train(training_data)\n",
    "# 打出结果\n",
    "print(model.classify(preprocess('this is a good book')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 词频统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['this', 'is', 'my', 'sentence', 'this', 'is', 'my', 'life', 'this', 'is', 'the', 'day']\n"
     ]
    }
   ],
   "source": [
    "from nltk import FreqDist\n",
    "# 做个词库先\n",
    "corpus = 'this is my sentence ' \\\n",
    "'this is my life ' \\\n",
    "'this is the day'\n",
    "# 随便tokenize⼀下\n",
    "# 显然, 正如上⽂提到,\n",
    "# 这⾥可以根据需要做任何的preprocessing:\n",
    "# stopwords, lemma, stemming, etc.\n",
    "tokens = nltk.word_tokenize(corpus)\n",
    "print(tokens)\n",
    "# 得到token好的word list\n",
    "# ['this', 'is', 'my', 'sentence',\n",
    "# 'this', 'is', 'my', 'life', 'this',\n",
    "# 'is', 'the', 'day']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "# 借⽤NLTK的FreqDist统计⼀下⽂字出现的频率\n",
    "fdist = FreqDist(tokens)\n",
    "# 它就类似于⼀个Dict\n",
    "# 带上某个单词, 可以看到它在整个⽂章中出现的次数\n",
    "print(fdist['this'])\n",
    "# 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "[('this', 3), ('is', 3), ('my', 2), ('sentence', 1), ('life', 1), ('the', 1), ('day', 1)]\n"
     ]
    }
   ],
   "source": [
    "# 词频最高的10个单词拿出来\n",
    "standard_freq_vector = fdist.most_common(10)\n",
    "size = len(standard_freq_vector)\n",
    "print(size)\n",
    "print(standard_freq_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'this': 0, 'is': 1, 'my': 2, 'sentence': 3, 'life': 4, 'the': 5, 'day': 6}\n"
     ]
    }
   ],
   "source": [
    "# Func: 按照出现频率⼤⼩, 记录下每⼀个单词的位置\n",
    "def position_lookup(v):\n",
    "    res = {}\n",
    "    counter = 0\n",
    "    for word in v:\n",
    "        res[word[0]] = counter\n",
    "        counter += 1\n",
    "    return res\n",
    "    # 把标准的单词位置记录下来\n",
    "\n",
    "standard_position_dict = position_lookup(standard_freq_vector)\n",
    "print(standard_position_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "# 这时, 如果我们有个新句⼦:\n",
    "sentence = 'this is cool'\n",
    "# 先新建⼀个跟我们的标准vector同样⼤⼩的向量\n",
    "freq_vector = [0] * size\n",
    "# 简单的Preprocessing\n",
    "tokens = nltk.word_tokenize(sentence)\n",
    "# 对于这个新句⼦⾥的每⼀个单词\n",
    "for word in tokens:\n",
    "    try:\n",
    "        # 如果在我们的词库⾥出现过\n",
    "        # 那么就在\"标准位置\"上+1\n",
    "        freq_vector[standard_position_dict[word]] += 1\n",
    "    except KeyError:\n",
    "        # 如果是个新词\n",
    "        # 就pass掉\n",
    "        continue\n",
    "print(freq_vector)\n",
    "# [1, 1, 0, 0, 0, 0, 0]\n",
    "# 第⼀个位置代表 is, 出现了⼀次\n",
    "# 第⼆个位置代表 this, 出现了⼀次"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 统计单词出现的数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding:utf-8 -*-\n",
    "import io\n",
    "import re\n",
    "\n",
    "\n",
    "class Counter:\n",
    "    def __init__(self, path):\n",
    "        self.mapping = dict()\n",
    "\n",
    "        with io.open(path, encoding=\"utf-8\") as f:\n",
    "            data = f.read()\n",
    "            words = [s.lower() for s in re.findall(\"\\w+\", data)]\n",
    "\n",
    "            for word in words:\n",
    "                #若字典中没有键对应的值则返回默认值\n",
    "                #终于明白如何解决字典的计数增加问题。\n",
    "                self.mapping[word] = self.mapping.get(word, default=0) + 1\n",
    "\n",
    "    def most_common(self, n):\n",
    "        assert n > 0, \"n should be large than 0\"\n",
    "        return sorted(self.mapping.items(), key=lambda item: item[1], reverse=True)[:n]\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    most_common_5 = Counter(\"importthis.txt\").most_common(5)\n",
    "    for item in most_common_5:\n",
    "        print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#上面的代码解决了我一直困惑的字典value计数问题"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF_IDF,  Term Frequency, Inverse Document Frequency\n",
    "字词的重要性随着它在文件中出现的次数成正比增加，但同时会随着它在语料库中出现的频率成反比下降。\n",
    "TF(t) = (t出现在⽂档中的次数) / (⽂档中的term总数).\n",
    "IDF(t) = log_e(⽂档总数 / 含有t的⽂档总数)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01930786229086497\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "from nltk.text import TextCollection\n",
    "\n",
    "# 首先，把所有的文档放到TextCollection类中\n",
    "# 这个类会自动帮你断句，做统计，做计算\n",
    "corpus = TextCollection(['this is sentence one',\n",
    "                         'this is sentence two',\n",
    "                         ' is sentence three'])\n",
    "\n",
    "# 直接就能算出tfidf\n",
    "# (term:一句话中的某个term,text:这句话)\n",
    "print(corpus.tf_idf('this', 'this is sentence four'))\n",
    "\n",
    "# 对于每个新句子\n",
    "new_sentence='this is sentence five'\n",
    "# 遍历一遍所有的vocabulary中的词：\n",
    "standard_vocab=['this' 'is' 'sentence' 'one' 'two' 'five']\n",
    "for word in standard_vocab:\n",
    "    print(corpus.tf_idf(word, new_sentence))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简单的⽂字预处理：\n",
    "# 1. 去除HTML\n",
    "# 这⾥⽤到BeautifulSoup这个库，\n",
    "# 当然，这种简单的事情，也可以⾃⼰做个字符串运算解决\n",
    "from bs4 import BeautifulSoup\n",
    "beautiful_text = BeautifulSoup(raw_text).get_text()\n",
    "#\n",
    "# 2. 把⾮字⺟的去除掉\n",
    "# 这⾥可以⽤正则表达式解决\n",
    "import re\n",
    "letters_only = re.sub(\"[^a-zA-Z]\", \" \", beautiful_text)\n",
    "#\n",
    "# 3. 全部⼩写化\n",
    "words = letters_only.lower().split()\n",
    "#\n",
    "# 4. 去除stopwords\n",
    "# 这⾥⽤到NLTK\n",
    "from nltk.corpus import stopwords\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "meaningful_words = [w for w in words if not w in stops]\n",
    "# ⾼阶⽂字处理：\n",
    "# 5. Lemmatization\n",
    "#\n",
    "# 这个⽐较复杂，下次NLTK的时候讲\n",
    "# 6. 搞回成⼀⻓串string\n",
    "return( \" \".join( meaningful_words ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizor: 把原来的string训练集，变成 list of lists：\n",
    "# 这个寒⽼师上堂课应该讲过：\n",
    "# 简单点的话，可以⽤这个：\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "# 达到这样的效果：\n",
    ">>> print sentences\n",
    "# 原⽂： ['Hello, how are you', 'im fine, thank you, and you?']\n",
    "[['hello', 'how'], ['fine', 'thank']]\n",
    "# 现在进⼊正题， w2v。\n",
    "# 我们⽤Gensim这个库来做，很⽅便。\n",
    "from gensim.models import word2vec\n",
    "# 先设⼀下param\n",
    "num_features = 1000 # 最多多少个不同的features\n",
    "min_word_count = 10 # ⼀个word，最少出现多少次 才被计⼊\n",
    "num_workers = 4 # 多少thread⼀起跑（快⼀点⼉）\n",
    "size = 256 # vec的size\n",
    "window = 5 # 前后观察多⻓的“语境”\n",
    "# 跑起来\n",
    "model = word2vec.Word2Vec(sentences, size=size, workers=num_workers,\\\n",
    "                          size=num_features, min_count = min_word_count,\\\n",
    "                          window = window)\n",
    "# 你可以save下来\n",
    "model.save('LOL.save')\n",
    "# ⽇后再load回来\n",
    "model = word2vec.Word2Vec.load('LOL.save')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 当然 你们也许会看到⾕歌也提供了⾃⼰的News包：\n",
    "# 要load 其他语⾔train出来的⽂件（⽐如C) 的Bin或者text⽂件\n",
    "# 那就这样：\n",
    "model = Word2Vec.load_word2vec_format('google_news.txt', binary=False) # C text format\n",
    "model = Word2Vec.load_word2vec_format('google_news.bin', binary=True) # C binary format\n",
    "# ⼏个常⽤的⽤法：\n",
    "# woman + king - man = queen\n",
    ">>> model.most_similar(positive=['woman', 'king'], negative=['man'])\n",
    "[('queen', 0.50882536), ...]\n",
    "# 求两个词的senmatics相似度\n",
    ">>> model.similarity('woman', 'man')\n",
    "0.73723527\n",
    "# 就更dict⼀样使⽤你train好的model\n",
    ">>> model['computer']\n",
    "array([-0.00449447, -0.00310097, 0.02421786, ...], dtype=float32)\n",
    "# 现在 你可以把这个model包装起来。把你所有的sentences token 过⼀遍\n",
    "def w2vmodel(sentences):\n",
    "...\n",
    "return vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这个时候你会发现，我们的vec是针对每个word的。⽽我们的训练集 是sen和label互相对应的，\n",
    "# ⼯业上，到了这⼀步，有三种解决⽅案：\n",
    "# 1. 平均化⼀个句⼦⾥所有词的vec。\n",
    "# sen_vec = [vec, vec, vec, ...] / n\n",
    "# 2. 排成⼀个⼤matrix (M * N)，等着CNN来搞\n",
    "# [ vec | vec | vec | vec | ... ]\n",
    "# 3. ⽤Doc2Vec。这是基于句⼦的vec，跟word2vec差不多思路，⽤起来也差不多。\n",
    "# 只对⻓篇 ⼤⽂章效果好。对头条新闻， twitter这种的东⻄，就不⾏了。每个“篇”的句⼦太少。\n",
    "# 具体可以看gensim。\n",
    "# Anyway, 这⼀步完成后，你会对于每个训练集的X，得到⼀个固定⻓度的vec或者matrix\n",
    "# 接下来的事情，⼤家就可以融会贯通了。\n",
    "# ⽐如，可以⽤前⾯冯⽼师讲的RF跑⼀遍 做classification。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算两个字符串之间的距离"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ratio=0.6153846153846154, dist=3\n"
     ]
    }
   ],
   "source": [
    "import Levenshtein\n",
    "s1 = 'kitten'\n",
    "s2 = 'sitting'\n",
    " \n",
    "ratio = Levenshtein.ratio(s1, s2)\n",
    "dist = Levenshtein.distance(s1, s2)\n",
    "print('ratio={0}, dist={1}'.format(ratio, dist))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "seqratio,计算两个字符串之间的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = ['1','2','3','4','5']\n",
    "b = ['2','3','4']\n",
    " \n",
    "Levenshtein.seqratio(a, b)# 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}