2.data_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.647 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "data_path='data/'\n",
    "\n",
    "ARTICLE_FILE = data_path+\"train_text.txt\"\n",
    "SUMMARRY_FILE = data_path+\"train_label.txt\"\n",
    "\n",
    "TRAIN_FILE = data_path+\"train_art_summ_prep.txt\"\n",
    "VAL_FILE = data_path+\"val_art_summ_prep.txt\"\n",
    "\n",
    "user_dict=data_path+'user_dict.txt'\n",
    "jieba.load_userdict(user_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def timer(func):\n",
    "    \"\"\"时间装饰器\"\"\"\n",
    "    def wrapper(*args, **kwargs):\n",
    "        start = time.time()\n",
    "        r = func(*args, **kwargs)\n",
    "        end = time.time()\n",
    "        cost = end - start\n",
    "        print(\"Cost time: {cost} s\")\n",
    "        return r\n",
    "\n",
    "    return wrapper\n",
    "\n",
    "\n",
    "\n",
    "@timer\n",
    "def load_data(filename):\n",
    "    \"\"\"加载数据文件，对文本进行分词\"\"\"\n",
    "    data_list = []\n",
    "    with open(filename, 'r', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            # jieba.enable_parallel()\n",
    "            words = jieba.cut(line.strip())\n",
    "            word_list = list(words)\n",
    "            # jieba.disable_parallel()\n",
    "            data_list.append(' '.join(word_list).strip())\n",
    "    return data_list\n",
    "\n",
    "\n",
    "def build_train_val(article_data, summary_data, train_num=600000):\n",
    "    \"\"\"划分训练和验证数据\"\"\"\n",
    "    train_list = []\n",
    "    val_list = []\n",
    "    n = 0\n",
    "    for text, summ in zip(article_data, summary_data):\n",
    "        n += 1\n",
    "        if n <= train_num:\n",
    "            train_list.append(text)\n",
    "            train_list.append(summ)\n",
    "        else:\n",
    "            val_list.append(text)\n",
    "            val_list.append(summ)\n",
    "    return train_list, val_list\n",
    "\n",
    "\n",
    "def save_file(filename, li):\n",
    "    \"\"\"预处理后的数据保存到文件\"\"\"\n",
    "    with open(filename, 'w+', encoding='utf-8') as f:\n",
    "        for item in li:\n",
    "            f.write(item + '\\n')\n",
    "    print(\"Save {filename} ok.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost time: {cost} s\n",
      "Cost time: {cost} s\n"
     ]
    }
   ],
   "source": [
    "article_data = load_data(ARTICLE_FILE)\n",
    "summary_data = load_data(SUMMARRY_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save {filename} ok.\n",
      "Save {filename} ok.\n"
     ]
    }
   ],
   "source": [
    "TRAIN_SPLIT = 80000\n",
    "train_list, val_list = build_train_val(article_data, summary_data, train_num=TRAIN_SPLIT)\n",
    "save_file(TRAIN_FILE, train_list)\n",
    "save_file(VAL_FILE, val_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch13_py37 Env",
   "language": "python",
   "name": "pytorch13_py37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}