China's Representatives.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\Anakin\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "# test for certain url\n",
    "url = \"http://www.npc.gov.cn/delegate/viewDelegate.action?dbid=200812000144\"\n",
    "bs = BeautifulSoup(requests.get(url).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "q = bs.find_all(\"td\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tdbg'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q[2][\"class\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<td>当前位置：<a href=\"http://www.npc.gov.cn/npc/xinwen/index.htm\" target=\"_blank\">首页</a> &gt;&gt; <a href=\"http://www.npc.gov.cn/delegate/delegateArea.action\">代表名单</a></td>\n",
      "<td rowspan=\"8\"><img class=\"img\" src=\"/delegate/delegateList/showimg1.action?dbid=120001\"/></td>\n",
      "<td class=\"tdbg\" rowspan=\"2\">姓名</td>\n",
      "<td class=\"bg2\" rowspan=\"2\">马旭</td>\n",
      "<td class=\"tdbg\">代表团</td>\n",
      "<td class=\"bg2\">北京</td>\n",
      "<td class=\"tdbg\">性别</td>\n",
      "<td class=\"bg2\">男</td>\n",
      "<td class=\"tdbg\">民族</td>\n",
      "<td class=\"bg2\">满族</td>\n",
      "<td class=\"tdbg\">籍贯</td>\n",
      "<td class=\"bg2\">湖北沙市</td>\n",
      "<td class=\"tdbg\">出生年月</td>\n",
      "<td class=\"bg2\">1963年09月</td>\n",
      "<td class=\"tdbg\">党派</td>\n",
      "<td class=\"bg2\">民建</td>\n",
      "<td class=\"tdbg\">毕业院校</td>\n",
      "<td class=\"bg2\">华西医科大学</td>\n",
      "<td class=\"tdbg\">所学专业</td>\n",
      "<td class=\"bg2\">医学遗传学专业</td>\n",
      "<td class=\"tdbg\">学历</td>\n",
      "<td class=\"bg2\">研究生</td>\n",
      "<td class=\"tdbg\">学位</td>\n",
      "<td class=\"bg2\">医学硕士</td>\n",
      "<td class=\"tdbg\">现任职务</td>\n",
      "<td class=\"bg2_1\" colspan=\"3\">    全国人大教育科学文化卫生委员会委员，国家卫生和计划生育委员会科学技术研究所所长\r\n",
      "</td>\n"
     ]
    }
   ],
   "source": [
    "for i in bs.find_all(\"td\"):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 根据上述写针对个人的网页的部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parsepage(url):\n",
    "    bs = BeautifulSoup(requests.get(url).content)\n",
    "    q = bs.find_all(\"td\")\n",
    "    tmp = dict()\n",
    "    for i in range(len(q)):\n",
    "        try:\n",
    "            if q[i][\"class\"][0]==\"tdbg\":\n",
    "                tmp[q[i].text]=(q[i+1].text).strip()\n",
    "        except:\n",
    "            pass\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\Anakin\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'代表团': '北京',\n",
       " '党派': '民盟',\n",
       " '出生年月': '1961年04月',\n",
       " '姓名': '王蓉蓉',\n",
       " '学位': '文学学士',\n",
       " '学历': '研究生',\n",
       " '性别': '女',\n",
       " '所学专业': '戏曲表演专业',\n",
       " '毕业院校': '中国戏曲学院',\n",
       " '民族': '汉族',\n",
       " '现任职务': '全国人大代表',\n",
       " '籍贯': '辽宁海城'}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 示例结果\n",
    "parsepage(\"http://www.npc.gov.cn/delegate/viewDelegate.action?dbid=200812001328\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get all Url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = \"http://www.npc.gov.cn/delegate/delegateArea.action\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from urllib.parse import urljoin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 显示到各省份的相对链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\Anakin\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "bs = BeautifulSoup(requests.get(url).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "qq = bs.find_all(\"li\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<li><a href=\"dbmd.action?id=a1\">北京</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a2\">天津</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a3\">河北</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a4\">山西</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a5\">内蒙古</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a6\">辽宁</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a7\">吉林</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a8\">黑龙江</a></li>,\n",
       " <li><a href=\"dbmd.action?id=a9\">上海</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b1\">江苏</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b2\">浙江</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b3\">安徽</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b4\">福建</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b5\">江西</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b6\">山东</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b7\">河南</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b8\">湖北</a></li>,\n",
       " <li><a href=\"dbmd.action?id=b9\">湖南</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c1\">广东</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c2\">广西</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c3\">海南</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c4\">重庆</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c5\">四川</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c6\">贵州</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c7\">云南</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c8\">西藏</a></li>,\n",
       " <li><a href=\"dbmd.action?id=c9\">陕西</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d1\">甘肃</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d2\">青海</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d3\">宁夏</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d4\">新疆</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d5\">香港</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d6\">澳门</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d7\">台湾</a></li>,\n",
       " <li><a href=\"dbmd.action?id=d8\">解放军</a> </li>]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urlprovince = dict()\n",
    "for i in qq:\n",
    "    urlprovince[i.text]=urljoin(url,i.a.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NameUrl = []\n",
    "urlalready = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\Anakin\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "ename": "ChunkedEncodingError",
     "evalue": "('Connection broken: IncompleteRead(1009 bytes read, 722 more expected)', IncompleteRead(1009 bytes read, 722 more expected))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIncompleteRead\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36m_error_catcher\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    297\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 298\u001b[0;31m                 \u001b[1;32myield\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    299\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m    592\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 593\u001b[0;31m                 \u001b[0mchunk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle_chunk\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    594\u001b[0m                 decoded = self._decode(chunk, decode_content=decode_content,\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36m_handle_chunk\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    557\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# amt > self.chunk_left\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 558\u001b[0;31m             \u001b[0mreturned_chunk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_safe_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchunk_left\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    559\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_safe_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# Toss the CRLF at the end of the chunk.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_safe_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    613\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mchunk\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 614\u001b[0;31m                 \u001b[1;32mraise\u001b[0m \u001b[0mIncompleteRead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb''\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mamt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    615\u001b[0m             \u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIncompleteRead\u001b[0m: IncompleteRead(1009 bytes read, 722 more expected)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mProtocolError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\models.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m()\u001b[0m\n\u001b[1;32m    718\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 719\u001b[0;31m                     \u001b[1;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mchunk_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecode_content\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    720\u001b[0m                         \u001b[1;32myield\u001b[0m \u001b[0mchunk\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36mstream\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m    427\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchunked\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msupports_chunked_reads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 428\u001b[0;31m             \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_chunked\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecode_content\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdecode_content\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    429\u001b[0m                 \u001b[1;32myield\u001b[0m \u001b[0mline\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36mread_chunked\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m    617\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_response\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 618\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_response\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"generator didn't stop after throw()\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\packages\\urllib3\\response.py\u001b[0m in \u001b[0;36m_error_catcher\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    315\u001b[0m                 \u001b[1;31m# This includes IncompleteRead.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m                 \u001b[1;32mraise\u001b[0m \u001b[0mProtocolError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Connection broken: %r'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mProtocolError\u001b[0m: ('Connection broken: IncompleteRead(1009 bytes read, 722 more expected)', IncompleteRead(1009 bytes read, 722 more expected))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mChunkedEncodingError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-87-532396e563ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'NameUrl = []\\nfor i in urlprovince:\\n    bs = BeautifulSoup(requests.get(urlprovince[i]).content)\\n    tmpname = bs.find_all(\"li\")\\n    for j in tmpname:\\n        url = urljoin(urlprovince[i],j.a.get(\\'href\\'))\\n        NameUrl.append(parsepage(url))'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2113\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   2114\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2115\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2116\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   2117\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\IPython\\core\\magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[1;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\IPython\\core\\magics\\execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1178\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m             \u001b[0mst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m             \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-62-601c0e0bc6e6>\u001b[0m in \u001b[0;36mparsepage\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mparsepage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mbs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mq\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"td\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mtmp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'allow_redirects'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m     \u001b[1;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'get'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[1;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[1;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    486\u001b[0m         }\n\u001b[1;32m    487\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    639\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 641\u001b[0;31m             \u001b[0mr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    642\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    643\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\models.py\u001b[0m in \u001b[0;36mcontent\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    795\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_content\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    796\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 797\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_content\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbytes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miter_content\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCONTENT_CHUNK_SIZE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mbytes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    798\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    799\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_content_consumed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\requests\\models.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m()\u001b[0m\n\u001b[1;32m    720\u001b[0m                         \u001b[1;32myield\u001b[0m \u001b[0mchunk\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mProtocolError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m                     \u001b[1;32mraise\u001b[0m \u001b[0mChunkedEncodingError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    723\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mDecodeError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    724\u001b[0m                     \u001b[1;32mraise\u001b[0m \u001b[0mContentDecodingError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mChunkedEncodingError\u001b[0m: ('Connection broken: IncompleteRead(1009 bytes read, 722 more expected)', IncompleteRead(1009 bytes read, 722 more expected))"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in urlprovince:\n",
    "    bs = BeautifulSoup(requests.get(urlprovince[i]).content)\n",
    "    tmpname = bs.find_all(\"li\")\n",
    "    for j in tmpname:\n",
    "        if j.a.get('href') not in urlalready:\n",
    "            urlalready.add(j.a.get('href'))\n",
    "            url = urljoin(urlprovince[i],j.a.get('href'))\n",
    "            NameUrl.append(parsepage(url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame(NameUrl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data2 = data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2919"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data2.to_csv(\"人大代表2.csv\",index=False,encoding=\"gbk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data2.to_csv(\"人大代表.csv\",index=False,encoding=\"utf8\",)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data2.to_excel(\"人大代表.xlsx\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 获取个人的链接测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Anakin\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\Anakin\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "bss =BeautifulSoup(requests.get(urljoin(url,qq[0].a.get('href'))).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<li><a href=\"viewDelegate.action?dbid=120001\" target=\"_blank\">马旭（满族）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120002\" target=\"_blank\">马振川</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120003\" target=\"_blank\">王万宾</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120004\" target=\"_blank\">王小珂（女，回族）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120005\" target=\"_blank\">王文京</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120006\" target=\"_blank\">王尔乘</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120007\" target=\"_blank\">王全</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120008\" target=\"_blank\">王安顺</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120009\" target=\"_blank\">王岐山</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120010\" target=\"_blank\">王青海</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=122251\" target=\"_blank\">王炳深</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120011\" target=\"_blank\">王晓初</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120012\" target=\"_blank\">王铮</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120013\" target=\"_blank\">王蓉蓉（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=123001\" target=\"_blank\">巨晓林</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120014\" target=\"_blank\">方新（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120015\" target=\"_blank\">邓中翰</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120016\" target=\"_blank\">冯乐平（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120017\" target=\"_blank\">朱良玉</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120018\" target=\"_blank\">朱惠刚</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120019\" target=\"_blank\">刘忠军</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120245\" target=\"_blank\">刘振所</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120020\" target=\"_blank\">刘晓晨</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=122804\" target=\"_blank\">刘晓翠（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120021\" target=\"_blank\">刘新成</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120022\" target=\"_blank\">闫傲霜（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120023\" target=\"_blank\">池强</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120024\" target=\"_blank\">苏辉（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120025\" target=\"_blank\">杜德印</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120026\" target=\"_blank\">李士祥</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120027\" target=\"_blank\">李大进</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=123051\" target=\"_blank\">李伟</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120028\" target=\"_blank\">李昭玲（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120029\" target=\"_blank\">李超钢</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120030\" target=\"_blank\">杨晓超</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120031\" target=\"_blank\">吴正宪（女，蒙古族）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120032\" target=\"_blank\">吴世雄（满族）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120033\" target=\"_blank\">吴碧霞（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120035\" target=\"_blank\">张大勇</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120036\" target=\"_blank\">张和平</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=121137\" target=\"_blank\">陈立国</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120037\" target=\"_blank\">陈吉宁</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120038\" target=\"_blank\">陈雨露</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120040\" target=\"_blank\">欧阳淞</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120041\" target=\"_blank\">周其凤</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120990\" target=\"_blank\">庞丽娟（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120042\" target=\"_blank\">赵郁</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120043\" target=\"_blank\">赵凯</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120044\" target=\"_blank\">荣华（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120188\" target=\"_blank\">段铁力</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120045\" target=\"_blank\">段强</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120046\" target=\"_blank\">钱渊（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120047\" target=\"_blank\">徐和谊（回族）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120049\" target=\"_blank\">高丽朴（女）</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120050\" target=\"_blank\">郭金龙</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120051\" target=\"_blank\">梁伟</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=121630\" target=\"_blank\">敬大力</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120052\" target=\"_blank\">韩晓武</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120053\" target=\"_blank\">程京</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120054\" target=\"_blank\">慕平</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=123052\" target=\"_blank\">蔡奇</a>\n",
       " </li>, <li><a href=\"viewDelegate.action?dbid=120055\" target=\"_blank\">魏哲</a>\n",
       " </li>]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bss.find_all(\"li\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "requests()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}