python学习笔记之爬虫.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 爬虫"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Tue May 16 14:07:25 2017\n",
    "\n",
    "@author: situ.st.1\n",
    "\"\"\"\n",
    "#Python通过re模块提供对正则表达式的支持。使用re的一般步骤是先将正则表达式的字符串形式编译为Pattern实例，\n",
    "#然后使用Pattern实例处理文本并获得匹配结果（一个Match实例），最后使用Match实例获得信息，进行其他的操作。\n",
    "\n",
    "#爬取网络图片------------------------------------\n",
    "import urllib\n",
    "import re\n",
    "\n",
    "# 运行脚本将得到整个页面中包含图片的URL地址。\n",
    "url = \"https://movie.douban.com/subject/26743573/photos?type=S&start=0&sortby=vote&size=a&subtype=a\"\n",
    "page = urllib.urlopen(url)\n",
    "html = page.read()\n",
    "\n",
    "reg = r'src=\"(.+?\\.jpg)\" pic_ext'\n",
    "imgre = re.compile(reg) #re.compile() 可以把正则表达式编译成一个正则表达式对象.\n",
    "imglist = re.findall(imgre,html) #re.findall() 方法读取html 中包含 imgre（正则表达式）的数据。\n",
    "print imglist#68条数据\n",
    "\n",
    "x = 0\n",
    "for imgurl in imglist:#\n",
    "    urllib.urlretrieve(imgurl,'%s.jpg' % x)\n",
    "    x+=1\n",
    "\n",
    "#这里的核心是用到了urllib.urlretrieve()方法，直接将远程数据下载到本地。\n",
    "#通过一个for循环对获取的图片连接进行遍历，为了使图片的文件名看上去更规范，对其进行重命名，命名规则通过x变量加1。保存的位置默认为程序的存放目录。\n",
    "\n",
    "#保存豆瓣欢乐颂图片\n",
    "import urllib\n",
    "import re\n",
    "\n",
    "url = \"https://movie.douban.com/subject/26743573/photos?type=S&start=0&sortby=vote&size=a&subtype=a\"\n",
    "page = urllib.urlopen(url)\n",
    "html = page.read()\n",
    "#说明: 正则表达式中 '.*?', 采用非贪婪模式匹配多个字符 \n",
    "reg = r'<a href=\"(https://movie.douban.com/photos/photo/.*?)\">'\n",
    "imgre = re.compile(reg) #re.compile() 可以把正则表达式编译成一个正则表达式对象.\n",
    "imghtmllist = re.findall(imgre,html) #re.findall() 方法读取html 中包含 imgre（正则表达式）的数据。\n",
    "for website in imghtmllist:  #stock_last：整理后的股票数据\n",
    "    if '#' in website:\n",
    "        imghtmllist.remove(website)\n",
    "                        \n",
    "print imghtmllist#68条数据\n",
    "\n",
    "#未运行成功\n",
    "for website in imghtmllist:\n",
    "    url = website\n",
    "    page = urllib.urlopen(url)\n",
    "    html = page.read()\n",
    "    reg = r'href=\"(https://img3.doubanio.com/view/photo/raw/public/.*?\\.jpg)\"'\n",
    "    imgre = re.compile(reg)\n",
    "    imglist = re.findall(imgre,html)\n",
    "    print imglist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import urllib\n",
    "import urllib2\n",
    "import re\n",
    "import httplib2\n",
    "\n",
    "url='http://quote.stockstar.com/stock/ranklist_a_3_1_1.html'  #目标网址\n",
    "headers={\"User-Agent\":\"Mozilla/5.0 (Windows NT 10.0; WOW64)\"}  #伪装浏览器请求报头\n",
    "request=urllib2.Request(url=url,headers=headers)  #请求服务器\n",
    "response=urllib2.urlopen(request)  #服务器应答\n",
    "content=response.read().decode('gbk')   #以一定的编码方式查看源码\n",
    "print(content)  #打印页面源码 \n",
    "\n",
    "pattern=re.compile(r'<tbody[\\s\\S]*</tbody>')  \n",
    "body=re.findall(pattern,content)  #匹配<tbody和</tbody>之间的所有代码\n",
    "pattern=re.compile('>(.*?)<')\n",
    "stock_page=re.findall(pattern,body[0])  #匹配>和<之间的所有信息\n",
    "\n",
    "# 通过非贪婪模式(.*?)匹配>和<之间的所有数据，会匹配出一些空白字符出来，所以我们采用如下代码把空白字符移除。\n",
    "stock_total = stock_page\n",
    "stock_last=stock_total[:] #stock_total：匹配出的股票数据\n",
    "for data in stock_total:  #stock_last：整理后的股票数据\n",
    "    if data=='':\n",
    "        stock_last.remove('')\n",
    "        \n",
    "        print('代码','\\t','简称','   ','\\t','最新价','\\t','涨跌幅','\\t','涨跌额','\\t','5分钟涨幅')\n",
    "for i in range(0,len(stock_last),13):        #网页总共有13列数据\n",
    "    print(stock_last[i],'\\t',stock_last[i+1],' ','\\t',stock_last[i+2],'  ','\\t',stock_last[i+3],'  ','\\t',stock_last[i+4],'  ','\\t',stock_last[i+5])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}