scrapy爬取豆瓣Top250电影

此次咱们爬取的内容

准备步骤

1. 找到html格式网页中须要爬取的数据的xpath
   例如咱们须要爬取图片的url

   
   
   这里用的是xPath Checker不会用的同窗请百度

2.而后咱们开始创建工程 打开cmd 而后在你想要创建工程的目录下面 输入 scrapy startproject douban
就会自动创建一个工程 而后去根目录创建一个run.py 去spiders这个目录里创建一个douban_spiders.py（注意这里的主爬虫文件和项目名称不能相同 否则会报错）

源码

# run.py
from scrapy import cmdline
cmdline.execute("scrapy crawl douban".split())

# douban_spiders.py
#coding:utf-8  
import scrapy  
from douban.items import DoubanItem  
   
from scrapy.crawler import CrawlerProcess  
   
class doubanSpider(scrapy.Spider):  
    name = 'douban'  
    allowed_domains = ["douban.com"]  
    start_urls = ["https://movie.douban.com/top250"]  
       
    def parse(self, response):  
        item = DoubanItem()  
        item['image_urls'] = response.xpath('//div[@class="pic"]//img//@src').extract()#提取图片连接  
        # print 'image_urls',item['image_urls']  
        item['title'] = response.xpath('//div[@class="hd"]/a/span[1]/text()').extract()#提取电影标题 
        # print 'title',item['title']  
        item['quote'] = response.xpath('//p[@class="quote"]/span/text()').extract()#提取简介
        # print 'quote',item['quote']  
        item['level'] = response.xpath('//em/text()').extract()#提取排名
        # print 'level',item['level']  
        yield item    
        new_url= "https://movie.douban.com/top250" + response.xpath('//span[@class="next"]/link/@href').extract_first()#翻页  
        # print 'new_url',new_url  
        if new_url:  
            yield scrapy.Request(new_url,callback=self.parse)

# items.py
import scrapy

class DoubanItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    image_urls = scrapy.Field()
    title = scrapy.Field()
    quote = scrapy.Field()
    level = scrapy.Field()

# pipelines.py
import os  
import urllib 

from douban import settings  

class DoubanPipeline(object):
    def process_item(self, item, spider):  
        i = 0
        dir_path = '%s/%s'%(settings.IMAGES_STORE,spider.name)#存储路径  
        print 'dir_path',dir_path  
        if not os.path.exists(dir_path):  
            os.makedirs(dir_path)  
        for image_url in item['image_urls']:  
            file_name = "Top" + item['level'][i] + ' ' +item['title'][i] + '('+item['quote'][i]+ ").jpg"#图片名称  
            i = i + 1
            # print 'filename',file_name  
            file_path = '%s/%s'%(dir_path,file_name)  
            # print 'file_path',file_path  
            if os.path.exists(file_name):  
                continue  
            with open(file_path,'wb') as file_writer:  
                conn = urllib.urlopen(image_url)#下载图片  
                file_writer.write(conn.read())  
            file_writer.close()  
        return item

# setting.py
BOT_NAME = 'douban'

SPIDER_MODULES = ['douban.spiders']
NEWSPIDER_MODULE = 'douban.spiders'

ITEM_PIPELINES = {  
   'douban.pipelines.DoubanPipeline': 1,  
}  
IMAGES_STORE='E:'  
DOWNLOAD_DELAY = 0.25

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'

最终爬取的结果