Python爬虫学习第一天--利用正则表达式爬取图片

 1 #!/usr/bin/env python
 2 # -*- coding: utf-8 -*-
 3 # @Time : 2018/7/10 22:34
 4 # @Author : chenxiaowei
 5 # @Email : chen1020xiaowei@163.com
 6 # @File : parse_meinv.py
 7 ###利用正则表达式匹配字符串爬取***的美女图片，保存相关数据到MongoDB而且把相关图片保存在本地###
 8 from parse_config import *
 9 import requests 10 import re 11 import json 12 import time 13 import pymongo 14 from requests.exceptions import RequestException 15 import hashlib 16 
17 db_client = pymongo.MongoClient(mongo_url)  # 初始化MongoDB数据库对象
18 db = db_client[mongo_database]  # 引用实例
19 
20 
21 def get_responses(url):  # 定义获取response函数
22     try: 23         responses = requests.get(url, headers=headers) 24         if responses.status_code == 200:  # 判断是否请求成功，利用.text方法返回html代码
25             return responses.text 26         else: 27             return None 28     except RequestException:  # 捕获父类异常
29         print('error1') 30         return None 31 
32 
33 def get_image_content(url):  # 定义函数
34     try: 35         responses = requests.get(url, headers=headers) 36         if responses.status_code == 200: 37             return responses.content  # 利用.content方法返回二进制文件
38         else: 39             return None 40     except RequestException: 41         print('error2') 42         return None 43 
44 
45 def download_image(content): 46     filename = '{0}.{1}'.format(hashlib.md5(content).hexdigest(), 'jpg')  # 字符串的通配方法
47     with open(path_image.format(filename), 'wb')as f:  # 'wb'保存图片
48  f.write(content) 49         print(filename,'下载成功！') 50  f.close() 51 
52 
53 def get_url_items(html): 54     pattern = re.compile('<li>.*?<a.*?href="(.*?)".*?class="TypeBigPics".*?src="(.*?)".*?<span>(.*?)</span>'
55                          + '.*?class="IcoList">(.*?)</em>.*?class="IcoTime">(.*?)</em>', re.S) 56     items = re.findall(pattern, html)  # 利用re库的compile方法构造正则表达式，findall方法获取items
57     for item in items: 58         yield { 59             '名称': item[2], 60             '壁纸': item[1], 61             '网址': item[0], 62             '发布日期': item[4], 63             '查看次数': item[3][3:] 64         }  # yield生成器，被调用时才赋值
65         content = get_image_content(item[1]) 66  download_image(content) 67 
68 
69 def save_to_file(filename,file_type,text):  # 保存至本地
70     with open('{}{}{}'.format(path_txt, filename,file_type), 'a', encoding='utf-8', )as wf:  # 以utf-8的编码方式追加到文件
71         wf.write(json.dumps(text, ensure_ascii=False) + '\n')  # 解码相关json格式
72         print(text,'写入到本地成功！') 73  wf.close() 74 
75 
76 def save_to_mongo(text):  # 存储到MongoDB
77     if db[mongo_table].insert(text): 78         print(text,'写入Mongo成功！') 79         return True 80     return False 81 
82 
83 def main(filename, page): 84     url = 'http://www.***/bizhitupian/meinvbizhi/{}.htm'.format(page) 85     html = get_responses(url) 86     items = get_url_items(html) 87     for item in items: 88  save_to_file(filename,file_type, item) 89  save_to_mongo(item) 90 
91 
92 if __name__ == '__main__': 93     for page in range(start_page, end_page + 1): 94  main(file, page) 95         time.sleep(15)#等待15秒，防止被识别

 1 #!/usr/bin/env python
 2 # -*- coding: utf-8 -*-
 3 # @Time : 2018/7/10 22:35
 4 # @Author : chenxiaowei
 5 # @Email : chen1020xiaowei@163.com
 6 # @File : parse_config.py
 7 mongo_url = 'localhost'
 8 mongo_database = 'youmeiwang'
 9 mongo_table = 'meinv'
10 headers = { 11     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.15 Safari/537.36'
12 } 13 file = '***美女图片.txt'
14 path_image = 'H:/Python_download/20180710/image/{}'
15 path_txt = 'H:/Python_download/20180710/file/'
16 filename = '***美女图片'
17 file_type = '.txt'
18 start_page = 1
19 end_page = 44