scrapy添加下载中间键 User-agent

设置 user -agent 和  打开（注意路径）

DOWNLOADER_MIDDLEWARES = {
   #'news.middlewares.NewsDownloaderMiddleware': 543,
   'news.middlewares.UserAgentMiddleware': 300,
}

#随机换user-agent
class UserAgentMiddleware(object):
    """
        给每个请求随机切换一个User-Agent
    """
    def process_request(self, request, spider):
        user_agent = random.choice(us_list)
        request.headers['User-Agent'] = user_agent
注意这样能够打印

request.headers['User-Agent']