ruby 爬虫 Spidr 用法

从一个网址开始爬:

Spidr.start_at('http://tenderlovemaking.com/')

爬一个host:

Spidr.host('coderrr.wordpress.com')

爬一个网站：

Spidr.site('http://rubyflow.com/')

多个 hosts:

Spidr.start_at(
  'http://company.com/',
  :hosts => [
    'company.com',
    /host\d\.company\.com/
  ]
)

忽略某个地址:

Spidr.site('http://matasano.com/', :ignore_links => [/log/])

忽略端口:

Spidr.site(
  'http://sketchy.content.com/',
  :ignore_ports => [8000, 8010, 8080]
)

打印出爬到的网址:

Spidr.site('http://rubyinside.org/') do |spider|
  spider.every_url { |url| puts url }
end

打印出失败的网址:

Spidr.site('http://sketchy.content.com/') do |spider|
  spider.every_failed_url { |url| puts url }
end

找出全部包含坏链的页面：

url_map = Hash.new { |hash,key| hash[key] = [] }

spider = Spidr.site('http://intranet.com/') do |spider|
  spider.every_link do |origin,dest|
    url_map[dest] << origin
  end
end

spider.failures.each do |url|
  puts "Broken link #{url} found in:"

  url_map[url].each { |page| puts "  #{page}" }
end

搜索 HTML 或 XML 页面:

Spidr.site('http://company.withablog.com/') do |spider|
  spider.every_page do |page|
    puts "[-] #{page.url}"

    page.search('//meta').each do |meta|
      name = (meta.attributes['name'] || meta.attributes['http-equiv'])
      value = meta.attributes['content']

      puts "    #{name} = #{value}"
    end
  end
end

打印出每一个页面的title:

Spidr.site('http://www.rubypulse.com/') do |spider|
  spider.every_html_page do |page|
    puts page.title
  end
end

访问表头来获取网站用的那种web servers:

servers = Set[]

Spidr.host('generic.company.com') do |spider|
  spider.all_headers do |headers|
    servers << headers['server']
  end
end

在被禁止的页面中止spider:

spider = Spidr.host('overnight.startup.com') do |spider|
  spider.every_forbidden_page do |page|
    spider.pause!
  end
end

跳过处理一些页面:

Spidr.host('sketchy.content.com') do |spider|
  spider.every_missing_page do |page|
    spider.skip_page!
  end
end

跳过处理一些连接:

Spidr.host('sketchy.content.com') do |spider|
  spider.every_url do |url|
    if url.path.split('/').find { |dir| dir.to_i > 1000 }
      spider.skip_link!
    end
  end
end