PHP爬虫 -- 020 做业解析 爬取BOSS直聘的招聘信息

做业代码

<?php

require 'vendor/autoload.php';
use Medoo\Medoo;
use QL\QueryList;

$ql = new QueryList();
// 计数, 用来估算进度
$count = 0;
// 建立数据库链接
$database = new medoo([
    'database_type' => 'mysql',
    'database_name' => 'demo_db',
    'server' => 'localhost',
    'username' => 'root',
    'password' => 'root',
    'charset' => 'utf8',
]);
// for循环, 循环给一个很大的数字, 判断是否有下一页, 若是没有, 则跳出循环
for ($i=1; $i < 99999; $i++) { 
    $url = "https://www.zhipin.com/c101180100/?query=php&page=".$i;
    $url_list = get_url_list($url);
    foreach ($url_list as $key => $value) {
        get_and_save_data($value['url']);
    }
    // 判断有没有下一页
    if(is_end($url)){
        break;
    }

}
/* * @Description: 判断目标url有没有下一页 * @param: 目标url * @return: 没有下一页则返回true, 不然false */ 
function is_end($url){
    global $ql;
    $class_content = $ql->html(get_html_source($url))->find('a[ka="page-next"]')->class;
    if($class_content == "next disabled"){
        return true;
    }else{
        return false;
    }
}
/* * @Description: 获取各个公司, 各个工做,详情页的超连接 * @param: 总的分页url * @return: 超连接列表 */ 
function get_url_list($url){
    global $ql;
    echo $url."\n";
    $data = $ql->html(get_html_source($url))->rules([
        "url"=>['#main > div > div.job-list > ul > li > div > div.info-primary > h3 > a','href','',function($content){
            return "https://www.zhipin.com".$content;
        }]
    ])->queryData();
    return $data;


}
/* * @Description: 从详情页获取相关数据并保存到数据库 * @param: 详情页url * @return: 没有返回值, 会显示进度 */ 
function get_and_save_data($url){
    global $ql,$database,$count;
    
    
    $html_source = get_html_source($url);
    $data = $ql->html($html_source)->rules([
        "company_name"=>['.job-sec>.name','text'],
        "job_name"=>['.info-primary>.name>h1','text'],
        "info"=>['.job-primary.detail-box>.info-primary>p','html'],
        "address"=>['.location-address','text'],
        "description"=>['.job-sec:nth-child(1)>.text','text'],
        "salary"=>['.salary','text']
    ])->queryData();
    foreach ($data as $key => $value) {
        // 若是公司名获取不到, 则从另一个位置获取
        if (!$value['company_name']) {
            $value['company_name'] = $ql->html($html_source)->find('a[ka="job-detail-company_custompage"]')->text();
        }
        $value['url'] = $url;
        $value['city'] = explode('<em class="dolt"></em>',$value['info'])[0];
        $value['work_year'] = explode('<em class="dolt"></em>',$value['info'])[1];
        $value['education'] = explode('<em class="dolt"></em>',$value['info'])[2];
        unset($value['info']); // 删除info, 保证能够写入数据库
        $data[$key] = $value;
    }

    
    $database->insert('jobs',$data);
    $count++;
    $percent = round($count/3);
    echo "当前进度: {$percent}%\n";
}

/* * @Description: IP代理, 使用代理获取html代码 * @param: 目标url * @return: 目标url的html代码 */ 
function get_html_source($url){
    $result = false;
    while (!$result) {
        $targetUrl = $url;
        $proxyServer = "http://http-dyn.abuyun.com:9020";
        $proxyUser = "H19D75L76VK89Q8D";
        $proxyPass = "8C17B0A80F475BD8";
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $targetUrl);
        curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, false);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
        curl_setopt($ch, CURLOPT_PROXY, $proxyServer);
        curl_setopt($ch, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
        curl_setopt($ch, CURLOPT_PROXYUSERPWD, "{$proxyUser}:{$proxyPass}");
        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727;)");
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 3);
        curl_setopt($ch, CURLOPT_TIMEOUT, 5);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $result = curl_exec($ch);
        if (!$result) {
            sleep(2);
        }
        curl_close($ch);
    }
    return $result;
}
复制代码

数据库中的数据