阅读 128

PHP爬虫 -- 018 作业源码(豆瓣top250)

首先, 创建数据表

CREATE TABLE `douban_movie` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `title` varchar(255) NOT NULL,
  `img` varchar(255) NOT NULL,
  `info` varchar(255) NOT NULL,
  `rank` int(11) NOT NULL,
  `score` decimal(2,1) NOT NULL,
  `comment` varchar(255) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
复制代码

编写爬虫代码

<?php
require 'vendor/autoload.php';
use Medoo\Medoo;
use QL\QueryList;
// 创建数据库连接
$database = new medoo([
    'database_type' => 'mysql',
    'database_name' => 'demo_db',
    'server' => 'localhost',
    'username' => 'root',
    'password' => 'root',
    'charset' => 'utf8',
]);

/* 
 * @Description: 通过目标url, 获取对应网页中我们需要的数据
 * @param: 目标url
 * @return: 包含数据的二维数组
*/ 
function get_data($url) {
    echo "function get_data is running .... \n";
    echo $url . "\n";
    $ql = new QueryList();
    $html_source = get_html_source($url);
    $data = $ql->html($html_source)->rules([
        'title' => ['#content > div > div.article > ol > li > div > div.info > div.hd > a > span:nth-child(1)', 'text'],
        'img' => ['#content > div > div.article > ol > li > div > div.pic > a > img', 'src'],
        'rank' => ['#content > div > div.article > ol > li > div > div.pic > em', 'text'],
        'score' => ['#content > div > div.article > ol > li > div > div.info > div.bd > div > span.rating_num', 'text'],
        'info' => ['#content > div > div.article > ol > li > div > div.info > div.bd > p:nth-child(1)', 'html'],
    ])->queryData();
    // 针对comment单独处理, 如果没有, 就自定义comment内容
    foreach ($data as $key => $value) {
        $num = $key + 1;
        $value['comment'] = $ql->html($html_source)->find("#content > div > div.article > ol > li:nth-child({$num}) > div > div.info > div.bd > p.quote > span")->text();
        if (!$value['comment']) {
            $value['comment'] = "好看到无话可说!";
        }
        $data[$key] = $value;
    }
    // 返回数据
    return $data;
}
/* 
 * @Description: 保存数据
 * @param: 二维数组
 * @return: 没有返回值
*/ 
function save_data($data) {
    echo "function save_data is running .... \n";
    global $database;
    $database->insert('douban_movie250', $data);
}
/* 
 * @Description: 使用IP代理访问url并获取html代码
 * @param: 目标url
 * @return: html代码
*/ 
function get_html_source($url) {
    echo "function get_html_source is running .... \n";
    $result = false;
    while (!$result) {
        $targetUrl = $url;
        $proxyServer = "http://http-dyn.abuyun.com:9020";
        $proxyUser = "H19D75L76VK89Q8D";
        $proxyPass = "8C17B0A80F475BD8";
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $targetUrl);
        curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, false);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
        curl_setopt($ch, CURLOPT_PROXY, $proxyServer);
        curl_setopt($ch, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
        curl_setopt($ch, CURLOPT_PROXYUSERPWD, "{$proxyUser}:{$proxyPass}");
        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727;)");
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 3);
        curl_setopt($ch, CURLOPT_TIMEOUT, 5);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $result = curl_exec($ch);
        if (!$result) {
            sleep(2);
        }
        curl_close($ch);
    }
    return $result;

}

// 遍历, 生成需要的10条url, 然后从每条url中获取数据, 并保存到数据库
for ($i = 0; $i < 10; $i++) {
    $page = $i * 25;
    $url = "https://movie.douban.com/top250?start=" . $page;
    save_data(get_data($url)); // 获取数据并保存
}
复制代码

看一下最终数据库的效果

从数据中读取数据, 写入markdown文件

<?php
require 'vendor/autoload.php';
use Medoo\Medoo;
// 创建数据库连接
$database = new medoo([
    'database_type' => 'mysql',
    'database_name' => 'demo_db',
    'server' => 'localhost',
    'username' => 'root',
    'password' => 'root',
    'charset' => 'utf8',
]);

/* 
 * @Description: 从数据库中获取数据
 * @return: 整理后的数据
*/ 
function get_data(){
    global $database;
    $data = $database->select('douban_movie250','*');
    // 我们需要把原来的电影信息, 分成两行
    foreach ($data as $key => $value) {
        $tmp = explode('<br>',$value['info']);
        $value['movie_info'][0] = trim($tmp[0]);
        $value['movie_info'][1] = trim($tmp[1]);
        $data[$key] = $value;
    }
    return $data;
}

/* 
 * @Description: 通过遍历数组, 生成markdown文档
 * @param: 二维数组
 * @return: 没有返回值
*/ 
function make_markdown($data){
    $md_obj = fopen('douban_markdown.md','w+');
    foreach($data as $key => $value){
        $title = $value['title'];
        $img = $value['img'];
        $rank = $value['rank'];
        $score = $value['score'];
        $info = $value['info'];
        $comment = $value['comment'];
        $movie_info = $value['movie_info'];
        fwrite($md_obj,"![]({$img})\n");
        fwrite($md_obj,"## {$rank}-{$title}-{$score}\n");
        fwrite($md_obj,"```\n");
        fwrite($md_obj,"{$movie_info[0]}\n");
        fwrite($md_obj,"{$movie_info[1]}\n");
        fwrite($md_obj,"```\n");
        fwrite($md_obj,"> ### {$comment}\n");
        fwrite($md_obj,"---\n");
    }
    fclose($md_obj);
}
// 获取数组, 并生成markdown文档
make_markdown(get_data());
复制代码

最终效果

关注下面的标签,发现更多相似文章
评论