> 安装 QueryList `composer require jaeger/querylist` ``` ~~~ <?php namespace app\common\service; use QL\QueryList; use think\Db; use think\Exception; class PaContent { /** * @return array * 获取数据 */ public static function save_data(){ $page=self::get_page();//获取分页 if ($page >27){ return ['code'=>0,'msg'=>'分页数据最大27页']; } ini_set('max_execution_time', 600);//秒为单位，自己根据需要定义 ini_set('memory_limit', '256M');//内存 $url = 'https://www.cbf.com.br/pelo-mundo/noticias/index?page='.$page; // 采集规则 $rules = [ // 文章标题 'title' => ['div>h2>a','text'], 'img' => ['.pull-left>a>img','src'], 'link' => ['div>h2>a','href'], ]; // 切片选择器 $range = '.news'; $data=QueryList::get($url)->rules($rules)->query()->range($range)->queryData();//多个数组 if (empty($data)){ return ['code'=>0,'msg'=>'未爬取到数据']; } $arr=[]; foreach ($data as $k=>$v){ $content=self::get_content_detail($v['link']); if ($content['code']==0){ continue ; } $arr[$k]['content']=$content['data']; $arr[$k]['title']=$v['title']; $img=self::getImage($v['img']); $arr[$k]['image']=$img['save_path']; //$arr[$k]['link']='https://www.cbf.com.br'.$v['link']; $arr[$k]['createtime']=time()-mt_rand(50,900); } return self::save_data_to_database($arr,$page); } protected static function set_page($page){ @file_put_contents('num.txt',$page+1); return true; } protected static function get_page(){ $page=@file_get_contents('num.txt'); return intval($page); } /** * @param $arr * @param $page * @return array * 保存数据 */ protected static function save_data_to_database($arr,$page){ if (empty($arr)) return ['code'=>0,'msg'=>'无数据可保存']; $result=Db::name('content')->insertAll($arr); if ($result){ self::set_page($page); //更新下次分页次数 return ['code'=>1,'msg'=>'数据保存成功条数：'.$result]; } return ['code'=>0,'msg'=>'数据保存失败']; } /** * @param $url * @return string|string[]|null * 获取内容详细html */ protected static function get_content_detail($url){ $url='https://www.cbf.com.br'.$url; try { $ql = QueryList::get($url, null, [ 'headers' => [ 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', 'Accept-Encoding' => 'gzip, deflate, br', ] ]); }catch (Exception $exception){ return ['code'=>0,'msg'=>'获取详细内容异常']; } //获取内容html $da=$ql->find(".article-inner-content")->html(); //去掉图片 $content= preg_replace('/<\s*img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i','',$da); return ['code'=>1,'msg'=>'ok','data'=>$content]; } /** * @param $url * @param string $save_dir * @param int $type * @return array * 远程图片本地化 */ public static function getImage($url,$save_dir='./content_img/',$type=0){ if(trim($url)==''){ return array('file_name'=>'','save_path'=>'','error'=>1); } $ext=strrchr($url,'.'); if($ext!='.gif'&&$ext!='.jpg'&&$ext!='.png'&&$ext!='.jpeg'){ return array('file_name'=>'','save_path'=>'','error'=>2); } $filename=time().$ext; //创建保存目录 if(!file_exists($save_dir)&&!mkdir($save_dir,0777,true)){ return array('file_name'=>'','save_path'=>'','error'=>3); } //获取远程文件所采用的方法 ob_start(); readfile($url); $img=ob_get_contents(); ob_end_clean(); //$size=strlen($img); //文件大小 $fp2=@fopen($save_dir.$filename,'a'); fwrite($fp2,$img); fclose($fp2); unset($img,$url); return array('file_name'=>$filename,'save_path'=>'/content_img/'.$filename,'error'=>0); } } ~~~ ```