php抓取网页内容-实现多线程下载工具
(2011-12-19 15:02:16)
标签:
it |
作者:赵鹏伟
闲时,研究了下php的curl,发现
用php的curl功能实现的多线程下载工具,比file_get_contents,以及linux自带的命令行curl、wget效率高多了,我亲自测试过。
function multiDownload($urlArray) {if (empty($urlArray)) return false; $isStr = false; if (is_string($urlArray)) { $urlArray = array($urlArray); $isStr = true; } self::log(sprintf("%s Multi thread download begin...", __METHOD__)); $mh = curl_multi_init(); //curl_multi_init -- Returns a new cURL multi handle $curlArray = array(); foreach ($urlArray as $i => $url) { self::log(sprintf("%s Download url: |%s|...", __METHOD__, $url)); $curlArray[$i] = curl_init($url); curl_setopt($curlArray[$i], CURLOPT_RETURNTRANSFER, true); //设置为true表示返回抓取的内容,而不是直接输出到浏览器上。TRUE to return the transfer as a string of the return value of curl_exec() instead of outputting it out directly curl_setopt($curlArray[$i], CURLOPT_AUTOREFERER, true); //自动设置referer。TRUE to automatically set the Referer: field in requests where it follows a Location: redirect. curl_setopt($curlArray[$i], CURLOPT_FOLLOWLOCATION, true); //跟踪url的跳转,比如301, 302等 curl_setopt($curlArray[$i], CURLOPT_MAXREDIRS, 2); //跟踪最大的跳转次数 curl_setopt($curlArray[$i], CURLOPT_HEADER, 0); //TRUE to include the header in the output. curl_setopt($curlArray[$i], CURLOPT_ENCODING, ""); //接受的编码类型,The contents of the "Accept-Encoding: " header. This enables decoding of the response. Supported encodings are "identity", "deflate", and "gzip". If an empty string, "", is set, a header containing all supported encoding types is sent. curl_setopt($curlArray[$i], CURLOPT_CONNECTTIMEOUT, 5); //连接超时时间 curl_multi_add_handle($mh, $curlArray[$i]); //curl_multi_add_handle -- Add a normal cURL handle to a cURL multi handle } $running = NULL; $count = 0; do { //10秒钟没退出,就超时退出 if ($count++>100) break; usleep(100000); curl_multi_exec($mh, $running); //curl_multi_exec -- Run the sub-connections of the current cURL handle } while($running > 0); $content = array(); foreach ($urlArray as $i => $url) { $content[$url] = curl_multi_getcontent($curlArray[$i]); //curl_multi_getcontent -- Return the content of a cURL handle if CURLOPT_RETURNTRANSFER is set } //curl_multi_remove_handle -- Remove a multi handle from a set of cURL handles foreach ($urlArray as $i => $url){ curl_multi_remove_handle($mh, $curlArray[$i]); } //curl_multi_close -- Close a set of cURL handles curl_multi_close($mh); self::log(sprintf("%s Multi thread download end...", __METHOD__)); //如果参数$urlArray是字符串,则将返回值也转换为字符串 if ($isStr) $content = implode('', $content); return $content; }