CRAWLER DE SITE EN PHP

Commentaire de Mcjo le 03/12/2009 15:17:24

bon je te renvoie un code source ou j'ai remplacé la constante par une variable et définie des fonctions sans public (mais ça ne change rien puisque en php si pas private -> public)

Maintenant si tu as un problème vérifie que tu sois en php et surtout que tu ais le module CURL pour php

:

<?php
/**
* Crawler
* botcrawler
*
* @package     Crawler
* @since       1.0
* @version     $Revision: 1 $
*/
class Crawler {

var $USER_AGENT = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14';
var $authentification_form;
var $authentification_url;
var $must_auth;
var $site_name;
var $cookies_path;
var $user_agent;

/**
* Constructor
*
* @param string $site_name name of site used for define the name of the cookie
* @param boolean $must_auth define if the site requiere an authenficiation - false by default,
* @param string $authentification_url url for authentification - empty by default
* @param array $authentification_form mixed value of the authenfication form
* @param string $user_agent set another user_agent
* @return void
*
*/
function Crawler($site_name, $must_auth = false, $authentification_url = "",  $authentification_form = array(), $user_agent = null, $debug = false) {

$this->user_agent = (!is_null($user_agent)?$user_agent:$this->USER_AGENT);

$this->site_name = $site_name;
$this->must_auth = $must_auth;
$this->authentification_url = $authentification_url;
$this->authentification_form = $authentification_form;
if (is_file("cookies/{$this->site_name}_cookie.txt")) {
unlink("cookies/{$this->site_name}_cookie.txt");
}
$fp_cookies = fopen("cookies/{$this->site_name}_cookie.txt", "w+");
fwrite($fp_cookies, '');
fclose($fp_cookies);

$this->cookies_path = realpath("cookies/{$this->site_name}_cookie.txt");

if ($this->must_auth) {
$ch = curl_init($authentification_url);
curl_setopt($ch, CURLOPT_POST, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS, $authentification_form);
curl_setopt($ch, CURLOPT_COOKIEFILE , $this->cookies_path);
curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookies_path);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_COOKIESESSION, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_USERAGENT, $this->user_agent);
$ret = curl_exec($ch);
if ($ret === FALSE) {
die(curl_error());
}
if ($debug) echo "<pre>Debug :\n".print_r($ret, true)."</pre>";
curl_close($ch);
if (preg_match('/(PHPSESSID=[0-9a-z,-]{32,40})/i', $ret, $m)) {
$sid = '?' . $m[1];
} else if (preg_match('#<input\s+type="hidden"\s+name="([^\r\n\t <>\'"\\\]+)"\s+value="([0-9a-z,-]{32,40})"\s*/?>#i', $ret, $m)) {
$sid = '?' . $m[1] . '=' . $m[2];
}
}
}

/**
* method to access to a page by get
*
* @param string $url
* @param integer $timeout
* @return content of page
*/
function http_fetch_url($url, $timeout = 1000)
{
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_AUTOREFERER, TRUE);
curl_setopt($ch, CURLOPT_COOKIEFILE , $this->cookies_path);
curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookies_path);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_COOKIESESSION, false);
curl_setopt($ch, CURLOPT_USERAGENT, $this->user_agent);
$data = curl_exec($ch);
curl_close($ch);

return $data;
}

/**
* method to access to a page by post
*
* @param stirng $url
* @param array $form array of string value of data post
* @param integer $timeout
* @return content of page
*/
function http_post_url($url, $form = array('none'=>'none'), $timeout = 1000)
{
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($ch, CURLOPT_COOKIEFILE , $this->cookies_path);
curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookies_path);
curl_setopt($ch, CURLOPT_POST, TRUE);
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($form));
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE);
curl_setopt($ch, CURLOPT_COOKIESESSION, false);
curl_setopt($ch, CURLOPT_USERAGENT, $this->user_agent);
$data = curl_exec($ch);
curl_close($ch);
return $data;
}

}

?>