crawler em php
Fala pessoal,
estou tentando fazer um crawler e consegui produzir algo funcional com o seguinte código:
<html>
<head>
<title>crawler</title>
<meta charset="utf-8">
</head>
<body>
</body>
</html>
<?php
include "connect.inc.php";
$to_crawl = $_POST["su"];
$c = array();
$numbrd = strlen($to_crawl);
function get_links($url){
global $c;
$input = @file_get_contents($url);
$regexp = "<a\s[^>]*href=(\"??)([^\" >]*?)\\1[^>]*>(.*)<\/a>";
preg_match_all("/$regexp/siU", $input, $matches);
$base_url = parse_url($url, PHP_URL_HOST);
$link = $matches[2];
foreach($link as $l){
if(strpos($l, "#")){
$l = substr($l, 0, strpos($l, "#"));
}
if(substr($l, 0, 1) == "."){
$l = substr($l, 1);
}
if(substr($l, 0, 7) == "http://"){
$l = $l;
} elseif(substr($l, 0, 8) == "https://"){
$l = $l;
} elseif(substr($l, 0, 2) == "//") {
$l = substr($l, 2);
} elseif(substr($l, 0, 1) == "#"){
$l = $url;
} elseif(substr($l, 0, 7) == "mailto:"){
$l = "[". $l ."]";
} else {
if(substr($l, 0, 1) != "/"){
$l = $base_url."/".$l;
} else {
$l = $base_url.$l;
}
}
if (substr($l, 0, 7) != "http://" && substr($l, 0, 1) != "[" && substr($l, 0, 8) != "https://"){
if (substr($url, 0, 8) == "https://"){
$l == "https//".$l;
} else {
$l == "http//".$l;
}
}
if(!in_array($l, $c)){
array_push($c, $l);
}
}
}
get_links($to_crawl);
foreach ($c as $page) {
get_links($page);
}
foreach ($c as $page) {
global $conteudo;
global $valor;
$pageurl = "http://".$page;
$content = @file_get_contents($pageurl);
preg_match_all('#<title>([^<\/]{1,})<\/title>#i', $content, $match);
foreach ($match[1] as $name => $valor) {
$valor = $valor;
}
preg_match_all('#<p>([^<\/]{1,})<\/p>#i', $content, $nmatch);
foreach ($nmatch[1] as $name2 => $valor2) {
$conteudo = $valor2 . " ";
}
preg_match_all('#<b>([^<\/]{1,})<\/b>#i', $content, $nmatch3);
foreach ($nmatch3[1] as $name3 => $valor3) {
$conteudo .= $valor3 . " ";
}
preg_match_all('#<h1>([^<\/]{1,})<\/h1>#i', $content, $nmatch4);
foreach ($nmatch4[1] as $name4 => $valor4) {
$conteudo .= $valor4;
}
if(substr($to_crawl, 0, $numbrd) == $to_crawl){
echo $conteudo;
$its = "INSERT INTO pages (name, url, content) VALUES ('$valor', '$pageurl', '$conteudo')";
mysqli_query($con, $its);
}
}
header('Location: congrats.php');
?>Discussão (0)
Carregando comentários...