I am encountering something pretty strange. I have a crawler that I am running locally when I var_dump an xpath result I will get:
object(DOMNodeList)#161 (1) {
["length"]=>
int(4)
}
But when I var_dump the xpath result in when I ported to laravel as a helper class I will get:
class DOMNodeList#741 (1) {
public $length =>
int(4)
}
I have no idea why it suddenly changes since my code is exactly the same.
<?php
namespace App\Http\Utilities;
class Crawler {
private $url;
private $class;
private $regex;
private $htmlStack;
// starting page
private $pageNumber = 1;
public $elementsArray;
public function startCrawler($url, $class, $regex=null) {
$this->url = $url;
$this->class = $class;
$this->regex = $regex;
$this->curlGet($this->url);
}
// start the crawler
private function curlGet($url) {
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($curl, CURLOPT_URL, $url);
$this->htmlStack .= curl_exec($curl);
$response = curl_getinfo($curl, CURLINFO_HTTP_CODE);
// check for more pages
$this->paginate($response);
}
// Keep finding more pages untill we get a 404
private function paginate($response) {
if($response === 200) {
$this->pageNumber++;
$url = $this->url . '?page=' . $this->pageNumber;
$this->curlGet($url);
} else {
$this->CreateDomDocument();
}
}
// Dive into a single products page
private function curlGetDeep($link) {
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
curl_setopt($curl, CURLOPT_URL, $link);
$product = curl_exec($curl);
$dom = new \Domdocument();
@$dom->loadHTML($product);
$xpath = new \DomXpath($dom);
// Find description
$descriptions = $xpath->query('//div[contains(@class, "description")]');
// Find Alcoholpercentage
$abv = $xpath->query('//*[text()="Alcoholpercentage"]/following-sibling::p');
var_dump($abv);
foreach($descriptions as $description) {
$results['description'] = $description->nodeValue;
$results['abv'] = $abv->item(0)->nodeValue;
return $results;
}
}
// Render the data
private function CreateDomDocument() {
$dom = new \Domdocument();
@$dom->loadHTML($this->htmlStack);
$xpath = new \DomXpath($dom);
// Find all the product blocks that contain beer attribute that we want to search for such as rating, title, price, etc.
$elements = $xpath->query('//article[contains(@class, "' . $this->class . '")]');
foreach($elements as $element) {
// get the node value from the .title class (the title of the product)
$title = $xpath->query('descendant::div[@class="title"]', $element);
$title = $title->item(0)->nodeValue;
// get the attribute value from the .link-overlay class (the actual link to the product)
$link = $xpath->query('descendant::a[@class="link-overlay"]', $element);
$link = $link->item(0)->getAttribute('href');
$link = 'https://www.gall.nl' . $link;
// get the image src value to rip the image from the site
$image = $xpath->query('descendant::div[@class="image"]/node()/node()', $element);
$image = $image->item(1)->getAttribute('src');
// Dive in the link that a single product has
$results = $this->curlGetDeep($link);
// if a regex option has been supplied run it on the results
if($this->regex) {
$title = preg_replace($this->regex, '', $title);
}
// temp fix, results have unwanted whitespace before and after the string.
$title = trim($title);
$description = trim($results['description']);
// remove the unwanted 6X 35x6(bulk beers) results with this regex
if(!preg_match('/\dX(\d+)?/', $title)) {
$this->elementsArray[] = [
'title' => $title,
'link' => $link,
'image' => $image,
'description' => $description,
'abv' => $results['abv']r
];
}
}
echo json_encode(['beers' => $this->elementsArray]);
}
}
I hope somebody can help me out, if you need more info please let me know. I am pretty baffled on why this happens.
via Chebli Mohamed
Aucun commentaire:
Enregistrer un commentaire