mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 00:01:21 +00:00
Move adult processing completely to services
This commit is contained in:
DariusIII
+3
-210
@@ -7,16 +7,14 @@ use App\Models\Genre;
|
||||
use App\Models\Release;
|
||||
use App\Models\Settings;
|
||||
use App\Models\XxxInfo;
|
||||
use Blacklight\processing\adult\ADE;
|
||||
use Blacklight\processing\adult\ADM;
|
||||
use Blacklight\processing\adult\AEBN;
|
||||
use Blacklight\processing\adult\Hotmovies;
|
||||
use Blacklight\processing\adult\Popporn;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
/**
|
||||
* Class XXX.
|
||||
*
|
||||
* This class handles display and browsing of adult content information.
|
||||
* For processing adult releases, use App\Services\AdultProcessing\AdultProcessingPipeline.
|
||||
*/
|
||||
class XXX
|
||||
{
|
||||
@@ -360,211 +358,6 @@ class XXX
|
||||
return $ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return false|int|string
|
||||
*
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function updateXXXInfo($movie): bool|int|string
|
||||
{
|
||||
$cover = $backdrop = 0;
|
||||
$xxxID = -2;
|
||||
$this->whichClass = 'aebn';
|
||||
$mov = new AEBN;
|
||||
$mov->cookie = $this->cookie;
|
||||
$this->colorCli->info('Checking AEBN for movie info');
|
||||
$res = $mov->processSite($movie);
|
||||
|
||||
if ($res === false) {
|
||||
$this->whichClass = 'pop';
|
||||
$mov = new Popporn;
|
||||
$mov->cookie = $this->cookie;
|
||||
$this->colorCli->info('Checking PopPorn for movie info');
|
||||
$res = $mov->processSite($movie);
|
||||
}
|
||||
|
||||
if ($res === false) {
|
||||
$this->whichClass = 'adm';
|
||||
$mov = new ADM;
|
||||
$mov->cookie = $this->cookie;
|
||||
$this->colorCli->info('Checking ADM for movie info');
|
||||
$res = $mov->processSite($movie);
|
||||
}
|
||||
|
||||
if ($res === false) {
|
||||
$this->whichClass = 'ade';
|
||||
$mov = new ADE;
|
||||
$this->colorCli->info('Checking ADE for movie info');
|
||||
$res = $mov->processSite($movie);
|
||||
}
|
||||
|
||||
if ($res === false) {
|
||||
$this->whichClass = 'hotm';
|
||||
$mov = new Hotmovies;
|
||||
$mov->cookie = $this->cookie;
|
||||
$this->colorCli->info('Checking HotMovies for movie info');
|
||||
$res = $mov->processSite($movie);
|
||||
}
|
||||
|
||||
// If a result is true getAll information.
|
||||
if ($res) {
|
||||
if ($this->echoOutput) {
|
||||
$fromstr = match ($this->whichClass) {
|
||||
'aebn' => 'Adult Entertainment Broadcast Network',
|
||||
'ade' => 'Adult DVD Empire',
|
||||
'pop' => 'PopPorn',
|
||||
'adm' => 'Adult DVD Marketplace',
|
||||
'hotm' => 'HotMovies',
|
||||
default => '',
|
||||
};
|
||||
$this->colorCli->primary('Fetching XXX info from: '.$fromstr);
|
||||
}
|
||||
$res = $mov->getAll();
|
||||
} else {
|
||||
// Nothing was found, go ahead and set to -2
|
||||
return -2;
|
||||
}
|
||||
|
||||
$res['cast'] = ! empty($res['cast']) ? implode(',', $res['cast']) : '';
|
||||
$res['genres'] = ! empty($res['genres']) ? $this->getGenreID($res['genres']) : '';
|
||||
|
||||
$mov = [
|
||||
'trailers' => ! empty($res['trailers']) ? serialize($res['trailers']) : '',
|
||||
'extras' => ! empty($res['extras']) ? serialize($res['extras']) : '',
|
||||
'productinfo' => ! empty($res['productinfo']) ? serialize($res['productinfo']) : '',
|
||||
'backdrop' => ! empty($res['backcover']) ? $res['backcover'] : 0,
|
||||
'cover' => ! empty($res['boxcover']) ? $res['boxcover'] : 0,
|
||||
'title' => ! empty($res['title']) ? html_entity_decode($res['title'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'plot' => ! empty($res['synopsis']) ? html_entity_decode($res['synopsis'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'tagline' => ! empty($res['tagline']) ? html_entity_decode($res['tagline'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'genre' => ! empty($res['genres']) ? html_entity_decode($res['genres'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'director' => ! empty($res['director']) ? html_entity_decode($res['director'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'actors' => ! empty($res['cast']) ? html_entity_decode($res['cast'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'directurl' => ! empty($res['directurl']) ? html_entity_decode($res['directurl'], ENT_QUOTES, 'UTF-8') : '',
|
||||
'classused' => $this->whichClass,
|
||||
];
|
||||
|
||||
$check = XxxInfo::query()->where('title', $mov['title'])->first(['id']);
|
||||
|
||||
if ($check !== null && $check['id'] > 0) {
|
||||
$xxxID = $check['id'];
|
||||
|
||||
// Update BoxCover.
|
||||
if (! empty($mov['cover'])) {
|
||||
$cover = $this->releaseImage->saveImage($xxxID.'-cover', $mov['cover'], $this->imgSavePath);
|
||||
}
|
||||
|
||||
// BackCover.
|
||||
if (! empty($mov['backdrop'])) {
|
||||
$backdrop = $this->releaseImage->saveImage($xxxID.'-backdrop', $mov['backdrop'], $this->imgSavePath, 1920, 1024);
|
||||
}
|
||||
|
||||
// Update Current XXX Information
|
||||
$this->update($check['id'], $mov['title'], $mov['tagline'], $mov['plot'], $mov['genre'], $mov['director'], $mov['actors'], $mov['extras'], $mov['productinfo'], $mov['trailers'], $mov['directurl'], $mov['classused'], $cover, $backdrop);
|
||||
}
|
||||
|
||||
// Insert New XXX Information
|
||||
if ($check === null) {
|
||||
$xxxID = XxxInfo::query()->insertGetId(
|
||||
[
|
||||
'title' => $mov['title'],
|
||||
'tagline' => $mov['tagline'],
|
||||
'plot' => "\x1f\x8b\x08\x00".gzcompress($mov['plot']),
|
||||
'genre' => substr($mov['genre'], 0, 64),
|
||||
'director' => $mov['director'],
|
||||
'actors' => $mov['actors'],
|
||||
'extras' => $mov['extras'],
|
||||
'productinfo' => $mov['productinfo'],
|
||||
'trailers' => $mov['trailers'],
|
||||
'directurl' => $mov['directurl'],
|
||||
'classused' => $mov['classused'],
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]
|
||||
);
|
||||
// Update BoxCover.
|
||||
if (! empty($mov['cover'])) {
|
||||
$cover = $this->releaseImage->saveImage($xxxID.'-cover', $mov['cover'], $this->imgSavePath);
|
||||
}
|
||||
|
||||
// BackCover.
|
||||
if (! empty($mov['backdrop'])) {
|
||||
$backdrop = $this->releaseImage->saveImage($xxxID.'-backdrop', $mov['backdrop'], $this->imgSavePath, 1920, 1024);
|
||||
}
|
||||
|
||||
XxxInfo::whereId($xxxID)->update(['cover' => $cover, 'backdrop' => $backdrop]);
|
||||
}
|
||||
|
||||
if ($this->echoOutput) {
|
||||
$this->colorCli->primary(($xxxID !== false ? 'Added/updated XXX movie: '.$mov['title'] : 'Nothing to update for XXX movie: '.$mov['title']), true);
|
||||
}
|
||||
|
||||
return $xxxID;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process XXX releases where xxxinfo is 0.
|
||||
*
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function processXXXReleases(): void
|
||||
{
|
||||
$res = Release::query()
|
||||
->where(['xxxinfo_id' => 0])
|
||||
->whereIn(
|
||||
'categories_id',
|
||||
[
|
||||
Category::XXX_DVD,
|
||||
Category::XXX_WMV,
|
||||
Category::XXX_XVID,
|
||||
Category::XXX_X264,
|
||||
Category::XXX_SD,
|
||||
Category::XXX_CLIPHD,
|
||||
Category::XXX_CLIPSD,
|
||||
Category::XXX_WEBDL,
|
||||
Category::XXX_UHD,
|
||||
Category::XXX_VR,
|
||||
]
|
||||
)
|
||||
->limit($this->movieQty)
|
||||
->get(['searchname', 'id']);
|
||||
|
||||
$movieCount = \count($res);
|
||||
|
||||
if ($movieCount > 0) {
|
||||
if ($this->echoOutput) {
|
||||
$this->colorCli->header('Processing '.$movieCount.' XXX releases.');
|
||||
}
|
||||
|
||||
// Loop over releases.
|
||||
foreach ($res as $arr) {
|
||||
$idcheck = -2;
|
||||
|
||||
// Try to get a name.
|
||||
if ($this->parseXXXSearchName($arr['searchname'])) {
|
||||
$check = $this->checkXXXInfoExists($this->currentTitle);
|
||||
if ($check === null) {
|
||||
if ($this->echoOutput) {
|
||||
$this->colorCli->info('Looking up: '.$this->currentTitle);
|
||||
}
|
||||
|
||||
$this->colorCli->info('Local match not found, checking web!');
|
||||
$idcheck = $this->updateXXXInfo($this->currentTitle);
|
||||
} else {
|
||||
$this->colorCli->info('Local match found for XXX Movie: '.$this->currentTitle);
|
||||
$idcheck = (int) $check['id'];
|
||||
}
|
||||
} else {
|
||||
$this->colorCli->primary('.');
|
||||
}
|
||||
Release::query()
|
||||
->where('id', $arr['id'])
|
||||
->update(['xxxinfo_id' => $idcheck]);
|
||||
}
|
||||
} elseif ($this->echoOutput) {
|
||||
$this->colorCli->header('No xxx releases to process.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks xxxinfo to make sure releases exist.
|
||||
|
||||
@@ -1,333 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
/**
|
||||
* Class ADE - Adult DVD Empire scraper
|
||||
* Handles movie information extraction from adultdvdempire.com
|
||||
*/
|
||||
class ADE extends AdultMovies
|
||||
{
|
||||
/**
|
||||
* If a direct link is given parse it rather than search.
|
||||
*/
|
||||
protected string $directLink = '';
|
||||
|
||||
/**
|
||||
* Search keyword.
|
||||
*/
|
||||
protected string $searchTerm = '';
|
||||
|
||||
/**
|
||||
* Define ADE Url here.
|
||||
*/
|
||||
private const ADE = 'https://www.adultdvdempire.com';
|
||||
|
||||
/**
|
||||
* Direct Url returned in getAll method.
|
||||
*/
|
||||
protected string $_directUrl = '';
|
||||
|
||||
/**
|
||||
* Sets the title in the getAll method.
|
||||
*/
|
||||
protected string $_title = '';
|
||||
|
||||
/** Trailing urls */
|
||||
protected string $_dvdQuery = '/dvd/search?q=';
|
||||
|
||||
protected string $_scenes = '/scenes';
|
||||
|
||||
protected string $_boxCover = '/boxcover';
|
||||
|
||||
protected string $_backCover = '/backcover';
|
||||
|
||||
protected string $_reviews = '/reviews';
|
||||
|
||||
protected string $_trailers = '/trailers';
|
||||
|
||||
protected $_response;
|
||||
|
||||
protected array $_res = [];
|
||||
|
||||
protected $_tmpResponse;
|
||||
|
||||
/**
|
||||
* Minimum similarity threshold for matching
|
||||
*/
|
||||
protected float $minimumSimilarity = 90.0;
|
||||
|
||||
/**
|
||||
* Gets Trailer Movies.
|
||||
*
|
||||
* @return array - url, streamid, basestreamingurl
|
||||
*/
|
||||
protected function trailers(): array
|
||||
{
|
||||
$this->_response = getRawHtml(self::ADE.$this->_trailers.$this->_directUrl);
|
||||
$this->_html->loadHtml($this->_response);
|
||||
if (preg_match("/([\"|'])(?P<swf>[^\"']+.swf)([\"|'])/i", $this->_response, $hits)) {
|
||||
$this->_res['trailers']['url'] = self::ADE.trim(trim($hits['swf']), '"');
|
||||
if (preg_match(
|
||||
'#(?:streamID:\s\")(?P<streamid>[0-9A-Z]+)(?:\")#',
|
||||
$this->_response,
|
||||
$hits
|
||||
)) {
|
||||
$this->_res['trailers']['streamid'] = trim($hits['streamid']);
|
||||
}
|
||||
if (preg_match(
|
||||
'#(?:BaseStreamingUrl:\s\")(?P<baseurl>[\d]+\.[\d]+\.[\d]+\.[\d]+)(?:\")#',
|
||||
$this->_response,
|
||||
$hits
|
||||
)) {
|
||||
$this->_res['trailers']['baseurl'] = $hits['baseurl'];
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets cover images for the xxx release.
|
||||
*
|
||||
* @return array - Boxcover and backcover
|
||||
*/
|
||||
protected function covers(): array
|
||||
{
|
||||
// Try multiple selectors for better reliability
|
||||
$selectors = [
|
||||
'div#Boxcover img[itemprop=image]',
|
||||
'img[itemprop=image]',
|
||||
'div#Boxcover img',
|
||||
'div.boxcover img',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$ret = $this->_html->findOne($selector);
|
||||
if ($ret && isset($ret->src)) {
|
||||
// Get high-resolution covers
|
||||
$this->_res['boxcover'] = preg_replace('/[ms]\.jpg$/', 'h.jpg', $ret->src);
|
||||
$this->_res['backcover'] = preg_replace('/[ms]\.jpg$/', 'bh.jpg', $ret->src);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the synopsis.
|
||||
*
|
||||
* @return array - plot
|
||||
*/
|
||||
protected function synopsis(): array
|
||||
{
|
||||
// Try multiple selectors in priority order
|
||||
$selectors = [
|
||||
'meta[property="og:description"]' => 'content',
|
||||
'meta[name="description"]' => 'content',
|
||||
'div[itemprop="description"]' => 'plaintext',
|
||||
'p.synopsis' => 'plaintext',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector => $property) {
|
||||
$meta = $this->_html->findOne($selector);
|
||||
if ($meta && isset($meta->$property) && $meta->$property !== false && ! empty(trim($meta->$property))) {
|
||||
$this->_res['synopsis'] = trim($meta->$property);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the cast members and/or awards.
|
||||
*
|
||||
* @return array - cast, awards
|
||||
*/
|
||||
protected function cast(): array
|
||||
{
|
||||
$cast = [];
|
||||
|
||||
// Try multiple selector strategies
|
||||
$selectors = [
|
||||
'div[itemprop="actor"] span[itemprop="name"]',
|
||||
'div.performer-list a',
|
||||
'a[href*="/performer/"]',
|
||||
'h3',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$elements = $this->_html->find($selector);
|
||||
if (! empty($elements)) {
|
||||
foreach ($elements as $a) {
|
||||
if ($a->plaintext !== false && ! empty(trim($a->plaintext))) {
|
||||
$cast[] = trim($a->plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($cast)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->_res['cast'] = array_values(array_unique($cast));
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets Genres, if exists return array else return false.
|
||||
*
|
||||
* @return mixed - Genres
|
||||
*/
|
||||
protected function genres(): mixed
|
||||
{
|
||||
$genres = [];
|
||||
|
||||
// Try multiple selector strategies
|
||||
$selectors = [
|
||||
'a[Label="Category"]',
|
||||
'div.categories a',
|
||||
'a[href*="/category/"]',
|
||||
'span[itemprop="genre"]',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$elements = $this->_html->find($selector);
|
||||
if (! empty($elements)) {
|
||||
foreach ($elements as $a) {
|
||||
if ($a->plaintext !== false && ! empty(trim($a->plaintext))) {
|
||||
$genres[] = trim($a->plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($genres)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->_res['genres'] = array_values(array_unique($genres));
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
protected function productInfo(bool $extras = false): mixed
|
||||
{
|
||||
$dofeature = null;
|
||||
$this->_tmpResponse = str_ireplace('Section ProductInfo', 'spdinfo', $this->_response);
|
||||
$this->_html->loadHtml($this->_tmpResponse);
|
||||
if ($ret = $this->_html->findOne('div[class=spdinfo]')) {
|
||||
$this->_tmpResponse = trim($ret->outertext);
|
||||
$ret = $this->_html->loadHtml($this->_tmpResponse);
|
||||
foreach ($ret->find('text') as $strong) {
|
||||
if (trim($strong->innertext) === 'Features') {
|
||||
$dofeature = true;
|
||||
}
|
||||
if ($dofeature !== true) {
|
||||
if (trim($strong->innertext) !== ' ') {
|
||||
$this->_res['productinfo'][] = trim($strong->innertext);
|
||||
}
|
||||
} else {
|
||||
if ($extras === true) {
|
||||
$this->_res['extras'][] = trim($strong->innertext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
array_shift($this->_res['productinfo']);
|
||||
array_shift($this->_res['productinfo']);
|
||||
$this->_res['productinfo'] = array_chunk($this->_res['productinfo'], 2, false);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches xxx name.
|
||||
*
|
||||
* @return bool - True if releases has 90% match, else false
|
||||
*/
|
||||
public function processSite(string $movie): bool
|
||||
{
|
||||
if (empty($movie)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_response = getRawHtml(self::ADE.$this->_dvdQuery.rawurlencode($movie));
|
||||
|
||||
if ($this->_response === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
// Try multiple search result selectors
|
||||
$resultSelectors = [
|
||||
'a[class=fancybox-button]',
|
||||
'div.card a.boxcover-link',
|
||||
'a[href*="/item/"]',
|
||||
];
|
||||
|
||||
$bestMatch = null;
|
||||
$highestSimilarity = 0;
|
||||
|
||||
foreach ($resultSelectors as $selector) {
|
||||
$res = $this->_html->find($selector);
|
||||
if (! empty($res)) {
|
||||
foreach ($res as $ret) {
|
||||
$title = $ret->title ?? $ret->getAttribute('title') ?? trim($ret->plaintext);
|
||||
$url = trim($ret->href ?? '');
|
||||
|
||||
if (empty($title) || empty($url)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Clean title for better matching
|
||||
$cleanTitle = str_replace('/XXX/', '', $title);
|
||||
$cleanTitle = preg_replace('/\(.*?\)|[._-]/', ' ', $cleanTitle);
|
||||
$cleanTitle = preg_replace('/\s+/', ' ', trim($cleanTitle));
|
||||
|
||||
similar_text(strtolower($movie), strtolower($cleanTitle), $p);
|
||||
|
||||
if ($p > $highestSimilarity) {
|
||||
$highestSimilarity = $p;
|
||||
$bestMatch = [
|
||||
'title' => trim($title),
|
||||
'url' => $url,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// If we found results with this selector, don't try others
|
||||
if ($bestMatch !== null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if best match meets threshold
|
||||
if ($bestMatch !== null && $highestSimilarity >= $this->minimumSimilarity) {
|
||||
$this->_directUrl = str_starts_with($bestMatch['url'], 'http')
|
||||
? $bestMatch['url']
|
||||
: self::ADE.$bestMatch['url'];
|
||||
$this->_title = $bestMatch['title'];
|
||||
|
||||
unset($this->_response);
|
||||
$this->_response = getRawHtml($this->_directUrl);
|
||||
|
||||
if ($this->_response !== false) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
use voku\helper\SimpleHtmlDomNodeBlank;
|
||||
|
||||
/**
|
||||
* Class ADM - Adult DVD Marketplace scraper
|
||||
* Handles movie information extraction from adultdvdmarketplace.com
|
||||
*/
|
||||
class ADM extends AdultMovies
|
||||
{
|
||||
/**
|
||||
* Override if 18 years+ or older
|
||||
* Define Adult DVD Marketplace url
|
||||
* Needed Search Queries Constant.
|
||||
*/
|
||||
private const ADMURL = 'https://www.adultdvdmarketplace.com';
|
||||
|
||||
private const TRAILINGSEARCH = '/xcart/adult_dvd/advanced_search.php?sort_by=relev&title=';
|
||||
|
||||
/**
|
||||
* Define a cookie file location for curl.
|
||||
*
|
||||
* @var string string
|
||||
*/
|
||||
public string $cookie = '';
|
||||
|
||||
/**
|
||||
* Direct Link given from outside url doesn't do a search.
|
||||
*/
|
||||
protected string $directLink = '';
|
||||
|
||||
/**
|
||||
* Set this for what you are searching for.
|
||||
*/
|
||||
protected string $searchTerm = '';
|
||||
|
||||
/**
|
||||
* Sets the directurl for the return results array.
|
||||
*/
|
||||
protected string $_directUrl = '';
|
||||
|
||||
/**
|
||||
* Results returned from each method.
|
||||
*/
|
||||
protected array $_res = [];
|
||||
|
||||
/**
|
||||
* Curl Raw Html.
|
||||
*/
|
||||
protected $_response;
|
||||
|
||||
/**
|
||||
* Add this to popurl to get results.
|
||||
*/
|
||||
protected string $_trailUrl = '';
|
||||
|
||||
/**
|
||||
* This is set in the getAll method.
|
||||
*/
|
||||
protected string $_title = '';
|
||||
|
||||
/**
|
||||
* Minimum similarity threshold for matching
|
||||
*/
|
||||
protected float $minimumSimilarity = 90.0;
|
||||
|
||||
/**
|
||||
* Get Box Cover Images.
|
||||
*
|
||||
* @return array - box cover,back cover
|
||||
*/
|
||||
protected function covers(): array
|
||||
{
|
||||
$baseUrl = 'https://www.adultdvdmarketplace.com/';
|
||||
|
||||
// Try fancybox link first
|
||||
if ($ret = $this->_html->findOne('a[rel=fancybox-button]')) {
|
||||
if (isset($ret->href) && preg_match('/images\/.*[\d]+\.jpg$/i', $ret->href, $hits)) {
|
||||
$this->_res['boxcover'] = str_starts_with($hits[0], 'http')
|
||||
? $hits[0]
|
||||
: $baseUrl.$hits[0];
|
||||
$this->_res['backcover'] = str_ireplace('/front/', '/back/', $this->_res['boxcover']);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
// Try license image
|
||||
if ($ret = $this->_html->findOne('img[rel=license]')) {
|
||||
if (isset($ret->src) && preg_match('/images\/.*[\d]+\.jpg$/i', $ret->src, $hits)) {
|
||||
$this->_res['boxcover'] = str_starts_with($hits[0], 'http')
|
||||
? $hits[0]
|
||||
: $baseUrl.$hits[0];
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the synopsis.
|
||||
*/
|
||||
protected function synopsis(): array
|
||||
{
|
||||
$this->_res['synopsis'] = 'N/A';
|
||||
|
||||
// Try to find Description heading
|
||||
foreach ($this->_html->find('h3') as $heading) {
|
||||
if (trim($heading->plaintext) === 'Description') {
|
||||
$nextElement = $heading->next_sibling();
|
||||
if ($nextElement && ! empty(trim($nextElement->plaintext))) {
|
||||
$this->_res['synopsis'] = trim($nextElement->plaintext);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: Try meta description
|
||||
$meta = $this->_html->findOne('meta[name="description"]');
|
||||
if ($meta && isset($meta->content) && ! empty(trim($meta->content))) {
|
||||
$this->_res['synopsis'] = trim($meta->content);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Product Information and Director.
|
||||
*/
|
||||
protected function productInfo(bool $extras = false): mixed
|
||||
{
|
||||
foreach ($this->_html->find('ul.list-unstyled li') as $li) {
|
||||
$category = explode(':', $li->plaintext);
|
||||
switch (trim($category[0])) {
|
||||
case 'Director':
|
||||
$this->_res['director'] = trim($category[1]);
|
||||
break;
|
||||
case 'Format':
|
||||
case 'Studio':
|
||||
case 'Released':
|
||||
case 'SKU':
|
||||
$this->_res['productinfo'][trim($category[0])] = trim($category[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the cast members.
|
||||
*/
|
||||
protected function cast(): array
|
||||
{
|
||||
$cast = [];
|
||||
foreach ($this->_html->find('h3') as $heading) {
|
||||
if (trim($heading->plaintext) === 'Cast') {
|
||||
foreach ($heading->nextSibling() as $next) {
|
||||
if (! $next instanceof SimpleHtmlDomNodeBlank && $next->nodeName !== 'h3') {
|
||||
$next = $next->nextSibling();
|
||||
}
|
||||
if (preg_match_all('/search_performerid/', $next->href, $hits)) {
|
||||
$cast[] = trim($next->plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->_res['cast'] = array_unique($cast);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets categories.
|
||||
*/
|
||||
protected function genres(): array
|
||||
{
|
||||
$genres = [];
|
||||
foreach ($this->_html->find('ul.list-unstyled') as $li) {
|
||||
$category = explode(':', $li->plaintext);
|
||||
if (trim($category[0]) === 'Category') {
|
||||
foreach (explode(',', $category[1]) as $genre) {
|
||||
$genres[] = trim($genre);
|
||||
}
|
||||
$this->_res['genres'] = $genres;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for match against search term.
|
||||
*
|
||||
* @return bool - true if search = 100%
|
||||
*/
|
||||
public function processSite(string $movie): bool
|
||||
{
|
||||
if (empty($movie)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_trailUrl = self::TRAILINGSEARCH.urlencode($movie);
|
||||
$this->_response = getRawHtml(self::ADMURL.$this->_trailUrl, $this->cookie);
|
||||
|
||||
if ($this->_response === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_html->loadHtml($this->_response);
|
||||
$check = $this->_html->find('img[rel=license]');
|
||||
|
||||
if (empty($check)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$bestMatch = null;
|
||||
$highestSimilarity = 0;
|
||||
|
||||
foreach ($check as $ret) {
|
||||
if (! isset($ret->alt) || ! isset($ret->src)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$title = trim($ret->alt, '"');
|
||||
$title = str_replace('/XXX/', '', $title);
|
||||
$comparetitle = preg_replace('/[\W]/', '', strtolower($title));
|
||||
$comparesearch = preg_replace('/[\W]/', '', strtolower($movie));
|
||||
|
||||
similar_text($comparetitle, $comparesearch, $p);
|
||||
|
||||
if ($p > $highestSimilarity && preg_match('/\/(?<sku>\d+)\.jpg$/i', $ret->src, $hits)) {
|
||||
$highestSimilarity = $p;
|
||||
$bestMatch = [
|
||||
'title' => trim($title),
|
||||
'sku' => $hits['sku'],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Check if best match meets threshold
|
||||
if ($bestMatch !== null && $highestSimilarity >= $this->minimumSimilarity) {
|
||||
$this->_title = $bestMatch['title'];
|
||||
$this->_trailUrl = '/dvd_view_'.$bestMatch['sku'].'.html';
|
||||
$this->_directUrl = self::ADMURL.$this->_trailUrl;
|
||||
|
||||
unset($this->_response);
|
||||
$this->_response = getRawHtml($this->_directUrl, $this->cookie);
|
||||
|
||||
if ($this->_response !== false) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
protected function trailers(): mixed
|
||||
{
|
||||
// TODO: Implement trailers() method.
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,288 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
use voku\helper\SimpleHtmlDomNodeBlank;
|
||||
|
||||
/**
|
||||
* Class AEBN - AEBN Theater scraper
|
||||
* Handles movie information extraction from straight.theater.aebn.net
|
||||
*/
|
||||
class AEBN extends AdultMovies
|
||||
{
|
||||
/**
|
||||
* Keyword to search.
|
||||
*/
|
||||
public string $searchTerm = '';
|
||||
|
||||
/**
|
||||
* Url Constants used within this class.
|
||||
*/
|
||||
private const AEBNSURL = 'https://straight.theater.aebn.net';
|
||||
|
||||
private const TRAILINGSEARCH = '/dispatcher/fts?theaterId=13992&genreId=101&locale=en&count=30&imageType=Large&targetSearchMode=basic&isAdvancedSearch=false&isFlushAdvancedSearchCriteria=false&sortType=Relevance&userQuery=title%3A+%2B';
|
||||
|
||||
private const TRAILERURL = '/dispatcher/previewPlayer?locale=en&theaterId=13992&genreId=101&movieId=';
|
||||
|
||||
/**
|
||||
* Direct Url in getAll method.
|
||||
*/
|
||||
protected string $_directUrl = '';
|
||||
|
||||
/**
|
||||
* Raw Html response from curl.
|
||||
*/
|
||||
protected $_response;
|
||||
|
||||
protected string $_trailerUrl = '';
|
||||
|
||||
/**
|
||||
* Returned results in all methods except search/geturl.
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $_res = [
|
||||
'backcover' => [],
|
||||
'boxcover' => [],
|
||||
'cast' => [],
|
||||
'director' => [],
|
||||
'genres' => [],
|
||||
'productinfo' => [],
|
||||
'synopsis' => [],
|
||||
'trailers' => ['url' => []],
|
||||
];
|
||||
|
||||
/**
|
||||
* Sets title in getAll method.
|
||||
*/
|
||||
protected string $_title = '';
|
||||
|
||||
/**
|
||||
* Minimum similarity threshold for matching
|
||||
*/
|
||||
protected float $minimumSimilarity = 90.0;
|
||||
|
||||
public string $cookie = '';
|
||||
|
||||
/**
|
||||
* Gets Trailer URL . will be processed in XXX insertswf.
|
||||
*/
|
||||
protected function trailers(): mixed
|
||||
{
|
||||
$ret = $this->_html->find('a[itemprop=trailer]', 0);
|
||||
if (! empty($ret) && preg_match('/movieId=(?<movieid>\d+)&/', trim($ret->href), $hits)) {
|
||||
$movieid = $hits['movieid'];
|
||||
$this->_res['trailers']['url'] = self::AEBNSURL.self::TRAILERURL.$movieid;
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the front and back cover of the box.
|
||||
*/
|
||||
protected function covers(): array
|
||||
{
|
||||
// Try multiple selectors
|
||||
$selectors = [
|
||||
'img[itemprop=thumbnailUrl]',
|
||||
'div#md-boxCover img',
|
||||
'img.boxcover',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$ret = $this->_html->findOne($selector);
|
||||
if ($ret && isset($ret->src)) {
|
||||
$coverUrl = trim($ret->src);
|
||||
|
||||
// Ensure URL has protocol
|
||||
if (str_starts_with($coverUrl, '//')) {
|
||||
$coverUrl = 'https:'.$coverUrl;
|
||||
}
|
||||
|
||||
// Get high-resolution versions
|
||||
$this->_res['boxcover'] = str_ireplace(['160w.jpg', '120w.jpg'], 'xlf.jpg', $coverUrl);
|
||||
$this->_res['backcover'] = str_ireplace(['160w.jpg', '120w.jpg'], 'xlb.jpg', $coverUrl);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Genres "Categories".
|
||||
*/
|
||||
protected function genres(): array
|
||||
{
|
||||
if ($ret = $this->_html->findOne('div.md-detailsCategories')) {
|
||||
foreach ($ret->find('a[itemprop=genre]') as $genre) {
|
||||
$this->_res['genres'][] = trim($genre->plaintext);
|
||||
}
|
||||
}
|
||||
if (! empty($this->_res['genres'])) {
|
||||
$this->_res['genres'] = array_unique($this->_res['genres']);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Cast Members "Stars" and Director if any.
|
||||
*/
|
||||
protected function cast(): array
|
||||
{
|
||||
// Do not reset the whole results array; only populate the cast key.
|
||||
$cast = [];
|
||||
$ret = $this->_html->findOne('div.starsFull');
|
||||
if (! $ret instanceof SimpleHtmlDomNodeBlank) {
|
||||
foreach ($ret->find('span[itemprop=name]') as $star) {
|
||||
$cast[] = trim($star->plaintext);
|
||||
}
|
||||
} else {
|
||||
$ret = $this->_html->findOne('div.detailsLink');
|
||||
if (! $ret instanceof SimpleHtmlDomNodeBlank) {
|
||||
foreach ($ret->find('span') as $star) {
|
||||
if (str_contains($star->plaintext, '/More/') && str_contains($star->plaintext, '/Stars/')) {
|
||||
$cast[] = trim($star->plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (! empty($cast)) {
|
||||
$this->_res['cast'] = $cast;
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the product information.
|
||||
*/
|
||||
protected function productInfo(bool $extras = false): mixed
|
||||
{
|
||||
if ($ret = $this->_html->find('div#md-detailsLeft', 0)) {
|
||||
foreach ($ret->find('div') as $div) {
|
||||
foreach ($div->find('span') as $span) {
|
||||
$text = rawurldecode($span->plaintext);
|
||||
$text = preg_replace('/ /', '', $text);
|
||||
$this->_res['productinfo'][] = trim($text);
|
||||
}
|
||||
}
|
||||
if (false !== $key = array_search('Running Time:', $this->_res['productinfo'], false)) {
|
||||
unset($this->_res['productinfo'][$key + 2]);
|
||||
}
|
||||
if (false !== $key = array_search('Director:', $this->_res['productinfo'], false)) {
|
||||
$this->_res['director'] = $this->_res['productinfo'][$key + 1];
|
||||
unset($this->_res['productinfo'][$key], $this->_res['productinfo'][$key + 1]);
|
||||
}
|
||||
$this->_res['productinfo'] = array_chunk($this->_res['productinfo'], 2, false);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the synopsis "plot".
|
||||
*/
|
||||
protected function synopsis(): array
|
||||
{
|
||||
// Prefer the modern schema attribute
|
||||
$ret = $this->_html->findOne('span[itemprop=about]');
|
||||
if ($ret && $ret->plaintext !== null) {
|
||||
$this->_res['synopsis'] = trim($ret->plaintext);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
// Fallback to legacy description container
|
||||
$ret = $this->_html->findOne('div.movieDetailDescription');
|
||||
if ($ret && $ret->plaintext !== null) {
|
||||
$text = trim($ret->plaintext);
|
||||
$this->_res['synopsis'] = preg_replace('/^Description:\s*/', '', $text);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for a XXX name.
|
||||
*/
|
||||
public function processSite(string $movie): bool
|
||||
{
|
||||
if (empty($movie)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_trailerUrl = self::TRAILINGSEARCH.urlencode($movie);
|
||||
$this->_response = getRawHtml(self::AEBNSURL.$this->_trailerUrl, $this->cookie);
|
||||
|
||||
if ($this->_response === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
$bestMatch = null;
|
||||
$highestSimilarity = 0;
|
||||
$i = 1;
|
||||
|
||||
foreach ($this->_html->find('div.movie') as $mov) {
|
||||
// Try multiple selector patterns
|
||||
$selectors = [
|
||||
'a#FTSMovieSearch_link_title_detail_'.$i,
|
||||
'a.title-link',
|
||||
'a[href*="/movie/"]',
|
||||
];
|
||||
|
||||
$ret = null;
|
||||
foreach ($selectors as $selector) {
|
||||
$ret = $mov->findOne($selector);
|
||||
if ($ret) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ($ret && isset($ret->href)) {
|
||||
$title = $ret->title ?? trim($ret->plaintext);
|
||||
|
||||
if (! empty($title)) {
|
||||
// Clean title for better matching
|
||||
$cleanTitle = str_replace('/XXX/', '', $title);
|
||||
$cleanTitle = preg_replace('/\(.*?\)|[._-]/', ' ', $cleanTitle);
|
||||
$cleanTitle = preg_replace('/\s+/', ' ', trim($cleanTitle));
|
||||
|
||||
similar_text(strtolower($movie), strtolower($cleanTitle), $p);
|
||||
|
||||
if ($p > $highestSimilarity) {
|
||||
$highestSimilarity = $p;
|
||||
$bestMatch = [
|
||||
'title' => trim($title),
|
||||
'url' => html_entity_decode($ret->href),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
$i++;
|
||||
}
|
||||
|
||||
// Check if best match meets threshold
|
||||
if ($bestMatch !== null && $highestSimilarity >= $this->minimumSimilarity) {
|
||||
$this->_title = $bestMatch['title'];
|
||||
$this->_trailerUrl = $bestMatch['url'];
|
||||
$this->_directUrl = self::AEBNSURL.$this->_trailerUrl;
|
||||
|
||||
unset($this->_response);
|
||||
$this->_response = getRawHtml(self::AEBNSURL.$this->_trailerUrl, $this->cookie);
|
||||
|
||||
if ($this->_response !== false) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
use voku\helper\HtmlDomParser;
|
||||
|
||||
abstract class AdultMovies
|
||||
{
|
||||
protected HtmlDomParser $_html;
|
||||
|
||||
protected string $_title;
|
||||
|
||||
protected string $_directUrl;
|
||||
|
||||
/**
|
||||
* AdultMovies constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->_html = new HtmlDomParser;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array|mixed
|
||||
*/
|
||||
abstract protected function productInfo(bool $extras = false): mixed;
|
||||
|
||||
abstract protected function covers(): mixed;
|
||||
|
||||
abstract protected function synopsis(): mixed;
|
||||
|
||||
abstract protected function cast(): mixed;
|
||||
|
||||
abstract protected function genres(): mixed;
|
||||
|
||||
abstract public function processSite(string $movie): mixed;
|
||||
|
||||
abstract protected function trailers(): mixed;
|
||||
|
||||
/**
|
||||
* Gets all information.
|
||||
*
|
||||
* @return array|false
|
||||
*/
|
||||
public function getAll(): bool|array
|
||||
{
|
||||
$results = [];
|
||||
|
||||
// Only include when present
|
||||
if (! empty($this->_directUrl)) {
|
||||
if (! empty($this->_title)) {
|
||||
$results['title'] = $this->_title;
|
||||
}
|
||||
$results['directurl'] = $this->_directUrl;
|
||||
}
|
||||
|
||||
$dummy = $this->synopsis();
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
|
||||
$dummy = $this->productInfo(true);
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
|
||||
$dummy = $this->cast();
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
|
||||
$dummy = $this->genres();
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
|
||||
$dummy = $this->covers();
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
|
||||
$dummy = $this->trailers();
|
||||
if (\is_array($dummy)) {
|
||||
$results = array_merge($results, $dummy);
|
||||
}
|
||||
if (empty($results)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
/**
|
||||
* Class Hotmovies - HotMovies.com scraper
|
||||
* Handles movie information extraction from hotmovies.com
|
||||
*/
|
||||
class Hotmovies extends AdultMovies
|
||||
{
|
||||
/**
|
||||
* Constant Urls used within this class
|
||||
* Needed Search Queries Variables.
|
||||
*/
|
||||
private const EXTRASEARCH = '&complete=on&search_in=video_title';
|
||||
|
||||
private const HMURL = 'https://www.hotmovies.com';
|
||||
|
||||
private const TRAILINGSEARCH = '/search.php?words=';
|
||||
|
||||
/**
|
||||
* Keyword Search.
|
||||
*/
|
||||
protected string $searchTerm = '';
|
||||
|
||||
/**
|
||||
* Define a cookie location.
|
||||
*/
|
||||
public string $cookie = '';
|
||||
|
||||
/**
|
||||
* If a direct link is set parse it instead of search for it.
|
||||
*/
|
||||
protected string $directLink = '';
|
||||
|
||||
/**
|
||||
* Sets the direct url in the getAll method.
|
||||
*/
|
||||
protected string $_directUrl = '';
|
||||
|
||||
/**
|
||||
* Sets the link to get in curl.
|
||||
*/
|
||||
protected string $_getLink = '';
|
||||
|
||||
/**
|
||||
* POST parameters used with curl.
|
||||
*/
|
||||
protected array $_postParams = [];
|
||||
|
||||
/**
|
||||
* Results return from some methods.
|
||||
*/
|
||||
protected array $_res = [];
|
||||
|
||||
/**
|
||||
* Raw Html from Curl.
|
||||
*/
|
||||
protected $_response;
|
||||
|
||||
/**
|
||||
* Sets the title in the getAll method.
|
||||
*/
|
||||
protected string $_title = '';
|
||||
|
||||
/**
|
||||
* Minimum similarity threshold for matching
|
||||
*/
|
||||
protected float $minimumSimilarity = 90.0;
|
||||
|
||||
protected function trailers(): false
|
||||
{
|
||||
// TODO: Implement trailers() method.
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the synopsis.
|
||||
*/
|
||||
protected function synopsis(): array
|
||||
{
|
||||
$this->_res['synopsis'] = 'N/A';
|
||||
|
||||
// Try multiple selectors
|
||||
$selectors = [
|
||||
'.video_description',
|
||||
'div.description',
|
||||
'div.synopsis',
|
||||
'meta[name="description"]',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$ret = $this->_html->findOne($selector);
|
||||
if ($ret) {
|
||||
$text = $ret->innerText ?? $ret->plaintext ?? $ret->content ?? '';
|
||||
if (! empty(trim($text))) {
|
||||
$this->_res['synopsis'] = trim($text);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process ProductInfo.
|
||||
*/
|
||||
protected function productInfo(bool $extras = false): mixed
|
||||
{
|
||||
$studio = false;
|
||||
$director = false;
|
||||
if (($ret = $this->_html->find('div.page_video_info')) && ! empty($ret->find('text'))) {
|
||||
foreach ($ret->find('text') as $e) {
|
||||
$e = trim($e->plaintext);
|
||||
$rArray = [',', '...', ' :'];
|
||||
$e = str_replace($rArray, '', $e);
|
||||
if (stripos($e, 'Studio:') !== false) {
|
||||
$studio = true;
|
||||
}
|
||||
if (str_contains($e, 'Director:')) {
|
||||
$director = true;
|
||||
$e = null;
|
||||
}
|
||||
if ($studio === true) {
|
||||
if ((stripos($e, 'Custodian of Records') === false) && stripos($e, 'Description') === false) {
|
||||
if ($director === true && ! empty($e)) {
|
||||
$this->_res['director'] = $e;
|
||||
$e = null;
|
||||
$director = false;
|
||||
}
|
||||
if (! empty($e)) {
|
||||
$this->_res['productinfo'][] = $e;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isset($this->_res['productinfo']) && \is_array($this->_res['productinfo'])) {
|
||||
$this->_res['productinfo'] = array_chunk($this->_res['productinfo'], 2, false);
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the cast members and director.
|
||||
*/
|
||||
protected function cast(): array
|
||||
{
|
||||
$cast = [];
|
||||
|
||||
// Prefer scoped search within stars container to avoid unrelated links
|
||||
if ($container = $this->_html->findOne('.stars')) {
|
||||
foreach ($container->find('a[title]') as $e) {
|
||||
$name = trim($e->title);
|
||||
$name = preg_replace('/\((.*)\)/', '', $name);
|
||||
$name = trim($name);
|
||||
if ($name !== '') {
|
||||
$cast[] = $name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: anchors that look like performer links
|
||||
if (empty($cast)) {
|
||||
foreach ($this->_html->find('a[href*="/performers/"]') as $e) {
|
||||
$name = trim($e->plaintext);
|
||||
if ($name !== '') {
|
||||
$cast[] = $name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($cast)) {
|
||||
$this->_res['cast'] = array_values(array_unique($cast));
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets categories.
|
||||
*/
|
||||
protected function genres(): array
|
||||
{
|
||||
$genres = [];
|
||||
if ($ret = $this->_html->findOne('div.categories')) {
|
||||
foreach ($ret->find('a') as $e) {
|
||||
if (str_contains($e->title, ' -> ')) {
|
||||
$e = explode(' -> ', $e->plaintext);
|
||||
$genres[] = trim($e[1]);
|
||||
}
|
||||
}
|
||||
$this->_res['genres'] = $genres;
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Box Cover Images.
|
||||
*/
|
||||
protected function covers(): array|false
|
||||
{
|
||||
// Try multiple selectors
|
||||
$selectors = [
|
||||
'img#cover',
|
||||
'div#large_cover img',
|
||||
'img.boxcover',
|
||||
'div.product-image img',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$ret = $this->_html->findOne($selector);
|
||||
if ($ret && isset($ret->src)) {
|
||||
$this->_res['boxcover'] = trim($ret->src);
|
||||
$this->_res['backcover'] = str_ireplace(['.cover', 'front'], ['.back', 'back'], trim($ret->src));
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for match against xxx movie name.
|
||||
*
|
||||
* @return bool , true if search >= 90%
|
||||
*/
|
||||
public function processSite(string $movie): bool
|
||||
{
|
||||
if (empty($movie)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_getLink = self::HMURL.self::TRAILINGSEARCH.urlencode($movie).self::EXTRASEARCH;
|
||||
$this->_response = getRawHtml($this->_getLink, $this->cookie);
|
||||
|
||||
if ($this->_response === false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
// Try multiple result selectors
|
||||
$resultSelectors = [
|
||||
'h3[class=title] a[title]',
|
||||
'h3.title a',
|
||||
'div.movie-title a',
|
||||
];
|
||||
|
||||
$bestMatch = null;
|
||||
$highestSimilarity = 0;
|
||||
|
||||
foreach ($resultSelectors as $selector) {
|
||||
$elements = $this->_html->find($selector);
|
||||
if (! empty($elements)) {
|
||||
foreach ($elements as $ret) {
|
||||
$title = $ret->title ?? $ret->plaintext ?? '';
|
||||
$url = $ret->href ?? '';
|
||||
|
||||
if (empty($title) || empty($url)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Clean title for better matching
|
||||
$cleanTitle = str_replace('/XXX/', '', $title);
|
||||
$cleanTitle = preg_replace('/\(.*?\)|[._-]/', ' ', $cleanTitle);
|
||||
$cleanTitle = preg_replace('/\s+/', ' ', trim($cleanTitle));
|
||||
|
||||
similar_text(strtolower($movie), strtolower($cleanTitle), $p);
|
||||
|
||||
if ($p > $highestSimilarity) {
|
||||
$highestSimilarity = $p;
|
||||
$bestMatch = [
|
||||
'title' => trim($title),
|
||||
'url' => trim($url),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// If we found results with this selector, don't try others
|
||||
if ($bestMatch !== null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if best match meets threshold
|
||||
if ($bestMatch !== null && $highestSimilarity >= $this->minimumSimilarity) {
|
||||
$this->_title = $bestMatch['title'];
|
||||
$this->_getLink = $bestMatch['url'];
|
||||
$this->_directUrl = str_starts_with($bestMatch['url'], 'http')
|
||||
? $bestMatch['url']
|
||||
: self::HMURL.$bestMatch['url'];
|
||||
|
||||
unset($this->_response);
|
||||
$this->_response = getRawHtml($this->_directUrl, $this->cookie);
|
||||
|
||||
if ($this->_response !== false) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,538 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
|
||||
class Popporn extends AdultMovies
|
||||
{
|
||||
/**
|
||||
* Define a cookie file location for curl.
|
||||
*/
|
||||
public string $cookie = '';
|
||||
|
||||
/**
|
||||
* Base URL for the site
|
||||
*/
|
||||
private const BASE_URL = 'https://www.popporn.com';
|
||||
|
||||
/**
|
||||
* Search endpoint
|
||||
*/
|
||||
private const SEARCH_ENDPOINT = '/search?q=';
|
||||
|
||||
/**
|
||||
* Age verification URL
|
||||
*/
|
||||
private const AGE_VERIFICATION_URL = 'https://www.popporn.com/popporn/4';
|
||||
|
||||
/**
|
||||
* Minimum similarity percentage to consider a match
|
||||
*/
|
||||
protected float $minimumSimilarity = 90.0;
|
||||
|
||||
/**
|
||||
* Direct URL for the movie
|
||||
*/
|
||||
protected string $_directUrl = '';
|
||||
|
||||
/**
|
||||
* Raw HTML response
|
||||
*/
|
||||
protected $_response;
|
||||
|
||||
/**
|
||||
* Results array
|
||||
*/
|
||||
protected array $_res = [];
|
||||
|
||||
/**
|
||||
* Movie title
|
||||
*/
|
||||
protected string $_title = '';
|
||||
|
||||
/**
|
||||
* Temporary URL for internal operations
|
||||
*/
|
||||
protected string $_trailUrl = '';
|
||||
|
||||
/**
|
||||
* POST parameters for API requests
|
||||
*/
|
||||
private $_postParams;
|
||||
|
||||
/**
|
||||
* Get Box Cover Images.
|
||||
*/
|
||||
protected function covers(): array|false
|
||||
{
|
||||
// Method 1: Try structured data
|
||||
if (preg_match('/"image":\s*"(.*?)"/is', $this->_response, $match)) {
|
||||
$this->_res['boxcover'] = trim($match[1]);
|
||||
// Try to determine backcover from boxcover pattern
|
||||
if (stripos(trim($match[1]), '_aa') !== false) {
|
||||
$this->_res['backcover'] = str_ireplace('_aa', '_bb', trim($match[1]));
|
||||
} else {
|
||||
$this->_res['backcover'] = str_ireplace('.jpg', '_b.jpg', trim($match[1]));
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
// Method 2: Try multiple selectors
|
||||
$selectors = [
|
||||
'div[id=box-art], a[rel=box-art]',
|
||||
'img.front',
|
||||
'div.box-cover img',
|
||||
'div.product-image img',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
if ($ret = $this->_html->findOne($selector)) {
|
||||
$this->_res['boxcover'] = $ret->href ?? $ret->src;
|
||||
|
||||
// Try to determine backcover
|
||||
if (stripos($this->_res['boxcover'], '_aa') !== false) {
|
||||
$this->_res['backcover'] = str_ireplace('_aa', '_bb', $this->_res['boxcover']);
|
||||
} else {
|
||||
$this->_res['backcover'] = str_ireplace('.jpg', '_b.jpg', $this->_res['boxcover']);
|
||||
}
|
||||
|
||||
// Also check for explicit back cover
|
||||
if (! isset($this->_res['backcover']) && $back = $this->_html->findOne('img.back')) {
|
||||
$this->_res['backcover'] = $back->src;
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the movie synopsis/description
|
||||
*/
|
||||
protected function synopsis(): array
|
||||
{
|
||||
// Method 1: Try structured data
|
||||
if (preg_match('/"description":\s*"(.*?)"/is', $this->_response, $match)) {
|
||||
$this->_res['synopsis'] = trim(html_entity_decode(str_replace('\\u', '\\u', $match[1])));
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
// Method 2: Try multiple selectors
|
||||
$selectors = [
|
||||
'div[id=product-info] h3[class=highlight] + *',
|
||||
'div.product-description',
|
||||
'div.synopsis',
|
||||
'meta[name="description"]',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
if ($ret = $this->_html->findOne($selector)) {
|
||||
$text = $ret->plaintext ?? $ret->content;
|
||||
|
||||
// Filter out "POPPORN EXCLUSIVE" text
|
||||
if (stripos(trim($text), 'POPPORN EXCLUSIVE') !== false) {
|
||||
if ($ret->next_sibling()) {
|
||||
$text = trim($ret->next_sibling()->plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($text)) {
|
||||
$this->_res['synopsis'] = trim($text);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Original method as fallback
|
||||
if ($ret = $this->_html->find('div[id=product-info] ,h3[class=highlight]', 1)) {
|
||||
if ($ret->next_sibling() && $ret->next_sibling()->plaintext) {
|
||||
if (stripos(trim($ret->next_sibling()->plaintext), 'POPPORN EXCLUSIVE') === false) {
|
||||
$this->_res['synopsis'] = trim($ret->next_sibling()->plaintext);
|
||||
} else {
|
||||
if ($ret->next_sibling()->next_sibling()) {
|
||||
$this->_res['synopsis'] = trim($ret->next_sibling()->next_sibling()->next_sibling()->plaintext);
|
||||
} else {
|
||||
$this->_res['synopsis'] = 'N/A';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets movie trailers
|
||||
*/
|
||||
protected function trailers(): array
|
||||
{
|
||||
// Method 1: Try structured data
|
||||
if (preg_match('/"contentUrl":\s*"(.*?)"/is', $this->_response, $match)) {
|
||||
$url = trim($match[1]);
|
||||
if (! empty($url)) {
|
||||
$this->_res['trailers']['url'] = $url;
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
// Method 2: Modern video embeds
|
||||
$videoSelectors = [
|
||||
'video source',
|
||||
'iframe[src*="trailer"]',
|
||||
'video[src]',
|
||||
];
|
||||
|
||||
foreach ($videoSelectors as $selector) {
|
||||
$ret = $this->_html->findOne($selector);
|
||||
if ($ret && isset($ret->src) && ! empty(trim($ret->src))) {
|
||||
$this->_res['trailers']['url'] = trim($ret->src);
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
}
|
||||
|
||||
// Method 3: Original flash-based trailer extraction
|
||||
$ret = $this->_html->findOne('input#thickbox-trailer-link');
|
||||
if ($ret && \property_exists($ret, 'value')) {
|
||||
$val = trim((string) $ret->value);
|
||||
if (! empty($val)) {
|
||||
$val = str_replace('..', '', $val);
|
||||
$tmprsp = $this->_response;
|
||||
$this->_trailUrl = $val;
|
||||
|
||||
if (preg_match_all('/productID="\+(?<id>\d+),/', $this->_response, $hits)) {
|
||||
$productid = $hits['id'][0];
|
||||
$random = ((float) mt_rand() / (float) mt_getrandmax()) * 5400000000000000;
|
||||
$this->_trailUrl = '/com/tlavideo/vod/FlvAjaxSupportService.cfc?random='.$random;
|
||||
$this->_postParams = 'method=pipeStreamLoc&productID='.$productid;
|
||||
|
||||
$response = getRawHtml(self::BASE_URL.$this->_trailUrl, $this->cookie, $this->_postParams);
|
||||
|
||||
if (! empty($response)) {
|
||||
$retJson = json_decode(json_decode($response, true), true);
|
||||
if ($retJson && isset($retJson['LOC']) && ! empty($retJson['LOC'])) {
|
||||
$this->_res['trailers']['baseurl'] = self::BASE_URL.'/flashmediaserver/trailerPlayer.swf';
|
||||
$this->_res['trailers']['flashvars'] = 'subscribe=false&image=&file='.self::BASE_URL.'/'.$retJson['LOC'].'&autostart=false';
|
||||
// Also provide a modern URL if possible
|
||||
$this->_res['trailers']['url'] = self::BASE_URL.'/'.$retJson['LOC'];
|
||||
}
|
||||
}
|
||||
|
||||
$this->_response = $tmprsp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets product information
|
||||
*/
|
||||
protected function productInfo(bool $extras = false): array
|
||||
{
|
||||
$productInfo = [];
|
||||
$director = '';
|
||||
|
||||
// Method 1: Try structured data
|
||||
if (preg_match('/"director":\s*{[^}]*"name":\s*"(.*?)"/is', $this->_response, $match)) {
|
||||
$director = trim($match[1]);
|
||||
}
|
||||
|
||||
// Method 2: Look for product details in various formats
|
||||
$selectors = [
|
||||
'div#lside',
|
||||
'div.product-details',
|
||||
'div.product-info',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
if ($ret = $this->_html->findOne($selector)) {
|
||||
// Extract country information
|
||||
$country = false;
|
||||
$rawInfo = [];
|
||||
|
||||
foreach ($ret->find('text') as $e) {
|
||||
$e = trim($e->innertext);
|
||||
$e = str_replace([', ', '...', ' '], '', $e);
|
||||
|
||||
if (stripos($e, 'Country:') !== false) {
|
||||
$country = true;
|
||||
}
|
||||
|
||||
if ($country === true) {
|
||||
if (stripos($e, 'addthis_config') === false) {
|
||||
if (! empty($e)) {
|
||||
$rawInfo[] = $e;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($rawInfo)) {
|
||||
$productInfo = array_chunk($rawInfo, 2, false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->_res['productinfo'] = $productInfo;
|
||||
$this->_res['director'] = $director;
|
||||
|
||||
// Get extras if requested
|
||||
if ($extras === true) {
|
||||
$features = false;
|
||||
$extrasData = [];
|
||||
|
||||
$featureSelectors = [
|
||||
'ul.stock-information',
|
||||
'div.features',
|
||||
'div.extras',
|
||||
];
|
||||
|
||||
foreach ($featureSelectors as $selector) {
|
||||
if ($ret = $this->_html->findOne($selector)) {
|
||||
foreach ($ret->find('li') as $e) {
|
||||
$text = trim($e->plaintext);
|
||||
if ($text === 'Features:') {
|
||||
$features = true;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($features === true && ! empty($text)) {
|
||||
$extrasData[] = $text;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($extrasData)) {
|
||||
$this->_res['extras'] = $extrasData;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the cast members
|
||||
*/
|
||||
protected function cast(): array
|
||||
{
|
||||
$cast = [];
|
||||
$director = '';
|
||||
|
||||
// Method 1: Try structured data
|
||||
if (preg_match_all('/"actor":\s*{[^}]*"name":\s*"(.*?)"/is', $this->_response, $matches)) {
|
||||
foreach ($matches[1] as $actor) {
|
||||
$cast[] = trim($actor);
|
||||
}
|
||||
}
|
||||
|
||||
if (preg_match('/"director":\s*{[^}]*"name":\s*"(.*?)"/is', $this->_response, $match)) {
|
||||
$director = trim($match[1]);
|
||||
}
|
||||
|
||||
// Method 2: Try multiple selectors
|
||||
if (empty($cast)) {
|
||||
$castSelectors = [
|
||||
'div.cast a',
|
||||
'div.stars a',
|
||||
'div.performers a',
|
||||
];
|
||||
|
||||
foreach ($castSelectors as $selector) {
|
||||
$elements = $this->_html->find($selector);
|
||||
if (! empty($elements)) {
|
||||
foreach ($elements as $element) {
|
||||
$cast[] = trim($element->plaintext);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Method 3: Original method (fallback)
|
||||
if (empty($cast)) {
|
||||
$castFound = false;
|
||||
$directorFound = false;
|
||||
$rawCast = [];
|
||||
|
||||
if ($ret = $this->_html->findOne('div#lside')) {
|
||||
foreach ($ret->find('text') as $e) {
|
||||
$e = trim($e->innertext);
|
||||
$e = str_replace([',', ' '], '', $e);
|
||||
|
||||
if (stripos($e, 'Cast') !== false) {
|
||||
$castFound = true;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$e = str_replace('Cast:', '', $e);
|
||||
|
||||
if ($castFound === true) {
|
||||
if (stripos($e, 'Director:') !== false) {
|
||||
$directorFound = true;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($directorFound === true && ! empty($e)) {
|
||||
$director = $e;
|
||||
$directorFound = false;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (stripos($e, 'Country:') === false && ! empty($e)) {
|
||||
$rawCast[] = $e;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$cast = $rawCast;
|
||||
}
|
||||
}
|
||||
|
||||
$this->_res['cast'] = array_unique(array_filter($cast));
|
||||
$this->_res['director'] = $director;
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets categories/genres
|
||||
*/
|
||||
protected function genres(): array
|
||||
{
|
||||
$genres = [];
|
||||
|
||||
// Method 1: Try structured data
|
||||
if (preg_match_all('/"genre":\s*"(.*?)"/is', $this->_response, $matches)) {
|
||||
foreach ($matches[1] as $genre) {
|
||||
$genres[] = trim($genre);
|
||||
}
|
||||
}
|
||||
|
||||
// Method 2: Try multiple selectors
|
||||
if (empty($genres)) {
|
||||
$selectors = [
|
||||
'div[id=thekeywords] a',
|
||||
'p[class=keywords] a',
|
||||
'div.categories a',
|
||||
'div.tags a',
|
||||
];
|
||||
|
||||
foreach ($selectors as $selector) {
|
||||
$elements = $this->_html->find($selector);
|
||||
if (! empty($elements)) {
|
||||
foreach ($elements as $e) {
|
||||
$genres[] = trim($e->plaintext);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->_res['genres'] = array_unique(array_filter($genres));
|
||||
|
||||
return $this->_res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for match against search term.
|
||||
*/
|
||||
public function processSite(string $movie): bool
|
||||
{
|
||||
if (empty($movie)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$searchUrl = self::BASE_URL.self::SEARCH_ENDPOINT.urlencode($movie);
|
||||
$this->_response = getRawHtml($searchUrl, $this->cookie);
|
||||
|
||||
if (empty($this->_response)) {
|
||||
// Try age verification URL
|
||||
$this->_response = getRawHtml(self::AGE_VERIFICATION_URL, $this->cookie);
|
||||
if (! empty($this->_response)) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return false; // Need to verify age first
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->_html->loadHtml($this->_response);
|
||||
$bestMatch = null;
|
||||
$highestSimilarity = 0;
|
||||
|
||||
// Try multiple selector patterns for search results
|
||||
$resultSelectors = [
|
||||
'div.product-info a, div.title a',
|
||||
'div.product-title a',
|
||||
'h3.product-title a',
|
||||
];
|
||||
|
||||
foreach ($resultSelectors as $selector) {
|
||||
$results = $this->_html->find($selector);
|
||||
|
||||
if (! empty($results)) {
|
||||
foreach ($results as $result) {
|
||||
$title = $result->title ?? $result->plaintext;
|
||||
$url = $result->href;
|
||||
|
||||
if (! empty($title)) {
|
||||
// Clean title for better comparison
|
||||
$cleanTitle = str_replace('XXX', '', $title);
|
||||
$cleanTitle = preg_replace('/\(.*?\)|[._\-]/i', ' ', $cleanTitle);
|
||||
$cleanTitle = trim($cleanTitle);
|
||||
|
||||
// Compare titles
|
||||
similar_text(strtolower($movie), strtolower($cleanTitle), $similarity);
|
||||
|
||||
if ($similarity > $highestSimilarity) {
|
||||
$highestSimilarity = $similarity;
|
||||
$bestMatch = [
|
||||
'title' => $title,
|
||||
'url' => $url,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
break; // If we found results with this selector, no need to try others
|
||||
}
|
||||
}
|
||||
|
||||
// If we found a match above our threshold
|
||||
if ($bestMatch && $highestSimilarity >= $this->minimumSimilarity) {
|
||||
$this->_title = trim($bestMatch['title']);
|
||||
$this->_directUrl = str_starts_with($bestMatch['url'], 'http')
|
||||
? $bestMatch['url']
|
||||
: self::BASE_URL.$bestMatch['url'];
|
||||
|
||||
// Fetch the movie details page
|
||||
$this->_response = getRawHtml($this->_directUrl, $this->cookie);
|
||||
if (! empty($this->_response)) {
|
||||
$this->_html->loadHtml($this->_response);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,6 @@ class ProcessAdultMovies extends Command
|
||||
protected $signature = 'nntmux:process-adult
|
||||
{--title= : Process a specific movie title}
|
||||
{--debug : Enable debug output}
|
||||
{--no-pipeline : Use legacy processing instead of pipeline}
|
||||
{--limit= : Limit number of releases to process}';
|
||||
|
||||
/**
|
||||
@@ -36,84 +35,77 @@ class ProcessAdultMovies extends Command
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$usePipeline = !$this->option('no-pipeline');
|
||||
$debug = $this->option('debug');
|
||||
$title = $this->option('title');
|
||||
$limit = $this->option('limit');
|
||||
|
||||
if ($usePipeline) {
|
||||
$pipeline = new AdultProcessingPipeline([], true);
|
||||
$pipeline = new AdultProcessingPipeline([], true);
|
||||
|
||||
if ($title) {
|
||||
// Process a single title
|
||||
$this->info("Looking up: {$title}");
|
||||
if ($title) {
|
||||
// Process a single title
|
||||
$this->info("Looking up: {$title}");
|
||||
|
||||
$result = $pipeline->processMovie($title, $debug);
|
||||
$result = $pipeline->processMovie($title, $debug);
|
||||
|
||||
if ($result['status'] === 'matched') {
|
||||
$this->info("Match found on {$result['provider']}!");
|
||||
$title_display = $result['movieData']['title'] ?? 'N/A';
|
||||
$synopsis_display = substr($result['movieData']['synopsis'] ?? 'N/A', 0, 200);
|
||||
$this->line("Title: {$title_display}");
|
||||
$this->line("Synopsis: {$synopsis_display}...");
|
||||
if ($result['status'] === 'matched') {
|
||||
$this->info("Match found on {$result['provider']}!");
|
||||
$title_display = $result['movieData']['title'] ?? 'N/A';
|
||||
$synopsis_display = substr($result['movieData']['synopsis'] ?? 'N/A', 0, 200);
|
||||
$this->line("Title: {$title_display}");
|
||||
$this->line("Synopsis: {$synopsis_display}...");
|
||||
|
||||
if (!empty($result['movieData']['boxcover'])) {
|
||||
$cover_url = $result['movieData']['boxcover'];
|
||||
$this->line("Cover: {$cover_url}");
|
||||
}
|
||||
if (!empty($result['movieData']['boxcover'])) {
|
||||
$cover_url = $result['movieData']['boxcover'];
|
||||
$this->line("Cover: {$cover_url}");
|
||||
}
|
||||
|
||||
if ($debug && !empty($result['debug'])) {
|
||||
$this->newLine();
|
||||
$this->line('Debug Info:');
|
||||
$this->line(json_encode($result['debug'], JSON_PRETTY_PRINT));
|
||||
}
|
||||
} else {
|
||||
$this->warn("No match found for: {$title}");
|
||||
|
||||
if ($debug && !empty($result['debug'])) {
|
||||
$this->newLine();
|
||||
$this->line('Debug Info:');
|
||||
$this->line(json_encode($result['debug'], JSON_PRETTY_PRINT));
|
||||
}
|
||||
if ($debug && !empty($result['debug'])) {
|
||||
$this->newLine();
|
||||
$this->line('Debug Info:');
|
||||
$this->line(json_encode($result['debug'], JSON_PRETTY_PRINT));
|
||||
}
|
||||
} else {
|
||||
// Process all pending releases
|
||||
$this->info('Processing adult movie releases using pipeline...');
|
||||
$this->warn("No match found for: {$title}");
|
||||
|
||||
if ($limit) {
|
||||
$this->info("Limited to {$limit} releases");
|
||||
}
|
||||
|
||||
$pipeline->processXXXReleases();
|
||||
|
||||
$stats = $pipeline->getStats();
|
||||
|
||||
$this->newLine();
|
||||
$this->table(
|
||||
['Metric', 'Value'],
|
||||
[
|
||||
['Processed', $stats['processed']],
|
||||
['Matched', $stats['matched']],
|
||||
['Failed', $stats['failed']],
|
||||
['Skipped', $stats['skipped']],
|
||||
['Duration', sprintf('%.2f seconds', $stats['duration'])],
|
||||
]
|
||||
);
|
||||
|
||||
if (!empty($stats['providers'])) {
|
||||
if ($debug && !empty($result['debug'])) {
|
||||
$this->newLine();
|
||||
$this->info('Provider Statistics:');
|
||||
$providerData = [];
|
||||
foreach ($stats['providers'] as $provider => $count) {
|
||||
$providerData[] = [$provider, $count];
|
||||
}
|
||||
$this->table(['Provider', 'Matches'], $providerData);
|
||||
$this->line('Debug Info:');
|
||||
$this->line(json_encode($result['debug'], JSON_PRETTY_PRINT));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Legacy processing
|
||||
$this->info('Using legacy processing...');
|
||||
(new \Blacklight\XXX())->processXXXReleases();
|
||||
// Process all pending releases
|
||||
$this->info('Processing adult movie releases using pipeline...');
|
||||
|
||||
if ($limit) {
|
||||
$this->info("Limited to {$limit} releases");
|
||||
}
|
||||
|
||||
$pipeline->processXXXReleases();
|
||||
|
||||
$stats = $pipeline->getStats();
|
||||
|
||||
$this->newLine();
|
||||
$this->table(
|
||||
['Metric', 'Value'],
|
||||
[
|
||||
['Processed', $stats['processed']],
|
||||
['Matched', $stats['matched']],
|
||||
['Failed', $stats['failed']],
|
||||
['Skipped', $stats['skipped']],
|
||||
['Duration', sprintf('%.2f seconds', $stats['duration'])],
|
||||
]
|
||||
);
|
||||
|
||||
if (!empty($stats['providers'])) {
|
||||
$this->newLine();
|
||||
$this->info('Provider Statistics:');
|
||||
$providerData = [];
|
||||
foreach ($stats['providers'] as $provider => $count) {
|
||||
$providerData[] = [$provider, $count];
|
||||
}
|
||||
$this->table(['Provider', 'Matches'], $providerData);
|
||||
}
|
||||
}
|
||||
|
||||
return Command::SUCCESS;
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* that automatically handles age verification for adult sites.
|
||||
*/
|
||||
|
||||
use Blacklight\processing\adult\AgeVerificationManager;
|
||||
use App\Services\AdultProcessing\AgeVerificationManager;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Exception\RequestException;
|
||||
|
||||
|
||||
@@ -40,11 +40,11 @@ if (! function_exists('getRawHtml')) {
|
||||
}
|
||||
|
||||
// For adult sites, use age verification manager if available
|
||||
if ($isAdultSite && class_exists('\Blacklight\processing\adult\AgeVerificationManager')) {
|
||||
if ($isAdultSite && class_exists('\App\Services\AdultProcessing\AgeVerificationManager')) {
|
||||
try {
|
||||
static $ageVerificationManager = null;
|
||||
if ($ageVerificationManager === null) {
|
||||
$ageVerificationManager = new \Blacklight\processing\adult\AgeVerificationManager;
|
||||
$ageVerificationManager = new \App\Services\AdultProcessing\AgeVerificationManager;
|
||||
}
|
||||
|
||||
if ($postData !== null) {
|
||||
|
||||
+3
-4
@@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight\processing\adult;
|
||||
namespace App\Services\AdultProcessing;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
@@ -78,9 +78,7 @@ class AgeVerificationManager
|
||||
if ($cookieDir !== null) {
|
||||
$this->cookieDir = $cookieDir;
|
||||
} else {
|
||||
// Use relative path from this file's location
|
||||
// This works both in Laravel and standalone contexts
|
||||
$this->cookieDir = __DIR__.'/../../../storage/app/cookies/adult_sites';
|
||||
$this->cookieDir = storage_path('app/cookies/adult_sites');
|
||||
}
|
||||
|
||||
// Create cookie directory if it doesn't exist
|
||||
@@ -346,3 +344,4 @@ class AgeVerificationManager
|
||||
return $this->cookieDir;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,30 +4,21 @@ namespace App\Services;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Services\AdultProcessing\AdultProcessingPipeline;
|
||||
use Blacklight\XXX;
|
||||
|
||||
class XXXProcessor
|
||||
{
|
||||
private bool $echooutput;
|
||||
private bool $usePipeline;
|
||||
|
||||
public function __construct(bool $echooutput, bool $usePipeline = true)
|
||||
public function __construct(bool $echooutput)
|
||||
{
|
||||
$this->echooutput = $echooutput;
|
||||
$this->usePipeline = $usePipeline;
|
||||
}
|
||||
|
||||
public function process(): void
|
||||
{
|
||||
if ((int) Settings::settingValue('lookupxxx') === 1) {
|
||||
if ($this->usePipeline) {
|
||||
// Use the new pipeline-based processing with async support
|
||||
$pipeline = new AdultProcessingPipeline([], $this->echooutput);
|
||||
$pipeline->processXXXReleases();
|
||||
} else {
|
||||
// Fall back to legacy processing
|
||||
(new XXX)->processXXXReleases();
|
||||
}
|
||||
$pipeline = new AdultProcessingPipeline([], $this->echooutput);
|
||||
$pipeline->processXXXReleases();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
require __DIR__.'/vendor/autoload.php';
|
||||
|
||||
use Blacklight\processing\adult\AgeVerificationManager;
|
||||
use App\Services\AdultProcessing\AgeVerificationManager;
|
||||
|
||||
$command = $argv[1] ?? 'help';
|
||||
$argument = $argv[2] ?? null;
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
// This script will update all records in the xxxinfo table where there is no cover
|
||||
require_once dirname(__DIR__, 3).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
|
||||
|
||||
use App\Services\AdultProcessing\AdultProcessingPipeline;
|
||||
use Blacklight\ColorCLI;
|
||||
use Blacklight\XXX;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
$movie = new XXX;
|
||||
$pipeline = new AdultProcessingPipeline([], true);
|
||||
$c = new ColorCLI;
|
||||
|
||||
$movies = DB::select('SELECT title FROM xxxinfo WHERE cover = 0');
|
||||
@@ -15,7 +15,13 @@ $movies = DB::select('SELECT title FROM xxxinfo WHERE cover = 0');
|
||||
echo $c->primary('Updating '.number_format(\count($movies)).' XXX movie covers.');
|
||||
foreach ($movies as $mov) {
|
||||
$starttime = now()->timestamp;
|
||||
$mov = $movie->updateXXXInfo($mov->title);
|
||||
$result = $pipeline->processMovie($mov->title);
|
||||
|
||||
if ($result['status'] === 'matched') {
|
||||
echo $c->primary('Updated: '.$mov->title);
|
||||
} else {
|
||||
echo $c->warning('No match found for: '.$mov->title);
|
||||
}
|
||||
|
||||
// sleep so that it's not ddos' the site
|
||||
$diff = floor((now()->timestamp - $starttime) * 1000000);
|
||||
|
||||
@@ -1,445 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Blacklight\Processing\Adult;
|
||||
|
||||
use Blacklight\processing\adult\ADE;
|
||||
use Blacklight\processing\adult\ADM;
|
||||
use Blacklight\processing\adult\AEBN;
|
||||
use Blacklight\processing\adult\Hotmovies;
|
||||
use Blacklight\processing\adult\Popporn;
|
||||
use Tests\TestCase;
|
||||
|
||||
/**
|
||||
* Adult Scrapers Real Data Tests
|
||||
*
|
||||
* These tests verify that scrapers can successfully retrieve and parse
|
||||
* real data from adult content sites. They help identify when sites
|
||||
* change their HTML structure and scraping needs to be updated.
|
||||
*
|
||||
* @group adult
|
||||
* @group scrapers
|
||||
* @group integration
|
||||
*/
|
||||
class AdultScrapersTest extends TestCase
|
||||
{
|
||||
/**
|
||||
* Test data for known movies that should be findable
|
||||
*/
|
||||
private array $testMovies = [
|
||||
'ade' => [
|
||||
['title' => 'Pirates', 'year' => 2005],
|
||||
['title' => 'The Masseuse', 'year' => 1990],
|
||||
['title' => 'Debbie Does Dallas', 'year' => 1978],
|
||||
],
|
||||
'adm' => [
|
||||
['title' => 'Pirates', 'year' => 2005],
|
||||
['title' => 'The Masseuse', 'year' => 1990],
|
||||
],
|
||||
'aebn' => [
|
||||
['title' => 'Pirates', 'year' => 2005],
|
||||
['title' => 'The Masseuse', 'year' => 1990],
|
||||
],
|
||||
'hotmovies' => [
|
||||
['title' => 'Pirates', 'year' => 2005],
|
||||
['title' => 'The Masseuse', 'year' => 1990],
|
||||
],
|
||||
'popporn' => [
|
||||
['title' => 'Pirates', 'year' => 2005],
|
||||
['title' => 'The Masseuse', 'year' => 1990],
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group ade
|
||||
*/
|
||||
public function ade_can_search_and_find_movies(): void
|
||||
{
|
||||
$scraper = new ADE;
|
||||
|
||||
foreach ($this->testMovies['ade'] as $movie) {
|
||||
$result = $scraper->processSite($movie['title']);
|
||||
|
||||
$this->assertTrue(
|
||||
$result,
|
||||
"ADE failed to find movie: {$movie['title']} ({$movie['year']})"
|
||||
);
|
||||
|
||||
if ($result) {
|
||||
$data = $scraper->getAll();
|
||||
$this->assertNotEmpty($data, 'ADE returned empty data');
|
||||
$this->assertArrayHasKey('title', $data, 'ADE missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'ADE missing directurl');
|
||||
|
||||
echo "\n✓ ADE found: {$data['title']}\n";
|
||||
echo " URL: {$data['directurl']}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group ade
|
||||
*/
|
||||
public function ade_extracts_complete_movie_data(): void
|
||||
{
|
||||
$scraper = new ADE;
|
||||
$result = $scraper->processSite('Pirates');
|
||||
|
||||
if (! $result) {
|
||||
$this->markTestSkipped('ADE could not find test movie "Pirates"');
|
||||
}
|
||||
|
||||
$data = $scraper->getAll();
|
||||
|
||||
// Required fields
|
||||
$this->assertArrayHasKey('title', $data, 'Missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Missing directurl');
|
||||
|
||||
// Optional but expected fields
|
||||
$expectedFields = ['synopsis', 'cast', 'genres', 'boxcover'];
|
||||
$missingFields = [];
|
||||
|
||||
foreach ($expectedFields as $field) {
|
||||
if (! isset($data[$field]) || empty($data[$field])) {
|
||||
$missingFields[] = $field;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($missingFields)) {
|
||||
echo "\n⚠ ADE missing or empty fields: ".implode(', ', $missingFields)."\n";
|
||||
}
|
||||
|
||||
// Log extracted data for inspection
|
||||
$this->logScrapedData('ADE', 'Pirates', $data);
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group adm
|
||||
*/
|
||||
public function adm_can_search_and_find_movies(): void
|
||||
{
|
||||
$scraper = new ADM;
|
||||
|
||||
foreach ($this->testMovies['adm'] as $movie) {
|
||||
$result = $scraper->processSite($movie['title']);
|
||||
|
||||
$this->assertTrue(
|
||||
$result,
|
||||
"ADM failed to find movie: {$movie['title']} ({$movie['year']})"
|
||||
);
|
||||
|
||||
if ($result) {
|
||||
$data = $scraper->getAll();
|
||||
$this->assertNotEmpty($data, 'ADM returned empty data');
|
||||
$this->assertArrayHasKey('title', $data, 'ADM missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'ADM missing directurl');
|
||||
|
||||
echo "\n✓ ADM found: {$data['title']}\n";
|
||||
echo " URL: {$data['directurl']}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group adm
|
||||
*/
|
||||
public function adm_extracts_complete_movie_data(): void
|
||||
{
|
||||
$scraper = new ADM;
|
||||
$result = $scraper->processSite('Pirates');
|
||||
|
||||
if (! $result) {
|
||||
$this->markTestSkipped('ADM could not find test movie "Pirates"');
|
||||
}
|
||||
|
||||
$data = $scraper->getAll();
|
||||
|
||||
// Required fields
|
||||
$this->assertArrayHasKey('title', $data, 'Missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Missing directurl');
|
||||
|
||||
// Optional but expected fields
|
||||
$expectedFields = ['synopsis', 'cast', 'genres', 'boxcover', 'director'];
|
||||
$missingFields = [];
|
||||
|
||||
foreach ($expectedFields as $field) {
|
||||
if (! isset($data[$field]) || empty($data[$field])) {
|
||||
$missingFields[] = $field;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($missingFields)) {
|
||||
echo "\n⚠ ADM missing or empty fields: ".implode(', ', $missingFields)."\n";
|
||||
}
|
||||
|
||||
// Log extracted data for inspection
|
||||
$this->logScrapedData('ADM', 'Pirates', $data);
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group aebn
|
||||
*/
|
||||
public function aebn_can_search_and_find_movies(): void
|
||||
{
|
||||
$scraper = new AEBN;
|
||||
|
||||
foreach ($this->testMovies['aebn'] as $movie) {
|
||||
$result = $scraper->processSite($movie['title']);
|
||||
|
||||
$this->assertTrue(
|
||||
$result,
|
||||
"AEBN failed to find movie: {$movie['title']} ({$movie['year']})"
|
||||
);
|
||||
|
||||
if ($result) {
|
||||
$data = $scraper->getAll();
|
||||
$this->assertNotEmpty($data, 'AEBN returned empty data');
|
||||
$this->assertArrayHasKey('title', $data, 'AEBN missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'AEBN missing directurl');
|
||||
|
||||
echo "\n✓ AEBN found: {$data['title']}\n";
|
||||
echo " URL: {$data['directurl']}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group aebn
|
||||
*/
|
||||
public function aebn_extracts_complete_movie_data(): void
|
||||
{
|
||||
$scraper = new AEBN;
|
||||
$result = $scraper->processSite('Pirates');
|
||||
|
||||
if (! $result) {
|
||||
$this->markTestSkipped('AEBN could not find test movie "Pirates"');
|
||||
}
|
||||
|
||||
$data = $scraper->getAll();
|
||||
|
||||
// Required fields
|
||||
$this->assertArrayHasKey('title', $data, 'Missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Missing directurl');
|
||||
|
||||
// Optional but expected fields
|
||||
$expectedFields = ['synopsis', 'cast', 'genres', 'boxcover', 'director'];
|
||||
$missingFields = [];
|
||||
|
||||
foreach ($expectedFields as $field) {
|
||||
if (! isset($data[$field]) || empty($data[$field])) {
|
||||
$missingFields[] = $field;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($missingFields)) {
|
||||
echo "\n⚠ AEBN missing or empty fields: ".implode(', ', $missingFields)."\n";
|
||||
}
|
||||
|
||||
// Log extracted data for inspection
|
||||
$this->logScrapedData('AEBN', 'Pirates', $data);
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group hotmovies
|
||||
*/
|
||||
public function hotmovies_can_search_and_find_movies(): void
|
||||
{
|
||||
$scraper = new Hotmovies;
|
||||
|
||||
foreach ($this->testMovies['hotmovies'] as $movie) {
|
||||
$result = $scraper->processSite($movie['title']);
|
||||
|
||||
$this->assertTrue(
|
||||
$result,
|
||||
"Hotmovies failed to find movie: {$movie['title']} ({$movie['year']})"
|
||||
);
|
||||
|
||||
if ($result) {
|
||||
$data = $scraper->getAll();
|
||||
$this->assertNotEmpty($data, 'Hotmovies returned empty data');
|
||||
$this->assertArrayHasKey('title', $data, 'Hotmovies missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Hotmovies missing directurl');
|
||||
|
||||
echo "\n✓ Hotmovies found: {$data['title']}\n";
|
||||
echo " URL: {$data['directurl']}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group hotmovies
|
||||
*/
|
||||
public function hotmovies_extracts_complete_movie_data(): void
|
||||
{
|
||||
$scraper = new Hotmovies;
|
||||
$result = $scraper->processSite('Pirates');
|
||||
|
||||
if (! $result) {
|
||||
$this->markTestSkipped('Hotmovies could not find test movie "Pirates"');
|
||||
}
|
||||
|
||||
$data = $scraper->getAll();
|
||||
|
||||
// Required fields
|
||||
$this->assertArrayHasKey('title', $data, 'Missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Missing directurl');
|
||||
|
||||
// Optional but expected fields
|
||||
$expectedFields = ['synopsis', 'cast', 'genres', 'boxcover', 'director'];
|
||||
$missingFields = [];
|
||||
|
||||
foreach ($expectedFields as $field) {
|
||||
if (! isset($data[$field]) || empty($data[$field])) {
|
||||
$missingFields[] = $field;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($missingFields)) {
|
||||
echo "\n⚠ Hotmovies missing or empty fields: ".implode(', ', $missingFields)."\n";
|
||||
}
|
||||
|
||||
// Log extracted data for inspection
|
||||
$this->logScrapedData('Hotmovies', 'Pirates', $data);
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group popporn
|
||||
*/
|
||||
public function popporn_can_search_and_find_movies(): void
|
||||
{
|
||||
$scraper = new Popporn;
|
||||
|
||||
foreach ($this->testMovies['popporn'] as $movie) {
|
||||
$result = $scraper->processSite($movie['title']);
|
||||
|
||||
$this->assertTrue(
|
||||
$result,
|
||||
"Popporn failed to find movie: {$movie['title']} ({$movie['year']})"
|
||||
);
|
||||
|
||||
if ($result) {
|
||||
$data = $scraper->getAll();
|
||||
$this->assertNotEmpty($data, 'Popporn returned empty data');
|
||||
$this->assertArrayHasKey('title', $data, 'Popporn missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Popporn missing directurl');
|
||||
|
||||
echo "\n✓ Popporn found: {$data['title']}\n";
|
||||
echo " URL: {$data['directurl']}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group popporn
|
||||
*/
|
||||
public function popporn_extracts_complete_movie_data(): void
|
||||
{
|
||||
$scraper = new Popporn;
|
||||
$result = $scraper->processSite('Pirates');
|
||||
|
||||
if (! $result) {
|
||||
$this->markTestSkipped('Popporn could not find test movie "Pirates"');
|
||||
}
|
||||
|
||||
$data = $scraper->getAll();
|
||||
|
||||
// Required fields
|
||||
$this->assertArrayHasKey('title', $data, 'Missing title');
|
||||
$this->assertArrayHasKey('directurl', $data, 'Missing directurl');
|
||||
|
||||
// Optional but expected fields
|
||||
$expectedFields = ['synopsis', 'cast', 'genres', 'boxcover', 'director'];
|
||||
$missingFields = [];
|
||||
|
||||
foreach ($expectedFields as $field) {
|
||||
if (! isset($data[$field]) || empty($data[$field])) {
|
||||
$missingFields[] = $field;
|
||||
}
|
||||
}
|
||||
|
||||
if (! empty($missingFields)) {
|
||||
echo "\n⚠ Popporn missing or empty fields: ".implode(', ', $missingFields)."\n";
|
||||
}
|
||||
|
||||
// Log extracted data for inspection
|
||||
$this->logScrapedData('Popporn', 'Pirates', $data);
|
||||
}
|
||||
|
||||
/**
|
||||
* @test
|
||||
*
|
||||
* @group similarity
|
||||
*/
|
||||
public function all_scrapers_have_configurable_similarity_threshold(): void
|
||||
{
|
||||
$scrapers = [
|
||||
'ADE' => new ADE,
|
||||
'ADM' => new ADM,
|
||||
'AEBN' => new AEBN,
|
||||
'Hotmovies' => new Hotmovies,
|
||||
'Popporn' => new Popporn,
|
||||
];
|
||||
|
||||
foreach ($scrapers as $name => $scraper) {
|
||||
$reflection = new \ReflectionClass($scraper);
|
||||
|
||||
$this->assertTrue(
|
||||
$reflection->hasProperty('minimumSimilarity'),
|
||||
"{$name} missing minimumSimilarity property"
|
||||
);
|
||||
|
||||
$property = $reflection->getProperty('minimumSimilarity');
|
||||
$property->setAccessible(true);
|
||||
$value = $property->getValue($scraper);
|
||||
|
||||
$this->assertIsFloat($value, "{$name} minimumSimilarity should be float");
|
||||
$this->assertEquals(90.0, $value, "{$name} default threshold should be 90.0");
|
||||
|
||||
echo "\n✓ {$name}: minimumSimilarity = {$value}\n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to log scraped data for manual inspection
|
||||
*/
|
||||
private function logScrapedData(string $scraper, string $movie, array $data): void
|
||||
{
|
||||
$logFile = storage_path('logs/scraper_test_'.date('Y-m-d').'.log');
|
||||
$logDir = dirname($logFile);
|
||||
|
||||
if (! is_dir($logDir)) {
|
||||
mkdir($logDir, 0755, true);
|
||||
}
|
||||
|
||||
$logEntry = sprintf(
|
||||
"[%s] %s - %s\n%s\n\n",
|
||||
date('Y-m-d H:i:s'),
|
||||
$scraper,
|
||||
$movie,
|
||||
json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)
|
||||
);
|
||||
|
||||
file_put_contents($logFile, $logEntry, FILE_APPEND);
|
||||
|
||||
echo "\n📝 Logged scraped data to: {$logFile}\n";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user