starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:01:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update movie scraping noiw that imdb blocks it

Browse Source
This commit is contained in:
DariusIII
2026-04-02 10:39:59 +02:00
parent e3ebdb8dce
commit bc282efae1
10 changed files with 1029 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/FetchMovieByImdb.php
+1
View File
@@ -45,6 +45,7 @@ class FetchMovieByImdb extends Command
Cache::forget('tmdb_movie_'.md5('tt'.$imdbId));
Cache::forget('trakt_movie_'.md5($imdbId));
Cache::forget('imdb_movie_'.md5($imdbId));
Cache::forget('imdb_scrape_id_'.$imdbId);
Cache::forget('omdb_movie_'.md5($imdbId));
$ok = $movie->updateMovieInfo($imdbId);
app/Services/ImdbScraper.php
+645 -140
View File
@@ -7,22 +7,40 @@ namespace App\Services;
use GuzzleHttp\Client;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use voku\helper\HtmlDomParser;
class ImdbScraper
{
protected const int SOFT_FAILURE_TTL_MINUTES = 30;
protected const int HARD_FAILURE_TTL_HOURS = 6;
protected Client $client;
public function __construct()
protected bool $lastRequestWasBlocked = false;
public function __construct(?Client $client = null)
{
$this->client = new Client([
$this->client = $client ?? new Client([
'timeout' => 10,
'connect_timeout' => 10,
'http_errors' => false,
'allow_redirects' => true,
'headers' => [
'User-Agent' => 'Mozilla/5.0 (compatible; NNTmuxBot/1.0; +https://github.com/NNTmux)',
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
'Accept-Language' => 'en-US,en;q=0.9',
'Cache-Control' => 'no-cache',
'Pragma' => 'no-cache',
],
]);
}
public function wasBlockedByWaf(): bool
{
return $this->lastRequestWasBlocked;
}
/**
* Fetch a movie by IMDB numeric ID.
*
@@ -31,144 +49,59 @@ class ImdbScraper
*/
public function fetchById(string $id): array|false
{
$id = preg_replace('/[^0-9]/', '', $id);
if ($id === '') {
$this->lastRequestWasBlocked = false;
$id = preg_replace('/[^0-9]/', '', $id) ?? '';
if ($id === '' || strlen($id) < 5 || strlen($id) > 8) {
return false;
}
$cacheKey = 'imdb_scrape_id_'.$id;
if (Cache::has($cacheKey)) {
return Cache::get($cacheKey);
}
$url = 'https://www.imdb.com/title/tt'.$id.'/';
try {
$res = $this->client->get($url);
$html = (string) $res->getBody();
$dom = HtmlDomParser::str_get_html($html);
if (! $dom) {
Cache::put($cacheKey, false, now()->addHours(6));
$response = $this->client->get($url, [
'headers' => [
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Referer' => 'https://www.imdb.com/',
'Upgrade-Insecure-Requests' => '1',
],
]);
$statusCode = $response->getStatusCode();
$html = (string) $response->getBody();
if ($this->isWafResponse($statusCode, $html)) {
$this->lastRequestWasBlocked = true;
Cache::put($cacheKey, false, now()->addMinutes(self::SOFT_FAILURE_TTL_MINUTES));
Log::notice('IMDb title fetch was challenged by WAF for tt'.$id);
return false;
}
$titleNode = $dom->findOne('h1');
$title = trim($titleNode->text() ?? '');
if ($title === '') {
Cache::put($cacheKey, false, now()->addHours(6));
if ($statusCode >= 400 || trim($html) === '') {
Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS));
return false;
}
// Year
$year = '';
$yearNode = $dom->findOne("span[data-testid='title-details-releasedate'] a");
if ($yearNode) {
if (preg_match('/(19|20)\d{2}/', $yearNode->text(), $m)) {
$year = $m[0];
}
}
if ($year === '') {
$altYearNode = $dom->findOne("span[data-testid='title-details-releasedate']");
if ($altYearNode && preg_match('/(19|20)\d{2}/', $altYearNode->text(), $m)) {
$year = $m[0];
}
$data = $this->parseTitleHtml($html, $id);
if ($data === false) {
Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS));
return false;
}
// Plot
$plot = '';
$plotNode = $dom->findOne("span[data-testid='plot-l']");
if (! $plotNode) {
$plotNode = $dom->findOne("span[data-testid='plot-xl']");
}
if ($plotNode) {
$plot = trim($plotNode->text());
}
// Rating
$rating = '';
$ratingNode = $dom->findOne("div[data-testid='hero-rating-bar__aggregate-rating__score'] span");
if ($ratingNode) {
$rating = trim($ratingNode->text());
}
$rating = preg_replace('/[^0-9\.]/', '', $rating);
// Poster
$cover = '';
$posterNode = $dom->findOne("div[data-testid='hero-media__poster'] img");
if ($posterNode) {
$cover = $posterNode->getAttribute('src');
}
if ($cover !== '' && str_contains($cover, '._V1_')) {
// Attempt higher res by removing size suffix
$cover = preg_replace('/\._V1_.*\./', '.', $cover);
}
// Genres
$genres = [];
foreach ($dom->find("div[data-testid='genres'] a") as $g) {
$gt = trim($g->text());
if ($gt !== '') {
$genres[] = $gt;
}
}
// Directors
$directors = [];
foreach ($dom->find("li[data-testid='title-pc-principal-credit']:contains(Director) a") as $d) {
$dt = trim($d->text());
if ($dt !== '') {
$directors[] = $dt;
}
}
if (empty($directors)) {
foreach ($dom->find("li[data-testid='title-pc-principal-credit']:contains(Creator) a") as $d) {
$dt = trim($d->text());
if ($dt !== '') {
$directors[] = $dt;
}
}
}
// Actors (top billed)
$actors = [];
foreach ($dom->find("div[data-testid='title-cast-item'] a[data-testid='title-cast-item__actor']") as $a) {
$at = trim($a->text());
if ($at !== '') {
$actors[] = $at;
}
if (count($actors) >= 10) {
break;
}
}
// Languages (from details section)
$language = '';
foreach ($dom->find("li[data-testid='title-details-languages'] a") as $ln) {
$lt = trim($ln->text());
if ($lt !== '') {
$language .= $lt.', ';
}
}
$language = rtrim($language, ', ');
$data = [
'imdbid' => $id,
'title' => $title,
'year' => $year,
'plot' => $plot,
'rating' => $rating,
'cover' => $cover,
'genre' => $genres,
'director' => $directors,
'actors' => $actors,
'language' => $language,
'type' => 'movie',
];
Cache::put($cacheKey, $data, now()->addDays(7));
return $data;
} catch (\Throwable $e) {
Log::debug('IMDb fetch error tt'.$id.': '.$e->getMessage());
Cache::put($cacheKey, false, now()->addHours(6));
Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS));
return false;
}
@@ -181,49 +114,621 @@ class ImdbScraper
*/
public function search(string $query): array
{
$this->lastRequestWasBlocked = false;
$query = trim($query);
if ($query === '') {
return [];
}
$norm = strtolower(preg_replace('/[^a-z0-9 ]/i', '', $query));
$slug = str_replace(' ', '_', $norm);
$norm = strtolower(preg_replace('/[^a-z0-9 ]/i', '', $query) ?? '');
$slug = preg_replace('/\s+/', '_', trim($norm)) ?? '';
if ($slug === '') {
return [];
}
$prefix = substr($slug, 0, 1);
$cacheKey = 'imdb_search_'.md5($slug);
if (Cache::has($cacheKey)) {
return Cache::get($cacheKey);
}
$url = 'https://v2.sg.media-imdb.com/suggestion/'.urlencode($prefix).'/'.urlencode($slug).'.json';
try {
$res = $this->client->get($url);
$json = json_decode((string) $res->getBody(), true);
$res = $this->client->get($url, [
'headers' => [
'Accept' => 'application/json,text/plain,*/*',
'Origin' => 'https://www.imdb.com',
'Referer' => 'https://www.imdb.com/find/?q='.rawurlencode($query).'&s=tt',
],
]);
$body = (string) $res->getBody();
$results = [];
foreach (($json['d'] ?? []) as $row) {
if (! isset($row['id']) || ! str_starts_with($row['id'], 'tt')) {
continue;
}
$id = substr($row['id'], 2);
$title = $row['l'] ?? '';
$year = $row['y'] ?? '';
if ($title === '') {
continue;
}
$results[] = [
'imdbid' => $id,
'title' => $title,
'year' => (string) $year,
];
if (count($results) >= 25) {
break;
if ($res->getStatusCode() === 200 && ! $this->isWafResponse($res->getStatusCode(), $body)) {
$results = $this->parseSuggestionJson($body);
}
if ($results === []) {
$htmlResults = $this->searchHtmlFallback($query);
if ($htmlResults !== []) {
$results = $htmlResults;
}
}
if ($results === []) {
$ttl = $this->lastRequestWasBlocked
? now()->addMinutes(self::SOFT_FAILURE_TTL_MINUTES)
: now()->addHours(self::HARD_FAILURE_TTL_HOURS);
Cache::put($cacheKey, [], $ttl);
return [];
}
Cache::put($cacheKey, $results, now()->addHours(12));
return $results; // @phpstan-ignore return.type
} catch (\Throwable $e) {
Log::debug('IMDb search error '.$query.': '.$e->getMessage());
Cache::put($cacheKey, [], now()->addHours(6));
Cache::put($cacheKey, [], now()->addHours(self::HARD_FAILURE_TTL_HOURS));
return [];
}
}
/**
* @return array<string, mixed>
*/
private function parseTitleHtml(string $html, string $id): array|false
{
if ($this->isWafResponse(200, $html)) {
$this->lastRequestWasBlocked = true;
return false;
}
$jsonLd = $this->extractPrimaryJsonLdEntity($html);
$dom = HtmlDomParser::str_get_html($html);
$title = $this->extractTitle($jsonLd, $dom);
if ($title === '') {
return false;
}
$year = $this->extractYearFromEntity($jsonLd);
if ($year === '') {
$year = $this->extractYearFromDom($dom);
}
$plot = $this->extractPlot($jsonLd, $dom);
$rating = $this->extractRating($jsonLd, $dom);
$cover = $this->extractCover($jsonLd, $dom);
$genres = $this->extractGenres($jsonLd, $dom);
$directors = $this->extractPeople($jsonLd['director'] ?? null);
$actors = array_slice($this->extractPeople($jsonLd['actor'] ?? null), 0, 10);
$language = $this->extractLanguage($jsonLd, $dom);
$type = $this->extractType($jsonLd);
if ($dom !== false) {
if ($directors === []) {
$directors = $this->extractPeopleFromDom($dom, [
"li[data-testid='title-pc-principal-credit']:contains(Director) a",
"li[data-testid='title-pc-principal-credit']:contains(Creator) a",
]);
}
if ($actors === []) {
$actors = array_slice($this->extractPeopleFromDom($dom, [
"div[data-testid='title-cast-item'] a[data-testid='title-cast-item__actor']",
]), 0, 10);
}
}
return [
'imdbid' => $id,
'title' => $title,
'year' => $year,
'plot' => $plot,
'rating' => $rating,
'cover' => $cover,
'genre' => $genres,
'director' => $directors,
'actors' => $actors,
'language' => $language,
'type' => $type,
];
}
private function isWafResponse(int $statusCode, string $body): bool
{
if ($statusCode === 202) {
return true;
}
if ($body === '') {
return false;
}
return Str::contains($body, [
'window.awsWafCookieDomainList',
'window.gokuProps',
'challenge.js',
'captcha',
'awswaf',
], true);
}
/**
* @return array<string, mixed>
*/
private function extractPrimaryJsonLdEntity(string $html): array
{
if (! preg_match_all('/<script[^>]*type=["\']application\/ld\+json["\'][^>]*>(.*?)<\/script>/is', $html, $matches)) {
return [];
}
foreach ($matches[1] as $json) {
$decoded = json_decode(html_entity_decode(trim($json), ENT_QUOTES | ENT_HTML5, 'UTF-8'), true);
if (! is_array($decoded)) {
continue;
}
$entity = $this->findTitleEntity($decoded);
if ($entity !== []) {
return $entity;
}
}
return [];
}
/**
* @param array<int|string, mixed> $payload
* @return array<string, mixed>
*/
private function findTitleEntity(array $payload): array
{
if ($this->looksLikeTitleEntity($payload)) {
/** @var array<string, mixed> $payload */
return $payload;
}
foreach ($payload as $value) {
if (! is_array($value)) {
continue;
}
$entity = $this->findTitleEntity($value);
if ($entity !== []) {
return $entity;
}
}
return [];
}
/**
* @param array<int|string, mixed> $payload
*/
private function looksLikeTitleEntity(array $payload): bool
{
$type = strtolower((string) ($payload['@type'] ?? ''));
return in_array($type, ['movie', 'tvseries', 'tvminiseries', 'tvepisode', 'tvmovie', 'video'], true)
|| (isset($payload['name']) && (isset($payload['actor']) || isset($payload['director']) || isset($payload['aggregateRating'])));
}
private function extractTitle(array $jsonLd, HtmlDomParser|false $dom): string
{
$title = trim((string) ($jsonLd['name'] ?? ''));
if ($title !== '') {
return $title;
}
if ($dom !== false) {
$titleNode = $dom->findOne('h1');
$title = trim((string) ($titleNode?->text() ?? ''));
if ($title !== '') {
return preg_replace('/\s+\(.*?\)$/', '', $title) ?? $title;
}
$ogTitle = $dom->findOne("meta[property='og:title']");
$title = trim((string) ($ogTitle?->getAttribute('content') ?? ''));
if ($title !== '') {
return preg_replace('/\s+-\s+IMDb$/', '', $title) ?? $title;
}
}
return '';
}
private function extractYearFromEntity(array $jsonLd): string
{
$year = $this->extractYearFromText((string) ($jsonLd['datePublished'] ?? ''));
if ($year !== '') {
return $year;
}
return $this->extractYearFromText((string) ($jsonLd['dateCreated'] ?? ''));
}
private function extractYearFromDom(HtmlDomParser|false $dom): string
{
if ($dom === false) {
return '';
}
$selectors = [
"span[data-testid='title-details-releasedate'] a",
"span[data-testid='title-details-releasedate']",
"ul[data-testid='hero-title-block__metadata'] li",
];
foreach ($selectors as $selector) {
foreach ($dom->find($selector) as $node) {
$year = $this->extractYearFromText($node->text());
if ($year !== '') {
return $year;
}
}
}
return '';
}
private function extractPlot(array $jsonLd, HtmlDomParser|false $dom): string
{
$plot = trim((string) ($jsonLd['description'] ?? ''));
if ($plot !== '') {
return $plot;
}
if ($dom === false) {
return '';
}
$plotNode = $dom->findOne("span[data-testid='plot-l']") ?: $dom->findOne("span[data-testid='plot-xl']");
$plot = trim((string) ($plotNode?->text() ?? ''));
if ($plot !== '') {
return $plot;
}
$html = $dom->html();
if (preg_match('/data-testid=["\']plot-(?:l|xl)["\'][^>]*>(.*?)<\/span>/is', $html, $matches) === 1) {
$plot = trim(html_entity_decode(strip_tags($matches[1]), ENT_QUOTES | ENT_HTML5, 'UTF-8'));
if ($plot !== '') {
return $plot;
}
}
$descriptionNode = $dom->findOne("meta[name='description']");
return trim((string) ($descriptionNode?->getAttribute('content') ?? ''));
}
private function extractRating(array $jsonLd, HtmlDomParser|false $dom): string
{
$rating = trim((string) ($jsonLd['aggregateRating']['ratingValue'] ?? ''));
if ($rating !== '') {
return $rating;
}
if ($dom === false) {
return '';
}
$ratingNode = $dom->findOne("div[data-testid='hero-rating-bar__aggregate-rating__score'] span");
$ratingText = trim((string) ($ratingNode?->text() ?? ''));
return preg_match('/\d+(?:\.\d+)?/', $ratingText, $matches) === 1 ? $matches[0] : '';
}
private function extractCover(array $jsonLd, HtmlDomParser|false $dom): string
{
$cover = $this->normalizeImageUrl($jsonLd['image'] ?? '');
if ($cover !== '') {
return $cover;
}
if ($dom === false) {
return '';
}
$posterNode = $dom->findOne("div[data-testid='hero-media__poster'] img") ?: $dom->findOne("meta[property='og:image']");
$cover = (string) ($posterNode?->getAttribute('src') ?? $posterNode?->getAttribute('content') ?? '');
return $this->normalizeImageUrl($cover);
}
/**
* @return array<int, string>
*/
private function extractGenres(array $jsonLd, HtmlDomParser|false $dom): array
{
$genres = $this->normalizeStringList($jsonLd['genre'] ?? []);
if ($genres !== []) {
return $genres;
}
if ($dom === false) {
return [];
}
$genres = [];
foreach ($dom->find("div[data-testid='genres'] a") as $genreNode) {
$genre = trim($genreNode->text());
if ($genre !== '') {
$genres[] = $genre;
}
}
return array_values(array_unique($genres));
}
/**
* @return array<int, string>
*/
private function extractPeople(mixed $value): array
{
$people = [];
foreach ($this->normalizeValueList($value) as $item) {
$name = '';
if (is_array($item)) {
$name = trim((string) ($item['name'] ?? ''));
} elseif (is_string($item)) {
$name = trim($item);
}
if ($name !== '') {
$people[] = $name;
}
}
return array_values(array_unique($people));
}
/**
* @param array<int, string> $selectors
* @return array<int, string>
*/
private function extractPeopleFromDom(HtmlDomParser $dom, array $selectors): array
{
$people = [];
foreach ($selectors as $selector) {
foreach ($dom->find($selector) as $node) {
$name = trim($node->text());
if ($name !== '') {
$people[] = $name;
}
}
}
return array_values(array_unique($people));
}
private function extractLanguage(array $jsonLd, HtmlDomParser|false $dom): string
{
$languages = $this->normalizeStringList($jsonLd['inLanguage'] ?? []);
if ($languages !== []) {
return implode(', ', $languages);
}
if ($dom === false) {
return '';
}
$languages = [];
foreach ($dom->find("li[data-testid='title-details-languages'] a") as $languageNode) {
$language = trim($languageNode->text());
if ($language !== '') {
$languages[] = $language;
}
}
return implode(', ', array_values(array_unique($languages)));
}
private function extractType(array $jsonLd): string
{
$type = strtolower((string) ($jsonLd['@type'] ?? 'movie'));
return match ($type) {
'tvseries', 'tvminiseries', 'tvepisode', 'tvmovie' => 'tv',
default => 'movie',
};
}
private function extractYearFromText(string $text): string
{
return preg_match('/(19|20)\d{2}/', $text, $matches) === 1 ? $matches[0] : '';
}
private function normalizeImageUrl(mixed $image): string
{
$url = '';
if (is_string($image)) {
$url = trim($image);
} elseif (is_array($image)) {
$url = trim((string) ($image['url'] ?? $image[0]['url'] ?? $image[0] ?? ''));
}
if ($url !== '' && str_contains($url, '._V1_')) {
$url = preg_replace('/\._V1_.*\./', '.', $url) ?? $url;
}
return $url;
}
/**
* @return array<int, string>
*/
private function normalizeStringList(mixed $value): array
{
$items = [];
foreach ($this->normalizeValueList($value) as $item) {
if (is_string($item)) {
$string = trim($item);
if ($string !== '') {
$items[] = $string;
}
}
}
return array_values(array_unique($items));
}
/**
* @return array<int, mixed>
*/
private function normalizeValueList(mixed $value): array
{
if ($value === null || $value === '') {
return [];
}
if (is_array($value)) {
return array_is_list($value) ? $value : [$value];
}
return [$value];
}
/**
* @return array<int, array{imdbid: string, title: string, year: string}>
*/
private function parseSuggestionJson(string $body): array
{
$json = json_decode($body, true);
if (! is_array($json)) {
return [];
}
$results = [];
foreach (($json['d'] ?? []) as $row) {
if (! is_array($row) || ! isset($row['id']) || ! is_string($row['id']) || ! str_starts_with($row['id'], 'tt')) {
continue;
}
$title = trim((string) ($row['l'] ?? ''));
if ($title === '') {
continue;
}
$results[] = [
'imdbid' => substr($row['id'], 2),
'title' => $title,
'year' => trim((string) ($row['y'] ?? '')),
];
if (count($results) >= 25) {
break;
}
}
return $results;
}
/**
* @return array<int, array{imdbid: string, title: string, year: string}>
*/
private function searchHtmlFallback(string $query): array
{
$url = 'https://www.imdb.com/find/?q='.rawurlencode($query).'&s=tt';
try {
$response = $this->client->get($url, [
'headers' => [
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Referer' => 'https://www.imdb.com/',
],
]);
$statusCode = $response->getStatusCode();
$html = (string) $response->getBody();
if ($this->isWafResponse($statusCode, $html)) {
$this->lastRequestWasBlocked = true;
return [];
}
if ($statusCode >= 400 || trim($html) === '') {
return [];
}
return $this->parseSearchHtml($html);
} catch (\Throwable $e) {
Log::debug('IMDb HTML search fallback failed for '.$query.': '.$e->getMessage());
return [];
}
}
/**
* @return array<int, array{imdbid: string, title: string, year: string}>
*/
private function parseSearchHtml(string $html): array
{
$results = [];
if (preg_match_all('#<a[^>]+href=["\']/title/tt(?P<id>\d{5,8})/["\'][^>]*>(?P<title>.*?)</a>#is', $html, $matches, PREG_OFFSET_CAPTURE)) {
foreach ($matches[0] as $index => $fullMatch) {
$title = trim(html_entity_decode(strip_tags($matches['title'][$index][0]), ENT_QUOTES | ENT_HTML5, 'UTF-8'));
if ($title === '') {
continue;
}
$offset = $fullMatch[1] + strlen($fullMatch[0]);
$context = substr($html, $offset, 200);
$results[] = [
'imdbid' => $matches['id'][$index][0],
'title' => $title,
'year' => $this->extractYearFromText(strip_tags($context)),
];
if (count($results) >= 25) {
break;
}
}
}
if ($results === []) {
$dom = HtmlDomParser::str_get_html($html);
if ($dom !== false) {
foreach ($dom->find("a[href^='/title/tt']") as $link) {
$href = (string) $link->getAttribute('href');
if (! preg_match('#/title/tt(?P<id>\d{5,8})/#', $href, $matches)) {
continue;
}
$title = trim($link->text());
if ($title === '') {
continue;
}
$context = trim(strip_tags((string) $html));
$results[] = [
'imdbid' => $matches['id'],
'title' => $title,
'year' => $this->extractYearFromText($context),
];
if (count($results) >= 25) {
break;
}
}
}
}
$unique = [];
foreach ($results as $result) {
$unique[$result['imdbid']] = $result;
}
return array_values($unique);
}
}
app/Services/MovieService.php
+15 -4
View File
@@ -73,14 +73,17 @@ class MovieService
public function __construct()
{
$this->releaseImage = new ReleaseImageService;
$this->traktcheck = config('nntmux_api.trakttv_api_key');
$traktApiKey = trim((string) config('nntmux_api.trakttv_api_key', ''));
$this->traktcheck = $traktApiKey !== '' ? $traktApiKey : null;
if ($this->traktcheck !== null) {
$this->traktTv = new TraktProvider;
}
$this->client = new Client;
$this->fanartapikey = config('nntmux_api.fanarttv_api_key');
$fanartApiKey = trim((string) config('nntmux_api.fanarttv_api_key', ''));
$this->fanartapikey = $fanartApiKey !== '' ? $fanartApiKey : null;
$this->fanart = new FanartTvService($this->fanartapikey);
$this->omdbapikey = config('nntmux_api.omdb_api_key');
$omdbApiKey = trim((string) config('nntmux_api.omdb_api_key', ''));
$this->omdbapikey = $omdbApiKey !== '' ? $omdbApiKey : null;
if ($this->omdbapikey !== null) {
$this->omdbApi = new OMDbAPI($this->omdbapikey);
}
@@ -696,7 +699,15 @@ class MovieService
$scraper = app(ImdbScraper::class);
$scraped = $scraper->fetchById($imdbId);
if ($scraped === false || empty($scraped['title'])) {
Cache::put($cacheKey, false, now()->addHours(6));
$ttl = $scraper->wasBlockedByWaf()
? now()->addMinutes(30)
: now()->addHours(6);
if ($scraper->wasBlockedByWaf()) {
Log::warning('IMDb title page fetch is currently blocked by WAF for '.$imdbId.', using shorter negative cache.');
}
Cache::put($cacheKey, false, $ttl);
return false;
}
app/Services/TmdbClient.php
+42
View File
@@ -160,6 +160,20 @@ class TmdbClient
*/
public function getMovie(int|string $id, array $appendToResponse = []): ?array
{
if (is_string($id) && str_starts_with($id, 'tt')) {
$movie = $this->findMovieByExternalId($id, 'imdb_id');
if ($movie === null) {
return null;
}
$tmdbId = self::getInt($movie, 'id');
if ($tmdbId <= 0) {
return null;
}
return $this->getMovie($tmdbId, $appendToResponse);
}
$params = [];
if (! empty($appendToResponse)) {
@@ -169,6 +183,34 @@ class TmdbClient
return $this->get('/movie/'.$id, $params);
}
/**
* Find a movie by external ID (IMDb, etc.)
*
* @param string $externalId The external ID value
* @param string $source The source: 'imdb_id'
* @return array<string, mixed>|null The TMDB movie data or null if not found
*/
public function findMovieByExternalId(string $externalId, string $source = 'imdb_id'): ?array
{
if (empty($externalId)) {
return null;
}
if ($source !== 'imdb_id') {
return null;
}
$result = $this->get('/find/'.$externalId, [
'external_source' => $source,
]);
if ($result === null || empty($result['movie_results'])) {
return null;
}
return $result['movie_results'][0] ?? null;
}
/**
* Get movie credits (cast and crew)
*
tests/Fixtures/imdb/search_inception.json
+23
View File
@@ -0,0 +1,23 @@
{
"d": [
{
"id": "tt1375666",
"l": "Inception",
"q": "feature",
"qid": "movie",
"rank": 188,
"s": "Leonardo DiCaprio, Joseph Gordon-Levitt",
"y": 2010
},
{
"id": "tt1790736",
"l": "Inception: The Cobol Job",
"q": "video",
"qid": "short",
"rank": 3227,
"s": "Leonardo DiCaprio, Joseph Gordon-Levitt",
"y": 2010
}
]
}
tests/Fixtures/imdb/title_jsonld.html
+41
View File
@@ -0,0 +1,41 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Example Movie - IMDb</title>
<meta property="og:title" content="Example Movie - IMDb">
<meta property="og:image" content="https://example.com/poster.jpg">
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Movie",
"name": "Example Movie",
"description": "An example plot goes here.",
"datePublished": "2024-05-17",
"image": "https://example.com/poster_from_jsonld.jpg",
"genre": ["Action", "Adventure"],
"aggregateRating": {
"@type": "AggregateRating",
"ratingValue": "7.3",
"ratingCount": 12345
},
"actor": [
{"@type": "Person", "name": "First Actor"},
{"@type": "Person", "name": "Second Actor"}
],
"director": [
{"@type": "Person", "name": "Famous Director"}
]
}
</script>
</head>
<body>
<li data-testid="title-details-languages">
<div>
<a href="/language/en">English</a>
<a href="/language/es">Spanish</a>
</div>
</li>
</body>
</html>
tests/Unit/ImdbScraperRealDataTest.php
+86 -34
View File
@@ -1,46 +1,24 @@
<?php
declare(strict_types=1);
namespace Tests\Unit;
use App\Services\ImdbScraper;
use Tests\TestCase;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
class ImdbScraperRealDataTest extends TestCase
class ImdbScraperRealDataTest extends ImdbScraperTestCase
{
protected function setUp(): void
{
parent::setUp();
config(['cache.default' => 'array']);
}
public function test_parse_shawshank_redemption_real_data(): void
{
$fixturePath = base_path('tests/Fixtures/imdb/shawshank_redemption.html');
if (! file_exists($fixturePath)) {
$this->markTestSkipped('Fixture file not found: tests/Fixtures/imdb/shawshank_redemption.html');
}
$this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper');
}
public function test_parse_handles_missing_optional_fields(): void
{
$this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper');
}
public function test_fetch_by_id_validates_format(): void
{
$scraper = new ImdbScraper;
$this->assertFalse($scraper->fetchById('1234'));
// Too long
$this->assertFalse($scraper->fetchById('123456789'));
// Non-numeric
$this->assertFalse($scraper->fetchById('abc1234'));
// Valid format (will fail to fetch, but format is valid)
// We can't test actual network call in unit tests
}
public function test_search_returns_empty_for_empty_query(): void
@@ -50,13 +28,87 @@ class ImdbScraperRealDataTest extends TestCase
$this->assertSame([], $scraper->search(' '));
}
public function test_parse_with_og_meta_fallback(): void
public function test_fetch_by_id_uses_dom_fallback_when_jsonld_is_missing(): void
{
$this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper');
$html = <<<'HTML'
<!doctype html>
<html lang="en">
<head>
<meta property="og:title" content="Fallback Movie - IMDb">
<meta property="og:image" content="https://example.com/fallback.jpg">
</head>
<body>
<h1>Fallback Movie</h1>
<ul data-testid="hero-title-block__metadata"><li>2025</li></ul>
<span data-testid="plot-xl">Fallback plot line.</span>
<div data-testid="hero-rating-bar__aggregate-rating__score"><span>8.1/10</span></div>
<div data-testid="genres"><a>Drama</a><a>Thriller</a></div>
<li data-testid="title-pc-principal-credit"><span>Director</span><a>Fallback Director</a></li>
<div data-testid="title-cast-item"><a data-testid="title-cast-item__actor">Fallback Actor</a></div>
<li data-testid="title-details-languages"><a>English</a><a>German</a></li>
</body>
</html>
HTML;
$scraper = $this->makeScraperWithResponses([
new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], $html),
]);
$data = $scraper->fetchById('7654321');
$this->assertIsArray($data);
$this->assertSame('Fallback Movie', $data['title']);
$this->assertSame('2025', $data['year']);
$this->assertSame('Fallback plot line.', $data['plot']);
$this->assertSame('8.1', $data['rating']);
$this->assertSame(['Drama', 'Thriller'], $data['genre']);
$this->assertSame(['Fallback Director'], $data['director']);
$this->assertSame(['Fallback Actor'], $data['actors']);
$this->assertSame('English, German', $data['language']);
}
public function test_parse_extracts_multiple_genres(): void
public function test_search_falls_back_to_html_results_when_suggestion_json_is_unavailable(): void
{
$this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper');
$html = <<<'HTML'
<!doctype html>
<html lang="en">
<body>
<section>
<a href="/title/tt1375666/">Inception</a>
<span>2010</span>
</section>
<section>
<a href="/title/tt1790736/">Inception: The Cobol Job</a>
<span>2010</span>
</section>
</body>
</html>
HTML;
$scraper = $this->makeScraperWithResponses([
new Response(500, ['Content-Type' => 'application/json; charset=UTF-8'], '{}'),
new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], $html),
]);
$results = $scraper->search('Inception');
$this->assertCount(2, $results);
$this->assertSame('1375666', $results[0]['imdbid']);
$this->assertSame('Inception', $results[0]['title']);
$this->assertSame('2010', $results[0]['year']);
}
/**
* @param array<int, Response> $responses
*/
private function makeScraperWithResponses(array $responses): ImdbScraper
{
$mock = new MockHandler($responses);
$client = new Client([
'handler' => HandlerStack::create($mock),
'http_errors' => false,
]);
return new ImdbScraper($client);
}
}
tests/Unit/ImdbScraperTest.php
+60 -18
View File
@@ -1,35 +1,63 @@
<?php
declare(strict_types=1);
namespace Tests\Unit;
use App\Services\ImdbScraper;
use Tests\TestCase;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
class ImdbScraperTest extends TestCase
class ImdbScraperTest extends ImdbScraperTestCase
{
protected function setUp(): void
public function test_fetch_by_id_parses_title_page_from_fixture(): void
{
parent::setUp();
config(['cache.default' => 'array']);
$scraper = $this->makeScraperWithResponses([
new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], file_get_contents(base_path('tests/Fixtures/imdb/title_jsonld.html')) ?: ''),
]);
$data = $scraper->fetchById('1234567');
$this->assertIsArray($data);
$this->assertSame('1234567', $data['imdbid']);
$this->assertSame('Example Movie', $data['title']);
$this->assertSame('2024', $data['year']);
$this->assertSame('An example plot goes here.', $data['plot']);
$this->assertSame('7.3', $data['rating']);
$this->assertSame('https://example.com/poster_from_jsonld.jpg', $data['cover']);
$this->assertSame(['Action', 'Adventure'], $data['genre']);
$this->assertSame(['Famous Director'], $data['director']);
$this->assertSame(['First Actor', 'Second Actor'], $data['actors']);
$this->assertSame('English, Spanish', $data['language']);
$this->assertSame('movie', $data['type']);
}
public function test_parse_title_page_from_fixture(): void
public function test_fetch_by_id_detects_waf_challenge_and_marks_temporary_block(): void
{
$fixturePath = base_path('tests/Fixtures/imdb/example_title.html');
if (! file_exists($fixturePath)) {
$this->markTestSkipped('Fixture file not found: tests/Fixtures/imdb/example_title.html');
}
$scraper = $this->makeScraperWithResponses([
new Response(202, ['Content-Type' => 'text/html; charset=UTF-8'], '<html><script>window.awsWafCookieDomainList=[];window.gokuProps={};</script></html>'),
]);
$this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper');
}
$data = $scraper->fetchById('1234567');
public function test_fetch_by_id_caches_false_on_failure(): void
{
$scraper = new ImdbScraper;
$data = $scraper->fetchById('123'); // invalid length triggers early false
$this->assertFalse($data);
$data2 = $scraper->fetchById('123');
$this->assertFalse($data2);
$this->assertTrue($scraper->wasBlockedByWaf());
}
public function test_search_parses_suggestion_json_results(): void
{
$scraper = $this->makeScraperWithResponses([
new Response(200, ['Content-Type' => 'application/json; charset=UTF-8'], file_get_contents(base_path('tests/Fixtures/imdb/search_inception.json')) ?: '{}'),
]);
$results = $scraper->search('Inception');
$this->assertNotEmpty($results);
$this->assertSame('1375666', $results[0]['imdbid']);
$this->assertSame('Inception', $results[0]['title']);
$this->assertSame('2010', $results[0]['year']);
}
public function test_search_empty_returns_empty_array(): void
@@ -37,4 +65,18 @@ class ImdbScraperTest extends TestCase
$scraper = new ImdbScraper;
$this->assertSame([], $scraper->search(''));
}
/**
* @param array<int, Response> $responses
*/
private function makeScraperWithResponses(array $responses): ImdbScraper
{
$mock = new MockHandler($responses);
$client = new Client([
'handler' => HandlerStack::create($mock),
'http_errors' => false,
]);
return new ImdbScraper($client);
}
}
tests/Unit/ImdbScraperTestCase.php
+97
View File
@@ -0,0 +1,97 @@
<?php
declare(strict_types=1);
namespace Tests\Unit;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use PDO;
use Tests\TestCase;
abstract class ImdbScraperTestCase extends TestCase
{
protected string $databasePath;
/**
* @var array<string, string|false>
*/
private array $originalEnvironment = [];
public function createApplication()
{
$this->databasePath = sys_get_temp_dir().'/nntmux-imdb-scraper-test.sqlite';
$this->originalEnvironment = [
'APP_ENV' => getenv('APP_ENV'),
'DB_CONNECTION' => getenv('DB_CONNECTION'),
'DB_DATABASE' => getenv('DB_DATABASE'),
];
if (file_exists($this->databasePath)) {
unlink($this->databasePath);
}
$pdo = new PDO('sqlite:'.$this->databasePath);
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
$pdo->exec("INSERT INTO settings (name, value) VALUES
('categorizeforeign', '0'),
('catwebdl', '0'),
('title', 'NNTmux Test'),
('home_link', '/')");
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
return $app;
}
protected function setUp(): void
{
parent::setUp();
config([
'database.default' => 'sqlite',
'database.connections.sqlite.database' => $this->databasePath,
'cache.default' => 'array',
'app.key' => 'base64:'.base64_encode(random_bytes(32)),
]);
DB::purge();
DB::reconnect();
Cache::flush();
}
protected function tearDown(): void
{
if (($this->databasePath ?? '') !== '' && file_exists($this->databasePath)) {
unlink($this->databasePath);
}
parent::tearDown();
foreach ($this->originalEnvironment as $key => $value) {
$this->setEnvironmentValue($key, $value === false ? null : $value);
}
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
}
tests/Unit/TmdbClientTest.php
+19 -1
View File
@@ -201,14 +201,32 @@ class TmdbClientTest extends TestCase
public function test_get_movie_with_imdb_id(): void
{
Http::fake([
'api.themoviedb.org/3/movie/tt0137523*' => Http::response([
'api.themoviedb.org/3/find/tt0137523*' => Http::response([
'movie_results' => [
['id' => 550],
],
]),
'api.themoviedb.org/3/movie/550*' => Http::response([
'id' => 550,
'title' => 'Fight Club',
'imdb_id' => 'tt0137523',
]),
]);
$result = $this->client->getMovie('tt0137523');
Http::assertSent(function ($request) {
return str_contains($request->url(), '/find/tt0137523')
&& str_contains($request->url(), 'external_source=imdb_id');
});
Http::assertSent(function ($request) {
return str_contains($request->url(), '/movie/550');
});
$this->assertNotNull($result);
$this->assertSame(550, $result['id']);
$this->assertSame('tt0137523', $result['imdb_id']);
}
public function test_get_movie_returns_null_on_404(): void