diff --git a/app/Console/Commands/FetchMovieByImdb.php b/app/Console/Commands/FetchMovieByImdb.php index eeca9ac8b..590777d1f 100644 --- a/app/Console/Commands/FetchMovieByImdb.php +++ b/app/Console/Commands/FetchMovieByImdb.php @@ -45,6 +45,7 @@ class FetchMovieByImdb extends Command Cache::forget('tmdb_movie_'.md5('tt'.$imdbId)); Cache::forget('trakt_movie_'.md5($imdbId)); Cache::forget('imdb_movie_'.md5($imdbId)); + Cache::forget('imdb_scrape_id_'.$imdbId); Cache::forget('omdb_movie_'.md5($imdbId)); $ok = $movie->updateMovieInfo($imdbId); diff --git a/app/Services/ImdbScraper.php b/app/Services/ImdbScraper.php index 020a5d17b..e4aa401e3 100644 --- a/app/Services/ImdbScraper.php +++ b/app/Services/ImdbScraper.php @@ -7,22 +7,40 @@ namespace App\Services; use GuzzleHttp\Client; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Str; use voku\helper\HtmlDomParser; class ImdbScraper { + protected const int SOFT_FAILURE_TTL_MINUTES = 30; + + protected const int HARD_FAILURE_TTL_HOURS = 6; + protected Client $client; - public function __construct() + protected bool $lastRequestWasBlocked = false; + + public function __construct(?Client $client = null) { - $this->client = new Client([ + $this->client = $client ?? new Client([ 'timeout' => 10, + 'connect_timeout' => 10, + 'http_errors' => false, + 'allow_redirects' => true, 'headers' => [ - 'User-Agent' => 'Mozilla/5.0 (compatible; NNTmuxBot/1.0; +https://github.com/NNTmux)', + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', + 'Accept-Language' => 'en-US,en;q=0.9', + 'Cache-Control' => 'no-cache', + 'Pragma' => 'no-cache', ], ]); } + public function wasBlockedByWaf(): bool + { + return $this->lastRequestWasBlocked; + } + /** * Fetch a movie by IMDB numeric ID. * @@ -31,144 +49,59 @@ class ImdbScraper */ public function fetchById(string $id): array|false { - $id = preg_replace('/[^0-9]/', '', $id); - if ($id === '') { + $this->lastRequestWasBlocked = false; + + $id = preg_replace('/[^0-9]/', '', $id) ?? ''; + if ($id === '' || strlen($id) < 5 || strlen($id) > 8) { return false; } + $cacheKey = 'imdb_scrape_id_'.$id; if (Cache::has($cacheKey)) { return Cache::get($cacheKey); } + $url = 'https://www.imdb.com/title/tt'.$id.'/'; + try { - $res = $this->client->get($url); - $html = (string) $res->getBody(); - $dom = HtmlDomParser::str_get_html($html); - if (! $dom) { - Cache::put($cacheKey, false, now()->addHours(6)); + $response = $this->client->get($url, [ + 'headers' => [ + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Referer' => 'https://www.imdb.com/', + 'Upgrade-Insecure-Requests' => '1', + ], + ]); + + $statusCode = $response->getStatusCode(); + $html = (string) $response->getBody(); + + if ($this->isWafResponse($statusCode, $html)) { + $this->lastRequestWasBlocked = true; + Cache::put($cacheKey, false, now()->addMinutes(self::SOFT_FAILURE_TTL_MINUTES)); + Log::notice('IMDb title fetch was challenged by WAF for tt'.$id); return false; } - $titleNode = $dom->findOne('h1'); - $title = trim($titleNode->text() ?? ''); - if ($title === '') { - Cache::put($cacheKey, false, now()->addHours(6)); + if ($statusCode >= 400 || trim($html) === '') { + Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS)); return false; } - // Year - $year = ''; - $yearNode = $dom->findOne("span[data-testid='title-details-releasedate'] a"); - if ($yearNode) { - if (preg_match('/(19|20)\d{2}/', $yearNode->text(), $m)) { - $year = $m[0]; - } - } - if ($year === '') { - $altYearNode = $dom->findOne("span[data-testid='title-details-releasedate']"); - if ($altYearNode && preg_match('/(19|20)\d{2}/', $altYearNode->text(), $m)) { - $year = $m[0]; - } + $data = $this->parseTitleHtml($html, $id); + if ($data === false) { + Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS)); + + return false; } - // Plot - $plot = ''; - $plotNode = $dom->findOne("span[data-testid='plot-l']"); - if (! $plotNode) { - $plotNode = $dom->findOne("span[data-testid='plot-xl']"); - } - if ($plotNode) { - $plot = trim($plotNode->text()); - } - - // Rating - $rating = ''; - $ratingNode = $dom->findOne("div[data-testid='hero-rating-bar__aggregate-rating__score'] span"); - if ($ratingNode) { - $rating = trim($ratingNode->text()); - } - $rating = preg_replace('/[^0-9\.]/', '', $rating); - - // Poster - $cover = ''; - $posterNode = $dom->findOne("div[data-testid='hero-media__poster'] img"); - if ($posterNode) { - $cover = $posterNode->getAttribute('src'); - } - if ($cover !== '' && str_contains($cover, '._V1_')) { - // Attempt higher res by removing size suffix - $cover = preg_replace('/\._V1_.*\./', '.', $cover); - } - - // Genres - $genres = []; - foreach ($dom->find("div[data-testid='genres'] a") as $g) { - $gt = trim($g->text()); - if ($gt !== '') { - $genres[] = $gt; - } - } - - // Directors - $directors = []; - foreach ($dom->find("li[data-testid='title-pc-principal-credit']:contains(Director) a") as $d) { - $dt = trim($d->text()); - if ($dt !== '') { - $directors[] = $dt; - } - } - if (empty($directors)) { - foreach ($dom->find("li[data-testid='title-pc-principal-credit']:contains(Creator) a") as $d) { - $dt = trim($d->text()); - if ($dt !== '') { - $directors[] = $dt; - } - } - } - - // Actors (top billed) - $actors = []; - foreach ($dom->find("div[data-testid='title-cast-item'] a[data-testid='title-cast-item__actor']") as $a) { - $at = trim($a->text()); - if ($at !== '') { - $actors[] = $at; - } - if (count($actors) >= 10) { - break; - } - } - - // Languages (from details section) - $language = ''; - foreach ($dom->find("li[data-testid='title-details-languages'] a") as $ln) { - $lt = trim($ln->text()); - if ($lt !== '') { - $language .= $lt.', '; - } - } - $language = rtrim($language, ', '); - - $data = [ - 'imdbid' => $id, - 'title' => $title, - 'year' => $year, - 'plot' => $plot, - 'rating' => $rating, - 'cover' => $cover, - 'genre' => $genres, - 'director' => $directors, - 'actors' => $actors, - 'language' => $language, - 'type' => 'movie', - ]; Cache::put($cacheKey, $data, now()->addDays(7)); return $data; } catch (\Throwable $e) { Log::debug('IMDb fetch error tt'.$id.': '.$e->getMessage()); - Cache::put($cacheKey, false, now()->addHours(6)); + Cache::put($cacheKey, false, now()->addHours(self::HARD_FAILURE_TTL_HOURS)); return false; } @@ -181,49 +114,621 @@ class ImdbScraper */ public function search(string $query): array { + $this->lastRequestWasBlocked = false; + $query = trim($query); if ($query === '') { return []; } - $norm = strtolower(preg_replace('/[^a-z0-9 ]/i', '', $query)); - $slug = str_replace(' ', '_', $norm); + + $norm = strtolower(preg_replace('/[^a-z0-9 ]/i', '', $query) ?? ''); + $slug = preg_replace('/\s+/', '_', trim($norm)) ?? ''; + if ($slug === '') { + return []; + } + $prefix = substr($slug, 0, 1); $cacheKey = 'imdb_search_'.md5($slug); if (Cache::has($cacheKey)) { return Cache::get($cacheKey); } + $url = 'https://v2.sg.media-imdb.com/suggestion/'.urlencode($prefix).'/'.urlencode($slug).'.json'; + try { - $res = $this->client->get($url); - $json = json_decode((string) $res->getBody(), true); + $res = $this->client->get($url, [ + 'headers' => [ + 'Accept' => 'application/json,text/plain,*/*', + 'Origin' => 'https://www.imdb.com', + 'Referer' => 'https://www.imdb.com/find/?q='.rawurlencode($query).'&s=tt', + ], + ]); + $body = (string) $res->getBody(); + $results = []; - foreach (($json['d'] ?? []) as $row) { - if (! isset($row['id']) || ! str_starts_with($row['id'], 'tt')) { - continue; - } - $id = substr($row['id'], 2); - $title = $row['l'] ?? ''; - $year = $row['y'] ?? ''; - if ($title === '') { - continue; - } - $results[] = [ - 'imdbid' => $id, - 'title' => $title, - 'year' => (string) $year, - ]; - if (count($results) >= 25) { - break; + if ($res->getStatusCode() === 200 && ! $this->isWafResponse($res->getStatusCode(), $body)) { + $results = $this->parseSuggestionJson($body); + } + + if ($results === []) { + $htmlResults = $this->searchHtmlFallback($query); + if ($htmlResults !== []) { + $results = $htmlResults; } } + + if ($results === []) { + $ttl = $this->lastRequestWasBlocked + ? now()->addMinutes(self::SOFT_FAILURE_TTL_MINUTES) + : now()->addHours(self::HARD_FAILURE_TTL_HOURS); + Cache::put($cacheKey, [], $ttl); + + return []; + } + Cache::put($cacheKey, $results, now()->addHours(12)); return $results; // @phpstan-ignore return.type } catch (\Throwable $e) { Log::debug('IMDb search error '.$query.': '.$e->getMessage()); - Cache::put($cacheKey, [], now()->addHours(6)); + Cache::put($cacheKey, [], now()->addHours(self::HARD_FAILURE_TTL_HOURS)); return []; } } + + /** + * @return array + */ + private function parseTitleHtml(string $html, string $id): array|false + { + if ($this->isWafResponse(200, $html)) { + $this->lastRequestWasBlocked = true; + + return false; + } + + $jsonLd = $this->extractPrimaryJsonLdEntity($html); + $dom = HtmlDomParser::str_get_html($html); + + $title = $this->extractTitle($jsonLd, $dom); + if ($title === '') { + return false; + } + + $year = $this->extractYearFromEntity($jsonLd); + if ($year === '') { + $year = $this->extractYearFromDom($dom); + } + + $plot = $this->extractPlot($jsonLd, $dom); + $rating = $this->extractRating($jsonLd, $dom); + $cover = $this->extractCover($jsonLd, $dom); + $genres = $this->extractGenres($jsonLd, $dom); + $directors = $this->extractPeople($jsonLd['director'] ?? null); + $actors = array_slice($this->extractPeople($jsonLd['actor'] ?? null), 0, 10); + $language = $this->extractLanguage($jsonLd, $dom); + $type = $this->extractType($jsonLd); + + if ($dom !== false) { + if ($directors === []) { + $directors = $this->extractPeopleFromDom($dom, [ + "li[data-testid='title-pc-principal-credit']:contains(Director) a", + "li[data-testid='title-pc-principal-credit']:contains(Creator) a", + ]); + } + + if ($actors === []) { + $actors = array_slice($this->extractPeopleFromDom($dom, [ + "div[data-testid='title-cast-item'] a[data-testid='title-cast-item__actor']", + ]), 0, 10); + } + } + + return [ + 'imdbid' => $id, + 'title' => $title, + 'year' => $year, + 'plot' => $plot, + 'rating' => $rating, + 'cover' => $cover, + 'genre' => $genres, + 'director' => $directors, + 'actors' => $actors, + 'language' => $language, + 'type' => $type, + ]; + } + + private function isWafResponse(int $statusCode, string $body): bool + { + if ($statusCode === 202) { + return true; + } + + if ($body === '') { + return false; + } + + return Str::contains($body, [ + 'window.awsWafCookieDomainList', + 'window.gokuProps', + 'challenge.js', + 'captcha', + 'awswaf', + ], true); + } + + /** + * @return array + */ + private function extractPrimaryJsonLdEntity(string $html): array + { + if (! preg_match_all('/]*type=["\']application\/ld\+json["\'][^>]*>(.*?)<\/script>/is', $html, $matches)) { + return []; + } + + foreach ($matches[1] as $json) { + $decoded = json_decode(html_entity_decode(trim($json), ENT_QUOTES | ENT_HTML5, 'UTF-8'), true); + if (! is_array($decoded)) { + continue; + } + + $entity = $this->findTitleEntity($decoded); + if ($entity !== []) { + return $entity; + } + } + + return []; + } + + /** + * @param array $payload + * @return array + */ + private function findTitleEntity(array $payload): array + { + if ($this->looksLikeTitleEntity($payload)) { + /** @var array $payload */ + return $payload; + } + + foreach ($payload as $value) { + if (! is_array($value)) { + continue; + } + + $entity = $this->findTitleEntity($value); + if ($entity !== []) { + return $entity; + } + } + + return []; + } + + /** + * @param array $payload + */ + private function looksLikeTitleEntity(array $payload): bool + { + $type = strtolower((string) ($payload['@type'] ?? '')); + + return in_array($type, ['movie', 'tvseries', 'tvminiseries', 'tvepisode', 'tvmovie', 'video'], true) + || (isset($payload['name']) && (isset($payload['actor']) || isset($payload['director']) || isset($payload['aggregateRating']))); + } + + private function extractTitle(array $jsonLd, HtmlDomParser|false $dom): string + { + $title = trim((string) ($jsonLd['name'] ?? '')); + if ($title !== '') { + return $title; + } + + if ($dom !== false) { + $titleNode = $dom->findOne('h1'); + $title = trim((string) ($titleNode?->text() ?? '')); + if ($title !== '') { + return preg_replace('/\s+\(.*?\)$/', '', $title) ?? $title; + } + + $ogTitle = $dom->findOne("meta[property='og:title']"); + $title = trim((string) ($ogTitle?->getAttribute('content') ?? '')); + if ($title !== '') { + return preg_replace('/\s+-\s+IMDb$/', '', $title) ?? $title; + } + } + + return ''; + } + + private function extractYearFromEntity(array $jsonLd): string + { + $year = $this->extractYearFromText((string) ($jsonLd['datePublished'] ?? '')); + if ($year !== '') { + return $year; + } + + return $this->extractYearFromText((string) ($jsonLd['dateCreated'] ?? '')); + } + + private function extractYearFromDom(HtmlDomParser|false $dom): string + { + if ($dom === false) { + return ''; + } + + $selectors = [ + "span[data-testid='title-details-releasedate'] a", + "span[data-testid='title-details-releasedate']", + "ul[data-testid='hero-title-block__metadata'] li", + ]; + + foreach ($selectors as $selector) { + foreach ($dom->find($selector) as $node) { + $year = $this->extractYearFromText($node->text()); + if ($year !== '') { + return $year; + } + } + } + + return ''; + } + + private function extractPlot(array $jsonLd, HtmlDomParser|false $dom): string + { + $plot = trim((string) ($jsonLd['description'] ?? '')); + if ($plot !== '') { + return $plot; + } + + if ($dom === false) { + return ''; + } + + $plotNode = $dom->findOne("span[data-testid='plot-l']") ?: $dom->findOne("span[data-testid='plot-xl']"); + $plot = trim((string) ($plotNode?->text() ?? '')); + if ($plot !== '') { + return $plot; + } + + $html = $dom->html(); + if (preg_match('/data-testid=["\']plot-(?:l|xl)["\'][^>]*>(.*?)<\/span>/is', $html, $matches) === 1) { + $plot = trim(html_entity_decode(strip_tags($matches[1]), ENT_QUOTES | ENT_HTML5, 'UTF-8')); + if ($plot !== '') { + return $plot; + } + } + + $descriptionNode = $dom->findOne("meta[name='description']"); + + return trim((string) ($descriptionNode?->getAttribute('content') ?? '')); + } + + private function extractRating(array $jsonLd, HtmlDomParser|false $dom): string + { + $rating = trim((string) ($jsonLd['aggregateRating']['ratingValue'] ?? '')); + if ($rating !== '') { + return $rating; + } + + if ($dom === false) { + return ''; + } + + $ratingNode = $dom->findOne("div[data-testid='hero-rating-bar__aggregate-rating__score'] span"); + $ratingText = trim((string) ($ratingNode?->text() ?? '')); + + return preg_match('/\d+(?:\.\d+)?/', $ratingText, $matches) === 1 ? $matches[0] : ''; + } + + private function extractCover(array $jsonLd, HtmlDomParser|false $dom): string + { + $cover = $this->normalizeImageUrl($jsonLd['image'] ?? ''); + if ($cover !== '') { + return $cover; + } + + if ($dom === false) { + return ''; + } + + $posterNode = $dom->findOne("div[data-testid='hero-media__poster'] img") ?: $dom->findOne("meta[property='og:image']"); + $cover = (string) ($posterNode?->getAttribute('src') ?? $posterNode?->getAttribute('content') ?? ''); + + return $this->normalizeImageUrl($cover); + } + + /** + * @return array + */ + private function extractGenres(array $jsonLd, HtmlDomParser|false $dom): array + { + $genres = $this->normalizeStringList($jsonLd['genre'] ?? []); + if ($genres !== []) { + return $genres; + } + + if ($dom === false) { + return []; + } + + $genres = []; + foreach ($dom->find("div[data-testid='genres'] a") as $genreNode) { + $genre = trim($genreNode->text()); + if ($genre !== '') { + $genres[] = $genre; + } + } + + return array_values(array_unique($genres)); + } + + /** + * @return array + */ + private function extractPeople(mixed $value): array + { + $people = []; + foreach ($this->normalizeValueList($value) as $item) { + $name = ''; + if (is_array($item)) { + $name = trim((string) ($item['name'] ?? '')); + } elseif (is_string($item)) { + $name = trim($item); + } + + if ($name !== '') { + $people[] = $name; + } + } + + return array_values(array_unique($people)); + } + + /** + * @param array $selectors + * @return array + */ + private function extractPeopleFromDom(HtmlDomParser $dom, array $selectors): array + { + $people = []; + + foreach ($selectors as $selector) { + foreach ($dom->find($selector) as $node) { + $name = trim($node->text()); + if ($name !== '') { + $people[] = $name; + } + } + } + + return array_values(array_unique($people)); + } + + private function extractLanguage(array $jsonLd, HtmlDomParser|false $dom): string + { + $languages = $this->normalizeStringList($jsonLd['inLanguage'] ?? []); + if ($languages !== []) { + return implode(', ', $languages); + } + + if ($dom === false) { + return ''; + } + + $languages = []; + foreach ($dom->find("li[data-testid='title-details-languages'] a") as $languageNode) { + $language = trim($languageNode->text()); + if ($language !== '') { + $languages[] = $language; + } + } + + return implode(', ', array_values(array_unique($languages))); + } + + private function extractType(array $jsonLd): string + { + $type = strtolower((string) ($jsonLd['@type'] ?? 'movie')); + + return match ($type) { + 'tvseries', 'tvminiseries', 'tvepisode', 'tvmovie' => 'tv', + default => 'movie', + }; + } + + private function extractYearFromText(string $text): string + { + return preg_match('/(19|20)\d{2}/', $text, $matches) === 1 ? $matches[0] : ''; + } + + private function normalizeImageUrl(mixed $image): string + { + $url = ''; + + if (is_string($image)) { + $url = trim($image); + } elseif (is_array($image)) { + $url = trim((string) ($image['url'] ?? $image[0]['url'] ?? $image[0] ?? '')); + } + + if ($url !== '' && str_contains($url, '._V1_')) { + $url = preg_replace('/\._V1_.*\./', '.', $url) ?? $url; + } + + return $url; + } + + /** + * @return array + */ + private function normalizeStringList(mixed $value): array + { + $items = []; + foreach ($this->normalizeValueList($value) as $item) { + if (is_string($item)) { + $string = trim($item); + if ($string !== '') { + $items[] = $string; + } + } + } + + return array_values(array_unique($items)); + } + + /** + * @return array + */ + private function normalizeValueList(mixed $value): array + { + if ($value === null || $value === '') { + return []; + } + + if (is_array($value)) { + return array_is_list($value) ? $value : [$value]; + } + + return [$value]; + } + + /** + * @return array + */ + private function parseSuggestionJson(string $body): array + { + $json = json_decode($body, true); + if (! is_array($json)) { + return []; + } + + $results = []; + foreach (($json['d'] ?? []) as $row) { + if (! is_array($row) || ! isset($row['id']) || ! is_string($row['id']) || ! str_starts_with($row['id'], 'tt')) { + continue; + } + + $title = trim((string) ($row['l'] ?? '')); + if ($title === '') { + continue; + } + + $results[] = [ + 'imdbid' => substr($row['id'], 2), + 'title' => $title, + 'year' => trim((string) ($row['y'] ?? '')), + ]; + + if (count($results) >= 25) { + break; + } + } + + return $results; + } + + /** + * @return array + */ + private function searchHtmlFallback(string $query): array + { + $url = 'https://www.imdb.com/find/?q='.rawurlencode($query).'&s=tt'; + + try { + $response = $this->client->get($url, [ + 'headers' => [ + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Referer' => 'https://www.imdb.com/', + ], + ]); + + $statusCode = $response->getStatusCode(); + $html = (string) $response->getBody(); + + if ($this->isWafResponse($statusCode, $html)) { + $this->lastRequestWasBlocked = true; + + return []; + } + + if ($statusCode >= 400 || trim($html) === '') { + return []; + } + + return $this->parseSearchHtml($html); + } catch (\Throwable $e) { + Log::debug('IMDb HTML search fallback failed for '.$query.': '.$e->getMessage()); + + return []; + } + } + + /** + * @return array + */ + private function parseSearchHtml(string $html): array + { + $results = []; + + if (preg_match_all('#]+href=["\']/title/tt(?P\d{5,8})/["\'][^>]*>(?P.*?)</a>#is', $html, $matches, PREG_OFFSET_CAPTURE)) { + foreach ($matches[0] as $index => $fullMatch) { + $title = trim(html_entity_decode(strip_tags($matches['title'][$index][0]), ENT_QUOTES | ENT_HTML5, 'UTF-8')); + if ($title === '') { + continue; + } + + $offset = $fullMatch[1] + strlen($fullMatch[0]); + $context = substr($html, $offset, 200); + + $results[] = [ + 'imdbid' => $matches['id'][$index][0], + 'title' => $title, + 'year' => $this->extractYearFromText(strip_tags($context)), + ]; + + if (count($results) >= 25) { + break; + } + } + } + + if ($results === []) { + $dom = HtmlDomParser::str_get_html($html); + if ($dom !== false) { + foreach ($dom->find("a[href^='/title/tt']") as $link) { + $href = (string) $link->getAttribute('href'); + if (! preg_match('#/title/tt(?P<id>\d{5,8})/#', $href, $matches)) { + continue; + } + + $title = trim($link->text()); + if ($title === '') { + continue; + } + + $context = trim(strip_tags((string) $html)); + $results[] = [ + 'imdbid' => $matches['id'], + 'title' => $title, + 'year' => $this->extractYearFromText($context), + ]; + + if (count($results) >= 25) { + break; + } + } + } + } + + $unique = []; + foreach ($results as $result) { + $unique[$result['imdbid']] = $result; + } + + return array_values($unique); + } } diff --git a/app/Services/MovieService.php b/app/Services/MovieService.php index e795bc425..be0dfe804 100644 --- a/app/Services/MovieService.php +++ b/app/Services/MovieService.php @@ -73,14 +73,17 @@ class MovieService public function __construct() { $this->releaseImage = new ReleaseImageService; - $this->traktcheck = config('nntmux_api.trakttv_api_key'); + $traktApiKey = trim((string) config('nntmux_api.trakttv_api_key', '')); + $this->traktcheck = $traktApiKey !== '' ? $traktApiKey : null; if ($this->traktcheck !== null) { $this->traktTv = new TraktProvider; } $this->client = new Client; - $this->fanartapikey = config('nntmux_api.fanarttv_api_key'); + $fanartApiKey = trim((string) config('nntmux_api.fanarttv_api_key', '')); + $this->fanartapikey = $fanartApiKey !== '' ? $fanartApiKey : null; $this->fanart = new FanartTvService($this->fanartapikey); - $this->omdbapikey = config('nntmux_api.omdb_api_key'); + $omdbApiKey = trim((string) config('nntmux_api.omdb_api_key', '')); + $this->omdbapikey = $omdbApiKey !== '' ? $omdbApiKey : null; if ($this->omdbapikey !== null) { $this->omdbApi = new OMDbAPI($this->omdbapikey); } @@ -696,7 +699,15 @@ class MovieService $scraper = app(ImdbScraper::class); $scraped = $scraper->fetchById($imdbId); if ($scraped === false || empty($scraped['title'])) { - Cache::put($cacheKey, false, now()->addHours(6)); + $ttl = $scraper->wasBlockedByWaf() + ? now()->addMinutes(30) + : now()->addHours(6); + + if ($scraper->wasBlockedByWaf()) { + Log::warning('IMDb title page fetch is currently blocked by WAF for '.$imdbId.', using shorter negative cache.'); + } + + Cache::put($cacheKey, false, $ttl); return false; } diff --git a/app/Services/TmdbClient.php b/app/Services/TmdbClient.php index 5a3c1f7eb..bdb275eeb 100644 --- a/app/Services/TmdbClient.php +++ b/app/Services/TmdbClient.php @@ -160,6 +160,20 @@ class TmdbClient */ public function getMovie(int|string $id, array $appendToResponse = []): ?array { + if (is_string($id) && str_starts_with($id, 'tt')) { + $movie = $this->findMovieByExternalId($id, 'imdb_id'); + if ($movie === null) { + return null; + } + + $tmdbId = self::getInt($movie, 'id'); + if ($tmdbId <= 0) { + return null; + } + + return $this->getMovie($tmdbId, $appendToResponse); + } + $params = []; if (! empty($appendToResponse)) { @@ -169,6 +183,34 @@ class TmdbClient return $this->get('/movie/'.$id, $params); } + /** + * Find a movie by external ID (IMDb, etc.) + * + * @param string $externalId The external ID value + * @param string $source The source: 'imdb_id' + * @return array<string, mixed>|null The TMDB movie data or null if not found + */ + public function findMovieByExternalId(string $externalId, string $source = 'imdb_id'): ?array + { + if (empty($externalId)) { + return null; + } + + if ($source !== 'imdb_id') { + return null; + } + + $result = $this->get('/find/'.$externalId, [ + 'external_source' => $source, + ]); + + if ($result === null || empty($result['movie_results'])) { + return null; + } + + return $result['movie_results'][0] ?? null; + } + /** * Get movie credits (cast and crew) * diff --git a/tests/Fixtures/imdb/search_inception.json b/tests/Fixtures/imdb/search_inception.json new file mode 100644 index 000000000..da5c6acc5 --- /dev/null +++ b/tests/Fixtures/imdb/search_inception.json @@ -0,0 +1,23 @@ +{ + "d": [ + { + "id": "tt1375666", + "l": "Inception", + "q": "feature", + "qid": "movie", + "rank": 188, + "s": "Leonardo DiCaprio, Joseph Gordon-Levitt", + "y": 2010 + }, + { + "id": "tt1790736", + "l": "Inception: The Cobol Job", + "q": "video", + "qid": "short", + "rank": 3227, + "s": "Leonardo DiCaprio, Joseph Gordon-Levitt", + "y": 2010 + } + ] +} + diff --git a/tests/Fixtures/imdb/title_jsonld.html b/tests/Fixtures/imdb/title_jsonld.html new file mode 100644 index 000000000..011101e11 --- /dev/null +++ b/tests/Fixtures/imdb/title_jsonld.html @@ -0,0 +1,41 @@ +<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <title>Example Movie - IMDb + + + + + +
  • + +
    • + + + diff --git a/tests/Unit/ImdbScraperRealDataTest.php b/tests/Unit/ImdbScraperRealDataTest.php index ed243a86f..e62a99d76 100644 --- a/tests/Unit/ImdbScraperRealDataTest.php +++ b/tests/Unit/ImdbScraperRealDataTest.php @@ -1,46 +1,24 @@ 'array']); - } - - public function test_parse_shawshank_redemption_real_data(): void - { - $fixturePath = base_path('tests/Fixtures/imdb/shawshank_redemption.html'); - if (! file_exists($fixturePath)) { - $this->markTestSkipped('Fixture file not found: tests/Fixtures/imdb/shawshank_redemption.html'); - } - - $this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper'); - } - - public function test_parse_handles_missing_optional_fields(): void - { - $this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper'); - } - public function test_fetch_by_id_validates_format(): void { $scraper = new ImdbScraper; + $this->assertFalse($scraper->fetchById('1234')); - - // Too long $this->assertFalse($scraper->fetchById('123456789')); - - // Non-numeric $this->assertFalse($scraper->fetchById('abc1234')); - - // Valid format (will fail to fetch, but format is valid) - // We can't test actual network call in unit tests } public function test_search_returns_empty_for_empty_query(): void @@ -50,13 +28,87 @@ class ImdbScraperRealDataTest extends TestCase $this->assertSame([], $scraper->search(' ')); } - public function test_parse_with_og_meta_fallback(): void + public function test_fetch_by_id_uses_dom_fallback_when_jsonld_is_missing(): void { - $this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper'); + $html = <<<'HTML' + + + + + + + +

    Fallback Movie

    +
    • 2025
    + Fallback plot line. +
    8.1/10
    + +
  • DirectorFallback Director
    • + +
  • EnglishGerman
    • + + +HTML; + + $scraper = $this->makeScraperWithResponses([ + new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], $html), + ]); + + $data = $scraper->fetchById('7654321'); + + $this->assertIsArray($data); + $this->assertSame('Fallback Movie', $data['title']); + $this->assertSame('2025', $data['year']); + $this->assertSame('Fallback plot line.', $data['plot']); + $this->assertSame('8.1', $data['rating']); + $this->assertSame(['Drama', 'Thriller'], $data['genre']); + $this->assertSame(['Fallback Director'], $data['director']); + $this->assertSame(['Fallback Actor'], $data['actors']); + $this->assertSame('English, German', $data['language']); } - public function test_parse_extracts_multiple_genres(): void + public function test_search_falls_back_to_html_results_when_suggestion_json_is_unavailable(): void { - $this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper'); + $html = <<<'HTML' + + + +
    + Inception + 2010 +
    +
    + Inception: The Cobol Job + 2010 +
    + + +HTML; + + $scraper = $this->makeScraperWithResponses([ + new Response(500, ['Content-Type' => 'application/json; charset=UTF-8'], '{}'), + new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], $html), + ]); + + $results = $scraper->search('Inception'); + + $this->assertCount(2, $results); + $this->assertSame('1375666', $results[0]['imdbid']); + $this->assertSame('Inception', $results[0]['title']); + $this->assertSame('2010', $results[0]['year']); + } + + /** + * @param array $responses + */ + private function makeScraperWithResponses(array $responses): ImdbScraper + { + $mock = new MockHandler($responses); + $client = new Client([ + 'handler' => HandlerStack::create($mock), + 'http_errors' => false, + ]); + + return new ImdbScraper($client); } } diff --git a/tests/Unit/ImdbScraperTest.php b/tests/Unit/ImdbScraperTest.php index 9dbb5715c..624bf59e9 100644 --- a/tests/Unit/ImdbScraperTest.php +++ b/tests/Unit/ImdbScraperTest.php @@ -1,35 +1,63 @@ 'array']); + $scraper = $this->makeScraperWithResponses([ + new Response(200, ['Content-Type' => 'text/html; charset=UTF-8'], file_get_contents(base_path('tests/Fixtures/imdb/title_jsonld.html')) ?: ''), + ]); + + $data = $scraper->fetchById('1234567'); + + $this->assertIsArray($data); + $this->assertSame('1234567', $data['imdbid']); + $this->assertSame('Example Movie', $data['title']); + $this->assertSame('2024', $data['year']); + $this->assertSame('An example plot goes here.', $data['plot']); + $this->assertSame('7.3', $data['rating']); + $this->assertSame('https://example.com/poster_from_jsonld.jpg', $data['cover']); + $this->assertSame(['Action', 'Adventure'], $data['genre']); + $this->assertSame(['Famous Director'], $data['director']); + $this->assertSame(['First Actor', 'Second Actor'], $data['actors']); + $this->assertSame('English, Spanish', $data['language']); + $this->assertSame('movie', $data['type']); } - public function test_parse_title_page_from_fixture(): void + public function test_fetch_by_id_detects_waf_challenge_and_marks_temporary_block(): void { - $fixturePath = base_path('tests/Fixtures/imdb/example_title.html'); - if (! file_exists($fixturePath)) { - $this->markTestSkipped('Fixture file not found: tests/Fixtures/imdb/example_title.html'); - } + $scraper = $this->makeScraperWithResponses([ + new Response(202, ['Content-Type' => 'text/html; charset=UTF-8'], ''), + ]); - $this->markTestSkipped('Test requires parseForTest() method which is not implemented in ImdbScraper'); - } + $data = $scraper->fetchById('1234567'); - public function test_fetch_by_id_caches_false_on_failure(): void - { - $scraper = new ImdbScraper; - $data = $scraper->fetchById('123'); // invalid length triggers early false $this->assertFalse($data); - $data2 = $scraper->fetchById('123'); - $this->assertFalse($data2); + $this->assertTrue($scraper->wasBlockedByWaf()); + } + + public function test_search_parses_suggestion_json_results(): void + { + $scraper = $this->makeScraperWithResponses([ + new Response(200, ['Content-Type' => 'application/json; charset=UTF-8'], file_get_contents(base_path('tests/Fixtures/imdb/search_inception.json')) ?: '{}'), + ]); + + $results = $scraper->search('Inception'); + + $this->assertNotEmpty($results); + $this->assertSame('1375666', $results[0]['imdbid']); + $this->assertSame('Inception', $results[0]['title']); + $this->assertSame('2010', $results[0]['year']); } public function test_search_empty_returns_empty_array(): void @@ -37,4 +65,18 @@ class ImdbScraperTest extends TestCase $scraper = new ImdbScraper; $this->assertSame([], $scraper->search('')); } + + /** + * @param array $responses + */ + private function makeScraperWithResponses(array $responses): ImdbScraper + { + $mock = new MockHandler($responses); + $client = new Client([ + 'handler' => HandlerStack::create($mock), + 'http_errors' => false, + ]); + + return new ImdbScraper($client); + } } diff --git a/tests/Unit/ImdbScraperTestCase.php b/tests/Unit/ImdbScraperTestCase.php new file mode 100644 index 000000000..bbdf4c8c1 --- /dev/null +++ b/tests/Unit/ImdbScraperTestCase.php @@ -0,0 +1,97 @@ + + */ + private array $originalEnvironment = []; + + public function createApplication() + { + $this->databasePath = sys_get_temp_dir().'/nntmux-imdb-scraper-test.sqlite'; + + $this->originalEnvironment = [ + 'APP_ENV' => getenv('APP_ENV'), + 'DB_CONNECTION' => getenv('DB_CONNECTION'), + 'DB_DATABASE' => getenv('DB_DATABASE'), + ]; + + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings (name, value) VALUES + ('categorizeforeign', '0'), + ('catwebdl', '0'), + ('title', 'NNTmux Test'), + ('home_link', '/')"); + + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $this->databasePath, + 'cache.default' => 'array', + 'app.key' => 'base64:'.base64_encode(random_bytes(32)), + ]); + + DB::purge(); + DB::reconnect(); + Cache::flush(); + } + + protected function tearDown(): void + { + if (($this->databasePath ?? '') !== '' && file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + parent::tearDown(); + + foreach ($this->originalEnvironment as $key => $value) { + $this->setEnvironmentValue($key, $value === false ? null : $value); + } + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv($key.'='.$value); + $_ENV[$key] = $value; + $_SERVER[$key] = $value; + } +} diff --git a/tests/Unit/TmdbClientTest.php b/tests/Unit/TmdbClientTest.php index dcd602ed9..67f3d9040 100644 --- a/tests/Unit/TmdbClientTest.php +++ b/tests/Unit/TmdbClientTest.php @@ -201,14 +201,32 @@ class TmdbClientTest extends TestCase public function test_get_movie_with_imdb_id(): void { Http::fake([ - 'api.themoviedb.org/3/movie/tt0137523*' => Http::response([ + 'api.themoviedb.org/3/find/tt0137523*' => Http::response([ + 'movie_results' => [ + ['id' => 550], + ], + ]), + 'api.themoviedb.org/3/movie/550*' => Http::response([ 'id' => 550, 'title' => 'Fight Club', + 'imdb_id' => 'tt0137523', ]), ]); $result = $this->client->getMovie('tt0137523'); + + Http::assertSent(function ($request) { + return str_contains($request->url(), '/find/tt0137523') + && str_contains($request->url(), 'external_source=imdb_id'); + }); + + Http::assertSent(function ($request) { + return str_contains($request->url(), '/movie/550'); + }); + $this->assertNotNull($result); + $this->assertSame(550, $result['id']); + $this->assertSame('tt0137523', $result['imdb_id']); } public function test_get_movie_returns_null_on_404(): void