diff --git a/.env.example b/.env.example index d2dc2202f..d74ce60a1 100644 --- a/.env.example +++ b/.env.example @@ -197,6 +197,7 @@ TVDB_APIKEY=ef6fb572-bcee-4d99-9b52-90549ad7553a TVDB_PIN= ANIDB_APIKEY= FANARTTV_APIKEY= +ISBNDB_API_KEY= OMDB_APIKEY= TRAKTTV_APIKEY= diff --git a/app/Services/BookService.php b/app/Services/BookService.php index 60f32d73f..83a9d81cb 100644 --- a/app/Services/BookService.php +++ b/app/Services/BookService.php @@ -23,12 +23,6 @@ class BookService { public bool $echooutput; - public ?string $pubkey; - - public ?string $privkey; - - public ?string $asstag; - public int $bookqty; public int $sleeptime; @@ -44,6 +38,8 @@ class BookService */ public array $failCache; + public ?string $parsedIsbn; + /** * @throws \Exception */ @@ -51,9 +47,6 @@ class BookService { $this->echooutput = config('nntmux.echocli'); - $this->pubkey = Settings::settingValue('amazonpubkey'); - $this->privkey = Settings::settingValue('amazonprivkey'); - $this->asstag = Settings::settingValue('amazonassociatetag'); $this->bookqty = Settings::settingValue('maxbooksprocessed') !== '' ? (int) Settings::settingValue('maxbooksprocessed') : 300; $this->sleeptime = Settings::settingValue('amazonsleep') !== '' ? (int) Settings::settingValue('amazonsleep') : 1000; $this->imgSavePath = storage_path('covers/book/'); @@ -62,6 +55,7 @@ class BookService $this->renamed = (int) Settings::settingValue('lookupbooks') === 2 ? 'AND isrenamed = 1' : ''; $this->failCache = []; + $this->parsedIsbn = null; } /** @@ -418,7 +412,7 @@ class BookService $bookId = -2; foreach ($res as $arr) { $startTime = now()->timestamp; - $usedAmazon = false; + $usedExternalApi = false; // audiobooks are also books and should be handled in an identical manor, even though it falls under a music category if ($arr['categories_id'] === (int) Category::MUSIC_AUDIOBOOK) { // audiobook @@ -443,8 +437,8 @@ class BookService } $bookId = -2; } elseif ($bookCheck === null) { - $bookId = $this->updateBookInfo($bookInfo); - $usedAmazon = true; + $bookId = $this->updateBookInfo($bookInfo, $this->parsedIsbn); + $usedExternalApi = true; if ($bookId === -2) { $this->failCache[] = $bookInfo; } @@ -460,9 +454,9 @@ class BookService echo '.'; } } - // Sleep to not flood amazon. + // Sleep to avoid flooding external book metadata providers. $diff = floor((now()->timestamp - $startTime) * 1000000); - if ($this->sleeptime * 1000 - $diff > 0 && $usedAmazon === true) { + if ($this->sleeptime * 1000 - $diff > 0 && $usedExternalApi === true) { usleep((int) ($this->sleeptime * 1000 - $diff)); } } @@ -478,10 +472,12 @@ class BookService */ public function parseTitle(mixed $release_name, mixed $releaseID, mixed $releasetype) { + $this->parsedIsbn = $this->extractIsbn((string) $release_name); + $a = preg_replace('/\d{1,2} \d{1,2} \d{2,4}|(19|20)\d\d|anybody got .+?[a-z]\? |[ ._-](Novel|TIA)([ ._-]|$)|([ \.])HQ([-\. ])|[\(\)\.\-_ ](AVI|AZW3?|DOC|EPUB|LIT|MOBI|NFO|RETAIL|(si)?PDF|RTF|TXT)[\)\]\.\-_ ](?![a-z0-9])|compleet|DAGSTiDNiNGEN|DiRFiX|\+ extra|r?e ?Books?([\.\-_ ]English|ers)?|azw3?|ePu([bp])s?|html|mobi|^NEW[\.\-_ ]|PDF([\.\-_ ]English)?|Please post more|Post description|Proper|Repack(fix)?|[\.\-_ ](Chinese|English|French|German|Italian|Retail|Scan|Swedish)|^R4 |Repost|Skytwohigh|TIA!+|TruePDF|V413HAV|(would someone )?please (re)?post.+? "|with the authors name right/i', '', $release_name); $b = preg_replace('/^(As Req |conversion |eq |Das neue Abenteuer \d+|Fixed version( ignore previous post)?|Full |Per Req As Found|(\s+)?R4 |REQ |revised |version |\d+(\s+)?$)|(COMPLETE|INTERNAL|RELOADED| (AZW3|eB|docx|ENG?|exe|FR|Fix|gnv64|MU|NIV|R\d\s+\d{1,2} \d{1,2}|R\d|Req|TTL|UC|v(\s+)?\d))(\s+)?$/i', '', $a); - // remove book series from title as this gets more matches on amazon + // Remove book series from title to improve metadata matching. $c = preg_replace('/ - \[.+\]|\[.+\]/', '', $b); // remove any brackets left behind @@ -517,7 +513,8 @@ class BookService } if ($releasetype === 'audiobook') { if (! empty($releasename) && ! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)) { - // we can skip category for audiobooks, since we already know it, so as long as the release name is valid return it so that it is postprocessed by amazon. In the future, determining the type of audiobook could be added (Lecture or book), since we can skip lookups on lectures, but for now handle them all the same way + // We can skip category checks for audiobooks since the category is known. + // As long as the release name is valid, it should still be post-processed. return $releasename; } @@ -527,6 +524,25 @@ class BookService return false; } + public function extractIsbn(string $releaseName): ?string + { + if (preg_match('/\b(97[89](?:[-\s]?\d){10})\b/', $releaseName, $matches) === 1) { + $isbn = preg_replace('/[^0-9]/', '', $matches[1]); + if (is_string($isbn) && strlen($isbn) === 13) { + return $isbn; + } + } + + if (preg_match('/\b(\d(?:[-\s]?\d){8}[-\s]?[\dXx])\b/', $releaseName, $matches) === 1) { + $isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $matches[1])); + if (strlen($isbn) === 10) { + return $isbn; + } + } + + return null; + } + /** * Update book info from external sources. * @@ -534,23 +550,41 @@ class BookService * * @throws \Exception */ - public function updateBookInfo(string $bookInfo = '', mixed $amazdata = null) + public function updateBookInfo(string $bookInfo = '', ?string $isbn = null) { $ri = new ReleaseImageService; + $isbnDb = new IsbnDbService; $bookId = -2; $book = false; if ($bookInfo !== '') { - cli()->info('Fetching data from iTunes for '.$bookInfo); - $book = $this->fetchItunesBookProperties($bookInfo); + if ($isbnDb->isConfigured()) { + if ($isbn !== null && $isbn !== '') { + cli()->info('Fetching data from ISBNdb by ISBN '.$isbn); + $book = $isbnDb->findByIsbn($isbn); + } + + if ($book === false || $book === null) { + cli()->info('Fetching data from ISBNdb for '.$bookInfo); + $book = $isbnDb->searchBook($bookInfo); + } + } + + if ($book === false || $book === null) { + cli()->info('Fetching data from iTunes for '.$bookInfo); + $book = $this->fetchItunesBookProperties($bookInfo); + } } if (empty($book)) { - return false; + return -2; } $check = BookInfo::query()->where('asin', $book['asin'])->first(); + if ($check === null && ! empty($book['isbn'])) { + $check = BookInfo::query()->where('isbn', $book['isbn'])->first(); + } if ($check === null) { $bookId = BookInfo::query()->insertGetId( [ @@ -599,7 +633,7 @@ class BookService cli()->alternateOver(' Author: ').cli()->primary($book['author']); } cli()->alternateOver(' Title: ').cli()->primary(' '.$book['title']); - if ($book['genre'] !== 'null') { + if ($book['genre'] !== '') { cli()->alternateOver(' Genre: ').cli()->primary(' '.$book['genre']); } } @@ -637,8 +671,8 @@ class BookService 'title' => $iTunesBook['name'], 'author' => $iTunesBook['author'], 'asin' => $iTunesBook['id'], - 'isbn' => 'null', - 'ean' => 'null', + 'isbn' => null, + 'ean' => null, 'url' => $iTunesBook['store_url'], 'salesrank' => '', 'publisher' => '', diff --git a/app/Services/IsbnDbService.php b/app/Services/IsbnDbService.php new file mode 100644 index 000000000..09305cedb --- /dev/null +++ b/app/Services/IsbnDbService.php @@ -0,0 +1,238 @@ +apiKey = trim($apiKey ?? (string) config('nntmux_api.isbndb_api_key', '')); + + $this->client = $client ?? new Client([ + 'base_uri' => self::API_URL, + 'timeout' => 15, + 'connect_timeout' => 10, + 'http_errors' => false, + 'headers' => [ + 'Accept' => 'application/json', + 'User-Agent' => 'NNTmux/1.0', + ], + ]); + } + + public function isConfigured(): bool + { + return $this->apiKey !== ''; + } + + /** + * @return array|null + */ + public function searchBook(string $query): ?array + { + $query = trim($query); + if ($query === '' || ! $this->isConfigured()) { + return null; + } + + $cacheKey = 'isbndb_search_'.md5($query.'_'.$this->pageSize); + $cached = Cache::get($cacheKey); + if (is_array($cached)) { + return $cached; + } + + $response = $this->request('/books/'.rawurlencode($query), [ + 'shouldMatchAll' => true, + 'pageSize' => $this->pageSize, + ]); + + $books = $response['data'] ?? null; + if (! is_array($books) || $books === [] || ! isset($books[0]) || ! is_array($books[0])) { + return null; + } + + $book = $this->normalizeBookResult($books[0]); + Cache::put($cacheKey, $book, now()->addHours(24)); + + return $book; + } + + /** + * @return array|null + */ + public function findByIsbn(string $isbn): ?array + { + $isbn = $this->normalizeIsbn($isbn); + if ($isbn === '' || ! $this->isConfigured()) { + return null; + } + + $cacheKey = 'isbndb_isbn_'.$isbn; + $cached = Cache::get($cacheKey); + if (is_array($cached)) { + return $cached; + } + + $response = $this->request('/book/'.rawurlencode($isbn)); + $bookRaw = $response['book'] ?? null; + if (! is_array($bookRaw) || $bookRaw === []) { + return null; + } + + $book = $this->normalizeBookResult($bookRaw); + Cache::put($cacheKey, $book, now()->addHours(24)); + + return $book; + } + + /** + * @param array $query + * @return array|null + */ + protected function request(string $path, array $query = []): ?array + { + if (! $this->isConfigured()) { + return null; + } + + try { + $response = $this->client->get($path, [ + 'headers' => [ + 'Authorization' => $this->apiKey, + ], + 'query' => $query, + ]); + + $statusCode = $response->getStatusCode(); + $this->logRateLimitHeaders($response); + + if ($statusCode === 404) { + return null; + } + + if ($statusCode === 401 || $statusCode === 429) { + Log::warning("ISBNdb request failed with status {$statusCode}"); + + return null; + } + + if ($statusCode !== 200) { + Log::warning("ISBNdb request returned status {$statusCode}"); + + return null; + } + + $data = json_decode($response->getBody()->getContents(), true); + + return is_array($data) ? $data : null; + } catch (GuzzleException $e) { + Log::error('ISBNdb API request error: '.$e->getMessage()); + + return null; + } + } + + /** + * @param array $book + * @return array + */ + private function normalizeBookResult(array $book): array + { + $isbn13 = isset($book['isbn13']) ? $this->normalizeIsbn((string) $book['isbn13']) : ''; + if ($isbn13 === '' && isset($book['isbn'])) { + $isbn13 = $this->normalizeIsbn((string) $book['isbn']); + } + + $isbn10 = isset($book['isbn10']) ? $this->normalizeIsbn((string) $book['isbn10']) : ''; + + $authors = is_array($book['authors'] ?? null) + ? array_values(array_filter(array_map('strval', $book['authors']))) + : []; + $subjects = is_array($book['subjects'] ?? null) + ? array_values(array_filter(array_map('strval', $book['subjects']))) + : []; + + $identifier = $isbn13 !== '' ? $isbn13 : ($isbn10 !== '' ? $isbn10 : md5((string) json_encode($book))); + $coverUrl = (string) ($book['image'] ?? $book['image_original'] ?? ''); + $overview = trim((string) ($book['synopsis'] ?? $book['overview'] ?? $book['excerpt'] ?? '')); + + return [ + 'title' => (string) ($book['title'] ?? ''), + 'author' => implode(', ', $authors), + 'asin' => 'isbndb:'.$identifier, + 'isbn' => $isbn13 !== '' ? $isbn13 : null, + 'ean' => $isbn10 !== '' ? $isbn10 : null, + 'url' => '', + 'salesrank' => '', + 'publisher' => (string) ($book['publisher'] ?? ''), + 'publishdate' => $this->normalizePublishDate($book['date_published'] ?? null), + 'pages' => isset($book['pages']) && is_numeric($book['pages']) ? (string) ((int) $book['pages']) : '', + 'overview' => strip_tags($overview), + 'genre' => implode(', ', $subjects), + 'coverurl' => $coverUrl, + 'cover' => $coverUrl !== '' ? 1 : 0, + ]; + } + + private function normalizePublishDate(mixed $date): ?string + { + if (! is_string($date) || trim($date) === '') { + return null; + } + + try { + return Carbon::parse($date)->format('Y-m-d'); + } catch (\Exception) { + return null; + } + } + + private function normalizeIsbn(string $isbn): string + { + return strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn)); + } + + private function logRateLimitHeaders(ResponseInterface $response): void + { + $rateLimit = $response->getHeaderLine('ratelimit'); + $rateLimitPolicy = $response->getHeaderLine('ratelimit-policy'); + + if ($rateLimit === '' && $rateLimitPolicy === '') { + return; + } + + preg_match('/remaining[=:\s"]+(\d+)/i', $rateLimit, $remainingMatches); + $remaining = isset($remainingMatches[1]) ? (int) $remainingMatches[1] : null; + + if ($remaining !== null && $remaining <= 50) { + Log::warning('ISBNdb API rate limit getting low', [ + 'ratelimit' => $rateLimit, + 'ratelimit_policy' => $rateLimitPolicy, + ]); + + return; + } + + Log::debug('ISBNdb API rate limit status', [ + 'ratelimit' => $rateLimit, + 'ratelimit_policy' => $rateLimitPolicy, + ]); + } +} diff --git a/config/nntmux_api.php b/config/nntmux_api.php index 08ee48e08..9695a2996 100644 --- a/config/nntmux_api.php +++ b/config/nntmux_api.php @@ -3,6 +3,7 @@ return [ 'anidb_api_key' => env('ANIDB_APIKEY', ''), 'fanarttv_api_key' => env('FANARTTV_APIKEY', ''), + 'isbndb_api_key' => env('ISBNDB_API_KEY', ''), 'imdbapi_dev_enabled' => env('IMDBAPI_DEV_ENABLED', true), 'imdbapi_dev_base_url' => env('IMDBAPI_DEV_BASE_URL', 'https://api.imdbapi.dev'), 'imdbapi_dev_min_interval_seconds' => env('IMDBAPI_DEV_MIN_INTERVAL_SECONDS', 15), diff --git a/resources/views/admin/site/edit.blade.php b/resources/views/admin/site/edit.blade.php index 0668c5bcb..257893d99 100644 --- a/resources/views/admin/site/edit.blade.php +++ b/resources/views/admin/site/edit.blade.php @@ -323,7 +323,7 @@ @endforeach -

Whether to attempt to lookup book information from Amazon.

+

Whether to attempt to lookup book information from ISBNdb, with iTunes fallback.

diff --git a/tests/Integration/IsbnDbServiceIntegrationTest.php b/tests/Integration/IsbnDbServiceIntegrationTest.php new file mode 100644 index 000000000..ef857170b --- /dev/null +++ b/tests/Integration/IsbnDbServiceIntegrationTest.php @@ -0,0 +1,42 @@ +isConfigured()) { + $this->markTestSkipped('ISBNDB_API_KEY is not configured.'); + } + + $book = $service->searchBook('The Hobbit J. R. R. Tolkien'); + + $this->assertNotNull($book, 'ISBNdb search should return a known book.'); + $this->assertNotSame('', $book['title']); + $this->assertArrayHasKey('author', $book); + $this->assertArrayHasKey('isbn', $book); + } + + public function test_find_by_isbn_with_live_isbndb_api(): void + { + $service = new IsbnDbService; + + if (! $service->isConfigured()) { + $this->markTestSkipped('ISBNDB_API_KEY is not configured.'); + } + + $book = $service->findByIsbn('9780140328721'); + + $this->assertNotNull($book, 'ISBNdb ISBN lookup should return a known book.'); + $this->assertNotSame('', $book['title']); + $this->assertSame('9780140328721', $book['isbn']); + } +} diff --git a/tests/Unit/Services/IsbnDbServiceTest.php b/tests/Unit/Services/IsbnDbServiceTest.php new file mode 100644 index 000000000..96e5c8ffc --- /dev/null +++ b/tests/Unit/Services/IsbnDbServiceTest.php @@ -0,0 +1,108 @@ +assertFalse($service->isConfigured()); + } + + public function test_search_book_maps_response_to_internal_shape(): void + { + $mock = new MockHandler([ + new Response(200, [ + 'ratelimit' => 'limit=500, remaining=499', + 'ratelimit-policy' => '500;w=86400', + ], json_encode([ + 'total' => 1, + 'data' => [[ + 'title' => 'Domain-Driven Design', + 'isbn13' => '9780321125217', + 'isbn10' => '0321125215', + 'authors' => ['Eric Evans'], + 'publisher' => 'Addison-Wesley Professional', + 'date_published' => '2003-08-30', + 'pages' => 560, + 'synopsis' => '

A practical guide.

', + 'subjects' => ['Software', 'Architecture'], + 'image' => 'https://example.com/cover.jpg', + ]], + ])), + ]); + + $service = new IsbnDbService( + new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']), + 'test-key' + ); + + $book = $service->searchBook('domain driven design'); + + $this->assertNotNull($book); + $this->assertSame('Domain-Driven Design', $book['title']); + $this->assertSame('Eric Evans', $book['author']); + $this->assertSame('9780321125217', $book['isbn']); + $this->assertSame('0321125215', $book['ean']); + $this->assertSame('Addison-Wesley Professional', $book['publisher']); + $this->assertSame('2003-08-30', $book['publishdate']); + $this->assertSame('560', $book['pages']); + $this->assertSame('A practical guide.', $book['overview']); + $this->assertSame('Software, Architecture', $book['genre']); + $this->assertSame('https://example.com/cover.jpg', $book['coverurl']); + $this->assertSame('isbndb:9780321125217', $book['asin']); + } + + public function test_find_by_isbn_reads_book_payload(): void + { + $mock = new MockHandler([ + new Response(200, [], json_encode([ + 'book' => [ + 'title' => 'Clean Code', + 'isbn13' => '9780132350884', + 'authors' => ['Robert C. Martin'], + 'date_published' => '2008', + 'subjects' => ['Programming'], + ], + ])), + ]); + + $service = new IsbnDbService( + new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']), + 'test-key' + ); + + $book = $service->findByIsbn('978-0132350884'); + + $this->assertNotNull($book); + $this->assertSame('Clean Code', $book['title']); + $this->assertSame('9780132350884', $book['isbn']); + $this->assertSame('Robert C. Martin', $book['author']); + } + + public function test_book_service_extracts_isbn_13_and_isbn_10(): void + { + /** @var BookService $bookService */ + $bookService = (new \ReflectionClass(BookService::class))->newInstanceWithoutConstructor(); + + $isbn13 = $bookService->extractIsbn('Some.Book.Title.978-0132350884.RETAIL.ePub'); + $isbn10 = $bookService->extractIsbn('Another Book 0132350882 mobi'); + $none = $bookService->extractIsbn('No ISBN in this release name'); + + $this->assertSame('9780132350884', $isbn13); + $this->assertSame('0132350882', $isbn10); + $this->assertNull($none); + } +}