Improve book matching

This commit is contained in:
DariusIII
2026-08-13 15:53:31 +02:00
parent 658937bee1
commit 992a0007f6
21 changed files with 2070 additions and 341 deletions
+21
View File
@@ -0,0 +1,21 @@
<?php
declare(strict_types=1);
namespace App\Exceptions;
use RuntimeException;
use Throwable;
class BookProviderException extends RuntimeException
{
public function __construct(
public readonly string $provider,
string $message,
public readonly ?int $statusCode = null,
public readonly ?int $retryAfterSeconds = null,
?Throwable $previous = null,
) {
parent::__construct($message, $statusCode ?? 0, $previous);
}
}
+525 -217
View File
@@ -6,6 +6,7 @@ namespace App\Services;
use App\Enums\ImageAssetProfile; use App\Enums\ImageAssetProfile;
use App\Enums\SecondarySearchIndex; use App\Enums\SecondarySearchIndex;
use App\Exceptions\BookProviderException;
use App\Facades\Search; use App\Facades\Search;
use App\Models\BookInfo; use App\Models\BookInfo;
use App\Models\Category; use App\Models\Category;
@@ -13,6 +14,7 @@ use App\Models\Release;
use App\Models\Settings; use App\Models\Settings;
use App\Services\NameFixing\Extractors\ObfuscatedSubjectExtractor; use App\Services\NameFixing\Extractors\ObfuscatedSubjectExtractor;
use App\Services\Releases\ReleaseBrowseService; use App\Services\Releases\ReleaseBrowseService;
use App\Support\BookIsbn;
use App\Support\BookMatchScorer; use App\Support\BookMatchScorer;
use App\Support\Data\BookParseResult; use App\Support\Data\BookParseResult;
use App\Support\MetadataSearchLookup; use App\Support\MetadataSearchLookup;
@@ -26,6 +28,8 @@ use Illuminate\Support\Facades\Log;
*/ */
class BookService class BookService
{ {
private const FAILURE_CACHE_VERSION = 2;
public bool $echooutput; public bool $echooutput;
public int $bookqty; public int $bookqty;
@@ -42,11 +46,29 @@ class BookService
private ObfuscatedSubjectExtractor $obfuscatedSubjectExtractor; private ObfuscatedSubjectExtractor $obfuscatedSubjectExtractor;
private IsbnDbService $isbnDb;
private GoogleBooksService $googleBooks;
private OpenLibraryService $openLibrary;
private ItunesService $itunes;
private BookMatchScorer $scorer;
private ReleaseImageService $releaseImageService;
/** /**
* @throws \Exception * @throws \Exception
*/ */
public function __construct() public function __construct(
{ ?IsbnDbService $isbnDb = null,
?GoogleBooksService $googleBooks = null,
?OpenLibraryService $openLibrary = null,
?ItunesService $itunes = null,
?BookMatchScorer $scorer = null,
?ReleaseImageService $releaseImageService = null,
) {
$this->echooutput = config('nntmux.echocli'); $this->echooutput = config('nntmux.echocli');
$this->bookqty = Settings::settingValue('maxbooksprocessed') !== '' ? (int) Settings::settingValue('maxbooksprocessed') : 300; $this->bookqty = Settings::settingValue('maxbooksprocessed') !== '' ? (int) Settings::settingValue('maxbooksprocessed') : 300;
@@ -58,6 +80,12 @@ class BookService
$this->parsedIsbn = null; $this->parsedIsbn = null;
$this->parsedBookResult = null; $this->parsedBookResult = null;
$this->obfuscatedSubjectExtractor = new ObfuscatedSubjectExtractor; $this->obfuscatedSubjectExtractor = new ObfuscatedSubjectExtractor;
$this->isbnDb = $isbnDb ?? new IsbnDbService;
$this->googleBooks = $googleBooks ?? new GoogleBooksService;
$this->openLibrary = $openLibrary ?? new OpenLibraryService;
$this->itunes = $itunes ?? new ItunesService;
$this->scorer = $scorer ?? new BookMatchScorer;
$this->releaseImageService = $releaseImageService ?? new ReleaseImageService;
} }
/** /**
@@ -118,6 +146,21 @@ class BookService
return $this->pickBestExistingBook($results->all(), $parsed); return $this->pickBestExistingBook($results->all(), $parsed);
} }
public function getBookInfoByIsbn(?string $isbn): ?BookInfo
{
$identifiers = BookIsbn::equivalents($isbn);
if ($identifiers === []) {
return null;
}
return BookInfo::query()
->where(static function ($query) use ($identifiers): void {
$query->whereIn('isbn', $identifiers)
->orWhereIn('ean', $identifiers);
})
->first();
}
/** /**
* Get book range with pagination. * Get book range with pagination.
* *
@@ -386,6 +429,10 @@ class BookService
$query->where('groups_id', $groupID); $query->where('groups_id', $groupID);
} }
if ($this->renamed !== '') {
$query->where('isrenamed', 1);
}
$this->processBookReleasesHelper( $this->processBookReleasesHelper(
$query->get(['searchname', 'id', 'categories_id']), $query->get(['searchname', 'id', 'categories_id']),
sprintf( sprintf(
@@ -406,8 +453,10 @@ class BookService
->orWhere('categories_id', Category::MUSIC_AUDIOBOOK); ->orWhere('categories_id', Category::MUSIC_AUDIOBOOK);
}) })
->where(function ($builder): void { ->where(function ($builder): void {
$builder->where('isrenamed', 0) if ($this->renamed === '') {
->orWhere('searchname', 'like', 'N:/NZB%') $builder->where('isrenamed', 0);
}
$builder->orWhere('searchname', 'like', 'N:/NZB%')
->orWhere('searchname', 'like', 'N_NZB_%') ->orWhere('searchname', 'like', 'N_NZB_%')
->orWhere('name', 'like', 'N:/NZB%') ->orWhere('name', 'like', 'N:/NZB%')
->orWhere('name', 'like', 'N_NZB_%'); ->orWhere('name', 'like', 'N_NZB_%');
@@ -453,8 +502,8 @@ class BookService
cli()->header('Processing '.$res->count().' book release(s) for categories id '.$categoryID); cli()->header('Processing '.$res->count().' book release(s) for categories id '.$categoryID);
} }
$bookId = -2;
foreach ($res as $arr) { foreach ($res as $arr) {
$bookId = -2;
$startTime = now()->timestamp; $startTime = now()->timestamp;
$usedExternalApi = false; $usedExternalApi = false;
// audiobooks are also books and should be handled in an identical manor, even though it falls under a music category // audiobooks are also books and should be handled in an identical manor, even though it falls under a music category
@@ -471,9 +520,10 @@ class BookService
cli()->info('Looking up: '.$bookInfo); cli()->info('Looking up: '.$bookInfo);
} }
// Do a local lookup first // Prefer an exact local identifier before any fuzzy title match.
$bookCheck = $this->getBookInfoByName($bookInfo, $this->parsedBookResult); $bookCheck = $this->getBookInfoByIsbn($this->parsedIsbn)
$failCacheKey = 'book_lookup_fail_'.md5(strtolower($bookInfo)); ?? $this->getBookInfoByName($bookInfo, $this->parsedBookResult);
$failCacheKey = $this->failureCacheKey($this->parsedBookResult, $bookInfo);
if ($bookCheck === null && Cache::has($failCacheKey)) { if ($bookCheck === null && Cache::has($failCacheKey)) {
// Lookup recently failed, no point trying again // Lookup recently failed, no point trying again
@@ -485,8 +535,8 @@ class BookService
$bookId = $this->updateBookInfo($bookInfo, $this->parsedIsbn); $bookId = $this->updateBookInfo($bookInfo, $this->parsedIsbn);
$usedExternalApi = true; $usedExternalApi = true;
if ($bookId === -2) { if ($bookId === -2) {
Cache::put($failCacheKey, true, now()->addDays(7)); Cache::put($failCacheKey, true, now()->addDay());
} else { } elseif ($bookId !== null) {
Cache::forget($failCacheKey); Cache::forget($failCacheKey);
} }
} else { } else {
@@ -554,9 +604,12 @@ class BookService
return false; return false;
} }
if (! empty($releasename) && ! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)) { if (! empty($releasename)
&& (! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)
|| $parsed->hasAuthor()
|| $parsed->isbn !== null)) {
$wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []); $wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []);
if ($wordCount >= 2) { if ($wordCount >= 2 || $parsed->hasAuthor() || $parsed->isbn !== null) {
return $parsed->searchQuery(); return $parsed->searchQuery();
} }
} }
@@ -568,9 +621,12 @@ class BookService
return false; return false;
} }
if (! empty($releasename) && ! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)) { if (! empty($releasename)
&& (! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)
|| $parsed->hasAuthor()
|| $parsed->isbn !== null)) {
$wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []); $wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []);
if ($wordCount >= 2) { if ($wordCount >= 2 || $parsed->hasAuthor() || $parsed->isbn !== null) {
return $parsed->searchQuery(); return $parsed->searchQuery();
} }
} }
@@ -589,6 +645,10 @@ class BookService
$releaseName = $quotedMatch[1]; $releaseName = $quotedMatch[1];
} }
$isbn = $this->extractIsbn($releaseName); $isbn = $this->extractIsbn($releaseName);
$year = null;
if (preg_match('/\b(19|20)\d{2}\b/', $releaseName, $yearMatch) === 1) {
$year = (int) $yearMatch[0];
}
$a = preg_replace('/\d{1,2} \d{1,2} \d{2,4}|(19|20)\d\d|anybody got .+?[a-z]\? |[ ._-](Novel|TIA)([ ._-]|$)|([ \.])HQ([-\. ])|[\(\)\.\-_ ](AVI|AZW3?|DOC|EPUB|LIT|MOBI|NFO|RETAIL|(si)?PDF|RTF|TXT)[\)\]\.\-_ ](?![a-z0-9])|compleet|DAGSTiDNiNGEN|DiRFiX|\+ extra|r?e ?Books?([\.\-_ ]English|ers)?|azw3?|ePu([bp])s?|html|mobi|^NEW[\.\-_ ]|PDF([\.\-_ ]English)?|Please post more|Post description|Proper|Repack(fix)?|[\.\-_ ](Chinese|English|French|German|Italian|Retail|Scan|Swedish|Multilingual)|^R4 |Repost|Skytwohigh|TIA!+|TruePDF|V413HAV|(would someone )?please (re)?post.+? "|with the authors name right/i', '', $releaseName); $a = preg_replace('/\d{1,2} \d{1,2} \d{2,4}|(19|20)\d\d|anybody got .+?[a-z]\? |[ ._-](Novel|TIA)([ ._-]|$)|([ \.])HQ([-\. ])|[\(\)\.\-_ ](AVI|AZW3?|DOC|EPUB|LIT|MOBI|NFO|RETAIL|(si)?PDF|RTF|TXT)[\)\]\.\-_ ](?![a-z0-9])|compleet|DAGSTiDNiNGEN|DiRFiX|\+ extra|r?e ?Books?([\.\-_ ]English|ers)?|azw3?|ePu([bp])s?|html|mobi|^NEW[\.\-_ ]|PDF([\.\-_ ]English)?|Please post more|Post description|Proper|Repack(fix)?|[\.\-_ ](Chinese|English|French|German|Italian|Retail|Scan|Swedish|Multilingual)|^R4 |Repost|Skytwohigh|TIA!+|TruePDF|V413HAV|(would someone )?please (re)?post.+? "|with the authors name right/i', '', $releaseName);
$b = preg_replace('/^(As Req |conversion |eq |Das neue Abenteuer \d+|Fixed version( ignore previous post)?|Full |Per Req As Found|(\s+)?R4 |REQ |revised |version |\d+(\s+)?$)|(COMPLETE|INTERNAL|RELOADED| (AZW3|eB|docx|ENG?|exe|FR|Fix|gnv64|MU|NIV|R\d\s+\d{1,2} \d{1,2}|R\d|Req|TTL|UC|v(\s+)?\d))(\s+)?$/i', '', (string) $a); $b = preg_replace('/^(As Req |conversion |eq |Das neue Abenteuer \d+|Fixed version( ignore previous post)?|Full |Per Req As Found|(\s+)?R4 |REQ |revised |version |\d+(\s+)?$)|(COMPLETE|INTERNAL|RELOADED| (AZW3|eB|docx|ENG?|exe|FR|Fix|gnv64|MU|NIV|R\d\s+\d{1,2} \d{1,2}|R\d|Req|TTL|UC|v(\s+)?\d))(\s+)?$/i', '', (string) $a);
$c = preg_replace('/ - \[.+\]|\[.+\]/', '', (string) $b); $c = preg_replace('/ - \[.+\]|\[.+\]/', '', (string) $b);
@@ -599,7 +659,6 @@ class BookService
$normalized = (string) preg_replace('/\b(S\d{1,3}E\d{1,3}|Season\s*\d+|Temporada\s*\d+|Saison\s*\d+|Staffel\s*\d+|Episode\s*\d+|HDTV|WEB[\s-]?DL|WEBRip|BluRay|BDRip|DVDRip|BRRip|XviD|[xh][\s.]?26[45]|HEVC|10bit|1080[pi]|720p|2160p|4K|UHD|HDR|REMUX|AAC5?\s*\d|DDP?5\s*\d|ATMOS|PAR2?|vol\d+\+\d+|NTb|FLUX|PSA|RARBG|YTS|YIFY|AMZN|DSNP|NF|ATVP|HMAX)\b/i', '', $normalized); $normalized = (string) preg_replace('/\b(S\d{1,3}E\d{1,3}|Season\s*\d+|Temporada\s*\d+|Saison\s*\d+|Staffel\s*\d+|Episode\s*\d+|HDTV|WEB[\s-]?DL|WEBRip|BluRay|BDRip|DVDRip|BRRip|XviD|[xh][\s.]?26[45]|HEVC|10bit|1080[pi]|720p|2160p|4K|UHD|HDR|REMUX|AAC5?\s*\d|DDP?5\s*\d|ATMOS|PAR2?|vol\d+\+\d+|NTb|FLUX|PSA|RARBG|YTS|YIFY|AMZN|DSNP|NF|ATVP|HMAX)\b/i', '', $normalized);
$normalized = trim((string) preg_replace('/\s+/', ' ', $normalized)); $normalized = trim((string) preg_replace('/\s+/', ' ', $normalized));
$year = null;
$specialMagazineTitle = null; $specialMagazineTitle = null;
if (preg_match('/^MCN[._ -](?<month>[A-Za-z]+)[._ -](?<day>\d{1,2})[._ -](?<year>(?:19|20)\d{2})/i', $rawReleaseName, $mcnMatch) === 1) { if (preg_match('/^MCN[._ -](?<month>[A-Za-z]+)[._ -](?<day>\d{1,2})[._ -](?<year>(?:19|20)\d{2})/i', $rawReleaseName, $mcnMatch) === 1) {
$month = ucfirst(strtolower($mcnMatch['month'])); $month = ucfirst(strtolower($mcnMatch['month']));
@@ -627,22 +686,15 @@ class BookService
$author = $candidateAuthor; $author = $candidateAuthor;
$title = $candidateTitle; $title = $candidateTitle;
} }
} elseif (preg_match('/^(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})\s+by\s+(?<title>.+)$/i', $normalized, $matches) === 1) {
$author = trim($matches['author']);
$title = trim($matches['title']);
} elseif (preg_match('/^(?<title>.+?)\s+by\s+(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})$/i', $normalized, $matches) === 1) { } elseif (preg_match('/^(?<title>.+?)\s+by\s+(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})$/i', $normalized, $matches) === 1) {
$author = trim($matches['author']); $author = trim($matches['author']);
$title = trim($matches['title']); $title = trim($matches['title']);
} }
$title = trim((string) preg_replace('/\s+/', ' ', (string) preg_replace('/\((Book|Series)\s*\d+\)/i', '', $title))); $title = trim((string) preg_replace('/\s+/', ' ', $title));
if ($author === null && preg_match('/^(?<title>.+?)[\s._-]+(?<year>(19|20)\d{2})[\s._-]+(?<format>EPUB|MOBI|AZW3?|PDF|FB2|DJVU|LIT)$/i', $normalized, $matches) === 1) { if ($author === null && preg_match('/^(?<title>.+?)[\s._-]+(?<year>(19|20)\d{2})[\s._-]+(?<format>EPUB|MOBI|AZW3?|PDF|FB2|DJVU|LIT)$/i', $normalized, $matches) === 1) {
$title = trim(str_replace(['.', '_'], ' ', $matches['title'])); $title = trim(str_replace(['.', '_'], ' ', $matches['title']));
} }
if ($author === null && preg_match('/^(?<title>.+?)\s+Vol\.?\s*\d{1,3}$/i', $title, $matches) === 1) {
$title = trim($matches['title']);
}
$isJunk = false; $isJunk = false;
$junkPattern = '/^([a-z0-9] )+$|ArtofUsenet|ekiosk|(ebook|mobi).+collection|erotica|Full Video|ImwithJamie|linkoff org|Mega.+pack|^[a-z0-9]+ (?!((January|February|March|April|May|June|July|August|September|O([ck])tober|November|De([cz])ember)))[a-z]+( (ebooks?|The))?$|NY Times|(Book|Massive) Dump|Sexual/i'; $junkPattern = '/^([a-z0-9] )+$|ArtofUsenet|ekiosk|(ebook|mobi).+collection|erotica|Full Video|ImwithJamie|linkoff org|Mega.+pack|^[a-z0-9]+ (?!((January|February|March|April|May|June|July|August|September|O([ck])tober|November|De([cz])ember)))[a-z]+( (ebooks?|The))?$|NY Times|(Book|Massive) Dump|Sexual/i';
if (preg_match($junkPattern, $normalized) === 1) { if (preg_match($junkPattern, $normalized) === 1) {
@@ -694,16 +746,15 @@ class BookService
{ {
if (preg_match('/\b(97[89](?:[-\s]?\d){10})\b/', $releaseName, $matches) === 1) { if (preg_match('/\b(97[89](?:[-\s]?\d){10})\b/', $releaseName, $matches) === 1) {
$isbn = preg_replace('/[^0-9]/', '', $matches[1]); $isbn = preg_replace('/[^0-9]/', '', $matches[1]);
if (is_string($isbn) && strlen($isbn) === 13) { if (is_string($isbn)) {
return $isbn; return BookIsbn::normalize($isbn);
} }
} }
if (preg_match('/\b(\d(?:[-\s]?\d){8}[-\s]?[\dXx])\b/', $releaseName, $matches) === 1) { if (preg_match('/\b(\d(?:[-\s]?\d){8}[-\s]?[\dXx])\b/', $releaseName, $matches) === 1) {
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $matches[1])); $isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $matches[1]));
if (strlen($isbn) === 10) {
return $isbn; return BookIsbn::normalize($isbn);
}
} }
return null; return null;
@@ -745,185 +796,160 @@ class BookService
/** /**
* Update book info from external sources. * Update book info from external sources.
* *
* @return false|int|string
*
* @throws \Exception * @throws \Exception
*/ */
public function updateBookInfo(string $bookInfo = '', ?string $isbn = null) public function updateBookInfo(string $bookInfo = '', ?string $isbn = null): ?int
{ {
$ri = new ReleaseImageService; $bookInfo = trim($bookInfo);
$isbnDb = new IsbnDbService; if ($bookInfo === '') {
$googleBooks = new GoogleBooksService;
$openLibrary = new OpenLibraryService;
$itunes = new ItunesService;
$scorer = new BookMatchScorer;
$bookId = -2;
$book = null;
if ($bookInfo !== '') {
$wordCount = count(array_filter(preg_split('/\s+/', trim($bookInfo)) ?: []));
if ($wordCount < 3) {
return -2;
}
$parsed = $this->parsedBookResult ?? $this->parseReleaseName($bookInfo);
$candidates = [];
$resolvedSource = null;
if ($isbnDb->isConfigured()) {
if ($isbn !== null && $isbn !== '') {
cli()->info('Fetching data from ISBNdb by ISBN '.$isbn);
$candidate = $isbnDb->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'isbndb_isbn';
}
}
if ($candidates === []) {
cli()->info('Fetching data from ISBNdb for '.$bookInfo);
$candidates = array_merge($candidates, $isbnDb->searchBooks($bookInfo));
if ($candidates !== []) {
$resolvedSource = 'isbndb_search';
}
}
}
if ($candidates === []) {
if ($isbn !== null && $isbn !== '') {
cli()->info('Fetching data from Google Books by ISBN '.$isbn);
$candidate = $googleBooks->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'google_books_isbn';
}
}
}
if ($candidates === []) {
cli()->info('Fetching data from Google Books for '.$bookInfo);
$candidates = array_merge(
$candidates,
$googleBooks->searchBooks(
$bookInfo,
$parsed->title,
$parsed->author,
$isbn
)
);
if ($candidates !== []) {
$resolvedSource = 'google_books_search';
}
}
if ($candidates === []) {
cli()->info('Fetching data from iTunes for '.$bookInfo);
$candidates = array_merge($candidates, $this->fetchItunesBookProperties($bookInfo, $itunes));
if ($candidates !== []) {
$resolvedSource = 'itunes_search';
}
}
if ($candidates === []) {
cli()->info('Fetching data from Open Library for '.$bookInfo);
if ($isbn !== null && $isbn !== '') {
$candidate = $openLibrary->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'open_library_isbn';
}
}
if ($candidates === []) {
$candidates = array_merge($candidates, $openLibrary->searchBooks($bookInfo));
if ($candidates !== []) {
$resolvedSource = 'open_library_search';
}
}
}
$book = $this->pickBestCandidate($candidates, $parsed, $scorer);
if (is_array($book)) {
Log::debug('Book metadata source resolved', [
'book_info' => $bookInfo,
'source' => $resolvedSource,
'isbn' => $isbn,
]);
}
}
if (empty($book)) {
return -2; return -2;
} }
$check = BookInfo::query()->where('asin', $book['asin'])->first(); $parsed = $this->parsedBookResult ?? $this->parseReleaseName($bookInfo);
if ($check === null && ! empty($book['isbn'])) { $isbn = BookIsbn::normalize($isbn ?? $parsed->isbn);
$check = BookInfo::query()->where('isbn', $book['isbn'])->first(); if ($isbn === null && ! $parsed->hasAuthor() && $this->scorer->meaningfulTitleTokenCount($parsed->title) < 2) {
} return -2;
if ($check === null) {
$bookId = BookInfo::query()->insertGetId(
[
'title' => $book['title'],
'author' => $book['author'],
'asin' => $book['asin'],
'isbn' => $book['isbn'],
'ean' => $book['ean'],
'url' => $book['url'],
'salesrank' => $book['salesrank'],
'publisher' => $book['publisher'],
'publishdate' => $book['publishdate'],
'pages' => $book['pages'],
'overview' => $book['overview'],
'genre' => $book['genre'],
'cover' => $book['cover'],
'created_at' => now(),
'updated_at' => now(),
]
);
} else {
$bookId = $check['id'];
BookInfo::query()->where('id', $bookId)->update(
[
'title' => $book['title'],
'author' => $book['author'],
'asin' => $book['asin'],
'isbn' => $book['isbn'],
'ean' => $book['ean'],
'url' => $book['url'],
'salesrank' => $book['salesrank'],
'publisher' => $book['publisher'],
'publishdate' => $book['publishdate'],
'pages' => $book['pages'],
'overview' => $book['overview'],
'genre' => $book['genre'],
'cover' => $book['cover'],
]
);
} }
if ($bookId && $bookId !== -2) { $candidates = [];
if ($this->echooutput) { $book = null;
cli()->header('Added/updated book: '); $hadProviderFailure = false;
if ($book['author'] !== '') {
cli()->alternateOver(' Author: ').cli()->primary($book['author']); if ($this->isbnDb->isConfigured()) {
try {
if ($isbn !== null) {
cli()->info('Fetching data from ISBNdb by ISBN '.$isbn);
$candidate = $this->isbnDb->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'isbndb_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} }
cli()->alternateOver(' Title: ').cli()->primary(' '.$book['title']);
if ($book['genre'] !== '') { if ($book === null) {
cli()->alternateOver(' Genre: ').cli()->primary(' '.$book['genre']); cli()->info('Fetching data from ISBNdb for '.$parsed->title);
$this->appendCandidates(
$candidates,
$this->isbnDb->searchBooks($parsed->title, $parsed->author, true),
'isbndb_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
} }
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
cli()->info('Retrying ISBNdb with title-only hypothesis '.$titleOnlyHypothesis->title);
$this->appendCandidates(
$candidates,
$this->isbnDb->searchBooks($titleOnlyHypothesis->title, null, true),
'isbndb_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
} }
$book['cover'] = (int) $ri->saveRemoteImage(
(string) $bookId,
$book['coverurl'],
$this->imgSavePath,
ImageAssetProfile::MetadataCover,
)->success;
} elseif ($this->echooutput) {
cli()->header('Nothing to update: ').
cli()->header($book['author'].
' - '.
$book['title']);
} }
if ($book === null) {
try {
if ($isbn !== null) {
cli()->info('Fetching data from Google Books by ISBN '.$isbn);
$candidate = $this->googleBooks->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'google_books_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
}
if ($book === null) {
cli()->info('Fetching data from Google Books for '.$parsed->title);
$this->appendCandidates(
$candidates,
$this->googleBooks->searchBooks($parsed->searchQuery(), $parsed->title, $parsed->author),
'google_books_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
$this->appendCandidates(
$candidates,
$this->googleBooks->searchBooks(
$titleOnlyHypothesis->title,
$titleOnlyHypothesis->title,
null,
),
'google_books_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
}
}
if ($book === null) {
try {
cli()->info('Fetching data from Open Library for '.$parsed->title);
if ($isbn !== null) {
$candidate = $this->openLibrary->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'open_library_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
}
if ($book === null) {
$this->appendCandidates(
$candidates,
$this->openLibrary->searchBooks($parsed->searchQuery()),
'open_library_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
$this->appendCandidates(
$candidates,
$this->openLibrary->searchBooks($titleOnlyHypothesis->title),
'open_library_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
}
}
if ($book === null) {
cli()->info('Fetching data from iTunes for '.$parsed->title);
$itunesCandidates = $this->fetchItunesBookProperties($parsed->searchQuery(), $this->itunes);
$this->appendCandidates(
$candidates,
$itunesCandidates,
'itunes_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
if ($itunesCandidates === [] && $this->itunes->lastRequestFailed()) {
$hadProviderFailure = true;
}
}
if ($book === null) {
return $hadProviderFailure ? null : -2;
}
Log::debug('Book metadata source resolved', [
'book_info' => $bookInfo,
'source' => $book['_source'] ?? null,
'isbn' => $isbn,
]);
$bookId = $this->persistBookCandidate($book);
$this->outputBookMatch($book);
return $bookId; return $bookId;
} }
@@ -980,18 +1006,28 @@ class BookService
return $books[0]; return $books[0];
} }
$scorer = new BookMatchScorer; $candidates = [];
$best = null;
$bestScore = 0.0;
foreach ($books as $book) { foreach ($books as $book) {
$score = $scorer->scoreBookInfo($book, $parsed); $candidates[] = [
if ($score > $bestScore) { '_book_id' => $book->id,
$best = $book; 'title' => $book->title,
$bestScore = $score; 'author' => $book->author,
} 'isbn' => $book->isbn,
'ean' => $book->ean,
'publishdate' => $book->publishdate,
'publisher' => $book->publisher,
'cover' => $book->cover ? 1 : 0,
];
} }
return $bestScore >= $this->minimumMatchScore($parsed) ? $best : null; $best = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
if ($best === null) {
return null;
}
$bookId = (int) ($best['_book_id'] ?? 0);
return $bookId > 0 ? BookInfo::query()->find($bookId) : null;
} }
/** /**
@@ -1004,30 +1040,302 @@ class BookService
return null; return null;
} }
$best = null; foreach ($this->matchingHypotheses($parsed) as $hypothesis) {
$bestScore = 0.0; $best = $this->pickBestCandidateForParse($candidates, $hypothesis, $scorer);
foreach ($candidates as $candidate) { if ($best !== null) {
$score = $scorer->score($candidate, $parsed); return $best;
if ($score > $bestScore) {
$bestScore = $score;
$best = $candidate;
} }
} }
if ($best === null || $bestScore < $this->minimumMatchScore($parsed)) { return null;
}
/**
* @param array<int, array<string, mixed>> $candidates
* @return array<string, mixed>|null
*/
private function pickBestCandidateForParse(
array $candidates,
BookParseResult $parsed,
BookMatchScorer $scorer,
): ?array {
/** @var array<string, array{candidate: array<string, mixed>, score: float, year_tie: int, completeness: int}> $byWork */
$byWork = [];
foreach ($candidates as $candidate) {
$score = $scorer->score($candidate, $parsed);
$key = $scorer->workKey($candidate);
if ($key === '|') {
$key = (string) ($candidate['asin'] ?? md5((string) json_encode($candidate)));
}
$completeness = $this->candidateCompleteness($candidate);
$yearTie = $this->candidateYearTie($candidate, $parsed);
$existing = $byWork[$key] ?? null;
if ($existing === null
|| $score > $existing['score']
|| ($score === $existing['score'] && $yearTie > $existing['year_tie'])
|| ($score === $existing['score']
&& $yearTie === $existing['year_tie']
&& $completeness > $existing['completeness'])) {
$byWork[$key] = [
'candidate' => $candidate,
'score' => $score,
'year_tie' => $yearTie,
'completeness' => $completeness,
];
}
}
$ranked = array_values($byWork);
usort($ranked, static function (array $left, array $right): int {
$scoreComparison = $right['score'] <=> $left['score'];
return $scoreComparison !== 0
? $scoreComparison
: (($right['year_tie'] <=> $left['year_tie'])
?: ($right['completeness'] <=> $left['completeness']));
});
$best = $ranked[0] ?? null;
if ($best === null || ! $this->candidatePassesIdentityRules(
$best['candidate'],
$best['score'],
$ranked[1]['score'] ?? null,
$parsed,
$scorer,
)) {
return null; return null;
} }
return $best; return $best['candidate'];
} }
private function minimumMatchScore(?BookParseResult $parsed): float /**
* @return list<BookParseResult>
*/
private function matchingHypotheses(BookParseResult $parsed): array
{ {
if ($parsed === null) { $hypotheses = [$parsed];
return 0.55; $titleOnly = $this->titleOnlyHypothesis($parsed);
if ($titleOnly !== null) {
$hypotheses[] = $titleOnly;
} }
// No-author parses are more ambiguous and need a stricter cutoff. return $hypotheses;
return $parsed->hasAuthor() ? 0.55 : 0.68; }
private function titleOnlyHypothesis(BookParseResult $parsed): ?BookParseResult
{
if (! $parsed->hasAuthor() || ! str_contains($parsed->rawName, ' - ')) {
return null;
}
return new BookParseResult(
rawName: $parsed->rawName,
title: trim($parsed->author.' - '.$parsed->title),
author: null,
isbn: $parsed->isbn,
year: $parsed->year,
format: $parsed->format,
isJunk: $parsed->isJunk,
isMagazine: $parsed->isMagazine,
);
}
/**
* @param array<string, mixed> $candidate
*/
private function candidatePassesIdentityRules(
array $candidate,
float $score,
?float $runnerUpScore,
BookParseResult $parsed,
BookMatchScorer $scorer,
): bool {
if ($score === 1.0 && $parsed->isbn !== null) {
return true;
}
$titleScore = $scorer->titleScore($parsed->title, (string) ($candidate['title'] ?? ''));
if ($parsed->hasAuthor()) {
$authorScore = $scorer->authorScore((string) $parsed->author, (string) ($candidate['author'] ?? ''));
if ($titleScore < 0.78 || $authorScore < 0.75 || $score < 0.82) {
return false;
}
return $runnerUpScore === null || ($score - $runnerUpScore) >= 0.08;
}
if ($scorer->meaningfulTitleTokenCount($parsed->title) < 3 || $titleScore < 0.92) {
return false;
}
return $runnerUpScore === null || ($score - $runnerUpScore) >= 0.12;
}
/**
* @param array<int, array<string, mixed>> $target
* @param array<int, array<string, mixed>> $incoming
*/
private function appendCandidates(array &$target, array $incoming, string $source): void
{
foreach ($incoming as $candidate) {
$candidate['_source'] = $source;
$target[] = $candidate;
}
}
/**
* @param array<string, mixed> $candidate
*/
private function candidateCompleteness(array $candidate): int
{
$score = 0;
foreach (['isbn', 'ean', 'publisher', 'publishdate', 'pages', 'overview', 'genre', 'coverurl'] as $field) {
if (isset($candidate[$field]) && $candidate[$field] !== '') {
$score++;
}
}
return $score;
}
/**
* @param array<string, mixed> $candidate
*/
private function candidateYearTie(array $candidate, BookParseResult $parsed): int
{
if ($parsed->year === null || empty($candidate['publishdate'])) {
return 0;
}
if (preg_match('/\b(19|20)\d{2}\b/', (string) $candidate['publishdate'], $match) !== 1) {
return 0;
}
return max(0, 100 - abs($parsed->year - (int) $match[0]));
}
/**
* @param array<string, mixed> $book
*/
private function persistBookCandidate(array $book): int
{
$check = null;
$asin = trim((string) ($book['asin'] ?? ''));
if ($asin !== '') {
$check = BookInfo::query()->where('asin', $asin)->first();
}
foreach (['isbn', 'ean'] as $identifierField) {
if ($check !== null) {
break;
}
$identifiers = BookIsbn::equivalents(isset($book[$identifierField]) ? (string) $book[$identifierField] : null);
if ($identifiers !== []) {
$check = BookInfo::query()
->where(static function ($query) use ($identifiers): void {
$query->whereIn('isbn', $identifiers)->orWhereIn('ean', $identifiers);
})
->first();
}
}
$attributes = [
'title' => (string) ($book['title'] ?? ''),
'author' => (string) ($book['author'] ?? ''),
'asin' => $asin !== '' ? $asin : null,
'isbn' => BookIsbn::normalize(isset($book['isbn']) ? (string) $book['isbn'] : null),
'ean' => BookIsbn::normalize(isset($book['ean']) ? (string) $book['ean'] : null),
'url' => $book['url'] ?? null,
'salesrank' => $book['salesrank'] ?? null,
'publisher' => $book['publisher'] ?? null,
'publishdate' => $book['publishdate'] ?? null,
'pages' => $book['pages'] ?? null,
'overview' => $book['overview'] ?? null,
'genre' => (string) ($book['genre'] ?? ''),
];
if ($check === null) {
$bookId = BookInfo::query()->insertGetId(array_merge($attributes, [
'cover' => 0,
'created_at' => now(),
'updated_at' => now(),
]));
} else {
$bookId = (int) $check->id;
$updates = [];
foreach ($attributes as $field => $value) {
if ($value === null || $value === '') {
continue;
}
if ($field === 'asin' && ! empty($check->asin)) {
continue;
}
$updates[$field] = $value;
}
if ($updates !== []) {
$updates['updated_at'] = now();
BookInfo::query()->where('id', $bookId)->update($updates);
}
}
$coverUrl = trim((string) ($book['coverurl'] ?? ''));
if ($coverUrl !== '') {
$coverSaved = $this->releaseImageService->saveRemoteImage(
(string) $bookId,
$coverUrl,
$this->imgSavePath,
ImageAssetProfile::MetadataCover,
)->success;
if ($coverSaved) {
BookInfo::query()->where('id', $bookId)->update(['cover' => 1]);
}
}
return $bookId;
}
/**
* @param array<string, mixed> $book
*/
private function outputBookMatch(array $book): void
{
if (! $this->echooutput) {
return;
}
cli()->header('Added/updated book: ');
if (! empty($book['author'])) {
cli()->alternateOver(' Author: ').cli()->primary((string) $book['author']);
}
cli()->alternateOver(' Title: ').cli()->primary(' '.(string) $book['title']);
if (! empty($book['genre'])) {
cli()->alternateOver(' Genre: ').cli()->primary(' '.(string) $book['genre']);
}
}
private function failureCacheKey(?BookParseResult $parsed, string $bookInfo): string
{
$identity = [
(string) self::FAILURE_CACHE_VERSION,
mb_strtolower(trim($parsed === null ? $bookInfo : $parsed->title)),
mb_strtolower(trim((string) ($parsed === null ? '' : $parsed->author))),
(string) ($parsed === null ? '' : $parsed->year),
(string) ($parsed === null ? '' : $parsed->isbn),
];
return 'book_lookup_fail_v'.self::FAILURE_CACHE_VERSION.'_'.md5(implode('|', $identity));
}
private function logProviderFailure(BookProviderException $exception, string $bookInfo): void
{
Log::warning('Book metadata provider unavailable', [
'provider' => $exception->provider,
'status' => $exception->statusCode,
'retry_after_seconds' => $exception->retryAfterSeconds,
'book_info' => $bookInfo,
'message' => $exception->getMessage(),
]);
} }
} }
+38 -14
View File
@@ -4,11 +4,13 @@ declare(strict_types=1);
namespace App\Services; namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon; use Carbon\Carbon;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log; use JsonException;
class GoogleBooksService class GoogleBooksService
{ {
@@ -73,7 +75,7 @@ class GoogleBooksService
$query = trim($query); $query = trim($query);
$title = $title !== null ? trim($title) : null; $title = $title !== null ? trim($title) : null;
$author = $author !== null ? trim($author) : null; $author = $author !== null ? trim($author) : null;
$isbn = $isbn !== null ? strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn)) : null; $isbn = BookIsbn::normalize($isbn);
$searchQuery = $this->buildSearchQuery($query, $title, $author, $isbn); $searchQuery = $this->buildSearchQuery($query, $title, $author, $isbn);
if ($searchQuery === '') { if ($searchQuery === '') {
@@ -118,20 +120,42 @@ class GoogleBooksService
$query['key'] = $this->apiKey; $query['key'] = $this->apiKey;
} }
$response = $this->client->get(self::API_URL, ['query' => $query]); $response = $this->client->get(self::API_URL, [
'http_errors' => false,
'query' => $query,
]);
if ($response->getStatusCode() !== 200) { if ($response->getStatusCode() !== 200) {
Log::warning('Google Books API request failed', ['status' => $response->getStatusCode()]); throw new BookProviderException(
'google_books',
return null; 'Google Books API request failed.',
$response->getStatusCode(),
);
} }
$decoded = json_decode($response->getBody()->getContents(), true); $decoded = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($decoded)) {
throw new BookProviderException('google_books', 'Google Books returned an invalid response payload.', 200);
}
return is_array($decoded) ? $decoded : null; return $decoded;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException(
'google_books',
'Google Books returned malformed JSON.',
200,
null,
$exception,
);
} catch (GuzzleException $e) { } catch (GuzzleException $e) {
Log::error('Google Books API request error: '.$e->getMessage()); throw new BookProviderException(
'google_books',
return null; 'Google Books API request error: '.$e->getMessage(),
null,
null,
$e,
);
} }
} }
@@ -172,11 +196,11 @@ class GoogleBooksService
continue; continue;
} }
$type = (string) ($identifier['type'] ?? ''); $type = (string) ($identifier['type'] ?? '');
$value = strtoupper((string) preg_replace('/[^0-9X]/i', '', (string) ($identifier['identifier'] ?? ''))); $value = BookIsbn::normalize((string) ($identifier['identifier'] ?? ''));
if ($type === 'ISBN_13' && $value !== '') { if ($type === 'ISBN_13' && $value !== null && strlen($value) === 13) {
$isbn13 = $value; $isbn13 = $value;
} }
if ($type === 'ISBN_10' && $value !== '') { if ($type === 'ISBN_10' && $value !== null && strlen($value) === 10) {
$isbn10 = $value; $isbn10 = $value;
} }
} }
+109 -36
View File
@@ -4,22 +4,27 @@ declare(strict_types=1);
namespace App\Services; namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon; use Carbon\Carbon;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use JsonException;
use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ResponseInterface;
class IsbnDbService class IsbnDbService
{ {
public const COOLDOWN_CACHE_KEY = 'isbndb_provider_cooldown';
protected const API_URL = 'https://api2.isbndb.com'; protected const API_URL = 'https://api2.isbndb.com';
protected Client $client; protected Client $client;
protected string $apiKey; protected string $apiKey;
protected int $pageSize = 5; protected int $pageSize = 10;
public function __construct(?Client $client = null, ?string $apiKey = null) public function __construct(?Client $client = null, ?string $apiKey = null)
{ {
@@ -45,9 +50,9 @@ class IsbnDbService
/** /**
* @return array<string, mixed>|null * @return array<string, mixed>|null
*/ */
public function searchBook(string $query): ?array public function searchBook(string $query, ?string $author = null, bool $shouldMatchAll = true): ?array
{ {
$books = $this->searchBooks($query); $books = $this->searchBooks($query, $author, $shouldMatchAll);
return $books[0] ?? null; return $books[0] ?? null;
} }
@@ -55,26 +60,46 @@ class IsbnDbService
/** /**
* @return array<int, array<string, mixed>> * @return array<int, array<string, mixed>>
*/ */
public function searchBooks(string $query): array public function searchBooks(string $query, ?string $author = null, bool $shouldMatchAll = true): array
{ {
$query = trim($query); $query = trim($query);
$author = $author !== null ? trim($author) : null;
if ($query === '' || ! $this->isConfigured()) { if ($query === '' || ! $this->isConfigured()) {
return []; return [];
} }
$cacheKey = 'isbndb_search_'.md5($query.'_'.$this->pageSize); if (Cache::has(self::COOLDOWN_CACHE_KEY)) {
throw new BookProviderException('isbndb', 'ISBNdb is temporarily cooling down after a failed request.');
}
$cacheKey = 'isbndb_search_'.md5(implode('|', [
mb_strtolower((string) preg_replace('/\s+/', ' ', $query)),
mb_strtolower((string) preg_replace('/\s+/', ' ', (string) $author)),
$shouldMatchAll ? 'all' : 'any',
(string) $this->pageSize,
]));
$cached = Cache::get($cacheKey); $cached = Cache::get($cacheKey);
if (is_array($cached)) { if (is_array($cached)) {
return $cached; return $cached;
} }
$response = $this->request('/books/'.rawurlencode($query), [ $requestQuery = [
'shouldMatchAll' => true, 'shouldMatchAll' => $shouldMatchAll ? 'true' : 'false',
'pageSize' => $this->pageSize, 'pageSize' => $this->pageSize,
]); 'column' => 'title',
];
$response = $this->request('/books/'.rawurlencode($query), $requestQuery);
$books = $response['data'] ?? null; $books = $response['data'] ?? null;
if (! is_array($books) || $books === []) { if (! is_array($books)) {
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException('isbndb', 'ISBNdb search response did not contain a data array.', 200, 300);
}
if ($books === []) {
Cache::put($cacheKey, [], now()->addHours(6));
return []; return [];
} }
@@ -95,11 +120,15 @@ class IsbnDbService
*/ */
public function findByIsbn(string $isbn): ?array public function findByIsbn(string $isbn): ?array
{ {
$isbn = $this->normalizeIsbn($isbn); $isbn = BookIsbn::normalize($isbn);
if ($isbn === '' || ! $this->isConfigured()) { if ($isbn === null || ! $this->isConfigured()) {
return null; return null;
} }
if (Cache::has(self::COOLDOWN_CACHE_KEY)) {
throw new BookProviderException('isbndb', 'ISBNdb is temporarily cooling down after a failed request.');
}
$cacheKey = 'isbndb_isbn_'.$isbn; $cacheKey = 'isbndb_isbn_'.$isbn;
$cached = Cache::get($cacheKey); $cached = Cache::get($cacheKey);
if (is_array($cached)) { if (is_array($cached)) {
@@ -109,7 +138,9 @@ class IsbnDbService
$response = $this->request('/book/'.rawurlencode($isbn)); $response = $this->request('/book/'.rawurlencode($isbn));
$bookRaw = $response['book'] ?? null; $bookRaw = $response['book'] ?? null;
if (! is_array($bookRaw) || $bookRaw === []) { if (! is_array($bookRaw) || $bookRaw === []) {
return null; Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException('isbndb', 'ISBNdb ISBN response did not contain a book object.', 200, 300);
} }
$book = $this->normalizeBookResult($bookRaw); $book = $this->normalizeBookResult($bookRaw);
@@ -130,6 +161,7 @@ class IsbnDbService
try { try {
$response = $this->client->get($path, [ $response = $this->client->get($path, [
'http_errors' => false,
'headers' => [ 'headers' => [
'Authorization' => $this->apiKey, 'Authorization' => $this->apiKey,
], ],
@@ -143,25 +175,50 @@ class IsbnDbService
return null; return null;
} }
if ($statusCode === 401 || $statusCode === 429) {
Log::warning("ISBNdb request failed with status {$statusCode}");
return null;
}
if ($statusCode !== 200) { if ($statusCode !== 200) {
Log::warning("ISBNdb request returned status {$statusCode}"); $retryAfter = $this->retryAfterSeconds($response);
if ($statusCode === 429 || $statusCode >= 500) {
$retryAfter ??= 300;
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addSeconds($retryAfter));
}
return null; throw new BookProviderException(
'isbndb',
"ISBNdb request returned status {$statusCode}.",
$statusCode,
$retryAfter,
);
} }
$data = json_decode($response->getBody()->getContents(), true); $data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
return is_array($data) ? $data : null; if (! is_array($data)) {
throw new BookProviderException('isbndb', 'ISBNdb returned an invalid response payload.', $statusCode);
}
return $data;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException(
'isbndb',
'ISBNdb returned malformed JSON.',
200,
300,
$exception,
);
} catch (GuzzleException $e) { } catch (GuzzleException $e) {
Log::error('ISBNdb API request error: '.$e->getMessage()); Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
return null; throw new BookProviderException(
'isbndb',
'ISBNdb API request error: '.$e->getMessage(),
null,
300,
$e,
);
} }
} }
@@ -171,12 +228,18 @@ class IsbnDbService
*/ */
private function normalizeBookResult(array $book): array private function normalizeBookResult(array $book): array
{ {
$isbn13 = isset($book['isbn13']) ? $this->normalizeIsbn((string) $book['isbn13']) : ''; $isbn13 = isset($book['isbn13']) ? BookIsbn::normalize((string) $book['isbn13']) : null;
if ($isbn13 === '' && isset($book['isbn'])) { if ($isbn13 === null && isset($book['isbn'])) {
$isbn13 = $this->normalizeIsbn((string) $book['isbn']); $isbn13 = BookIsbn::normalize((string) $book['isbn']);
}
if ($isbn13 !== null && strlen($isbn13) === 10) {
$isbn13 = BookIsbn::toIsbn13($isbn13);
} }
$isbn10 = isset($book['isbn10']) ? $this->normalizeIsbn((string) $book['isbn10']) : ''; $isbn10 = isset($book['isbn10']) ? BookIsbn::normalize((string) $book['isbn10']) : null;
if ($isbn10 !== null && strlen($isbn10) === 13) {
$isbn10 = BookIsbn::toIsbn10($isbn10);
}
$authors = is_array($book['authors'] ?? null) $authors = is_array($book['authors'] ?? null)
? array_values(array_filter(array_map('strval', $book['authors']))) ? array_values(array_filter(array_map('strval', $book['authors'])))
@@ -185,7 +248,7 @@ class IsbnDbService
? array_values(array_filter(array_map('strval', $book['subjects']))) ? array_values(array_filter(array_map('strval', $book['subjects'])))
: []; : [];
$identifier = $isbn13 !== '' ? $isbn13 : ($isbn10 !== '' ? $isbn10 : md5((string) json_encode($book))); $identifier = $isbn13 ?? $isbn10 ?? md5((string) json_encode($book));
$coverUrl = (string) ($book['image'] ?? $book['image_original'] ?? ''); $coverUrl = (string) ($book['image'] ?? $book['image_original'] ?? '');
$overview = trim((string) ($book['synopsis'] ?? $book['overview'] ?? $book['excerpt'] ?? '')); $overview = trim((string) ($book['synopsis'] ?? $book['overview'] ?? $book['excerpt'] ?? ''));
@@ -193,8 +256,8 @@ class IsbnDbService
'title' => (string) ($book['title'] ?? ''), 'title' => (string) ($book['title'] ?? ''),
'author' => implode(', ', $authors), 'author' => implode(', ', $authors),
'asin' => 'isbndb:'.$identifier, 'asin' => 'isbndb:'.$identifier,
'isbn' => $isbn13 !== '' ? $isbn13 : null, 'isbn' => $isbn13,
'ean' => $isbn10 !== '' ? $isbn10 : null, 'ean' => $isbn10,
'url' => '', 'url' => '',
'salesrank' => '', 'salesrank' => '',
'publisher' => (string) ($book['publisher'] ?? ''), 'publisher' => (string) ($book['publisher'] ?? ''),
@@ -220,11 +283,6 @@ class IsbnDbService
} }
} }
private function normalizeIsbn(string $isbn): string
{
return strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn));
}
private function logRateLimitHeaders(ResponseInterface $response): void private function logRateLimitHeaders(ResponseInterface $response): void
{ {
$rateLimit = $response->getHeaderLine('ratelimit'); $rateLimit = $response->getHeaderLine('ratelimit');
@@ -251,4 +309,19 @@ class IsbnDbService
'ratelimit_policy' => $rateLimitPolicy, 'ratelimit_policy' => $rateLimitPolicy,
]); ]);
} }
private function retryAfterSeconds(ResponseInterface $response): ?int
{
$retryAfter = trim($response->getHeaderLine('Retry-After'));
if ($retryAfter === '') {
return null;
}
if (ctype_digit($retryAfter)) {
return max(1, (int) $retryAfter);
}
$retryAt = strtotime($retryAfter);
return $retryAt === false ? null : max(1, $retryAt - time());
}
} }
+12
View File
@@ -29,6 +29,8 @@ class ItunesService
protected string $country = 'US'; protected string $country = 'US';
private bool $lastRequestFailed = false;
public function __construct() public function __construct()
{ {
$this->client = new Client([ $this->client = new Client([
@@ -62,6 +64,11 @@ class ItunesService
return $this; return $this;
} }
public function lastRequestFailed(): bool
{
return $this->lastRequestFailed;
}
/** /**
* Search for music albums. * Search for music albums.
* *
@@ -240,6 +247,7 @@ class ItunesService
*/ */
protected function search(string $term, string $entity, string $media): ?array protected function search(string $term, string $entity, string $media): ?array
{ {
$this->lastRequestFailed = false;
$term = trim($term); $term = trim($term);
if (empty($term)) { if (empty($term)) {
return null; return null;
@@ -265,6 +273,7 @@ class ItunesService
$statusCode = $response->getStatusCode(); $statusCode = $response->getStatusCode();
if ($statusCode !== 200) { if ($statusCode !== 200) {
$this->lastRequestFailed = true;
Log::warning("iTunes API search returned status {$statusCode}"); Log::warning("iTunes API search returned status {$statusCode}");
return null; return null;
@@ -273,6 +282,8 @@ class ItunesService
$data = json_decode($response->getBody()->getContents(), true); $data = json_decode($response->getBody()->getContents(), true);
if (! isset($data['results'])) { if (! isset($data['results'])) {
$this->lastRequestFailed = true;
return null; return null;
} }
@@ -281,6 +292,7 @@ class ItunesService
return $results; return $results;
} catch (GuzzleException $e) { } catch (GuzzleException $e) {
$this->lastRequestFailed = true;
Log::error('iTunes API search error: '.$e->getMessage()); Log::error('iTunes API search error: '.$e->getMessage());
return null; return null;
+39 -18
View File
@@ -4,11 +4,13 @@ declare(strict_types=1);
namespace App\Services; namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon; use Carbon\Carbon;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log; use JsonException;
class OpenLibraryService class OpenLibraryService
{ {
@@ -38,8 +40,8 @@ class OpenLibraryService
*/ */
public function findByIsbn(string $isbn): ?array public function findByIsbn(string $isbn): ?array
{ {
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn)); $isbn = BookIsbn::normalize($isbn);
if ($isbn === '') { if ($isbn === null) {
return null; return null;
} }
@@ -50,24 +52,33 @@ class OpenLibraryService
} }
try { try {
$response = $this->client->get(self::ISBN_URL.rawurlencode($isbn).'.json'); $response = $this->client->get(self::ISBN_URL.rawurlencode($isbn).'.json', ['http_errors' => false]);
if ($response->getStatusCode() !== 200) { if ($response->getStatusCode() === 404) {
return null; return null;
} }
if ($response->getStatusCode() !== 200) {
throw new BookProviderException(
'open_library',
'Open Library ISBN request failed.',
$response->getStatusCode(),
);
}
$data = json_decode($response->getBody()->getContents(), true); $data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($data)) { if (! is_array($data)) {
return null; throw new BookProviderException('open_library', 'Open Library returned an invalid ISBN payload.', 200);
} }
$book = $this->normalizeIsbnResult($data, $isbn); $book = $this->normalizeIsbnResult($data, $isbn);
Cache::put($cacheKey, $book, now()->addHours(24)); Cache::put($cacheKey, $book, now()->addHours(24));
return $book; return $book;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException('open_library', 'Open Library returned malformed JSON.', 200, null, $exception);
} catch (GuzzleException $e) { } catch (GuzzleException $e) {
Log::error('OpenLibrary ISBN lookup error: '.$e->getMessage()); throw new BookProviderException('open_library', 'Open Library ISBN lookup error: '.$e->getMessage(), null, null, $e);
return null;
} }
} }
@@ -99,18 +110,26 @@ class OpenLibraryService
try { try {
$response = $this->client->get(self::SEARCH_URL, [ $response = $this->client->get(self::SEARCH_URL, [
'http_errors' => false,
'query' => [ 'query' => [
'q' => $query, 'q' => $query,
'limit' => $this->limit, 'limit' => $this->limit,
], ],
]); ]);
if ($response->getStatusCode() !== 200) { if ($response->getStatusCode() === 404) {
return []; return [];
} }
if ($response->getStatusCode() !== 200) {
throw new BookProviderException(
'open_library',
'Open Library search request failed.',
$response->getStatusCode(),
);
}
$data = json_decode($response->getBody()->getContents(), true); $data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($data['docs'] ?? null)) { if (! is_array($data['docs'] ?? null)) {
return []; throw new BookProviderException('open_library', 'Open Library returned an invalid search payload.', 200);
} }
$results = []; $results = [];
@@ -123,10 +142,12 @@ class OpenLibraryService
Cache::put($cacheKey, $results, now()->addHours(12)); Cache::put($cacheKey, $results, now()->addHours(12));
return $results; return $results;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException('open_library', 'Open Library returned malformed JSON.', 200, null, $exception);
} catch (GuzzleException $e) { } catch (GuzzleException $e) {
Log::error('OpenLibrary search error: '.$e->getMessage()); throw new BookProviderException('open_library', 'Open Library search error: '.$e->getMessage(), null, null, $e);
return [];
} }
} }
@@ -180,8 +201,8 @@ class OpenLibraryService
$isbn10 = null; $isbn10 = null;
if (is_array($doc['isbn'] ?? null)) { if (is_array($doc['isbn'] ?? null)) {
foreach ($doc['isbn'] as $isbn) { foreach ($doc['isbn'] as $isbn) {
$normalized = strtoupper((string) preg_replace('/[^0-9X]/i', '', (string) $isbn)); $normalized = BookIsbn::normalize((string) $isbn);
if ($normalized === '') { if ($normalized === null) {
continue; continue;
} }
if (strlen($normalized) === 13 && $isbn13 === null) { if (strlen($normalized) === 13 && $isbn13 === null) {
+13 -3
View File
@@ -155,6 +155,7 @@ class PostProcessRunner extends BaseRunner
private function getBooksBuckets(): array private function getBooksBuckets(): array
{ {
$bucketExpr = $this->guidBucketExpression(); $bucketExpr = $this->guidBucketExpression();
$pendingCondition = $this->bookPendingCondition();
return DB::select(' return DB::select('
SELECT DISTINCT '.$bucketExpr.' AS id SELECT DISTINCT '.$bucketExpr.' AS id
@@ -164,7 +165,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = '.Category::MUSIC_AUDIOBOOK.' OR categories_id = '.Category::MUSIC_AUDIOBOOK.'
) )
AND ( AND (
bookinfo_id IS NULL '.$pendingCondition.'
OR searchname LIKE "N:/NZB%" OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%" OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%" OR name LIKE "N:/NZB%"
@@ -600,6 +601,8 @@ class PostProcessRunner extends BaseRunner
return; return;
} }
$pendingCondition = $this->bookPendingCondition();
$checkSql = ' $checkSql = '
SELECT id SELECT id
FROM releases FROM releases
@@ -608,7 +611,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = 3030 OR categories_id = 3030
) )
AND ( AND (
bookinfo_id IS NULL '.$pendingCondition.'
OR searchname LIKE "N:/NZB%" OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%" OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%" OR name LIKE "N:/NZB%"
@@ -630,7 +633,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = 3030 OR categories_id = 3030
) )
AND ( AND (
bookinfo_id IS NULL '.$pendingCondition.'
OR searchname LIKE "N:/NZB%" OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%" OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%" OR name LIKE "N:/NZB%"
@@ -643,6 +646,13 @@ class PostProcessRunner extends BaseRunner
$this->runPostProcess($queue, $maxProcesses, 'books', 'books postprocessing'); $this->runPostProcess($queue, $maxProcesses, 'books', 'books postprocessing');
} }
private function bookPendingCondition(): string
{
return (int) Settings::settingValue('lookupbooks') === 2
? '(bookinfo_id IS NULL AND isrenamed = 1)'
: 'bookinfo_id IS NULL';
}
public function processMusic(): void public function processMusic(): void
{ {
if ((int) Settings::settingValue('lookupmusic') <= 0) { if ((int) Settings::settingValue('lookupmusic') <= 0) {
+114
View File
@@ -0,0 +1,114 @@
<?php
declare(strict_types=1);
namespace App\Support;
final class BookIsbn
{
public static function normalize(?string $value): ?string
{
if ($value === null) {
return null;
}
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $value));
return match (strlen($isbn)) {
10 => self::isValidIsbn10($isbn) ? $isbn : null,
13 => self::isValidIsbn13($isbn) ? $isbn : null,
default => null,
};
}
public static function equivalent(?string $left, ?string $right): bool
{
$leftIdentifiers = self::equivalents($left);
$rightIdentifiers = self::equivalents($right);
return $leftIdentifiers !== [] && array_intersect($leftIdentifiers, $rightIdentifiers) !== [];
}
/**
* @return list<string>
*/
public static function equivalents(?string $value): array
{
$isbn = self::normalize($value);
if ($isbn === null) {
return [];
}
$identifiers = [$isbn];
$converted = strlen($isbn) === 10 ? self::toIsbn13($isbn) : self::toIsbn10($isbn);
if ($converted !== null) {
$identifiers[] = $converted;
}
sort($identifiers);
return array_values(array_unique($identifiers));
}
public static function toIsbn13(?string $value): ?string
{
$isbn10 = self::normalize($value);
if ($isbn10 === null || strlen($isbn10) !== 10) {
return null;
}
$body = '978'.substr($isbn10, 0, 9);
$sum = 0;
for ($index = 0; $index < 12; $index++) {
$sum += ((int) $body[$index]) * ($index % 2 === 0 ? 1 : 3);
}
return $body.(string) ((10 - ($sum % 10)) % 10);
}
public static function toIsbn10(?string $value): ?string
{
$isbn13 = self::normalize($value);
if ($isbn13 === null || strlen($isbn13) !== 13 || ! str_starts_with($isbn13, '978')) {
return null;
}
$body = substr($isbn13, 3, 9);
$sum = 0;
for ($index = 0; $index < 9; $index++) {
$sum += ((int) $body[$index]) * (10 - $index);
}
$check = (11 - ($sum % 11)) % 11;
return $body.($check === 10 ? 'X' : (string) $check);
}
private static function isValidIsbn10(string $isbn): bool
{
if (preg_match('/^\d{9}[\dX]$/', $isbn) !== 1) {
return false;
}
$sum = 0;
for ($index = 0; $index < 10; $index++) {
$digit = $isbn[$index] === 'X' ? 10 : (int) $isbn[$index];
$sum += $digit * (10 - $index);
}
return $sum % 11 === 0;
}
private static function isValidIsbn13(string $isbn): bool
{
if (preg_match('/^97[89]\d{10}$/', $isbn) !== 1) {
return false;
}
$sum = 0;
for ($index = 0; $index < 13; $index++) {
$sum += ((int) $isbn[$index]) * ($index % 2 === 0 ? 1 : 3);
}
return $sum % 10 === 0;
}
}
+149 -53
View File
@@ -14,34 +14,32 @@ class BookMatchScorer
*/ */
public function score(array $candidate, BookParseResult $parsed): float public function score(array $candidate, BookParseResult $parsed): float
{ {
$candidateIsbn = $this->normalizeIsbn((string) ($candidate['isbn'] ?? '')); $parsedIsbn = BookIsbn::normalize($parsed->isbn);
$parsedIsbn = $this->normalizeIsbn((string) ($parsed->isbn ?? '')); $candidateIdentifiers = $this->candidateIdentifiers($candidate);
if ($parsedIsbn !== '' && $candidateIsbn !== '' && $parsedIsbn === $candidateIsbn) { if ($parsedIsbn !== null && $candidateIdentifiers !== []) {
foreach ($candidateIdentifiers as $candidateIdentifier) {
if (! BookIsbn::equivalent($parsedIsbn, $candidateIdentifier)) {
return 0.0;
}
}
return 1.0; return 1.0;
} }
$titleScore = $this->titleSimilarity($parsed->title, (string) ($candidate['title'] ?? '')); if ($this->hasStructuralConflict($parsed, (string) ($candidate['title'] ?? ''))) {
$authorScore = $parsed->hasAuthor() return 0.0;
? $this->similarity((string) $parsed->author, (string) ($candidate['author'] ?? ''))
: 0.0;
$yearScore = $this->yearScore($parsed->year, (string) ($candidate['publishdate'] ?? ''));
$publisherScore = ! empty($candidate['publisher']) ? 1.0 : 0.0;
$coverScore = (int) ($candidate['cover'] ?? 0) === 1 ? 1.0 : 0.0;
if ($parsed->hasAuthor()) {
return (0.45 * $titleScore)
+ (0.30 * $authorScore)
+ (0.10 * $yearScore)
+ (0.08 * $publisherScore)
+ (0.07 * $coverScore);
} }
// When the release has no reliable author, avoid over-trusting $titleScore = $this->titleScore($parsed->title, (string) ($candidate['title'] ?? ''));
// "has cover/publisher" signals for weak title matches. $authorScore = $parsed->hasAuthor()
return (0.80 * $titleScore) ? $this->authorScore((string) $parsed->author, (string) ($candidate['author'] ?? ''))
+ (0.10 * $yearScore) : 0.0;
+ (0.05 * $publisherScore)
+ (0.05 * $coverScore); if ($parsed->hasAuthor()) {
return (0.65 * $titleScore) + (0.35 * $authorScore);
}
return $titleScore;
} }
public function scoreBookInfo(BookInfo $book, BookParseResult $parsed): float public function scoreBookInfo(BookInfo $book, BookParseResult $parsed): float
@@ -50,14 +48,28 @@ class BookMatchScorer
'title' => $book->title, 'title' => $book->title,
'author' => $book->author, 'author' => $book->author,
'isbn' => $book->isbn, 'isbn' => $book->isbn,
'ean' => $book->ean,
'publishdate' => $book->publishdate, 'publishdate' => $book->publishdate,
'publisher' => $book->publisher, 'publisher' => $book->publisher,
'cover' => $book->cover ? 1 : 0, 'cover' => $book->cover ? 1 : 0,
], $parsed); ], $parsed);
} }
private function titleSimilarity(string $left, string $right): float public function titleScore(string $left, string $right): float
{ {
$leftVariants = $this->titleVariants($left);
$rightVariants = $this->titleVariants($right);
foreach ($leftVariants as $leftVariant) {
foreach ($rightVariants as $rightVariant) {
if ($leftVariant !== '' && $leftVariant === $rightVariant) {
return $leftVariant === $this->normalizeText($left)
&& $rightVariant === $this->normalizeText($right)
? 1.0
: 0.97;
}
}
}
$left = $this->normalizeText($left); $left = $this->normalizeText($left);
$right = $this->normalizeText($right); $right = $this->normalizeText($right);
if ($left === '' || $right === '') { if ($left === '' || $right === '') {
@@ -74,52 +86,62 @@ class BookMatchScorer
return 0.0; return 0.0;
} }
$shared = array_intersect($leftWords, $rightWords); $shared = array_intersect(array_unique($leftWords), array_unique($rightWords));
$total = max(count($leftWords), count($rightWords)); $tokenScore = (2 * count($shared)) / (count(array_unique($leftWords)) + count(array_unique($rightWords)));
$jaccardScore = count($shared) / $total;
similar_text($left, $right, $percent); similar_text($left, $right, $percent);
$simTextScore = max(0.0, min(1.0, $percent / 100)); $simTextScore = max(0.0, min(1.0, $percent / 100));
$lengthRatio = min(mb_strlen($left), mb_strlen($right)) / max(mb_strlen($left), mb_strlen($right)); return min(1.0, max($tokenScore, $simTextScore));
$lengthPenalty = $lengthRatio < 0.5 ? 0.7 : 1.0;
return min(1.0, ((0.5 * $jaccardScore) + (0.5 * $simTextScore)) * $lengthPenalty);
} }
private function similarity(string $left, string $right): float public function authorScore(string $left, string $right): float
{ {
$left = $this->normalizeText($left); $leftTokens = $this->tokens($left);
$right = $this->normalizeText($right); $rightTokens = $this->tokens($right);
if ($left === '' || $right === '') { if ($leftTokens === [] || $rightTokens === []) {
return 0.0; return 0.0;
} }
similar_text($left, $right, $percent); $leftCoverage = $this->authorTokenCoverage($leftTokens, $rightTokens);
$rightCoverage = $this->authorTokenCoverage($rightTokens, $leftTokens);
return max(0.0, min(1.0, $percent / 100)); return ($leftCoverage + $rightCoverage) / 2;
} }
private function yearScore(?int $parsedYear, string $candidatePublishDate): float public function hasStructuralConflict(BookParseResult $parsed, string $candidateTitle): bool
{ {
if ($parsedYear === null) { $parsedMarkers = $this->structuralMarkers($parsed->rawName.' '.$parsed->title);
return 0.5; $candidateMarkers = $this->structuralMarkers($candidateTitle);
foreach ($parsedMarkers as $type => $number) {
if (isset($candidateMarkers[$type]) && $candidateMarkers[$type] !== $number) {
return true;
}
} }
if (preg_match('/\b(19|20)\d{2}\b/', $candidatePublishDate, $matches) !== 1) { return false;
return 0.0; }
}
$candidateYear = (int) $matches[0]; public function meaningfulTitleTokenCount(string $title): int
if ($parsedYear === $candidateYear) { {
return 1.0; $stopWords = ['a', 'an', 'and', 'by', 'for', 'in', 'of', 'on', 'or', 'the', 'to', 'with'];
}
if (abs($parsedYear - $candidateYear) <= 1) { return count(array_filter(
return 0.6; $this->tokens($title),
} static fn (string $token): bool => mb_strlen($token) > 1 && ! in_array($token, $stopWords, true)
));
}
return 0.0; /**
* @param array<string, mixed> $candidate
*/
public function workKey(array $candidate): string
{
$authorTokens = $this->tokens((string) ($candidate['author'] ?? ''));
sort($authorTokens);
return $this->normalizeText((string) ($candidate['title'] ?? '')).'|'
.implode(' ', $authorTokens);
} }
private function normalizeText(string $value): string private function normalizeText(string $value): string
@@ -132,8 +154,82 @@ class BookMatchScorer
return trim($value); return trim($value);
} }
private function normalizeIsbn(string $value): string /**
* @return list<string>
*/
private function titleVariants(string $value): array
{ {
return strtoupper((string) preg_replace('/[^0-9X]/i', '', $value)); $variants = [$this->normalizeText($value)];
$parts = preg_split('/\s+(?::|-)\s+|:\s*/u', $value, 2);
if (is_array($parts) && count($parts) === 2) {
$variants[] = $this->normalizeText($parts[0]);
}
return array_values(array_unique(array_filter($variants)));
}
/**
* @param array<string, mixed> $candidate
* @return list<string>
*/
private function candidateIdentifiers(array $candidate): array
{
$identifiers = [];
foreach (['isbn', 'ean'] as $field) {
$isbn = BookIsbn::normalize(isset($candidate[$field]) ? (string) $candidate[$field] : null);
if ($isbn !== null) {
$identifiers[] = $isbn;
}
}
return array_values(array_unique($identifiers));
}
/**
* @return list<string>
*/
private function tokens(string $value): array
{
$normalized = $this->normalizeText($value);
return $normalized === '' ? [] : array_values(array_unique(explode(' ', $normalized)));
}
/**
* @param list<string> $needles
* @param list<string> $haystack
*/
private function authorTokenCoverage(array $needles, array $haystack): float
{
$matched = 0;
foreach ($needles as $needle) {
foreach ($haystack as $candidate) {
if ($needle === $candidate
|| (mb_strlen($needle) === 1 && str_starts_with($candidate, $needle))
|| (mb_strlen($candidate) === 1 && str_starts_with($needle, $candidate))) {
$matched++;
break;
}
}
}
return $matched / count($needles);
}
/**
* @return array<string, int>
*/
private function structuralMarkers(string $value): array
{
$markers = [];
if (preg_match('/\b(?:book|series|vol(?:ume)?)\.?\s*#?\s*(\d{1,3})\b/i', $value, $match) === 1) {
$markers['volume'] = (int) $match[1];
}
if (preg_match('/\b(\d{1,3})(?:st|nd|rd|th)?\s+(?:edition|ed\.?)\b/i', $value, $match) === 1) {
$markers['edition'] = (int) $match[1];
}
return $markers;
} }
} }
@@ -0,0 +1,50 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
private const ISBN_INDEX = 'ix_bookinfo_isbn';
private const EAN_INDEX = 'ix_bookinfo_ean';
/**
* Run the migrations.
*/
public function up(): void
{
if (! Schema::hasTable('bookinfo')) {
return;
}
Schema::table('bookinfo', static function (Blueprint $table): void {
if (! Schema::hasIndex('bookinfo', self::ISBN_INDEX)) {
$table->index('isbn', self::ISBN_INDEX);
}
if (! Schema::hasIndex('bookinfo', self::EAN_INDEX)) {
$table->index('ean', self::EAN_INDEX);
}
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
if (! Schema::hasTable('bookinfo')) {
return;
}
Schema::table('bookinfo', static function (Blueprint $table): void {
if (Schema::hasIndex('bookinfo', self::ISBN_INDEX)) {
$table->dropIndex(self::ISBN_INDEX);
}
if (Schema::hasIndex('bookinfo', self::EAN_INDEX)) {
$table->dropIndex(self::EAN_INDEX);
}
});
}
};
+2
View File
@@ -119,6 +119,8 @@ CREATE TABLE `bookinfo` (
`updated_at` timestamp NULL DEFAULT NULL, `updated_at` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `ix_bookinfo_asin` (`asin`), UNIQUE KEY `ix_bookinfo_asin` (`asin`),
KEY `ix_bookinfo_isbn` (`isbn`),
KEY `ix_bookinfo_ean` (`ean`),
FULLTEXT KEY `ix_bookinfo_author_title_ft` (`author`,`title`) FULLTEXT KEY `ix_bookinfo_author_title_ft` (`author`,`title`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
+2
View File
@@ -119,6 +119,8 @@ CREATE TABLE `bookinfo` (
`updated_at` timestamp NULL DEFAULT NULL, `updated_at` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `ix_bookinfo_asin` (`asin`), UNIQUE KEY `ix_bookinfo_asin` (`asin`),
KEY `ix_bookinfo_isbn` (`isbn`),
KEY `ix_bookinfo_ean` (`ean`),
FULLTEXT KEY `ix_bookinfo_author_title_ft` (`author`,`title`) FULLTEXT KEY `ix_bookinfo_author_title_ft` (`author`,`title`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
@@ -0,0 +1,107 @@
<?php
declare(strict_types=1);
namespace Tests\Feature;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
use PDO;
use Tests\TestCase;
class BookInfoIsbnIndexesMigrationTest extends TestCase
{
private string $databasePath;
/**
* @var array<string, string|false>
*/
private array $originalEnvironment = [];
public function createApplication()
{
$this->databasePath = sys_get_temp_dir().'/nntmux-bookinfo-isbn-indexes.sqlite';
$this->originalEnvironment = [
'APP_ENV' => getenv('APP_ENV'),
'DB_CONNECTION' => getenv('DB_CONNECTION'),
'DB_DATABASE' => getenv('DB_DATABASE'),
];
if (file_exists($this->databasePath)) {
unlink($this->databasePath);
}
$pdo = new PDO('sqlite:'.$this->databasePath);
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
$pdo->exec("INSERT INTO settings (name, value) VALUES ('categorizeforeign', '0'), ('catwebdl', '1')");
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
return $app;
}
protected function setUp(): void
{
parent::setUp();
config([
'database.default' => 'sqlite',
'database.connections.sqlite.database' => $this->databasePath,
]);
Schema::create('bookinfo', function (Blueprint $table): void {
$table->increments('id');
$table->string('isbn')->nullable();
$table->string('ean')->nullable();
});
}
protected function tearDown(): void
{
if ($this->databasePath !== '' && file_exists($this->databasePath)) {
unlink($this->databasePath);
}
parent::tearDown();
foreach ($this->originalEnvironment as $key => $value) {
$this->setEnvironmentValue($key, $value === false ? null : $value);
}
}
public function test_it_adds_and_removes_isbn_indexes_idempotently(): void
{
$migration = require database_path('migrations/2026_08_13_133021_add_isbn_indexes_to_bookinfo_table.php');
$migration->up();
$migration->up();
$this->assertTrue(Schema::hasIndex('bookinfo', 'ix_bookinfo_isbn'));
$this->assertTrue(Schema::hasIndex('bookinfo', 'ix_bookinfo_ean'));
$migration->down();
$this->assertFalse(Schema::hasIndex('bookinfo', 'ix_bookinfo_isbn'));
$this->assertFalse(Schema::hasIndex('bookinfo', 'ix_bookinfo_ean'));
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
}
+433
View File
@@ -0,0 +1,433 @@
<?php
declare(strict_types=1);
namespace Tests\Feature;
use App\Exceptions\BookProviderException;
use App\Models\BookInfo;
use App\Services\BookService;
use App\Services\GoogleBooksService;
use App\Services\IsbnDbService;
use App\Services\ItunesService;
use App\Services\OpenLibraryService;
use App\Services\ReleaseImageService;
use App\Support\BookMatchScorer;
use App\Support\Data\BookParseResult;
use App\Support\Data\ImageProcessingResult;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Mockery;
use PDO;
use Tests\TestCase;
class BookServiceMatchingTest extends TestCase
{
private string $databasePath;
/**
* @var array<string, string|false>
*/
private array $originalEnvironment = [];
public function createApplication()
{
$this->databasePath = sys_get_temp_dir().'/nntmux-book-service-matching.sqlite';
$this->originalEnvironment = [
'APP_ENV' => getenv('APP_ENV'),
'DB_CONNECTION' => getenv('DB_CONNECTION'),
'DB_DATABASE' => getenv('DB_DATABASE'),
];
if (file_exists($this->databasePath)) {
unlink($this->databasePath);
}
$pdo = new PDO('sqlite:'.$this->databasePath);
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
$pdo->exec("INSERT INTO settings (name, value) VALUES ('maxbooksprocessed', '50'), ('amazonsleep', '0'), ('lookupbooks', '1')");
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
return $app;
}
protected function setUp(): void
{
parent::setUp();
config([
'database.default' => 'sqlite',
'database.connections.sqlite.database' => $this->databasePath,
]);
DB::purge();
DB::reconnect();
Schema::create('bookinfo', function (Blueprint $table): void {
$table->increments('id');
$table->string('title');
$table->string('author')->default('');
$table->string('asin')->nullable()->unique();
$table->string('isbn')->nullable()->index();
$table->string('ean')->nullable()->index();
$table->string('url')->nullable();
$table->unsignedInteger('salesrank')->nullable();
$table->string('publisher')->nullable();
$table->dateTime('publishdate')->nullable();
$table->string('pages')->nullable();
$table->text('overview')->nullable();
$table->string('genre')->default('');
$table->boolean('cover')->default(false);
$table->timestamps();
});
}
protected function tearDown(): void
{
if ($this->databasePath !== '' && file_exists($this->databasePath)) {
unlink($this->databasePath);
}
parent::tearDown();
foreach ($this->originalEnvironment as $key => $value) {
$this->setEnvironmentValue($key, $value === false ? null : $value);
}
}
public function test_rejected_isbndb_candidates_fall_through_to_google_books(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')->once()->andReturn([
$this->candidate('Unrelated Cookbook', 'Different Author', 'isbndb:wrong'),
]);
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldReceive('searchBooks')->once()->andReturn([
$this->candidate('Clean Code', 'Robert C. Martin', 'googlebooks:clean-code'),
]);
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldNotReceive('searchBooks');
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldNotReceive('findEbooks');
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $this->cleanCodeParseResult();
$bookId = $service->updateBookInfo('Robert C Martin Clean Code');
$this->assertIsInt($bookId);
$this->assertGreaterThan(0, $bookId);
$this->assertSame('Clean Code', BookInfo::query()->findOrFail($bookId)->title);
}
public function test_accepted_isbndb_candidate_stops_the_provider_cascade(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')->once()->andReturn([
$this->candidate('Clean Code', 'Robert C. Martin', 'isbndb:clean-code'),
]);
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldNotReceive('searchBooks');
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldNotReceive('searchBooks');
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldNotReceive('findEbooks');
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $this->cleanCodeParseResult();
$this->assertGreaterThan(0, $service->updateBookInfo('Robert C Martin Clean Code'));
}
public function test_duplicate_editions_of_the_same_work_do_not_create_false_ambiguity(): void
{
$first = $this->candidate('Clean Code', 'Robert C. Martin', 'isbndb:edition-one');
$second = $this->candidate('Clean Code', 'Martin, Robert C.', 'isbndb:edition-two');
$second['publisher'] = 'Prentice Hall';
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')->once()->andReturn([$first, $second]);
$service = $this->makeService(
$isbnDb,
Mockery::mock(GoogleBooksService::class),
Mockery::mock(OpenLibraryService::class),
Mockery::mock(ItunesService::class),
);
$service->parsedBookResult = $this->cleanCodeParseResult();
$this->assertGreaterThan(0, $service->updateBookInfo('Robert C Martin Clean Code'));
}
public function test_close_distinct_runner_up_prevents_a_premature_match(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')->once()->andReturn([
$this->candidate('Clean Code', 'Robert C. Martin', 'isbndb:clean-code'),
$this->candidate('Clean Coder', 'Robert C. Martin', 'isbndb:clean-coder'),
]);
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldReceive('searchBooks')->once()->andReturn([]);
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldReceive('searchBooks')->once()->andReturn([]);
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldReceive('findEbooks')->once()->andReturn([]);
$itunes->shouldReceive('lastRequestFailed')->once()->andReturnFalse();
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $this->cleanCodeParseResult();
$this->assertSame(-2, $service->updateBookInfo('Robert C Martin Clean Code'));
}
public function test_ambiguous_author_dash_title_parse_retries_as_a_title_only_search(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')
->once()
->with('From Journeyman to Master', 'The Pragmatic Programmer', true)
->andReturn([]);
$isbnDb->shouldReceive('searchBooks')
->once()
->with('The Pragmatic Programmer - From Journeyman to Master', null, true)
->andReturn([
$this->candidate(
'The Pragmatic Programmer: From Journeyman to Master',
'Andrew Hunt, David Thomas',
'isbndb:pragmatic-programmer',
),
]);
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldNotReceive('searchBooks');
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldNotReceive('searchBooks');
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldNotReceive('findEbooks');
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $service->parseReleaseName(
'The Pragmatic Programmer - From Journeyman to Master EPUB'
);
$this->assertGreaterThan(0, $service->updateBookInfo($service->parsedBookResult->searchQuery()));
}
public function test_transient_provider_failure_leaves_the_match_pending(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('searchBooks')->once()->andThrow(
new BookProviderException('isbndb', 'Unavailable', 429, 300)
);
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldReceive('searchBooks')->once()->andReturn([]);
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldReceive('searchBooks')->once()->andReturn([]);
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldReceive('findEbooks')->once()->andReturn([]);
$itunes->shouldReceive('lastRequestFailed')->once()->andReturnFalse();
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $this->cleanCodeParseResult();
$this->assertNull($service->updateBookInfo('Robert C Martin Clean Code'));
}
public function test_deterministic_exhaustion_returns_failed_sentinel(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnFalse();
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldReceive('searchBooks')->once()->andReturn([]);
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldReceive('searchBooks')->once()->andReturn([]);
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldReceive('findEbooks')->once()->andReturn([]);
$itunes->shouldReceive('lastRequestFailed')->once()->andReturnFalse();
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = $this->cleanCodeParseResult();
$this->assertSame(-2, $service->updateBookInfo('Robert C Martin Clean Code'));
}
public function test_exact_local_isbn_lookup_checks_both_identifier_versions(): void
{
$bookId = DB::table('bookinfo')->insertGetId([
'title' => 'Clean Code',
'author' => 'Robert C. Martin',
'asin' => 'existing:clean-code',
'isbn' => '9780132350884',
'ean' => '0132350882',
'genre' => '',
'cover' => 0,
'created_at' => now(),
'updated_at' => now(),
]);
$service = $this->makeService(
Mockery::mock(IsbnDbService::class),
Mockery::mock(GoogleBooksService::class),
Mockery::mock(OpenLibraryService::class),
Mockery::mock(ItunesService::class),
);
$this->assertSame($bookId, $service->getBookInfoByIsbn('0-13-235088-2')?->id);
}
public function test_equivalent_isbn_deduplication_preserves_metadata_and_failed_cover_state(): void
{
$bookId = DB::table('bookinfo')->insertGetId([
'title' => 'Clean Code',
'author' => 'Robert C. Martin',
'asin' => 'legacy:clean-code',
'isbn' => null,
'ean' => '0132350882',
'publisher' => 'Prentice Hall',
'genre' => 'Programming',
'cover' => 0,
'created_at' => now(),
'updated_at' => now(),
]);
$candidate = $this->candidate('Clean Code', 'Robert C. Martin', 'isbndb:clean-code');
$candidate['isbn'] = '9780132350884';
$candidate['publisher'] = '';
$candidate['genre'] = '';
$candidate['coverurl'] = 'https://example.com/cover.jpg';
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnTrue();
$isbnDb->shouldReceive('findByIsbn')->once()->with('9780132350884')->andReturn($candidate);
$isbnDb->shouldNotReceive('searchBooks');
$service = $this->makeService(
$isbnDb,
Mockery::mock(GoogleBooksService::class),
Mockery::mock(OpenLibraryService::class),
Mockery::mock(ItunesService::class),
);
$service->parsedBookResult = new BookParseResult(
rawName: 'Clean Code 9780132350884',
title: 'Clean Code',
author: 'Robert C Martin',
isbn: '9780132350884',
);
$this->assertSame($bookId, $service->updateBookInfo('Clean Code', '9780132350884'));
$book = BookInfo::query()->findOrFail($bookId);
$this->assertSame('legacy:clean-code', $book->asin);
$this->assertSame('Prentice Hall', $book->publisher);
$this->assertSame('Programming', $book->genre);
$this->assertFalse((bool) $book->cover);
$this->assertSame(1, BookInfo::query()->count());
}
public function test_two_word_unauthored_title_reaches_external_providers_but_is_not_loosely_accepted(): void
{
$isbnDb = Mockery::mock(IsbnDbService::class);
$isbnDb->shouldReceive('isConfigured')->andReturnFalse();
$google = Mockery::mock(GoogleBooksService::class);
$google->shouldReceive('searchBooks')->once()->andReturn([]);
$openLibrary = Mockery::mock(OpenLibraryService::class);
$openLibrary->shouldReceive('searchBooks')->once()->andReturn([]);
$itunes = Mockery::mock(ItunesService::class);
$itunes->shouldReceive('findEbooks')->once()->andReturn([]);
$itunes->shouldReceive('lastRequestFailed')->once()->andReturnFalse();
$service = $this->makeService($isbnDb, $google, $openLibrary, $itunes);
$service->parsedBookResult = new BookParseResult(
rawName: 'Atomic Habits EPUB',
title: 'Atomic Habits',
);
$this->assertSame(-2, $service->updateBookInfo('Atomic Habits'));
}
private function makeService(
IsbnDbService $isbnDb,
GoogleBooksService $googleBooks,
OpenLibraryService $openLibrary,
ItunesService $itunes,
): BookService {
$images = Mockery::mock(ReleaseImageService::class);
$images->shouldReceive('saveRemoteImage')
->zeroOrMoreTimes()
->andReturn(ImageProcessingResult::failure('No cover URL.'));
return new BookService(
isbnDb: $isbnDb,
googleBooks: $googleBooks,
openLibrary: $openLibrary,
itunes: $itunes,
scorer: new BookMatchScorer,
releaseImageService: $images,
);
}
private function cleanCodeParseResult(): BookParseResult
{
return new BookParseResult(
rawName: 'Clean Code by Robert C Martin',
title: 'Clean Code',
author: 'Robert C Martin',
year: 2008,
);
}
/**
* @return array<string, mixed>
*/
private function candidate(string $title, string $author, string $asin): array
{
return [
'title' => $title,
'author' => $author,
'asin' => $asin,
'isbn' => null,
'ean' => null,
'url' => '',
'salesrank' => '',
'publisher' => '',
'publishdate' => null,
'pages' => '',
'overview' => '',
'genre' => '',
'coverurl' => '',
'cover' => 0,
];
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
}
@@ -122,6 +122,46 @@ class PostProcessRunnerBooksGateTest extends TestCase
$this->assertStringContainsString('artisan postprocess:guid books a', $runner->captured[0]); $this->assertStringContainsString('artisan postprocess:guid books a', $runner->captured[0]);
} }
public function test_renamed_only_mode_skips_unrenamed_pending_books(): void
{
DB::table('settings')->where('name', 'lookupbooks')->update(['value' => '2']);
DB::table('releases')->insert([
'id' => 2,
'name' => 'A Plain Unrenamed Book',
'searchname' => 'A Plain Unrenamed Book',
'groups_id' => 1,
'size' => 1,
'postdate' => now(),
'adddate' => now(),
'guid' => str_repeat('b', 40),
'leftguid' => 'b',
'fromname' => 'poster@example.com',
'categories_id' => Category::BOOKS_EBOOK,
'bookinfo_id' => null,
'isrenamed' => 0,
]);
$runner = new class extends PostProcessRunner
{
public array $captured = [];
public function headerNone(): void {}
protected function headerStart(string $workType, int $count, int $maxProcesses): void {}
protected function executeCommand(string $command): string
{
$this->captured[] = $command;
return '';
}
};
$runner->processBooks();
$this->assertSame([], $runner->captured);
}
private function setEnvironmentValue(string $key, ?string $value): void private function setEnvironmentValue(string $key, ?string $value): void
{ {
if ($value === null) { if ($value === null) {
@@ -159,6 +199,7 @@ class PostProcessRunnerBooksGateTest extends TestCase
$table->string('fromname')->nullable(); $table->string('fromname')->nullable();
$table->integer('categories_id')->default(Category::OTHER_MISC); $table->integer('categories_id')->default(Category::OTHER_MISC);
$table->integer('bookinfo_id')->nullable(); $table->integer('bookinfo_id')->nullable();
$table->tinyInteger('isrenamed')->default(0);
}); });
} }
} }
+33
View File
@@ -32,6 +32,39 @@ class BookServiceTest extends TestCase
$this->assertSame('9780321125217', $parsed->isbn); $this->assertSame('9780321125217', $parsed->isbn);
} }
public function test_parse_release_name_parses_title_by_author_in_conventional_order(): void
{
$parsed = $this->makeService()->parseReleaseName('Clean Code by Robert C. Martin 2008 EPUB');
$this->assertSame('Clean Code', $parsed->title);
$this->assertSame('Robert C Martin', $parsed->author);
$this->assertSame(2008, $parsed->year);
}
public function test_parse_release_name_captures_year_before_removing_release_noise(): void
{
$parsed = $this->makeService()->parseReleaseName('The Pragmatic Programmer 1999 EPUB');
$this->assertSame('The Pragmatic Programmer', $parsed->title);
$this->assertSame(1999, $parsed->year);
}
public function test_parse_release_name_preserves_series_and_volume_evidence(): void
{
$parsed = $this->makeService()->parseReleaseName('Frank Herbert - Dune (Book 2) EPUB');
$this->assertSame('Dune (Book 2)', $parsed->title);
$this->assertSame('Frank Herbert', $parsed->author);
}
public function test_extract_isbn_ignores_invalid_check_digits(): void
{
$service = $this->makeService();
$this->assertNull($service->extractIsbn('Fake Book 978-0-13-235088-5 EPUB'));
$this->assertNull($service->extractIsbn('Fake Book 0132350883 EPUB'));
}
public function test_parse_release_name_flags_software_as_junk(): void public function test_parse_release_name_flags_software_as_junk(): void
{ {
$service = $this->makeService(); $service = $this->makeService();
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace Tests\Unit\Services; namespace Tests\Unit\Services;
use App\Exceptions\BookProviderException;
use App\Services\GoogleBooksService; use App\Services\GoogleBooksService;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\Handler\MockHandler;
@@ -61,4 +62,19 @@ class GoogleBooksServiceTest extends TestCase
$this->assertTrue($service->isConfigured()); $this->assertTrue($service->isConfigured());
$this->assertFalse($service->hasApiKey()); $this->assertFalse($service->hasApiKey());
} }
public function test_server_error_is_reported_as_provider_unavailability(): void
{
$service = new GoogleBooksService(
new Client([
'handler' => HandlerStack::create(new MockHandler([new Response(503)])),
'http_errors' => false,
]),
null
);
$this->expectException(BookProviderException::class);
$service->searchBooks('unavailable query');
}
} }
+169
View File
@@ -4,16 +4,28 @@ declare(strict_types=1);
namespace Tests\Unit\Services; namespace Tests\Unit\Services;
use App\Exceptions\BookProviderException;
use App\Services\BookService; use App\Services\BookService;
use App\Services\IsbnDbService; use App\Services\IsbnDbService;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack; use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Response;
use Illuminate\Support\Facades\Cache;
use Tests\TestCase; use Tests\TestCase;
class IsbnDbServiceTest extends TestCase class IsbnDbServiceTest extends TestCase
{ {
protected function setUp(): void
{
parent::setUp();
Cache::flush();
}
public function test_is_configured_returns_false_when_key_is_missing(): void public function test_is_configured_returns_false_when_key_is_missing(): void
{ {
$service = new IsbnDbService(null, ''); $service = new IsbnDbService(null, '');
@@ -67,6 +79,163 @@ class IsbnDbServiceTest extends TestCase
$this->assertSame('isbndb:9780321125217', $book['asin']); $this->assertSame('isbndb:9780321125217', $book['asin']);
} }
public function test_search_books_uses_the_documented_title_and_match_parameters(): void
{
$mock = new MockHandler([
new Response(200, [], json_encode(['data' => []])),
]);
$history = [];
$handler = HandlerStack::create($mock);
$handler->push(Middleware::history($history));
$service = new IsbnDbService(
new Client(['handler' => $handler, 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
$this->assertSame([], $service->searchBooks('Clean Code', 'Robert C Martin', true));
$this->assertCount(1, $history);
parse_str($history[0]['request']->getUri()->getQuery(), $query);
$this->assertArrayNotHasKey('author', $query);
$this->assertSame('title', $query['column']);
$this->assertSame('true', $query['shouldMatchAll']);
$this->assertSame('10', $query['pageSize']);
}
public function test_successful_empty_search_is_cached(): void
{
$mock = new MockHandler([
new Response(200, [], json_encode(['data' => []])),
]);
$history = [];
$handler = HandlerStack::create($mock);
$handler->push(Middleware::history($history));
$service = new IsbnDbService(
new Client(['handler' => $handler, 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
$this->assertSame([], $service->searchBooks('Unique Empty Book Query'));
$this->assertSame([], $service->searchBooks('Unique Empty Book Query'));
$this->assertCount(1, $history);
}
public function test_rate_limit_response_throws_provider_exception_and_sets_cooldown(): void
{
$mock = new MockHandler([
new Response(429, ['Retry-After' => '120']),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
try {
$service->searchBooks('Rate Limited Query');
$this->fail('Expected the ISBNdb request to report provider unavailability.');
} catch (BookProviderException $exception) {
$this->assertSame('isbndb', $exception->provider);
$this->assertSame(429, $exception->statusCode);
$this->assertSame(120, $exception->retryAfterSeconds);
}
$this->assertTrue(Cache::has(IsbnDbService::COOLDOWN_CACHE_KEY));
}
public function test_malformed_success_response_throws_provider_exception(): void
{
$mock = new MockHandler([
new Response(200, [], '{not-json'),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
$this->expectException(BookProviderException::class);
$service->searchBooks('Malformed Response Query');
}
public function test_success_response_without_documented_data_envelope_is_retryable(): void
{
$mock = new MockHandler([
new Response(200, [], json_encode(['books' => []])),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
$this->expectException(BookProviderException::class);
$service->searchBooks('Missing Data Envelope Query');
}
public function test_authentication_failure_is_distinguished_without_transient_cooldown(): void
{
$mock = new MockHandler([
new Response(401),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
try {
$service->searchBooks('Authentication Failure Query');
$this->fail('Expected an authentication provider exception.');
} catch (BookProviderException $exception) {
$this->assertSame(401, $exception->statusCode);
$this->assertNull($exception->retryAfterSeconds);
}
$this->assertFalse(Cache::has(IsbnDbService::COOLDOWN_CACHE_KEY));
}
public function test_server_failure_applies_default_cooldown(): void
{
$mock = new MockHandler([
new Response(503),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
try {
$service->searchBooks('Server Failure Query');
$this->fail('Expected a transient provider exception.');
} catch (BookProviderException $exception) {
$this->assertSame(503, $exception->statusCode);
$this->assertSame(300, $exception->retryAfterSeconds);
}
$this->assertTrue(Cache::has(IsbnDbService::COOLDOWN_CACHE_KEY));
}
public function test_network_failure_applies_default_cooldown(): void
{
$request = new Request('GET', 'https://api2.isbndb.com/books/Network');
$mock = new MockHandler([
new ConnectException('Connection failed', $request),
]);
$service = new IsbnDbService(
new Client(['handler' => HandlerStack::create($mock), 'base_uri' => 'https://api2.isbndb.com']),
'test-key'
);
try {
$service->searchBooks('Network Failure Query');
$this->fail('Expected a network provider exception.');
} catch (BookProviderException $exception) {
$this->assertNull($exception->statusCode);
$this->assertSame(300, $exception->retryAfterSeconds);
}
$this->assertTrue(Cache::has(IsbnDbService::COOLDOWN_CACHE_KEY));
}
public function test_find_by_isbn_reads_book_payload(): void public function test_find_by_isbn_reads_book_payload(): void
{ {
$mock = new MockHandler([ $mock = new MockHandler([
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace Tests\Unit\Services; namespace Tests\Unit\Services;
use App\Exceptions\BookProviderException;
use App\Services\OpenLibraryService; use App\Services\OpenLibraryService;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\Handler\MockHandler;
@@ -66,4 +67,17 @@ class OpenLibraryServiceTest extends TestCase
$this->assertSame('Robert C. Martin', $book['author']); $this->assertSame('Robert C. Martin', $book['author']);
$this->assertSame('9780134494166', $book['isbn']); $this->assertSame('9780134494166', $book['isbn']);
} }
public function test_malformed_search_response_is_reported_as_provider_unavailability(): void
{
$service = new OpenLibraryService(
new Client(['handler' => HandlerStack::create(new MockHandler([
new Response(200, [], '{invalid-json'),
]))])
);
$this->expectException(BookProviderException::class);
$service->searchBooks('malformed query');
}
} }
+35
View File
@@ -0,0 +1,35 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Support;
use App\Support\BookIsbn;
use PHPUnit\Framework\TestCase;
class BookIsbnTest extends TestCase
{
public function test_normalizes_and_validates_isbn_10_and_isbn_13(): void
{
$this->assertSame('0132350882', BookIsbn::normalize('0-13-235088-2'));
$this->assertSame('9780132350884', BookIsbn::normalize('978-0-13-235088-4'));
$this->assertNull(BookIsbn::normalize('0132350883'));
$this->assertNull(BookIsbn::normalize('9780132350885'));
}
public function test_converts_and_compares_equivalent_isbn_versions(): void
{
$this->assertSame('9780132350884', BookIsbn::toIsbn13('0132350882'));
$this->assertSame('0132350882', BookIsbn::toIsbn10('9780132350884'));
$this->assertTrue(BookIsbn::equivalent('0132350882', '9780132350884'));
$this->assertFalse(BookIsbn::equivalent('0132350882', '9780321125217'));
}
public function test_returns_all_equivalent_identifiers_without_duplicates(): void
{
$this->assertSame(
['0132350882', '9780132350884'],
BookIsbn::equivalents('9780132350884')
);
}
}
+148
View File
@@ -31,6 +31,154 @@ class BookMatchScorerTest extends TestCase
$this->assertSame(1.0, $score); $this->assertSame(1.0, $score);
} }
public function test_returns_perfect_score_for_equivalent_isbn_10_candidate(): void
{
$parsed = new BookParseResult(
rawName: 'Clean Code 9780132350884',
title: 'Clean Code',
isbn: '9780132350884'
);
$score = (new BookMatchScorer)->score([
'title' => 'Clean Code',
'author' => 'Robert C. Martin',
'isbn' => null,
'ean' => '0132350882',
], $parsed);
$this->assertSame(1.0, $score);
}
public function test_rejects_candidate_with_conflicting_explicit_isbn(): void
{
$parsed = new BookParseResult(
rawName: 'Clean Code 9780132350884',
title: 'Clean Code',
author: 'Robert C. Martin',
isbn: '9780132350884'
);
$score = (new BookMatchScorer)->score([
'title' => 'Clean Code',
'author' => 'Robert C. Martin',
'isbn' => '9780321125217',
'ean' => '0321125215',
'publisher' => 'Excellent Publisher',
'cover' => 1,
], $parsed);
$this->assertSame(0.0, $score);
}
public function test_rejects_candidate_when_only_one_of_two_explicit_identifiers_matches(): void
{
$parsed = new BookParseResult(
rawName: 'Clean Code 9780132350884',
title: 'Clean Code',
isbn: '9780132350884'
);
$score = (new BookMatchScorer)->score([
'title' => 'Clean Code',
'author' => 'Robert C. Martin',
'isbn' => '9780132350884',
'ean' => '0321125215',
], $parsed);
$this->assertSame(0.0, $score);
}
public function test_author_matching_is_order_insensitive_and_supports_initials(): void
{
$parsed = new BookParseResult(
rawName: 'Clean Code by Robert C Martin',
title: 'Clean Code',
author: 'Robert C Martin'
);
$score = (new BookMatchScorer)->score([
'title' => 'Clean Code',
'author' => 'Martin, Robert C.',
], $parsed);
$this->assertGreaterThanOrEqual(0.82, $score);
}
public function test_cover_and_publisher_do_not_increase_identity_score(): void
{
$parsed = new BookParseResult(
rawName: 'Completely Different Book',
title: 'Completely Different Book'
);
$scorer = new BookMatchScorer;
$withoutMetadata = $scorer->score([
'title' => 'Another Unrelated Work',
'publisher' => '',
'cover' => 0,
], $parsed);
$withMetadata = $scorer->score([
'title' => 'Another Unrelated Work',
'publisher' => 'Publisher',
'cover' => 1,
], $parsed);
$this->assertSame($withoutMetadata, $withMetadata);
}
public function test_rejects_conflicting_volume_numbers(): void
{
$parsed = new BookParseResult(
rawName: 'Dune Book 2',
title: 'Dune Book 2',
author: 'Frank Herbert'
);
$score = (new BookMatchScorer)->score([
'title' => 'Dune Book 3',
'author' => 'Frank Herbert',
], $parsed);
$this->assertSame(0.0, $score);
}
public function test_rejects_conflicting_edition_numbers(): void
{
$parsed = new BookParseResult(
rawName: 'Effective Java 3rd Edition',
title: 'Effective Java 3rd Edition',
author: 'Joshua Bloch'
);
$score = (new BookMatchScorer)->score([
'title' => 'Effective Java 2nd Edition',
'author' => 'Joshua Bloch',
], $parsed);
$this->assertSame(0.0, $score);
}
public function test_multilingual_punctuation_normalizes_without_metadata_boosts(): void
{
$parsed = new BookParseResult(
rawName: 'Cien años de soledad by Gabriel García Márquez',
title: 'Cien años de soledad',
author: 'Gabriel García Márquez'
);
$score = (new BookMatchScorer)->score([
'title' => 'Cien años de soledad!',
'author' => 'Márquez, Gabriel García',
], $parsed);
$this->assertSame(1.0, $score);
}
public function test_generic_stop_words_do_not_count_as_significant_title_evidence(): void
{
$this->assertSame(2, (new BookMatchScorer)->meaningfulTitleTokenCount('The Art of War'));
}
public function test_higher_score_for_better_title_author_match(): void public function test_higher_score_for_better_title_author_match(): void
{ {
$parsed = new BookParseResult( $parsed = new BookParseResult(