starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve book matching

Browse Source
This commit is contained in:
DariusIII
2026-08-13 15:53:31 +02:00
parent 658937bee1
commit 992a0007f6
21 changed files with 2070 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Exceptions/BookProviderException.php
+21
View File
@@ -0,0 +1,21 @@
<?php
declare(strict_types=1);
namespace App\Exceptions;
use RuntimeException;
use Throwable;
class BookProviderException extends RuntimeException
{
public function __construct(
public readonly string $provider,
string $message,
public readonly ?int $statusCode = null,
public readonly ?int $retryAfterSeconds = null,
?Throwable $previous = null,
) {
parent::__construct($message, $statusCode ?? 0, $previous);
}
}
app/Services/BookService.php
+525 -217
View File
@@ -6,6 +6,7 @@ namespace App\Services;
use App\Enums\ImageAssetProfile;
use App\Enums\SecondarySearchIndex;
use App\Exceptions\BookProviderException;
use App\Facades\Search;
use App\Models\BookInfo;
use App\Models\Category;
@@ -13,6 +14,7 @@ use App\Models\Release;
use App\Models\Settings;
use App\Services\NameFixing\Extractors\ObfuscatedSubjectExtractor;
use App\Services\Releases\ReleaseBrowseService;
use App\Support\BookIsbn;
use App\Support\BookMatchScorer;
use App\Support\Data\BookParseResult;
use App\Support\MetadataSearchLookup;
@@ -26,6 +28,8 @@ use Illuminate\Support\Facades\Log;
*/
class BookService
{
private const FAILURE_CACHE_VERSION = 2;
public bool $echooutput;
public int $bookqty;
@@ -42,11 +46,29 @@ class BookService
private ObfuscatedSubjectExtractor $obfuscatedSubjectExtractor;
private IsbnDbService $isbnDb;
private GoogleBooksService $googleBooks;
private OpenLibraryService $openLibrary;
private ItunesService $itunes;
private BookMatchScorer $scorer;
private ReleaseImageService $releaseImageService;
/**
* @throws \Exception
*/
public function __construct()
{
public function __construct(
?IsbnDbService $isbnDb = null,
?GoogleBooksService $googleBooks = null,
?OpenLibraryService $openLibrary = null,
?ItunesService $itunes = null,
?BookMatchScorer $scorer = null,
?ReleaseImageService $releaseImageService = null,
) {
$this->echooutput = config('nntmux.echocli');
$this->bookqty = Settings::settingValue('maxbooksprocessed') !== '' ? (int) Settings::settingValue('maxbooksprocessed') : 300;
@@ -58,6 +80,12 @@ class BookService
$this->parsedIsbn = null;
$this->parsedBookResult = null;
$this->obfuscatedSubjectExtractor = new ObfuscatedSubjectExtractor;
$this->isbnDb = $isbnDb ?? new IsbnDbService;
$this->googleBooks = $googleBooks ?? new GoogleBooksService;
$this->openLibrary = $openLibrary ?? new OpenLibraryService;
$this->itunes = $itunes ?? new ItunesService;
$this->scorer = $scorer ?? new BookMatchScorer;
$this->releaseImageService = $releaseImageService ?? new ReleaseImageService;
}
/**
@@ -118,6 +146,21 @@ class BookService
return $this->pickBestExistingBook($results->all(), $parsed);
}
public function getBookInfoByIsbn(?string $isbn): ?BookInfo
{
$identifiers = BookIsbn::equivalents($isbn);
if ($identifiers === []) {
return null;
}
return BookInfo::query()
->where(static function ($query) use ($identifiers): void {
$query->whereIn('isbn', $identifiers)
->orWhereIn('ean', $identifiers);
})
->first();
}
/**
* Get book range with pagination.
*
@@ -386,6 +429,10 @@ class BookService
$query->where('groups_id', $groupID);
}
if ($this->renamed !== '') {
$query->where('isrenamed', 1);
}
$this->processBookReleasesHelper(
$query->get(['searchname', 'id', 'categories_id']),
sprintf(
@@ -406,8 +453,10 @@ class BookService
->orWhere('categories_id', Category::MUSIC_AUDIOBOOK);
})
->where(function ($builder): void {
$builder->where('isrenamed', 0)
->orWhere('searchname', 'like', 'N:/NZB%')
if ($this->renamed === '') {
$builder->where('isrenamed', 0);
}
$builder->orWhere('searchname', 'like', 'N:/NZB%')
->orWhere('searchname', 'like', 'N_NZB_%')
->orWhere('name', 'like', 'N:/NZB%')
->orWhere('name', 'like', 'N_NZB_%');
@@ -453,8 +502,8 @@ class BookService
cli()->header('Processing '.$res->count().' book release(s) for categories id '.$categoryID);
}
$bookId = -2;
foreach ($res as $arr) {
$bookId = -2;
$startTime = now()->timestamp;
$usedExternalApi = false;
// audiobooks are also books and should be handled in an identical manor, even though it falls under a music category
@@ -471,9 +520,10 @@ class BookService
cli()->info('Looking up: '.$bookInfo);
}
// Do a local lookup first
$bookCheck = $this->getBookInfoByName($bookInfo, $this->parsedBookResult);
$failCacheKey = 'book_lookup_fail_'.md5(strtolower($bookInfo));
// Prefer an exact local identifier before any fuzzy title match.
$bookCheck = $this->getBookInfoByIsbn($this->parsedIsbn)
?? $this->getBookInfoByName($bookInfo, $this->parsedBookResult);
$failCacheKey = $this->failureCacheKey($this->parsedBookResult, $bookInfo);
if ($bookCheck === null && Cache::has($failCacheKey)) {
// Lookup recently failed, no point trying again
@@ -485,8 +535,8 @@ class BookService
$bookId = $this->updateBookInfo($bookInfo, $this->parsedIsbn);
$usedExternalApi = true;
if ($bookId === -2) {
Cache::put($failCacheKey, true, now()->addDays(7));
} else {
Cache::put($failCacheKey, true, now()->addDay());
} elseif ($bookId !== null) {
Cache::forget($failCacheKey);
}
} else {
@@ -554,9 +604,12 @@ class BookService
return false;
}
if (! empty($releasename) && ! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)) {
if (! empty($releasename)
&& (! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)
|| $parsed->hasAuthor()
|| $parsed->isbn !== null)) {
$wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []);
if ($wordCount >= 2) {
if ($wordCount >= 2 || $parsed->hasAuthor() || $parsed->isbn !== null) {
return $parsed->searchQuery();
}
}
@@ -568,9 +621,12 @@ class BookService
return false;
}
if (! empty($releasename) && ! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)) {
if (! empty($releasename)
&& (! preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $releasename)
|| $parsed->hasAuthor()
|| $parsed->isbn !== null)) {
$wordCount = count(preg_split('/\s+/', trim($releasename)) ?: []);
if ($wordCount >= 2) {
if ($wordCount >= 2 || $parsed->hasAuthor() || $parsed->isbn !== null) {
return $parsed->searchQuery();
}
}
@@ -589,6 +645,10 @@ class BookService
$releaseName = $quotedMatch[1];
}
$isbn = $this->extractIsbn($releaseName);
$year = null;
if (preg_match('/\b(19|20)\d{2}\b/', $releaseName, $yearMatch) === 1) {
$year = (int) $yearMatch[0];
}
$a = preg_replace('/\d{1,2} \d{1,2} \d{2,4}|(19|20)\d\d|anybody got .+?[a-z]\? |[ ._-](Novel|TIA)([ ._-]|$)|([ \.])HQ([-\. ])|[\(\)\.\-_ ](AVI|AZW3?|DOC|EPUB|LIT|MOBI|NFO|RETAIL|(si)?PDF|RTF|TXT)[\)\]\.\-_ ](?![a-z0-9])|compleet|DAGSTiDNiNGEN|DiRFiX|\+ extra|r?e ?Books?([\.\-_ ]English|ers)?|azw3?|ePu([bp])s?|html|mobi|^NEW[\.\-_ ]|PDF([\.\-_ ]English)?|Please post more|Post description|Proper|Repack(fix)?|[\.\-_ ](Chinese|English|French|German|Italian|Retail|Scan|Swedish|Multilingual)|^R4 |Repost|Skytwohigh|TIA!+|TruePDF|V413HAV|(would someone )?please (re)?post.+? "|with the authors name right/i', '', $releaseName);
$b = preg_replace('/^(As Req |conversion |eq |Das neue Abenteuer \d+|Fixed version( ignore previous post)?|Full |Per Req As Found|(\s+)?R4 |REQ |revised |version |\d+(\s+)?$)|(COMPLETE|INTERNAL|RELOADED| (AZW3|eB|docx|ENG?|exe|FR|Fix|gnv64|MU|NIV|R\d\s+\d{1,2} \d{1,2}|R\d|Req|TTL|UC|v(\s+)?\d))(\s+)?$/i', '', (string) $a);
$c = preg_replace('/ - \[.+\]|\[.+\]/', '', (string) $b);
@@ -599,7 +659,6 @@ class BookService
$normalized = (string) preg_replace('/\b(S\d{1,3}E\d{1,3}|Season\s*\d+|Temporada\s*\d+|Saison\s*\d+|Staffel\s*\d+|Episode\s*\d+|HDTV|WEB[\s-]?DL|WEBRip|BluRay|BDRip|DVDRip|BRRip|XviD|[xh][\s.]?26[45]|HEVC|10bit|1080[pi]|720p|2160p|4K|UHD|HDR|REMUX|AAC5?\s*\d|DDP?5\s*\d|ATMOS|PAR2?|vol\d+\+\d+|NTb|FLUX|PSA|RARBG|YTS|YIFY|AMZN|DSNP|NF|ATVP|HMAX)\b/i', '', $normalized);
$normalized = trim((string) preg_replace('/\s+/', ' ', $normalized));
$year = null;
$specialMagazineTitle = null;
if (preg_match('/^MCN[._ -](?<month>[A-Za-z]+)[._ -](?<day>\d{1,2})[._ -](?<year>(?:19|20)\d{2})/i', $rawReleaseName, $mcnMatch) === 1) {
$month = ucfirst(strtolower($mcnMatch['month']));
@@ -627,22 +686,15 @@ class BookService
$author = $candidateAuthor;
$title = $candidateTitle;
}
} elseif (preg_match('/^(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})\s+by\s+(?<title>.+)$/i', $normalized, $matches) === 1) {
$author = trim($matches['author']);
$title = trim($matches['title']);
} elseif (preg_match('/^(?<title>.+?)\s+by\s+(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})$/i', $normalized, $matches) === 1) {
$author = trim($matches['author']);
$title = trim($matches['title']);
}
$title = trim((string) preg_replace('/\s+/', ' ', (string) preg_replace('/\((Book|Series)\s*\d+\)/i', '', $title)));
$title = trim((string) preg_replace('/\s+/', ' ', $title));
if ($author === null && preg_match('/^(?<title>.+?)[\s._-]+(?<year>(19|20)\d{2})[\s._-]+(?<format>EPUB|MOBI|AZW3?|PDF|FB2|DJVU|LIT)$/i', $normalized, $matches) === 1) {
$title = trim(str_replace(['.', '_'], ' ', $matches['title']));
}
if ($author === null && preg_match('/^(?<title>.+?)\s+Vol\.?\s*\d{1,3}$/i', $title, $matches) === 1) {
$title = trim($matches['title']);
}
$isJunk = false;
$junkPattern = '/^([a-z0-9] )+$|ArtofUsenet|ekiosk|(ebook|mobi).+collection|erotica|Full Video|ImwithJamie|linkoff org|Mega.+pack|^[a-z0-9]+ (?!((January|February|March|April|May|June|July|August|September|O([ck])tober|November|De([cz])ember)))[a-z]+( (ebooks?|The))?$|NY Times|(Book|Massive) Dump|Sexual/i';
if (preg_match($junkPattern, $normalized) === 1) {
@@ -694,16 +746,15 @@ class BookService
{
if (preg_match('/\b(97[89](?:[-\s]?\d){10})\b/', $releaseName, $matches) === 1) {
$isbn = preg_replace('/[^0-9]/', '', $matches[1]);
if (is_string($isbn) && strlen($isbn) === 13) {
return $isbn;
if (is_string($isbn)) {
return BookIsbn::normalize($isbn);
}
}
if (preg_match('/\b(\d(?:[-\s]?\d){8}[-\s]?[\dXx])\b/', $releaseName, $matches) === 1) {
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $matches[1]));
if (strlen($isbn) === 10) {
return $isbn;
}
return BookIsbn::normalize($isbn);
}
return null;
@@ -745,185 +796,160 @@ class BookService
/**
* Update book info from external sources.
*
* @return false|int|string
*
* @throws \Exception
*/
public function updateBookInfo(string $bookInfo = '', ?string $isbn = null)
public function updateBookInfo(string $bookInfo = '', ?string $isbn = null): ?int
{
$ri = new ReleaseImageService;
$isbnDb = new IsbnDbService;
$googleBooks = new GoogleBooksService;
$openLibrary = new OpenLibraryService;
$itunes = new ItunesService;
$scorer = new BookMatchScorer;
$bookId = -2;
$book = null;
if ($bookInfo !== '') {
$wordCount = count(array_filter(preg_split('/\s+/', trim($bookInfo)) ?: []));
if ($wordCount < 3) {
return -2;
}
$parsed = $this->parsedBookResult ?? $this->parseReleaseName($bookInfo);
$candidates = [];
$resolvedSource = null;
if ($isbnDb->isConfigured()) {
if ($isbn !== null && $isbn !== '') {
cli()->info('Fetching data from ISBNdb by ISBN '.$isbn);
$candidate = $isbnDb->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'isbndb_isbn';
}
}
if ($candidates === []) {
cli()->info('Fetching data from ISBNdb for '.$bookInfo);
$candidates = array_merge($candidates, $isbnDb->searchBooks($bookInfo));
if ($candidates !== []) {
$resolvedSource = 'isbndb_search';
}
}
}
if ($candidates === []) {
if ($isbn !== null && $isbn !== '') {
cli()->info('Fetching data from Google Books by ISBN '.$isbn);
$candidate = $googleBooks->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'google_books_isbn';
}
}
}
if ($candidates === []) {
cli()->info('Fetching data from Google Books for '.$bookInfo);
$candidates = array_merge(
$candidates,
$googleBooks->searchBooks(
$bookInfo,
$parsed->title,
$parsed->author,
$isbn
)
);
if ($candidates !== []) {
$resolvedSource = 'google_books_search';
}
}
if ($candidates === []) {
cli()->info('Fetching data from iTunes for '.$bookInfo);
$candidates = array_merge($candidates, $this->fetchItunesBookProperties($bookInfo, $itunes));
if ($candidates !== []) {
$resolvedSource = 'itunes_search';
}
}
if ($candidates === []) {
cli()->info('Fetching data from Open Library for '.$bookInfo);
if ($isbn !== null && $isbn !== '') {
$candidate = $openLibrary->findByIsbn($isbn);
if (is_array($candidate)) {
$candidates[] = $candidate;
$resolvedSource = 'open_library_isbn';
}
}
if ($candidates === []) {
$candidates = array_merge($candidates, $openLibrary->searchBooks($bookInfo));
if ($candidates !== []) {
$resolvedSource = 'open_library_search';
}
}
}
$book = $this->pickBestCandidate($candidates, $parsed, $scorer);
if (is_array($book)) {
Log::debug('Book metadata source resolved', [
'book_info' => $bookInfo,
'source' => $resolvedSource,
'isbn' => $isbn,
]);
}
}
if (empty($book)) {
$bookInfo = trim($bookInfo);
if ($bookInfo === '') {
return -2;
}
$check = BookInfo::query()->where('asin', $book['asin'])->first();
if ($check === null && ! empty($book['isbn'])) {
$check = BookInfo::query()->where('isbn', $book['isbn'])->first();
}
if ($check === null) {
$bookId = BookInfo::query()->insertGetId(
[
'title' => $book['title'],
'author' => $book['author'],
'asin' => $book['asin'],
'isbn' => $book['isbn'],
'ean' => $book['ean'],
'url' => $book['url'],
'salesrank' => $book['salesrank'],
'publisher' => $book['publisher'],
'publishdate' => $book['publishdate'],
'pages' => $book['pages'],
'overview' => $book['overview'],
'genre' => $book['genre'],
'cover' => $book['cover'],
'created_at' => now(),
'updated_at' => now(),
]
);
} else {
$bookId = $check['id'];
BookInfo::query()->where('id', $bookId)->update(
[
'title' => $book['title'],
'author' => $book['author'],
'asin' => $book['asin'],
'isbn' => $book['isbn'],
'ean' => $book['ean'],
'url' => $book['url'],
'salesrank' => $book['salesrank'],
'publisher' => $book['publisher'],
'publishdate' => $book['publishdate'],
'pages' => $book['pages'],
'overview' => $book['overview'],
'genre' => $book['genre'],
'cover' => $book['cover'],
]
);
$parsed = $this->parsedBookResult ?? $this->parseReleaseName($bookInfo);
$isbn = BookIsbn::normalize($isbn ?? $parsed->isbn);
if ($isbn === null && ! $parsed->hasAuthor() && $this->scorer->meaningfulTitleTokenCount($parsed->title) < 2) {
return -2;
}
if ($bookId && $bookId !== -2) {
if ($this->echooutput) {
cli()->header('Added/updated book: ');
if ($book['author'] !== '') {
cli()->alternateOver(' Author: ').cli()->primary($book['author']);
$candidates = [];
$book = null;
$hadProviderFailure = false;
if ($this->isbnDb->isConfigured()) {
try {
if ($isbn !== null) {
cli()->info('Fetching data from ISBNdb by ISBN '.$isbn);
$candidate = $this->isbnDb->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'isbndb_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
}
cli()->alternateOver(' Title: ').cli()->primary(' '.$book['title']);
if ($book['genre'] !== '') {
cli()->alternateOver(' Genre: ').cli()->primary(' '.$book['genre']);
if ($book === null) {
cli()->info('Fetching data from ISBNdb for '.$parsed->title);
$this->appendCandidates(
$candidates,
$this->isbnDb->searchBooks($parsed->title, $parsed->author, true),
'isbndb_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
cli()->info('Retrying ISBNdb with title-only hypothesis '.$titleOnlyHypothesis->title);
$this->appendCandidates(
$candidates,
$this->isbnDb->searchBooks($titleOnlyHypothesis->title, null, true),
'isbndb_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
}
$book['cover'] = (int) $ri->saveRemoteImage(
(string) $bookId,
$book['coverurl'],
$this->imgSavePath,
ImageAssetProfile::MetadataCover,
)->success;
} elseif ($this->echooutput) {
cli()->header('Nothing to update: ').
cli()->header($book['author'].
' - '.
$book['title']);
}
if ($book === null) {
try {
if ($isbn !== null) {
cli()->info('Fetching data from Google Books by ISBN '.$isbn);
$candidate = $this->googleBooks->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'google_books_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
}
if ($book === null) {
cli()->info('Fetching data from Google Books for '.$parsed->title);
$this->appendCandidates(
$candidates,
$this->googleBooks->searchBooks($parsed->searchQuery(), $parsed->title, $parsed->author),
'google_books_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
$this->appendCandidates(
$candidates,
$this->googleBooks->searchBooks(
$titleOnlyHypothesis->title,
$titleOnlyHypothesis->title,
null,
),
'google_books_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
}
}
if ($book === null) {
try {
cli()->info('Fetching data from Open Library for '.$parsed->title);
if ($isbn !== null) {
$candidate = $this->openLibrary->findByIsbn($isbn);
if (is_array($candidate)) {
$this->appendCandidates($candidates, [$candidate], 'open_library_isbn');
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
}
if ($book === null) {
$this->appendCandidates(
$candidates,
$this->openLibrary->searchBooks($parsed->searchQuery()),
'open_library_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
$titleOnlyHypothesis = $this->titleOnlyHypothesis($parsed);
if ($book === null && $titleOnlyHypothesis !== null) {
$this->appendCandidates(
$candidates,
$this->openLibrary->searchBooks($titleOnlyHypothesis->title),
'open_library_title_fallback',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
}
} catch (BookProviderException $exception) {
$hadProviderFailure = true;
$this->logProviderFailure($exception, $bookInfo);
}
}
if ($book === null) {
cli()->info('Fetching data from iTunes for '.$parsed->title);
$itunesCandidates = $this->fetchItunesBookProperties($parsed->searchQuery(), $this->itunes);
$this->appendCandidates(
$candidates,
$itunesCandidates,
'itunes_search',
);
$book = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
if ($itunesCandidates === [] && $this->itunes->lastRequestFailed()) {
$hadProviderFailure = true;
}
}
if ($book === null) {
return $hadProviderFailure ? null : -2;
}
Log::debug('Book metadata source resolved', [
'book_info' => $bookInfo,
'source' => $book['_source'] ?? null,
'isbn' => $isbn,
]);
$bookId = $this->persistBookCandidate($book);
$this->outputBookMatch($book);
return $bookId;
}
@@ -980,18 +1006,28 @@ class BookService
return $books[0];
}
$scorer = new BookMatchScorer;
$best = null;
$bestScore = 0.0;
$candidates = [];
foreach ($books as $book) {
$score = $scorer->scoreBookInfo($book, $parsed);
if ($score > $bestScore) {
$best = $book;
$bestScore = $score;
}
$candidates[] = [
'_book_id' => $book->id,
'title' => $book->title,
'author' => $book->author,
'isbn' => $book->isbn,
'ean' => $book->ean,
'publishdate' => $book->publishdate,
'publisher' => $book->publisher,
'cover' => $book->cover ? 1 : 0,
];
}
return $bestScore >= $this->minimumMatchScore($parsed) ? $best : null;
$best = $this->pickBestCandidate($candidates, $parsed, $this->scorer);
if ($best === null) {
return null;
}
$bookId = (int) ($best['_book_id'] ?? 0);
return $bookId > 0 ? BookInfo::query()->find($bookId) : null;
}
/**
@@ -1004,30 +1040,302 @@ class BookService
return null;
}
$best = null;
$bestScore = 0.0;
foreach ($candidates as $candidate) {
$score = $scorer->score($candidate, $parsed);
if ($score > $bestScore) {
$bestScore = $score;
$best = $candidate;
foreach ($this->matchingHypotheses($parsed) as $hypothesis) {
$best = $this->pickBestCandidateForParse($candidates, $hypothesis, $scorer);
if ($best !== null) {
return $best;
}
}
if ($best === null || $bestScore < $this->minimumMatchScore($parsed)) {
return null;
}
/**
* @param array<int, array<string, mixed>> $candidates
* @return array<string, mixed>|null
*/
private function pickBestCandidateForParse(
array $candidates,
BookParseResult $parsed,
BookMatchScorer $scorer,
): ?array {
/** @var array<string, array{candidate: array<string, mixed>, score: float, year_tie: int, completeness: int}> $byWork */
$byWork = [];
foreach ($candidates as $candidate) {
$score = $scorer->score($candidate, $parsed);
$key = $scorer->workKey($candidate);
if ($key === '|') {
$key = (string) ($candidate['asin'] ?? md5((string) json_encode($candidate)));
}
$completeness = $this->candidateCompleteness($candidate);
$yearTie = $this->candidateYearTie($candidate, $parsed);
$existing = $byWork[$key] ?? null;
if ($existing === null
|| $score > $existing['score']
|| ($score === $existing['score'] && $yearTie > $existing['year_tie'])
|| ($score === $existing['score']
&& $yearTie === $existing['year_tie']
&& $completeness > $existing['completeness'])) {
$byWork[$key] = [
'candidate' => $candidate,
'score' => $score,
'year_tie' => $yearTie,
'completeness' => $completeness,
];
}
}
$ranked = array_values($byWork);
usort($ranked, static function (array $left, array $right): int {
$scoreComparison = $right['score'] <=> $left['score'];
return $scoreComparison !== 0
? $scoreComparison
: (($right['year_tie'] <=> $left['year_tie'])
?: ($right['completeness'] <=> $left['completeness']));
});
$best = $ranked[0] ?? null;
if ($best === null || ! $this->candidatePassesIdentityRules(
$best['candidate'],
$best['score'],
$ranked[1]['score'] ?? null,
$parsed,
$scorer,
)) {
return null;
}
return $best;
return $best['candidate'];
}
private function minimumMatchScore(?BookParseResult $parsed): float
/**
* @return list<BookParseResult>
*/
private function matchingHypotheses(BookParseResult $parsed): array
{
if ($parsed === null) {
return 0.55;
$hypotheses = [$parsed];
$titleOnly = $this->titleOnlyHypothesis($parsed);
if ($titleOnly !== null) {
$hypotheses[] = $titleOnly;
}
// No-author parses are more ambiguous and need a stricter cutoff.
return $parsed->hasAuthor() ? 0.55 : 0.68;
return $hypotheses;
}
private function titleOnlyHypothesis(BookParseResult $parsed): ?BookParseResult
{
if (! $parsed->hasAuthor() || ! str_contains($parsed->rawName, ' - ')) {
return null;
}
return new BookParseResult(
rawName: $parsed->rawName,
title: trim($parsed->author.' - '.$parsed->title),
author: null,
isbn: $parsed->isbn,
year: $parsed->year,
format: $parsed->format,
isJunk: $parsed->isJunk,
isMagazine: $parsed->isMagazine,
);
}
/**
* @param array<string, mixed> $candidate
*/
private function candidatePassesIdentityRules(
array $candidate,
float $score,
?float $runnerUpScore,
BookParseResult $parsed,
BookMatchScorer $scorer,
): bool {
if ($score === 1.0 && $parsed->isbn !== null) {
return true;
}
$titleScore = $scorer->titleScore($parsed->title, (string) ($candidate['title'] ?? ''));
if ($parsed->hasAuthor()) {
$authorScore = $scorer->authorScore((string) $parsed->author, (string) ($candidate['author'] ?? ''));
if ($titleScore < 0.78 || $authorScore < 0.75 || $score < 0.82) {
return false;
}
return $runnerUpScore === null || ($score - $runnerUpScore) >= 0.08;
}
if ($scorer->meaningfulTitleTokenCount($parsed->title) < 3 || $titleScore < 0.92) {
return false;
}
return $runnerUpScore === null || ($score - $runnerUpScore) >= 0.12;
}
/**
* @param array<int, array<string, mixed>> $target
* @param array<int, array<string, mixed>> $incoming
*/
private function appendCandidates(array &$target, array $incoming, string $source): void
{
foreach ($incoming as $candidate) {
$candidate['_source'] = $source;
$target[] = $candidate;
}
}
/**
* @param array<string, mixed> $candidate
*/
private function candidateCompleteness(array $candidate): int
{
$score = 0;
foreach (['isbn', 'ean', 'publisher', 'publishdate', 'pages', 'overview', 'genre', 'coverurl'] as $field) {
if (isset($candidate[$field]) && $candidate[$field] !== '') {
$score++;
}
}
return $score;
}
/**
* @param array<string, mixed> $candidate
*/
private function candidateYearTie(array $candidate, BookParseResult $parsed): int
{
if ($parsed->year === null || empty($candidate['publishdate'])) {
return 0;
}
if (preg_match('/\b(19|20)\d{2}\b/', (string) $candidate['publishdate'], $match) !== 1) {
return 0;
}
return max(0, 100 - abs($parsed->year - (int) $match[0]));
}
/**
* @param array<string, mixed> $book
*/
private function persistBookCandidate(array $book): int
{
$check = null;
$asin = trim((string) ($book['asin'] ?? ''));
if ($asin !== '') {
$check = BookInfo::query()->where('asin', $asin)->first();
}
foreach (['isbn', 'ean'] as $identifierField) {
if ($check !== null) {
break;
}
$identifiers = BookIsbn::equivalents(isset($book[$identifierField]) ? (string) $book[$identifierField] : null);
if ($identifiers !== []) {
$check = BookInfo::query()
->where(static function ($query) use ($identifiers): void {
$query->whereIn('isbn', $identifiers)->orWhereIn('ean', $identifiers);
})
->first();
}
}
$attributes = [
'title' => (string) ($book['title'] ?? ''),
'author' => (string) ($book['author'] ?? ''),
'asin' => $asin !== '' ? $asin : null,
'isbn' => BookIsbn::normalize(isset($book['isbn']) ? (string) $book['isbn'] : null),
'ean' => BookIsbn::normalize(isset($book['ean']) ? (string) $book['ean'] : null),
'url' => $book['url'] ?? null,
'salesrank' => $book['salesrank'] ?? null,
'publisher' => $book['publisher'] ?? null,
'publishdate' => $book['publishdate'] ?? null,
'pages' => $book['pages'] ?? null,
'overview' => $book['overview'] ?? null,
'genre' => (string) ($book['genre'] ?? ''),
];
if ($check === null) {
$bookId = BookInfo::query()->insertGetId(array_merge($attributes, [
'cover' => 0,
'created_at' => now(),
'updated_at' => now(),
]));
} else {
$bookId = (int) $check->id;
$updates = [];
foreach ($attributes as $field => $value) {
if ($value === null || $value === '') {
continue;
}
if ($field === 'asin' && ! empty($check->asin)) {
continue;
}
$updates[$field] = $value;
}
if ($updates !== []) {
$updates['updated_at'] = now();
BookInfo::query()->where('id', $bookId)->update($updates);
}
}
$coverUrl = trim((string) ($book['coverurl'] ?? ''));
if ($coverUrl !== '') {
$coverSaved = $this->releaseImageService->saveRemoteImage(
(string) $bookId,
$coverUrl,
$this->imgSavePath,
ImageAssetProfile::MetadataCover,
)->success;
if ($coverSaved) {
BookInfo::query()->where('id', $bookId)->update(['cover' => 1]);
}
}
return $bookId;
}
/**
* @param array<string, mixed> $book
*/
private function outputBookMatch(array $book): void
{
if (! $this->echooutput) {
return;
}
cli()->header('Added/updated book: ');
if (! empty($book['author'])) {
cli()->alternateOver(' Author: ').cli()->primary((string) $book['author']);
}
cli()->alternateOver(' Title: ').cli()->primary(' '.(string) $book['title']);
if (! empty($book['genre'])) {
cli()->alternateOver(' Genre: ').cli()->primary(' '.(string) $book['genre']);
}
}
private function failureCacheKey(?BookParseResult $parsed, string $bookInfo): string
{
$identity = [
(string) self::FAILURE_CACHE_VERSION,
mb_strtolower(trim($parsed === null ? $bookInfo : $parsed->title)),
mb_strtolower(trim((string) ($parsed === null ? '' : $parsed->author))),
(string) ($parsed === null ? '' : $parsed->year),
(string) ($parsed === null ? '' : $parsed->isbn),
];
return 'book_lookup_fail_v'.self::FAILURE_CACHE_VERSION.'_'.md5(implode('|', $identity));
}
private function logProviderFailure(BookProviderException $exception, string $bookInfo): void
{
Log::warning('Book metadata provider unavailable', [
'provider' => $exception->provider,
'status' => $exception->statusCode,
'retry_after_seconds' => $exception->retryAfterSeconds,
'book_info' => $bookInfo,
'message' => $exception->getMessage(),
]);
}
}
app/Services/GoogleBooksService.php
+38 -14
View File
@@ -4,11 +4,13 @@ declare(strict_types=1);
namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
use JsonException;
class GoogleBooksService
{
@@ -73,7 +75,7 @@ class GoogleBooksService
$query = trim($query);
$title = $title !== null ? trim($title) : null;
$author = $author !== null ? trim($author) : null;
$isbn = $isbn !== null ? strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn)) : null;
$isbn = BookIsbn::normalize($isbn);
$searchQuery = $this->buildSearchQuery($query, $title, $author, $isbn);
if ($searchQuery === '') {
@@ -118,20 +120,42 @@ class GoogleBooksService
$query['key'] = $this->apiKey;
}
$response = $this->client->get(self::API_URL, ['query' => $query]);
$response = $this->client->get(self::API_URL, [
'http_errors' => false,
'query' => $query,
]);
if ($response->getStatusCode() !== 200) {
Log::warning('Google Books API request failed', ['status' => $response->getStatusCode()]);
return null;
throw new BookProviderException(
'google_books',
'Google Books API request failed.',
$response->getStatusCode(),
);
}
$decoded = json_decode($response->getBody()->getContents(), true);
$decoded = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($decoded)) {
throw new BookProviderException('google_books', 'Google Books returned an invalid response payload.', 200);
}
return is_array($decoded) ? $decoded : null;
return $decoded;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException(
'google_books',
'Google Books returned malformed JSON.',
200,
null,
$exception,
);
} catch (GuzzleException $e) {
Log::error('Google Books API request error: '.$e->getMessage());
return null;
throw new BookProviderException(
'google_books',
'Google Books API request error: '.$e->getMessage(),
null,
null,
$e,
);
}
}
@@ -172,11 +196,11 @@ class GoogleBooksService
continue;
}
$type = (string) ($identifier['type'] ?? '');
$value = strtoupper((string) preg_replace('/[^0-9X]/i', '', (string) ($identifier['identifier'] ?? '')));
if ($type === 'ISBN_13' && $value !== '') {
$value = BookIsbn::normalize((string) ($identifier['identifier'] ?? ''));
if ($type === 'ISBN_13' && $value !== null && strlen($value) === 13) {
$isbn13 = $value;
}
if ($type === 'ISBN_10' && $value !== '') {
if ($type === 'ISBN_10' && $value !== null && strlen($value) === 10) {
$isbn10 = $value;
}
}
app/Services/IsbnDbService.php
+109 -36
View File
@@ -4,22 +4,27 @@ declare(strict_types=1);
namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
use JsonException;
use Psr\Http\Message\ResponseInterface;
class IsbnDbService
{
public const COOLDOWN_CACHE_KEY = 'isbndb_provider_cooldown';
protected const API_URL = 'https://api2.isbndb.com';
protected Client $client;
protected string $apiKey;
protected int $pageSize = 5;
protected int $pageSize = 10;
public function __construct(?Client $client = null, ?string $apiKey = null)
{
@@ -45,9 +50,9 @@ class IsbnDbService
/**
* @return array<string, mixed>|null
*/
public function searchBook(string $query): ?array
public function searchBook(string $query, ?string $author = null, bool $shouldMatchAll = true): ?array
{
$books = $this->searchBooks($query);
$books = $this->searchBooks($query, $author, $shouldMatchAll);
return $books[0] ?? null;
}
@@ -55,26 +60,46 @@ class IsbnDbService
/**
* @return array<int, array<string, mixed>>
*/
public function searchBooks(string $query): array
public function searchBooks(string $query, ?string $author = null, bool $shouldMatchAll = true): array
{
$query = trim($query);
$author = $author !== null ? trim($author) : null;
if ($query === '' || ! $this->isConfigured()) {
return [];
}
$cacheKey = 'isbndb_search_'.md5($query.'_'.$this->pageSize);
if (Cache::has(self::COOLDOWN_CACHE_KEY)) {
throw new BookProviderException('isbndb', 'ISBNdb is temporarily cooling down after a failed request.');
}
$cacheKey = 'isbndb_search_'.md5(implode('|', [
mb_strtolower((string) preg_replace('/\s+/', ' ', $query)),
mb_strtolower((string) preg_replace('/\s+/', ' ', (string) $author)),
$shouldMatchAll ? 'all' : 'any',
(string) $this->pageSize,
]));
$cached = Cache::get($cacheKey);
if (is_array($cached)) {
return $cached;
}
$response = $this->request('/books/'.rawurlencode($query), [
'shouldMatchAll' => true,
$requestQuery = [
'shouldMatchAll' => $shouldMatchAll ? 'true' : 'false',
'pageSize' => $this->pageSize,
]);
'column' => 'title',
];
$response = $this->request('/books/'.rawurlencode($query), $requestQuery);
$books = $response['data'] ?? null;
if (! is_array($books) || $books === []) {
if (! is_array($books)) {
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException('isbndb', 'ISBNdb search response did not contain a data array.', 200, 300);
}
if ($books === []) {
Cache::put($cacheKey, [], now()->addHours(6));
return [];
}
@@ -95,11 +120,15 @@ class IsbnDbService
*/
public function findByIsbn(string $isbn): ?array
{
$isbn = $this->normalizeIsbn($isbn);
if ($isbn === '' || ! $this->isConfigured()) {
$isbn = BookIsbn::normalize($isbn);
if ($isbn === null || ! $this->isConfigured()) {
return null;
}
if (Cache::has(self::COOLDOWN_CACHE_KEY)) {
throw new BookProviderException('isbndb', 'ISBNdb is temporarily cooling down after a failed request.');
}
$cacheKey = 'isbndb_isbn_'.$isbn;
$cached = Cache::get($cacheKey);
if (is_array($cached)) {
@@ -109,7 +138,9 @@ class IsbnDbService
$response = $this->request('/book/'.rawurlencode($isbn));
$bookRaw = $response['book'] ?? null;
if (! is_array($bookRaw) || $bookRaw === []) {
return null;
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException('isbndb', 'ISBNdb ISBN response did not contain a book object.', 200, 300);
}
$book = $this->normalizeBookResult($bookRaw);
@@ -130,6 +161,7 @@ class IsbnDbService
try {
$response = $this->client->get($path, [
'http_errors' => false,
'headers' => [
'Authorization' => $this->apiKey,
],
@@ -143,25 +175,50 @@ class IsbnDbService
return null;
}
if ($statusCode === 401 || $statusCode === 429) {
Log::warning("ISBNdb request failed with status {$statusCode}");
return null;
}
if ($statusCode !== 200) {
Log::warning("ISBNdb request returned status {$statusCode}");
$retryAfter = $this->retryAfterSeconds($response);
if ($statusCode === 429 || $statusCode >= 500) {
$retryAfter ??= 300;
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addSeconds($retryAfter));
}
return null;
throw new BookProviderException(
'isbndb',
"ISBNdb request returned status {$statusCode}.",
$statusCode,
$retryAfter,
);
}
$data = json_decode($response->getBody()->getContents(), true);
$data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
return is_array($data) ? $data : null;
if (! is_array($data)) {
throw new BookProviderException('isbndb', 'ISBNdb returned an invalid response payload.', $statusCode);
}
return $data;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
throw new BookProviderException(
'isbndb',
'ISBNdb returned malformed JSON.',
200,
300,
$exception,
);
} catch (GuzzleException $e) {
Log::error('ISBNdb API request error: '.$e->getMessage());
Cache::put(self::COOLDOWN_CACHE_KEY, true, now()->addMinutes(5));
return null;
throw new BookProviderException(
'isbndb',
'ISBNdb API request error: '.$e->getMessage(),
null,
300,
$e,
);
}
}
@@ -171,12 +228,18 @@ class IsbnDbService
*/
private function normalizeBookResult(array $book): array
{
$isbn13 = isset($book['isbn13']) ? $this->normalizeIsbn((string) $book['isbn13']) : '';
if ($isbn13 === '' && isset($book['isbn'])) {
$isbn13 = $this->normalizeIsbn((string) $book['isbn']);
$isbn13 = isset($book['isbn13']) ? BookIsbn::normalize((string) $book['isbn13']) : null;
if ($isbn13 === null && isset($book['isbn'])) {
$isbn13 = BookIsbn::normalize((string) $book['isbn']);
}
if ($isbn13 !== null && strlen($isbn13) === 10) {
$isbn13 = BookIsbn::toIsbn13($isbn13);
}
$isbn10 = isset($book['isbn10']) ? $this->normalizeIsbn((string) $book['isbn10']) : '';
$isbn10 = isset($book['isbn10']) ? BookIsbn::normalize((string) $book['isbn10']) : null;
if ($isbn10 !== null && strlen($isbn10) === 13) {
$isbn10 = BookIsbn::toIsbn10($isbn10);
}
$authors = is_array($book['authors'] ?? null)
? array_values(array_filter(array_map('strval', $book['authors'])))
@@ -185,7 +248,7 @@ class IsbnDbService
? array_values(array_filter(array_map('strval', $book['subjects'])))
: [];
$identifier = $isbn13 !== '' ? $isbn13 : ($isbn10 !== '' ? $isbn10 : md5((string) json_encode($book)));
$identifier = $isbn13 ?? $isbn10 ?? md5((string) json_encode($book));
$coverUrl = (string) ($book['image'] ?? $book['image_original'] ?? '');
$overview = trim((string) ($book['synopsis'] ?? $book['overview'] ?? $book['excerpt'] ?? ''));
@@ -193,8 +256,8 @@ class IsbnDbService
'title' => (string) ($book['title'] ?? ''),
'author' => implode(', ', $authors),
'asin' => 'isbndb:'.$identifier,
'isbn' => $isbn13 !== '' ? $isbn13 : null,
'ean' => $isbn10 !== '' ? $isbn10 : null,
'isbn' => $isbn13,
'ean' => $isbn10,
'url' => '',
'salesrank' => '',
'publisher' => (string) ($book['publisher'] ?? ''),
@@ -220,11 +283,6 @@ class IsbnDbService
}
}
private function normalizeIsbn(string $isbn): string
{
return strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn));
}
private function logRateLimitHeaders(ResponseInterface $response): void
{
$rateLimit = $response->getHeaderLine('ratelimit');
@@ -251,4 +309,19 @@ class IsbnDbService
'ratelimit_policy' => $rateLimitPolicy,
]);
}
private function retryAfterSeconds(ResponseInterface $response): ?int
{
$retryAfter = trim($response->getHeaderLine('Retry-After'));
if ($retryAfter === '') {
return null;
}
if (ctype_digit($retryAfter)) {
return max(1, (int) $retryAfter);
}
$retryAt = strtotime($retryAfter);
return $retryAt === false ? null : max(1, $retryAt - time());
}
}
app/Services/ItunesService.php
+12
View File
@@ -29,6 +29,8 @@ class ItunesService
protected string $country = 'US';
private bool $lastRequestFailed = false;
public function __construct()
{
$this->client = new Client([
@@ -62,6 +64,11 @@ class ItunesService
return $this;
}
public function lastRequestFailed(): bool
{
return $this->lastRequestFailed;
}
/**
* Search for music albums.
*
@@ -240,6 +247,7 @@ class ItunesService
*/
protected function search(string $term, string $entity, string $media): ?array
{
$this->lastRequestFailed = false;
$term = trim($term);
if (empty($term)) {
return null;
@@ -265,6 +273,7 @@ class ItunesService
$statusCode = $response->getStatusCode();
if ($statusCode !== 200) {
$this->lastRequestFailed = true;
Log::warning("iTunes API search returned status {$statusCode}");
return null;
@@ -273,6 +282,8 @@ class ItunesService
$data = json_decode($response->getBody()->getContents(), true);
if (! isset($data['results'])) {
$this->lastRequestFailed = true;
return null;
}
@@ -281,6 +292,7 @@ class ItunesService
return $results;
} catch (GuzzleException $e) {
$this->lastRequestFailed = true;
Log::error('iTunes API search error: '.$e->getMessage());
return null;
app/Services/OpenLibraryService.php
+39 -18
View File
@@ -4,11 +4,13 @@ declare(strict_types=1);
namespace App\Services;
use App\Exceptions\BookProviderException;
use App\Support\BookIsbn;
use Carbon\Carbon;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
use JsonException;
class OpenLibraryService
{
@@ -38,8 +40,8 @@ class OpenLibraryService
*/
public function findByIsbn(string $isbn): ?array
{
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $isbn));
if ($isbn === '') {
$isbn = BookIsbn::normalize($isbn);
if ($isbn === null) {
return null;
}
@@ -50,24 +52,33 @@ class OpenLibraryService
}
try {
$response = $this->client->get(self::ISBN_URL.rawurlencode($isbn).'.json');
if ($response->getStatusCode() !== 200) {
$response = $this->client->get(self::ISBN_URL.rawurlencode($isbn).'.json', ['http_errors' => false]);
if ($response->getStatusCode() === 404) {
return null;
}
if ($response->getStatusCode() !== 200) {
throw new BookProviderException(
'open_library',
'Open Library ISBN request failed.',
$response->getStatusCode(),
);
}
$data = json_decode($response->getBody()->getContents(), true);
$data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($data)) {
return null;
throw new BookProviderException('open_library', 'Open Library returned an invalid ISBN payload.', 200);
}
$book = $this->normalizeIsbnResult($data, $isbn);
Cache::put($cacheKey, $book, now()->addHours(24));
return $book;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException('open_library', 'Open Library returned malformed JSON.', 200, null, $exception);
} catch (GuzzleException $e) {
Log::error('OpenLibrary ISBN lookup error: '.$e->getMessage());
return null;
throw new BookProviderException('open_library', 'Open Library ISBN lookup error: '.$e->getMessage(), null, null, $e);
}
}
@@ -99,18 +110,26 @@ class OpenLibraryService
try {
$response = $this->client->get(self::SEARCH_URL, [
'http_errors' => false,
'query' => [
'q' => $query,
'limit' => $this->limit,
],
]);
if ($response->getStatusCode() !== 200) {
if ($response->getStatusCode() === 404) {
return [];
}
if ($response->getStatusCode() !== 200) {
throw new BookProviderException(
'open_library',
'Open Library search request failed.',
$response->getStatusCode(),
);
}
$data = json_decode($response->getBody()->getContents(), true);
$data = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (! is_array($data['docs'] ?? null)) {
return [];
throw new BookProviderException('open_library', 'Open Library returned an invalid search payload.', 200);
}
$results = [];
@@ -123,10 +142,12 @@ class OpenLibraryService
Cache::put($cacheKey, $results, now()->addHours(12));
return $results;
} catch (BookProviderException $exception) {
throw $exception;
} catch (JsonException $exception) {
throw new BookProviderException('open_library', 'Open Library returned malformed JSON.', 200, null, $exception);
} catch (GuzzleException $e) {
Log::error('OpenLibrary search error: '.$e->getMessage());
return [];
throw new BookProviderException('open_library', 'Open Library search error: '.$e->getMessage(), null, null, $e);
}
}
@@ -180,8 +201,8 @@ class OpenLibraryService
$isbn10 = null;
if (is_array($doc['isbn'] ?? null)) {
foreach ($doc['isbn'] as $isbn) {
$normalized = strtoupper((string) preg_replace('/[^0-9X]/i', '', (string) $isbn));
if ($normalized === '') {
$normalized = BookIsbn::normalize((string) $isbn);
if ($normalized === null) {
continue;
}
if (strlen($normalized) === 13 && $isbn13 === null) {
app/Services/Runners/PostProcessRunner.php
+13 -3
View File
@@ -155,6 +155,7 @@ class PostProcessRunner extends BaseRunner
private function getBooksBuckets(): array
{
$bucketExpr = $this->guidBucketExpression();
$pendingCondition = $this->bookPendingCondition();
return DB::select('
SELECT DISTINCT '.$bucketExpr.' AS id
@@ -164,7 +165,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = '.Category::MUSIC_AUDIOBOOK.'
)
AND (
bookinfo_id IS NULL
'.$pendingCondition.'
OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%"
@@ -600,6 +601,8 @@ class PostProcessRunner extends BaseRunner
return;
}
$pendingCondition = $this->bookPendingCondition();
$checkSql = '
SELECT id
FROM releases
@@ -608,7 +611,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = 3030
)
AND (
bookinfo_id IS NULL
'.$pendingCondition.'
OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%"
@@ -630,7 +633,7 @@ class PostProcessRunner extends BaseRunner
OR categories_id = 3030
)
AND (
bookinfo_id IS NULL
'.$pendingCondition.'
OR searchname LIKE "N:/NZB%"
OR searchname LIKE "N_NZB_%"
OR name LIKE "N:/NZB%"
@@ -643,6 +646,13 @@ class PostProcessRunner extends BaseRunner
$this->runPostProcess($queue, $maxProcesses, 'books', 'books postprocessing');
}
private function bookPendingCondition(): string
{
return (int) Settings::settingValue('lookupbooks') === 2
? '(bookinfo_id IS NULL AND isrenamed = 1)'
: 'bookinfo_id IS NULL';
}
public function processMusic(): void
{
if ((int) Settings::settingValue('lookupmusic') <= 0) {
app/Support/BookIsbn.php
+114
View File
@@ -0,0 +1,114 @@
<?php
declare(strict_types=1);
namespace App\Support;
final class BookIsbn
{
public static function normalize(?string $value): ?string
{
if ($value === null) {
return null;
}
$isbn = strtoupper((string) preg_replace('/[^0-9X]/i', '', $value));
return match (strlen($isbn)) {
10 => self::isValidIsbn10($isbn) ? $isbn : null,
13 => self::isValidIsbn13($isbn) ? $isbn : null,
default => null,
};
}
public static function equivalent(?string $left, ?string $right): bool
{
$leftIdentifiers = self::equivalents($left);
$rightIdentifiers = self::equivalents($right);
return $leftIdentifiers !== [] && array_intersect($leftIdentifiers, $rightIdentifiers) !== [];
}
/**
* @return list<string>
*/
public static function equivalents(?string $value): array
{
$isbn = self::normalize($value);
if ($isbn === null) {
return [];
}
$identifiers = [$isbn];
$converted = strlen($isbn) === 10 ? self::toIsbn13($isbn) : self::toIsbn10($isbn);
if ($converted !== null) {
$identifiers[] = $converted;
}
sort($identifiers);
return array_values(array_unique($identifiers));
}
public static function toIsbn13(?string $value): ?string
{
$isbn10 = self::normalize($value);
if ($isbn10 === null || strlen($isbn10) !== 10) {
return null;
}
$body = '978'.substr($isbn10, 0, 9);
$sum = 0;
for ($index = 0; $index < 12; $index++) {
$sum += ((int) $body[$index]) * ($index % 2 === 0 ? 1 : 3);
}
return $body.(string) ((10 - ($sum % 10)) % 10);
}
public static function toIsbn10(?string $value): ?string
{
$isbn13 = self::normalize($value);
if ($isbn13 === null || strlen($isbn13) !== 13 || ! str_starts_with($isbn13, '978')) {
return null;
}
$body = substr($isbn13, 3, 9);
$sum = 0;
for ($index = 0; $index < 9; $index++) {
$sum += ((int) $body[$index]) * (10 - $index);
}
$check = (11 - ($sum % 11)) % 11;
return $body.($check === 10 ? 'X' : (string) $check);
}
private static function isValidIsbn10(string $isbn): bool
{
if (preg_match('/^\d{9}[\dX]$/', $isbn) !== 1) {
return false;
}
$sum = 0;
for ($index = 0; $index < 10; $index++) {
$digit = $isbn[$index] === 'X' ? 10 : (int) $isbn[$index];
$sum += $digit * (10 - $index);
}
return $sum % 11 === 0;
}
private static function isValidIsbn13(string $isbn): bool
{
if (preg_match('/^97[89]\d{10}$/', $isbn) !== 1) {
return false;
}
$sum = 0;
for ($index = 0; $index < 13; $index++) {
$sum += ((int) $isbn[$index]) * ($index % 2 === 0 ? 1 : 3);
}
return $sum % 10 === 0;
}
}
app/Support/BookMatchScorer.php
+149 -53
View File
@@ -14,34 +14,32 @@ class BookMatchScorer
*/
public function score(array $candidate, BookParseResult $parsed): float
{
$candidateIsbn = $this->normalizeIsbn((string) ($candidate['isbn'] ?? ''));
$parsedIsbn = $this->normalizeIsbn((string) ($parsed->isbn ?? ''));
if ($parsedIsbn !== '' && $candidateIsbn !== '' && $parsedIsbn === $candidateIsbn) {
$parsedIsbn = BookIsbn::normalize($parsed->isbn);
$candidateIdentifiers = $this->candidateIdentifiers($candidate);
if ($parsedIsbn !== null && $candidateIdentifiers !== []) {
foreach ($candidateIdentifiers as $candidateIdentifier) {
if (! BookIsbn::equivalent($parsedIsbn, $candidateIdentifier)) {
return 0.0;
}
}
return 1.0;
}
$titleScore = $this->titleSimilarity($parsed->title, (string) ($candidate['title'] ?? ''));
$authorScore = $parsed->hasAuthor()
? $this->similarity((string) $parsed->author, (string) ($candidate['author'] ?? ''))
: 0.0;
$yearScore = $this->yearScore($parsed->year, (string) ($candidate['publishdate'] ?? ''));
$publisherScore = ! empty($candidate['publisher']) ? 1.0 : 0.0;
$coverScore = (int) ($candidate['cover'] ?? 0) === 1 ? 1.0 : 0.0;
if ($parsed->hasAuthor()) {
return (0.45 * $titleScore)
+ (0.30 * $authorScore)
+ (0.10 * $yearScore)
+ (0.08 * $publisherScore)
+ (0.07 * $coverScore);
if ($this->hasStructuralConflict($parsed, (string) ($candidate['title'] ?? ''))) {
return 0.0;
}
// When the release has no reliable author, avoid over-trusting
// "has cover/publisher" signals for weak title matches.
return (0.80 * $titleScore)
+ (0.10 * $yearScore)
+ (0.05 * $publisherScore)
+ (0.05 * $coverScore);
$titleScore = $this->titleScore($parsed->title, (string) ($candidate['title'] ?? ''));
$authorScore = $parsed->hasAuthor()
? $this->authorScore((string) $parsed->author, (string) ($candidate['author'] ?? ''))
: 0.0;
if ($parsed->hasAuthor()) {
return (0.65 * $titleScore) + (0.35 * $authorScore);
}
return $titleScore;
}
public function scoreBookInfo(BookInfo $book, BookParseResult $parsed): float
@@ -50,14 +48,28 @@ class BookMatchScorer
'title' => $book->title,
'author' => $book->author,
'isbn' => $book->isbn,
'ean' => $book->ean,
'publishdate' => $book->publishdate,
'publisher' => $book->publisher,
'cover' => $book->cover ? 1 : 0,
], $parsed);
}
private function titleSimilarity(string $left, string $right): float
public function titleScore(string $left, string $right): float
{
$leftVariants = $this->titleVariants($left);
$rightVariants = $this->titleVariants($right);
foreach ($leftVariants as $leftVariant) {
foreach ($rightVariants as $rightVariant) {
if ($leftVariant !== '' && $leftVariant === $rightVariant) {
return $leftVariant === $this->normalizeText($left)
&& $rightVariant === $this->normalizeText($right)
? 1.0
: 0.97;
}
}
}
$left = $this->normalizeText($left);
$right = $this->normalizeText($right);
if ($left === '' || $right === '') {
@@ -74,52 +86,62 @@ class BookMatchScorer
return 0.0;
}
$shared = array_intersect($leftWords, $rightWords);
$total = max(count($leftWords), count($rightWords));
$jaccardScore = count($shared) / $total;
$shared = array_intersect(array_unique($leftWords), array_unique($rightWords));
$tokenScore = (2 * count($shared)) / (count(array_unique($leftWords)) + count(array_unique($rightWords)));
similar_text($left, $right, $percent);
$simTextScore = max(0.0, min(1.0, $percent / 100));
$lengthRatio = min(mb_strlen($left), mb_strlen($right)) / max(mb_strlen($left), mb_strlen($right));
$lengthPenalty = $lengthRatio < 0.5 ? 0.7 : 1.0;
return min(1.0, ((0.5 * $jaccardScore) + (0.5 * $simTextScore)) * $lengthPenalty);
return min(1.0, max($tokenScore, $simTextScore));
}
private function similarity(string $left, string $right): float
public function authorScore(string $left, string $right): float
{
$left = $this->normalizeText($left);
$right = $this->normalizeText($right);
if ($left === '' || $right === '') {
$leftTokens = $this->tokens($left);
$rightTokens = $this->tokens($right);
if ($leftTokens === [] || $rightTokens === []) {
return 0.0;
}
similar_text($left, $right, $percent);
$leftCoverage = $this->authorTokenCoverage($leftTokens, $rightTokens);
$rightCoverage = $this->authorTokenCoverage($rightTokens, $leftTokens);
return max(0.0, min(1.0, $percent / 100));
return ($leftCoverage + $rightCoverage) / 2;
}
private function yearScore(?int $parsedYear, string $candidatePublishDate): float
public function hasStructuralConflict(BookParseResult $parsed, string $candidateTitle): bool
{
if ($parsedYear === null) {
return 0.5;
$parsedMarkers = $this->structuralMarkers($parsed->rawName.' '.$parsed->title);
$candidateMarkers = $this->structuralMarkers($candidateTitle);
foreach ($parsedMarkers as $type => $number) {
if (isset($candidateMarkers[$type]) && $candidateMarkers[$type] !== $number) {
return true;
}
}
if (preg_match('/\b(19|20)\d{2}\b/', $candidatePublishDate, $matches) !== 1) {
return 0.0;
}
return false;
}
$candidateYear = (int) $matches[0];
if ($parsedYear === $candidateYear) {
return 1.0;
}
public function meaningfulTitleTokenCount(string $title): int
{
$stopWords = ['a', 'an', 'and', 'by', 'for', 'in', 'of', 'on', 'or', 'the', 'to', 'with'];
if (abs($parsedYear - $candidateYear) <= 1) {
return 0.6;
}
return count(array_filter(
$this->tokens($title),
static fn (string $token): bool => mb_strlen($token) > 1 && ! in_array($token, $stopWords, true)
));
}
return 0.0;
/**
* @param array<string, mixed> $candidate
*/
public function workKey(array $candidate): string
{
$authorTokens = $this->tokens((string) ($candidate['author'] ?? ''));
sort($authorTokens);
return $this->normalizeText((string) ($candidate['title'] ?? '')).'|'
.implode(' ', $authorTokens);
}
private function normalizeText(string $value): string
@@ -132,8 +154,82 @@ class BookMatchScorer
return trim($value);
}
private function normalizeIsbn(string $value): string
/**
* @return list<string>
*/
private function titleVariants(string $value): array
{
return strtoupper((string) preg_replace('/[^0-9X]/i', '', $value));
$variants = [$this->normalizeText($value)];
$parts = preg_split('/\s+(?::|-)\s+|:\s*/u', $value, 2);
if (is_array($parts) && count($parts) === 2) {
$variants[] = $this->normalizeText($parts[0]);
}
return array_values(array_unique(array_filter($variants)));
}
/**
* @param array<string, mixed> $candidate
* @return list<string>
*/
private function candidateIdentifiers(array $candidate): array
{
$identifiers = [];
foreach (['isbn', 'ean'] as $field) {
$isbn = BookIsbn::normalize(isset($candidate[$field]) ? (string) $candidate[$field] : null);
if ($isbn !== null) {
$identifiers[] = $isbn;
}
}
return array_values(array_unique($identifiers));
}
/**
* @return list<string>
*/
private function tokens(string $value): array
{
$normalized = $this->normalizeText($value);
return $normalized === '' ? [] : array_values(array_unique(explode(' ', $normalized)));
}
/**
* @param list<string> $needles
* @param list<string> $haystack
*/
private function authorTokenCoverage(array $needles, array $haystack): float
{
$matched = 0;
foreach ($needles as $needle) {
foreach ($haystack as $candidate) {
if ($needle === $candidate
|| (mb_strlen($needle) === 1 && str_starts_with($candidate, $needle))
|| (mb_strlen($candidate) === 1 && str_starts_with($needle, $candidate))) {
$matched++;
break;
}
}
}
return $matched / count($needles);
}
/**
* @return array<string, int>
*/
private function structuralMarkers(string $value): array
{
$markers = [];
if (preg_match('/\b(?:book|series|vol(?:ume)?)\.?\s*#?\s*(\d{1,3})\b/i', $value, $match) === 1) {
$markers['volume'] = (int) $match[1];
}
if (preg_match('/\b(\d{1,3})(?:st|nd|rd|th)?\s+(?:edition|ed\.?)\b/i', $value, $match) === 1) {
$markers['edition'] = (int) $match[1];
}
return $markers;
}
}