From b20e01a00910c01d6624d619eb9bc8ef1ec3ee3b Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 23 Apr 2026 17:01:28 +0200 Subject: [PATCH] Update book processing further --- app/Console/Commands/RequeueFailedBooks.php | 151 ++++++++++++++++++ .../ReleaseFileManager.php | 18 ++- app/Services/BookService.php | 130 +++++++++++++-- .../Extractors/ObfuscatedSubjectExtractor.php | 14 ++ app/Services/NameFixing/FileNameCleaner.php | 38 +++++ .../NameFixing/ReleaseUpdateService.php | 7 +- app/Services/Runners/PostProcessRunner.php | 62 +++++-- app/Support/BookMatchScorer.php | 10 +- .../views/admin/releases/index.blade.php | 4 +- ...AdditionalProcessingNzbSplitRenameTest.php | 56 +++++++ .../ReleaseNameFixedRecategorizationTest.php | 46 ++++++ tests/Unit/Services/BookServiceTest.php | 29 ++++ .../NameFixing/FileNameCleanerTest.php | 10 ++ .../ObfuscatedSubjectExtractorTest.php | 30 ++++ tests/Unit/Support/BookMatchScorerTest.php | 19 +++ 15 files changed, 594 insertions(+), 30 deletions(-) create mode 100644 app/Console/Commands/RequeueFailedBooks.php diff --git a/app/Console/Commands/RequeueFailedBooks.php b/app/Console/Commands/RequeueFailedBooks.php new file mode 100644 index 000000000..0b0e3756b --- /dev/null +++ b/app/Console/Commands/RequeueFailedBooks.php @@ -0,0 +1,151 @@ +option('limit')); + $dryRun = (bool) $this->option('dry-run'); + $onlyObfuscated = (bool) $this->option('only-obfuscated'); + $normalizeObfuscatedAll = (bool) $this->option('normalize-obfuscated-all'); + + $query = Release::query() + ->select(['id', 'name', 'searchname', 'categories_id', 'bookinfo_id']) + ->where(function ($builder): void { + $builder->whereBetween('categories_id', [Category::BOOKS_ROOT, Category::BOOKS_UNKNOWN]) + ->orWhere('categories_id', Category::MUSIC_AUDIOBOOK); + }) + ->orderBy('id'); + + if ($normalizeObfuscatedAll) { + $query->where(function ($builder): void { + $builder + ->where('searchname', 'like', 'N:/NZB%') + ->orWhere('searchname', 'like', 'N_NZB_%') + ->orWhere('name', 'like', 'N:/NZB%') + ->orWhere('name', 'like', 'N_NZB_%'); + }); + } else { + $query->where('bookinfo_id', -2); + } + + if ($limit > 0) { + $query->limit($limit); + } + + $releases = $query->get(); + $total = $releases->count(); + + if ($total === 0) { + $this->info('No failed book releases found (bookinfo_id = -2).'); + + return self::SUCCESS; + } + + $queued = 0; + $renamed = 0; + $skipped = 0; + + $bar = $this->output->createProgressBar($total); + $bar->start(); + + foreach ($releases as $release) { + $bar->advance(); + + $normalized = $extractor->extract((string) $release->searchname) + ?? $extractor->extract((string) $release->name); + + if ($onlyObfuscated && $normalized === null) { + $skipped++; + + continue; + } + + $updates = []; + $shouldRequeue = (int) $release->bookinfo_id === -2; + if ($shouldRequeue) { + $updates['bookinfo_id'] = null; + } + if ($normalized !== null && $normalized !== $release->searchname) { + $updates['searchname'] = $normalized; + $updates['isrenamed'] = 1; + } + + if ($updates === []) { + $skipped++; + + continue; + } + + if ($dryRun) { + if (isset($updates['searchname'])) { + $this->newLine(); + $this->line(sprintf( + '[dry-run] #%d searchname: "%s" -> "%s"', + (int) $release->id, + (string) $release->searchname, + (string) $updates['searchname'] + )); + } + + $queued++; + if (isset($updates['searchname'])) { + $renamed++; + } + + continue; + } + + Release::query()->where('id', $release->id)->update($updates); + Search::updateRelease((int) $release->id); + + $queued++; + if (isset($updates['searchname'])) { + $renamed++; + } + } + + $bar->finish(); + $this->newLine(2); + + if ($dryRun) { + $this->info(sprintf( + 'Dry run complete. Inspected: %d, would apply updates: %d, would normalize names: %d, skipped: %d.', + $total, + $queued, + $renamed, + $skipped + )); + + return self::SUCCESS; + } + + $this->info(sprintf( + 'Requeued %d failed book release(s). Normalized %d searchname value(s). Skipped: %d.', + $queued, + $renamed, + $skipped + )); + $this->line('Next step: run your books postprocess (e.g. `php artisan update:postprocess book`).'); + + return self::SUCCESS; + } +} diff --git a/app/Services/AdditionalProcessing/ReleaseFileManager.php b/app/Services/AdditionalProcessing/ReleaseFileManager.php index d08e56e18..7f49f9e7d 100644 --- a/app/Services/AdditionalProcessing/ReleaseFileManager.php +++ b/app/Services/AdditionalProcessing/ReleaseFileManager.php @@ -169,10 +169,14 @@ class ReleaseFileManager */ public function processReleaseNameFromNzbContents(array $nzbContents, ReleaseProcessingContext $context): bool { + if (! $this->releaseHasNzbSplitWrapper($context->release)) { + return false; + } + foreach ($nzbContents as $nzbFile) { $title = (string) ($nzbFile['title'] ?? ''); - if ($title === '' || ! str_contains($title, '__NZBSPLIT__')) { + if ($title === '') { continue; } @@ -184,7 +188,8 @@ class ReleaseFileManager $candidate = $this->fileNameCleaner->normalizeCandidateTitle($candidate); - if (! $this->fileNameCleaner->isPlausibleReleaseTitle($candidate)) { + $isPlausible = $this->fileNameCleaner->isPlausibleReleaseTitle($candidate); + if (! $isPlausible) { continue; } @@ -651,6 +656,15 @@ class ReleaseFileManager return $this->fileNameCleaner->normalizeCandidateTitle($title); } + private function releaseHasNzbSplitWrapper(Release $release): bool + { + $name = (string) ($release->name ?? ''); + $searchName = (string) ($release->searchname ?? ''); + + return str_contains(strtoupper($name), 'NZBSPLIT') + || str_contains(strtoupper($searchName), 'NZBSPLIT'); + } + /** * Check if a title is plausible for release naming. */ diff --git a/app/Services/BookService.php b/app/Services/BookService.php index d6fa621b6..0a0b04954 100644 --- a/app/Services/BookService.php +++ b/app/Services/BookService.php @@ -191,13 +191,13 @@ class BookService } // Step 2: Get paginated book entity list with only needed columns - $bookSql = 'SELECT boo.id, boo.title, boo.author, boo.cover, boo.publisher, boo.publishdate, boo.review, boo.url, ' + $bookSql = 'SELECT boo.id, boo.title, boo.author, boo.cover, boo.publisher, boo.publishdate, boo.overview, boo.url, ' .'MAX(r.postdate) AS latest_postdate, ' .'COUNT(r.id) AS total_releases ' .'FROM bookinfo boo ' .'INNER JOIN releases r ON boo.id = r.bookinfo_id ' .'WHERE '.$baseWhere.' ' - .'GROUP BY boo.id, boo.title, boo.author, boo.cover, boo.publisher, boo.publishdate, boo.review, boo.url ' + .'GROUP BY boo.id, boo.title, boo.author, boo.cover, boo.publisher, boo.publishdate, boo.overview, boo.url ' ."ORDER BY {$order[0]} {$order[1]} " ."LIMIT {$num} OFFSET {$start}"; @@ -366,6 +366,8 @@ class BookService */ public function processBookReleases(string $groupID = '', string $guidChar = ''): void { + $this->normalizeBookSearchNames($groupID, $guidChar); + $query = Release::query() ->whereNull('bookinfo_id') ->where(static function ($builder): void { @@ -394,6 +396,50 @@ class BookService ); } + protected function normalizeBookSearchNames(string $groupID = '', string $guidChar = ''): void + { + $query = Release::query() + ->select(['id', 'name', 'searchname', 'categories_id', 'isrenamed']) + ->where(static function ($builder): void { + $builder->whereBetween('categories_id', [Category::BOOKS_ROOT, Category::BOOKS_UNKNOWN]) + ->orWhere('categories_id', Category::MUSIC_AUDIOBOOK); + }) + ->where(function ($builder): void { + $builder->where('isrenamed', 0) + ->orWhere('searchname', 'like', 'N:/NZB%') + ->orWhere('searchname', 'like', 'N_NZB_%') + ->orWhere('name', 'like', 'N:/NZB%') + ->orWhere('name', 'like', 'N_NZB_%'); + }) + ->orderByDesc('postdate') + ->limit($this->bookqty); + + if ($guidChar !== '') { + $query->where('leftguid', 'like', $guidChar.'%'); + } + + if ($groupID !== '') { + $query->where('groups_id', $groupID); + } + + foreach ($query->get() as $release) { + $releaseType = (int) $release->categories_id === Category::MUSIC_AUDIOBOOK ? 'audiobook' : 'ebook'; + $sourceName = $this->preferredBookSourceName((string) $release->searchname, (string) $release->name); + $parsed = $this->parseReleaseName($sourceName, $releaseType); + $normalizedSearchName = $this->determineReadableBookSearchName($sourceName, $parsed); + + if ($normalizedSearchName === null || $normalizedSearchName === $release->searchname) { + continue; + } + + Release::query()->where('id', (int) $release->id)->update([ + 'searchname' => $normalizedSearchName, + 'isrenamed' => 1, + ]); + Search::updateRelease((int) $release->id); + } + } + /** * Process book releases helper. * @@ -472,8 +518,16 @@ class BookService */ public function parseTitle(mixed $release_name, mixed $releaseID, mixed $releasetype) { - $normalizedReleaseName = $this->obfuscatedSubjectExtractor->extract((string) $release_name) ?? (string) $release_name; - $parsed = $this->parseReleaseName($normalizedReleaseName, (string) $releasetype); + $rawReleaseName = (string) $release_name; + $parsed = $this->parseReleaseName($rawReleaseName, (string) $releasetype); + $normalizedReleaseName = $this->determineReadableBookSearchName($rawReleaseName, $parsed) ?? $rawReleaseName; + if ($normalizedReleaseName !== $rawReleaseName) { + Release::query()->where('id', (int) $releaseID)->update([ + 'searchname' => $normalizedReleaseName, + 'isrenamed' => 1, + ]); + Search::updateRelease((int) $releaseID); + } $this->parsedBookResult = $parsed; $this->parsedIsbn = $parsed->isbn; $releasename = $parsed->title; @@ -528,6 +582,7 @@ class BookService public function parseReleaseName(string $releaseName, string $releaseType = 'ebook'): BookParseResult { + $rawReleaseName = $releaseName; $releaseName = $this->obfuscatedSubjectExtractor->extract($releaseName) ?? $releaseName; if (preg_match('/"([^"]{3,240})"/', $releaseName, $quotedMatch) === 1) { $releaseName = $quotedMatch[1]; @@ -544,6 +599,13 @@ class BookService $normalized = trim((string) preg_replace('/\s+/', ' ', $normalized)); $year = null; + $specialMagazineTitle = null; + if (preg_match('/^MCN[._ -](?[A-Za-z]+)[._ -](?\d{1,2})[._ -](?(?:19|20)\d{2})/i', $rawReleaseName, $mcnMatch) === 1) { + $month = ucfirst(strtolower($mcnMatch['month'])); + $day = ltrim($mcnMatch['day'], '0'); + $specialMagazineTitle = sprintf('MCN - %s %s, %s', $month, $day === '' ? $mcnMatch['day'] : $day, $mcnMatch['year']); + $year = (int) $mcnMatch['year']; + } if (preg_match('/\b(19|20)\d{2}\b/', $normalized, $yearMatch) === 1) { $year = (int) $yearMatch[0]; } @@ -554,11 +616,13 @@ class BookService } $author = null; - $title = $normalized; - if (preg_match('/^(?[A-Za-z0-9&\'\.\-\s]{3,80})\s-\s(?.+)$/', $normalized, $matches) === 1) { + $title = $specialMagazineTitle ?? $normalized; + if ($specialMagazineTitle === null && preg_match('/^(?<author>[A-Za-z0-9&\'\.\-\s]{3,80})\s-\s(?<title>.+)$/', $normalized, $matches) === 1) { $candidateAuthor = trim($matches['author']); $candidateTitle = trim($matches['title']); - if (preg_match('/^(Issue\s+\d+|\d+\w*\s+Edition\b)/i', $candidateTitle) !== 1) { + $isMcnMagazineSplit = preg_match('/^MCN$/i', $candidateAuthor) === 1 + && preg_match('/^\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\b/i', $candidateTitle) === 1; + if ($isMcnMagazineSplit === false && preg_match('/^(Issue\s+\d+|\d+\w*\s+Edition\b)/i', $candidateTitle) !== 1) { $author = $candidateAuthor; $title = $candidateTitle; } @@ -606,6 +670,11 @@ class BookService preg_match('/^([a-z0-9ΓΌ!]+ ){1,2}(N|Vol)?\d{1,4}([abc])?$|^([a-z0-9]+ ){1,2}(Jan( |unar|$)|Feb( |ruary|$)|Mar( |ch|$)|Apr( |il|$)|May(?![a-z0-9])|Jun([ e$])|Jul([ y$])|Aug( |ust|$)|Sep( |tember|$)|O([ck])t( |ober|$)|Nov( |ember|$)|De([cz])( |ember|$))/ui', $normalized) === 1 || preg_match('/\bIssue[\._\- ]?\d{1,4}\b.*\b(19|20)\d{2}\b/i', $normalized) === 1 || preg_match('/\bIssue[\._\- ]?\d{1,4}\b/i', $normalized) === 1 + || (preg_match('/\bMCN[._ -]/i', $rawReleaseName) === 1 + && preg_match('/\bMAGAZINE\b/i', $rawReleaseName) === 1) + || (preg_match('/^MCN\b/i', $normalized) === 1 + && preg_match('/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\b/i', $normalized) === 1 + && preg_match('/\b\d{1,2}\b/', $normalized) === 1) ) && preg_match('/Part \d+/i', $normalized) !== 1; return new BookParseResult( @@ -639,6 +708,39 @@ class BookService return null; } + protected function determineReadableBookSearchName(string $rawReleaseName, BookParseResult $parsed): ?string + { + if ($this->looksLikeObfuscatedBookSubject($rawReleaseName)) { + $normalized = $this->obfuscatedSubjectExtractor->extract($rawReleaseName); + if ($normalized !== null) { + return $normalized; + } + } + + if ($parsed->isJunk) { + return null; + } + + $candidate = trim($parsed->searchQuery()); + if ($candidate === '' || preg_match('/^[a-z0-9]+$|^([0-9]+ ){1,}$|Part \d+/i', $candidate)) { + return null; + } + + $wordCount = count(preg_split('/\s+/', $candidate) ?: []); + + return $wordCount >= 2 ? $candidate : null; + } + + protected function preferredBookSourceName(string $searchName, string $originalName): string + { + return trim($searchName) !== '' ? $searchName : $originalName; + } + + protected function looksLikeObfuscatedBookSubject(string $value): bool + { + return preg_match('/^\s*(?:N:\/NZB\b|N[\s._:-]*NZB[\s._-]*\[|(?:re\s*)?posted\s+by\b)/i', $value) === 1; + } + /** * Update book info from external sources. * @@ -883,7 +985,7 @@ class BookService } } - return $bestScore >= 0.55 ? $best : null; + return $bestScore >= $this->minimumMatchScore($parsed) ? $best : null; } /** @@ -906,10 +1008,20 @@ class BookService } } - if ($best === null || $bestScore < 0.55) { + if ($best === null || $bestScore < $this->minimumMatchScore($parsed)) { return null; } return $best; } + + private function minimumMatchScore(?BookParseResult $parsed): float + { + if ($parsed === null) { + return 0.55; + } + + // No-author parses are more ambiguous and need a stricter cutoff. + return $parsed->hasAuthor() ? 0.55 : 0.68; + } } diff --git a/app/Services/NameFixing/Extractors/ObfuscatedSubjectExtractor.php b/app/Services/NameFixing/Extractors/ObfuscatedSubjectExtractor.php index f727464e0..98f1e6eeb 100644 --- a/app/Services/NameFixing/Extractors/ObfuscatedSubjectExtractor.php +++ b/app/Services/NameFixing/Extractors/ObfuscatedSubjectExtractor.php @@ -11,6 +11,7 @@ final class ObfuscatedSubjectExtractor */ private const array PREFIX_PATTERNS = [ '/^\s*N:\/NZB\s*\[\d+\/\d+\]\s*-\s*/i', + '/^\s*N[\s._:-]*NZB[\s._-]*\[\d+(?:[\/_]\d+)?\][\s._-]*-\s*/i', '/^\s*\[[^\]]{2,80}\]\s*/', '/^\s*(?:re\s*)?posted\s+by\s+[^-]{2,120}\s*-\s*/i', ]; @@ -25,6 +26,7 @@ final class ObfuscatedSubjectExtractor '/\.7z$/i', '/\.rar$/i', '/\.par2?$/i', + '/\.part$/i', '/\.vol\d+[+\-]\d+\.par2?$/i', '/\.\d{3}$/', ]; @@ -64,6 +66,7 @@ final class ObfuscatedSubjectExtractor $normalized = preg_replace('/\.(?=[A-Za-z0-9])/', ' ', $normalized) ?? $normalized; $normalized = preg_replace('/\s{2,}/', ' ', $normalized) ?? $normalized; $normalized = trim($normalized, " \t\n\r\0\x0B\"'`-_."); + $normalized = $this->toReadableTitle($normalized); if ($normalized === '' || $normalized === $original) { return null; @@ -71,4 +74,15 @@ final class ObfuscatedSubjectExtractor return $normalized; } + + private function toReadableTitle(string $value): string + { + // If the candidate looks fully lowercase, present a cleaner title-cased + // variant for UI/searchname while preserving separators and numbers. + if (preg_match('/\p{Lu}/u', $value) === 1) { + return $value; + } + + return mb_convert_case($value, MB_CASE_TITLE, 'UTF-8'); + } } diff --git a/app/Services/NameFixing/FileNameCleaner.php b/app/Services/NameFixing/FileNameCleaner.php index 2f2cf432a..283973dd6 100644 --- a/app/Services/NameFixing/FileNameCleaner.php +++ b/app/Services/NameFixing/FileNameCleaner.php @@ -184,6 +184,44 @@ class FileNameCleaner return trim($t, " .-_\t\r\n"); } + /** + * Format a candidate for storage in searchname. + */ + public function formatSearchName(string $title, ?string $normalizedFallback = null): string + { + $formatted = trim($title); + + $formatted = $this->extractNzbSplitName($formatted) ?? $formatted; + + // Remove common media/archive extensions while keeping scene separators. + $formatted = preg_replace(self::VIDEO_EXTENSIONS, '', $formatted) ?? $formatted; + $formatted = preg_replace(self::AUDIO_EXTENSIONS, '', $formatted) ?? $formatted; + $formatted = preg_replace(self::IMAGE_EXTENSIONS, '', $formatted) ?? $formatted; + $formatted = preg_replace(self::EBOOK_EXTENSIONS, '', $formatted) ?? $formatted; + $formatted = preg_replace(self::GAMEAPP_EXTENSIONS, '', $formatted) ?? $formatted; + $formatted = preg_replace(self::SUBTITLE_EXTENSIONS, '', $formatted) ?? $formatted; + + foreach (self::ARCHIVE_PATTERNS as $pattern) { + $formatted = preg_replace($pattern, '', $formatted) ?? $formatted; + } + + $formatted = preg_replace('/[.\-_ ](?:part|vol|r)\d+(?:\+\d+)?$/i', '', $formatted) ?? $formatted; + $formatted = trim($formatted, " .-_\t\r\n"); + + if ($formatted === '') { + return $normalizedFallback ?? $this->normalizeCandidateTitle($title); + } + + if (! $this->looksLikeSceneRelease($formatted)) { + return $normalizedFallback ?? $this->normalizeCandidateTitle($title); + } + + $formatted = preg_replace('/[ _]+/', '.', $formatted) ?? $formatted; + $formatted = preg_replace('/\.{2,}/', '.', $formatted) ?? $formatted; + + return trim($formatted, " .-_\t\r\n"); + } + /** * Check if a title is plausible for a release. * diff --git a/app/Services/NameFixing/ReleaseUpdateService.php b/app/Services/NameFixing/ReleaseUpdateService.php index 00a70f34b..fc75cf858 100644 --- a/app/Services/NameFixing/ReleaseUpdateService.php +++ b/app/Services/NameFixing/ReleaseUpdateService.php @@ -137,14 +137,15 @@ class ReleaseUpdateService } if ($this->relid !== $release->releases_id) { - $newName = (new ReleaseCleaningService)->fixerCleaner($name); + $cleanedName = (new ReleaseCleaningService)->fixerCleaner($name); // Normalize and sanity-check candidate for non-trusted sources - $newName = $this->fileNameCleaner->normalizeCandidateTitle($newName); + $normalizedName = $this->fileNameCleaner->normalizeCandidateTitle($cleanedName); + $newName = $this->fileNameCleaner->formatSearchName($cleanedName, $normalizedName); // Determine if the source is trusted enough to bypass plausibility checks $trustedSource = $this->isTrustedSource($type, $method, $preId); - if (! $trustedSource && ! $this->fileNameCleaner->isPlausibleReleaseTitle($newName)) { + if (! $trustedSource && ! $this->fileNameCleaner->isPlausibleReleaseTitle($normalizedName)) { // Skip low-quality rename candidates for untrusted sources $this->done = true; diff --git a/app/Services/Runners/PostProcessRunner.php b/app/Services/Runners/PostProcessRunner.php index 20d75598f..2974c3fb3 100644 --- a/app/Services/Runners/PostProcessRunner.php +++ b/app/Services/Runners/PostProcessRunner.php @@ -12,6 +12,13 @@ use Illuminate\Support\Facades\Log; class PostProcessRunner extends BaseRunner { + private function guidBucketExpression(string $column = 'leftguid'): string + { + return DB::getDriverName() === 'sqlite' + ? 'substr('.$column.', 1, 1)' + : 'LEFT('.$column.', 1)'; + } + /** * @param array<string, mixed> $releases */ @@ -40,6 +47,17 @@ class PostProcessRunner extends BaseRunner $count = count($releases); $this->headerStart('postprocess: '.$desc, $count, $maxProcesses); + if ($count <= 1 || $maxProcesses <= 1) { + foreach ($releases as $release) { + $char = isset($release->id) ? substr((string) $release->id, 0, 1) : ''; + $command = PHP_BINARY.' artisan postprocess:guid '.$type.' '.$char; + echo $this->executeCommand($command); + cli()->primary('Finished task for '.$desc); + } + + return; + } + // Process in batches using Laravel's native Concurrency facade $batches = array_chunk($releases, max(1, $maxProcesses)); @@ -68,13 +86,14 @@ class PostProcessRunner extends BaseRunner public function processAdditional(): void { + $bucketExpr = $this->guidBucketExpression('r.leftguid'); $ppAddMinSize = Settings::settingValue('minsizetopostprocess') !== '' ? (int) Settings::settingValue('minsizetopostprocess') : 1; $ppAddMinSize = ($ppAddMinSize > 0 ? ('AND r.size > '.($ppAddMinSize * 1048576)) : ''); $ppAddMaxSize = (Settings::settingValue('maxsizetopostprocess') !== '') ? (int) Settings::settingValue('maxsizetopostprocess') : 100; $ppAddMaxSize = ($ppAddMaxSize > 0 ? ('AND r.size < '.($ppAddMaxSize * 1073741824)) : ''); $sql = ' - SELECT DISTINCT LEFT(r.leftguid, 1) AS id + SELECT DISTINCT '.$bucketExpr.' AS id FROM releases r LEFT JOIN categories c ON c.id = r.categories_id WHERE r.passwordstatus = -1 @@ -105,8 +124,9 @@ class PostProcessRunner extends BaseRunner return; } + $bucketExpr = $this->guidBucketExpression('r.leftguid'); $sql = ' - SELECT DISTINCT LEFT(r.leftguid, 1) AS id + SELECT DISTINCT '.$bucketExpr.' AS id FROM releases r WHERE 1=1 '.$nfoQuery.' LIMIT 16'; @@ -141,8 +161,9 @@ class PostProcessRunner extends BaseRunner } $renamedFlag = ($renamedOnly ? 2 : 1); + $bucketExpr = $this->guidBucketExpression(); $sql = ' - SELECT DISTINCT LEFT(leftguid, 1) AS id, '.$renamedFlag.' AS renamed + SELECT DISTINCT '.$bucketExpr.' AS id, '.$renamedFlag.' AS renamed FROM releases WHERE categories_id BETWEEN 2000 AND 2999 AND '.imdb_id_needs_lookup_sql('imdbid').' @@ -182,8 +203,9 @@ class PostProcessRunner extends BaseRunner } $renamedFlag = ($renamedOnly ? 2 : 1); + $bucketExpr = $this->guidBucketExpression(); $sql = ' - SELECT DISTINCT LEFT(leftguid, 1) AS id, '.$renamedFlag.' AS renamed + SELECT DISTINCT '.$bucketExpr.' AS id, '.$renamedFlag.' AS renamed FROM releases WHERE categories_id BETWEEN 5000 AND 5999 AND categories_id != 5070 @@ -304,8 +326,9 @@ class PostProcessRunner extends BaseRunner return; } + $bucketExpr = $this->guidBucketExpression(); $sql = ' - SELECT DISTINCT LEFT(leftguid, 1) AS id + SELECT DISTINCT '.$bucketExpr.' AS id FROM releases WHERE categories_id = 5070 AND anidbid IS NULL @@ -327,8 +350,17 @@ class PostProcessRunner extends BaseRunner $checkSql = ' SELECT id FROM releases - WHERE categories_id BETWEEN 7000 AND 7999 - AND bookinfo_id IS NULL + WHERE ( + categories_id BETWEEN 7000 AND 7999 + OR categories_id = 3030 + ) + AND ( + bookinfo_id IS NULL + OR searchname LIKE "N:/NZB%" + OR searchname LIKE "N_NZB_%" + OR name LIKE "N:/NZB%" + OR name LIKE "N_NZB_%" + ) LIMIT 1'; if (count(DB::select($checkSql)) === 0) { $this->headerNone(); @@ -336,11 +368,21 @@ class PostProcessRunner extends BaseRunner return; } + $bucketExpr = $this->guidBucketExpression(); $sql = ' - SELECT DISTINCT LEFT(leftguid, 1) AS id + SELECT DISTINCT '.$bucketExpr.' AS id FROM releases - WHERE categories_id BETWEEN 7000 AND 7999 - AND bookinfo_id IS NULL + WHERE ( + categories_id BETWEEN 7000 AND 7999 + OR categories_id = 3030 + ) + AND ( + bookinfo_id IS NULL + OR searchname LIKE "N:/NZB%" + OR searchname LIKE "N_NZB_%" + OR name LIKE "N:/NZB%" + OR name LIKE "N_NZB_%" + ) LIMIT 16'; $queue = DB::select($sql); diff --git a/app/Support/BookMatchScorer.php b/app/Support/BookMatchScorer.php index 8f79c0970..08ecef68c 100644 --- a/app/Support/BookMatchScorer.php +++ b/app/Support/BookMatchScorer.php @@ -36,10 +36,12 @@ class BookMatchScorer + (0.07 * $coverScore); } - return (0.65 * $titleScore) - + (0.15 * $yearScore) - + (0.10 * $publisherScore) - + (0.10 * $coverScore); + // When the release has no reliable author, avoid over-trusting + // "has cover/publisher" signals for weak title matches. + return (0.80 * $titleScore) + + (0.10 * $yearScore) + + (0.05 * $publisherScore) + + (0.05 * $coverScore); } public function scoreBookInfo(BookInfo $book, BookParseResult $parsed): float diff --git a/resources/views/admin/releases/index.blade.php b/resources/views/admin/releases/index.blade.php index f4293821d..687ad80c6 100644 --- a/resources/views/admin/releases/index.blade.php +++ b/resources/views/admin/releases/index.blade.php @@ -26,8 +26,8 @@ <div class="text-gray-900 dark:text-gray-100 font-medium max-w-md wrap-break-word break-all" title="{{ $release->searchname }}"> {{ $release->searchname }} </div> - <div class="text-gray-500 dark:text-gray-400 text-xs mt-1 max-w-md truncate" title="{{ $release->name }}"> - {{ $release->name }} + <div class="text-gray-500 dark:text-gray-400 text-xs mt-1 max-w-md truncate" title="{{ $release->searchname }}"> + {{ $release->searchname }} </div> </td> <td class="px-6 py-4 whitespace-nowrap"> diff --git a/tests/Feature/AdditionalProcessingNzbSplitRenameTest.php b/tests/Feature/AdditionalProcessingNzbSplitRenameTest.php index 21f9fbab8..b9f467605 100644 --- a/tests/Feature/AdditionalProcessingNzbSplitRenameTest.php +++ b/tests/Feature/AdditionalProcessingNzbSplitRenameTest.php @@ -59,6 +59,35 @@ class AdditionalProcessingNzbSplitRenameTest extends TestCase $this->assertSame($expected, $context->release->searchname); } + public function test_additional_postprocessing_skips_non_nzbsplit_wrapped_titles(): void + { + $wrapped = "N_NZB_[1_6]_-_Woman's_Day_New_Zealand_-_Issue_45_April_27_2026.par2"; + + $release = new Release([ + 'id' => 125, + 'releases_id' => 125, + 'name' => $wrapped, + 'searchname' => $wrapped, + 'fromname' => 'poster@example.com', + 'guid' => str_repeat('b', 40), + 'groups_id' => 1, + 'categories_id' => Category::OTHER_HASHED, + ]); + + $context = new ReleaseProcessingContext($release); + $updateService = $this->createMock(ReleaseUpdateService::class); + $updateService->expects($this->never())->method('updateRelease'); + + $manager = $this->makeReleaseFileManager($updateService); + + $renamed = $manager->processReleaseNameFromNzbContents([ + ['title' => $wrapped], + ], $context); + + $this->assertFalse($renamed); + $this->assertSame($wrapped, $context->release->searchname); + } + public function test_additional_postprocessing_skips_low_information_nzb_split_payloads(): void { $wrapped = 'TEST__NZBSPLIT__1234567890abcdef__NZBSPLIT__setup.7z.001'; @@ -85,6 +114,33 @@ class AdditionalProcessingNzbSplitRenameTest extends TestCase $this->assertSame($wrapped, $context->release->searchname); } + public function test_additional_postprocessing_does_not_rename_movie_style_obfuscated_subjects(): void + { + $wrapped = '[1/6] - "Yoh! Bestie 2026 1080p NF WEB-DL H 264 DDP5 1-UBWEB.par2" yEnc'; + $existingName = 'Yoh!_Bestie_2026_1080p_NF_WEB-DL_H_264_DDP5_1-UBWEB'; + $release = new Release([ + 'id' => 126, + 'releases_id' => 126, + 'name' => $wrapped, + 'searchname' => $existingName, + 'groups_id' => 1, + 'categories_id' => Category::MOVIE_HD, + ]); + + $context = new ReleaseProcessingContext($release); + $updateService = $this->createMock(ReleaseUpdateService::class); + $updateService->expects($this->never())->method('updateRelease'); + + $manager = $this->makeReleaseFileManager($updateService); + + $renamed = $manager->processReleaseNameFromNzbContents([ + ['title' => $wrapped], + ], $context); + + $this->assertFalse($renamed); + $this->assertSame($existingName, $context->release->searchname); + } + public function test_archive_name_extractor_unwraps_nzb_split_file_names(): void { $manager = $this->makeReleaseFileManager(); diff --git a/tests/Feature/ReleaseNameFixedRecategorizationTest.php b/tests/Feature/ReleaseNameFixedRecategorizationTest.php index 093127922..45b3623df 100644 --- a/tests/Feature/ReleaseNameFixedRecategorizationTest.php +++ b/tests/Feature/ReleaseNameFixedRecategorizationTest.php @@ -179,6 +179,52 @@ class ReleaseNameFixedRecategorizationTest extends TestCase $this->assertSame(1, (int) $release->isrenamed); } + public function test_renaming_space_separated_scene_title_does_not_overwrite_dotted_searchname(): void + { + Search::shouldReceive('updateRelease')->once(); + + $group = UsenetGroup::query()->create([ + 'name' => 'alt.binaries.multimedia', + 'active' => 1, + 'backfill' => 0, + ]); + + $oldName = 'Southern.Charm.S11E12.Even.Further.South.720p.AMZN.WEB-DL.DDP2.0.H.264-NTb'; + $release = Release::factory()->create([ + 'name' => '[1/25] - "Southern.Charm.S11E12.Even.Further.South.720p.AMZN.WEB-DL.DDP2.0.H.264-NTb.par2" yEnc', + 'searchname' => $oldName, + 'fromname' => 'poster@example.com', + 'groups_id' => $group->id, + 'categories_id' => Category::TV_HD, + 'iscategorized' => 1, + 'isrenamed' => 1, + 'guid' => str_repeat('c', 40), + 'leftguid' => 'c', + 'nzb_guid' => 'test-southern-charm', + 'size' => 1, + 'postdate' => now(), + 'adddate' => now(), + ]); + + $service = app(ReleaseUpdateService::class); + $service->updateRelease( + $release->fresh(), + 'Southern Charm S11E12 Even Further South 720p AMZN WEB-DL DDP2 0 H 264-NTb', + 'RarInfo FileName Match', + true, + 'Filenames, ', + true, + false, + ); + + $release->refresh(); + + $this->assertSame($oldName, $release->searchname); + $this->assertSame(Category::TV_HD, $release->categories_id); + $this->assertSame(1, (int) $release->iscategorized); + $this->assertSame(1, (int) $release->isrenamed); + } + private function setEnvironmentValue(string $key, ?string $value): void { if ($value === null) { diff --git a/tests/Unit/Services/BookServiceTest.php b/tests/Unit/Services/BookServiceTest.php index 46f25b206..d83829771 100644 --- a/tests/Unit/Services/BookServiceTest.php +++ b/tests/Unit/Services/BookServiceTest.php @@ -126,4 +126,33 @@ class BookServiceTest extends TestCase $this->assertSame('History of War - Issue 158,', $parsed->title); $this->assertTrue($parsed->isMagazine); } + + public function test_parse_release_name_marks_mcn_hybrid_magazine_as_magazine(): void + { + $service = $this->makeService(); + + $parsed = $service->parseReleaseName('MCN.April.22.2026.HYBRID.MAGAZINE.eBook-21A1', 'ebook'); + + $this->assertTrue($parsed->isMagazine); + $this->assertSame('MCN - April 22, 2026', $parsed->title); + } + + public function test_parse_release_name_marks_normalized_mcn_date_as_magazine(): void + { + $service = $this->makeService(); + + $parsed = $service->parseReleaseName('MCN April 22, 2026', 'ebook'); + + $this->assertTrue($parsed->isMagazine); + } + + public function test_parse_release_name_keeps_mcn_prefix_in_title_for_magazine(): void + { + $service = $this->makeService(); + + $parsed = $service->parseReleaseName('MCN - April 22, 2026', 'ebook'); + + $this->assertTrue($parsed->isMagazine); + $this->assertStringStartsWith('MCN', $parsed->title); + } } diff --git a/tests/Unit/Services/NameFixing/FileNameCleanerTest.php b/tests/Unit/Services/NameFixing/FileNameCleanerTest.php index d5d92c801..0d2df6da8 100644 --- a/tests/Unit/Services/NameFixing/FileNameCleanerTest.php +++ b/tests/Unit/Services/NameFixing/FileNameCleanerTest.php @@ -38,4 +38,14 @@ class FileNameCleanerTest extends TestCase $this->assertSame('WOB Klassik 4.25', $cleaner->normalizeCandidateTitle('WOB Klassik 4.25.Pdf')); } + + public function test_format_search_name_keeps_scene_titles_dotted(): void + { + $cleaner = new FileNameCleaner; + + $this->assertSame( + 'Southern.Charm.S11E12.Even.Further.South.720p.AMZN.WEB-DL.DDP2.0.H.264-NTb', + $cleaner->formatSearchName('Southern Charm S11E12 Even Further South 720p AMZN WEB-DL DDP2 0 H 264-NTb.mkv') + ); + } } diff --git a/tests/Unit/Services/NameFixing/ObfuscatedSubjectExtractorTest.php b/tests/Unit/Services/NameFixing/ObfuscatedSubjectExtractorTest.php index e3827bd97..0ff3f7c16 100644 --- a/tests/Unit/Services/NameFixing/ObfuscatedSubjectExtractorTest.php +++ b/tests/Unit/Services/NameFixing/ObfuscatedSubjectExtractorTest.php @@ -49,4 +49,34 @@ class ObfuscatedSubjectExtractorTest extends TestCase $this->assertNull($result); } + + #[Test] + public function it_extracts_underscore_nzb_prefix_without_quotes(): void + { + $extractor = new ObfuscatedSubjectExtractor; + + $result = $extractor->extract('N_NZB_[6]_-_Woman\'s_Day_New_Zealand_-_Issue_45_April_27_2026.par2'); + + $this->assertSame('Woman\'s Day New Zealand - Issue 45 April 27 2026', $result); + } + + #[Test] + public function it_extracts_underscore_fraction_nzb_prefix_without_quotes(): void + { + $extractor = new ObfuscatedSubjectExtractor; + + $result = $extractor->extract('N_NZB_[1_6]_-_Woman\'s_Day_New_Zealand_-_Issue_45_April_27_2026.par2'); + + $this->assertSame('Woman\'s Day New Zealand - Issue 45 April 27 2026', $result); + } + + #[Test] + public function it_title_cases_lowercase_obfuscated_candidates(): void + { + $extractor = new ObfuscatedSubjectExtractor; + + $result = $extractor->extract('N:/NZB [02/11] - "landscape.garden.design.issue.2.2026.part1.rar" yEnc'); + + $this->assertSame('Landscape Garden Design Issue 2 2026', $result); + } } diff --git a/tests/Unit/Support/BookMatchScorerTest.php b/tests/Unit/Support/BookMatchScorerTest.php index 145f48bda..1bf1c8cb3 100644 --- a/tests/Unit/Support/BookMatchScorerTest.php +++ b/tests/Unit/Support/BookMatchScorerTest.php @@ -78,4 +78,23 @@ class BookMatchScorerTest extends TestCase $this->assertLessThan(0.55, $score, 'Unrelated book with partial word overlap should not reach match threshold'); } + + public function test_diabetes_title_collision_scores_below_no_author_cutoff(): void + { + $parsed = new BookParseResult( + rawName: 'Living With Diabetes - 1st Edition 2026', + title: 'Living With Diabetes - 1st Edition 2026', + year: 2026 + ); + + $score = (new BookMatchScorer)->score([ + 'title' => 'Healthy Lifestyle Against Diabetes 1st Edition', + 'author' => 'Amie Armstrong', + 'publishdate' => '2018-11-04', + 'publisher' => 'Independently Published', + 'cover' => 1, + ], $parsed); + + $this->assertLessThan(0.68, $score, 'Ambiguous no-author title collision should stay below stricter cutoff'); + } }