From 282070a2afae0a82df4a5dbfde239e2223280482 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Wed, 13 May 2026 08:51:11 +0200 Subject: [PATCH] Update search and indexing --- app/Console/Commands/NntmuxSearchDiag.php | 217 ++++++++++++++++++ .../Commands/NntmuxSearchReconcile.php | 161 +++++++++++++ app/Services/Nzb/NzbImportService.php | 103 ++++++--- app/Services/ReleaseCreationService.php | 94 +++++--- app/Services/ReleaseProcessingService.php | 7 +- .../Releases/ReleaseDuplicateFinder.php | 74 ++++++ .../Search/Drivers/ManticoreSearchDriver.php | 106 +++++++-- config/nntmux.php | 11 + ...5_12_000000_add_ix_releases_searchname.php | 40 ++++ tests/Feature/CbpCleanupServiceTest.php | 176 +++++++++++++- 10 files changed, 903 insertions(+), 86 deletions(-) create mode 100644 app/Console/Commands/NntmuxSearchDiag.php create mode 100644 app/Console/Commands/NntmuxSearchReconcile.php create mode 100644 app/Services/Releases/ReleaseDuplicateFinder.php create mode 100644 database/migrations/2026_05_12_000000_add_ix_releases_searchname.php diff --git a/app/Console/Commands/NntmuxSearchDiag.php b/app/Console/Commands/NntmuxSearchDiag.php new file mode 100644 index 000000000..7f12870bc --- /dev/null +++ b/app/Console/Commands/NntmuxSearchDiag.php @@ -0,0 +1,217 @@ +error('This command only supports SEARCH_DRIVER=manticore.'); + + return self::FAILURE; + } + + if (! $manticore->isAvailable()) { + $this->error('Manticore is not reachable (isAvailable() returned false).'); + + return self::FAILURE; + } + + $index = $manticore->getReleasesIndex(); + $nameFilter = (string) $this->option('name'); + $ids = $this->argument('ids'); + + $releaseIds = []; + if ($nameFilter !== '') { + $limit = max(1, min(500, (int) $this->option('limit'))); + $like = '%'.addcslashes($nameFilter, '%_\\').'%'; + $releaseIds = Release::query() + ->where(function ($q) use ($like): void { + $q->where('name', 'LIKE', $like) + ->orWhere('searchname', 'LIKE', $like); + }) + ->orderByDesc('id') + ->limit($limit) + ->pluck('id') + ->map(static fn ($id): int => (int) $id) + ->all(); + if ($releaseIds === []) { + $this->warn('No releases matched --name filter.'); + + return self::SUCCESS; + } + } else { + if ($ids === [] || $ids === null) { + $this->error('Provide at least one id/guid or use --name='); + + return self::FAILURE; + } + foreach ($ids as $token) { + $resolved = $this->resolveReleaseId((string) $token); + if ($resolved === null) { + $this->warn("No release found for token: {$token}"); + + continue; + } + $releaseIds[] = $resolved; + } + } + + if ($releaseIds === []) { + return self::FAILURE; + } + + $showPasswords = (int) Settings::settingValue('showpasswordedrelease') === 1; + $verbose = (bool) $this->option('show-fields'); + + foreach ($releaseIds as $rid) { + $this->diagnoseOne((int) $rid, $manticore, $index, $showPasswords, $verbose); + } + + return self::SUCCESS; + } + + private function resolveReleaseId(string $token): ?int + { + if (ctype_digit($token)) { + $id = (int) $token; + $exists = Release::query()->whereKey($id)->exists(); + + return $exists ? $id : null; + } + + return Release::query()->where('guid', $token)->value('id'); + } + + /** + * @return array> + */ + private function manticoreSqlRows(ManticoreSearchDriver $driver, string $sql): array + { + try { + $response = $driver->manticoreSearch->sql($sql, true); + } catch (Throwable $e) { + $this->error('Manticore SQL error: '.$e->getMessage()); + + return []; + } + + if (! \is_array($response)) { + return []; + } + + if (isset($response['data']) && \is_array($response['data'])) { + return array_values($response['data']); + } + + return []; + } + + private function diagnoseOne( + int $releaseId, + ManticoreSearchDriver $driver, + string $index, + bool $showPasswords, + bool $verbose, + ): void { + $release = Release::query()->find($releaseId); + if ($release === null) { + $this->line(sprintf('%d | NOT_IN_DB', $releaseId)); + + return; + } + + $sql = sprintf('SELECT * FROM `%s` WHERE id = %d', str_replace('`', '``', $index), $releaseId); + $rows = $this->manticoreSqlRows($driver, $sql); + $indexRow = $rows[0] ?? null; + + $status = $this->classify($release, $indexRow, $showPasswords, $verbose); + + $this->line(sprintf( + '%d | %s | searchname=%s | size=%s | passwordstatus=%s', + $releaseId, + $status, + $release->searchname, + (string) $release->size, + (string) $release->passwordstatus + )); + + if ($verbose) { + $this->line('DB: '.json_encode($release->only([ + 'id', 'name', 'searchname', 'categories_id', 'passwordstatus', 'nzbstatus', + 'groups_id', 'size', 'postdate', 'adddate', + ]))); + $this->line('IX: '.json_encode($indexRow)); + } + } + + /** + * @param array|null $indexRow + */ + private function classify(Release $release, ?array $indexRow, bool $showPasswords, bool $verbose): string + { + if ($indexRow === null) { + return 'MISSING_IN_INDEX'; + } + + $hidden = false; + if (! $showPasswords && (int) $release->passwordstatus !== 0) { + $hidden = true; + } + + $expected = ReleaseSearchIndexDocument::normalize($release->toArray()); + unset($expected['id']); + + $drift = false; + foreach (['searchname', 'name', 'categories_id', 'passwordstatus', 'size', 'groups_id', 'nzbstatus'] as $field) { + $dbVal = $expected[$field] ?? null; + $ixVal = $indexRow[$field] ?? null; + if ((string) $dbVal !== (string) $ixVal) { + $drift = true; + if ($verbose) { + $this->line(" drift: {$field} db=".json_encode($dbVal).' ix='.json_encode($ixVal)); + } + } + } + + $dbPostTs = (int) ($expected['postdate_ts'] ?? 0); + $ixPostTs = isset($indexRow['postdate_ts']) ? (int) $indexRow['postdate_ts'] : 0; + if (abs($dbPostTs - $ixPostTs) > 2) { + $drift = true; + } + + $dbAddTs = (int) ($expected['adddate_ts'] ?? 0); + $ixAddTs = isset($indexRow['adddate_ts']) ? (int) $indexRow['adddate_ts'] : 0; + if (abs($dbAddTs - $ixAddTs) > 2) { + $drift = true; + } + + if ($drift) { + return 'STALE_INDEX'; + } + + if ($hidden) { + return 'STATUS_HIDDEN'; + } + + return 'OK'; + } +} diff --git a/app/Console/Commands/NntmuxSearchReconcile.php b/app/Console/Commands/NntmuxSearchReconcile.php new file mode 100644 index 000000000..b92cb974a --- /dev/null +++ b/app/Console/Commands/NntmuxSearchReconcile.php @@ -0,0 +1,161 @@ +error('This command only supports SEARCH_DRIVER=manticore.'); + + return self::FAILURE; + } + + if (! $manticore->isAvailable()) { + $this->error('Manticore is not reachable.'); + + return self::FAILURE; + } + + $since = (string) $this->option('since'); + $cutoff = $this->parseSinceCutoff($since); + $chunk = max(50, min(5000, (int) $this->option('chunk'))); + $inBatch = max(50, min(1000, (int) $this->option('in-batch'))); + $dryRun = (bool) $this->option('dry-run'); + $reindex = (bool) $this->option('reindex'); + + if ($reindex && $dryRun) { + $this->warn('--reindex with --dry-run: no writes will be performed.'); + $reindex = false; + } + + $index = $manticore->getReleasesIndex(); + $missingAll = []; + $scanned = 0; + + $bar = $this->output->createProgressBar(); + $bar->start(); + + Release::query() + ->where('adddate', '>=', $cutoff) + ->orderBy('id') + ->chunkById($chunk, function ($releases) use ($manticore, $index, $inBatch, &$missingAll, &$scanned, $bar): void { + $ids = $releases->pluck('id')->map(static fn ($id): int => (int) $id)->all(); + $scanned += \count($ids); + foreach (array_chunk($ids, $inBatch) as $batch) { + $indexed = $this->fetchIndexedIds($manticore, $index, $batch); + foreach (array_diff($batch, $indexed) as $mid) { + $missingAll[] = (int) $mid; + } + } + $bar->advance($releases->count()); + }, 'id'); + + $bar->finish(); + $this->newLine(2); + + $missingTotal = \count($missingAll); + $this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} (adddate >= {$cutoff->toDateTimeString()})"); + if ($missingTotal > 0) { + $sample = \array_slice($missingAll, 0, 20); + $this->line('Sample ids: '.implode(', ', $sample)); + } + + if ($missingTotal === 0) { + return self::SUCCESS; + } + + if (! $reindex) { + $this->comment('Run with --reindex to push Search::updateRelease() for each missing id.'); + + return self::SUCCESS; + } + + $this->info('Reindexing missing releases...'); + $done = 0; + foreach ($missingAll as $mid) { + try { + Search::updateRelease($mid); + $done++; + } catch (Throwable $e) { + $this->error("Failed reindex id={$mid}: ".$e->getMessage()); + } + } + + $this->info("Reindexed {$done} release(s)."); + + return self::SUCCESS; + } + + private function parseSinceCutoff(string $since): Carbon + { + $since = trim($since); + if (preg_match('/^(\d+)h$/i', $since, $m)) { + return Carbon::now()->subHours((int) $m[1]); + } + if (preg_match('/^(\d+)d$/i', $since, $m)) { + return Carbon::now()->subDays((int) $m[1]); + } + + try { + return Carbon::parse($since, config('app.timezone')); + } catch (Throwable) { + $this->warn("Could not parse --since={$since}; defaulting to 24 hours."); + + return Carbon::now()->subHours(24); + } + } + + /** + * @param list $ids + * @return list + */ + private function fetchIndexedIds(ManticoreSearchDriver $driver, string $index, array $ids): array + { + if ($ids === []) { + return []; + } + + $list = implode(',', array_map(static fn (int $id): string => (string) $id, $ids)); + $safeIndex = str_replace('`', '``', $index); + $sql = "SELECT id FROM `{$safeIndex}` WHERE id IN ({$list})"; + + try { + $response = $driver->manticoreSearch->sql($sql, true); + } catch (Throwable) { + return []; + } + + if (! \is_array($response) || ! isset($response['data']) || ! \is_array($response['data'])) { + return []; + } + + $out = []; + foreach ($response['data'] as $row) { + if (isset($row['id'])) { + $out[] = (int) $row['id']; + } + } + + return $out; + } +} diff --git a/app/Services/Nzb/NzbImportService.php b/app/Services/Nzb/NzbImportService.php index 2ed3b3282..eea44e209 100644 --- a/app/Services/Nzb/NzbImportService.php +++ b/app/Services/Nzb/NzbImportService.php @@ -5,15 +5,19 @@ declare(strict_types=1); namespace App\Services\Nzb; use App\Enums\NzbImportStatus; +use App\Models\Predb; use App\Models\Release; use App\Models\Settings; use App\Models\UsenetGroup; use App\Services\BlacklistService; use App\Services\Categorization\CategorizationService; use App\Services\ReleaseCleaningService; +use App\Services\Releases\ReleaseDuplicateFinder; +use App\Support\Utf8; use Illuminate\Contracts\Filesystem\FileNotFoundException; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; /** @@ -31,6 +35,8 @@ class NzbImportService protected CategorizationService $category; + protected ReleaseDuplicateFinder $releaseDuplicateFinder; + /** * List of all the group names/ids in the DB. * @@ -67,6 +73,7 @@ class NzbImportService $this->category = new CategorizationService; $this->nzb = app(NzbService::class); $this->releaseCleaner = new ReleaseCleaningService; + $this->releaseDuplicateFinder = app(ReleaseDuplicateFinder::class); $this->crossPostt = Settings::settingValue('crossposttime') !== '' ? Settings::settingValue('crossposttime') : 2; // Set properties from options @@ -386,51 +393,89 @@ class NzbImportService $subject = mb_convert_encoding(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess)), 'UTF-8', mb_list_encodings()); $renamed = 0; + $cleanedMeta = null; if ($nzbDetails['useFName'] !== '') { - // If we are using the filename as the subject. We don't need to clean it. $cleanName = $nzbDetails['useFName']; $renamed = 1; } else { - // Pass the subject through release cleaner to get a nicer name. - $cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['groupName']); - if (isset($cleanName['properlynamed'])) { - $cleanName = $cleanName['cleansubject']; - $renamed = (isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0); + $cleanedMeta = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['groupName']); + if (\is_array($cleanedMeta)) { + $cleanName = $cleanedMeta['cleansubject'] ?? $subject; + $renamed = (isset($cleanedMeta['properlynamed']) && $cleanedMeta['properlynamed'] === true) ? 1 : 0; + } else { + $cleanName = \is_string($cleanedMeta) ? $cleanedMeta : $subject; + } + } + + if (! \is_string($cleanName)) { + $cleanName = $subject; + } + + $preIdFromCleaner = 0; + if (\is_array($cleanedMeta) && isset($cleanedMeta['predb']) && (int) $cleanedMeta['predb'] > 0) { + $preIdFromCleaner = (int) $cleanedMeta['predb']; + } + + $predbIdInt = $preIdFromCleaner; + if ($predbIdInt === 0 && $cleanName !== '') { + $preMatch = Predb::matchPre($cleanName); + if ($preMatch !== false) { + $cleanName = $preMatch['title']; + $predbIdInt = (int) $preMatch['predb_id']; + $renamed = 1; } } $escapedSubject = $subject; $escapedFromName = $nzbDetails['from']; + $escapedSearchName = Utf8::clean($cleanName); + if ($escapedSearchName === '') { + $escapedSearchName = $escapedSubject; + } - // Look for a duplicate on name, poster and size. - $dupeCheck = Release::query()->where(['name' => $escapedSubject, 'fromname' => $escapedFromName])->whereBetween('size', [$nzbDetails['totalSize'] * 0.99, $nzbDetails['totalSize'] * 1.01])->first(['id']); + [$dupeCheck, $dupeReason] = $this->releaseDuplicateFinder->findDuplicate( + $escapedSubject, + $escapedSearchName, + $predbIdInt, + (int) $nzbDetails['totalSize'] + ); - if ($dupeCheck === null) { - $escapedSearchName = $cleanName; - $determinedCategory = $this->category->determineCategory($nzbDetails['groups_id'], $cleanName, $escapedFromName); - // Insert the release into the DB. - $relID = Release::insertRelease( - [ - 'name' => $escapedSubject, - 'searchname' => $escapedSearchName ?? $escapedSubject, - 'totalpart' => $nzbDetails['totalFiles'], - 'groups_id' => $nzbDetails['groups_id'], - 'guid' => $this->relGuid, - 'postdate' => $nzbDetails['postDate'], - 'fromname' => $escapedFromName, - 'size' => $nzbDetails['totalSize'], - 'categories_id' => $determinedCategory['categories_id'], - 'isrenamed' => $renamed, - 'predb_id' => 0, - 'nzbstatus' => NzbService::NZB_ADDED, - ] - ); - } else { + if ($dupeCheck !== null) { + Log::info('NZB import skipped as duplicate', [ + 'reason' => $dupeReason, + 'matched_release_id' => $dupeCheck->id, + 'new_searchname' => $escapedSearchName, + 'existing_searchname' => $dupeCheck->searchname, + 'new_size' => (int) $nzbDetails['totalSize'], + 'existing_size' => (int) $dupeCheck->size, + 'new_fromname' => $escapedFromName, + 'existing_fromname' => $dupeCheck->fromname, + 'new_name' => $escapedSubject, + 'existing_name' => $dupeCheck->name, + ]); $this->echoOut('This release is already in our DB so skipping: '.$subject); return NzbImportStatus::Duplicate; } + $determinedCategory = $this->category->determineCategory($nzbDetails['groups_id'], $cleanName, $escapedFromName); + $relID = Release::insertRelease( + [ + 'name' => $escapedSubject, + 'searchname' => $escapedSearchName, + 'totalpart' => $nzbDetails['totalFiles'], + 'groups_id' => $nzbDetails['groups_id'], + 'guid' => $this->relGuid, + 'postdate' => $nzbDetails['postDate'], + 'fromname' => $escapedFromName, + 'size' => $nzbDetails['totalSize'], + 'categories_id' => $determinedCategory['categories_id'], + 'isrenamed' => $renamed, + 'predb_id' => $predbIdInt, + 'nzbstatus' => NzbService::NZB_ADDED, + ] + ); + if ($relID === null) { $this->echoOut('ERROR: Problem inserting: '.$subject); diff --git a/app/Services/ReleaseCreationService.php b/app/Services/ReleaseCreationService.php index 1c4c3a04a..ae8efece1 100644 --- a/app/Services/ReleaseCreationService.php +++ b/app/Services/ReleaseCreationService.php @@ -14,8 +14,10 @@ use App\Models\ReleasesGroups; use App\Models\UsenetGroup; use App\Services\Categorization\CategorizationService; use App\Services\Nzb\NzbService; +use App\Services\Releases\ReleaseDuplicateFinder; use App\Support\Utf8; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; class ReleaseCreationService @@ -23,6 +25,7 @@ class ReleaseCreationService public function __construct( private readonly ReleaseCleaningService $releaseCleaning, private readonly CollectionCleanupService $collectionCleanupService, + private readonly ReleaseDuplicateFinder $releaseDuplicateFinder, ) {} /** @@ -62,47 +65,49 @@ class ReleaseCreationService $cleanRelName = Utf8::clean(str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection->subject)); $fromName = Utf8::clean(trim($collection->fromname, "'")); - // Deduplicate by name, from, and ~size - $dupeCheck = Release::query() - ->where(['name' => $cleanRelName, 'fromname' => $fromName]) - ->whereBetween('size', [$collection->filesize * .99, $collection->filesize * 1.01]) - ->first(['id']); + $cleanedMeta = $this->releaseCleaning->releaseCleaner( + $collection->subject, + $collection->fromname, + $collection->gname + ); + + $namingRegexId = 0; + if (\is_array($cleanedMeta)) { + $namingRegexId = isset($cleanedMeta['id']) ? (int) $cleanedMeta['id'] : 0; + } + + if (\is_array($cleanedMeta)) { + $properName = $cleanedMeta['properlynamed'] ?? false; + $preID = $cleanedMeta['predb'] ?? false; + $cleanedName = $cleanedMeta['cleansubject'] ?? $cleanRelName; + } else { + $properName = true; + $preID = false; + $cleanedName = $cleanRelName; + } + + if ($preID === false && $cleanedName !== '') { + $preMatch = Predb::matchPre($cleanedName); + if ($preMatch !== false) { + $cleanedName = $preMatch['title']; + $preID = $preMatch['predb_id']; + $properName = true; + } + } + + $searchName = ! empty($cleanedName) ? Utf8::clean($cleanedName) : $cleanRelName; + $predbIdInt = $preID === false ? 0 : (int) $preID; + + [$dupeCheck, $dupeReason] = $this->releaseDuplicateFinder->findDuplicate( + $cleanRelName, + $searchName, + $predbIdInt, + (int) $collection->filesize + ); if ($dupeCheck === null) { - $cleanedMeta = $this->releaseCleaning->releaseCleaner( - $collection->subject, - $collection->fromname, - $collection->gname - ); - - $namingRegexId = 0; - if (\is_array($cleanedMeta)) { - $namingRegexId = isset($cleanedMeta['id']) ? (int) $cleanedMeta['id'] : 0; - } - - if (\is_array($cleanedMeta)) { - $properName = $cleanedMeta['properlynamed'] ?? false; - $preID = $cleanedMeta['predb'] ?? false; - $cleanedName = $cleanedMeta['cleansubject'] ?? $cleanRelName; - } else { - $properName = true; - $preID = false; - $cleanedName = $cleanRelName; - } - - if ($preID === false && $cleanedName !== '') { - $preMatch = Predb::matchPre($cleanedName); - if ($preMatch !== false) { - $cleanedName = $preMatch['title']; - $preID = $preMatch['predb_id']; - $properName = true; - } - } - $determinedCategory = $categorize->determineCategory($collection->groups_id, $cleanedName, $fromName); - $searchName = ! empty($cleanedName) ? Utf8::clean($cleanedName) : $cleanRelName; - $releaseID = Release::insertRelease([ 'name' => $cleanRelName, 'searchname' => $searchName, @@ -114,7 +119,7 @@ class ReleaseCreationService 'size' => $collection->filesize, 'categories_id' => $determinedCategory['categories_id'] ?? Category::OTHER_MISC, 'isrenamed' => $properName === true ? 1 : 0, - 'predb_id' => $preID === false ? 0 : $preID, + 'predb_id' => $predbIdInt, 'nzbstatus' => NzbService::NZB_NONE, ]); @@ -172,6 +177,19 @@ class ReleaseCreationService } } } else { + Log::info('Release import skipped as duplicate', [ + 'reason' => $dupeReason, + 'matched_release_id' => $dupeCheck->id, + 'new_searchname' => $searchName, + 'existing_searchname' => $dupeCheck->searchname, + 'new_size' => (int) $collection->filesize, + 'existing_size' => (int) $dupeCheck->size, + 'new_fromname' => $fromName, + 'existing_fromname' => $dupeCheck->fromname, + 'new_name' => $cleanRelName, + 'existing_name' => $dupeCheck->name, + ]); + $this->collectionCleanupService->deleteCollectionsAndDescendants( [$collection->id], 'Duplicate cleanup', diff --git a/app/Services/ReleaseProcessingService.php b/app/Services/ReleaseProcessingService.php index 802df2f1a..535fe0135 100644 --- a/app/Services/ReleaseProcessingService.php +++ b/app/Services/ReleaseProcessingService.php @@ -16,6 +16,7 @@ use App\Services\Categorization\CategorizationService; use App\Services\NNTP\NNTPService; use App\Services\Nzb\NzbService; use App\Services\Releases\ReleaseBrowseService; +use App\Services\Releases\ReleaseDuplicateFinder; use App\Services\Releases\ReleaseManagementService; use App\Support\Data\ProcessReleasesSettings; use App\Support\Data\ReleaseCreationResult; @@ -89,7 +90,11 @@ final class ReleaseProcessingService ?? new CollectionCleanupService; $this->releaseCreationService = $releaseCreationService - ?? new ReleaseCreationService($this->releaseCleaning, $this->collectionCleanupService); + ?? new ReleaseCreationService( + $this->releaseCleaning, + $this->collectionCleanupService, + app(ReleaseDuplicateFinder::class) + ); $this->postProcessService = $postProcessService; $this->settings = $this->loadSettings(); diff --git a/app/Services/Releases/ReleaseDuplicateFinder.php b/app/Services/Releases/ReleaseDuplicateFinder.php new file mode 100644 index 000000000..9ad17aa41 --- /dev/null +++ b/app/Services/Releases/ReleaseDuplicateFinder.php @@ -0,0 +1,74 @@ +whereBetween('size', [$lowSize, $highSize]); + + if ($predbId > 0 || $searchName !== '') { + $query->where(function (Builder $w) use ($searchName, $predbId): void { + if ($predbId > 0) { + $w->where('predb_id', $predbId); + if ($searchName !== '') { + $w->orWhere('searchname', $searchName); + } + } elseif ($searchName !== '') { + $w->where('searchname', $searchName); + } else { + $w->whereRaw('1 = 0'); + } + }); + } else { + $query->where('name', $cleanRelName); + } + + $dup = $query->first(['id', 'predb_id', 'searchname', 'fromname', 'size', 'name']); + + if ($dup === null) { + return [null, null]; + } + + $reason = $this->resolveReason($dup, $searchName, $predbId); + + return [$dup, $reason]; + } + + private function resolveReason(Release $dup, string $searchName, int $predbId): string + { + if ($predbId > 0 && (int) $dup->predb_id === $predbId) { + return 'predb_id_match'; + } + + if ($searchName !== '' && (string) $dup->searchname === $searchName) { + return 'searchname_match'; + } + + return 'name_match_fallback'; + } +} diff --git a/app/Services/Search/Drivers/ManticoreSearchDriver.php b/app/Services/Search/Drivers/ManticoreSearchDriver.php index 7f1ef5076..af64d9ab3 100644 --- a/app/Services/Search/Drivers/ManticoreSearchDriver.php +++ b/app/Services/Search/Drivers/ManticoreSearchDriver.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace App\Services\Search\Drivers; use App\Enums\SecondarySearchIndex; +use App\Jobs\ReindexReleaseJob; use App\Models\BookInfo; use App\Models\ConsoleInfo; use App\Models\GamesInfo; @@ -168,26 +169,95 @@ class ManticoreSearchDriver implements SearchDriverInterface return; } - try { - $document = ReleaseSearchIndexDocument::normalize($parameters); - unset($document['id']); + $releaseId = (int) $parameters['id']; + if (! $this->replaceReleaseDocumentWithRetry($parameters)) { + $this->recordReleaseIndexFailure($releaseId, 'insertRelease'); + try { + ReindexReleaseJob::dispatch($releaseId)->delay(now()->addSeconds(2)); + } catch (\Throwable $e) { + Log::error('ManticoreSearch: failed to queue ReindexReleaseJob', [ + 'release_id' => $releaseId, + 'error' => $e->getMessage(), + ]); + } + } + } - $this->manticoreSearch->table($this->config['indexes']['releases']) - ->replaceDocument($document, $parameters['id']); + /** + * @param array $parameters + */ + private function replaceReleaseDocumentWithRetry(array $parameters): bool + { + $document = ReleaseSearchIndexDocument::normalize($parameters); + unset($document['id']); - } catch (ResponseException $e) { - Log::error('ManticoreSearch insertRelease ResponseException: '.$e->getMessage(), [ - 'release_id' => $parameters['id'], - 'index' => $this->config['indexes']['releases'], + $indexName = $this->config['indexes']['releases']; + $releaseId = (int) $parameters['id']; + + for ($attempt = 0; $attempt < 2; $attempt++) { + try { + $this->manticoreSearch->table($indexName) + ->replaceDocument($document, $releaseId); + + return true; + } catch (\Throwable $e) { + $isResponse = $e instanceof ResponseException; + $isManticoreRuntime = $e instanceof RuntimeException; + + if (($isResponse || $isManticoreRuntime) && $attempt === 0) { + usleep(100_000); + + continue; + } + + if ($isResponse) { + Log::error('ManticoreSearch insertRelease ResponseException: '.$e->getMessage(), [ + 'release_id' => $releaseId, + 'index' => $indexName, + ]); + + return false; + } + + if ($isManticoreRuntime) { + Log::error('ManticoreSearch insertRelease RuntimeException: '.$e->getMessage(), [ + 'release_id' => $releaseId, + ]); + + return false; + } + + Log::error('ManticoreSearch insertRelease unexpected error: '.$e->getMessage(), [ + 'release_id' => $releaseId, + 'trace' => $e->getTraceAsString(), + ]); + + return false; + } + } + + return false; + } + + private function recordReleaseIndexFailure(int $releaseId, string $phase): void + { + Cache::increment('search:index:failures:releases'); + if (Cache::add('search:index:release_index_warn_lock', true, 60)) { + Log::warning('ManticoreSearch: release indexing failure', [ + 'release_id_sample' => $releaseId, + 'phase' => $phase, + 'failures_total' => (int) Cache::get('search:index:failures:releases', 0), ]); - } catch (RuntimeException $e) { - Log::error('ManticoreSearch insertRelease RuntimeException: '.$e->getMessage(), [ - 'release_id' => $parameters['id'], - ]); - } catch (\Throwable $e) { - Log::error('ManticoreSearch insertRelease unexpected error: '.$e->getMessage(), [ - 'release_id' => $parameters['id'], - 'trace' => $e->getTraceAsString(), + } + } + + private function recordReleaseNotFoundForIndex(int|string $releaseID): void + { + Cache::increment('search:index:failures:release_not_found'); + if (Cache::add('search:index:release_not_found_warn_lock', true, 60)) { + Log::warning('ManticoreSearch: release not found when updating index', [ + 'release_id' => $releaseID, + 'not_found_total' => (int) Cache::get('search:index:failures:release_not_found', 0), ]); } } @@ -648,11 +718,13 @@ class ManticoreSearchDriver implements SearchDriverInterface $this->insertRelease($release->toArray()); } else { Log::warning('ManticoreSearch: Release not found for update', ['id' => $releaseID]); + $this->recordReleaseNotFoundForIndex($releaseID); } } catch (\Throwable $e) { Log::error('ManticoreSearch updateRelease error: '.$e->getMessage(), [ 'release_id' => $releaseID, ]); + $this->recordReleaseIndexFailure((int) $releaseID, 'updateRelease_query'); } } diff --git a/config/nntmux.php b/config/nntmux.php index 18b187edb..19f67a000 100644 --- a/config/nntmux.php +++ b/config/nntmux.php @@ -21,6 +21,17 @@ return [ 'purge_inactive_users' => env('PURGE_INACTIVE_USERS', false), 'purge_inactive_users_days' => env('PURGE_INACTIVE_USERS_DAYS', 180), 'mysql_search_fallback' => env('MYSQL_SEARCH_FALLBACK', false), // Disable MySQL LIKE fallback when Manticore/Elasticsearch return no results + + /* + |-------------------------------------------------------------------------- + | Release dedupe (import-time) + |-------------------------------------------------------------------------- + | + | Size tolerance for matching an existing release when deduping imports + | (collections / NZB). Default 0.05 = ±5% on total bytes (par2/RAR drift). + | + */ + 'release_dedupe_size_tolerance' => (float) env('RELEASE_DEDUPE_SIZE_TOLERANCE', 0.05), 'btcpay_webhook_secret' => env('BTCPAY_SECRET'), 'tmp_unrar_path' => env('TEMP_UNRAR_PATH', storage_path('tmp/unrar/')), 'tmp_unzip_path' => env('TEMP_UNZIP_PATH', storage_path('tmp/unzip/')), diff --git a/database/migrations/2026_05_12_000000_add_ix_releases_searchname.php b/database/migrations/2026_05_12_000000_add_ix_releases_searchname.php new file mode 100644 index 000000000..c302cdf9a --- /dev/null +++ b/database/migrations/2026_05_12_000000_add_ix_releases_searchname.php @@ -0,0 +1,40 @@ +indexExists('ix_releases_searchname')) { + return; + } + + Schema::table('releases', function (Blueprint $table): void { + $table->index('searchname', 'ix_releases_searchname'); + }); + } + + public function down(): void + { + if (! $this->indexExists('ix_releases_searchname')) { + return; + } + + Schema::table('releases', function (Blueprint $table): void { + $table->dropIndex('ix_releases_searchname'); + }); + } + + private function indexExists(string $keyName): bool + { + $rows = DB::select('SHOW INDEX FROM `releases` WHERE Key_name = ?', [$keyName]); + + return $rows !== []; + } +}; diff --git a/tests/Feature/CbpCleanupServiceTest.php b/tests/Feature/CbpCleanupServiceTest.php index 4950bbaeb..ba2a12b21 100644 --- a/tests/Feature/CbpCleanupServiceTest.php +++ b/tests/Feature/CbpCleanupServiceTest.php @@ -8,6 +8,7 @@ use App\Services\CollectionCleanupService; use App\Services\Nzb\NzbService; use App\Services\ReleaseCleaningService; use App\Services\ReleaseCreationService; +use App\Services\Releases\ReleaseDuplicateFinder; use Illuminate\Support\Facades\DB; use Tests\TestCase; @@ -21,6 +22,20 @@ class CbpCleanupServiceTest extends TestCase DB::purge(); DB::reconnect(); DB::connection()->getPdo()->sqliteCreateFunction('UNIX_TIMESTAMP', static fn (?string $value): int => strtotime((string) $value)); + DB::connection()->getPdo()->sqliteCreateFunction( + 'REGEXP', + static function (?string $subject, ?string $pattern): int { + if ($subject === null || $pattern === null || $pattern === '') { + return 0; + } + set_error_handler(static fn (): true => true); + $ok = @preg_match($pattern, $subject); + restore_error_handler(); + + return $ok ? 1 : 0; + }, + 2 + ); $this->createTables(); $this->seedSettings(); @@ -192,7 +207,8 @@ class CbpCleanupServiceTest extends TestCase $service = new ReleaseCreationService( app(ReleaseCleaningService::class), - app(CollectionCleanupService::class) + app(CollectionCleanupService::class), + app(ReleaseDuplicateFinder::class) ); $result = $service->createReleases(null, 10, false); @@ -202,6 +218,145 @@ class CbpCleanupServiceTest extends TestCase $this->assertSame(0, DB::table('collections')->count()); } + public function test_release_duplicate_finder_matches_searchname_within_size_band(): void + { + DB::table('releases')->insert([ + 'id' => 20, + 'name' => 'raw-obfuscated-a', + 'searchname' => 'Unified.Scene.S01E01.1080p', + 'totalpart' => 1, + 'groups_id' => 1, + 'adddate' => now()->format('Y-m-d H:i:s'), + 'guid' => str_repeat('c', 36), + 'leftguid' => 'c', + 'postdate' => now()->format('Y-m-d H:i:s'), + 'fromname' => 'poster-a@example.com', + 'size' => 1_000_000, + 'passwordstatus' => 0, + 'haspreview' => -1, + 'categories_id' => 1, + 'nfostatus' => -1, + 'nzbstatus' => NzbService::NZB_NONE, + 'isrenamed' => 1, + 'iscategorized' => 1, + 'predb_id' => 0, + 'source' => null, + ]); + + $finder = app(ReleaseDuplicateFinder::class); + [$dup, $reason] = $finder->findDuplicate( + 'raw-obfuscated-b', + 'Unified.Scene.S01E01.1080p', + 0, + 1_020_000 + ); + + $this->assertNotNull($dup); + $this->assertSame('searchname_match', $reason); + } + + public function test_release_duplicate_finder_matches_predb_id_when_searchname_differs(): void + { + DB::table('releases')->insert([ + 'id' => 21, + 'name' => 'old', + 'searchname' => 'Old Style Name', + 'totalpart' => 1, + 'groups_id' => 1, + 'adddate' => now()->format('Y-m-d H:i:s'), + 'guid' => str_repeat('d', 36), + 'leftguid' => 'd', + 'postdate' => now()->format('Y-m-d H:i:s'), + 'fromname' => 'p@example.com', + 'size' => 2_000_000, + 'passwordstatus' => 0, + 'haspreview' => -1, + 'categories_id' => 1, + 'nfostatus' => -1, + 'nzbstatus' => NzbService::NZB_NONE, + 'isrenamed' => 1, + 'iscategorized' => 1, + 'predb_id' => 9001, + 'source' => null, + ]); + + $finder = app(ReleaseDuplicateFinder::class); + [$dup, $reason] = $finder->findDuplicate( + 'new', + 'New Style Name', + 9001, + 2_050_000 + ); + + $this->assertNotNull($dup); + $this->assertSame('predb_id_match', $reason); + } + + public function test_release_duplicate_finder_does_not_match_outside_size_tolerance(): void + { + config(['nntmux.release_dedupe_size_tolerance' => 0.05]); + + DB::table('releases')->insert([ + 'id' => 22, + 'name' => 'x', + 'searchname' => 'Same.Search', + 'totalpart' => 1, + 'groups_id' => 1, + 'adddate' => now()->format('Y-m-d H:i:s'), + 'guid' => str_repeat('e', 36), + 'leftguid' => 'e', + 'postdate' => now()->format('Y-m-d H:i:s'), + 'fromname' => 'p@example.com', + 'size' => 1_000_000, + 'passwordstatus' => 0, + 'haspreview' => -1, + 'categories_id' => 1, + 'nfostatus' => -1, + 'nzbstatus' => NzbService::NZB_NONE, + 'isrenamed' => 1, + 'iscategorized' => 1, + 'predb_id' => 0, + 'source' => null, + ]); + + $finder = app(ReleaseDuplicateFinder::class); + [$dup] = $finder->findDuplicate('x', 'Same.Search', 0, 1_200_000); + + $this->assertNull($dup); + } + + public function test_release_duplicate_finder_falls_back_to_name_when_searchname_empty(): void + { + DB::table('releases')->insert([ + 'id' => 23, + 'name' => 'fallback.unique', + 'searchname' => '', + 'totalpart' => 1, + 'groups_id' => 1, + 'adddate' => now()->format('Y-m-d H:i:s'), + 'guid' => str_repeat('f', 36), + 'leftguid' => 'f', + 'postdate' => now()->format('Y-m-d H:i:s'), + 'fromname' => 'p@example.com', + 'size' => 500, + 'passwordstatus' => 0, + 'haspreview' => -1, + 'categories_id' => 1, + 'nfostatus' => -1, + 'nzbstatus' => NzbService::NZB_NONE, + 'isrenamed' => 1, + 'iscategorized' => 1, + 'predb_id' => 0, + 'source' => null, + ]); + + $finder = app(ReleaseDuplicateFinder::class); + [$dup, $reason] = $finder->findDuplicate('fallback.unique', '', 0, 500); + + $this->assertNotNull($dup); + $this->assertSame('name_match_fallback', $reason); + } + private function seedSettings(): void { $settings = [ @@ -274,6 +429,25 @@ class CbpCleanupServiceTest extends TestCase partnumber INTEGER, size INTEGER )'); + DB::statement('CREATE TABLE release_naming_regexes ( + id INTEGER PRIMARY KEY, + group_regex VARCHAR(255), + regex VARCHAR(255), + status INTEGER DEFAULT 1, + ordinal INTEGER DEFAULT 0 + )'); + DB::statement('CREATE TABLE collection_regexes ( + id INTEGER PRIMARY KEY, + group_regex VARCHAR(255), + regex VARCHAR(255), + status INTEGER DEFAULT 1, + ordinal INTEGER DEFAULT 0 + )'); + DB::statement('CREATE TABLE predb ( + id INTEGER PRIMARY KEY, + title VARCHAR(255), + filename VARCHAR(255) + )'); DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.test']); DB::table('categories')->insert(['id' => 1, 'title' => 'Misc', 'parent_categories_id' => null]); }