Update search and indexing

2026-08-28 17:01:16 +00:00 · 2026-05-13 08:51:11 +02:00
parent 5e74a4d769
commit 282070a2af
10 changed files with 903 additions and 86 deletions
@@ -0,0 +1,217 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Models\Release;
+use App\Models\Settings;
+use App\Services\Search\Drivers\ManticoreSearchDriver;
+use App\Support\ReleaseSearchIndexDocument;
+use Illuminate\Console\Command;
+use Throwable;
+
+class NntmuxSearchDiag extends Command
+{
+    protected $signature = 'nntmux:search-diag
+                            {ids?* : Release numeric ids or guids}
+                            {--name= : Match releases where name or searchname contains this substring (MySQL)}
+                            {--limit=20 : Max rows when using --name}
+                            {--show-fields : Dump DB row and index document as JSON lines}';
+
+    protected $description = 'Compare releases table rows to Manticore releases_rt documents (diagnostics)';
+
+    public function handle(ManticoreSearchDriver $manticore): int
+    {
+        if (config('search.default') !== 'manticore') {
+            $this->error('This command only supports SEARCH_DRIVER=manticore.');
+
+            return self::FAILURE;
+        }
+
+        if (! $manticore->isAvailable()) {
+            $this->error('Manticore is not reachable (isAvailable() returned false).');
+
+            return self::FAILURE;
+        }
+
+        $index = $manticore->getReleasesIndex();
+        $nameFilter = (string) $this->option('name');
+        $ids = $this->argument('ids');
+
+        $releaseIds = [];
+        if ($nameFilter !== '') {
+            $limit = max(1, min(500, (int) $this->option('limit')));
+            $like = '%'.addcslashes($nameFilter, '%_\\').'%';
+            $releaseIds = Release::query()
+                ->where(function ($q) use ($like): void {
+                    $q->where('name', 'LIKE', $like)
+                        ->orWhere('searchname', 'LIKE', $like);
+                })
+                ->orderByDesc('id')
+                ->limit($limit)
+                ->pluck('id')
+                ->map(static fn ($id): int => (int) $id)
+                ->all();
+            if ($releaseIds === []) {
+                $this->warn('No releases matched --name filter.');
+
+                return self::SUCCESS;
+            }
+        } else {
+            if ($ids === [] || $ids === null) {
+                $this->error('Provide at least one id/guid or use --name=');
+
+                return self::FAILURE;
+            }
+            foreach ($ids as $token) {
+                $resolved = $this->resolveReleaseId((string) $token);
+                if ($resolved === null) {
+                    $this->warn("No release found for token: {$token}");
+
+                    continue;
+                }
+                $releaseIds[] = $resolved;
+            }
+        }
+
+        if ($releaseIds === []) {
+            return self::FAILURE;
+        }
+
+        $showPasswords = (int) Settings::settingValue('showpasswordedrelease') === 1;
+        $verbose = (bool) $this->option('show-fields');
+
+        foreach ($releaseIds as $rid) {
+            $this->diagnoseOne((int) $rid, $manticore, $index, $showPasswords, $verbose);
+        }
+
+        return self::SUCCESS;
+    }
+
+    private function resolveReleaseId(string $token): ?int
+    {
+        if (ctype_digit($token)) {
+            $id = (int) $token;
+            $exists = Release::query()->whereKey($id)->exists();
+
+            return $exists ? $id : null;
+        }
+
+        return Release::query()->where('guid', $token)->value('id');
+    }
+
+    /**
+     * @return array<int, array<string, mixed>>
+     */
+    private function manticoreSqlRows(ManticoreSearchDriver $driver, string $sql): array
+    {
+        try {
+            $response = $driver->manticoreSearch->sql($sql, true);
+        } catch (Throwable $e) {
+            $this->error('Manticore SQL error: '.$e->getMessage());
+
+            return [];
+        }
+
+        if (! \is_array($response)) {
+            return [];
+        }
+
+        if (isset($response['data']) && \is_array($response['data'])) {
+            return array_values($response['data']);
+        }
+
+        return [];
+    }
+
+    private function diagnoseOne(
+        int $releaseId,
+        ManticoreSearchDriver $driver,
+        string $index,
+        bool $showPasswords,
+        bool $verbose,
+    ): void {
+        $release = Release::query()->find($releaseId);
+        if ($release === null) {
+            $this->line(sprintf('%d | NOT_IN_DB', $releaseId));
+
+            return;
+        }
+
+        $sql = sprintf('SELECT * FROM `%s` WHERE id = %d', str_replace('`', '``', $index), $releaseId);
+        $rows = $this->manticoreSqlRows($driver, $sql);
+        $indexRow = $rows[0] ?? null;
+
+        $status = $this->classify($release, $indexRow, $showPasswords, $verbose);
+
+        $this->line(sprintf(
+            '%d | %s | searchname=%s | size=%s | passwordstatus=%s',
+            $releaseId,
+            $status,
+            $release->searchname,
+            (string) $release->size,
+            (string) $release->passwordstatus
+        ));
+
+        if ($verbose) {
+            $this->line('DB: '.json_encode($release->only([
+                'id', 'name', 'searchname', 'categories_id', 'passwordstatus', 'nzbstatus',
+                'groups_id', 'size', 'postdate', 'adddate',
+            ])));
+            $this->line('IX: '.json_encode($indexRow));
+        }
+    }
+
+    /**
+     * @param  array<string, mixed>|null  $indexRow
+     */
+    private function classify(Release $release, ?array $indexRow, bool $showPasswords, bool $verbose): string
+    {
+        if ($indexRow === null) {
+            return 'MISSING_IN_INDEX';
+        }
+
+        $hidden = false;
+        if (! $showPasswords && (int) $release->passwordstatus !== 0) {
+            $hidden = true;
+        }
+
+        $expected = ReleaseSearchIndexDocument::normalize($release->toArray());
+        unset($expected['id']);
+
+        $drift = false;
+        foreach (['searchname', 'name', 'categories_id', 'passwordstatus', 'size', 'groups_id', 'nzbstatus'] as $field) {
+            $dbVal = $expected[$field] ?? null;
+            $ixVal = $indexRow[$field] ?? null;
+            if ((string) $dbVal !== (string) $ixVal) {
+                $drift = true;
+                if ($verbose) {
+                    $this->line("  drift: {$field} db=".json_encode($dbVal).' ix='.json_encode($ixVal));
+                }
+            }
+        }
+
+        $dbPostTs = (int) ($expected['postdate_ts'] ?? 0);
+        $ixPostTs = isset($indexRow['postdate_ts']) ? (int) $indexRow['postdate_ts'] : 0;
+        if (abs($dbPostTs - $ixPostTs) > 2) {
+            $drift = true;
+        }
+
+        $dbAddTs = (int) ($expected['adddate_ts'] ?? 0);
+        $ixAddTs = isset($indexRow['adddate_ts']) ? (int) $indexRow['adddate_ts'] : 0;
+        if (abs($dbAddTs - $ixAddTs) > 2) {
+            $drift = true;
+        }
+
+        if ($drift) {
+            return 'STALE_INDEX';
+        }
+
+        if ($hidden) {
+            return 'STATUS_HIDDEN';
+        }
+
+        return 'OK';
+    }
+}
@@ -0,0 +1,161 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Facades\Search;
+use App\Models\Release;
+use App\Services\Search\Drivers\ManticoreSearchDriver;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Throwable;
+
+class NntmuxSearchReconcile extends Command
+{
+    protected $signature = 'nntmux:search-reconcile
+                            {--since=24h : Only releases with adddate after this window (e.g. 24h, 7d)}
+                            {--chunk=1000 : MySQL chunk size when scanning releases}
+                            {--in-batch=500 : Max ids per Manticore IN() query}
+                            {--reindex : Call Search::updateRelease for each missing id}
+                            {--dry-run : Report only; do not write to the index}';
+
+    protected $description = 'Find releases rows missing from Manticore releases_rt and optionally reindex them';
+
+    public function handle(ManticoreSearchDriver $manticore): int
+    {
+        if (config('search.default') !== 'manticore') {
+            $this->error('This command only supports SEARCH_DRIVER=manticore.');
+
+            return self::FAILURE;
+        }
+
+        if (! $manticore->isAvailable()) {
+            $this->error('Manticore is not reachable.');
+
+            return self::FAILURE;
+        }
+
+        $since = (string) $this->option('since');
+        $cutoff = $this->parseSinceCutoff($since);
+        $chunk = max(50, min(5000, (int) $this->option('chunk')));
+        $inBatch = max(50, min(1000, (int) $this->option('in-batch')));
+        $dryRun = (bool) $this->option('dry-run');
+        $reindex = (bool) $this->option('reindex');
+
+        if ($reindex && $dryRun) {
+            $this->warn('--reindex with --dry-run: no writes will be performed.');
+            $reindex = false;
+        }
+
+        $index = $manticore->getReleasesIndex();
+        $missingAll = [];
+        $scanned = 0;
+
+        $bar = $this->output->createProgressBar();
+        $bar->start();
+
+        Release::query()
+            ->where('adddate', '>=', $cutoff)
+            ->orderBy('id')
+            ->chunkById($chunk, function ($releases) use ($manticore, $index, $inBatch, &$missingAll, &$scanned, $bar): void {
+                $ids = $releases->pluck('id')->map(static fn ($id): int => (int) $id)->all();
+                $scanned += \count($ids);
+                foreach (array_chunk($ids, $inBatch) as $batch) {
+                    $indexed = $this->fetchIndexedIds($manticore, $index, $batch);
+                    foreach (array_diff($batch, $indexed) as $mid) {
+                        $missingAll[] = (int) $mid;
+                    }
+                }
+                $bar->advance($releases->count());
+            }, 'id');
+
+        $bar->finish();
+        $this->newLine(2);
+
+        $missingTotal = \count($missingAll);
+        $this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} (adddate >= {$cutoff->toDateTimeString()})");
+        if ($missingTotal > 0) {
+            $sample = \array_slice($missingAll, 0, 20);
+            $this->line('Sample ids: '.implode(', ', $sample));
+        }
+
+        if ($missingTotal === 0) {
+            return self::SUCCESS;
+        }
+
+        if (! $reindex) {
+            $this->comment('Run with --reindex to push Search::updateRelease() for each missing id.');
+
+            return self::SUCCESS;
+        }
+
+        $this->info('Reindexing missing releases...');
+        $done = 0;
+        foreach ($missingAll as $mid) {
+            try {
+                Search::updateRelease($mid);
+                $done++;
+            } catch (Throwable $e) {
+                $this->error("Failed reindex id={$mid}: ".$e->getMessage());
+            }
+        }
+
+        $this->info("Reindexed {$done} release(s).");
+
+        return self::SUCCESS;
+    }
+
+    private function parseSinceCutoff(string $since): Carbon
+    {
+        $since = trim($since);
+        if (preg_match('/^(\d+)h$/i', $since, $m)) {
+            return Carbon::now()->subHours((int) $m[1]);
+        }
+        if (preg_match('/^(\d+)d$/i', $since, $m)) {
+            return Carbon::now()->subDays((int) $m[1]);
+        }
+
+        try {
+            return Carbon::parse($since, config('app.timezone'));
+        } catch (Throwable) {
+            $this->warn("Could not parse --since={$since}; defaulting to 24 hours.");
+
+            return Carbon::now()->subHours(24);
+        }
+    }
+
+    /**
+     * @param  list<int>  $ids
+     * @return list<int>
+     */
+    private function fetchIndexedIds(ManticoreSearchDriver $driver, string $index, array $ids): array
+    {
+        if ($ids === []) {
+            return [];
+        }
+
+        $list = implode(',', array_map(static fn (int $id): string => (string) $id, $ids));
+        $safeIndex = str_replace('`', '``', $index);
+        $sql = "SELECT id FROM `{$safeIndex}` WHERE id IN ({$list})";
+
+        try {
+            $response = $driver->manticoreSearch->sql($sql, true);
+        } catch (Throwable) {
+            return [];
+        }
+
+        if (! \is_array($response) || ! isset($response['data']) || ! \is_array($response['data'])) {
+            return [];
+        }
+
+        $out = [];
+        foreach ($response['data'] as $row) {
+            if (isset($row['id'])) {
+                $out[] = (int) $row['id'];
+            }
+        }
+
+        return $out;
+    }
+}
@@ -5,15 +5,19 @@ declare(strict_types=1);
 namespace App\Services\Nzb;

 use App\Enums\NzbImportStatus;
+use App\Models\Predb;
 use App\Models\Release;
 use App\Models\Settings;
 use App\Models\UsenetGroup;
 use App\Services\BlacklistService;
 use App\Services\Categorization\CategorizationService;
 use App\Services\ReleaseCleaningService;
+use App\Services\Releases\ReleaseDuplicateFinder;
+use App\Support\Utf8;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;

 /**
@@ -31,6 +35,8 @@ class NzbImportService

    protected CategorizationService $category;

+    protected ReleaseDuplicateFinder $releaseDuplicateFinder;
+
    /**
     * List of all the group names/ids in the DB.
     *
@@ -67,6 +73,7 @@ class NzbImportService
        $this->category = new CategorizationService;
        $this->nzb = app(NzbService::class);
        $this->releaseCleaner = new ReleaseCleaningService;
+        $this->releaseDuplicateFinder = app(ReleaseDuplicateFinder::class);
        $this->crossPostt = Settings::settingValue('crossposttime') !== '' ? Settings::settingValue('crossposttime') : 2;

        // Set properties from options
@@ -386,51 +393,89 @@ class NzbImportService
        $subject = mb_convert_encoding(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess)), 'UTF-8', mb_list_encodings());

        $renamed = 0;
+        $cleanedMeta = null;
        if ($nzbDetails['useFName'] !== '') {
-            // If we are using the filename as the subject. We don't need to clean it.
            $cleanName = $nzbDetails['useFName'];
            $renamed = 1;
        } else {
-            // Pass the subject through release cleaner to get a nicer name.
-            $cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['groupName']);
-            if (isset($cleanName['properlynamed'])) {
-                $cleanName = $cleanName['cleansubject'];
-                $renamed = (isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0);
+            $cleanedMeta = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['groupName']);
+            if (\is_array($cleanedMeta)) {
+                $cleanName = $cleanedMeta['cleansubject'] ?? $subject;
+                $renamed = (isset($cleanedMeta['properlynamed']) && $cleanedMeta['properlynamed'] === true) ? 1 : 0;
+            } else {
+                $cleanName = \is_string($cleanedMeta) ? $cleanedMeta : $subject;
+            }
+        }
+
+        if (! \is_string($cleanName)) {
+            $cleanName = $subject;
+        }
+
+        $preIdFromCleaner = 0;
+        if (\is_array($cleanedMeta) && isset($cleanedMeta['predb']) && (int) $cleanedMeta['predb'] > 0) {
+            $preIdFromCleaner = (int) $cleanedMeta['predb'];
+        }
+
+        $predbIdInt = $preIdFromCleaner;
+        if ($predbIdInt === 0 && $cleanName !== '') {
+            $preMatch = Predb::matchPre($cleanName);
+            if ($preMatch !== false) {
+                $cleanName = $preMatch['title'];
+                $predbIdInt = (int) $preMatch['predb_id'];
+                $renamed = 1;
            }
        }

        $escapedSubject = $subject;
        $escapedFromName = $nzbDetails['from'];
+        $escapedSearchName = Utf8::clean($cleanName);
+        if ($escapedSearchName === '') {
+            $escapedSearchName = $escapedSubject;
+        }

-        // Look for a duplicate on name, poster and size.
-        $dupeCheck = Release::query()->where(['name' => $escapedSubject, 'fromname' => $escapedFromName])->whereBetween('size', [$nzbDetails['totalSize'] * 0.99, $nzbDetails['totalSize'] * 1.01])->first(['id']);
+        [$dupeCheck, $dupeReason] = $this->releaseDuplicateFinder->findDuplicate(
+            $escapedSubject,
+            $escapedSearchName,
+            $predbIdInt,
+            (int) $nzbDetails['totalSize']
+        );

-        if ($dupeCheck === null) {
-            $escapedSearchName = $cleanName;
-            $determinedCategory = $this->category->determineCategory($nzbDetails['groups_id'], $cleanName, $escapedFromName);
-            // Insert the release into the DB.
-            $relID = Release::insertRelease(
-                [
-                    'name' => $escapedSubject,
-                    'searchname' => $escapedSearchName ?? $escapedSubject,
-                    'totalpart' => $nzbDetails['totalFiles'],
-                    'groups_id' => $nzbDetails['groups_id'],
-                    'guid' => $this->relGuid,
-                    'postdate' => $nzbDetails['postDate'],
-                    'fromname' => $escapedFromName,
-                    'size' => $nzbDetails['totalSize'],
-                    'categories_id' => $determinedCategory['categories_id'],
-                    'isrenamed' => $renamed,
-                    'predb_id' => 0,
-                    'nzbstatus' => NzbService::NZB_ADDED,
-                ]
-            );
-        } else {
+        if ($dupeCheck !== null) {
+            Log::info('NZB import skipped as duplicate', [
+                'reason' => $dupeReason,
+                'matched_release_id' => $dupeCheck->id,
+                'new_searchname' => $escapedSearchName,
+                'existing_searchname' => $dupeCheck->searchname,
+                'new_size' => (int) $nzbDetails['totalSize'],
+                'existing_size' => (int) $dupeCheck->size,
+                'new_fromname' => $escapedFromName,
+                'existing_fromname' => $dupeCheck->fromname,
+                'new_name' => $escapedSubject,
+                'existing_name' => $dupeCheck->name,
+            ]);
            $this->echoOut('This release is already in our DB so skipping: '.$subject);

            return NzbImportStatus::Duplicate;
        }

+        $determinedCategory = $this->category->determineCategory($nzbDetails['groups_id'], $cleanName, $escapedFromName);
+        $relID = Release::insertRelease(
+            [
+                'name' => $escapedSubject,
+                'searchname' => $escapedSearchName,
+                'totalpart' => $nzbDetails['totalFiles'],
+                'groups_id' => $nzbDetails['groups_id'],
+                'guid' => $this->relGuid,
+                'postdate' => $nzbDetails['postDate'],
+                'fromname' => $escapedFromName,
+                'size' => $nzbDetails['totalSize'],
+                'categories_id' => $determinedCategory['categories_id'],
+                'isrenamed' => $renamed,
+                'predb_id' => $predbIdInt,
+                'nzbstatus' => NzbService::NZB_ADDED,
+            ]
+        );
+
        if ($relID === null) {
            $this->echoOut('ERROR: Problem inserting: '.$subject);

@@ -14,8 +14,10 @@ use App\Models\ReleasesGroups;
 use App\Models\UsenetGroup;
 use App\Services\Categorization\CategorizationService;
 use App\Services\Nzb\NzbService;
+use App\Services\Releases\ReleaseDuplicateFinder;
 use App\Support\Utf8;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;

 class ReleaseCreationService
@@ -23,6 +25,7 @@ class ReleaseCreationService
    public function __construct(
        private readonly ReleaseCleaningService $releaseCleaning,
        private readonly CollectionCleanupService $collectionCleanupService,
+        private readonly ReleaseDuplicateFinder $releaseDuplicateFinder,
    ) {}

    /**
@@ -62,47 +65,49 @@ class ReleaseCreationService
            $cleanRelName = Utf8::clean(str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection->subject));
            $fromName = Utf8::clean(trim($collection->fromname, "'"));

-            // Deduplicate by name, from, and ~size
-            $dupeCheck = Release::query()
-                ->where(['name' => $cleanRelName, 'fromname' => $fromName])
-                ->whereBetween('size', [$collection->filesize * .99, $collection->filesize * 1.01])
-                ->first(['id']);
+            $cleanedMeta = $this->releaseCleaning->releaseCleaner(
+                $collection->subject,
+                $collection->fromname,
+                $collection->gname
+            );
+
+            $namingRegexId = 0;
+            if (\is_array($cleanedMeta)) {
+                $namingRegexId = isset($cleanedMeta['id']) ? (int) $cleanedMeta['id'] : 0;
+            }
+
+            if (\is_array($cleanedMeta)) {
+                $properName = $cleanedMeta['properlynamed'] ?? false;
+                $preID = $cleanedMeta['predb'] ?? false;
+                $cleanedName = $cleanedMeta['cleansubject'] ?? $cleanRelName;
+            } else {
+                $properName = true;
+                $preID = false;
+                $cleanedName = $cleanRelName;
+            }
+
+            if ($preID === false && $cleanedName !== '') {
+                $preMatch = Predb::matchPre($cleanedName);
+                if ($preMatch !== false) {
+                    $cleanedName = $preMatch['title'];
+                    $preID = $preMatch['predb_id'];
+                    $properName = true;
+                }
+            }
+
+            $searchName = ! empty($cleanedName) ? Utf8::clean($cleanedName) : $cleanRelName;
+            $predbIdInt = $preID === false ? 0 : (int) $preID;
+
+            [$dupeCheck, $dupeReason] = $this->releaseDuplicateFinder->findDuplicate(
+                $cleanRelName,
+                $searchName,
+                $predbIdInt,
+                (int) $collection->filesize
+            );

            if ($dupeCheck === null) {
-                $cleanedMeta = $this->releaseCleaning->releaseCleaner(
-                    $collection->subject,
-                    $collection->fromname,
-                    $collection->gname
-                );
-
-                $namingRegexId = 0;
-                if (\is_array($cleanedMeta)) {
-                    $namingRegexId = isset($cleanedMeta['id']) ? (int) $cleanedMeta['id'] : 0;
-                }
-
-                if (\is_array($cleanedMeta)) {
-                    $properName = $cleanedMeta['properlynamed'] ?? false;
-                    $preID = $cleanedMeta['predb'] ?? false;
-                    $cleanedName = $cleanedMeta['cleansubject'] ?? $cleanRelName;
-                } else {
-                    $properName = true;
-                    $preID = false;
-                    $cleanedName = $cleanRelName;
-                }
-
-                if ($preID === false && $cleanedName !== '') {
-                    $preMatch = Predb::matchPre($cleanedName);
-                    if ($preMatch !== false) {
-                        $cleanedName = $preMatch['title'];
-                        $preID = $preMatch['predb_id'];
-                        $properName = true;
-                    }
-                }
-
                $determinedCategory = $categorize->determineCategory($collection->groups_id, $cleanedName, $fromName);

-                $searchName = ! empty($cleanedName) ? Utf8::clean($cleanedName) : $cleanRelName;
-
                $releaseID = Release::insertRelease([
                    'name' => $cleanRelName,
                    'searchname' => $searchName,
@@ -114,7 +119,7 @@ class ReleaseCreationService
                    'size' => $collection->filesize,
                    'categories_id' => $determinedCategory['categories_id'] ?? Category::OTHER_MISC,
                    'isrenamed' => $properName === true ? 1 : 0,
-                    'predb_id' => $preID === false ? 0 : $preID,
+                    'predb_id' => $predbIdInt,
                    'nzbstatus' => NzbService::NZB_NONE,
                ]);

@@ -172,6 +177,19 @@ class ReleaseCreationService
                    }
                }
            } else {
+                Log::info('Release import skipped as duplicate', [
+                    'reason' => $dupeReason,
+                    'matched_release_id' => $dupeCheck->id,
+                    'new_searchname' => $searchName,
+                    'existing_searchname' => $dupeCheck->searchname,
+                    'new_size' => (int) $collection->filesize,
+                    'existing_size' => (int) $dupeCheck->size,
+                    'new_fromname' => $fromName,
+                    'existing_fromname' => $dupeCheck->fromname,
+                    'new_name' => $cleanRelName,
+                    'existing_name' => $dupeCheck->name,
+                ]);
+
                $this->collectionCleanupService->deleteCollectionsAndDescendants(
                    [$collection->id],
                    'Duplicate cleanup',
@@ -16,6 +16,7 @@ use App\Services\Categorization\CategorizationService;
 use App\Services\NNTP\NNTPService;
 use App\Services\Nzb\NzbService;
 use App\Services\Releases\ReleaseBrowseService;
+use App\Services\Releases\ReleaseDuplicateFinder;
 use App\Services\Releases\ReleaseManagementService;
 use App\Support\Data\ProcessReleasesSettings;
 use App\Support\Data\ReleaseCreationResult;
@@ -89,7 +90,11 @@ final class ReleaseProcessingService
            ?? new CollectionCleanupService;

        $this->releaseCreationService = $releaseCreationService
-            ?? new ReleaseCreationService($this->releaseCleaning, $this->collectionCleanupService);
+            ?? new ReleaseCreationService(
+                $this->releaseCleaning,
+                $this->collectionCleanupService,
+                app(ReleaseDuplicateFinder::class)
+            );
        $this->postProcessService = $postProcessService;

        $this->settings = $this->loadSettings();
@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Services\Releases;
+
+use App\Models\Release;
+use Illuminate\Database\Eloquent\Builder;
+
+/**
+ * Finds an existing release that should be treated as a duplicate of an incoming import.
+ *
+ * Uses (predb_id OR searchname) within a configurable size band. Falls back to raw {@see Release::$name}
+ * when {@see Release::$searchname} is empty and there is no predb id.
+ */
+final class ReleaseDuplicateFinder
+{
+    /**
+     * @return array{0: ?Release, 1: ?string} Tuple of matched release (if any) and dedupe reason for logging.
+     */
+    public function findDuplicate(
+        string $cleanRelName,
+        string $searchName,
+        int $predbId,
+        int $filesize,
+    ): array {
+        $tolerance = (float) config('nntmux.release_dedupe_size_tolerance', 0.05);
+        $lowSize = (int) floor($filesize * (1 - $tolerance));
+        $highSize = (int) ceil($filesize * (1 + $tolerance));
+
+        $query = Release::query()
+            ->whereBetween('size', [$lowSize, $highSize]);
+
+        if ($predbId > 0 || $searchName !== '') {
+            $query->where(function (Builder $w) use ($searchName, $predbId): void {
+                if ($predbId > 0) {
+                    $w->where('predb_id', $predbId);
+                    if ($searchName !== '') {
+                        $w->orWhere('searchname', $searchName);
+                    }
+                } elseif ($searchName !== '') {
+                    $w->where('searchname', $searchName);
+                } else {
+                    $w->whereRaw('1 = 0');
+                }
+            });
+        } else {
+            $query->where('name', $cleanRelName);
+        }
+
+        $dup = $query->first(['id', 'predb_id', 'searchname', 'fromname', 'size', 'name']);
+
+        if ($dup === null) {
+            return [null, null];
+        }
+
+        $reason = $this->resolveReason($dup, $searchName, $predbId);
+
+        return [$dup, $reason];
+    }
+
+    private function resolveReason(Release $dup, string $searchName, int $predbId): string
+    {
+        if ($predbId > 0 && (int) $dup->predb_id === $predbId) {
+            return 'predb_id_match';
+        }
+
+        if ($searchName !== '' && (string) $dup->searchname === $searchName) {
+            return 'searchname_match';
+        }
+
+        return 'name_match_fallback';
+    }
+}
@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace App\Services\Search\Drivers;

 use App\Enums\SecondarySearchIndex;
+use App\Jobs\ReindexReleaseJob;
 use App\Models\BookInfo;
 use App\Models\ConsoleInfo;
 use App\Models\GamesInfo;
@@ -168,26 +169,95 @@ class ManticoreSearchDriver implements SearchDriverInterface
            return;
        }

-        try {
-            $document = ReleaseSearchIndexDocument::normalize($parameters);
-            unset($document['id']);
+        $releaseId = (int) $parameters['id'];
+        if (! $this->replaceReleaseDocumentWithRetry($parameters)) {
+            $this->recordReleaseIndexFailure($releaseId, 'insertRelease');
+            try {
+                ReindexReleaseJob::dispatch($releaseId)->delay(now()->addSeconds(2));
+            } catch (\Throwable $e) {
+                Log::error('ManticoreSearch: failed to queue ReindexReleaseJob', [
+                    'release_id' => $releaseId,
+                    'error' => $e->getMessage(),
+                ]);
+            }
+        }
+    }

-            $this->manticoreSearch->table($this->config['indexes']['releases'])
-                ->replaceDocument($document, $parameters['id']);
+    /**
+     * @param  array<string, mixed>  $parameters
+     */
+    private function replaceReleaseDocumentWithRetry(array $parameters): bool
+    {
+        $document = ReleaseSearchIndexDocument::normalize($parameters);
+        unset($document['id']);

-        } catch (ResponseException $e) {
-            Log::error('ManticoreSearch insertRelease ResponseException: '.$e->getMessage(), [
-                'release_id' => $parameters['id'],
-                'index' => $this->config['indexes']['releases'],
+        $indexName = $this->config['indexes']['releases'];
+        $releaseId = (int) $parameters['id'];
+
+        for ($attempt = 0; $attempt < 2; $attempt++) {
+            try {
+                $this->manticoreSearch->table($indexName)
+                    ->replaceDocument($document, $releaseId);
+
+                return true;
+            } catch (\Throwable $e) {
+                $isResponse = $e instanceof ResponseException;
+                $isManticoreRuntime = $e instanceof RuntimeException;
+
+                if (($isResponse || $isManticoreRuntime) && $attempt === 0) {
+                    usleep(100_000);
+
+                    continue;
+                }
+
+                if ($isResponse) {
+                    Log::error('ManticoreSearch insertRelease ResponseException: '.$e->getMessage(), [
+                        'release_id' => $releaseId,
+                        'index' => $indexName,
+                    ]);
+
+                    return false;
+                }
+
+                if ($isManticoreRuntime) {
+                    Log::error('ManticoreSearch insertRelease RuntimeException: '.$e->getMessage(), [
+                        'release_id' => $releaseId,
+                    ]);
+
+                    return false;
+                }
+
+                Log::error('ManticoreSearch insertRelease unexpected error: '.$e->getMessage(), [
+                    'release_id' => $releaseId,
+                    'trace' => $e->getTraceAsString(),
+                ]);
+
+                return false;
+            }
+        }
+
+        return false;
+    }
+
+    private function recordReleaseIndexFailure(int $releaseId, string $phase): void
+    {
+        Cache::increment('search:index:failures:releases');
+        if (Cache::add('search:index:release_index_warn_lock', true, 60)) {
+            Log::warning('ManticoreSearch: release indexing failure', [
+                'release_id_sample' => $releaseId,
+                'phase' => $phase,
+                'failures_total' => (int) Cache::get('search:index:failures:releases', 0),
            ]);
-        } catch (RuntimeException $e) {
-            Log::error('ManticoreSearch insertRelease RuntimeException: '.$e->getMessage(), [
-                'release_id' => $parameters['id'],
-            ]);
-        } catch (\Throwable $e) {
-            Log::error('ManticoreSearch insertRelease unexpected error: '.$e->getMessage(), [
-                'release_id' => $parameters['id'],
-                'trace' => $e->getTraceAsString(),
+        }
+    }
+
+    private function recordReleaseNotFoundForIndex(int|string $releaseID): void
+    {
+        Cache::increment('search:index:failures:release_not_found');
+        if (Cache::add('search:index:release_not_found_warn_lock', true, 60)) {
+            Log::warning('ManticoreSearch: release not found when updating index', [
+                'release_id' => $releaseID,
+                'not_found_total' => (int) Cache::get('search:index:failures:release_not_found', 0),
            ]);
        }
    }
@@ -648,11 +718,13 @@ class ManticoreSearchDriver implements SearchDriverInterface
                $this->insertRelease($release->toArray());
            } else {
                Log::warning('ManticoreSearch: Release not found for update', ['id' => $releaseID]);
+                $this->recordReleaseNotFoundForIndex($releaseID);
            }
        } catch (\Throwable $e) {
            Log::error('ManticoreSearch updateRelease error: '.$e->getMessage(), [
                'release_id' => $releaseID,
            ]);
+            $this->recordReleaseIndexFailure((int) $releaseID, 'updateRelease_query');
        }
    }

@@ -21,6 +21,17 @@ return [
    'purge_inactive_users' => env('PURGE_INACTIVE_USERS', false),
    'purge_inactive_users_days' => env('PURGE_INACTIVE_USERS_DAYS', 180),
    'mysql_search_fallback' => env('MYSQL_SEARCH_FALLBACK', false), // Disable MySQL LIKE fallback when Manticore/Elasticsearch return no results
+
+    /*
+    |--------------------------------------------------------------------------
+    | Release dedupe (import-time)
+    |--------------------------------------------------------------------------
+    |
+    | Size tolerance for matching an existing release when deduping imports
+    | (collections / NZB). Default 0.05 = ±5% on total bytes (par2/RAR drift).
+    |
+    */
+    'release_dedupe_size_tolerance' => (float) env('RELEASE_DEDUPE_SIZE_TOLERANCE', 0.05),
    'btcpay_webhook_secret' => env('BTCPAY_SECRET'),
    'tmp_unrar_path' => env('TEMP_UNRAR_PATH', storage_path('tmp/unrar/')),
    'tmp_unzip_path' => env('TEMP_UNZIP_PATH', storage_path('tmp/unzip/')),
@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        if ($this->indexExists('ix_releases_searchname')) {
+            return;
+        }
+
+        Schema::table('releases', function (Blueprint $table): void {
+            $table->index('searchname', 'ix_releases_searchname');
+        });
+    }
+
+    public function down(): void
+    {
+        if (! $this->indexExists('ix_releases_searchname')) {
+            return;
+        }
+
+        Schema::table('releases', function (Blueprint $table): void {
+            $table->dropIndex('ix_releases_searchname');
+        });
+    }
+
+    private function indexExists(string $keyName): bool
+    {
+        $rows = DB::select('SHOW INDEX FROM `releases` WHERE Key_name = ?', [$keyName]);
+
+        return $rows !== [];
+    }
+};
@@ -8,6 +8,7 @@ use App\Services\CollectionCleanupService;
 use App\Services\Nzb\NzbService;
 use App\Services\ReleaseCleaningService;
 use App\Services\ReleaseCreationService;
+use App\Services\Releases\ReleaseDuplicateFinder;
 use Illuminate\Support\Facades\DB;
 use Tests\TestCase;

@@ -21,6 +22,20 @@ class CbpCleanupServiceTest extends TestCase
        DB::purge();
        DB::reconnect();
        DB::connection()->getPdo()->sqliteCreateFunction('UNIX_TIMESTAMP', static fn (?string $value): int => strtotime((string) $value));
+        DB::connection()->getPdo()->sqliteCreateFunction(
+            'REGEXP',
+            static function (?string $subject, ?string $pattern): int {
+                if ($subject === null || $pattern === null || $pattern === '') {
+                    return 0;
+                }
+                set_error_handler(static fn (): true => true);
+                $ok = @preg_match($pattern, $subject);
+                restore_error_handler();
+
+                return $ok ? 1 : 0;
+            },
+            2
+        );

        $this->createTables();
        $this->seedSettings();
@@ -192,7 +207,8 @@ class CbpCleanupServiceTest extends TestCase

        $service = new ReleaseCreationService(
            app(ReleaseCleaningService::class),
-            app(CollectionCleanupService::class)
+            app(CollectionCleanupService::class),
+            app(ReleaseDuplicateFinder::class)
        );
        $result = $service->createReleases(null, 10, false);

@@ -202,6 +218,145 @@ class CbpCleanupServiceTest extends TestCase
        $this->assertSame(0, DB::table('collections')->count());
    }

+    public function test_release_duplicate_finder_matches_searchname_within_size_band(): void
+    {
+        DB::table('releases')->insert([
+            'id' => 20,
+            'name' => 'raw-obfuscated-a',
+            'searchname' => 'Unified.Scene.S01E01.1080p',
+            'totalpart' => 1,
+            'groups_id' => 1,
+            'adddate' => now()->format('Y-m-d H:i:s'),
+            'guid' => str_repeat('c', 36),
+            'leftguid' => 'c',
+            'postdate' => now()->format('Y-m-d H:i:s'),
+            'fromname' => 'poster-a@example.com',
+            'size' => 1_000_000,
+            'passwordstatus' => 0,
+            'haspreview' => -1,
+            'categories_id' => 1,
+            'nfostatus' => -1,
+            'nzbstatus' => NzbService::NZB_NONE,
+            'isrenamed' => 1,
+            'iscategorized' => 1,
+            'predb_id' => 0,
+            'source' => null,
+        ]);
+
+        $finder = app(ReleaseDuplicateFinder::class);
+        [$dup, $reason] = $finder->findDuplicate(
+            'raw-obfuscated-b',
+            'Unified.Scene.S01E01.1080p',
+            0,
+            1_020_000
+        );
+
+        $this->assertNotNull($dup);
+        $this->assertSame('searchname_match', $reason);
+    }
+
+    public function test_release_duplicate_finder_matches_predb_id_when_searchname_differs(): void
+    {
+        DB::table('releases')->insert([
+            'id' => 21,
+            'name' => 'old',
+            'searchname' => 'Old Style Name',
+            'totalpart' => 1,
+            'groups_id' => 1,
+            'adddate' => now()->format('Y-m-d H:i:s'),
+            'guid' => str_repeat('d', 36),
+            'leftguid' => 'd',
+            'postdate' => now()->format('Y-m-d H:i:s'),
+            'fromname' => 'p@example.com',
+            'size' => 2_000_000,
+            'passwordstatus' => 0,
+            'haspreview' => -1,
+            'categories_id' => 1,
+            'nfostatus' => -1,
+            'nzbstatus' => NzbService::NZB_NONE,
+            'isrenamed' => 1,
+            'iscategorized' => 1,
+            'predb_id' => 9001,
+            'source' => null,
+        ]);
+
+        $finder = app(ReleaseDuplicateFinder::class);
+        [$dup, $reason] = $finder->findDuplicate(
+            'new',
+            'New Style Name',
+            9001,
+            2_050_000
+        );
+
+        $this->assertNotNull($dup);
+        $this->assertSame('predb_id_match', $reason);
+    }
+
+    public function test_release_duplicate_finder_does_not_match_outside_size_tolerance(): void
+    {
+        config(['nntmux.release_dedupe_size_tolerance' => 0.05]);
+
+        DB::table('releases')->insert([
+            'id' => 22,
+            'name' => 'x',
+            'searchname' => 'Same.Search',
+            'totalpart' => 1,
+            'groups_id' => 1,
+            'adddate' => now()->format('Y-m-d H:i:s'),
+            'guid' => str_repeat('e', 36),
+            'leftguid' => 'e',
+            'postdate' => now()->format('Y-m-d H:i:s'),
+            'fromname' => 'p@example.com',
+            'size' => 1_000_000,
+            'passwordstatus' => 0,
+            'haspreview' => -1,
+            'categories_id' => 1,
+            'nfostatus' => -1,
+            'nzbstatus' => NzbService::NZB_NONE,
+            'isrenamed' => 1,
+            'iscategorized' => 1,
+            'predb_id' => 0,
+            'source' => null,
+        ]);
+
+        $finder = app(ReleaseDuplicateFinder::class);
+        [$dup] = $finder->findDuplicate('x', 'Same.Search', 0, 1_200_000);
+
+        $this->assertNull($dup);
+    }
+
+    public function test_release_duplicate_finder_falls_back_to_name_when_searchname_empty(): void
+    {
+        DB::table('releases')->insert([
+            'id' => 23,
+            'name' => 'fallback.unique',
+            'searchname' => '',
+            'totalpart' => 1,
+            'groups_id' => 1,
+            'adddate' => now()->format('Y-m-d H:i:s'),
+            'guid' => str_repeat('f', 36),
+            'leftguid' => 'f',
+            'postdate' => now()->format('Y-m-d H:i:s'),
+            'fromname' => 'p@example.com',
+            'size' => 500,
+            'passwordstatus' => 0,
+            'haspreview' => -1,
+            'categories_id' => 1,
+            'nfostatus' => -1,
+            'nzbstatus' => NzbService::NZB_NONE,
+            'isrenamed' => 1,
+            'iscategorized' => 1,
+            'predb_id' => 0,
+            'source' => null,
+        ]);
+
+        $finder = app(ReleaseDuplicateFinder::class);
+        [$dup, $reason] = $finder->findDuplicate('fallback.unique', '', 0, 500);
+
+        $this->assertNotNull($dup);
+        $this->assertSame('name_match_fallback', $reason);
+    }
+
    private function seedSettings(): void
    {
        $settings = [
@@ -274,6 +429,25 @@ class CbpCleanupServiceTest extends TestCase
            partnumber INTEGER,
            size INTEGER
        )');
+        DB::statement('CREATE TABLE release_naming_regexes (
+            id INTEGER PRIMARY KEY,
+            group_regex VARCHAR(255),
+            regex VARCHAR(255),
+            status INTEGER DEFAULT 1,
+            ordinal INTEGER DEFAULT 0
+        )');
+        DB::statement('CREATE TABLE collection_regexes (
+            id INTEGER PRIMARY KEY,
+            group_regex VARCHAR(255),
+            regex VARCHAR(255),
+            status INTEGER DEFAULT 1,
+            ordinal INTEGER DEFAULT 0
+        )');
+        DB::statement('CREATE TABLE predb (
+            id INTEGER PRIMARY KEY,
+            title VARCHAR(255),
+            filename VARCHAR(255)
+        )');
        DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.test']);
        DB::table('categories')->insert(['id' => 1, 'title' => 'Misc', 'parent_categories_id' => null]);
    }