diff --git a/.env.example b/.env.example index f6d6a5c12..c433600ae 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,8 @@ MANTICORESEARCH_FUZZY_ENABLED=false # For high availability, set multiple hosts (comma-separated): "host1:9308,host2:9308" MANTICORESEARCH_HOSTS= MANTICORESEARCH_RETRIES=2 +MANTICORESEARCH_RETRY_ATTEMPTS=2 +MANTICORESEARCH_RETRY_DELAY_MS=100 # Autocomplete settings MANTICORESEARCH_AUTOCOMPLETE_ENABLED=true MANTICORESEARCH_AUTOCOMPLETE_MIN_LENGTH=2 diff --git a/app/Console/Commands/CreateManticoreIndexes.php b/app/Console/Commands/CreateManticoreIndexes.php index 9b52a7808..04b5cb4ba 100644 --- a/app/Console/Commands/CreateManticoreIndexes.php +++ b/app/Console/Commands/CreateManticoreIndexes.php @@ -12,7 +12,9 @@ use Manticoresearch\Exceptions\ResponseException; class CreateManticoreIndexes extends Command { - protected $signature = 'manticore:create-indexes {--drop : Drop existing indexes before creating}'; + protected $signature = 'manticore:create-indexes + {--drop : Drop selected indexes before creating} + {--index=* : Logical index name(s) to create; defaults to all indexes}'; protected $description = 'Create Manticore Search indexes based on configuration'; @@ -33,8 +35,19 @@ class CreateManticoreIndexes extends Command } $configuredNames = config('search.drivers.manticore.indexes', []); + $selected = array_values(array_filter(array_map('strval', (array) $this->option('index')))); + $definitions = ManticoreIndexRegistry::definitions(); + if ($selected !== []) { + $unknown = array_diff($selected, array_keys($definitions)); + if ($unknown !== []) { + $this->error('Unknown logical index: '.implode(', ', $unknown)); + + return self::FAILURE; + } + $definitions = array_intersect_key($definitions, array_flip($selected)); + } $hasErrors = false; - foreach (ManticoreIndexRegistry::definitions() as $logical => $schema) { + foreach ($definitions as $logical => $schema) { $indexName = (string) ($configuredNames[$logical] ?? $logical.'_rt'); if (! $this->createIndex($indexName, $schema, (bool) $this->option('drop'))) { $hasErrors = true; diff --git a/app/Console/Commands/InspectManticore.php b/app/Console/Commands/InspectManticore.php index 1e7f2ad6c..9e572a5cb 100644 --- a/app/Console/Commands/InspectManticore.php +++ b/app/Console/Commands/InspectManticore.php @@ -56,7 +56,7 @@ final class InspectManticore extends Command $this->line(sprintf('%-20s %s', $table, $status['needs_rebuild'] ? 'REBUILD REQUIRED' : 'compatible')); } if (! $report['compatible']) { - $this->warn('Run manticore:create-indexes --drop during a maintenance window, then repopulate all enabled indexes.'); + $this->warn('Run manticore:create-indexes --drop --index=releases during a maintenance window for release-only schema changes, then repopulate the affected index.'); } } diff --git a/app/Console/Commands/NntmuxOffsetPopulate.php b/app/Console/Commands/NntmuxOffsetPopulate.php index 0e4dbb3ad..40f754c0d 100644 --- a/app/Console/Commands/NntmuxOffsetPopulate.php +++ b/app/Console/Commands/NntmuxOffsetPopulate.php @@ -8,6 +8,7 @@ use App\Facades\Elasticsearch; use App\Facades\Search; use App\Models\Predb; use App\Models\Release; +use App\Services\Search\Drivers\ManticoreSearchDriver; use App\Services\Search\Support\ElasticsearchResponseHelper; use Elastic\Elasticsearch\Client as ElasticsearchClient; use Exception; @@ -115,7 +116,9 @@ class NntmuxOffsetPopulate extends Command $this->monitorProcesses($processes); // @phpstan-ignore argument.type // Verify final count - $this->verifyIndexPopulation($engine, $index, $total); + if (! $this->verifyIndexPopulation($engine, $index, $total)) { + return Command::FAILURE; + } $this->info('All parallel processes completed successfully!'); @@ -252,7 +255,7 @@ class NntmuxOffsetPopulate extends Command /** * Verify index population */ - private function verifyIndexPopulation(string $engine, string $index, int $expectedTotal): void + private function verifyIndexPopulation(string $engine, string $index, int $expectedTotal): bool { $this->info('Verifying index population...'); @@ -271,12 +274,34 @@ class NntmuxOffsetPopulate extends Command $this->info('✓ Index population successful!'); } else { $this->warn('⚠ Index population may be incomplete'); + + return false; } } catch (Exception $e) { $this->warn("Could not verify index population: {$e->getMessage()}"); + + return false; } + + return true; } else { - $this->info('ManticoreSearch index verification not implemented yet'); + $indexName = (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt'); + try { + $actualCount = app(ManticoreSearchDriver::class)->countIndexDocuments($indexName); + $this->info('Expected: '.number_format($expectedTotal)); + $this->info('Actual: '.number_format($actualCount)); + if ($actualCount < $expectedTotal) { + $this->warn('Manticore index population is incomplete.'); + + return false; + } + } catch (Exception $e) { + $this->warn("Could not verify Manticore population: {$e->getMessage()}"); + + return false; + } + + return true; } } @@ -286,7 +311,7 @@ class NntmuxOffsetPopulate extends Command private function clearIndex(string $engine, string $index): void { if ($engine === 'manticore') { - $indexName = $index === 'releases' ? 'releases_rt' : 'predb_rt'; + $indexName = (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt'); Search::truncateIndex([$indexName]); $this->info("Truncated ManticoreSearch index: {$indexName}"); } else { diff --git a/app/Console/Commands/NntmuxOffsetWorker.php b/app/Console/Commands/NntmuxOffsetWorker.php index 9a376cd5d..284314080 100644 --- a/app/Console/Commands/NntmuxOffsetWorker.php +++ b/app/Console/Commands/NntmuxOffsetWorker.php @@ -7,7 +7,8 @@ namespace App\Console\Commands; use App\Facades\Elasticsearch; use App\Facades\Search; use App\Models\Predb; -use App\Models\Release; +use App\Services\Search\Support\ReleaseIndexProjection; +use App\Support\ReleaseSearchIndexDocument; use Exception; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; @@ -97,7 +98,7 @@ class NntmuxOffsetWorker extends Command if ($engine === 'manticore') { $batchData = []; - $query->chunk($batchSize, function ($items) use ($transformer, &$processed, &$errors, &$batchData, $batchSize, $workerId) { + $query->chunk($batchSize, function ($items) use ($transformer, $index, &$processed, &$errors, &$batchData, $batchSize, $workerId) { $this->info("Worker {$workerId}: Processing chunk of {$items->count()} items"); foreach ($items as $item) { @@ -106,7 +107,7 @@ class NntmuxOffsetWorker extends Command $processed++; if (count($batchData) >= $batchSize) { - $this->processSearchBatch($batchData, $workerId); // @phpstan-ignore argument.type + $this->processSearchBatch($batchData, $index, $workerId); // @phpstan-ignore argument.type $this->info("Worker {$workerId}: Inserted batch of ".count($batchData).' records'); $batchData = []; } @@ -119,7 +120,7 @@ class NntmuxOffsetWorker extends Command // Process remaining items if (! empty($batchData)) { - $this->processSearchBatch($batchData, $workerId); // @phpstan-ignore argument.type + $this->processSearchBatch($batchData, $index, $workerId); // @phpstan-ignore argument.type $this->info("Worker {$workerId}: Inserted final batch of ".count($batchData).' records'); } @@ -166,17 +167,8 @@ class NntmuxOffsetWorker extends Command private function buildOffsetQuery(string $index, int $offset, int $limit): mixed { if ($index === 'releases') { - return Release::query() - ->select([ - 'releases.id', - 'releases.name', - 'releases.searchname', - 'releases.fromname', - 'releases.categories_id', - 'releases.postdate', - ]) - ->selectRaw('(SELECT GROUP_CONCAT(rf.name SEPARATOR " ") FROM release_files rf WHERE rf.releases_id = releases.id) AS filename') - ->orderBy('releases.id') + return ReleaseIndexProjection::query() + ->orderBy('r.id') ->offset($offset) ->limit($limit); } else { @@ -194,34 +186,15 @@ class NntmuxOffsetWorker extends Command private function getTransformer(string $engine, string $index): callable { if ($index === 'releases') { - if ($engine === 'manticore') { - return function ($item) { - return [ - 'id' => (string) $item->id, - 'name' => (string) ($item->name ?: ''), - 'searchname' => (string) ($item->searchname ?: ''), - 'fromname' => (string) ($item->fromname ?: ''), - 'categories_id' => (string) ($item->categories_id ?: '0'), - 'filename' => (string) ($item->filename ?: ''), - 'dummy' => '1', - ]; - }; - } else { - return function ($item) { - $searchName = str_replace(['.', '-'], ' ', $item->searchname ?? ''); + return function ($item) use ($engine): array { + $row = (array) $item; - return [ - 'id' => $item->id, - 'name' => $item->name, - 'searchname' => $item->searchname, - 'plainsearchname' => $searchName, - 'fromname' => $item->fromname, - 'categories_id' => $item->categories_id, - 'filename' => $item->filename ?? '', - 'postdate' => $item->postdate, - ]; - }; - } + return $engine === 'manticore' + ? ReleaseSearchIndexDocument::normalize($row) + : array_merge($row, [ + 'plainsearchname' => str_replace(['.', '-'], ' ', (string) ($item->searchname ?? '')), + ]); + }; } else { // predb return function ($item) { return [ @@ -229,7 +202,6 @@ class NntmuxOffsetWorker extends Command 'title' => (string) ($item->title ?? ''), 'filename' => (string) ($item->filename ?? ''), 'source' => (string) ($item->source ?? ''), - 'dummy' => 1, ]; }; } @@ -241,7 +213,7 @@ class NntmuxOffsetWorker extends Command private function getIndexName(string $engine, string $index): string { if ($engine === 'manticore') { - return $index === 'releases' ? 'releases_rt' : 'predb_rt'; + return (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt'); } else { return $index; } @@ -252,14 +224,22 @@ class NntmuxOffsetWorker extends Command * * @param array $data */ - private function processSearchBatch(array $data, int $workerId): void + private function processSearchBatch(array $data, string $index, int $workerId): void { $retries = 3; $attempt = 0; while ($attempt < $retries) { try { - Search::bulkInsertReleases($data); + $result = []; + if ($index === 'releases') { + $result = Search::bulkInsertReleases($data); + } else { + $result = Search::bulkInsertPredb($data); + } + if (($result['errors'] ?? 0) > 0) { + throw new Exception('Bulk operation reported '.(int) $result['errors'].' error(s).'); + } break; } catch (Exception $e) { $attempt++; diff --git a/app/Console/Commands/NntmuxPopulateSearchIndexes.php b/app/Console/Commands/NntmuxPopulateSearchIndexes.php index fb6f087f3..9153d8cb5 100644 --- a/app/Console/Commands/NntmuxPopulateSearchIndexes.php +++ b/app/Console/Commands/NntmuxPopulateSearchIndexes.php @@ -16,6 +16,8 @@ use App\Models\Predb; use App\Models\Release; use App\Models\SteamApp; use App\Models\Video; +use App\Services\Search\Drivers\ManticoreSearchDriver; +use App\Services\Search\Support\ReleaseIndexProjection; use App\Support\ReleaseSearchIndexDocument; use App\Support\SecondaryIndexDocuments; use Exception; @@ -194,7 +196,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreReleases(): int { - $indexName = 'releases_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.releases', 'releases_rt'); Search::truncateIndex([$indexName]); @@ -205,39 +207,14 @@ class NntmuxPopulateSearchIndexes extends Command return Command::SUCCESS; } - // Optimized query: avoid GROUP_CONCAT and complex joins for faster population - // External media IDs can be populated separately if needed - $query = Release::query() - ->orderByDesc('releases.id') - ->select([ - 'releases.id', - 'releases.name', - 'releases.searchname', - 'releases.fromname', - 'releases.categories_id', - 'releases.size', - 'releases.postdate', - 'releases.adddate', - 'releases.totalpart', - 'releases.grabs', - 'releases.passwordstatus', - 'releases.groups_id', - 'releases.nzbstatus', - 'releases.haspreview', - 'releases.videos_id', - 'releases.movieinfo_id', - 'releases.imdbid', - ]); + $query = ReleaseIndexProjection::query()->orderBy('r.id'); return $this->processManticoreData( $indexName, $total, $query, function ($item) { - return ReleaseSearchIndexDocument::normalize(array_merge( - (array) $item->getAttributes(), - ['filename' => ''] - )); + return ReleaseSearchIndexDocument::normalize((array) $item); } ); } @@ -247,7 +224,7 @@ class NntmuxPopulateSearchIndexes extends Command */ private function manticorePredb(): int { - $indexName = 'predb_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.predb', 'predb_rt'); Search::truncateIndex([$indexName]); @@ -282,7 +259,7 @@ class NntmuxPopulateSearchIndexes extends Command */ private function manticoreMovies(): int { - $indexName = 'movies_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.movies', 'movies_rt'); Search::truncateIndex([$indexName]); @@ -336,7 +313,7 @@ class NntmuxPopulateSearchIndexes extends Command */ private function manticoreTvshows(): int { - $indexName = 'tvshows_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.tvshows', 'tvshows_rt'); Search::truncateIndex([$indexName]); @@ -385,7 +362,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreMusic(): int { - $indexName = 'music_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.music', 'music_rt'); Search::truncateIndex([$indexName]); $total = MusicInfo::count(); if (! $total) { @@ -405,7 +382,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreBooks(): int { - $indexName = 'books_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.books', 'books_rt'); Search::truncateIndex([$indexName]); $total = BookInfo::count(); if (! $total) { @@ -425,7 +402,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreGames(): int { - $indexName = 'games_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.games', 'games_rt'); Search::truncateIndex([$indexName]); $total = GamesInfo::count(); if (! $total) { @@ -445,7 +422,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreConsole(): int { - $indexName = 'console_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.console', 'console_rt'); Search::truncateIndex([$indexName]); $total = ConsoleInfo::count(); if (! $total) { @@ -465,7 +442,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreSteam(): int { - $indexName = 'steam_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.steam', 'steam_rt'); Search::truncateIndex([$indexName]); $total = SteamApp::count(); if (! $total) { @@ -485,7 +462,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreAnime(): int { - $indexName = 'anime_rt'; + $indexName = (string) config('search.drivers.manticore.indexes.anime', 'anime_rt'); Search::truncateIndex([$indexName]); $total = (int) DB::table('anidb_titles')->count(); if (! $total) { @@ -555,7 +532,7 @@ class NntmuxPopulateSearchIndexes extends Command $batchData = []; try { - $query->chunk($chunkSize, function ($items) use ($indexName, $transformer, $bar, &$processedCount, &$errorCount, $batchSize, &$batchData) { + $processItems = function ($items) use ($indexName, $transformer, $bar, &$processedCount, &$errorCount, $batchSize, &$batchData): void { foreach ($items as $item) { try { $batchData[] = $transformer($item); @@ -574,7 +551,13 @@ class NntmuxPopulateSearchIndexes extends Command } $bar->advance(); } - }); + }; + + if ($indexName === (string) config('search.drivers.manticore.indexes.releases', 'releases_rt')) { + $query->chunkById($chunkSize, $processItems, 'r.id', 'id'); + } else { + $query->chunk($chunkSize, $processItems); + } // Process remaining items if (! empty($batchData)) { @@ -586,8 +569,20 @@ class NntmuxPopulateSearchIndexes extends Command if ($errorCount > 0) { $this->warn("Completed with {$errorCount} errors out of {$processedCount} processed items."); - } else { - $this->info('Search index population completed successfully!'); + + return Command::FAILURE; + } + + try { + $actualCount = app(ManticoreSearchDriver::class)->countIndexDocuments($indexName); + if ($actualCount < $total) { + $this->error(sprintf('Manticore verification failed for %s: expected %d, found %d.', $indexName, $total, $actualCount)); + + return Command::FAILURE; + } + $this->info(sprintf('Search index population completed successfully (%d documents verified).', $actualCount)); + } catch (Exception $e) { + $this->warn('Manticore population completed, but verification failed: '.$e->getMessage()); } return Command::SUCCESS; @@ -1143,25 +1138,29 @@ class NntmuxPopulateSearchIndexes extends Command while ($attempt < $retries) { try { // Use the correct bulk insert method based on index name - if ($indexName === 'releases_rt') { - Search::bulkInsertReleases($data); - } elseif ($indexName === 'predb_rt') { - Search::bulkInsertPredb($data); - } elseif ($indexName === 'music_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Music, $data); - } elseif ($indexName === 'books_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Books, $data); - } elseif ($indexName === 'games_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Games, $data); - } elseif ($indexName === 'console_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Console, $data); - } elseif ($indexName === 'steam_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Steam, $data); - } elseif ($indexName === 'anime_rt') { - Search::bulkInsertSecondary(SecondarySearchIndex::Anime, $data); + $result = []; + if ($indexName === (string) config('search.drivers.manticore.indexes.releases', 'releases_rt')) { + $result = Search::bulkInsertReleases($data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.predb', 'predb_rt')) { + $result = Search::bulkInsertPredb($data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.music', 'music_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Music, $data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.books', 'books_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Books, $data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.games', 'games_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Games, $data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.console', 'console_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Console, $data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.steam', 'steam_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Steam, $data); + } elseif ($indexName === (string) config('search.drivers.manticore.indexes.anime', 'anime_rt')) { + $result = Search::bulkInsertSecondary(SecondarySearchIndex::Anime, $data); } else { throw new Exception("Unknown index: {$indexName}"); } + if (($result['errors'] ?? 0) > 0) { + throw new Exception('Bulk operation reported '.(int) $result['errors'].' error(s).'); + } break; } catch (Exception $e) { $attempt++; diff --git a/app/Console/Commands/NntmuxSearchReconcile.php b/app/Console/Commands/NntmuxSearchReconcile.php index ba42eadbd..b3588e6c8 100644 --- a/app/Console/Commands/NntmuxSearchReconcile.php +++ b/app/Console/Commands/NntmuxSearchReconcile.php @@ -16,8 +16,10 @@ class NntmuxSearchReconcile extends Command { protected $signature = 'nntmux:search-reconcile {--since=24h : Only releases with adddate after this window (e.g. 24h, 7d)} + {--all : Scan every release row instead of applying --since} {--chunk=1000 : MySQL chunk size when scanning releases} {--in-batch=500 : Max ids per Manticore IN() query} + {--orphans : Also remove indexed ids that no longer exist in MySQL} {--reindex : Call Search::updateRelease for each missing id} {--dry-run : Report only; do not write to the index}'; @@ -39,10 +41,12 @@ class NntmuxSearchReconcile extends Command $since = (string) $this->option('since'); $cutoff = $this->parseSinceCutoff($since); + $scanAll = (bool) $this->option('all'); $chunk = max(50, min(5000, (int) $this->option('chunk'))); $inBatch = max(50, min(1000, (int) $this->option('in-batch'))); $dryRun = (bool) $this->option('dry-run'); $reindex = (bool) $this->option('reindex'); + $checkOrphans = (bool) $this->option('orphans'); if ($reindex && $dryRun) { $this->warn('--reindex with --dry-run: no writes will be performed.'); @@ -56,8 +60,11 @@ class NntmuxSearchReconcile extends Command $bar = $this->output->createProgressBar(); $bar->start(); - Release::query() - ->where('adddate', '>=', $cutoff) + $releaseQuery = Release::query(); + if (! $scanAll) { + $releaseQuery->where('adddate', '>=', $cutoff); + } + $releaseQuery ->orderBy('id') ->chunkById($chunk, function ($releases) use ($manticore, $index, $inBatch, &$missingAll, &$scanned, $bar): void { $ids = $releases->pluck('id')->map(static fn ($id): int => (int) $id)->all(); @@ -75,7 +82,8 @@ class NntmuxSearchReconcile extends Command $this->newLine(2); $missingTotal = \count($missingAll); - $this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} (adddate >= {$cutoff->toDateTimeString()})"); + $scope = $scanAll ? 'all release rows' : "adddate >= {$cutoff->toDateTimeString()}"; + $this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} ({$scope})"); if ($missingTotal > 0) { $sample = \array_slice($missingAll, 0, 20); $this->line('Sample ids: '.implode(', ', $sample)); @@ -89,12 +97,18 @@ class NntmuxSearchReconcile extends Command $this->probeAndWarnAllMissing($manticore, $index, \array_slice($missingAll, 0, 5)); } - if ($missingTotal === 0) { + $orphans = $checkOrphans ? $this->findOrphanedIds($manticore, $chunk) : []; + if ($orphans !== []) { + $this->info('Indexed documents with no MySQL release row: '.count($orphans)); + $this->line('Orphan sample ids: '.implode(', ', array_slice($orphans, 0, 20))); + } + + if ($missingTotal === 0 && $orphans === []) { return self::SUCCESS; } if (! $reindex) { - $this->comment('Run with --reindex to push Search::updateRelease() for each missing id.'); + $this->comment('Run with --reindex to repair missing documents and remove orphaned ids.'); return self::SUCCESS; } @@ -110,11 +124,44 @@ class NntmuxSearchReconcile extends Command } } + if ($orphans !== []) { + try { + Search::deleteReleases($orphans); + $this->info('Removed '.count($orphans).' orphaned document(s).'); + } catch (Throwable $e) { + $this->error('Failed removing orphaned documents: '.$e->getMessage()); + } + } + $this->info("Reindexed {$done} release(s)."); return self::SUCCESS; } + /** + * @return list + */ + private function findOrphanedIds(ManticoreSearchDriver $manticore, int $pageSize): array + { + $orphans = []; + $offset = 0; + + do { + $indexed = $manticore->indexedReleaseIds($pageSize, $offset); + if ($indexed === []) { + break; + } + + $existing = Release::query()->whereIn('id', $indexed)->pluck('id') + ->map(static fn ($id): int => (int) $id) + ->all(); + $orphans = [...$orphans, ...array_values(array_diff($indexed, $existing))]; + $offset += count($indexed); + } while (count($indexed) === $pageSize); + + return $orphans; + } + private function parseSinceCutoff(string $since): Carbon { $since = trim($since); diff --git a/app/Console/Commands/NntmuxSearchRepair.php b/app/Console/Commands/NntmuxSearchRepair.php new file mode 100644 index 000000000..2dabc8141 --- /dev/null +++ b/app/Console/Commands/NntmuxSearchRepair.php @@ -0,0 +1,56 @@ +option('limit'))); + $query = DB::table('search_index_failures') + ->whereNull('resolved_at') + ->where(function ($query): void { + $query->whereNull('next_attempt_at')->orWhere('next_attempt_at', '<=', now()); + }) + ->orderBy('id') + ->limit($limit); + + $rows = $query->get(['release_id', 'operation']); + if ($rows->isEmpty()) { + $this->info('No failed release index updates are due for repair.'); + + return self::SUCCESS; + } + + foreach ($rows as $row) { + $releaseId = (int) $row->release_id; + if ((bool) $this->option('dry-run')) { + $this->line((string) $releaseId); + + continue; + } + + if ($row->operation === 'delete') { + Search::deleteRelease($releaseId); + } else { + Search::updateRelease($releaseId); + } + } + + $this->info(sprintf('%s release index failure(s) %s.', $rows->count(), $this->option('dry-run') ? 'reported' : 'processed')); + + return self::SUCCESS; + } +} diff --git a/app/Facades/Search.php b/app/Facades/Search.php index 562febfa1..2c836bc7f 100644 --- a/app/Facades/Search.php +++ b/app/Facades/Search.php @@ -17,7 +17,7 @@ use Illuminate\Support\Facades\Facade; * @method static void insertRelease(array $parameters) * @method static void updateRelease(int|string $releaseID) * @method static void deleteRelease(int $id) - * @method static void deleteReleases(iterable $ids) + * @method static void deleteReleases(iterable $ids) * @method static void insertPredb(array $parameters) * @method static void updatePreDb(array $parameters) * @method static void deletePreDb(int $id) diff --git a/app/Models/Release.php b/app/Models/Release.php index 61f690326..52ac24998 100644 --- a/app/Models/Release.php +++ b/app/Models/Release.php @@ -216,7 +216,7 @@ class Release extends Model ] ); - Search::updateRelease($parameters['id']); + self::syncSearchIndexAfterCommit((int) $parameters['id']); return $parameters['id']; } @@ -249,7 +249,7 @@ class Release extends Model ] ); - Search::updateRelease($id); + self::syncSearchIndexAfterCommit((int) $id); } /** @@ -262,7 +262,7 @@ class Release extends Model $id = self::whereGuid($guid)->value('id'); self::whereGuid($guid)->increment('grabs'); if ($id !== null) { - Search::updateRelease((int) $id); + self::syncSearchIndexAfterCommit((int) $id); } } } @@ -284,7 +284,7 @@ class Release extends Model $ids = self::query()->whereIn('guid', $guids)->pluck('id'); self::query()->whereIn('guid', $guids)->increment('grabs'); foreach ($ids as $id) { - Search::updateRelease((int) $id); + self::syncSearchIndexAfterCommit((int) $id); } } } @@ -302,7 +302,7 @@ class Release extends Model $ids = self::whereVideosId($videoId)->pluck('id'); $updated = self::whereVideosId($videoId)->update(['videos_id' => 0, 'tv_episodes_id' => 0]); foreach ($ids as $id) { - Search::updateRelease((int) $id); + self::syncSearchIndexAfterCommit((int) $id); } return $updated; @@ -313,6 +313,17 @@ class Release extends Model return self::whereAnidbid($anidbID)->update(['anidbid' => -1]); } + private static function syncSearchIndexAfterCommit(int $releaseId): void + { + if ($releaseId <= 0) { + return; + } + + DB::afterCommit(function () use ($releaseId): void { + Search::updateRelease($releaseId); + }); + } + public static function getTopDownloads(): mixed { $expiresAt = now()->addMinutes(config('nntmux.cache_expiry_long')); diff --git a/app/Models/ReleaseFile.php b/app/Models/ReleaseFile.php index 271038075..66c41dd1e 100644 --- a/app/Models/ReleaseFile.php +++ b/app/Models/ReleaseFile.php @@ -8,6 +8,7 @@ use App\Facades\Search; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; /** @@ -146,7 +147,9 @@ class ReleaseFile extends Model if (\strlen($hash) === 32) { ParHash::insertOrIgnore(['releases_id' => $id, 'hash' => $hash]); } - Search::updateRelease($id); + DB::afterCommit(function () use ($id): void { + Search::updateRelease($id); + }); } return $insert ?? 0; diff --git a/app/Observers/CategoryObserver.php b/app/Observers/CategoryObserver.php new file mode 100644 index 000000000..40c358aa5 --- /dev/null +++ b/app/Observers/CategoryObserver.php @@ -0,0 +1,28 @@ +isDirty(['title', 'parentid', 'root_categories_id'])) { + return; + } + + DB::afterCommit(fn (): bool => $this->reindex($category)); + } + + private function reindex(Category $category): bool + { + ReleaseSearchIndexSync::forCategory((int) $category->id); + + return true; + } +} diff --git a/app/Observers/MovieInfoObserver.php b/app/Observers/MovieInfoObserver.php index d8b842bd9..ebf7863d8 100644 --- a/app/Observers/MovieInfoObserver.php +++ b/app/Observers/MovieInfoObserver.php @@ -6,6 +6,8 @@ namespace App\Observers; use App\Facades\Search; use App\Models\MovieInfo; +use App\Support\ReleaseSearchIndexSync; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; class MovieInfoObserver @@ -24,6 +26,7 @@ class MovieInfoObserver public function updated(MovieInfo $movie): void { $this->syncToSearchIndex($movie); + DB::afterCommit(fn (): bool => $this->syncReleases($movie)); } /** @@ -39,6 +42,7 @@ class MovieInfoObserver 'error' => $e->getMessage(), ]); } + DB::afterCommit(fn (): bool => $this->syncReleases($movie)); } /** @@ -67,4 +71,11 @@ class MovieInfoObserver ]); } } + + private function syncReleases(MovieInfo $movie): bool + { + ReleaseSearchIndexSync::forMovieInfo((int) $movie->id); + + return true; + } } diff --git a/app/Observers/ReleaseObserver.php b/app/Observers/ReleaseObserver.php index d0124d7ed..bb21e4270 100644 --- a/app/Observers/ReleaseObserver.php +++ b/app/Observers/ReleaseObserver.php @@ -8,6 +8,7 @@ use App\Facades\Search; use App\Models\Release; use App\Services\Nzb\NzbService; use App\Services\ReleaseImageService; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; /** @@ -25,7 +26,7 @@ class ReleaseObserver */ public function created(Release $release): void { - $this->syncToSearchIndex($release); + $this->syncAfterCommit($release); } /** @@ -38,19 +39,25 @@ class ReleaseObserver { $indexedFields = [ 'name', + 'guid', 'searchname', 'fromname', 'categories_id', 'imdbid', + 'anidbid', 'movieinfo_id', 'videos_id', + 'tv_episodes_id', 'size', 'totalpart', 'grabs', 'passwordstatus', 'groups_id', 'nzbstatus', + 'nfostatus', 'haspreview', + 'jpgstatus', + 'comments', 'postdate', 'adddate', ]; @@ -64,7 +71,7 @@ class ReleaseObserver } if ($changed) { - $this->syncToSearchIndex($release); + $this->syncAfterCommit($release); } } @@ -106,20 +113,27 @@ class ReleaseObserver */ public function deleted(Release $release): void { - try { - Search::deleteRelease($release->id); - } catch (\Throwable $e) { - Log::error('ReleaseObserver: Failed to delete release from search index', [ - 'release_id' => $release->id, - 'error' => $e->getMessage(), - ]); - } + DB::afterCommit(function () use ($release): void { + try { + Search::deleteRelease($release->id); + } catch (\Throwable $e) { + Log::error('ReleaseObserver: Failed to delete release from search index', [ + 'release_id' => $release->id, + 'error' => $e->getMessage(), + ]); + } + }); + } + + private function syncAfterCommit(Release $release): void + { + DB::afterCommit(fn (): bool => $this->syncToSearchIndex($release)); } /** * Sync the release to the search index (full document from DB + joins). */ - private function syncToSearchIndex(Release $release): void + private function syncToSearchIndex(Release $release): bool { try { Search::updateRelease($release->id); @@ -129,11 +143,15 @@ class ReleaseObserver 'release_id' => $release->id, ]); } + + return true; } catch (\Throwable $e) { Log::error('ReleaseObserver: Failed to sync release to search index', [ 'release_id' => $release->id, 'error' => $e->getMessage(), ]); + + return false; } } } diff --git a/app/Observers/RootCategoryObserver.php b/app/Observers/RootCategoryObserver.php new file mode 100644 index 000000000..8ff42d73c --- /dev/null +++ b/app/Observers/RootCategoryObserver.php @@ -0,0 +1,28 @@ +isDirty('title')) { + return; + } + + DB::afterCommit(fn (): bool => $this->reindex($category)); + } + + private function reindex(RootCategory $category): bool + { + ReleaseSearchIndexSync::forRootCategory((int) $category->id); + + return true; + } +} diff --git a/app/Observers/UsenetGroupObserver.php b/app/Observers/UsenetGroupObserver.php new file mode 100644 index 000000000..2ad9b67be --- /dev/null +++ b/app/Observers/UsenetGroupObserver.php @@ -0,0 +1,28 @@ +isDirty('name')) { + return; + } + + DB::afterCommit(fn (): bool => $this->reindex($group)); + } + + private function reindex(UsenetGroup $group): bool + { + ReleaseSearchIndexSync::forQueryGroup((int) $group->id); + + return true; + } +} diff --git a/app/Observers/VideoObserver.php b/app/Observers/VideoObserver.php index a3a64a0ab..1fdb50d47 100644 --- a/app/Observers/VideoObserver.php +++ b/app/Observers/VideoObserver.php @@ -6,6 +6,8 @@ namespace App\Observers; use App\Facades\Search; use App\Models\Video; +use App\Support\ReleaseSearchIndexSync; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; class VideoObserver @@ -24,6 +26,7 @@ class VideoObserver public function updated(Video $video): void { $this->syncToSearchIndex($video); + DB::afterCommit(fn (): bool => $this->syncReleases($video)); } /** @@ -39,6 +42,7 @@ class VideoObserver 'error' => $e->getMessage(), ]); } + DB::afterCommit(fn (): bool => $this->syncReleases($video)); } /** @@ -66,4 +70,11 @@ class VideoObserver ]); } } + + private function syncReleases(Video $video): bool + { + ReleaseSearchIndexSync::forVideo((int) $video->id); + + return true; + } } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 7f8d5cfd2..cfec610bd 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -9,25 +9,31 @@ use App\Listeners\RecategorizeReleaseAfterNameFix; use App\Models\AnidbInfo; use App\Models\AnidbTitle; use App\Models\BookInfo; +use App\Models\Category; use App\Models\ConsoleInfo; use App\Models\GamesInfo; use App\Models\MovieInfo; use App\Models\MusicInfo; use App\Models\Release; use App\Models\RolePromotion; +use App\Models\RootCategory; use App\Models\SteamApp; +use App\Models\UsenetGroup; use App\Models\User; use App\Models\Video; use App\Observers\AnidbInfoObserver; use App\Observers\AnidbTitleObserver; use App\Observers\BookInfoObserver; +use App\Observers\CategoryObserver; use App\Observers\ConsoleInfoObserver; use App\Observers\GamesInfoObserver; use App\Observers\MovieInfoObserver; use App\Observers\MusicInfoObserver; use App\Observers\ReleaseObserver; use App\Observers\RolePromotionObserver; +use App\Observers\RootCategoryObserver; use App\Observers\SteamAppObserver; +use App\Observers\UsenetGroupObserver; use App\Observers\VideoObserver; use App\View\Composers\AdminDataComposer; use App\View\Composers\GlobalDataComposer; @@ -74,6 +80,9 @@ class AppServiceProvider extends ServiceProvider MovieInfo::observe(MovieInfoObserver::class); Video::observe(VideoObserver::class); Release::observe(ReleaseObserver::class); + Category::observe(CategoryObserver::class); + RootCategory::observe(RootCategoryObserver::class); + UsenetGroup::observe(UsenetGroupObserver::class); MusicInfo::observe(MusicInfoObserver::class); BookInfo::observe(BookInfoObserver::class); GamesInfo::observe(GamesInfoObserver::class); diff --git a/app/Services/ConsoleService.php b/app/Services/ConsoleService.php index 00bbcdfee..4a5692711 100644 --- a/app/Services/ConsoleService.php +++ b/app/Services/ConsoleService.php @@ -632,7 +632,7 @@ class ConsoleService } $result['release'] = $releaseName; - array_map('trim', $result); + $result = array_map('trim', $result); return (isset($result['title'], $result['platform']) && ! empty($result['title'])) ? $result : false; } diff --git a/app/Services/Releases/ReleaseBrowseService.php b/app/Services/Releases/ReleaseBrowseService.php index e3cb3aef4..caa65f318 100644 --- a/app/Services/Releases/ReleaseBrowseService.php +++ b/app/Services/Releases/ReleaseBrowseService.php @@ -9,6 +9,7 @@ use App\Models\Category; use App\Models\Release; use App\Models\Settings; use App\Models\UsenetGroup; +use App\Support\ReleaseSearchIndexDocument; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; @@ -375,6 +376,7 @@ class ReleaseBrowseService 'sort_field' => $indexSortField, 'sort_dir' => $orderBy[1] ?? 'desc', 'try_fuzzy' => false, + 'include_documents' => true, ]; $filtered = Search::searchReleasesFiltered($criteria, (int) $num, (int) $start); @@ -382,6 +384,24 @@ class ReleaseBrowseService return []; } + if (! empty($filtered['documents'])) { + $documents = array_map(static function (array $document): object { + $row = ReleaseSearchIndexDocument::toReleaseRow($document); + $row['title'] = $row['episode_title']; + $row['firstaired'] = $row['firstaired_ts'] > 0 + ? date('Y-m-d H:i:s', $row['firstaired_ts']) + : null; + + return (object) $row; + }, $filtered['documents']); + + if ($documents !== []) { + $documents[0]->_totalcount = $documents[0]->_totalrows = (int) ($filtered['total'] ?? count($documents)); + } + + return $documents; + } + $ids = array_map(static fn (int|string $id): int => (int) $id, $filtered['ids']); $idList = implode(',', $ids); $fieldOrder = implode(',', $ids); diff --git a/app/Services/Releases/ReleaseSearchService.php b/app/Services/Releases/ReleaseSearchService.php index 8a8280abb..2aef27569 100644 --- a/app/Services/Releases/ReleaseSearchService.php +++ b/app/Services/Releases/ReleaseSearchService.php @@ -13,6 +13,7 @@ use App\Models\Settings; use App\Models\UsenetGroup; use App\Services\Search\DTO\ReleaseSearchQuery; use App\Services\Search\DTO\SearchCursor; +use App\Support\ReleaseSearchIndexDocument; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; @@ -259,11 +260,31 @@ class ReleaseSearchService 'sort_field' => $this->browseOrderToIndexSortField($orderField), 'sort_dir' => $orderDir, 'try_fuzzy' => true, + 'include_documents' => true, ]; $criteria['track_total'] = $cursor === null; $searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset, $cursor)); $filtered = $searchPage->legacy(); + + if ($searchPage->documents !== []) { + $releases = collect(array_map(function (array $document): object { + $row = ReleaseSearchIndexDocument::toReleaseRow($document); + $row['title'] = $row['episode_title']; + $row['firstaired'] = $row['firstaired_ts'] > 0 + ? date('Y-m-d H:i:s', $row['firstaired_ts']) + : null; + + return (object) $row; + }, $searchPage->documents)); + if ($releases->isNotEmpty()) { + $releases[0]->_totalrows = (int) $filtered['total']; + $releases[0]->_search_last_sort = $searchPage->lastSortValues; + $releases[0]->_search_has_more = $searchPage->hasMore; + } + + return $releases; + } if ($cursor !== null && $filtered['total'] === 0) { $filtered['total'] = $cursor->total; } diff --git a/app/Services/Search/DTO/ReleaseSearchQuery.php b/app/Services/Search/DTO/ReleaseSearchQuery.php index b5de31b63..8806bff55 100644 --- a/app/Services/Search/DTO/ReleaseSearchQuery.php +++ b/app/Services/Search/DTO/ReleaseSearchQuery.php @@ -32,6 +32,7 @@ final readonly class ReleaseSearchQuery public int $offset = 0, public ?SearchCursor $cursor = null, public bool $trackTotal = true, + public bool $includeDocuments = false, ) {} /** @param array $criteria */ @@ -57,6 +58,7 @@ final readonly class ReleaseSearchQuery offset: max(0, $offset), cursor: $cursor, trackTotal: (bool) ($criteria['track_total'] ?? true), + includeDocuments: (bool) ($criteria['include_documents'] ?? false), ); } @@ -81,6 +83,7 @@ final readonly class ReleaseSearchQuery 'try_fuzzy' => $this->tryFuzzy, 'cursor_sort' => $this->cursor?->sortValues, 'track_total' => $this->trackTotal, + 'include_documents' => $this->includeDocuments, ]; } diff --git a/app/Services/Search/DTO/SearchPage.php b/app/Services/Search/DTO/SearchPage.php index 0e52bf362..d0c6fad0a 100644 --- a/app/Services/Search/DTO/SearchPage.php +++ b/app/Services/Search/DTO/SearchPage.php @@ -9,6 +9,7 @@ final readonly class SearchPage /** * @param list $ids * @param list $lastSortValues + * @param list> $documents */ public function __construct( public array $ids, @@ -19,6 +20,7 @@ final readonly class SearchPage public float $durationMs = 0.0, public array $lastSortValues = [], public bool $hasMore = false, + public array $documents = [], ) {} /** @return array{ids: list, total: int, fuzzy: bool} */ diff --git a/app/Services/Search/Drivers/ElasticSearchDriver.php b/app/Services/Search/Drivers/ElasticSearchDriver.php index e2548c37c..74fa7b5c0 100644 --- a/app/Services/Search/Drivers/ElasticSearchDriver.php +++ b/app/Services/Search/Drivers/ElasticSearchDriver.php @@ -109,6 +109,7 @@ class ElasticSearchDriver implements SearchDriverInterface durationMs: (hrtime(true) - $startedAt) / 1_000_000, lastSortValues: $result['last_sort'] ?? [], hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])), + documents: $result['documents'] ?? [], ); } diff --git a/app/Services/Search/Drivers/ManticoreSearchDriver.php b/app/Services/Search/Drivers/ManticoreSearchDriver.php index 6ec1ecaef..c073c4280 100644 --- a/app/Services/Search/Drivers/ManticoreSearchDriver.php +++ b/app/Services/Search/Drivers/ManticoreSearchDriver.php @@ -19,6 +19,7 @@ use App\Services\Search\DTO\ReleaseSearchQuery; use App\Services\Search\DTO\SearchPage; use App\Services\Search\Support\ManticoreClientFactory; use App\Services\Search\Support\ManticoreIndexRegistry; +use App\Services\Search\Support\ReleaseIndexProjection; use App\Support\ReleaseSearchIndexDocument; use App\Support\SecondaryIndexDocuments; use Illuminate\Support\Facades\Cache; @@ -121,9 +122,87 @@ class ManticoreSearchDriver implements SearchDriverInterface durationMs: (hrtime(true) - $startedAt) / 1_000_000, lastSortValues: $result['last_sort'] ?? [], hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])), + documents: $result['documents'] ?? [], ); } + /** + * Return the current document count for an RT table. + */ + public function countIndexDocuments(string $index): int + { + if (preg_match('/^[a-zA-Z0-9_]+$/', $index) !== 1) { + throw new \InvalidArgumentException('Unsafe Manticore index name.'); + } + + $response = $this->manticoreSearch->sql("SELECT COUNT(*) AS total FROM `{$index}`", true); + if (! is_array($response)) { + return 0; + } + + foreach ($response as $key => $value) { + if (is_array($value) && isset($value['total'])) { + return (int) $value['total']; + } + if ($key === 'total' && is_numeric($value)) { + return (int) $value; + } + if (is_numeric($value)) { + return (int) $value; + } + } + + return 0; + } + + /** + * Return a stable page of release document ids for drift/orphan checks. + * + * @return list + */ + public function indexedReleaseIds(int $limit = 1000, int $offset = 0): array + { + $limit = max(1, min(10000, $limit)); + $offset = max(0, $offset); + $index = $this->getReleasesIndex(); + + if (preg_match('/^[a-zA-Z0-9_]+$/', $index) !== 1) { + throw new \InvalidArgumentException('Unsafe Manticore index name.'); + } + + $sql = "SELECT id FROM {$index} ORDER BY id ASC LIMIT {$offset},{$limit} OPTION max_matches=".($offset + $limit); + $response = $this->manticoreSearch->sql($sql, true); + if (! is_array($response)) { + return []; + } + + $ids = []; + foreach ($response as $key => $value) { + if (ctype_digit((string) $key)) { + $ids[(int) $key] = true; + } + if (is_array($value) && isset($value['id'])) { + $ids[(int) $value['id']] = true; + } + if ( + ($key === 'id' || is_int($key) || ctype_digit((string) $key)) + && (is_int($value) || (is_string($value) && ctype_digit($value))) + ) { + $ids[(int) $value] = true; + } + } + + if (isset($response['data']) && is_array($response['data'])) { + foreach ($response['data'] as $row) { + if (is_array($row) && isset($row['id'])) { + $ids[(int) $row['id']] = true; + } + } + } + + return array_values(array_filter(array_map('intval', array_keys($ids)), static fn (int $id): bool => $id > 0)); + } + /** * Check if autocomplete is enabled. */ @@ -209,7 +288,7 @@ class ManticoreSearchDriver implements SearchDriverInterface $releaseId = (int) $parameters['id']; if (! $this->replaceReleaseDocumentWithRetry($parameters)) { - $this->recordReleaseIndexFailure($releaseId, 'insertRelease'); + $this->recordReleaseIndexFailure($releaseId, 'insertRelease', 'upsert'); try { ReindexReleaseJob::dispatch($releaseId)->delay(now()->addSeconds(2)); } catch (\Throwable $e) { @@ -226,24 +305,34 @@ class ManticoreSearchDriver implements SearchDriverInterface */ private function replaceReleaseDocumentWithRetry(array $parameters): bool { - $document = ReleaseSearchIndexDocument::normalize($parameters); + // updateRelease() supplies the already-normalized canonical projection. + // Preserve its *_ts fields instead of normalizing a second time and + // replacing post/add timestamps with zero. + $document = ReleaseSearchIndexDocument::normalizeForBulk($parameters); unset($document['id']); $indexName = $this->config['indexes']['releases']; $releaseId = (int) $parameters['id']; - for ($attempt = 0; $attempt < 2; $attempt++) { + $attempts = max(1, (int) ($this->config['retry_attempts'] ?? config('search.drivers.manticore.retry_attempts', 3))); + $delayMs = max(0, (int) ($this->config['retry_delay_ms'] ?? config('search.drivers.manticore.retry_delay_ms', 100))); + + for ($attempt = 0; $attempt < $attempts; $attempt++) { try { $this->manticoreSearch->table($indexName) ->replaceDocument($document, $releaseId); + $this->resolveReleaseIndexFailure($releaseId); + return true; } catch (\Throwable $e) { $isResponse = $e instanceof ResponseException; $isManticoreRuntime = $e instanceof RuntimeException; - if (($isResponse || $isManticoreRuntime) && $attempt === 0) { - usleep(100_000); + if (($isResponse || $isManticoreRuntime) && $attempt < $attempts - 1) { + if ($delayMs > 0) { + usleep($delayMs * 1000 * ($attempt + 1)); + } continue; } @@ -277,9 +366,33 @@ class ManticoreSearchDriver implements SearchDriverInterface return false; } - private function recordReleaseIndexFailure(int $releaseId, string $phase): void + private function recordReleaseIndexFailure(int $releaseId, string $phase, string $operation = 'upsert'): void { Cache::increment('search:index:failures:releases'); + try { + $existing = DB::table('search_index_failures') + ->where('release_id', $releaseId) + ->first(['attempts']); + $attempts = ((int) ($existing->attempts ?? 0)) + 1; + DB::table('search_index_failures')->updateOrInsert( + ['release_id' => $releaseId], + [ + 'operation' => $operation, + 'attempts' => $attempts, + 'last_error' => $phase, + 'next_attempt_at' => now()->addSeconds(min(3600, 2 ** min($attempts, 10))), + 'resolved_at' => null, + 'updated_at' => now(), + 'created_at' => now(), + ] + ); + } catch (\Throwable $e) { + Log::error('ManticoreSearch: unable to persist release index failure', [ + 'release_id' => $releaseId, + 'phase' => $phase, + 'error' => $e->getMessage(), + ]); + } if (Cache::add('search:index:release_index_warn_lock', true, 60)) { Log::warning('ManticoreSearch: release indexing failure', [ 'release_id_sample' => $releaseId, @@ -289,6 +402,28 @@ class ManticoreSearchDriver implements SearchDriverInterface } } + private function resolveReleaseIndexFailure(int $releaseId): void + { + if ($releaseId <= 0) { + return; + } + + try { + DB::table('search_index_failures') + ->where('release_id', $releaseId) + ->update([ + 'resolved_at' => now(), + 'next_attempt_at' => null, + 'updated_at' => now(), + ]); + } catch (\Throwable $e) { + Log::debug('ManticoreSearch: unable to mark release index failure resolved', [ + 'release_id' => $releaseId, + 'error' => $e->getMessage(), + ]); + } + } + private function recordReleaseNotFoundForIndex(int|string $releaseID): void { Cache::increment('search:index:failures:release_not_found'); @@ -361,13 +496,34 @@ class ManticoreSearchDriver implements SearchDriverInterface return; } - try { - $this->manticoreSearch->table($this->getReleasesIndex()) - ->deleteDocumentsByIds($ids); - } catch (ResponseException $e) { - Log::error('ManticoreSearch deleteReleases error: '.$e->getMessage(), [ - 'ids' => $ids, - ]); + $attempts = max(1, (int) ($this->config['retry_attempts'] ?? config('search.drivers.manticore.retry_attempts', 2))); + $delayMs = max(0, (int) ($this->config['retry_delay_ms'] ?? config('search.drivers.manticore.retry_delay_ms', 100))); + + for ($attempt = 0; $attempt < $attempts; $attempt++) { + try { + $this->manticoreSearch->table($this->getReleasesIndex()) + ->deleteDocumentsByIds($ids); + foreach ($ids as $id) { + $this->resolveReleaseIndexFailure((int) $id); + } + + return; + } catch (\Throwable $e) { + if ($attempt < $attempts - 1) { + if ($delayMs > 0) { + usleep($delayMs * 1000 * ($attempt + 1)); + } + + continue; + } + + Log::error('ManticoreSearch deleteReleases error: '.$e->getMessage(), [ + 'ids' => $ids, + ]); + foreach ($ids as $id) { + $this->recordReleaseIndexFailure((int) $id, 'deleteReleases: '.$e->getMessage(), 'delete'); + } + } } } @@ -665,6 +821,42 @@ class ManticoreSearchDriver implements SearchDriverInterface return $fieldSelector.' '.$scopedQuery; } + /** + * Build a release-name query that also handles punctuation used as token + * separators in Usenet subjects (for example, WEB-DL.x265). + * + * The prepared query deliberately escapes mid-word hyphens so user search + * operators remain safe. Manticore tokenizes those separators instead of + * indexing them as part of a word, though, so an ordinary release-name + * query needs a normalized-token alternative as well. + */ + private static function scopeReleaseSearchQuery(string $rawQuery, string $preparedQuery, string $fieldSelector = ''): string + { + $primary = self::scopePreparedQueryToField($preparedQuery, $fieldSelector); + + if ($fieldSelector !== '@searchname' || self::queryHasNegation($rawQuery)) { + return $primary; + } + + // Preserve advanced query syntax and only add the fallback for plain + // release-name input whose punctuation is a token separator. + if (preg_match('/[^\p{L}\p{N}\s._-]/u', $rawQuery) === 1 || ! preg_match('/[._-]/', $rawQuery)) { + return $primary; + } + + $normalized = preg_replace('/[._-]+/u', ' ', trim($rawQuery)); + $normalized = is_string($normalized) ? trim(preg_replace('/\s+/', ' ', $normalized) ?? '') : ''; + $normalizedPrepared = self::prepareUserSearchQuery($normalized); + + if ($normalizedPrepared === '' || $normalizedPrepared === $preparedQuery) { + return $primary; + } + + $normalizedQuery = self::scopePreparedQueryToField($normalizedPrepared, $fieldSelector); + + return '('.$primary.' | '.$normalizedQuery.')'; + } + /** * Check if a search query contains negation operators (! or - prefix on words). * @@ -706,64 +898,10 @@ class ManticoreSearchDriver implements SearchDriverInterface } try { - $release = Release::query() - ->where('releases.id', $releaseID) - ->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id') - ->leftJoin('movieinfo as mi', 'releases.movieinfo_id', '=', 'mi.id') - ->leftJoin('videos as v', 'releases.videos_id', '=', 'v.id') - ->select([ - 'releases.id', - 'releases.name', - 'releases.searchname', - 'releases.fromname', - 'releases.categories_id', - 'releases.size', - 'releases.postdate', - 'releases.adddate', - 'releases.totalpart', - 'releases.grabs', - 'releases.passwordstatus', - 'releases.groups_id', - 'releases.nzbstatus', - 'releases.haspreview', - 'releases.imdbid', - 'releases.videos_id', - 'releases.movieinfo_id', - DB::raw('IFNULL(mi.tmdbid, 0) AS tmdbid'), - DB::raw('IFNULL(mi.traktid, 0) AS traktid'), - DB::raw('IFNULL(v.tvdb, 0) AS tvdb'), - DB::raw('IFNULL(v.tvmaze, 0) AS tvmaze'), - DB::raw('IFNULL(v.tvrage, 0) AS tvrage'), - DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename'), - ]) - ->groupBy([ - 'releases.id', - 'releases.name', - 'releases.searchname', - 'releases.fromname', - 'releases.categories_id', - 'releases.size', - 'releases.postdate', - 'releases.adddate', - 'releases.totalpart', - 'releases.grabs', - 'releases.passwordstatus', - 'releases.groups_id', - 'releases.nzbstatus', - 'releases.haspreview', - 'releases.imdbid', - 'releases.videos_id', - 'releases.movieinfo_id', - 'mi.tmdbid', - 'mi.traktid', - 'v.tvdb', - 'v.tvmaze', - 'v.tvrage', - ]) - ->first(); + $release = ReleaseIndexProjection::forId((int) $releaseID); if ($release !== null) { - $this->insertRelease($release->toArray()); + $this->insertRelease($release); } else { Log::warning('ManticoreSearch: Release not found for update, removing from index', ['id' => $releaseID]); $this->recordReleaseNotFoundForIndex($releaseID); @@ -1260,9 +1398,10 @@ class ManticoreSearchDriver implements SearchDriverInterface $terms = []; foreach ($searchArray as $key => $value) { if (! empty($value)) { - $preparedValue = self::prepareUserSearchQuery($value); + $rawValue = (string) $value; + $preparedValue = self::prepareUserSearchQuery($rawValue); if (! empty($preparedValue)) { - $terms[] = '@@relaxed '.self::scopePreparedQueryToField($preparedValue, '@'.$key); + $terms[] = '@@relaxed '.self::scopeReleaseSearchQuery($rawValue, $preparedValue, '@'.$key); } } } @@ -1294,7 +1433,7 @@ class ManticoreSearchDriver implements SearchDriverInterface } } - $searchExpr = '@@relaxed '.self::scopePreparedQueryToField($preparedSearch, $searchColumns); + $searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery($searchString, $preparedSearch, $searchColumns); } else { return []; } @@ -2401,7 +2540,7 @@ class ManticoreSearchDriver implements SearchDriverInterface return $this->searchReleasesByCategory($categoryIds, $limit); } - $searchExpr = '@@relaxed '.self::scopePreparedQueryToField($preparedSearch, '@searchname'); + $searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery($searchTerm, $preparedSearch, '@searchname'); $query = (new Search($this->manticoreSearch)) ->setTable($this->getReleasesIndex()) @@ -2486,7 +2625,7 @@ class ManticoreSearchDriver implements SearchDriverInterface } $prepared = self::prepareUserSearchQuery((string) $value); if ($prepared !== '') { - $terms[] = '@@relaxed '.self::scopePreparedQueryToField($prepared, '@'.$key); + $terms[] = '@@relaxed '.self::scopeReleaseSearchQuery((string) $value, $prepared, '@'.$key); } } if ($terms === []) { @@ -2532,8 +2671,15 @@ class ManticoreSearchDriver implements SearchDriverInterface } $ids = []; + $documents = []; foreach ($results as $doc) { - $ids[] = $doc->getId(); + $id = (int) $doc->getId(); + $ids[] = $id; + if ((bool) ($criteria['include_documents'] ?? false)) { + $document = $doc->getData(); + $document['id'] = $id; + $documents[] = $document; + } } return [ @@ -2545,6 +2691,7 @@ class ManticoreSearchDriver implements SearchDriverInterface // supports a compound search-after predicate without raw SQL. 'last_sort' => [], 'has_more' => $offset + count($ids) < (int) $results->getTotal(), + 'documents' => $documents, ]; }; diff --git a/app/Services/Search/Support/ManticoreIndexRegistry.php b/app/Services/Search/Support/ManticoreIndexRegistry.php index 754c123ba..dc84fb873 100644 --- a/app/Services/Search/Support/ManticoreIndexRegistry.php +++ b/app/Services/Search/Support/ManticoreIndexRegistry.php @@ -17,15 +17,27 @@ final class ManticoreIndexRegistry return [ 'releases' => ['settings' => $settings, 'columns' => [ - 'name' => ['type' => 'text'], 'searchname' => ['type' => 'text'], + 'guid' => ['type' => 'string'], 'name' => ['type' => 'text'], + 'searchname' => ['type' => 'text'], 'plainsearchname' => ['type' => 'text'], 'fromname' => ['type' => 'text'], 'filename' => ['type' => 'text'], + 'category_name' => ['type' => 'string'], 'parent_category' => ['type' => 'string'], + 'sub_category' => ['type' => 'string'], 'group_name' => ['type' => 'string'], 'categories_id' => ['type' => 'integer'], 'imdbid' => ['type' => 'string'], 'tmdbid' => ['type' => 'integer'], 'traktid' => ['type' => 'integer'], 'tvdb' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'], - 'tvrage' => ['type' => 'integer'], 'videos_id' => ['type' => 'integer'], - 'movieinfo_id' => ['type' => 'integer'], 'size' => ['type' => 'bigint'], + 'tvrage' => ['type' => 'integer'], 'trakt' => ['type' => 'integer'], + 'imdb' => ['type' => 'string'], 'tmdb' => ['type' => 'integer'], + 'videos_id' => ['type' => 'integer'], + 'tv_episodes_id' => ['type' => 'integer'], 'movieinfo_id' => ['type' => 'integer'], + 'anidbid' => ['type' => 'integer'], 'parentid' => ['type' => 'integer'], + 'episode_title' => ['type' => 'text'], 'series' => ['type' => 'integer'], + 'episode' => ['type' => 'integer'], 'firstaired_ts' => ['type' => 'bigint'], + 'size' => ['type' => 'bigint'], 'postdate_ts' => ['type' => 'bigint'], 'adddate_ts' => ['type' => 'bigint'], 'totalpart' => ['type' => 'integer'], 'grabs' => ['type' => 'integer'], + 'comments' => ['type' => 'integer'], 'nfostatus' => ['type' => 'integer'], + 'jpgstatus' => ['type' => 'integer'], 'nfoid' => ['type' => 'integer'], + 'reid' => ['type' => 'integer'], 'passwordstatus' => ['type' => 'bigint'], 'groups_id' => ['type' => 'integer'], 'nzbstatus' => ['type' => 'integer'], 'haspreview' => ['type' => 'bigint'], ]], @@ -56,7 +68,7 @@ final class ManticoreIndexRegistry public static function profile(string $logical): array { $fields = match ($logical) { - 'releases' => ['searchname' => 12, 'name' => 8, 'filename' => 5, 'fromname' => 1], + 'releases' => ['searchname' => 12, 'plainsearchname' => 10, 'name' => 8, 'filename' => 5, 'fromname' => 1], 'predb' => ['title' => 12, 'filename' => 5], 'movies' => ['title' => 12, 'director' => 5, 'actors' => 3, 'genre' => 2, 'plot' => 1], 'tvshows' => ['title' => 12], diff --git a/app/Services/Search/Support/ReleaseIndexProjection.php b/app/Services/Search/Support/ReleaseIndexProjection.php new file mode 100644 index 000000000..04eceb20a --- /dev/null +++ b/app/Services/Search/Support/ReleaseIndexProjection.php @@ -0,0 +1,90 @@ +getDriverName() === 'sqlite' + ? "GROUP_CONCAT(rf.name, ' ')" + : "GROUP_CONCAT(rf.name SEPARATOR ' ')"; + $categoryName = DB::connection()->getDriverName() === 'sqlite' + ? "cp.title || ' > ' || c.title" + : "CONCAT(cp.title, ' > ', c.title)"; + + return DB::table('releases as r') + ->leftJoin('usenet_groups as g', 'g.id', '=', 'r.groups_id') + ->leftJoin('categories as c', 'c.id', '=', 'r.categories_id') + ->leftJoin('root_categories as cp', 'cp.id', '=', 'c.root_categories_id') + ->leftJoin('movieinfo as mi', function ($join): void { + $join->on('mi.id', '=', 'r.movieinfo_id') + ->where('r.movieinfo_id', '>', 0); + }) + ->leftJoin('videos as v', function ($join): void { + $join->on('v.id', '=', 'r.videos_id') + ->where('r.videos_id', '>', 0); + }) + ->leftJoin('tv_episodes as tve', function ($join): void { + $join->on('tve.id', '=', 'r.tv_episodes_id') + ->where('r.tv_episodes_id', '>', 0); + }) + ->leftJoin('release_files as rf', 'rf.releases_id', '=', 'r.id') + ->leftJoin('release_nfos as rn', 'rn.releases_id', '=', 'r.id') + ->leftJoin('video_data as vd', 'vd.releases_id', '=', 'r.id') + ->select([ + 'r.id', 'r.guid', 'r.name', 'r.searchname', 'r.fromname', 'r.categories_id', + 'r.groups_id', 'r.size', 'r.postdate', 'r.adddate', 'r.totalpart', 'r.grabs', + 'r.comments', 'r.passwordstatus', 'r.nzbstatus', 'r.nfostatus', 'r.haspreview', + 'r.jpgstatus', 'r.videos_id', 'r.tv_episodes_id', 'r.movieinfo_id', 'r.imdbid', + 'r.anidbid', 'g.name as group_name', 'c.root_categories_id as parentid', + 'c.title as sub_category', 'tve.title as episode_title', 'tve.series', + 'tve.episode', 'tve.firstaired', + 'cp.title as parent_category', DB::raw("{$categoryName} AS category_name"), + DB::raw("COALESCE({$groupConcat}, '') AS filename"), + DB::raw('COALESCE(mi.tmdbid, 0) AS tmdbid'), + DB::raw('COALESCE(mi.traktid, 0) AS traktid'), + DB::raw('COALESCE(v.tvdb, 0) AS tvdb'), + DB::raw('COALESCE(v.tvmaze, 0) AS tvmaze'), + DB::raw('COALESCE(v.tvrage, 0) AS tvrage'), + DB::raw('COALESCE(v.trakt, 0) AS trakt'), + DB::raw("COALESCE(v.imdb, '') AS imdb"), + DB::raw('COALESCE(v.tmdb, 0) AS tmdb'), + DB::raw('COALESCE(rn.releases_id, 0) AS nfoid'), + DB::raw('COALESCE(vd.releases_id, 0) AS reid'), + ]) + ->groupBy([ + 'r.id', 'r.guid', 'r.name', 'r.searchname', 'r.fromname', 'r.categories_id', + 'r.groups_id', 'r.size', 'r.postdate', 'r.adddate', 'r.totalpart', 'r.grabs', + 'r.comments', 'r.passwordstatus', 'r.nzbstatus', 'r.nfostatus', 'r.haspreview', + 'r.jpgstatus', 'r.videos_id', 'r.tv_episodes_id', 'r.movieinfo_id', 'r.imdbid', + 'r.anidbid', 'g.name', 'c.root_categories_id', 'c.title', 'tve.title', + 'tve.series', 'tve.episode', 'tve.firstaired', 'cp.title', 'mi.tmdbid', + 'mi.traktid', 'v.tvdb', 'v.tvmaze', 'v.tvrage', 'v.trakt', 'v.imdb', 'v.tmdb', + 'rn.releases_id', 'vd.releases_id', + ]); + } + + /** + * @return array|null + */ + public static function forId(int $releaseId): ?array + { + if ($releaseId <= 0) { + return null; + } + + $row = self::query()->where('r.id', $releaseId)->first(); + + return $row === null ? null : ReleaseSearchIndexDocument::normalize((array) $row); + } +} diff --git a/app/Support/ReleaseSearchIndexDocument.php b/app/Support/ReleaseSearchIndexDocument.php index f36dd7e45..396f22f19 100644 --- a/app/Support/ReleaseSearchIndexDocument.php +++ b/app/Support/ReleaseSearchIndexDocument.php @@ -9,6 +9,26 @@ namespace App\Support; */ final class ReleaseSearchIndexDocument { + /** + * Fields that may be returned to browse/search consumers. + * Sensitive release data must never be added to this list. + * + * @return list + */ + public static function fields(): array + { + return [ + 'id', 'guid', 'name', 'searchname', 'plainsearchname', 'fromname', 'filename', + 'categories_id', 'category_name', 'parent_category', 'sub_category', + 'groups_id', 'group_name', 'imdbid', 'tmdbid', 'traktid', 'tvdb', 'tvmaze', + 'tvrage', 'trakt', 'imdb', 'tmdb', 'videos_id', 'tv_episodes_id', 'movieinfo_id', 'anidbid', + 'parentid', 'episode_title', 'series', 'episode', 'firstaired_ts', + 'size', 'postdate_ts', 'adddate_ts', 'totalpart', 'grabs', 'comments', + 'passwordstatus', 'nzbstatus', 'nfostatus', 'haspreview', 'jpgstatus', + 'nfoid', 'reid', + ]; + } + /** * Normalize a release row for bulk indexing. Rows already produced by {@see normalize()} * (e.g. from release search populate) lack `postdate` / `adddate` keys; @@ -40,10 +60,16 @@ final class ReleaseSearchIndexDocument return [ 'id' => (int) ($row['id'] ?? 0), + 'guid' => (string) ($row['guid'] ?? ''), 'name' => (string) ($row['name'] ?? ''), 'searchname' => (string) ($row['searchname'] ?? ''), + 'plainsearchname' => (string) ($row['plainsearchname'] ?? self::plainSearchName($row['searchname'] ?? '')), 'fromname' => (string) ($row['fromname'] ?? ''), 'categories_id' => (int) ($row['categories_id'] ?? 0), + 'category_name' => (string) ($row['category_name'] ?? ''), + 'parent_category' => (string) ($row['parent_category'] ?? ''), + 'sub_category' => (string) ($row['sub_category'] ?? ''), + 'group_name' => (string) ($row['group_name'] ?? ''), 'filename' => (string) ($row['filename'] ?? ''), 'imdbid' => (string) ($row['imdbid'] ?? ''), 'tmdbid' => (int) ($row['tmdbid'] ?? 0), @@ -51,20 +77,50 @@ final class ReleaseSearchIndexDocument 'tvdb' => (int) ($row['tvdb'] ?? 0), 'tvmaze' => (int) ($row['tvmaze'] ?? 0), 'tvrage' => (int) ($row['tvrage'] ?? 0), + 'trakt' => (int) ($row['trakt'] ?? 0), + 'imdb' => (string) ($row['imdb'] ?? ''), + 'tmdb' => (int) ($row['tmdb'] ?? 0), 'videos_id' => (int) ($row['videos_id'] ?? 0), + 'tv_episodes_id' => (int) ($row['tv_episodes_id'] ?? 0), 'movieinfo_id' => (int) ($row['movieinfo_id'] ?? 0), + 'anidbid' => (int) ($row['anidbid'] ?? 0), + 'parentid' => (int) ($row['parentid'] ?? 0), + 'episode_title' => (string) ($row['episode_title'] ?? ''), + 'series' => (int) ($row['series'] ?? 0), + 'episode' => (int) ($row['episode'] ?? 0), + 'firstaired_ts' => self::datetimeToUnix($row['firstaired'] ?? ($row['firstaired_ts'] ?? null)), 'size' => (int) ($row['size'] ?? 0), 'postdate_ts' => $postdateTs, 'adddate_ts' => $adddateTs, 'totalpart' => (int) ($row['totalpart'] ?? 0), 'grabs' => (int) ($row['grabs'] ?? 0), + 'comments' => (int) ($row['comments'] ?? 0), 'passwordstatus' => (int) ($row['passwordstatus'] ?? 0), 'groups_id' => (int) ($row['groups_id'] ?? 0), 'nzbstatus' => (int) ($row['nzbstatus'] ?? 0), + 'nfostatus' => (int) ($row['nfostatus'] ?? 0), 'haspreview' => (int) ($row['haspreview'] ?? 0), + 'jpgstatus' => (int) ($row['jpgstatus'] ?? 0), + 'nfoid' => (int) ($row['nfoid'] ?? 0), + 'reid' => (int) ($row['reid'] ?? 0), ]; } + /** + * Convert an indexed document into the date/attribute names expected by the views. + * + * @param array $document + * @return array + */ + public static function toReleaseRow(array $document): array + { + $row = self::normalizeForBulk($document); + $row['postdate'] = self::unixToDatetime((int) $row['postdate_ts']); + $row['adddate'] = self::unixToDatetime((int) $row['adddate_ts']); + + return $row; + } + private static function datetimeToUnix(mixed $value): int { if ($value === null || $value === '') { @@ -77,4 +133,14 @@ final class ReleaseSearchIndexDocument return $ts !== false ? $ts : 0; } + + private static function plainSearchName(mixed $value): string + { + return trim((string) preg_replace('/[._-]+/', ' ', (string) $value)); + } + + private static function unixToDatetime(int $timestamp): ?string + { + return $timestamp > 0 ? date('Y-m-d H:i:s', $timestamp) : null; + } } diff --git a/app/Support/ReleaseSearchIndexSync.php b/app/Support/ReleaseSearchIndexSync.php index f75c58d6d..6623e1de4 100644 --- a/app/Support/ReleaseSearchIndexSync.php +++ b/app/Support/ReleaseSearchIndexSync.php @@ -5,8 +5,10 @@ declare(strict_types=1); namespace App\Support; use App\Facades\Search; +use App\Models\Category; use App\Models\Release; use App\Observers\ReleaseObserver; +use Illuminate\Database\Eloquent\Builder; /** * Re-sync releases_rt / ES release documents after query-builder or raw SQL updates @@ -27,6 +29,61 @@ final class ReleaseSearchIndexSync } } + public static function forMovieInfo(int $movieInfoId): void + { + if ($movieInfoId > 0) { + self::forQuery(Release::query()->where('movieinfo_id', $movieInfoId)); + } + } + + public static function forVideo(int $videoId): void + { + if ($videoId > 0) { + self::forQuery(Release::query()->where('videos_id', $videoId)); + } + } + + public static function forCategory(int $categoryId): void + { + if ($categoryId > 0) { + self::forQuery(Release::query()->where('categories_id', $categoryId)); + } + } + + public static function forRootCategory(int $rootCategoryId): void + { + if ($rootCategoryId <= 0) { + return; + } + + $categoryIds = Category::query() + ->where('root_categories_id', $rootCategoryId) + ->pluck('id'); + + if ($categoryIds->isNotEmpty()) { + self::forQuery(Release::query()->whereIn('categories_id', $categoryIds)); + } + } + + public static function forQueryGroup(int $groupId): void + { + if ($groupId > 0) { + self::forQuery(Release::query()->where('groups_id', $groupId)); + } + } + + /** + * @param Builder $query + */ + private static function forQuery($query): void + { + $query->select('releases.id')->orderBy('releases.id')->chunkById(500, function ($releases): bool { + self::forIds($releases->pluck('id')); + + return true; + }, 'releases.id'); + } + /** * Reindex every release (chunked). Use after mass UPDATEs that match an optional raw WHERE suffix. * diff --git a/config/search.php b/config/search.php index 1b2b9aa74..0fec82e54 100644 --- a/config/search.php +++ b/config/search.php @@ -50,6 +50,8 @@ return [ 'anime' => env('MANTICORESEARCH_INDEX_ANIME', 'anime_rt'), ], 'max_matches' => env('MANTICORESEARCH_MAX_MATCHES', 10000), + 'retry_attempts' => (int) env('MANTICORESEARCH_RETRY_ATTEMPTS', 2), + 'retry_delay_ms' => (int) env('MANTICORESEARCH_RETRY_DELAY_MS', 100), 'cache_minutes' => env('MANTICORESEARCH_CACHE_MINUTES', 5), 'autocomplete' => [ 'enabled' => env('MANTICORESEARCH_AUTOCOMPLETE_ENABLED', true), diff --git a/database/migrations/2026_08_02_000000_create_search_index_failures_table.php b/database/migrations/2026_08_02_000000_create_search_index_failures_table.php new file mode 100644 index 000000000..735590f10 --- /dev/null +++ b/database/migrations/2026_08_02_000000_create_search_index_failures_table.php @@ -0,0 +1,33 @@ +id(); + $table->unsignedBigInteger('release_id')->unique(); + $table->string('operation', 32)->default('upsert'); + $table->unsignedInteger('attempts')->default(0); + $table->text('last_error')->nullable(); + $table->timestamp('next_attempt_at')->nullable()->index(); + $table->timestamp('resolved_at')->nullable()->index(); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists('search_index_failures'); + } +}; diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 5def260fe..478244a27 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -928,7 +928,7 @@ parameters: - message: '#^Parameter \#1 \$params of method Elastic\\Elasticsearch\\Client\:\:delete\(\) expects array\{id\: string, index\: string, wait_for_active_shards\?\: string, refresh\?\: string, routing\?\: string, timeout\?\: int\|string, if_seq_no\?\: int, if_primary_term\?\: int, \.\.\.\}\|null, array\{index\: string, id\: int\\|int\<1, max\>\} given\.$#' identifier: argument.type - count: 4 + count: 3 path: app/Services/Search/Drivers/ElasticSearchDriver.php - @@ -937,12 +937,6 @@ parameters: count: 2 path: app/Services/Search/Drivers/ElasticSearchDriver.php - - - message: '#^Method App\\Services\\Search\\Drivers\\ManticoreSearchDriver\:\:phrasesToSearchArray\(\) return type has no value type specified in iterable type array\.$#' - identifier: missingType.iterableValue - count: 1 - path: app/Services/Search/Drivers/ManticoreSearchDriver.php - - message: '#^Method App\\Services\\Search\\Drivers\\ManticoreSearchDriver\:\:searchReleasesByMultipleExternalIds\(\) should return array\ but returns list\\.$#' identifier: return.type diff --git a/routes/console.php b/routes/console.php index fa5506a2b..6b9cf4efc 100644 --- a/routes/console.php +++ b/routes/console.php @@ -61,6 +61,7 @@ Schedule::call(function () { // Check tmux health and auto-restart if monitor pane is dead Schedule::command('tmux:health-check --auto-restart')->everyThirtyMinutes()->withoutOverlapping(); Schedule::command('nntmux:check-service-health')->everyMinute()->withoutOverlapping(); +Schedule::command('nntmux:search-repair --limit=100')->everyMinute()->withoutOverlapping(); // Keep the admin dashboard snapshot (Cache::flexible) hot so admins never pay // the cold-cache cost when opening /admin/index. Schedule::command('admin:warm-dashboard')->everyMinute()->withoutOverlapping(); diff --git a/tests/Unit/ManticoreSearchQueryTest.php b/tests/Unit/ManticoreSearchQueryTest.php index be2c813a9..d854f791a 100644 --- a/tests/Unit/ManticoreSearchQueryTest.php +++ b/tests/Unit/ManticoreSearchQueryTest.php @@ -18,6 +18,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function negationQueriesProvider(): array { return [ @@ -37,6 +38,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function phraseQueriesProvider(): array { return [ @@ -55,6 +57,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function orQueriesProvider(): array { return [ @@ -71,6 +74,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function wildcardQueriesProvider(): array { return [ @@ -88,6 +92,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function groupingQueriesProvider(): array { return [ @@ -104,6 +109,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array */ public static function escapingQueriesProvider(): array { return [ @@ -143,6 +149,24 @@ class ManticoreSearchQueryTest extends TestCase ); } + #[Test] + public function it_adds_a_normalized_release_name_query_for_punctuation_separators(): void + { + $reflection = new ReflectionClass(ManticoreSearchDriver::class); + $method = $reflection->getMethod('scopeReleaseSearchQuery'); + + $query = $method->invoke( + null, + 'Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB-DL.HEVC.x265-BONE', + ManticoreSearchDriver::prepareUserSearchQuery('Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB-DL.HEVC.x265-BONE'), + '@searchname' + ); + + $this->assertStringContainsString('@searchname (Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB\\-DL.HEVC.x265\\-BONE)', $query); + $this->assertStringContainsString('@searchname (Love Is A Dogs Best Friend 2025 1080p WEB DL HEVC x265 BONE)', $query); + } + + /** @return array */ public static function edgeCaseQueriesProvider(): array { return [ @@ -163,6 +187,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertStringContainsString('circus', $result); } + /** @param array|string $input */ #[Test] #[DataProvider('negationDetectionProvider')] public function it_detects_negation_operators(array|string $input, bool $expected): void @@ -171,6 +196,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertSame($expected, $result); } + /** @return array|string, bool}> */ public static function negationDetectionProvider(): array { return [ @@ -244,7 +270,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertIsString($driverSource); $this->assertStringContainsString( - "\$terms[] = '@@relaxed '.self::scopePreparedQueryToField(\$prepared, '@'.\$key);", + "\$terms[] = '@@relaxed '.self::scopeReleaseSearchQuery((string) \$value, \$prepared, '@'.\$key);", $driverSource ); $this->assertStringNotContainsString( @@ -260,7 +286,7 @@ class ManticoreSearchQueryTest extends TestCase $this->assertIsString($driverSource); $this->assertStringContainsString( - "\$searchExpr = '@@relaxed '.self::scopePreparedQueryToField(\$preparedSearch, '@searchname');", + "\$searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery(\$searchTerm, \$preparedSearch, '@searchname');", $driverSource ); $this->assertStringNotContainsString( diff --git a/tests/Unit/NntmuxSearchReconcileFetchIndexedIdsTest.php b/tests/Unit/NntmuxSearchReconcileFetchIndexedIdsTest.php index 55d0d3bac..1d921873d 100644 --- a/tests/Unit/NntmuxSearchReconcileFetchIndexedIdsTest.php +++ b/tests/Unit/NntmuxSearchReconcileFetchIndexedIdsTest.php @@ -121,7 +121,7 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase public function __construct() {} - public function sql(...$params): mixed + public function sql(mixed ...$params): mixed { $this->capturedSql = $params[0] ?? null; @@ -143,6 +143,9 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase $this->assertNotNull($client->capturedSql); $this->assertStringContainsString('LIMIT 500', $client->capturedSql); $this->assertStringContainsString('OPTION max_matches=500', $client->capturedSql); + $driverSource = file_get_contents(__DIR__.'/../../app/Services/Search/Drivers/ManticoreSearchDriver.php'); + self::assertIsString($driverSource); + self::assertStringContainsString('ORDER BY id ASC', $driverSource); } /** @@ -159,7 +162,7 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase // Bypass parent ctor: we never hit the network in this test. } - public function sql(...$params): mixed + public function sql(mixed ...$params): mixed { return $this->stubResponse; } diff --git a/tests/Unit/ReleaseSearchIndexDocumentTest.php b/tests/Unit/ReleaseSearchIndexDocumentTest.php index dc179fb64..1330bce37 100644 --- a/tests/Unit/ReleaseSearchIndexDocumentTest.php +++ b/tests/Unit/ReleaseSearchIndexDocumentTest.php @@ -10,6 +10,61 @@ use PHPUnit\Framework\TestCase; final class ReleaseSearchIndexDocumentTest extends TestCase { + #[Test] + public function normalize_builds_the_complete_public_projection_without_sensitive_fields(): void + { + $document = ReleaseSearchIndexDocument::normalize([ + 'id' => '42', + 'guid' => 'guid-42', + 'name' => 'Release.Name', + 'searchname' => 'Release.Name', + 'fromname' => 'poster', + 'filename' => 'file.nzb', + 'categories_id' => '1', + 'category_name' => 'TV', + 'parent_category' => 'Video', + 'sub_category' => 'TV', + 'groups_id' => '2', + 'group_name' => 'alt.example', + 'tmdbid' => '123', + 'tv_episodes_id' => '456', + 'passwordstatus' => '-1', + 'comments' => '3', + 'postdate' => '2026-01-02 03:04:05', + 'adddate' => '2026-01-03 04:05:06', + 'nzb_password' => 'must-not-be-indexed', + ]); + + self::assertSame(42, $document['id']); + self::assertSame('Release Name', $document['plainsearchname']); + self::assertSame(123, $document['tmdbid']); + self::assertSame(456, $document['tv_episodes_id']); + self::assertSame(-1, $document['passwordstatus']); + self::assertGreaterThan(0, $document['postdate_ts']); + self::assertGreaterThan(0, $document['adddate_ts']); + self::assertArrayNotHasKey('nzb_password', $document); + $documentFields = array_keys($document); + $declaredFields = ReleaseSearchIndexDocument::fields(); + sort($documentFields); + sort($declaredFields); + self::assertSame($declaredFields, $documentFields); + } + + #[Test] + public function indexed_documents_can_be_hydrated_for_legacy_release_consumers(): void + { + $row = ReleaseSearchIndexDocument::toReleaseRow([ + 'id' => 42, + 'postdate_ts' => 1767323045, + 'adddate_ts' => 1767413106, + 'name' => 'Release', + ]); + + self::assertSame('2026-01-02 03:04:05', $row['postdate']); + self::assertSame('2026-01-03 04:05:06', $row['adddate']); + self::assertSame(42, $row['id']); + } + #[Test] public function normalize_for_bulk_preserves_timestamps_when_row_is_already_normalized(): void { diff --git a/tests/Unit/Services/Search/ManticoreDeleteReleasesTest.php b/tests/Unit/Services/Search/ManticoreDeleteReleasesTest.php new file mode 100644 index 000000000..31bf48e11 --- /dev/null +++ b/tests/Unit/Services/Search/ManticoreDeleteReleasesTest.php @@ -0,0 +1,70 @@ + '127.0.0.1', + 'port' => 9308, + 'indexes' => [ + 'releases' => 'releases_rt', + 'predb' => 'predb_rt', + ], + ]; + + $table = $this->createMock(Table::class); + $table->expects($this->once()) + ->method('deleteDocumentsByIds') + ->with([149331415, 42]); + + $client = $this->createMock(Client::class); + $client->expects($this->once()) + ->method('table') + ->with('releases_rt') + ->willReturn($table); + $client->expects($this->never())->method('sql'); + + $driver = new ManticoreSearchDriver($config); + $prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch'); + $prop->setAccessible(true); + $prop->setValue($driver, $client); + + $driver->deleteReleases([149331415, 42, 0, 149331415]); + } + + #[Test] + public function delete_releases_skips_empty_id_list(): void + { + $config = [ + 'host' => '127.0.0.1', + 'port' => 9308, + 'indexes' => [ + 'releases' => 'releases_rt', + 'predb' => 'predb_rt', + ], + ]; + + $client = $this->createMock(Client::class); + $client->expects($this->never())->method('table'); + $client->expects($this->never())->method('sql'); + + $driver = new ManticoreSearchDriver($config); + $prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch'); + $prop->setAccessible(true); + $prop->setValue($driver, $client); + + $driver->deleteReleases([0, -1]); + } +} diff --git a/tests/Unit/Services/Search/ManticoreInsertRetryTest.php b/tests/Unit/Services/Search/ManticoreInsertRetryTest.php index bba453539..91000da9b 100644 --- a/tests/Unit/Services/Search/ManticoreInsertRetryTest.php +++ b/tests/Unit/Services/Search/ManticoreInsertRetryTest.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace Tests\Unit\Services\Search; use App\Services\Search\Drivers\ManticoreSearchDriver; +use App\Support\ReleaseSearchIndexDocument; use Manticoresearch\Client; use Manticoresearch\Exceptions\ResponseException; use Manticoresearch\Request; @@ -130,4 +131,40 @@ final class ManticoreInsertRetryTest extends TestCase $ok = $refP->invoke($driver, $this->releaseRow(7)); $this->assertFalse($ok); } + + public function test_replace_release_document_preserves_pre_normalized_timestamps(): void + { + $config = [ + 'host' => '127.0.0.1', + 'port' => 9308, + 'retry_attempts' => 1, + 'indexes' => ['releases' => 'releases_rt', 'predb' => 'predb_rt'], + ]; + $document = ReleaseSearchIndexDocument::normalize($this->releaseRow(42)); + + $table = $this->createMock(Table::class); + $table->expects($this->once()) + ->method('replaceDocument') + ->with( + $this->callback(static fn (array $indexed): bool => + $indexed['postdate_ts'] === $document['postdate_ts'] + && $indexed['adddate_ts'] === $document['adddate_ts'] + ), + 42 + ); + + $client = $this->createMock(Client::class); + $client->expects($this->once()) + ->method('table') + ->with('releases_rt') + ->willReturn($table); + + $driver = new ManticoreSearchDriver($config); + $prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch'); + $prop->setValue($driver, $client); + + $method = new ReflectionMethod(ManticoreSearchDriver::class, 'replaceReleaseDocumentWithRetry'); + + $this->assertTrue($method->invoke($driver, $document)); + } }