starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update manticore search usage

Browse Source
This commit is contained in:
DariusIII
2026-08-02 22:49:50 +02:00
parent 7d3ad20c7b
commit afb06af625
38 changed files with 1133 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+2
View File
@@ -30,6 +30,8 @@ MANTICORESEARCH_FUZZY_ENABLED=false
# For high availability, set multiple hosts (comma-separated): "host1:9308,host2:9308"
MANTICORESEARCH_HOSTS=
MANTICORESEARCH_RETRIES=2
MANTICORESEARCH_RETRY_ATTEMPTS=2
MANTICORESEARCH_RETRY_DELAY_MS=100
# Autocomplete settings
MANTICORESEARCH_AUTOCOMPLETE_ENABLED=true
MANTICORESEARCH_AUTOCOMPLETE_MIN_LENGTH=2
app/Console/Commands/CreateManticoreIndexes.php
+15 -2
View File
@@ -12,7 +12,9 @@ use Manticoresearch\Exceptions\ResponseException;
class CreateManticoreIndexes extends Command
{
protected $signature = 'manticore:create-indexes {--drop : Drop existing indexes before creating}';
protected $signature = 'manticore:create-indexes
{--drop : Drop selected indexes before creating}
{--index=* : Logical index name(s) to create; defaults to all indexes}';
protected $description = 'Create Manticore Search indexes based on configuration';
@@ -33,8 +35,19 @@ class CreateManticoreIndexes extends Command
}
$configuredNames = config('search.drivers.manticore.indexes', []);
$selected = array_values(array_filter(array_map('strval', (array) $this->option('index'))));
$definitions = ManticoreIndexRegistry::definitions();
if ($selected !== []) {
$unknown = array_diff($selected, array_keys($definitions));
if ($unknown !== []) {
$this->error('Unknown logical index: '.implode(', ', $unknown));
return self::FAILURE;
}
$definitions = array_intersect_key($definitions, array_flip($selected));
}
$hasErrors = false;
foreach (ManticoreIndexRegistry::definitions() as $logical => $schema) {
foreach ($definitions as $logical => $schema) {
$indexName = (string) ($configuredNames[$logical] ?? $logical.'_rt');
if (! $this->createIndex($indexName, $schema, (bool) $this->option('drop'))) {
$hasErrors = true;
app/Console/Commands/InspectManticore.php
+1 -1
View File
@@ -56,7 +56,7 @@ final class InspectManticore extends Command
$this->line(sprintf('%-20s %s', $table, $status['needs_rebuild'] ? 'REBUILD REQUIRED' : 'compatible'));
}
if (! $report['compatible']) {
$this->warn('Run manticore:create-indexes --drop during a maintenance window, then repopulate all enabled indexes.');
$this->warn('Run manticore:create-indexes --drop --index=releases during a maintenance window for release-only schema changes, then repopulate the affected index.');
}
}
app/Console/Commands/NntmuxOffsetPopulate.php
+29 -4
View File
@@ -8,6 +8,7 @@ use App\Facades\Elasticsearch;
use App\Facades\Search;
use App\Models\Predb;
use App\Models\Release;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use App\Services\Search\Support\ElasticsearchResponseHelper;
use Elastic\Elasticsearch\Client as ElasticsearchClient;
use Exception;
@@ -115,7 +116,9 @@ class NntmuxOffsetPopulate extends Command
$this->monitorProcesses($processes); // @phpstan-ignore argument.type
// Verify final count
$this->verifyIndexPopulation($engine, $index, $total);
if (! $this->verifyIndexPopulation($engine, $index, $total)) {
return Command::FAILURE;
}
$this->info('All parallel processes completed successfully!');
@@ -252,7 +255,7 @@ class NntmuxOffsetPopulate extends Command
/**
* Verify index population
*/
private function verifyIndexPopulation(string $engine, string $index, int $expectedTotal): void
private function verifyIndexPopulation(string $engine, string $index, int $expectedTotal): bool
{
$this->info('Verifying index population...');
@@ -271,12 +274,34 @@ class NntmuxOffsetPopulate extends Command
$this->info('✓ Index population successful!');
} else {
$this->warn('⚠ Index population may be incomplete');
return false;
}
} catch (Exception $e) {
$this->warn("Could not verify index population: {$e->getMessage()}");
return false;
}
return true;
} else {
$this->info('ManticoreSearch index verification not implemented yet');
$indexName = (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt');
try {
$actualCount = app(ManticoreSearchDriver::class)->countIndexDocuments($indexName);
$this->info('Expected: '.number_format($expectedTotal));
$this->info('Actual: '.number_format($actualCount));
if ($actualCount < $expectedTotal) {
$this->warn('Manticore index population is incomplete.');
return false;
}
} catch (Exception $e) {
$this->warn("Could not verify Manticore population: {$e->getMessage()}");
return false;
}
return true;
}
}
@@ -286,7 +311,7 @@ class NntmuxOffsetPopulate extends Command
private function clearIndex(string $engine, string $index): void
{
if ($engine === 'manticore') {
$indexName = $index === 'releases' ? 'releases_rt' : 'predb_rt';
$indexName = (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt');
Search::truncateIndex([$indexName]);
$this->info("Truncated ManticoreSearch index: {$indexName}");
} else {
app/Console/Commands/NntmuxOffsetWorker.php
+26 -46
View File
@@ -7,7 +7,8 @@ namespace App\Console\Commands;
use App\Facades\Elasticsearch;
use App\Facades\Search;
use App\Models\Predb;
use App\Models\Release;
use App\Services\Search\Support\ReleaseIndexProjection;
use App\Support\ReleaseSearchIndexDocument;
use Exception;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
@@ -97,7 +98,7 @@ class NntmuxOffsetWorker extends Command
if ($engine === 'manticore') {
$batchData = [];
$query->chunk($batchSize, function ($items) use ($transformer, &$processed, &$errors, &$batchData, $batchSize, $workerId) {
$query->chunk($batchSize, function ($items) use ($transformer, $index, &$processed, &$errors, &$batchData, $batchSize, $workerId) {
$this->info("Worker {$workerId}: Processing chunk of {$items->count()} items");
foreach ($items as $item) {
@@ -106,7 +107,7 @@ class NntmuxOffsetWorker extends Command
$processed++;
if (count($batchData) >= $batchSize) {
$this->processSearchBatch($batchData, $workerId); // @phpstan-ignore argument.type
$this->processSearchBatch($batchData, $index, $workerId); // @phpstan-ignore argument.type
$this->info("Worker {$workerId}: Inserted batch of ".count($batchData).' records');
$batchData = [];
}
@@ -119,7 +120,7 @@ class NntmuxOffsetWorker extends Command
// Process remaining items
if (! empty($batchData)) {
$this->processSearchBatch($batchData, $workerId); // @phpstan-ignore argument.type
$this->processSearchBatch($batchData, $index, $workerId); // @phpstan-ignore argument.type
$this->info("Worker {$workerId}: Inserted final batch of ".count($batchData).' records');
}
@@ -166,17 +167,8 @@ class NntmuxOffsetWorker extends Command
private function buildOffsetQuery(string $index, int $offset, int $limit): mixed
{
if ($index === 'releases') {
return Release::query()
->select([
'releases.id',
'releases.name',
'releases.searchname',
'releases.fromname',
'releases.categories_id',
'releases.postdate',
])
->selectRaw('(SELECT GROUP_CONCAT(rf.name SEPARATOR " ") FROM release_files rf WHERE rf.releases_id = releases.id) AS filename')
->orderBy('releases.id')
return ReleaseIndexProjection::query()
->orderBy('r.id')
->offset($offset)
->limit($limit);
} else {
@@ -194,34 +186,15 @@ class NntmuxOffsetWorker extends Command
private function getTransformer(string $engine, string $index): callable
{
if ($index === 'releases') {
if ($engine === 'manticore') {
return function ($item) {
return [
'id' => (string) $item->id,
'name' => (string) ($item->name ?: ''),
'searchname' => (string) ($item->searchname ?: ''),
'fromname' => (string) ($item->fromname ?: ''),
'categories_id' => (string) ($item->categories_id ?: '0'),
'filename' => (string) ($item->filename ?: ''),
'dummy' => '1',
];
};
} else {
return function ($item) {
$searchName = str_replace(['.', '-'], ' ', $item->searchname ?? '');
return function ($item) use ($engine): array {
$row = (array) $item;
return [
'id' => $item->id,
'name' => $item->name,
'searchname' => $item->searchname,
'plainsearchname' => $searchName,
'fromname' => $item->fromname,
'categories_id' => $item->categories_id,
'filename' => $item->filename ?? '',
'postdate' => $item->postdate,
];
};
}
return $engine === 'manticore'
? ReleaseSearchIndexDocument::normalize($row)
: array_merge($row, [
'plainsearchname' => str_replace(['.', '-'], ' ', (string) ($item->searchname ?? '')),
]);
};
} else { // predb
return function ($item) {
return [
@@ -229,7 +202,6 @@ class NntmuxOffsetWorker extends Command
'title' => (string) ($item->title ?? ''),
'filename' => (string) ($item->filename ?? ''),
'source' => (string) ($item->source ?? ''),
'dummy' => 1,
];
};
}
@@ -241,7 +213,7 @@ class NntmuxOffsetWorker extends Command
private function getIndexName(string $engine, string $index): string
{
if ($engine === 'manticore') {
return $index === 'releases' ? 'releases_rt' : 'predb_rt';
return (string) config("search.drivers.manticore.indexes.{$index}", $index.'_rt');
} else {
return $index;
}
@@ -252,14 +224,22 @@ class NntmuxOffsetWorker extends Command
*
* @param array<string, mixed> $data
*/
private function processSearchBatch(array $data, int $workerId): void
private function processSearchBatch(array $data, string $index, int $workerId): void
{
$retries = 3;
$attempt = 0;
while ($attempt < $retries) {
try {
Search::bulkInsertReleases($data);
$result = [];
if ($index === 'releases') {
$result = Search::bulkInsertReleases($data);
} else {
$result = Search::bulkInsertPredb($data);
}
if (($result['errors'] ?? 0) > 0) {
throw new Exception('Bulk operation reported '.(int) $result['errors'].' error(s).');
}
break;
} catch (Exception $e) {
$attempt++;
app/Console/Commands/NntmuxPopulateSearchIndexes.php
+56 -57
View File
@@ -16,6 +16,8 @@ use App\Models\Predb;
use App\Models\Release;
use App\Models\SteamApp;
use App\Models\Video;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use App\Services\Search\Support\ReleaseIndexProjection;
use App\Support\ReleaseSearchIndexDocument;
use App\Support\SecondaryIndexDocuments;
use Exception;
@@ -194,7 +196,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreReleases(): int
{
$indexName = 'releases_rt';
$indexName = (string) config('search.drivers.manticore.indexes.releases', 'releases_rt');
Search::truncateIndex([$indexName]);
@@ -205,39 +207,14 @@ class NntmuxPopulateSearchIndexes extends Command
return Command::SUCCESS;
}
// Optimized query: avoid GROUP_CONCAT and complex joins for faster population
// External media IDs can be populated separately if needed
$query = Release::query()
->orderByDesc('releases.id')
->select([
'releases.id',
'releases.name',
'releases.searchname',
'releases.fromname',
'releases.categories_id',
'releases.size',
'releases.postdate',
'releases.adddate',
'releases.totalpart',
'releases.grabs',
'releases.passwordstatus',
'releases.groups_id',
'releases.nzbstatus',
'releases.haspreview',
'releases.videos_id',
'releases.movieinfo_id',
'releases.imdbid',
]);
$query = ReleaseIndexProjection::query()->orderBy('r.id');
return $this->processManticoreData(
$indexName,
$total,
$query,
function ($item) {
return ReleaseSearchIndexDocument::normalize(array_merge(
(array) $item->getAttributes(),
['filename' => '']
));
return ReleaseSearchIndexDocument::normalize((array) $item);
}
);
}
@@ -247,7 +224,7 @@ class NntmuxPopulateSearchIndexes extends Command
*/
private function manticorePredb(): int
{
$indexName = 'predb_rt';
$indexName = (string) config('search.drivers.manticore.indexes.predb', 'predb_rt');
Search::truncateIndex([$indexName]);
@@ -282,7 +259,7 @@ class NntmuxPopulateSearchIndexes extends Command
*/
private function manticoreMovies(): int
{
$indexName = 'movies_rt';
$indexName = (string) config('search.drivers.manticore.indexes.movies', 'movies_rt');
Search::truncateIndex([$indexName]);
@@ -336,7 +313,7 @@ class NntmuxPopulateSearchIndexes extends Command
*/
private function manticoreTvshows(): int
{
$indexName = 'tvshows_rt';
$indexName = (string) config('search.drivers.manticore.indexes.tvshows', 'tvshows_rt');
Search::truncateIndex([$indexName]);
@@ -385,7 +362,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreMusic(): int
{
$indexName = 'music_rt';
$indexName = (string) config('search.drivers.manticore.indexes.music', 'music_rt');
Search::truncateIndex([$indexName]);
$total = MusicInfo::count();
if (! $total) {
@@ -405,7 +382,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreBooks(): int
{
$indexName = 'books_rt';
$indexName = (string) config('search.drivers.manticore.indexes.books', 'books_rt');
Search::truncateIndex([$indexName]);
$total = BookInfo::count();
if (! $total) {
@@ -425,7 +402,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreGames(): int
{
$indexName = 'games_rt';
$indexName = (string) config('search.drivers.manticore.indexes.games', 'games_rt');
Search::truncateIndex([$indexName]);
$total = GamesInfo::count();
if (! $total) {
@@ -445,7 +422,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreConsole(): int
{
$indexName = 'console_rt';
$indexName = (string) config('search.drivers.manticore.indexes.console', 'console_rt');
Search::truncateIndex([$indexName]);
$total = ConsoleInfo::count();
if (! $total) {
@@ -465,7 +442,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreSteam(): int
{
$indexName = 'steam_rt';
$indexName = (string) config('search.drivers.manticore.indexes.steam', 'steam_rt');
Search::truncateIndex([$indexName]);
$total = SteamApp::count();
if (! $total) {
@@ -485,7 +462,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function manticoreAnime(): int
{
$indexName = 'anime_rt';
$indexName = (string) config('search.drivers.manticore.indexes.anime', 'anime_rt');
Search::truncateIndex([$indexName]);
$total = (int) DB::table('anidb_titles')->count();
if (! $total) {
@@ -555,7 +532,7 @@ class NntmuxPopulateSearchIndexes extends Command
$batchData = [];
try {
$query->chunk($chunkSize, function ($items) use ($indexName, $transformer, $bar, &$processedCount, &$errorCount, $batchSize, &$batchData) {
$processItems = function ($items) use ($indexName, $transformer, $bar, &$processedCount, &$errorCount, $batchSize, &$batchData): void {
foreach ($items as $item) {
try {
$batchData[] = $transformer($item);
@@ -574,7 +551,13 @@ class NntmuxPopulateSearchIndexes extends Command
}
$bar->advance();
}
});
};
if ($indexName === (string) config('search.drivers.manticore.indexes.releases', 'releases_rt')) {
$query->chunkById($chunkSize, $processItems, 'r.id', 'id');
} else {
$query->chunk($chunkSize, $processItems);
}
// Process remaining items
if (! empty($batchData)) {
@@ -586,8 +569,20 @@ class NntmuxPopulateSearchIndexes extends Command
if ($errorCount > 0) {
$this->warn("Completed with {$errorCount} errors out of {$processedCount} processed items.");
} else {
$this->info('Search index population completed successfully!');
return Command::FAILURE;
}
try {
$actualCount = app(ManticoreSearchDriver::class)->countIndexDocuments($indexName);
if ($actualCount < $total) {
$this->error(sprintf('Manticore verification failed for %s: expected %d, found %d.', $indexName, $total, $actualCount));
return Command::FAILURE;
}
$this->info(sprintf('Search index population completed successfully (%d documents verified).', $actualCount));
} catch (Exception $e) {
$this->warn('Manticore population completed, but verification failed: '.$e->getMessage());
}
return Command::SUCCESS;
@@ -1143,25 +1138,29 @@ class NntmuxPopulateSearchIndexes extends Command
while ($attempt < $retries) {
try {
// Use the correct bulk insert method based on index name
if ($indexName === 'releases_rt') {
Search::bulkInsertReleases($data);
} elseif ($indexName === 'predb_rt') {
Search::bulkInsertPredb($data);
} elseif ($indexName === 'music_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Music, $data);
} elseif ($indexName === 'books_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Books, $data);
} elseif ($indexName === 'games_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Games, $data);
} elseif ($indexName === 'console_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Console, $data);
} elseif ($indexName === 'steam_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Steam, $data);
} elseif ($indexName === 'anime_rt') {
Search::bulkInsertSecondary(SecondarySearchIndex::Anime, $data);
$result = [];
if ($indexName === (string) config('search.drivers.manticore.indexes.releases', 'releases_rt')) {
$result = Search::bulkInsertReleases($data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.predb', 'predb_rt')) {
$result = Search::bulkInsertPredb($data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.music', 'music_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Music, $data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.books', 'books_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Books, $data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.games', 'games_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Games, $data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.console', 'console_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Console, $data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.steam', 'steam_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Steam, $data);
} elseif ($indexName === (string) config('search.drivers.manticore.indexes.anime', 'anime_rt')) {
$result = Search::bulkInsertSecondary(SecondarySearchIndex::Anime, $data);
} else {
throw new Exception("Unknown index: {$indexName}");
}
if (($result['errors'] ?? 0) > 0) {
throw new Exception('Bulk operation reported '.(int) $result['errors'].' error(s).');
}
break;
} catch (Exception $e) {
$attempt++;
app/Console/Commands/NntmuxSearchReconcile.php
+52 -5
View File
@@ -16,8 +16,10 @@ class NntmuxSearchReconcile extends Command
{
protected $signature = 'nntmux:search-reconcile
{--since=24h : Only releases with adddate after this window (e.g. 24h, 7d)}
{--all : Scan every release row instead of applying --since}
{--chunk=1000 : MySQL chunk size when scanning releases}
{--in-batch=500 : Max ids per Manticore IN() query}
{--orphans : Also remove indexed ids that no longer exist in MySQL}
{--reindex : Call Search::updateRelease for each missing id}
{--dry-run : Report only; do not write to the index}';
@@ -39,10 +41,12 @@ class NntmuxSearchReconcile extends Command
$since = (string) $this->option('since');
$cutoff = $this->parseSinceCutoff($since);
$scanAll = (bool) $this->option('all');
$chunk = max(50, min(5000, (int) $this->option('chunk')));
$inBatch = max(50, min(1000, (int) $this->option('in-batch')));
$dryRun = (bool) $this->option('dry-run');
$reindex = (bool) $this->option('reindex');
$checkOrphans = (bool) $this->option('orphans');
if ($reindex && $dryRun) {
$this->warn('--reindex with --dry-run: no writes will be performed.');
@@ -56,8 +60,11 @@ class NntmuxSearchReconcile extends Command
$bar = $this->output->createProgressBar();
$bar->start();
Release::query()
->where('adddate', '>=', $cutoff)
$releaseQuery = Release::query();
if (! $scanAll) {
$releaseQuery->where('adddate', '>=', $cutoff);
}
$releaseQuery
->orderBy('id')
->chunkById($chunk, function ($releases) use ($manticore, $index, $inBatch, &$missingAll, &$scanned, $bar): void {
$ids = $releases->pluck('id')->map(static fn ($id): int => (int) $id)->all();
@@ -75,7 +82,8 @@ class NntmuxSearchReconcile extends Command
$this->newLine(2);
$missingTotal = \count($missingAll);
$this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} (adddate >= {$cutoff->toDateTimeString()})");
$scope = $scanAll ? 'all release rows' : "adddate >= {$cutoff->toDateTimeString()}";
$this->info("Scanned {$scanned} release row(s); missing in Manticore index: {$missingTotal} ({$scope})");
if ($missingTotal > 0) {
$sample = \array_slice($missingAll, 0, 20);
$this->line('Sample ids: '.implode(', ', $sample));
@@ -89,12 +97,18 @@ class NntmuxSearchReconcile extends Command
$this->probeAndWarnAllMissing($manticore, $index, \array_slice($missingAll, 0, 5));
}
if ($missingTotal === 0) {
$orphans = $checkOrphans ? $this->findOrphanedIds($manticore, $chunk) : [];
if ($orphans !== []) {
$this->info('Indexed documents with no MySQL release row: '.count($orphans));
$this->line('Orphan sample ids: '.implode(', ', array_slice($orphans, 0, 20)));
}
if ($missingTotal === 0 && $orphans === []) {
return self::SUCCESS;
}
if (! $reindex) {
$this->comment('Run with --reindex to push Search::updateRelease() for each missing id.');
$this->comment('Run with --reindex to repair missing documents and remove orphaned ids.');
return self::SUCCESS;
}
@@ -110,11 +124,44 @@ class NntmuxSearchReconcile extends Command
}
}
if ($orphans !== []) {
try {
Search::deleteReleases($orphans);
$this->info('Removed '.count($orphans).' orphaned document(s).');
} catch (Throwable $e) {
$this->error('Failed removing orphaned documents: '.$e->getMessage());
}
}
$this->info("Reindexed {$done} release(s).");
return self::SUCCESS;
}
/**
* @return list<int>
*/
private function findOrphanedIds(ManticoreSearchDriver $manticore, int $pageSize): array
{
$orphans = [];
$offset = 0;
do {
$indexed = $manticore->indexedReleaseIds($pageSize, $offset);
if ($indexed === []) {
break;
}
$existing = Release::query()->whereIn('id', $indexed)->pluck('id')
->map(static fn ($id): int => (int) $id)
->all();
$orphans = [...$orphans, ...array_values(array_diff($indexed, $existing))];
$offset += count($indexed);
} while (count($indexed) === $pageSize);
return $orphans;
}
private function parseSinceCutoff(string $since): Carbon
{
$since = trim($since);
app/Console/Commands/NntmuxSearchRepair.php
+56
View File
@@ -0,0 +1,56 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Facades\Search;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
final class NntmuxSearchRepair extends Command
{
protected $signature = 'nntmux:search-repair
{--limit=100 : Maximum failed releases to retry}
{--dry-run : Report due failures without writing to Manticore}';
protected $description = 'Retry failed release search-index updates';
public function handle(): int
{
$limit = max(1, min(5000, (int) $this->option('limit')));
$query = DB::table('search_index_failures')
->whereNull('resolved_at')
->where(function ($query): void {
$query->whereNull('next_attempt_at')->orWhere('next_attempt_at', '<=', now());
})
->orderBy('id')
->limit($limit);
$rows = $query->get(['release_id', 'operation']);
if ($rows->isEmpty()) {
$this->info('No failed release index updates are due for repair.');
return self::SUCCESS;
}
foreach ($rows as $row) {
$releaseId = (int) $row->release_id;
if ((bool) $this->option('dry-run')) {
$this->line((string) $releaseId);
continue;
}
if ($row->operation === 'delete') {
Search::deleteRelease($releaseId);
} else {
Search::updateRelease($releaseId);
}
}
$this->info(sprintf('%s release index failure(s) %s.', $rows->count(), $this->option('dry-run') ? 'reported' : 'processed'));
return self::SUCCESS;
}
}
app/Facades/Search.php
+1 -1
View File
@@ -17,7 +17,7 @@ use Illuminate\Support\Facades\Facade;
* @method static void insertRelease(array $parameters)
* @method static void updateRelease(int|string $releaseID)
* @method static void deleteRelease(int $id)
* @method static void deleteReleases(iterable $ids)
* @method static void deleteReleases(iterable<int|string> $ids)
* @method static void insertPredb(array $parameters)
* @method static void updatePreDb(array $parameters)
* @method static void deletePreDb(int $id)
app/Models/Release.php
+16 -5
View File
@@ -216,7 +216,7 @@ class Release extends Model
]
);
Search::updateRelease($parameters['id']);
self::syncSearchIndexAfterCommit((int) $parameters['id']);
return $parameters['id'];
}
@@ -249,7 +249,7 @@ class Release extends Model
]
);
Search::updateRelease($id);
self::syncSearchIndexAfterCommit((int) $id);
}
/**
@@ -262,7 +262,7 @@ class Release extends Model
$id = self::whereGuid($guid)->value('id');
self::whereGuid($guid)->increment('grabs');
if ($id !== null) {
Search::updateRelease((int) $id);
self::syncSearchIndexAfterCommit((int) $id);
}
}
}
@@ -284,7 +284,7 @@ class Release extends Model
$ids = self::query()->whereIn('guid', $guids)->pluck('id');
self::query()->whereIn('guid', $guids)->increment('grabs');
foreach ($ids as $id) {
Search::updateRelease((int) $id);
self::syncSearchIndexAfterCommit((int) $id);
}
}
}
@@ -302,7 +302,7 @@ class Release extends Model
$ids = self::whereVideosId($videoId)->pluck('id');
$updated = self::whereVideosId($videoId)->update(['videos_id' => 0, 'tv_episodes_id' => 0]);
foreach ($ids as $id) {
Search::updateRelease((int) $id);
self::syncSearchIndexAfterCommit((int) $id);
}
return $updated;
@@ -313,6 +313,17 @@ class Release extends Model
return self::whereAnidbid($anidbID)->update(['anidbid' => -1]);
}
private static function syncSearchIndexAfterCommit(int $releaseId): void
{
if ($releaseId <= 0) {
return;
}
DB::afterCommit(function () use ($releaseId): void {
Search::updateRelease($releaseId);
});
}
public static function getTopDownloads(): mixed
{
$expiresAt = now()->addMinutes(config('nntmux.cache_expiry_long'));
app/Models/ReleaseFile.php
+4 -1
View File
@@ -8,6 +8,7 @@ use App\Facades\Search;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
/**
@@ -146,7 +147,9 @@ class ReleaseFile extends Model
if (\strlen($hash) === 32) {
ParHash::insertOrIgnore(['releases_id' => $id, 'hash' => $hash]);
}
Search::updateRelease($id);
DB::afterCommit(function () use ($id): void {
Search::updateRelease($id);
});
}
return $insert ?? 0;
app/Observers/CategoryObserver.php
+28
View File
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
namespace App\Observers;
use App\Models\Category;
use App\Support\ReleaseSearchIndexSync;
use Illuminate\Support\Facades\DB;
final class CategoryObserver
{
public function updated(Category $category): void
{
if (! $category->isDirty(['title', 'parentid', 'root_categories_id'])) {
return;
}
DB::afterCommit(fn (): bool => $this->reindex($category));
}
private function reindex(Category $category): bool
{
ReleaseSearchIndexSync::forCategory((int) $category->id);
return true;
}
}
app/Observers/MovieInfoObserver.php
+11
View File
@@ -6,6 +6,8 @@ namespace App\Observers;
use App\Facades\Search;
use App\Models\MovieInfo;
use App\Support\ReleaseSearchIndexSync;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class MovieInfoObserver
@@ -24,6 +26,7 @@ class MovieInfoObserver
public function updated(MovieInfo $movie): void
{
$this->syncToSearchIndex($movie);
DB::afterCommit(fn (): bool => $this->syncReleases($movie));
}
/**
@@ -39,6 +42,7 @@ class MovieInfoObserver
'error' => $e->getMessage(),
]);
}
DB::afterCommit(fn (): bool => $this->syncReleases($movie));
}
/**
@@ -67,4 +71,11 @@ class MovieInfoObserver
]);
}
}
private function syncReleases(MovieInfo $movie): bool
{
ReleaseSearchIndexSync::forMovieInfo((int) $movie->id);
return true;
}
}
app/Observers/ReleaseObserver.php
+29 -11
View File
@@ -8,6 +8,7 @@ use App\Facades\Search;
use App\Models\Release;
use App\Services\Nzb\NzbService;
use App\Services\ReleaseImageService;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
/**
@@ -25,7 +26,7 @@ class ReleaseObserver
*/
public function created(Release $release): void
{
$this->syncToSearchIndex($release);
$this->syncAfterCommit($release);
}
/**
@@ -38,19 +39,25 @@ class ReleaseObserver
{
$indexedFields = [
'name',
'guid',
'searchname',
'fromname',
'categories_id',
'imdbid',
'anidbid',
'movieinfo_id',
'videos_id',
'tv_episodes_id',
'size',
'totalpart',
'grabs',
'passwordstatus',
'groups_id',
'nzbstatus',
'nfostatus',
'haspreview',
'jpgstatus',
'comments',
'postdate',
'adddate',
];
@@ -64,7 +71,7 @@ class ReleaseObserver
}
if ($changed) {
$this->syncToSearchIndex($release);
$this->syncAfterCommit($release);
}
}
@@ -106,20 +113,27 @@ class ReleaseObserver
*/
public function deleted(Release $release): void
{
try {
Search::deleteRelease($release->id);
} catch (\Throwable $e) {
Log::error('ReleaseObserver: Failed to delete release from search index', [
'release_id' => $release->id,
'error' => $e->getMessage(),
]);
}
DB::afterCommit(function () use ($release): void {
try {
Search::deleteRelease($release->id);
} catch (\Throwable $e) {
Log::error('ReleaseObserver: Failed to delete release from search index', [
'release_id' => $release->id,
'error' => $e->getMessage(),
]);
}
});
}
private function syncAfterCommit(Release $release): void
{
DB::afterCommit(fn (): bool => $this->syncToSearchIndex($release));
}
/**
* Sync the release to the search index (full document from DB + joins).
*/
private function syncToSearchIndex(Release $release): void
private function syncToSearchIndex(Release $release): bool
{
try {
Search::updateRelease($release->id);
@@ -129,11 +143,15 @@ class ReleaseObserver
'release_id' => $release->id,
]);
}
return true;
} catch (\Throwable $e) {
Log::error('ReleaseObserver: Failed to sync release to search index', [
'release_id' => $release->id,
'error' => $e->getMessage(),
]);
return false;
}
}
}
app/Observers/RootCategoryObserver.php
+28
View File
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
namespace App\Observers;
use App\Models\RootCategory;
use App\Support\ReleaseSearchIndexSync;
use Illuminate\Support\Facades\DB;
final class RootCategoryObserver
{
public function updated(RootCategory $category): void
{
if (! $category->isDirty('title')) {
return;
}
DB::afterCommit(fn (): bool => $this->reindex($category));
}
private function reindex(RootCategory $category): bool
{
ReleaseSearchIndexSync::forRootCategory((int) $category->id);
return true;
}
}
app/Observers/UsenetGroupObserver.php
+28
View File
@@ -0,0 +1,28 @@
<?php
declare(strict_types=1);
namespace App\Observers;
use App\Models\UsenetGroup;
use App\Support\ReleaseSearchIndexSync;
use Illuminate\Support\Facades\DB;
final class UsenetGroupObserver
{
public function updated(UsenetGroup $group): void
{
if (! $group->isDirty('name')) {
return;
}
DB::afterCommit(fn (): bool => $this->reindex($group));
}
private function reindex(UsenetGroup $group): bool
{
ReleaseSearchIndexSync::forQueryGroup((int) $group->id);
return true;
}
}
app/Observers/VideoObserver.php
+11
View File
@@ -6,6 +6,8 @@ namespace App\Observers;
use App\Facades\Search;
use App\Models\Video;
use App\Support\ReleaseSearchIndexSync;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class VideoObserver
@@ -24,6 +26,7 @@ class VideoObserver
public function updated(Video $video): void
{
$this->syncToSearchIndex($video);
DB::afterCommit(fn (): bool => $this->syncReleases($video));
}
/**
@@ -39,6 +42,7 @@ class VideoObserver
'error' => $e->getMessage(),
]);
}
DB::afterCommit(fn (): bool => $this->syncReleases($video));
}
/**
@@ -66,4 +70,11 @@ class VideoObserver
]);
}
}
private function syncReleases(Video $video): bool
{
ReleaseSearchIndexSync::forVideo((int) $video->id);
return true;
}
}
app/Providers/AppServiceProvider.php
+9
View File
@@ -9,25 +9,31 @@ use App\Listeners\RecategorizeReleaseAfterNameFix;
use App\Models\AnidbInfo;
use App\Models\AnidbTitle;
use App\Models\BookInfo;
use App\Models\Category;
use App\Models\ConsoleInfo;
use App\Models\GamesInfo;
use App\Models\MovieInfo;
use App\Models\MusicInfo;
use App\Models\Release;
use App\Models\RolePromotion;
use App\Models\RootCategory;
use App\Models\SteamApp;
use App\Models\UsenetGroup;
use App\Models\User;
use App\Models\Video;
use App\Observers\AnidbInfoObserver;
use App\Observers\AnidbTitleObserver;
use App\Observers\BookInfoObserver;
use App\Observers\CategoryObserver;
use App\Observers\ConsoleInfoObserver;
use App\Observers\GamesInfoObserver;
use App\Observers\MovieInfoObserver;
use App\Observers\MusicInfoObserver;
use App\Observers\ReleaseObserver;
use App\Observers\RolePromotionObserver;
use App\Observers\RootCategoryObserver;
use App\Observers\SteamAppObserver;
use App\Observers\UsenetGroupObserver;
use App\Observers\VideoObserver;
use App\View\Composers\AdminDataComposer;
use App\View\Composers\GlobalDataComposer;
@@ -74,6 +80,9 @@ class AppServiceProvider extends ServiceProvider
MovieInfo::observe(MovieInfoObserver::class);
Video::observe(VideoObserver::class);
Release::observe(ReleaseObserver::class);
Category::observe(CategoryObserver::class);
RootCategory::observe(RootCategoryObserver::class);
UsenetGroup::observe(UsenetGroupObserver::class);
MusicInfo::observe(MusicInfoObserver::class);
BookInfo::observe(BookInfoObserver::class);
GamesInfo::observe(GamesInfoObserver::class);
app/Services/ConsoleService.php
+1 -1
View File
@@ -632,7 +632,7 @@ class ConsoleService
}
$result['release'] = $releaseName;
array_map('trim', $result);
$result = array_map('trim', $result);
return (isset($result['title'], $result['platform']) && ! empty($result['title'])) ? $result : false;
}
app/Services/Releases/ReleaseBrowseService.php
+20
View File
@@ -9,6 +9,7 @@ use App\Models\Category;
use App\Models\Release;
use App\Models\Settings;
use App\Models\UsenetGroup;
use App\Support\ReleaseSearchIndexDocument;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
@@ -375,6 +376,7 @@ class ReleaseBrowseService
'sort_field' => $indexSortField,
'sort_dir' => $orderBy[1] ?? 'desc',
'try_fuzzy' => false,
'include_documents' => true,
];
$filtered = Search::searchReleasesFiltered($criteria, (int) $num, (int) $start);
@@ -382,6 +384,24 @@ class ReleaseBrowseService
return [];
}
if (! empty($filtered['documents'])) {
$documents = array_map(static function (array $document): object {
$row = ReleaseSearchIndexDocument::toReleaseRow($document);
$row['title'] = $row['episode_title'];
$row['firstaired'] = $row['firstaired_ts'] > 0
? date('Y-m-d H:i:s', $row['firstaired_ts'])
: null;
return (object) $row;
}, $filtered['documents']);
if ($documents !== []) {
$documents[0]->_totalcount = $documents[0]->_totalrows = (int) ($filtered['total'] ?? count($documents));
}
return $documents;
}
$ids = array_map(static fn (int|string $id): int => (int) $id, $filtered['ids']);
$idList = implode(',', $ids);
$fieldOrder = implode(',', $ids);
app/Services/Releases/ReleaseSearchService.php
+21
View File
@@ -13,6 +13,7 @@ use App\Models\Settings;
use App\Models\UsenetGroup;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchCursor;
use App\Support\ReleaseSearchIndexDocument;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Cache;
@@ -259,11 +260,31 @@ class ReleaseSearchService
'sort_field' => $this->browseOrderToIndexSortField($orderField),
'sort_dir' => $orderDir,
'try_fuzzy' => true,
'include_documents' => true,
];
$criteria['track_total'] = $cursor === null;
$searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset, $cursor));
$filtered = $searchPage->legacy();
if ($searchPage->documents !== []) {
$releases = collect(array_map(function (array $document): object {
$row = ReleaseSearchIndexDocument::toReleaseRow($document);
$row['title'] = $row['episode_title'];
$row['firstaired'] = $row['firstaired_ts'] > 0
? date('Y-m-d H:i:s', $row['firstaired_ts'])
: null;
return (object) $row;
}, $searchPage->documents));
if ($releases->isNotEmpty()) {
$releases[0]->_totalrows = (int) $filtered['total'];
$releases[0]->_search_last_sort = $searchPage->lastSortValues;
$releases[0]->_search_has_more = $searchPage->hasMore;
}
return $releases;
}
if ($cursor !== null && $filtered['total'] === 0) {
$filtered['total'] = $cursor->total;
}
app/Services/Search/DTO/ReleaseSearchQuery.php
+3
View File
@@ -32,6 +32,7 @@ final readonly class ReleaseSearchQuery
public int $offset = 0,
public ?SearchCursor $cursor = null,
public bool $trackTotal = true,
public bool $includeDocuments = false,
) {}
/** @param array<string, mixed> $criteria */
@@ -57,6 +58,7 @@ final readonly class ReleaseSearchQuery
offset: max(0, $offset),
cursor: $cursor,
trackTotal: (bool) ($criteria['track_total'] ?? true),
includeDocuments: (bool) ($criteria['include_documents'] ?? false),
);
}
@@ -81,6 +83,7 @@ final readonly class ReleaseSearchQuery
'try_fuzzy' => $this->tryFuzzy,
'cursor_sort' => $this->cursor?->sortValues,
'track_total' => $this->trackTotal,
'include_documents' => $this->includeDocuments,
];
}
app/Services/Search/DTO/SearchPage.php
+2
View File
@@ -9,6 +9,7 @@ final readonly class SearchPage
/**
* @param list<int> $ids
* @param list<int|float|string> $lastSortValues
* @param list<array<string, mixed>> $documents
*/
public function __construct(
public array $ids,
@@ -19,6 +20,7 @@ final readonly class SearchPage
public float $durationMs = 0.0,
public array $lastSortValues = [],
public bool $hasMore = false,
public array $documents = [],
) {}
/** @return array{ids: list<int>, total: int, fuzzy: bool} */
app/Services/Search/Drivers/ElasticSearchDriver.php
+1
View File
@@ -109,6 +109,7 @@ class ElasticSearchDriver implements SearchDriverInterface
durationMs: (hrtime(true) - $startedAt) / 1_000_000,
lastSortValues: $result['last_sort'] ?? [],
hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])),
documents: $result['documents'] ?? [],
);
}
app/Services/Search/Drivers/ManticoreSearchDriver.php
+222 -75
View File
@@ -19,6 +19,7 @@ use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use App\Services\Search\Support\ManticoreClientFactory;
use App\Services\Search\Support\ManticoreIndexRegistry;
use App\Services\Search\Support\ReleaseIndexProjection;
use App\Support\ReleaseSearchIndexDocument;
use App\Support\SecondaryIndexDocuments;
use Illuminate\Support\Facades\Cache;
@@ -121,9 +122,87 @@ class ManticoreSearchDriver implements SearchDriverInterface
durationMs: (hrtime(true) - $startedAt) / 1_000_000,
lastSortValues: $result['last_sort'] ?? [],
hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])),
documents: $result['documents'] ?? [],
);
}
/**
* Return the current document count for an RT table.
*/
public function countIndexDocuments(string $index): int
{
if (preg_match('/^[a-zA-Z0-9_]+$/', $index) !== 1) {
throw new \InvalidArgumentException('Unsafe Manticore index name.');
}
$response = $this->manticoreSearch->sql("SELECT COUNT(*) AS total FROM `{$index}`", true);
if (! is_array($response)) {
return 0;
}
foreach ($response as $key => $value) {
if (is_array($value) && isset($value['total'])) {
return (int) $value['total'];
}
if ($key === 'total' && is_numeric($value)) {
return (int) $value;
}
if (is_numeric($value)) {
return (int) $value;
}
}
return 0;
}
/**
* Return a stable page of release document ids for drift/orphan checks.
*
* @return list<int>
*/
public function indexedReleaseIds(int $limit = 1000, int $offset = 0): array
{
$limit = max(1, min(10000, $limit));
$offset = max(0, $offset);
$index = $this->getReleasesIndex();
if (preg_match('/^[a-zA-Z0-9_]+$/', $index) !== 1) {
throw new \InvalidArgumentException('Unsafe Manticore index name.');
}
$sql = "SELECT id FROM {$index} ORDER BY id ASC LIMIT {$offset},{$limit} OPTION max_matches=".($offset + $limit);
$response = $this->manticoreSearch->sql($sql, true);
if (! is_array($response)) {
return [];
}
$ids = [];
foreach ($response as $key => $value) {
if (ctype_digit((string) $key)) {
$ids[(int) $key] = true;
}
if (is_array($value) && isset($value['id'])) {
$ids[(int) $value['id']] = true;
}
if (
($key === 'id' || is_int($key) || ctype_digit((string) $key))
&& (is_int($value) || (is_string($value) && ctype_digit($value)))
) {
$ids[(int) $value] = true;
}
}
if (isset($response['data']) && is_array($response['data'])) {
foreach ($response['data'] as $row) {
if (is_array($row) && isset($row['id'])) {
$ids[(int) $row['id']] = true;
}
}
}
return array_values(array_filter(array_map('intval', array_keys($ids)), static fn (int $id): bool => $id > 0));
}
/**
* Check if autocomplete is enabled.
*/
@@ -209,7 +288,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
$releaseId = (int) $parameters['id'];
if (! $this->replaceReleaseDocumentWithRetry($parameters)) {
$this->recordReleaseIndexFailure($releaseId, 'insertRelease');
$this->recordReleaseIndexFailure($releaseId, 'insertRelease', 'upsert');
try {
ReindexReleaseJob::dispatch($releaseId)->delay(now()->addSeconds(2));
} catch (\Throwable $e) {
@@ -226,24 +305,34 @@ class ManticoreSearchDriver implements SearchDriverInterface
*/
private function replaceReleaseDocumentWithRetry(array $parameters): bool
{
$document = ReleaseSearchIndexDocument::normalize($parameters);
// updateRelease() supplies the already-normalized canonical projection.
// Preserve its *_ts fields instead of normalizing a second time and
// replacing post/add timestamps with zero.
$document = ReleaseSearchIndexDocument::normalizeForBulk($parameters);
unset($document['id']);
$indexName = $this->config['indexes']['releases'];
$releaseId = (int) $parameters['id'];
for ($attempt = 0; $attempt < 2; $attempt++) {
$attempts = max(1, (int) ($this->config['retry_attempts'] ?? config('search.drivers.manticore.retry_attempts', 3)));
$delayMs = max(0, (int) ($this->config['retry_delay_ms'] ?? config('search.drivers.manticore.retry_delay_ms', 100)));
for ($attempt = 0; $attempt < $attempts; $attempt++) {
try {
$this->manticoreSearch->table($indexName)
->replaceDocument($document, $releaseId);
$this->resolveReleaseIndexFailure($releaseId);
return true;
} catch (\Throwable $e) {
$isResponse = $e instanceof ResponseException;
$isManticoreRuntime = $e instanceof RuntimeException;
if (($isResponse || $isManticoreRuntime) && $attempt === 0) {
usleep(100_000);
if (($isResponse || $isManticoreRuntime) && $attempt < $attempts - 1) {
if ($delayMs > 0) {
usleep($delayMs * 1000 * ($attempt + 1));
}
continue;
}
@@ -277,9 +366,33 @@ class ManticoreSearchDriver implements SearchDriverInterface
return false;
}
private function recordReleaseIndexFailure(int $releaseId, string $phase): void
private function recordReleaseIndexFailure(int $releaseId, string $phase, string $operation = 'upsert'): void
{
Cache::increment('search:index:failures:releases');
try {
$existing = DB::table('search_index_failures')
->where('release_id', $releaseId)
->first(['attempts']);
$attempts = ((int) ($existing->attempts ?? 0)) + 1;
DB::table('search_index_failures')->updateOrInsert(
['release_id' => $releaseId],
[
'operation' => $operation,
'attempts' => $attempts,
'last_error' => $phase,
'next_attempt_at' => now()->addSeconds(min(3600, 2 ** min($attempts, 10))),
'resolved_at' => null,
'updated_at' => now(),
'created_at' => now(),
]
);
} catch (\Throwable $e) {
Log::error('ManticoreSearch: unable to persist release index failure', [
'release_id' => $releaseId,
'phase' => $phase,
'error' => $e->getMessage(),
]);
}
if (Cache::add('search:index:release_index_warn_lock', true, 60)) {
Log::warning('ManticoreSearch: release indexing failure', [
'release_id_sample' => $releaseId,
@@ -289,6 +402,28 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
}
private function resolveReleaseIndexFailure(int $releaseId): void
{
if ($releaseId <= 0) {
return;
}
try {
DB::table('search_index_failures')
->where('release_id', $releaseId)
->update([
'resolved_at' => now(),
'next_attempt_at' => null,
'updated_at' => now(),
]);
} catch (\Throwable $e) {
Log::debug('ManticoreSearch: unable to mark release index failure resolved', [
'release_id' => $releaseId,
'error' => $e->getMessage(),
]);
}
}
private function recordReleaseNotFoundForIndex(int|string $releaseID): void
{
Cache::increment('search:index:failures:release_not_found');
@@ -361,13 +496,34 @@ class ManticoreSearchDriver implements SearchDriverInterface
return;
}
try {
$this->manticoreSearch->table($this->getReleasesIndex())
->deleteDocumentsByIds($ids);
} catch (ResponseException $e) {
Log::error('ManticoreSearch deleteReleases error: '.$e->getMessage(), [
'ids' => $ids,
]);
$attempts = max(1, (int) ($this->config['retry_attempts'] ?? config('search.drivers.manticore.retry_attempts', 2)));
$delayMs = max(0, (int) ($this->config['retry_delay_ms'] ?? config('search.drivers.manticore.retry_delay_ms', 100)));
for ($attempt = 0; $attempt < $attempts; $attempt++) {
try {
$this->manticoreSearch->table($this->getReleasesIndex())
->deleteDocumentsByIds($ids);
foreach ($ids as $id) {
$this->resolveReleaseIndexFailure((int) $id);
}
return;
} catch (\Throwable $e) {
if ($attempt < $attempts - 1) {
if ($delayMs > 0) {
usleep($delayMs * 1000 * ($attempt + 1));
}
continue;
}
Log::error('ManticoreSearch deleteReleases error: '.$e->getMessage(), [
'ids' => $ids,
]);
foreach ($ids as $id) {
$this->recordReleaseIndexFailure((int) $id, 'deleteReleases: '.$e->getMessage(), 'delete');
}
}
}
}
@@ -665,6 +821,42 @@ class ManticoreSearchDriver implements SearchDriverInterface
return $fieldSelector.' '.$scopedQuery;
}
/**
* Build a release-name query that also handles punctuation used as token
* separators in Usenet subjects (for example, WEB-DL.x265).
*
* The prepared query deliberately escapes mid-word hyphens so user search
* operators remain safe. Manticore tokenizes those separators instead of
* indexing them as part of a word, though, so an ordinary release-name
* query needs a normalized-token alternative as well.
*/
private static function scopeReleaseSearchQuery(string $rawQuery, string $preparedQuery, string $fieldSelector = ''): string
{
$primary = self::scopePreparedQueryToField($preparedQuery, $fieldSelector);
if ($fieldSelector !== '@searchname' || self::queryHasNegation($rawQuery)) {
return $primary;
}
// Preserve advanced query syntax and only add the fallback for plain
// release-name input whose punctuation is a token separator.
if (preg_match('/[^\p{L}\p{N}\s._-]/u', $rawQuery) === 1 || ! preg_match('/[._-]/', $rawQuery)) {
return $primary;
}
$normalized = preg_replace('/[._-]+/u', ' ', trim($rawQuery));
$normalized = is_string($normalized) ? trim(preg_replace('/\s+/', ' ', $normalized) ?? '') : '';
$normalizedPrepared = self::prepareUserSearchQuery($normalized);
if ($normalizedPrepared === '' || $normalizedPrepared === $preparedQuery) {
return $primary;
}
$normalizedQuery = self::scopePreparedQueryToField($normalizedPrepared, $fieldSelector);
return '('.$primary.' | '.$normalizedQuery.')';
}
/**
* Check if a search query contains negation operators (! or - prefix on words).
*
@@ -706,64 +898,10 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
try {
$release = Release::query()
->where('releases.id', $releaseID)
->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id')
->leftJoin('movieinfo as mi', 'releases.movieinfo_id', '=', 'mi.id')
->leftJoin('videos as v', 'releases.videos_id', '=', 'v.id')
->select([
'releases.id',
'releases.name',
'releases.searchname',
'releases.fromname',
'releases.categories_id',
'releases.size',
'releases.postdate',
'releases.adddate',
'releases.totalpart',
'releases.grabs',
'releases.passwordstatus',
'releases.groups_id',
'releases.nzbstatus',
'releases.haspreview',
'releases.imdbid',
'releases.videos_id',
'releases.movieinfo_id',
DB::raw('IFNULL(mi.tmdbid, 0) AS tmdbid'),
DB::raw('IFNULL(mi.traktid, 0) AS traktid'),
DB::raw('IFNULL(v.tvdb, 0) AS tvdb'),
DB::raw('IFNULL(v.tvmaze, 0) AS tvmaze'),
DB::raw('IFNULL(v.tvrage, 0) AS tvrage'),
DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename'),
])
->groupBy([
'releases.id',
'releases.name',
'releases.searchname',
'releases.fromname',
'releases.categories_id',
'releases.size',
'releases.postdate',
'releases.adddate',
'releases.totalpart',
'releases.grabs',
'releases.passwordstatus',
'releases.groups_id',
'releases.nzbstatus',
'releases.haspreview',
'releases.imdbid',
'releases.videos_id',
'releases.movieinfo_id',
'mi.tmdbid',
'mi.traktid',
'v.tvdb',
'v.tvmaze',
'v.tvrage',
])
->first();
$release = ReleaseIndexProjection::forId((int) $releaseID);
if ($release !== null) {
$this->insertRelease($release->toArray());
$this->insertRelease($release);
} else {
Log::warning('ManticoreSearch: Release not found for update, removing from index', ['id' => $releaseID]);
$this->recordReleaseNotFoundForIndex($releaseID);
@@ -1260,9 +1398,10 @@ class ManticoreSearchDriver implements SearchDriverInterface
$terms = [];
foreach ($searchArray as $key => $value) {
if (! empty($value)) {
$preparedValue = self::prepareUserSearchQuery($value);
$rawValue = (string) $value;
$preparedValue = self::prepareUserSearchQuery($rawValue);
if (! empty($preparedValue)) {
$terms[] = '@@relaxed '.self::scopePreparedQueryToField($preparedValue, '@'.$key);
$terms[] = '@@relaxed '.self::scopeReleaseSearchQuery($rawValue, $preparedValue, '@'.$key);
}
}
}
@@ -1294,7 +1433,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
}
$searchExpr = '@@relaxed '.self::scopePreparedQueryToField($preparedSearch, $searchColumns);
$searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery($searchString, $preparedSearch, $searchColumns);
} else {
return [];
}
@@ -2401,7 +2540,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
return $this->searchReleasesByCategory($categoryIds, $limit);
}
$searchExpr = '@@relaxed '.self::scopePreparedQueryToField($preparedSearch, '@searchname');
$searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery($searchTerm, $preparedSearch, '@searchname');
$query = (new Search($this->manticoreSearch))
->setTable($this->getReleasesIndex())
@@ -2486,7 +2625,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$prepared = self::prepareUserSearchQuery((string) $value);
if ($prepared !== '') {
$terms[] = '@@relaxed '.self::scopePreparedQueryToField($prepared, '@'.$key);
$terms[] = '@@relaxed '.self::scopeReleaseSearchQuery((string) $value, $prepared, '@'.$key);
}
}
if ($terms === []) {
@@ -2532,8 +2671,15 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$ids = [];
$documents = [];
foreach ($results as $doc) {
$ids[] = $doc->getId();
$id = (int) $doc->getId();
$ids[] = $id;
if ((bool) ($criteria['include_documents'] ?? false)) {
$document = $doc->getData();
$document['id'] = $id;
$documents[] = $document;
}
}
return [
@@ -2545,6 +2691,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
// supports a compound search-after predicate without raw SQL.
'last_sort' => [],
'has_more' => $offset + count($ids) < (int) $results->getTotal(),
'documents' => $documents,
];
};
app/Services/Search/Support/ManticoreIndexRegistry.php
+16 -4
View File
@@ -17,15 +17,27 @@ final class ManticoreIndexRegistry
return [
'releases' => ['settings' => $settings, 'columns' => [
'name' => ['type' => 'text'], 'searchname' => ['type' => 'text'],
'guid' => ['type' => 'string'], 'name' => ['type' => 'text'],
'searchname' => ['type' => 'text'], 'plainsearchname' => ['type' => 'text'],
'fromname' => ['type' => 'text'], 'filename' => ['type' => 'text'],
'category_name' => ['type' => 'string'], 'parent_category' => ['type' => 'string'],
'sub_category' => ['type' => 'string'], 'group_name' => ['type' => 'string'],
'categories_id' => ['type' => 'integer'], 'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'], 'traktid' => ['type' => 'integer'],
'tvdb' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'], 'videos_id' => ['type' => 'integer'],
'movieinfo_id' => ['type' => 'integer'], 'size' => ['type' => 'bigint'],
'tvrage' => ['type' => 'integer'], 'trakt' => ['type' => 'integer'],
'imdb' => ['type' => 'string'], 'tmdb' => ['type' => 'integer'],
'videos_id' => ['type' => 'integer'],
'tv_episodes_id' => ['type' => 'integer'], 'movieinfo_id' => ['type' => 'integer'],
'anidbid' => ['type' => 'integer'], 'parentid' => ['type' => 'integer'],
'episode_title' => ['type' => 'text'], 'series' => ['type' => 'integer'],
'episode' => ['type' => 'integer'], 'firstaired_ts' => ['type' => 'bigint'],
'size' => ['type' => 'bigint'],
'postdate_ts' => ['type' => 'bigint'], 'adddate_ts' => ['type' => 'bigint'],
'totalpart' => ['type' => 'integer'], 'grabs' => ['type' => 'integer'],
'comments' => ['type' => 'integer'], 'nfostatus' => ['type' => 'integer'],
'jpgstatus' => ['type' => 'integer'], 'nfoid' => ['type' => 'integer'],
'reid' => ['type' => 'integer'],
'passwordstatus' => ['type' => 'bigint'], 'groups_id' => ['type' => 'integer'],
'nzbstatus' => ['type' => 'integer'], 'haspreview' => ['type' => 'bigint'],
]],
@@ -56,7 +68,7 @@ final class ManticoreIndexRegistry
public static function profile(string $logical): array
{
$fields = match ($logical) {
'releases' => ['searchname' => 12, 'name' => 8, 'filename' => 5, 'fromname' => 1],
'releases' => ['searchname' => 12, 'plainsearchname' => 10, 'name' => 8, 'filename' => 5, 'fromname' => 1],
'predb' => ['title' => 12, 'filename' => 5],
'movies' => ['title' => 12, 'director' => 5, 'actors' => 3, 'genre' => 2, 'plot' => 1],
'tvshows' => ['title' => 12],
app/Services/Search/Support/ReleaseIndexProjection.php
+90
View File
@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\Support;
use App\Support\ReleaseSearchIndexDocument;
use Illuminate\Database\Query\Builder;
use Illuminate\Support\Facades\DB;
/**
* Single database projection used to build complete release search documents.
*/
final class ReleaseIndexProjection
{
public static function query(): Builder
{
$groupConcat = DB::connection()->getDriverName() === 'sqlite'
? "GROUP_CONCAT(rf.name, ' ')"
: "GROUP_CONCAT(rf.name SEPARATOR ' ')";
$categoryName = DB::connection()->getDriverName() === 'sqlite'
? "cp.title || ' > ' || c.title"
: "CONCAT(cp.title, ' > ', c.title)";
return DB::table('releases as r')
->leftJoin('usenet_groups as g', 'g.id', '=', 'r.groups_id')
->leftJoin('categories as c', 'c.id', '=', 'r.categories_id')
->leftJoin('root_categories as cp', 'cp.id', '=', 'c.root_categories_id')
->leftJoin('movieinfo as mi', function ($join): void {
$join->on('mi.id', '=', 'r.movieinfo_id')
->where('r.movieinfo_id', '>', 0);
})
->leftJoin('videos as v', function ($join): void {
$join->on('v.id', '=', 'r.videos_id')
->where('r.videos_id', '>', 0);
})
->leftJoin('tv_episodes as tve', function ($join): void {
$join->on('tve.id', '=', 'r.tv_episodes_id')
->where('r.tv_episodes_id', '>', 0);
})
->leftJoin('release_files as rf', 'rf.releases_id', '=', 'r.id')
->leftJoin('release_nfos as rn', 'rn.releases_id', '=', 'r.id')
->leftJoin('video_data as vd', 'vd.releases_id', '=', 'r.id')
->select([
'r.id', 'r.guid', 'r.name', 'r.searchname', 'r.fromname', 'r.categories_id',
'r.groups_id', 'r.size', 'r.postdate', 'r.adddate', 'r.totalpart', 'r.grabs',
'r.comments', 'r.passwordstatus', 'r.nzbstatus', 'r.nfostatus', 'r.haspreview',
'r.jpgstatus', 'r.videos_id', 'r.tv_episodes_id', 'r.movieinfo_id', 'r.imdbid',
'r.anidbid', 'g.name as group_name', 'c.root_categories_id as parentid',
'c.title as sub_category', 'tve.title as episode_title', 'tve.series',
'tve.episode', 'tve.firstaired',
'cp.title as parent_category', DB::raw("{$categoryName} AS category_name"),
DB::raw("COALESCE({$groupConcat}, '') AS filename"),
DB::raw('COALESCE(mi.tmdbid, 0) AS tmdbid'),
DB::raw('COALESCE(mi.traktid, 0) AS traktid'),
DB::raw('COALESCE(v.tvdb, 0) AS tvdb'),
DB::raw('COALESCE(v.tvmaze, 0) AS tvmaze'),
DB::raw('COALESCE(v.tvrage, 0) AS tvrage'),
DB::raw('COALESCE(v.trakt, 0) AS trakt'),
DB::raw("COALESCE(v.imdb, '') AS imdb"),
DB::raw('COALESCE(v.tmdb, 0) AS tmdb'),
DB::raw('COALESCE(rn.releases_id, 0) AS nfoid'),
DB::raw('COALESCE(vd.releases_id, 0) AS reid'),
])
->groupBy([
'r.id', 'r.guid', 'r.name', 'r.searchname', 'r.fromname', 'r.categories_id',
'r.groups_id', 'r.size', 'r.postdate', 'r.adddate', 'r.totalpart', 'r.grabs',
'r.comments', 'r.passwordstatus', 'r.nzbstatus', 'r.nfostatus', 'r.haspreview',
'r.jpgstatus', 'r.videos_id', 'r.tv_episodes_id', 'r.movieinfo_id', 'r.imdbid',
'r.anidbid', 'g.name', 'c.root_categories_id', 'c.title', 'tve.title',
'tve.series', 'tve.episode', 'tve.firstaired', 'cp.title', 'mi.tmdbid',
'mi.traktid', 'v.tvdb', 'v.tvmaze', 'v.tvrage', 'v.trakt', 'v.imdb', 'v.tmdb',
'rn.releases_id', 'vd.releases_id',
]);
}
/**
* @return array<string, mixed>|null
*/
public static function forId(int $releaseId): ?array
{
if ($releaseId <= 0) {
return null;
}
$row = self::query()->where('r.id', $releaseId)->first();
return $row === null ? null : ReleaseSearchIndexDocument::normalize((array) $row);
}
}
app/Support/ReleaseSearchIndexDocument.php
+66
View File
@@ -9,6 +9,26 @@ namespace App\Support;
*/
final class ReleaseSearchIndexDocument
{
/**
* Fields that may be returned to browse/search consumers.
* Sensitive release data must never be added to this list.
*
* @return list<string>
*/
public static function fields(): array
{
return [
'id', 'guid', 'name', 'searchname', 'plainsearchname', 'fromname', 'filename',
'categories_id', 'category_name', 'parent_category', 'sub_category',
'groups_id', 'group_name', 'imdbid', 'tmdbid', 'traktid', 'tvdb', 'tvmaze',
'tvrage', 'trakt', 'imdb', 'tmdb', 'videos_id', 'tv_episodes_id', 'movieinfo_id', 'anidbid',
'parentid', 'episode_title', 'series', 'episode', 'firstaired_ts',
'size', 'postdate_ts', 'adddate_ts', 'totalpart', 'grabs', 'comments',
'passwordstatus', 'nzbstatus', 'nfostatus', 'haspreview', 'jpgstatus',
'nfoid', 'reid',
];
}
/**
* Normalize a release row for bulk indexing. Rows already produced by {@see normalize()}
* (e.g. from release search populate) lack `postdate` / `adddate` keys;
@@ -40,10 +60,16 @@ final class ReleaseSearchIndexDocument
return [
'id' => (int) ($row['id'] ?? 0),
'guid' => (string) ($row['guid'] ?? ''),
'name' => (string) ($row['name'] ?? ''),
'searchname' => (string) ($row['searchname'] ?? ''),
'plainsearchname' => (string) ($row['plainsearchname'] ?? self::plainSearchName($row['searchname'] ?? '')),
'fromname' => (string) ($row['fromname'] ?? ''),
'categories_id' => (int) ($row['categories_id'] ?? 0),
'category_name' => (string) ($row['category_name'] ?? ''),
'parent_category' => (string) ($row['parent_category'] ?? ''),
'sub_category' => (string) ($row['sub_category'] ?? ''),
'group_name' => (string) ($row['group_name'] ?? ''),
'filename' => (string) ($row['filename'] ?? ''),
'imdbid' => (string) ($row['imdbid'] ?? ''),
'tmdbid' => (int) ($row['tmdbid'] ?? 0),
@@ -51,20 +77,50 @@ final class ReleaseSearchIndexDocument
'tvdb' => (int) ($row['tvdb'] ?? 0),
'tvmaze' => (int) ($row['tvmaze'] ?? 0),
'tvrage' => (int) ($row['tvrage'] ?? 0),
'trakt' => (int) ($row['trakt'] ?? 0),
'imdb' => (string) ($row['imdb'] ?? ''),
'tmdb' => (int) ($row['tmdb'] ?? 0),
'videos_id' => (int) ($row['videos_id'] ?? 0),
'tv_episodes_id' => (int) ($row['tv_episodes_id'] ?? 0),
'movieinfo_id' => (int) ($row['movieinfo_id'] ?? 0),
'anidbid' => (int) ($row['anidbid'] ?? 0),
'parentid' => (int) ($row['parentid'] ?? 0),
'episode_title' => (string) ($row['episode_title'] ?? ''),
'series' => (int) ($row['series'] ?? 0),
'episode' => (int) ($row['episode'] ?? 0),
'firstaired_ts' => self::datetimeToUnix($row['firstaired'] ?? ($row['firstaired_ts'] ?? null)),
'size' => (int) ($row['size'] ?? 0),
'postdate_ts' => $postdateTs,
'adddate_ts' => $adddateTs,
'totalpart' => (int) ($row['totalpart'] ?? 0),
'grabs' => (int) ($row['grabs'] ?? 0),
'comments' => (int) ($row['comments'] ?? 0),
'passwordstatus' => (int) ($row['passwordstatus'] ?? 0),
'groups_id' => (int) ($row['groups_id'] ?? 0),
'nzbstatus' => (int) ($row['nzbstatus'] ?? 0),
'nfostatus' => (int) ($row['nfostatus'] ?? 0),
'haspreview' => (int) ($row['haspreview'] ?? 0),
'jpgstatus' => (int) ($row['jpgstatus'] ?? 0),
'nfoid' => (int) ($row['nfoid'] ?? 0),
'reid' => (int) ($row['reid'] ?? 0),
];
}
/**
* Convert an indexed document into the date/attribute names expected by the views.
*
* @param array<string, mixed> $document
* @return array<string, mixed>
*/
public static function toReleaseRow(array $document): array
{
$row = self::normalizeForBulk($document);
$row['postdate'] = self::unixToDatetime((int) $row['postdate_ts']);
$row['adddate'] = self::unixToDatetime((int) $row['adddate_ts']);
return $row;
}
private static function datetimeToUnix(mixed $value): int
{
if ($value === null || $value === '') {
@@ -77,4 +133,14 @@ final class ReleaseSearchIndexDocument
return $ts !== false ? $ts : 0;
}
private static function plainSearchName(mixed $value): string
{
return trim((string) preg_replace('/[._-]+/', ' ', (string) $value));
}
private static function unixToDatetime(int $timestamp): ?string
{
return $timestamp > 0 ? date('Y-m-d H:i:s', $timestamp) : null;
}
}
app/Support/ReleaseSearchIndexSync.php
+57
View File
@@ -5,8 +5,10 @@ declare(strict_types=1);
namespace App\Support;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Release;
use App\Observers\ReleaseObserver;
use Illuminate\Database\Eloquent\Builder;
/**
* Re-sync releases_rt / ES release documents after query-builder or raw SQL updates
@@ -27,6 +29,61 @@ final class ReleaseSearchIndexSync
}
}
public static function forMovieInfo(int $movieInfoId): void
{
if ($movieInfoId > 0) {
self::forQuery(Release::query()->where('movieinfo_id', $movieInfoId));
}
}
public static function forVideo(int $videoId): void
{
if ($videoId > 0) {
self::forQuery(Release::query()->where('videos_id', $videoId));
}
}
public static function forCategory(int $categoryId): void
{
if ($categoryId > 0) {
self::forQuery(Release::query()->where('categories_id', $categoryId));
}
}
public static function forRootCategory(int $rootCategoryId): void
{
if ($rootCategoryId <= 0) {
return;
}
$categoryIds = Category::query()
->where('root_categories_id', $rootCategoryId)
->pluck('id');
if ($categoryIds->isNotEmpty()) {
self::forQuery(Release::query()->whereIn('categories_id', $categoryIds));
}
}
public static function forQueryGroup(int $groupId): void
{
if ($groupId > 0) {
self::forQuery(Release::query()->where('groups_id', $groupId));
}
}
/**
* @param Builder<Release> $query
*/
private static function forQuery($query): void
{
$query->select('releases.id')->orderBy('releases.id')->chunkById(500, function ($releases): bool {
self::forIds($releases->pluck('id'));
return true;
}, 'releases.id');
}
/**
* Reindex every release (chunked). Use after mass UPDATEs that match an optional raw WHERE suffix.
*
config/search.php
+2
View File
@@ -50,6 +50,8 @@ return [
'anime' => env('MANTICORESEARCH_INDEX_ANIME', 'anime_rt'),
],
'max_matches' => env('MANTICORESEARCH_MAX_MATCHES', 10000),
'retry_attempts' => (int) env('MANTICORESEARCH_RETRY_ATTEMPTS', 2),
'retry_delay_ms' => (int) env('MANTICORESEARCH_RETRY_DELAY_MS', 100),
'cache_minutes' => env('MANTICORESEARCH_CACHE_MINUTES', 5),
'autocomplete' => [
'enabled' => env('MANTICORESEARCH_AUTOCOMPLETE_ENABLED', true),
database/migrations/2026_08_02_000000_create_search_index_failures_table.php
+33
View File
@@ -0,0 +1,33 @@
<?php
declare(strict_types=1);
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
if (Schema::hasTable('search_index_failures')) {
return;
}
Schema::create('search_index_failures', function (Blueprint $table): void {
$table->id();
$table->unsignedBigInteger('release_id')->unique();
$table->string('operation', 32)->default('upsert');
$table->unsignedInteger('attempts')->default(0);
$table->text('last_error')->nullable();
$table->timestamp('next_attempt_at')->nullable()->index();
$table->timestamp('resolved_at')->nullable()->index();
$table->timestamps();
});
}
public function down(): void
{
Schema::dropIfExists('search_index_failures');
}
};
phpstan-baseline.neon
+1 -7
View File
@@ -928,7 +928,7 @@ parameters:
-
message: '#^Parameter \#1 \$params of method Elastic\\Elasticsearch\\Client\:\:delete\(\) expects array\{id\: string, index\: string, wait_for_active_shards\?\: string, refresh\?\: string, routing\?\: string, timeout\?\: int\|string, if_seq_no\?\: int, if_primary_term\?\: int, \.\.\.\}\|null, array\{index\: string, id\: int\<min, \-1\>\|int\<1, max\>\} given\.$#'
identifier: argument.type
count: 4
count: 3
path: app/Services/Search/Drivers/ElasticSearchDriver.php
-
@@ -937,12 +937,6 @@ parameters:
count: 2
path: app/Services/Search/Drivers/ElasticSearchDriver.php
-
message: '#^Method App\\Services\\Search\\Drivers\\ManticoreSearchDriver\:\:phrasesToSearchArray\(\) return type has no value type specified in iterable type array\.$#'
identifier: missingType.iterableValue
count: 1
path: app/Services/Search/Drivers/ManticoreSearchDriver.php
-
message: '#^Method App\\Services\\Search\\Drivers\\ManticoreSearchDriver\:\:searchReleasesByMultipleExternalIds\(\) should return array\<string, mixed\> but returns list\<int\>\.$#'
identifier: return.type
routes/console.php
+1
View File
@@ -61,6 +61,7 @@ Schedule::call(function () {
// Check tmux health and auto-restart if monitor pane is dead
Schedule::command('tmux:health-check --auto-restart')->everyThirtyMinutes()->withoutOverlapping();
Schedule::command('nntmux:check-service-health')->everyMinute()->withoutOverlapping();
Schedule::command('nntmux:search-repair --limit=100')->everyMinute()->withoutOverlapping();
// Keep the admin dashboard snapshot (Cache::flexible) hot so admins never pay
// the cold-cache cost when opening /admin/index.
Schedule::command('admin:warm-dashboard')->everyMinute()->withoutOverlapping();
tests/Unit/ManticoreSearchQueryTest.php
+28 -2
View File
@@ -18,6 +18,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function negationQueriesProvider(): array
{
return [
@@ -37,6 +38,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function phraseQueriesProvider(): array
{
return [
@@ -55,6 +57,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function orQueriesProvider(): array
{
return [
@@ -71,6 +74,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function wildcardQueriesProvider(): array
{
return [
@@ -88,6 +92,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function groupingQueriesProvider(): array
{
return [
@@ -104,6 +109,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{string, string}> */
public static function escapingQueriesProvider(): array
{
return [
@@ -143,6 +149,24 @@ class ManticoreSearchQueryTest extends TestCase
);
}
#[Test]
public function it_adds_a_normalized_release_name_query_for_punctuation_separators(): void
{
$reflection = new ReflectionClass(ManticoreSearchDriver::class);
$method = $reflection->getMethod('scopeReleaseSearchQuery');
$query = $method->invoke(
null,
'Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB-DL.HEVC.x265-BONE',
ManticoreSearchDriver::prepareUserSearchQuery('Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB-DL.HEVC.x265-BONE'),
'@searchname'
);
$this->assertStringContainsString('@searchname (Love.Is.A.Dogs.Best.Friend.2025.1080p.WEB\\-DL.HEVC.x265\\-BONE)', $query);
$this->assertStringContainsString('@searchname (Love Is A Dogs Best Friend 2025 1080p WEB DL HEVC x265 BONE)', $query);
}
/** @return array<string, array{string, string}> */
public static function edgeCaseQueriesProvider(): array
{
return [
@@ -163,6 +187,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertStringContainsString('circus', $result);
}
/** @param array<string, string>|string $input */
#[Test]
#[DataProvider('negationDetectionProvider')]
public function it_detects_negation_operators(array|string $input, bool $expected): void
@@ -171,6 +196,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertSame($expected, $result);
}
/** @return array<string, array{array<string, string>|string, bool}> */
public static function negationDetectionProvider(): array
{
return [
@@ -244,7 +270,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertIsString($driverSource);
$this->assertStringContainsString(
"\$terms[] = '@@relaxed '.self::scopePreparedQueryToField(\$prepared, '@'.\$key);",
"\$terms[] = '@@relaxed '.self::scopeReleaseSearchQuery((string) \$value, \$prepared, '@'.\$key);",
$driverSource
);
$this->assertStringNotContainsString(
@@ -260,7 +286,7 @@ class ManticoreSearchQueryTest extends TestCase
$this->assertIsString($driverSource);
$this->assertStringContainsString(
"\$searchExpr = '@@relaxed '.self::scopePreparedQueryToField(\$preparedSearch, '@searchname');",
"\$searchExpr = '@@relaxed '.self::scopeReleaseSearchQuery(\$searchTerm, \$preparedSearch, '@searchname');",
$driverSource
);
$this->assertStringNotContainsString(
tests/Unit/NntmuxSearchReconcileFetchIndexedIdsTest.php
+5 -2
View File
@@ -121,7 +121,7 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase
public function __construct() {}
public function sql(...$params): mixed
public function sql(mixed ...$params): mixed
{
$this->capturedSql = $params[0] ?? null;
@@ -143,6 +143,9 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase
$this->assertNotNull($client->capturedSql);
$this->assertStringContainsString('LIMIT 500', $client->capturedSql);
$this->assertStringContainsString('OPTION max_matches=500', $client->capturedSql);
$driverSource = file_get_contents(__DIR__.'/../../app/Services/Search/Drivers/ManticoreSearchDriver.php');
self::assertIsString($driverSource);
self::assertStringContainsString('ORDER BY id ASC', $driverSource);
}
/**
@@ -159,7 +162,7 @@ class NntmuxSearchReconcileFetchIndexedIdsTest extends TestCase
// Bypass parent ctor: we never hit the network in this test.
}
public function sql(...$params): mixed
public function sql(mixed ...$params): mixed
{
return $this->stubResponse;
}
tests/Unit/ReleaseSearchIndexDocumentTest.php
+55
View File
@@ -10,6 +10,61 @@ use PHPUnit\Framework\TestCase;
final class ReleaseSearchIndexDocumentTest extends TestCase
{
#[Test]
public function normalize_builds_the_complete_public_projection_without_sensitive_fields(): void
{
$document = ReleaseSearchIndexDocument::normalize([
'id' => '42',
'guid' => 'guid-42',
'name' => 'Release.Name',
'searchname' => 'Release.Name',
'fromname' => 'poster',
'filename' => 'file.nzb',
'categories_id' => '1',
'category_name' => 'TV',
'parent_category' => 'Video',
'sub_category' => 'TV',
'groups_id' => '2',
'group_name' => 'alt.example',
'tmdbid' => '123',
'tv_episodes_id' => '456',
'passwordstatus' => '-1',
'comments' => '3',
'postdate' => '2026-01-02 03:04:05',
'adddate' => '2026-01-03 04:05:06',
'nzb_password' => 'must-not-be-indexed',
]);
self::assertSame(42, $document['id']);
self::assertSame('Release Name', $document['plainsearchname']);
self::assertSame(123, $document['tmdbid']);
self::assertSame(456, $document['tv_episodes_id']);
self::assertSame(-1, $document['passwordstatus']);
self::assertGreaterThan(0, $document['postdate_ts']);
self::assertGreaterThan(0, $document['adddate_ts']);
self::assertArrayNotHasKey('nzb_password', $document);
$documentFields = array_keys($document);
$declaredFields = ReleaseSearchIndexDocument::fields();
sort($documentFields);
sort($declaredFields);
self::assertSame($declaredFields, $documentFields);
}
#[Test]
public function indexed_documents_can_be_hydrated_for_legacy_release_consumers(): void
{
$row = ReleaseSearchIndexDocument::toReleaseRow([
'id' => 42,
'postdate_ts' => 1767323045,
'adddate_ts' => 1767413106,
'name' => 'Release',
]);
self::assertSame('2026-01-02 03:04:05', $row['postdate']);
self::assertSame('2026-01-03 04:05:06', $row['adddate']);
self::assertSame(42, $row['id']);
}
#[Test]
public function normalize_for_bulk_preserves_timestamps_when_row_is_already_normalized(): void
{
tests/Unit/Services/Search/ManticoreDeleteReleasesTest.php
+70
View File
@@ -0,0 +1,70 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Services\Search;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use Manticoresearch\Client;
use Manticoresearch\Table;
use PHPUnit\Framework\Attributes\Test;
use Tests\TestCase;
final class ManticoreDeleteReleasesTest extends TestCase
{
#[Test]
public function delete_releases_uses_document_api_not_sql(): void
{
$config = [
'host' => '127.0.0.1',
'port' => 9308,
'indexes' => [
'releases' => 'releases_rt',
'predb' => 'predb_rt',
],
];
$table = $this->createMock(Table::class);
$table->expects($this->once())
->method('deleteDocumentsByIds')
->with([149331415, 42]);
$client = $this->createMock(Client::class);
$client->expects($this->once())
->method('table')
->with('releases_rt')
->willReturn($table);
$client->expects($this->never())->method('sql');
$driver = new ManticoreSearchDriver($config);
$prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch');
$prop->setAccessible(true);
$prop->setValue($driver, $client);
$driver->deleteReleases([149331415, 42, 0, 149331415]);
}
#[Test]
public function delete_releases_skips_empty_id_list(): void
{
$config = [
'host' => '127.0.0.1',
'port' => 9308,
'indexes' => [
'releases' => 'releases_rt',
'predb' => 'predb_rt',
],
];
$client = $this->createMock(Client::class);
$client->expects($this->never())->method('table');
$client->expects($this->never())->method('sql');
$driver = new ManticoreSearchDriver($config);
$prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch');
$prop->setAccessible(true);
$prop->setValue($driver, $client);
$driver->deleteReleases([0, -1]);
}
}
tests/Unit/Services/Search/ManticoreInsertRetryTest.php
+37
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace Tests\Unit\Services\Search;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use App\Support\ReleaseSearchIndexDocument;
use Manticoresearch\Client;
use Manticoresearch\Exceptions\ResponseException;
use Manticoresearch\Request;
@@ -130,4 +131,40 @@ final class ManticoreInsertRetryTest extends TestCase
$ok = $refP->invoke($driver, $this->releaseRow(7));
$this->assertFalse($ok);
}
public function test_replace_release_document_preserves_pre_normalized_timestamps(): void
{
$config = [
'host' => '127.0.0.1',
'port' => 9308,
'retry_attempts' => 1,
'indexes' => ['releases' => 'releases_rt', 'predb' => 'predb_rt'],
];
$document = ReleaseSearchIndexDocument::normalize($this->releaseRow(42));
$table = $this->createMock(Table::class);
$table->expects($this->once())
->method('replaceDocument')
->with(
$this->callback(static fn (array $indexed): bool =>
$indexed['postdate_ts'] === $document['postdate_ts']
&& $indexed['adddate_ts'] === $document['adddate_ts']
),
42
);
$client = $this->createMock(Client::class);
$client->expects($this->once())
->method('table')
->with('releases_rt')
->willReturn($table);
$driver = new ManticoreSearchDriver($config);
$prop = new \ReflectionProperty(ManticoreSearchDriver::class, 'manticoreSearch');
$prop->setValue($driver, $client);
$method = new ReflectionMethod(ManticoreSearchDriver::class, 'replaceReleaseDocumentWithRetry');
$this->assertTrue($method->invoke($driver, $document));
}
}