From 9643d4dd0bd49db8f3c2b2bd9739a940521dbfdb Mon Sep 17 00:00:00 2001 From: DariusIII Date: Wed, 15 Jul 2026 14:22:31 +0200 Subject: [PATCH] Update manticore indexes --- .../Commands/CreateManticoreIndexes.php | 230 ++---------------- app/Console/Commands/InspectManticore.php | 75 ++++++ .../Search/Drivers/ManticoreSearchDriver.php | 230 +++++++----------- .../Search/Support/ManticoreIndexRegistry.php | 133 ++++++++++ .../Support/ManticoreSchemaInspector.php | 52 ++++ docker-compose.yml.prod-dist | 4 +- docs/adr/0001-manticore-lexical-search.md | 21 ++ docs/manticore-28-upgrade.md | 25 ++ .../Search/manticore-relevance-v1.json | 24 ++ tests/Unit/ManticoreIndexRegistryTest.php | 68 ++++++ 10 files changed, 513 insertions(+), 349 deletions(-) create mode 100644 app/Console/Commands/InspectManticore.php create mode 100644 app/Services/Search/Support/ManticoreIndexRegistry.php create mode 100644 app/Services/Search/Support/ManticoreSchemaInspector.php create mode 100644 docs/adr/0001-manticore-lexical-search.md create mode 100644 docs/manticore-28-upgrade.md create mode 100644 tests/Fixtures/Search/manticore-relevance-v1.json create mode 100644 tests/Unit/ManticoreIndexRegistryTest.php diff --git a/app/Console/Commands/CreateManticoreIndexes.php b/app/Console/Commands/CreateManticoreIndexes.php index 62281384b..9b52a7808 100644 --- a/app/Console/Commands/CreateManticoreIndexes.php +++ b/app/Console/Commands/CreateManticoreIndexes.php @@ -4,263 +4,85 @@ declare(strict_types=1); namespace App\Console\Commands; -use App\Enums\SecondarySearchIndex; use App\Services\Search\Support\ManticoreClientFactory; +use App\Services\Search\Support\ManticoreIndexRegistry; use Illuminate\Console\Command; use Manticoresearch\Client; use Manticoresearch\Exceptions\ResponseException; class CreateManticoreIndexes extends Command { - /** - * The name and signature of the console command. - * - * @var string - */ - protected $signature = 'manticore:create-indexes - {--drop : Drop existing indexes before creating}'; + protected $signature = 'manticore:create-indexes {--drop : Drop existing indexes before creating}'; - /** - * The console command description. - * - * @var string - */ protected $description = 'Create Manticore Search indexes based on configuration'; protected Client $client; - /** - * Execute the console command. - */ public function handle(): int { $this->info('Creating Manticore Search indexes...'); - - $dropExisting = $this->option('drop'); - - $this->client = ManticoreClientFactory::make(config('search.drivers.manticore', config('manticoresearch', []))); - - // We'll skip checking for data_dir this way since it may not be accessible via API - // but instead provide better error handling during index creation - - // If you encounter data_dir errors, ensure it's properly set in config/manticore.conf: - // data_dir = /path/to/data - // And make sure the path exists and has proper permissions + $this->client = ManticoreClientFactory::make(config('search.drivers.manticore', [])); try { $this->client->nodes()->status(); - } catch (\Exception $e) { + } catch (\Throwable $e) { $this->error('Failed to connect to Manticore Search: '.$e->getMessage()); - $this->info('Please check if Manticore Search is running and properly configured.'); + $this->info('Check the configured host, authentication, and whether Manticore is running.'); - return 1; + return self::FAILURE; } - // Define indexes and their schema - $indexes = [ - 'releases_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'name' => ['type' => 'text'], - 'searchname' => ['type' => 'text'], - 'fromname' => ['type' => 'text'], - 'filename' => ['type' => 'text'], - 'categories_id' => ['type' => 'integer'], - // External media IDs for efficient searching - 'imdbid' => ['type' => 'string'], - 'tmdbid' => ['type' => 'integer'], - 'traktid' => ['type' => 'integer'], - 'tvdb' => ['type' => 'integer'], - 'tvmaze' => ['type' => 'integer'], - 'tvrage' => ['type' => 'integer'], - 'videos_id' => ['type' => 'integer'], - 'movieinfo_id' => ['type' => 'integer'], - 'size' => ['type' => 'bigint'], - 'postdate_ts' => ['type' => 'bigint'], - 'adddate_ts' => ['type' => 'bigint'], - 'totalpart' => ['type' => 'integer'], - 'grabs' => ['type' => 'integer'], - 'passwordstatus' => ['type' => 'bigint'], - 'groups_id' => ['type' => 'integer'], - 'nzbstatus' => ['type' => 'integer'], - 'haspreview' => ['type' => 'bigint'], - ], - ], - 'predb_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'title' => ['type' => 'text', 'attribute' => true], - 'filename' => ['type' => 'text', 'attribute' => true], - 'source' => ['type' => 'string', 'attribute' => true], - ], - ], - 'movies_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'imdbid' => ['type' => 'string'], - 'tmdbid' => ['type' => 'integer'], - 'traktid' => ['type' => 'integer'], - 'title' => ['type' => 'text'], - 'year' => ['type' => 'text'], - 'genre' => ['type' => 'text'], - 'actors' => ['type' => 'text'], - 'director' => ['type' => 'text'], - 'rating' => ['type' => 'text'], - 'plot' => ['type' => 'text'], - ], - ], - 'tvshows_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'title' => ['type' => 'text'], - 'tvdb' => ['type' => 'integer'], - 'trakt' => ['type' => 'integer'], - 'tvmaze' => ['type' => 'integer'], - 'tvrage' => ['type' => 'integer'], - 'imdb' => ['type' => 'string'], - 'tmdb' => ['type' => 'integer'], - 'started' => ['type' => 'text'], - 'type' => ['type' => 'integer'], - ], - ], - 'music_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Music->manticoreColumns(), - ], - 'books_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Books->manticoreColumns(), - ], - 'games_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Games->manticoreColumns(), - ], - 'console_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Console->manticoreColumns(), - ], - 'steam_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Steam->manticoreColumns(), - ], - 'anime_rt' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => SecondarySearchIndex::Anime->manticoreColumns(), - ], - ]; - + $configuredNames = config('search.drivers.manticore.indexes', []); $hasErrors = false; - - // Create each index - foreach ($indexes as $indexName => $schema) { - if (! $this->createIndex($indexName, $schema, $dropExisting)) { + foreach (ManticoreIndexRegistry::definitions() as $logical => $schema) { + $indexName = (string) ($configuredNames[$logical] ?? $logical.'_rt'); + if (! $this->createIndex($indexName, $schema, (bool) $this->option('drop'))) { $hasErrors = true; } } if ($hasErrors) { - $this->error('Some errors occurred during index creation.'); + $this->error('Some Manticore tables could not be created.'); - return 1; + return self::FAILURE; } - $this->info('All Manticore Search indexes created successfully!'); + $this->info('All Manticore Search tables are ready.'); - return 0; + return self::SUCCESS; } - /** - * Create a single index with error handling. - * - * @param array $schema - */ + /** @param array $schema */ protected function createIndex(string $indexName, array $schema, bool $dropExisting): bool { - $this->info("Creating {$indexName} index..."); - $indices = $this->client->tables(); + $tables = $this->client->tables(); try { - // Optionally drop existing index if ($dropExisting) { - try { - $this->info("Dropping existing {$indexName} index..."); - $indices->drop(['index' => $indexName, 'body' => ['silent' => true]]); - $this->info("Successfully dropped {$indexName} index."); - } catch (ResponseException $e) { - if (! str_contains($e->getMessage(), 'unknown index')) { - $this->warn("Warning when dropping {$indexName} index: ".$e->getMessage()); - } - } + $this->info("Dropping {$indexName}..."); + $tables->drop(['index' => $indexName, 'body' => ['silent' => true]]); } - // Instead of checking if index exists (which doesn't work), - // try to create it directly and handle any errors - // that might occur if it already exists - $response = $indices->create([ - 'index' => $indexName, - 'body' => $schema, - ]); - - $this->info("Successfully created {$indexName} index."); - $this->line('Response: '.json_encode($response, JSON_PRETTY_PRINT)); + $tables->create(['index' => $indexName, 'body' => $schema]); + $this->info("Created {$indexName}."); return true; } catch (ResponseException $e) { - // Check if the error is because the index already exists if (str_contains($e->getMessage(), 'already exists')) { - $this->warn("Index {$indexName} already exists. Use --drop option to recreate it."); + $this->warn("{$indexName} already exists; use --drop during a maintenance rebuild."); return true; } - // Check for data_dir configuration error if (str_contains($e->getMessage(), 'data_dir')) { - $this->error("Failed to create {$indexName} index: ".$e->getMessage()); - $this->newLine(); - $this->warn('ManticoreSearch requires data_dir to be set in its configuration file.'); - $this->info('To fix this, edit your ManticoreSearch config file (usually /etc/manticoresearch/manticore.conf):'); - $this->line(' 1. Add or uncomment: data_dir = /var/lib/manticore'); - $this->line(' 2. Ensure the directory exists and is writable by the manticore user'); - $this->line(' 3. Restart ManticoreSearch: sudo systemctl restart manticore'); - - return false; + $this->error("Failed to create {$indexName}: configure a writable Manticore data_dir."); + } else { + $this->error("Failed to create {$indexName}: ".$e->getMessage()); } - $this->error("Failed to create {$indexName} index: ".$e->getMessage()); - return false; - } catch (\Exception $e) { - $this->error("Failed to create {$indexName} index: ".$e->getMessage()); + } catch (\Throwable $e) { + $this->error("Failed to create {$indexName}: ".$e->getMessage()); return false; } diff --git a/app/Console/Commands/InspectManticore.php b/app/Console/Commands/InspectManticore.php new file mode 100644 index 000000000..667469645 --- /dev/null +++ b/app/Console/Commands/InspectManticore.php @@ -0,0 +1,75 @@ +sql('SELECT VERSION() AS version', true); + } catch (\Throwable $e) { + $this->error('Unable to connect to Manticore: '.$e->getMessage()); + + return self::FAILURE; + } + + $report = ['version' => self::extractVersion($versionResponse), 'compatible' => true, 'tables' => []]; + $configured = config('search.drivers.manticore.indexes', []); + + foreach (ManticoreIndexRegistry::definitions() as $logical => $definition) { + $table = (string) ($configured[$logical] ?? $logical.'_rt'); + try { + if (preg_match('/^[a-zA-Z0-9_]+$/', $table) !== 1) { + throw new \RuntimeException('Unsafe configured table name'); + } + $actual = $client->table($table)->describe(); + $comparison = ManticoreSchemaInspector::compareColumns(is_array($actual) ? $actual : [], $definition['columns']); + $actualSettings = $client->sql("SHOW TABLE {$table} SETTINGS", true); + $missingSettings = ManticoreSchemaInspector::missingSettings($actualSettings, $definition['settings']); + $needsRebuild = $comparison['missing'] !== [] || $comparison['incompatible'] !== [] || $missingSettings !== []; + $report['tables'][$table] = [...$comparison, 'missing_settings' => $missingSettings, 'needs_rebuild' => $needsRebuild]; + $report['compatible'] = $report['compatible'] && ! $needsRebuild; + } catch (\Throwable $e) { + $report['tables'][$table] = ['missing_table' => true, 'needs_rebuild' => true, 'error' => $e->getMessage()]; + $report['compatible'] = false; + } + } + + if ($this->option('json')) { + $this->line((string) json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } else { + $this->info('Manticore version: '.$report['version']); + foreach ($report['tables'] as $table => $status) { + $this->line(sprintf('%-20s %s', $table, $status['needs_rebuild'] ? 'REBUILD REQUIRED' : 'compatible')); + } + if (! $report['compatible']) { + $this->warn('Run manticore:create-indexes --drop during a maintenance window, then repopulate all enabled indexes.'); + } + } + + return $report['compatible'] ? self::SUCCESS : self::FAILURE; + } + + private static function extractVersion(mixed $response): string + { + $encoded = json_encode($response); + if (is_string($encoded) && preg_match('/\d+\.\d+(?:\.\d+)?/', $encoded, $match) === 1) { + return $match[0]; + } + + return 'unknown'; + } +} diff --git a/app/Services/Search/Drivers/ManticoreSearchDriver.php b/app/Services/Search/Drivers/ManticoreSearchDriver.php index 85966dee7..e0c97c6cb 100644 --- a/app/Services/Search/Drivers/ManticoreSearchDriver.php +++ b/app/Services/Search/Drivers/ManticoreSearchDriver.php @@ -18,6 +18,7 @@ use App\Services\Search\Contracts\SearchDriverInterface; use App\Services\Search\DTO\ReleaseSearchQuery; use App\Services\Search\DTO\SearchPage; use App\Services\Search\Support\ManticoreClientFactory; +use App\Services\Search\Support\ManticoreIndexRegistry; use App\Support\ReleaseSearchIndexDocument; use App\Support\SecondaryIndexDocuments; use Illuminate\Support\Facades\Cache; @@ -856,123 +857,17 @@ class ManticoreSearchDriver implements SearchDriverInterface } try { - $indices = $this->manticoreSearch->tables(); + $logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []); + $definition = $logical === null ? null : (ManticoreIndexRegistry::definitions()[$logical] ?? null); - if ($index === 'releases_rt') { - $indices->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'name' => ['type' => 'text'], - 'searchname' => ['type' => 'text'], - 'fromname' => ['type' => 'text'], - 'filename' => ['type' => 'text'], - 'categories_id' => ['type' => 'integer'], - 'imdbid' => ['type' => 'string'], - 'tmdbid' => ['type' => 'integer'], - 'traktid' => ['type' => 'integer'], - 'tvdb' => ['type' => 'integer'], - 'tvmaze' => ['type' => 'integer'], - 'tvrage' => ['type' => 'integer'], - 'videos_id' => ['type' => 'integer'], - 'movieinfo_id' => ['type' => 'integer'], - 'size' => ['type' => 'bigint'], - 'postdate_ts' => ['type' => 'bigint'], - 'adddate_ts' => ['type' => 'bigint'], - 'totalpart' => ['type' => 'integer'], - 'grabs' => ['type' => 'integer'], - 'passwordstatus' => ['type' => 'bigint'], - 'groups_id' => ['type' => 'integer'], - 'nzbstatus' => ['type' => 'integer'], - 'haspreview' => ['type' => 'bigint'], - ], - ], - ]); - cli()->info('Created releases_rt index with external ID fields and infix search support'); - } elseif ($index === 'predb_rt') { - $indices->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'title' => ['type' => 'text'], - 'filename' => ['type' => 'text'], - 'source' => ['type' => 'text'], - ], - ], - ]); - cli()->info('Created predb_rt index with infix search support'); - } elseif ($index === 'movies_rt') { - $indices->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'imdbid' => ['type' => 'string'], - 'tmdbid' => ['type' => 'integer'], - 'traktid' => ['type' => 'integer'], - 'title' => ['type' => 'text'], - 'year' => ['type' => 'text'], - 'genre' => ['type' => 'text'], - 'actors' => ['type' => 'text'], - 'director' => ['type' => 'text'], - 'rating' => ['type' => 'text'], - 'plot' => ['type' => 'text'], - ], - ], - ]); - cli()->info('Created movies_rt index with infix search support'); - } elseif ($index === 'tvshows_rt') { - $indices->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => [ - 'title' => ['type' => 'text'], - 'tvdb' => ['type' => 'integer'], - 'trakt' => ['type' => 'integer'], - 'tvmaze' => ['type' => 'integer'], - 'tvrage' => ['type' => 'integer'], - 'imdb' => ['type' => 'string'], - 'tmdb' => ['type' => 'integer'], - 'started' => ['type' => 'text'], - 'type' => ['type' => 'integer'], - ], - ], - ]); - cli()->info('Created tvshows_rt index with infix search support'); - } else { - foreach (SecondarySearchIndex::cases() as $secondary) { - if ($index === $this->getSecondaryIndexName($secondary)) { - $indices->create([ - 'index' => $index, - 'body' => [ - 'settings' => [ - 'min_prefix_len' => 0, - 'min_infix_len' => 2, - ], - 'columns' => $secondary->manticoreColumns(), - ], - ]); - cli()->info("Created {$index} secondary metadata index"); + if ($definition === null) { + cli()->error("No Manticore schema is registered for {$index}"); - break; - } - } + return; } + + $this->manticoreSearch->tables()->create(['index' => $index, 'body' => $definition]); + cli()->info("Created {$index} with the configured Manticore schema"); } catch (\Throwable $e) { if (! str_contains($e->getMessage(), 'already exists')) { cli()->error('Error creating index '.$index.': '.$e->getMessage()); @@ -1155,11 +1050,14 @@ class ManticoreSearchDriver implements SearchDriverInterface } $normalizedLimit = $this->normalizeSearchLimit($limit); + $logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []); + $profile = ManticoreIndexRegistry::profile($logical ?? ''); $fuzzyConfig = $this->getFuzzyConfig(); - $distance = $fuzzyConfig['max_distance'] ?? 2; + $distance = min((int) ($fuzzyConfig['max_distance'] ?? 2), $profile['fuzzy_distance']); // Create cache key for fuzzy search results $cacheKey = 'manticore:fuzzy:'.md5(serialize([ + 'generation' => config('search.index_generation', '1'), 'index' => $index, 'array' => $searchArray, 'limit' => $normalizedLimit, @@ -1215,6 +1113,7 @@ class ManticoreSearchDriver implements SearchDriverInterface ->search($searchExpr) ->option('fuzzy', true) ->option('distance', $distance) + ->option('field_weights', $profile['fields']) ->limit($normalizedLimit); $results = $query->get(); @@ -1308,6 +1207,8 @@ class ManticoreSearchDriver implements SearchDriverInterface } $normalizedLimit = $this->normalizeSearchLimit($limit); + $logical = ManticoreIndexRegistry::logicalName($rt_index, $this->config['indexes'] ?? []); + $profile = ManticoreIndexRegistry::profile($logical ?? ''); if (config('app.debug')) { Log::debug('ManticoreSearch::searchIndexes called', [ @@ -1321,11 +1222,13 @@ class ManticoreSearchDriver implements SearchDriverInterface // Create cache key for search results $cacheKey = md5(serialize([ + 'generation' => config('search.index_generation', '1'), 'index' => $rt_index, 'search' => $searchString, 'columns' => $column, 'array' => $searchArray, 'limit' => $normalizedLimit, + 'profile' => $profile, ])); $cached = Cache::get($cacheKey); @@ -1400,7 +1303,8 @@ class ManticoreSearchDriver implements SearchDriverInterface // Use a fresh Search instance for every query to avoid parameter accumulation across calls $query = (new Search($this->manticoreSearch)) ->setTable($rt_index) - ->option('ranker', 'sph04') + ->option('ranker', $profile['ranker']) + ->option('field_weights', $profile['fields']) ->maxMatches($normalizedLimit) ->limit($normalizedLimit) ->stripBadUtf8(true) @@ -1417,7 +1321,8 @@ class ManticoreSearchDriver implements SearchDriverInterface try { $query = (new Search($this->manticoreSearch)) ->setTable($rt_index) - ->option('ranker', 'sph04') + ->option('ranker', $profile['ranker']) + ->option('field_weights', $profile['fields']) ->maxMatches($normalizedLimit) ->limit($normalizedLimit) ->stripBadUtf8(true) @@ -1516,7 +1421,12 @@ class ManticoreSearchDriver implements SearchDriverInterface } $index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt'); - $cacheKey = 'manticore:autocomplete:'.md5($index.$query); + $logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []); + $fields = ManticoreIndexRegistry::autocompleteFields($logical ?? ''); + if ($fields === []) { + return []; + } + $cacheKey = 'manticore:autocomplete:'.config('search.index_generation', '1').':'.md5($index.$query); $cached = Cache::get($cacheKey); if ($cached !== null) { @@ -1533,8 +1443,7 @@ class ManticoreSearchDriver implements SearchDriverInterface return []; } - // Use relaxed search on searchname field - $searchExpr = '@@relaxed @searchname '.$escapedQuery; + $searchExpr = '@@relaxed @('.implode(',', $fields).') '.$escapedQuery.'*'; $search = (new Search($this->manticoreSearch)) ->setTable($index) @@ -1545,10 +1454,16 @@ class ManticoreSearchDriver implements SearchDriverInterface $results = $search->get(); - $seen = []; + $ranked = []; foreach ($results as $doc) { $data = $doc->getData(); - $searchname = $data['searchname'] ?? ''; + $searchname = ''; + foreach ($fields as $field) { + if (! empty($data[$field])) { + $searchname = (string) $data[$field]; + break; + } + } if (empty($searchname)) { continue; @@ -1556,20 +1471,29 @@ class ManticoreSearchDriver implements SearchDriverInterface // Create a clean suggestion from the searchname $suggestion = $this->extractSuggestion($searchname, $query); - - if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) { - $seen[strtolower($suggestion)] = true; - $suggestions[] = [ - 'suggest' => $suggestion, - 'distance' => 0, - 'docs' => 1, - ]; + if ($suggestion === null || $suggestion === '') { + continue; } - if (count($suggestions) >= $maxResults) { - break; + $normalized = mb_strtolower(trim((string) preg_replace('/\s+/', ' ', $suggestion))); + if ($normalized !== '') { + if (! isset($ranked[$normalized])) { + $ranked[$normalized] = [ + 'suggest' => $suggestion, + 'distance' => 0, + 'docs' => 0, + 'prefix' => str_starts_with($normalized, mb_strtolower($query)) ? 1 : 0, + ]; + } + $ranked[$normalized]['docs']++; } } + + uasort($ranked, static fn (array $a, array $b): int => [$b['prefix'], $b['docs']] <=> [$a['prefix'], $a['docs']]); + foreach (array_slice($ranked, 0, $maxResults) as $item) { + unset($item['prefix']); + $suggestions[] = $item; + } } catch (\Throwable $e) { if (config('app.debug')) { Log::warning('ManticoreSearch autocomplete error: '.$e->getMessage()); @@ -1594,13 +1518,13 @@ class ManticoreSearchDriver implements SearchDriverInterface private function extractSuggestion(string $searchname, string $query): ?string { // Clean up the searchname - remove file extensions, quality tags at the end - $clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname); + $clean = (string) preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname); // Replace dots and underscores with spaces for readability $clean = str_replace(['.', '_'], ' ', $clean); // Remove multiple spaces - $clean = preg_replace('/\s+/', ' ', $clean); + $clean = (string) preg_replace('/\s+/', ' ', $clean); $clean = trim($clean); if (empty($clean)) { @@ -1622,9 +1546,9 @@ class ManticoreSearchDriver implements SearchDriverInterface // Clean up - don't cut mid-word if ($start > 0) { - $extracted = preg_replace('/^\S*\s/', '', $extracted); + $extracted = (string) preg_replace('/^\S*\s/', '', $extracted); } - $extracted = preg_replace('/\s\S*$/', '', $extracted); + $extracted = (string) preg_replace('/\s\S*$/', '', $extracted); return trim($extracted); } @@ -1664,7 +1588,7 @@ class ManticoreSearchDriver implements SearchDriverInterface } $index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt'); - $cacheKey = 'manticore:suggest:'.md5($index.$query); + $cacheKey = 'manticore:suggest:'.config('search.index_generation', '1').':'.md5($index.$query); $cached = Cache::get($cacheKey); if ($cached !== null) { @@ -1730,8 +1654,12 @@ class ManticoreSearchDriver implements SearchDriverInterface return []; } - // Use relaxed search to find partial matches - $searchExpr = '@@relaxed @searchname '.$escapedQuery; + $logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []); + $fields = ManticoreIndexRegistry::autocompleteFields($logical ?? ''); + if ($fields === []) { + return []; + } + $searchExpr = '@@relaxed @('.implode(',', $fields).') '.$escapedQuery; $search = (new Search($this->manticoreSearch)) ->setTable($index) @@ -1745,10 +1673,16 @@ class ManticoreSearchDriver implements SearchDriverInterface $termCounts = []; foreach ($results as $doc) { $data = $doc->getData(); - $searchname = $data['searchname'] ?? ''; + $searchname = ''; + foreach ($fields as $field) { + if (! empty($data[$field])) { + $searchname = (string) $data[$field]; + break; + } + } // Extract words from searchname - $words = preg_split('/[\s.\-_]+/', strtolower($searchname)); + $words = preg_split('/[\s.\-_]+/', strtolower($searchname)) ?: []; foreach ($words as $word) { if (strlen($word) >= 3 && $word !== strtolower($query)) { // Check if word is similar to query (within edit distance) @@ -2524,9 +2458,15 @@ class ManticoreSearchDriver implements SearchDriverInterface if ($searchString === '') { return ['ids' => [], 'total' => 0, 'fuzzy' => false]; } + $profile = ManticoreIndexRegistry::profile('releases'); + $distance = min( + (int) ($this->getFuzzyConfig()['max_distance'] ?? 2), + $profile['fuzzy_distance'] + ); $query->search($searchString) ->option('fuzzy', true) - ->option('distance', (int) ($this->getFuzzyConfig()['max_distance'] ?? 2)); + ->option('distance', $distance) + ->option('field_weights', $profile['fields']); } else { $searchArray = $this->phrasesToSearchArray($phrases); $terms = []; @@ -2542,7 +2482,10 @@ class ManticoreSearchDriver implements SearchDriverInterface if ($terms === []) { return ['ids' => [], 'total' => 0, 'fuzzy' => false]; } - $query->search(implode(' ', $terms))->option('ranker', 'sph04'); + $profile = ManticoreIndexRegistry::profile('releases'); + $query->search(implode(' ', $terms)) + ->option('ranker', $profile['ranker']) + ->option('field_weights', $profile['fields']); } } @@ -2609,6 +2552,7 @@ class ManticoreSearchDriver implements SearchDriverInterface /** * @param array|string $phrases + * @return array */ private function phrasesToSearchArray(array|string $phrases): array { diff --git a/app/Services/Search/Support/ManticoreIndexRegistry.php b/app/Services/Search/Support/ManticoreIndexRegistry.php new file mode 100644 index 000000000..9b89f5808 --- /dev/null +++ b/app/Services/Search/Support/ManticoreIndexRegistry.php @@ -0,0 +1,133 @@ +, columns: array>}> + */ + public static function definitions(): array + { + $settings = self::fullTextSettings(); + + return [ + 'releases' => ['settings' => $settings, 'columns' => [ + 'name' => ['type' => 'text'], 'searchname' => ['type' => 'text'], + 'fromname' => ['type' => 'text'], 'filename' => ['type' => 'text'], + 'categories_id' => ['type' => 'integer'], 'imdbid' => ['type' => 'string'], + 'tmdbid' => ['type' => 'integer'], 'traktid' => ['type' => 'integer'], + 'tvdb' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'], + 'tvrage' => ['type' => 'integer'], 'videos_id' => ['type' => 'integer'], + 'movieinfo_id' => ['type' => 'integer'], 'size' => ['type' => 'bigint'], + 'postdate_ts' => ['type' => 'bigint'], 'adddate_ts' => ['type' => 'bigint'], + 'totalpart' => ['type' => 'integer'], 'grabs' => ['type' => 'integer'], + 'passwordstatus' => ['type' => 'bigint'], 'groups_id' => ['type' => 'integer'], + 'nzbstatus' => ['type' => 'integer'], 'haspreview' => ['type' => 'bigint'], + ]], + 'predb' => ['settings' => $settings, 'columns' => [ + 'title' => ['type' => 'text', 'attribute' => true], + 'filename' => ['type' => 'text', 'attribute' => true], + 'source' => ['type' => 'string', 'attribute' => true], + ]], + 'movies' => ['settings' => $settings, 'columns' => [ + 'imdbid' => ['type' => 'string'], 'tmdbid' => ['type' => 'integer'], + 'traktid' => ['type' => 'integer'], 'title' => ['type' => 'text'], + 'year' => ['type' => 'text'], 'genre' => ['type' => 'text'], + 'actors' => ['type' => 'text'], 'director' => ['type' => 'text'], + 'rating' => ['type' => 'text'], 'plot' => ['type' => 'text'], + ]], + 'tvshows' => ['settings' => $settings, 'columns' => [ + 'title' => ['type' => 'text'], 'tvdb' => ['type' => 'integer'], + 'trakt' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'], + 'tvrage' => ['type' => 'integer'], 'imdb' => ['type' => 'string'], + 'tmdb' => ['type' => 'integer'], 'started' => ['type' => 'text'], + 'type' => ['type' => 'integer'], + ]], + ...self::secondaryDefinitions($settings), + ]; + } + + /** @return array{ranker: string, fields: array, fuzzy_distance: int} */ + public static function profile(string $logical): array + { + $fields = match ($logical) { + 'releases' => ['searchname' => 12, 'name' => 8, 'filename' => 5, 'fromname' => 1], + 'predb' => ['title' => 12, 'filename' => 5], + 'movies' => ['title' => 12, 'director' => 5, 'actors' => 3, 'genre' => 2, 'plot' => 1], + 'tvshows' => ['title' => 12], + 'music' => ['title' => 12, 'artist' => 9], + 'books' => ['title' => 12, 'author' => 9], + 'console' => ['title' => 12], + 'steam' => ['name' => 12], + 'games', 'anime' => ['title' => 12], + default => [], + }; + + return ['ranker' => 'sph04', 'fields' => $fields, 'fuzzy_distance' => 2]; + } + + /** @return list */ + public static function autocompleteFields(string $logical): array + { + return match ($logical) { + 'releases' => ['searchname', 'name'], + 'predb' => ['title', 'filename'], + 'movies', 'tvshows', 'music', 'books', 'games', 'console', 'anime' => ['title'], + 'steam' => ['name'], + default => [], + }; + } + + /** @param array $configuredIndexes */ + public static function logicalName(string $table, array $configuredIndexes = []): ?string + { + foreach ($configuredIndexes as $logical => $configuredTable) { + if ($configuredTable === $table) { + return (string) $logical; + } + } + + return str_ends_with($table, '_rt') ? array_search($table, self::defaultTableNames(), true) ?: null : null; + } + + /** @return array */ + public static function defaultTableNames(): array + { + $names = []; + foreach (array_keys(self::definitions()) as $logical) { + $names[$logical] = $logical.'_rt'; + } + + return $names; + } + + /** @return array */ + private static function fullTextSettings(): array + { + return [ + 'min_prefix_len' => 0, + 'min_infix_len' => 2, + 'exact_words' => 1, + 'index_field_lengths' => 1, + ]; + } + + /** + * @param array $settings + * @return array, columns: array>}> + */ + private static function secondaryDefinitions(array $settings): array + { + $definitions = []; + foreach (SecondarySearchIndex::cases() as $index) { + $definitions[$index->value] = ['settings' => $settings, 'columns' => $index->manticoreColumns()]; + } + + return $definitions; + } +} diff --git a/app/Services/Search/Support/ManticoreSchemaInspector.php b/app/Services/Search/Support/ManticoreSchemaInspector.php new file mode 100644 index 000000000..a99657eb1 --- /dev/null +++ b/app/Services/Search/Support/ManticoreSchemaInspector.php @@ -0,0 +1,52 @@ + $expected + * @return list + */ + public static function missingSettings(mixed $actual, array $expected): array + { + $encoded = strtolower((string) json_encode($actual)); + + return array_values(array_filter(array_keys($expected), static function (string $setting) use ($encoded, $expected): bool { + return ! str_contains($encoded, strtolower($setting)) + || ! str_contains($encoded, strtolower((string) $expected[$setting])); + })); + } + + /** + * @param array> $actual + * @param array> $expected + * @return array{missing: list, incompatible: array} + */ + public static function compareColumns(array $actual, array $expected): array + { + $types = []; + foreach ($actual as $column) { + $name = (string) ($column['Field'] ?? $column['field'] ?? ''); + $type = strtolower((string) ($column['Type'] ?? $column['type'] ?? '')); + if ($name !== '') { + $types[$name] = $type; + } + } + + $missing = []; + $incompatible = []; + foreach ($expected as $name => $definition) { + $expectedType = strtolower((string) $definition['type']); + if (! isset($types[$name])) { + $missing[] = $name; + } elseif (! str_contains($types[$name], $expectedType)) { + $incompatible[$name] = ['expected' => $expectedType, 'actual' => $types[$name]]; + } + } + + return ['missing' => $missing, 'incompatible' => $incompatible]; + } +} diff --git a/docker-compose.yml.prod-dist b/docker-compose.yml.prod-dist index 685ac1e97..473bac033 100644 --- a/docker-compose.yml.prod-dist +++ b/docker-compose.yml.prod-dist @@ -111,7 +111,7 @@ services: retries: 3 timeout: 5s manticore: - image: manticoresearch/manticore + image: manticoresearch/manticore:28.4.4 profiles: - manticore environment: @@ -130,7 +130,7 @@ services: soft: -1 hard: -1 volumes: - - 'manticore:/var/lib/manticore' + - 'manticore:/var/lib/manticore/data' - ./config/manticore.conf:/etc/manticoresearch/manticore.conf networks: - nntmux diff --git a/docs/adr/0001-manticore-lexical-search.md b/docs/adr/0001-manticore-lexical-search.md new file mode 100644 index 000000000..a1fe21166 --- /dev/null +++ b/docs/adr/0001-manticore-lexical-search.md @@ -0,0 +1,21 @@ +# ADR 0001: Manticore lexical search and maintenance rebuilds + +## Status + +Accepted + +## Context + +NNTmux searches release and metadata tables through Manticore. Search must remain predictable for Newznab clients, tolerate release-name punctuation and misspellings, and remain operable without embedding services. Existing release browsing is newest-first and callers rely on that ordering. + +## Decision + +- Pin Manticore Search 28.4.4 and keep the official PHP client at 4.0.1. +- Use one table registry for schemas and per-table lexical field weights. +- Retain `sph04`, quoted phrases, exclusions, exact-word indexing, and exact-first/fuzzy-second retrieval. +- Keep release results ordered by the requested attribute, defaulting to `postdate_ts DESC, id DESC`; lexical score controls matching rather than presentation order. +- Rebuild incompatible tables in a maintenance window. Semantic, conversational, vector, and external-embedding search are deferred. + +## Consequences + +The deployment stays lightweight and backward compatible, and relevance changes can be measured with a stable fixture. Ranking cannot rescue an older result from date-first presentation. Settings that affect indexed tokens require a complete repopulation and a backup-based rollback. diff --git a/docs/manticore-28-upgrade.md b/docs/manticore-28-upgrade.md new file mode 100644 index 000000000..c6ed647df --- /dev/null +++ b/docs/manticore-28-upgrade.md @@ -0,0 +1,25 @@ +# Manticore Search 28.4.4 maintenance upgrade + +This upgrade rebuilds every configured Manticore table. Do not reuse data files written by 28.4.4 with an older daemon. + +## Before maintenance + +1. Set `SEARCH_INDEX_GENERATION` to a new value so cached results and cursors cannot cross the cutover. +2. Run `php artisan manticore:inspect --json` and save its output. +3. Stop the processing engine with `php artisan tmux:stop` and pause queue workers that update search documents. +4. Back up `/var/lib/manticore/data` with the container stopped, or use the supported Manticore backup tooling. Record the old image tag with the backup. + +## Upgrade and repopulate + +1. Pull and start `manticoresearch/manticore:28.4.4`. +2. Confirm HTTP connectivity and optional username/password or bearer-token authentication with `php artisan manticore:inspect`. +3. Run `php artisan manticore:create-indexes --drop`. +4. Run `php artisan nntmux:populate --manticore --all` (or explicitly populate every enabled releases, PreDB, movie, TV, and secondary metadata table). +5. Run `php artisan manticore:inspect`, index reconciliation, and representative searches from `tests/Fixtures/Search/manticore-relevance-v1.json`. +6. Resume queue workers and run `php artisan tmux:start` only after all tables report compatible. + +## Rollback + +Stop Manticore, restore the pre-upgrade data backup, and start the exact image version recorded with that backup. Do not point the older image at files modified by 28.4.4. Restore the previous `SEARCH_INDEX_GENERATION` only if application code was also rolled back. + +Local Docker may remain anonymous. Secured deployments use `MANTICORESEARCH_USERNAME` and `MANTICORESEARCH_PASSWORD`, or `MANTICORESEARCH_TOKEN`; never store credentials in Compose files. diff --git a/tests/Fixtures/Search/manticore-relevance-v1.json b/tests/Fixtures/Search/manticore-relevance-v1.json new file mode 100644 index 000000000..b6d699bd9 --- /dev/null +++ b/tests/Fixtures/Search/manticore-relevance-v1.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "thresholds": { + "top_5_precision": 0.8, + "top_10_precision": 0.8, + "max_zero_result_rate": 0.05, + "max_fuzzy_fallback_rate": 0.25, + "max_p95_latency_ms": 250 + }, + "queries": [ + {"index": "releases", "query": "The.Matrix.1999.1080p", "expected_terms": ["matrix", "1999"], "fuzzy": false}, + {"index": "releases", "query": "\"breaking bad\" -german", "expected_terms": ["breaking bad"], "excluded_terms": ["german"], "fuzzy": false}, + {"index": "releases", "query": "Oppenhiemer 2023", "expected_terms": ["oppenheimer", "2023"], "fuzzy": true}, + {"index": "predb", "query": "Ubuntu.24.04-LTS", "expected_terms": ["ubuntu", "24.04"], "fuzzy": false}, + {"index": "movies", "query": "Blade Runner 2049", "expected_terms": ["blade runner", "2049"], "fuzzy": false}, + {"index": "tvshows", "query": "Better Call Saul", "expected_terms": ["better call saul"], "fuzzy": false}, + {"index": "music", "query": "Daft Punk Discovery", "expected_terms": ["daft punk", "discovery"], "fuzzy": false}, + {"index": "books", "query": "Frank Herbert Dune", "expected_terms": ["frank herbert", "dune"], "fuzzy": false}, + {"index": "games", "query": "Baldur's Gate 3", "expected_terms": ["baldur", "gate"], "fuzzy": false}, + {"index": "console", "query": "Zelda Tears of the Kingdom", "expected_terms": ["zelda", "kingdom"], "fuzzy": false}, + {"index": "steam", "query": "Cyberpunk 2077", "expected_terms": ["cyberpunk", "2077"], "fuzzy": false}, + {"index": "anime", "query": "Fullmetal Alchemist Brotherhood", "expected_terms": ["fullmetal alchemist"], "fuzzy": false} + ] +} diff --git a/tests/Unit/ManticoreIndexRegistryTest.php b/tests/Unit/ManticoreIndexRegistryTest.php new file mode 100644 index 000000000..9006f60ae --- /dev/null +++ b/tests/Unit/ManticoreIndexRegistryTest.php @@ -0,0 +1,68 @@ + 'passwordstatus', 'Type' => 'uint']], + ['passwordstatus' => ['type' => 'bigint'], 'haspreview' => ['type' => 'bigint']] + ); + + self::assertSame(['haspreview'], $result['missing']); + self::assertSame('bigint', $result['incompatible']['passwordstatus']['expected']); + } + + #[Test] + public function benchmark_fixture_is_versioned_and_covers_every_table(): void + { + $fixture = json_decode((string) file_get_contents(__DIR__.'/../Fixtures/Search/manticore-relevance-v1.json'), true, flags: JSON_THROW_ON_ERROR); + + self::assertSame(1, $fixture['version']); + self::assertSame(array_keys(ManticoreIndexRegistry::definitions()), array_values(array_unique(array_column($fixture['queries'], 'index')))); + self::assertArrayHasKey('top_5_precision', $fixture['thresholds']); + } +}