starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update manticore indexes

Browse Source
This commit is contained in:
DariusIII
2026-07-15 14:22:31 +02:00
parent a37f3bd3c8
commit 9643d4dd0b
10 changed files with 513 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/CreateManticoreIndexes.php
+26 -204
View File
@@ -4,263 +4,85 @@ declare(strict_types=1);
namespace App\Console\Commands;
use App\Enums\SecondarySearchIndex;
use App\Services\Search\Support\ManticoreClientFactory;
use App\Services\Search\Support\ManticoreIndexRegistry;
use Illuminate\Console\Command;
use Manticoresearch\Client;
use Manticoresearch\Exceptions\ResponseException;
class CreateManticoreIndexes extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'manticore:create-indexes
{--drop : Drop existing indexes before creating}';
protected $signature = 'manticore:create-indexes {--drop : Drop existing indexes before creating}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Create Manticore Search indexes based on configuration';
protected Client $client;
/**
* Execute the console command.
*/
public function handle(): int
{
$this->info('Creating Manticore Search indexes...');
$dropExisting = $this->option('drop');
$this->client = ManticoreClientFactory::make(config('search.drivers.manticore', config('manticoresearch', [])));
// We'll skip checking for data_dir this way since it may not be accessible via API
// but instead provide better error handling during index creation
// If you encounter data_dir errors, ensure it's properly set in config/manticore.conf:
// data_dir = /path/to/data
// And make sure the path exists and has proper permissions
$this->client = ManticoreClientFactory::make(config('search.drivers.manticore', []));
try {
$this->client->nodes()->status();
} catch (\Exception $e) {
} catch (\Throwable $e) {
$this->error('Failed to connect to Manticore Search: '.$e->getMessage());
$this->info('Please check if Manticore Search is running and properly configured.');
$this->info('Check the configured host, authentication, and whether Manticore is running.');
return 1;
return self::FAILURE;
}
// Define indexes and their schema
$indexes = [
'releases_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'name' => ['type' => 'text'],
'searchname' => ['type' => 'text'],
'fromname' => ['type' => 'text'],
'filename' => ['type' => 'text'],
'categories_id' => ['type' => 'integer'],
// External media IDs for efficient searching
'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'],
'traktid' => ['type' => 'integer'],
'tvdb' => ['type' => 'integer'],
'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'],
'videos_id' => ['type' => 'integer'],
'movieinfo_id' => ['type' => 'integer'],
'size' => ['type' => 'bigint'],
'postdate_ts' => ['type' => 'bigint'],
'adddate_ts' => ['type' => 'bigint'],
'totalpart' => ['type' => 'integer'],
'grabs' => ['type' => 'integer'],
'passwordstatus' => ['type' => 'bigint'],
'groups_id' => ['type' => 'integer'],
'nzbstatus' => ['type' => 'integer'],
'haspreview' => ['type' => 'bigint'],
],
],
'predb_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'title' => ['type' => 'text', 'attribute' => true],
'filename' => ['type' => 'text', 'attribute' => true],
'source' => ['type' => 'string', 'attribute' => true],
],
],
'movies_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'],
'traktid' => ['type' => 'integer'],
'title' => ['type' => 'text'],
'year' => ['type' => 'text'],
'genre' => ['type' => 'text'],
'actors' => ['type' => 'text'],
'director' => ['type' => 'text'],
'rating' => ['type' => 'text'],
'plot' => ['type' => 'text'],
],
],
'tvshows_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'title' => ['type' => 'text'],
'tvdb' => ['type' => 'integer'],
'trakt' => ['type' => 'integer'],
'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'],
'imdb' => ['type' => 'string'],
'tmdb' => ['type' => 'integer'],
'started' => ['type' => 'text'],
'type' => ['type' => 'integer'],
],
],
'music_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Music->manticoreColumns(),
],
'books_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Books->manticoreColumns(),
],
'games_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Games->manticoreColumns(),
],
'console_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Console->manticoreColumns(),
],
'steam_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Steam->manticoreColumns(),
],
'anime_rt' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => SecondarySearchIndex::Anime->manticoreColumns(),
],
];
$configuredNames = config('search.drivers.manticore.indexes', []);
$hasErrors = false;
// Create each index
foreach ($indexes as $indexName => $schema) {
if (! $this->createIndex($indexName, $schema, $dropExisting)) {
foreach (ManticoreIndexRegistry::definitions() as $logical => $schema) {
$indexName = (string) ($configuredNames[$logical] ?? $logical.'_rt');
if (! $this->createIndex($indexName, $schema, (bool) $this->option('drop'))) {
$hasErrors = true;
}
}
if ($hasErrors) {
$this->error('Some errors occurred during index creation.');
$this->error('Some Manticore tables could not be created.');
return 1;
return self::FAILURE;
}
$this->info('All Manticore Search indexes created successfully!');
$this->info('All Manticore Search tables are ready.');
return 0;
return self::SUCCESS;
}
/**
* Create a single index with error handling.
*
* @param array<string, mixed> $schema
*/
/** @param array<string, mixed> $schema */
protected function createIndex(string $indexName, array $schema, bool $dropExisting): bool
{
$this->info("Creating {$indexName} index...");
$indices = $this->client->tables();
$tables = $this->client->tables();
try {
// Optionally drop existing index
if ($dropExisting) {
try {
$this->info("Dropping existing {$indexName} index...");
$indices->drop(['index' => $indexName, 'body' => ['silent' => true]]);
$this->info("Successfully dropped {$indexName} index.");
} catch (ResponseException $e) {
if (! str_contains($e->getMessage(), 'unknown index')) {
$this->warn("Warning when dropping {$indexName} index: ".$e->getMessage());
}
}
$this->info("Dropping {$indexName}...");
$tables->drop(['index' => $indexName, 'body' => ['silent' => true]]);
}
// Instead of checking if index exists (which doesn't work),
// try to create it directly and handle any errors
// that might occur if it already exists
$response = $indices->create([
'index' => $indexName,
'body' => $schema,
]);
$this->info("Successfully created {$indexName} index.");
$this->line('Response: '.json_encode($response, JSON_PRETTY_PRINT));
$tables->create(['index' => $indexName, 'body' => $schema]);
$this->info("Created {$indexName}.");
return true;
} catch (ResponseException $e) {
// Check if the error is because the index already exists
if (str_contains($e->getMessage(), 'already exists')) {
$this->warn("Index {$indexName} already exists. Use --drop option to recreate it.");
$this->warn("{$indexName} already exists; use --drop during a maintenance rebuild.");
return true;
}
// Check for data_dir configuration error
if (str_contains($e->getMessage(), 'data_dir')) {
$this->error("Failed to create {$indexName} index: ".$e->getMessage());
$this->newLine();
$this->warn('ManticoreSearch requires data_dir to be set in its configuration file.');
$this->info('To fix this, edit your ManticoreSearch config file (usually /etc/manticoresearch/manticore.conf):');
$this->line(' 1. Add or uncomment: data_dir = /var/lib/manticore');
$this->line(' 2. Ensure the directory exists and is writable by the manticore user');
$this->line(' 3. Restart ManticoreSearch: sudo systemctl restart manticore');
return false;
$this->error("Failed to create {$indexName}: configure a writable Manticore data_dir.");
} else {
$this->error("Failed to create {$indexName}: ".$e->getMessage());
}
$this->error("Failed to create {$indexName} index: ".$e->getMessage());
return false;
} catch (\Exception $e) {
$this->error("Failed to create {$indexName} index: ".$e->getMessage());
} catch (\Throwable $e) {
$this->error("Failed to create {$indexName}: ".$e->getMessage());
return false;
}
app/Console/Commands/InspectManticore.php
+75
View File
@@ -0,0 +1,75 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Services\Search\Support\ManticoreClientFactory;
use App\Services\Search\Support\ManticoreIndexRegistry;
use App\Services\Search\Support\ManticoreSchemaInspector;
use Illuminate\Console\Command;
final class InspectManticore extends Command
{
protected $signature = 'manticore:inspect {--json : Emit machine-readable JSON}';
protected $description = 'Report Manticore version and table schema compatibility';
public function handle(): int
{
try {
$client = ManticoreClientFactory::make(config('search.drivers.manticore', []));
$versionResponse = $client->sql('SELECT VERSION() AS version', true);
} catch (\Throwable $e) {
$this->error('Unable to connect to Manticore: '.$e->getMessage());
return self::FAILURE;
}
$report = ['version' => self::extractVersion($versionResponse), 'compatible' => true, 'tables' => []];
$configured = config('search.drivers.manticore.indexes', []);
foreach (ManticoreIndexRegistry::definitions() as $logical => $definition) {
$table = (string) ($configured[$logical] ?? $logical.'_rt');
try {
if (preg_match('/^[a-zA-Z0-9_]+$/', $table) !== 1) {
throw new \RuntimeException('Unsafe configured table name');
}
$actual = $client->table($table)->describe();
$comparison = ManticoreSchemaInspector::compareColumns(is_array($actual) ? $actual : [], $definition['columns']);
$actualSettings = $client->sql("SHOW TABLE {$table} SETTINGS", true);
$missingSettings = ManticoreSchemaInspector::missingSettings($actualSettings, $definition['settings']);
$needsRebuild = $comparison['missing'] !== [] || $comparison['incompatible'] !== [] || $missingSettings !== [];
$report['tables'][$table] = [...$comparison, 'missing_settings' => $missingSettings, 'needs_rebuild' => $needsRebuild];
$report['compatible'] = $report['compatible'] && ! $needsRebuild;
} catch (\Throwable $e) {
$report['tables'][$table] = ['missing_table' => true, 'needs_rebuild' => true, 'error' => $e->getMessage()];
$report['compatible'] = false;
}
}
if ($this->option('json')) {
$this->line((string) json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
} else {
$this->info('Manticore version: '.$report['version']);
foreach ($report['tables'] as $table => $status) {
$this->line(sprintf('%-20s %s', $table, $status['needs_rebuild'] ? 'REBUILD REQUIRED' : 'compatible'));
}
if (! $report['compatible']) {
$this->warn('Run manticore:create-indexes --drop during a maintenance window, then repopulate all enabled indexes.');
}
}
return $report['compatible'] ? self::SUCCESS : self::FAILURE;
}
private static function extractVersion(mixed $response): string
{
$encoded = json_encode($response);
if (is_string($encoded) && preg_match('/\d+\.\d+(?:\.\d+)?/', $encoded, $match) === 1) {
return $match[0];
}
return 'unknown';
}
}
app/Services/Search/Drivers/ManticoreSearchDriver.php
+87 -143
View File
@@ -18,6 +18,7 @@ use App\Services\Search\Contracts\SearchDriverInterface;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use App\Services\Search\Support\ManticoreClientFactory;
use App\Services\Search\Support\ManticoreIndexRegistry;
use App\Support\ReleaseSearchIndexDocument;
use App\Support\SecondaryIndexDocuments;
use Illuminate\Support\Facades\Cache;
@@ -856,123 +857,17 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
try {
$indices = $this->manticoreSearch->tables();
$logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []);
$definition = $logical === null ? null : (ManticoreIndexRegistry::definitions()[$logical] ?? null);
if ($index === 'releases_rt') {
$indices->create([
'index' => $index,
'body' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'name' => ['type' => 'text'],
'searchname' => ['type' => 'text'],
'fromname' => ['type' => 'text'],
'filename' => ['type' => 'text'],
'categories_id' => ['type' => 'integer'],
'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'],
'traktid' => ['type' => 'integer'],
'tvdb' => ['type' => 'integer'],
'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'],
'videos_id' => ['type' => 'integer'],
'movieinfo_id' => ['type' => 'integer'],
'size' => ['type' => 'bigint'],
'postdate_ts' => ['type' => 'bigint'],
'adddate_ts' => ['type' => 'bigint'],
'totalpart' => ['type' => 'integer'],
'grabs' => ['type' => 'integer'],
'passwordstatus' => ['type' => 'bigint'],
'groups_id' => ['type' => 'integer'],
'nzbstatus' => ['type' => 'integer'],
'haspreview' => ['type' => 'bigint'],
],
],
]);
cli()->info('Created releases_rt index with external ID fields and infix search support');
} elseif ($index === 'predb_rt') {
$indices->create([
'index' => $index,
'body' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'title' => ['type' => 'text'],
'filename' => ['type' => 'text'],
'source' => ['type' => 'text'],
],
],
]);
cli()->info('Created predb_rt index with infix search support');
} elseif ($index === 'movies_rt') {
$indices->create([
'index' => $index,
'body' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'],
'traktid' => ['type' => 'integer'],
'title' => ['type' => 'text'],
'year' => ['type' => 'text'],
'genre' => ['type' => 'text'],
'actors' => ['type' => 'text'],
'director' => ['type' => 'text'],
'rating' => ['type' => 'text'],
'plot' => ['type' => 'text'],
],
],
]);
cli()->info('Created movies_rt index with infix search support');
} elseif ($index === 'tvshows_rt') {
$indices->create([
'index' => $index,
'body' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => [
'title' => ['type' => 'text'],
'tvdb' => ['type' => 'integer'],
'trakt' => ['type' => 'integer'],
'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'],
'imdb' => ['type' => 'string'],
'tmdb' => ['type' => 'integer'],
'started' => ['type' => 'text'],
'type' => ['type' => 'integer'],
],
],
]);
cli()->info('Created tvshows_rt index with infix search support');
} else {
foreach (SecondarySearchIndex::cases() as $secondary) {
if ($index === $this->getSecondaryIndexName($secondary)) {
$indices->create([
'index' => $index,
'body' => [
'settings' => [
'min_prefix_len' => 0,
'min_infix_len' => 2,
],
'columns' => $secondary->manticoreColumns(),
],
]);
cli()->info("Created {$index} secondary metadata index");
if ($definition === null) {
cli()->error("No Manticore schema is registered for {$index}");
break;
}
}
return;
}
$this->manticoreSearch->tables()->create(['index' => $index, 'body' => $definition]);
cli()->info("Created {$index} with the configured Manticore schema");
} catch (\Throwable $e) {
if (! str_contains($e->getMessage(), 'already exists')) {
cli()->error('Error creating index '.$index.': '.$e->getMessage());
@@ -1155,11 +1050,14 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$normalizedLimit = $this->normalizeSearchLimit($limit);
$logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []);
$profile = ManticoreIndexRegistry::profile($logical ?? '');
$fuzzyConfig = $this->getFuzzyConfig();
$distance = $fuzzyConfig['max_distance'] ?? 2;
$distance = min((int) ($fuzzyConfig['max_distance'] ?? 2), $profile['fuzzy_distance']);
// Create cache key for fuzzy search results
$cacheKey = 'manticore:fuzzy:'.md5(serialize([
'generation' => config('search.index_generation', '1'),
'index' => $index,
'array' => $searchArray,
'limit' => $normalizedLimit,
@@ -1215,6 +1113,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
->search($searchExpr)
->option('fuzzy', true)
->option('distance', $distance)
->option('field_weights', $profile['fields'])
->limit($normalizedLimit);
$results = $query->get();
@@ -1308,6 +1207,8 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$normalizedLimit = $this->normalizeSearchLimit($limit);
$logical = ManticoreIndexRegistry::logicalName($rt_index, $this->config['indexes'] ?? []);
$profile = ManticoreIndexRegistry::profile($logical ?? '');
if (config('app.debug')) {
Log::debug('ManticoreSearch::searchIndexes called', [
@@ -1321,11 +1222,13 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Create cache key for search results
$cacheKey = md5(serialize([
'generation' => config('search.index_generation', '1'),
'index' => $rt_index,
'search' => $searchString,
'columns' => $column,
'array' => $searchArray,
'limit' => $normalizedLimit,
'profile' => $profile,
]));
$cached = Cache::get($cacheKey);
@@ -1400,7 +1303,8 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Use a fresh Search instance for every query to avoid parameter accumulation across calls
$query = (new Search($this->manticoreSearch))
->setTable($rt_index)
->option('ranker', 'sph04')
->option('ranker', $profile['ranker'])
->option('field_weights', $profile['fields'])
->maxMatches($normalizedLimit)
->limit($normalizedLimit)
->stripBadUtf8(true)
@@ -1417,7 +1321,8 @@ class ManticoreSearchDriver implements SearchDriverInterface
try {
$query = (new Search($this->manticoreSearch))
->setTable($rt_index)
->option('ranker', 'sph04')
->option('ranker', $profile['ranker'])
->option('field_weights', $profile['fields'])
->maxMatches($normalizedLimit)
->limit($normalizedLimit)
->stripBadUtf8(true)
@@ -1516,7 +1421,12 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt');
$cacheKey = 'manticore:autocomplete:'.md5($index.$query);
$logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []);
$fields = ManticoreIndexRegistry::autocompleteFields($logical ?? '');
if ($fields === []) {
return [];
}
$cacheKey = 'manticore:autocomplete:'.config('search.index_generation', '1').':'.md5($index.$query);
$cached = Cache::get($cacheKey);
if ($cached !== null) {
@@ -1533,8 +1443,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
return [];
}
// Use relaxed search on searchname field
$searchExpr = '@@relaxed @searchname '.$escapedQuery;
$searchExpr = '@@relaxed @('.implode(',', $fields).') '.$escapedQuery.'*';
$search = (new Search($this->manticoreSearch))
->setTable($index)
@@ -1545,10 +1454,16 @@ class ManticoreSearchDriver implements SearchDriverInterface
$results = $search->get();
$seen = [];
$ranked = [];
foreach ($results as $doc) {
$data = $doc->getData();
$searchname = $data['searchname'] ?? '';
$searchname = '';
foreach ($fields as $field) {
if (! empty($data[$field])) {
$searchname = (string) $data[$field];
break;
}
}
if (empty($searchname)) {
continue;
@@ -1556,20 +1471,29 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Create a clean suggestion from the searchname
$suggestion = $this->extractSuggestion($searchname, $query);
if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) {
$seen[strtolower($suggestion)] = true;
$suggestions[] = [
'suggest' => $suggestion,
'distance' => 0,
'docs' => 1,
];
if ($suggestion === null || $suggestion === '') {
continue;
}
if (count($suggestions) >= $maxResults) {
break;
$normalized = mb_strtolower(trim((string) preg_replace('/\s+/', ' ', $suggestion)));
if ($normalized !== '') {
if (! isset($ranked[$normalized])) {
$ranked[$normalized] = [
'suggest' => $suggestion,
'distance' => 0,
'docs' => 0,
'prefix' => str_starts_with($normalized, mb_strtolower($query)) ? 1 : 0,
];
}
$ranked[$normalized]['docs']++;
}
}
uasort($ranked, static fn (array $a, array $b): int => [$b['prefix'], $b['docs']] <=> [$a['prefix'], $a['docs']]);
foreach (array_slice($ranked, 0, $maxResults) as $item) {
unset($item['prefix']);
$suggestions[] = $item;
}
} catch (\Throwable $e) {
if (config('app.debug')) {
Log::warning('ManticoreSearch autocomplete error: '.$e->getMessage());
@@ -1594,13 +1518,13 @@ class ManticoreSearchDriver implements SearchDriverInterface
private function extractSuggestion(string $searchname, string $query): ?string
{
// Clean up the searchname - remove file extensions, quality tags at the end
$clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname);
$clean = (string) preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname);
// Replace dots and underscores with spaces for readability
$clean = str_replace(['.', '_'], ' ', $clean);
// Remove multiple spaces
$clean = preg_replace('/\s+/', ' ', $clean);
$clean = (string) preg_replace('/\s+/', ' ', $clean);
$clean = trim($clean);
if (empty($clean)) {
@@ -1622,9 +1546,9 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Clean up - don't cut mid-word
if ($start > 0) {
$extracted = preg_replace('/^\S*\s/', '', $extracted);
$extracted = (string) preg_replace('/^\S*\s/', '', $extracted);
}
$extracted = preg_replace('/\s\S*$/', '', $extracted);
$extracted = (string) preg_replace('/\s\S*$/', '', $extracted);
return trim($extracted);
}
@@ -1664,7 +1588,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
$index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt');
$cacheKey = 'manticore:suggest:'.md5($index.$query);
$cacheKey = 'manticore:suggest:'.config('search.index_generation', '1').':'.md5($index.$query);
$cached = Cache::get($cacheKey);
if ($cached !== null) {
@@ -1730,8 +1654,12 @@ class ManticoreSearchDriver implements SearchDriverInterface
return [];
}
// Use relaxed search to find partial matches
$searchExpr = '@@relaxed @searchname '.$escapedQuery;
$logical = ManticoreIndexRegistry::logicalName($index, $this->config['indexes'] ?? []);
$fields = ManticoreIndexRegistry::autocompleteFields($logical ?? '');
if ($fields === []) {
return [];
}
$searchExpr = '@@relaxed @('.implode(',', $fields).') '.$escapedQuery;
$search = (new Search($this->manticoreSearch))
->setTable($index)
@@ -1745,10 +1673,16 @@ class ManticoreSearchDriver implements SearchDriverInterface
$termCounts = [];
foreach ($results as $doc) {
$data = $doc->getData();
$searchname = $data['searchname'] ?? '';
$searchname = '';
foreach ($fields as $field) {
if (! empty($data[$field])) {
$searchname = (string) $data[$field];
break;
}
}
// Extract words from searchname
$words = preg_split('/[\s.\-_]+/', strtolower($searchname));
$words = preg_split('/[\s.\-_]+/', strtolower($searchname)) ?: [];
foreach ($words as $word) {
if (strlen($word) >= 3 && $word !== strtolower($query)) {
// Check if word is similar to query (within edit distance)
@@ -2524,9 +2458,15 @@ class ManticoreSearchDriver implements SearchDriverInterface
if ($searchString === '') {
return ['ids' => [], 'total' => 0, 'fuzzy' => false];
}
$profile = ManticoreIndexRegistry::profile('releases');
$distance = min(
(int) ($this->getFuzzyConfig()['max_distance'] ?? 2),
$profile['fuzzy_distance']
);
$query->search($searchString)
->option('fuzzy', true)
->option('distance', (int) ($this->getFuzzyConfig()['max_distance'] ?? 2));
->option('distance', $distance)
->option('field_weights', $profile['fields']);
} else {
$searchArray = $this->phrasesToSearchArray($phrases);
$terms = [];
@@ -2542,7 +2482,10 @@ class ManticoreSearchDriver implements SearchDriverInterface
if ($terms === []) {
return ['ids' => [], 'total' => 0, 'fuzzy' => false];
}
$query->search(implode(' ', $terms))->option('ranker', 'sph04');
$profile = ManticoreIndexRegistry::profile('releases');
$query->search(implode(' ', $terms))
->option('ranker', $profile['ranker'])
->option('field_weights', $profile['fields']);
}
}
@@ -2609,6 +2552,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
/**
* @param array<string, mixed>|string $phrases
* @return array<string, mixed>
*/
private function phrasesToSearchArray(array|string $phrases): array
{
app/Services/Search/Support/ManticoreIndexRegistry.php
+133
View File
@@ -0,0 +1,133 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\Support;
use App\Enums\SecondarySearchIndex;
final class ManticoreIndexRegistry
{
/**
* @return array<string, array{settings: array<string, int|string>, columns: array<string, array<string, mixed>>}>
*/
public static function definitions(): array
{
$settings = self::fullTextSettings();
return [
'releases' => ['settings' => $settings, 'columns' => [
'name' => ['type' => 'text'], 'searchname' => ['type' => 'text'],
'fromname' => ['type' => 'text'], 'filename' => ['type' => 'text'],
'categories_id' => ['type' => 'integer'], 'imdbid' => ['type' => 'string'],
'tmdbid' => ['type' => 'integer'], 'traktid' => ['type' => 'integer'],
'tvdb' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'], 'videos_id' => ['type' => 'integer'],
'movieinfo_id' => ['type' => 'integer'], 'size' => ['type' => 'bigint'],
'postdate_ts' => ['type' => 'bigint'], 'adddate_ts' => ['type' => 'bigint'],
'totalpart' => ['type' => 'integer'], 'grabs' => ['type' => 'integer'],
'passwordstatus' => ['type' => 'bigint'], 'groups_id' => ['type' => 'integer'],
'nzbstatus' => ['type' => 'integer'], 'haspreview' => ['type' => 'bigint'],
]],
'predb' => ['settings' => $settings, 'columns' => [
'title' => ['type' => 'text', 'attribute' => true],
'filename' => ['type' => 'text', 'attribute' => true],
'source' => ['type' => 'string', 'attribute' => true],
]],
'movies' => ['settings' => $settings, 'columns' => [
'imdbid' => ['type' => 'string'], 'tmdbid' => ['type' => 'integer'],
'traktid' => ['type' => 'integer'], 'title' => ['type' => 'text'],
'year' => ['type' => 'text'], 'genre' => ['type' => 'text'],
'actors' => ['type' => 'text'], 'director' => ['type' => 'text'],
'rating' => ['type' => 'text'], 'plot' => ['type' => 'text'],
]],
'tvshows' => ['settings' => $settings, 'columns' => [
'title' => ['type' => 'text'], 'tvdb' => ['type' => 'integer'],
'trakt' => ['type' => 'integer'], 'tvmaze' => ['type' => 'integer'],
'tvrage' => ['type' => 'integer'], 'imdb' => ['type' => 'string'],
'tmdb' => ['type' => 'integer'], 'started' => ['type' => 'text'],
'type' => ['type' => 'integer'],
]],
...self::secondaryDefinitions($settings),
];
}
/** @return array{ranker: string, fields: array<string, int>, fuzzy_distance: int} */
public static function profile(string $logical): array
{
$fields = match ($logical) {
'releases' => ['searchname' => 12, 'name' => 8, 'filename' => 5, 'fromname' => 1],
'predb' => ['title' => 12, 'filename' => 5],
'movies' => ['title' => 12, 'director' => 5, 'actors' => 3, 'genre' => 2, 'plot' => 1],
'tvshows' => ['title' => 12],
'music' => ['title' => 12, 'artist' => 9],
'books' => ['title' => 12, 'author' => 9],
'console' => ['title' => 12],
'steam' => ['name' => 12],
'games', 'anime' => ['title' => 12],
default => [],
};
return ['ranker' => 'sph04', 'fields' => $fields, 'fuzzy_distance' => 2];
}
/** @return list<string> */
public static function autocompleteFields(string $logical): array
{
return match ($logical) {
'releases' => ['searchname', 'name'],
'predb' => ['title', 'filename'],
'movies', 'tvshows', 'music', 'books', 'games', 'console', 'anime' => ['title'],
'steam' => ['name'],
default => [],
};
}
/** @param array<string, string> $configuredIndexes */
public static function logicalName(string $table, array $configuredIndexes = []): ?string
{
foreach ($configuredIndexes as $logical => $configuredTable) {
if ($configuredTable === $table) {
return (string) $logical;
}
}
return str_ends_with($table, '_rt') ? array_search($table, self::defaultTableNames(), true) ?: null : null;
}
/** @return array<string, string> */
public static function defaultTableNames(): array
{
$names = [];
foreach (array_keys(self::definitions()) as $logical) {
$names[$logical] = $logical.'_rt';
}
return $names;
}
/** @return array<string, int|string> */
private static function fullTextSettings(): array
{
return [
'min_prefix_len' => 0,
'min_infix_len' => 2,
'exact_words' => 1,
'index_field_lengths' => 1,
];
}
/**
* @param array<string, int|string> $settings
* @return array<string, array{settings: array<string, int|string>, columns: array<string, array<string, mixed>>}>
*/
private static function secondaryDefinitions(array $settings): array
{
$definitions = [];
foreach (SecondarySearchIndex::cases() as $index) {
$definitions[$index->value] = ['settings' => $settings, 'columns' => $index->manticoreColumns()];
}
return $definitions;
}
}
app/Services/Search/Support/ManticoreSchemaInspector.php
+52
View File
@@ -0,0 +1,52 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\Support;
final class ManticoreSchemaInspector
{
/**
* @param array<string, mixed> $expected
* @return list<string>
*/
public static function missingSettings(mixed $actual, array $expected): array
{
$encoded = strtolower((string) json_encode($actual));
return array_values(array_filter(array_keys($expected), static function (string $setting) use ($encoded, $expected): bool {
return ! str_contains($encoded, strtolower($setting))
|| ! str_contains($encoded, strtolower((string) $expected[$setting]));
}));
}
/**
* @param array<int, array<string, mixed>> $actual
* @param array<string, array<string, mixed>> $expected
* @return array{missing: list<string>, incompatible: array<string, array{expected: string, actual: string}>}
*/
public static function compareColumns(array $actual, array $expected): array
{
$types = [];
foreach ($actual as $column) {
$name = (string) ($column['Field'] ?? $column['field'] ?? '');
$type = strtolower((string) ($column['Type'] ?? $column['type'] ?? ''));
if ($name !== '') {
$types[$name] = $type;
}
}
$missing = [];
$incompatible = [];
foreach ($expected as $name => $definition) {
$expectedType = strtolower((string) $definition['type']);
if (! isset($types[$name])) {
$missing[] = $name;
} elseif (! str_contains($types[$name], $expectedType)) {
$incompatible[$name] = ['expected' => $expectedType, 'actual' => $types[$name]];
}
}
return ['missing' => $missing, 'incompatible' => $incompatible];
}
}
docker-compose.yml.prod-dist
+2 -2
View File
@@ -111,7 +111,7 @@ services:
retries: 3
timeout: 5s
manticore:
image: manticoresearch/manticore
image: manticoresearch/manticore:28.4.4
profiles:
- manticore
environment:
@@ -130,7 +130,7 @@ services:
soft: -1
hard: -1
volumes:
- 'manticore:/var/lib/manticore'
- 'manticore:/var/lib/manticore/data'
- ./config/manticore.conf:/etc/manticoresearch/manticore.conf
networks:
- nntmux
docs/adr/0001-manticore-lexical-search.md
+21
View File
@@ -0,0 +1,21 @@
# ADR 0001: Manticore lexical search and maintenance rebuilds
## Status
Accepted
## Context
NNTmux searches release and metadata tables through Manticore. Search must remain predictable for Newznab clients, tolerate release-name punctuation and misspellings, and remain operable without embedding services. Existing release browsing is newest-first and callers rely on that ordering.
## Decision
- Pin Manticore Search 28.4.4 and keep the official PHP client at 4.0.1.
- Use one table registry for schemas and per-table lexical field weights.
- Retain `sph04`, quoted phrases, exclusions, exact-word indexing, and exact-first/fuzzy-second retrieval.
- Keep release results ordered by the requested attribute, defaulting to `postdate_ts DESC, id DESC`; lexical score controls matching rather than presentation order.
- Rebuild incompatible tables in a maintenance window. Semantic, conversational, vector, and external-embedding search are deferred.
## Consequences
The deployment stays lightweight and backward compatible, and relevance changes can be measured with a stable fixture. Ranking cannot rescue an older result from date-first presentation. Settings that affect indexed tokens require a complete repopulation and a backup-based rollback.
docs/manticore-28-upgrade.md
+25
View File
@@ -0,0 +1,25 @@
# Manticore Search 28.4.4 maintenance upgrade
This upgrade rebuilds every configured Manticore table. Do not reuse data files written by 28.4.4 with an older daemon.
## Before maintenance
1. Set `SEARCH_INDEX_GENERATION` to a new value so cached results and cursors cannot cross the cutover.
2. Run `php artisan manticore:inspect --json` and save its output.
3. Stop the processing engine with `php artisan tmux:stop` and pause queue workers that update search documents.
4. Back up `/var/lib/manticore/data` with the container stopped, or use the supported Manticore backup tooling. Record the old image tag with the backup.
## Upgrade and repopulate
1. Pull and start `manticoresearch/manticore:28.4.4`.
2. Confirm HTTP connectivity and optional username/password or bearer-token authentication with `php artisan manticore:inspect`.
3. Run `php artisan manticore:create-indexes --drop`.
4. Run `php artisan nntmux:populate --manticore --all` (or explicitly populate every enabled releases, PreDB, movie, TV, and secondary metadata table).
5. Run `php artisan manticore:inspect`, index reconciliation, and representative searches from `tests/Fixtures/Search/manticore-relevance-v1.json`.
6. Resume queue workers and run `php artisan tmux:start` only after all tables report compatible.
## Rollback
Stop Manticore, restore the pre-upgrade data backup, and start the exact image version recorded with that backup. Do not point the older image at files modified by 28.4.4. Restore the previous `SEARCH_INDEX_GENERATION` only if application code was also rolled back.
Local Docker may remain anonymous. Secured deployments use `MANTICORESEARCH_USERNAME` and `MANTICORESEARCH_PASSWORD`, or `MANTICORESEARCH_TOKEN`; never store credentials in Compose files.
tests/Fixtures/Search/manticore-relevance-v1.json
+24
View File
@@ -0,0 +1,24 @@
{
"version": 1,
"thresholds": {
"top_5_precision": 0.8,
"top_10_precision": 0.8,
"max_zero_result_rate": 0.05,
"max_fuzzy_fallback_rate": 0.25,
"max_p95_latency_ms": 250
},
"queries": [
{"index": "releases", "query": "The.Matrix.1999.1080p", "expected_terms": ["matrix", "1999"], "fuzzy": false},
{"index": "releases", "query": "\"breaking bad\" -german", "expected_terms": ["breaking bad"], "excluded_terms": ["german"], "fuzzy": false},
{"index": "releases", "query": "Oppenhiemer 2023", "expected_terms": ["oppenheimer", "2023"], "fuzzy": true},
{"index": "predb", "query": "Ubuntu.24.04-LTS", "expected_terms": ["ubuntu", "24.04"], "fuzzy": false},
{"index": "movies", "query": "Blade Runner 2049", "expected_terms": ["blade runner", "2049"], "fuzzy": false},
{"index": "tvshows", "query": "Better Call Saul", "expected_terms": ["better call saul"], "fuzzy": false},
{"index": "music", "query": "Daft Punk Discovery", "expected_terms": ["daft punk", "discovery"], "fuzzy": false},
{"index": "books", "query": "Frank Herbert Dune", "expected_terms": ["frank herbert", "dune"], "fuzzy": false},
{"index": "games", "query": "Baldur's Gate 3", "expected_terms": ["baldur", "gate"], "fuzzy": false},
{"index": "console", "query": "Zelda Tears of the Kingdom", "expected_terms": ["zelda", "kingdom"], "fuzzy": false},
{"index": "steam", "query": "Cyberpunk 2077", "expected_terms": ["cyberpunk", "2077"], "fuzzy": false},
{"index": "anime", "query": "Fullmetal Alchemist Brotherhood", "expected_terms": ["fullmetal alchemist"], "fuzzy": false}
]
}
tests/Unit/ManticoreIndexRegistryTest.php
+68
View File
@@ -0,0 +1,68 @@
<?php
declare(strict_types=1);
namespace Tests\Unit;
use App\Services\Search\Support\ManticoreIndexRegistry;
use App\Services\Search\Support\ManticoreSchemaInspector;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
final class ManticoreIndexRegistryTest extends TestCase
{
#[Test]
public function it_defines_every_search_table_with_relevance_settings(): void
{
$definitions = ManticoreIndexRegistry::definitions();
self::assertSame(
['releases', 'predb', 'movies', 'tvshows', 'music', 'books', 'games', 'console', 'steam', 'anime'],
array_keys($definitions)
);
foreach ($definitions as $definition) {
self::assertSame(2, $definition['settings']['min_infix_len']);
self::assertSame(1, $definition['settings']['exact_words']);
self::assertSame(1, $definition['settings']['index_field_lengths']);
}
self::assertSame('bigint', $definitions['releases']['columns']['passwordstatus']['type']);
self::assertSame('bigint', $definitions['releases']['columns']['haspreview']['type']);
}
#[Test]
public function it_provides_title_first_field_weights(): void
{
self::assertGreaterThan(
ManticoreIndexRegistry::profile('movies')['fields']['plot'],
ManticoreIndexRegistry::profile('movies')['fields']['title']
);
self::assertGreaterThan(
ManticoreIndexRegistry::profile('releases')['fields']['fromname'],
ManticoreIndexRegistry::profile('releases')['fields']['searchname']
);
}
#[Test]
public function it_reports_schema_drift_that_requires_a_rebuild(): void
{
$result = ManticoreSchemaInspector::compareColumns(
[['Field' => 'passwordstatus', 'Type' => 'uint']],
['passwordstatus' => ['type' => 'bigint'], 'haspreview' => ['type' => 'bigint']]
);
self::assertSame(['haspreview'], $result['missing']);
self::assertSame('bigint', $result['incompatible']['passwordstatus']['expected']);
}
#[Test]
public function benchmark_fixture_is_versioned_and_covers_every_table(): void
{
$fixture = json_decode((string) file_get_contents(__DIR__.'/../Fixtures/Search/manticore-relevance-v1.json'), true, flags: JSON_THROW_ON_ERROR);
self::assertSame(1, $fixture['version']);
self::assertSame(array_keys(ManticoreIndexRegistry::definitions()), array_values(array_unique(array_column($fixture['queries'], 'index'))));
self::assertArrayHasKey('top_5_precision', $fixture['thresholds']);
}
}