diff --git a/app/Console/Commands/CreateManticoreIndexes.php b/app/Console/Commands/CreateManticoreIndexes.php new file mode 100644 index 000000000..d024a508b --- /dev/null +++ b/app/Console/Commands/CreateManticoreIndexes.php @@ -0,0 +1,168 @@ +info('Creating Manticore Search indexes...'); + + $dropExisting = $this->option('drop'); + + // Get connection details from config + $host = config('sphinxsearch.host', '127.0.0.1'); + $port = config('sphinxsearch.port', 9308); + + // Create client + $this->client = new Client([ + 'host' => $host, + 'port' => $port + ]); + + // We'll skip checking for data_dir this way since it may not be accessible via API + // but instead provide better error handling during index creation + + // If you encounter data_dir errors, ensure it's properly set in manticore.conf: + // data_dir = /path/to/data + // And make sure the path exists and has proper permissions + + try { + $this->client->nodes()->status(); + } catch (\Exception $e) { + $this->error('Failed to connect to Manticore Search: ' . $e->getMessage()); + $this->info('Please check if Manticore Search is running and properly configured.'); + return 1; + } + + // Define indexes and their schema + $indexes = [ + 'releases_rt' => [ + 'settings' => [ + 'min_prefix_len' => 0, + 'min_infix_len' => 2, + ], + 'columns' => [ + 'name' => ['type' => 'text'], + 'searchname' => ['type' => 'text'], + 'fromname' => ['type' => 'text'], + 'filename' => ['type' => 'text'], + 'categories_id' => ['type' => 'text'], + 'dummy' => ['type' => 'integer', 'attribute' => true] + ] + ], + 'predb_rt' => [ + 'settings' => [ + 'min_prefix_len' => 0, + 'min_infix_len' => 2, + ], + 'columns' => [ + 'title' => ['type' => 'text', 'attribute' => true], + 'filename' => ['type' => 'text', 'attribute' => true], + 'dummy' => ['type' => 'integer', 'attribute' => true], + 'source' => ['type' => 'string', 'attribute' => true] + ] + ] + ]; + + $hasErrors = false; + + // Create each index + foreach ($indexes as $indexName => $schema) { + if (!$this->createIndex($indexName, $schema, $dropExisting)) { + $hasErrors = true; + } + } + + if ($hasErrors) { + $this->error('Some errors occurred during index creation.'); + return 1; + } + + $this->info('All Manticore Search indexes created successfully!'); + return 0; + } + + /** + * Create a single index with error handling. + * + * @param string $indexName + * @param array $schema + * @param bool $dropExisting + * @return bool + */ + protected function createIndex(string $indexName, array $schema, bool $dropExisting): bool + { + $this->info("Creating {$indexName} index..."); + $indices = $this->client->tables(); + + try { + // Optionally drop existing index + if ($dropExisting) { + try { + $this->info("Dropping existing {$indexName} index..."); + $indices->drop(['index' => $indexName, 'body' => ['silent' => true]]); + $this->info("Successfully dropped {$indexName} index."); + } catch (ResponseException $e) { + if (!str_contains($e->getMessage(), 'unknown index')) { + $this->warn("Warning when dropping {$indexName} index: " . $e->getMessage()); + } + } + } + + // Instead of checking if index exists (which doesn't work), + // try to create it directly and handle any errors + // that might occur if it already exists + $response = $indices->create([ + 'index' => $indexName, + 'body' => $schema + ]); + + $this->info("Successfully created {$indexName} index."); + $this->line('Response: ' . json_encode($response, JSON_PRETTY_PRINT)); + return true; + } catch (ResponseException $e) { + // Check if the error is because the index already exists + if (str_contains($e->getMessage(), 'already exists')) { + $this->warn("Index {$indexName} already exists. Use --drop option to recreate it."); + return true; + } + + $this->error("Failed to create {$indexName} index: " . $e->getMessage()); + return false; + } catch (\Exception $e) { + $this->error("Failed to create {$indexName} index: " . $e->getMessage()); + return false; + } + } + } diff --git a/app/Console/Commands/NntmuxPopulateSearchIndexes.php b/app/Console/Commands/NntmuxPopulateSearchIndexes.php index f6f59440f..f4a72150e 100644 --- a/app/Console/Commands/NntmuxPopulateSearchIndexes.php +++ b/app/Console/Commands/NntmuxPopulateSearchIndexes.php @@ -58,11 +58,10 @@ class NntmuxPopulateSearchIndexes extends Command /** * Run releases. */ - private function manticoreReleases(): void + private function manticoreReleases(): void { $manticore = new ManticoreSearch; $manticore->truncateRTIndex(Arr::wrap('releases_rt')); - $data = []; $total = Release::count(); if (! $total) { $this->warn('Releases table is empty. Nothing to do.'); @@ -84,19 +83,23 @@ class NntmuxPopulateSearchIndexes extends Command ->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', 'releases.categories_id']) ->selectRaw('IFNULL(GROUP_CONCAT(release_files.name SEPARATOR " "),"") filename') ->groupBy('id') - ->chunk($max, function ($releases) use ($manticore, $bar, $data) { + ->chunk($max, function ($releases) use ($manticore, $bar) { + $data = []; foreach ($releases as $r) { $data[] = [ 'id' => $r->id, - 'name' => $r->name, - 'searchname' => $r->searchname, - 'fromname' => $r->fromname, - 'categories_id' => (string) $r->categories_id, - 'filename' => $r->filename, + 'name' => (string) ($r->name ?? ''), + 'searchname' => (string) ($r->searchname ?? ''), + 'fromname' => (string) ($r->fromname ?? ''), + 'categories_id' => (string) ($r->categories_id ?? '0'), + 'filename' => (string) ($r->filename ?? ''), + 'dummy' => 1, // Adding dummy integer field as required by schema ]; $bar->advance(); } - $manticore->manticoreSearch->table('releases_rt')->replaceDocuments($data); + if (!empty($data)) { + $manticore->manticoreSearch->table('releases_rt')->replaceDocuments($data); + } }); $bar->finish(); $this->newLine(); @@ -109,7 +112,6 @@ class NntmuxPopulateSearchIndexes extends Command { $manticore = new ManticoreSearch; $manticore->truncateRTIndex(['predb_rt']); - $data = []; $total = Predb::count(); if (! $total) { @@ -129,17 +131,21 @@ class NntmuxPopulateSearchIndexes extends Command ->select(['id', 'title', 'filename', 'source']) ->groupBy('id') ->orderBy('id') - ->chunk($max, function ($pre) use ($manticore, $bar, $data) { + ->chunk($max, function ($pre) use ($manticore, $bar) { + $data = []; foreach ($pre as $p) { $data[] = [ 'id' => $p->id, - 'title' => $p->title, - 'filename' => $p->filename, - 'source' => $p->source, + 'title' => (string) ($p->title ?? ''), + 'filename' => (string) ($p->filename ?? ''), + 'source' => (string) ($p->source ?? ''), + 'dummy' => 1, // Adding dummy integer field as required by schema ]; $bar->advance(); } - $manticore->manticoreSearch->table('predb_rt')->replaceDocuments($data); + if (!empty($data)) { + $manticore->manticoreSearch->table('predb_rt')->replaceDocuments($data); + } }); $bar->finish(); diff --git a/misc/manticoresearch/manticore.conf b/misc/manticoresearch/manticore.conf index 512d34f69..616781b80 100644 --- a/misc/manticoresearch/manticore.conf +++ b/misc/manticoresearch/manticore.conf @@ -1,120 +1,3 @@ -# !*: Do not change the "releases_rt" or "predb_rt" words. -index releases_rt -{ - # !*: Do not change this. - type = rt - - # dictionary type, 'crc' or 'keywords' - # crc is faster to index when no substring/wildcards searches are needed - # crc with substrings might be faster to search but is much slower to index - # (because all substrings are pre-extracted as individual keywords) - # keywords is much faster to index with substrings, and index is much (3-10x) smaller - # keywords supports wildcards, crc does not, and never will - # optional, default is 'keywords' - # dict = keywords - - # !*: These allow wildcard searches, they take more ram/disk space. - # !*: If you want to turn it off, put min_infix_len, min_prefix_len and enable_star to 0. - # - # !*: min_prefix_len Allows wildcard searches from the start of the word. ie: to search fishing you can - # do fish* ; This mode takes less ram/disk space than min_infix_len. To set it on, put min_infix_len to 0 - # and min_prefix_len to 1 - # - # !*: min_infix_len Allows wildcard searches from any position in the word, ie: *ish* would match fishing - # This comes at the expense of more ram/disk space. - # NOTE: min_infix_len DOES NOT WORK FOR RT INDEXES ON SOME VERSIONS OF SPHINX, IT WILL DEFAULT TO min_prefix_len - # IF YOUR SPHINX VERSION DOES NOT SUPPORT IT. - # - # - min_prefix_len = 0 - min_infix_len = 2 - - # Expands keywords with their exact forms (i.e. the forms of the keywords before applying any morphological modifications) and/or stars when possible. The supported values are: - # - # 1 - expand to both the exact form and the form with the stars. running will become (running | *running* | =running) - # exact - augment the keyword with only its exact form. running will become (running | =running) - # star - augment the keyword by adding * around it. running will become (running | *running*) Optional, default is 0 (do not expand keywords). - # expand_keywords = 1 - - # !*: Path to where the index files will be stored. - # !*: http://sphinxsearch.com/docs/current.html#conf-path - path = /var/lib/manticore/data/releases_rt - - # RAM chunk size limit - # RT index will keep at most this much data in RAM, then flush to disk - # !*: http://sphinxsearch.com/docs/current.html#conf-rt-mem-limit - rt_mem_limit = 1024M - - # Full-text field declarations. - # !*: Do not change these. - rt_field = name - rt_field = searchname - rt_field = fromname - rt_field = filename - rt_field = categories_id - rt_attr_uint = dummy - -} - -# !*: Do not change the "predb_rt" word. -index predb_rt -{ - # !*: Do not change this. - type = rt - - # dictionary type, 'crc' or 'keywords' - # crc is faster to index when no substring/wildcards searches are needed - # crc with substrings might be faster to search but is much slower to index - # (because all substrings are pre-extracted as individual keywords) - # keywords is much faster to index with substrings, and index is much (3-10x) smaller - # keywords supports wildcards, crc does not, and never will - # optional, default is 'keywords' - # dict = keywords - - # !*: These allow wildcard searches, they take more ram/disk space. - # !*: If you want to turn it off, put min_infix_len, min_prefix_len and enable_star to 0. - # - # !*: min_prefix_len Allows wildcard searches from the start of the word. ie: to search fishing you can - # do fish* ; This mode takes less ram/disk space than min_infix_len. To set it on, put min_infix_len to 0 - # and min_prefix_len to 1 - # - # !*: min_infix_len Allows wildcard searches from any position in the word, ie: *ish* would match fishing - # This comes at the expense of more ram/disk space. - # NOTE: min_infix_len DOES NOT WORK FOR RT INDEXES ON SOME VERSIONS OF SPHINX, IT WILL DEFAULT TO min_prefix_len - # IF YOUR SPHINX VERSION DOES NOT SUPPORT IT. - # - # !*: DO NOT ENABLE BOTH min_prefix_len AND min_infix_len AT THE SAME TIME. - # - min_prefix_len = 0 - min_infix_len = 2 - - # Expands keywords with their exact forms (i.e. the forms of the keywords before applying any morphological modifications) and/or stars when possible. The supported values are: - - # 1 - expand to both the exact form and the form with the stars. running will become (running | *running* | =running) - # exact - augment the keyword with only its exact form. running will become (running | =running) - # star - augment the keyword by adding * around it. running will become (running | *running*) Optional, default is 0 (do not expand keywords). - # expand_keywords = 1 - - # !*: Path to where the index files will be stored. - # !*: http://sphinxsearch.com/docs/current.html#conf-path - path = /var/lib/manticore/data/predb_rt - - # RAM chunk size limit - # RT index will keep at most this much data in RAM, then flush to disk - # !*: http://sphinxsearch.com/docs/current.html#conf-rt-mem-limit - rt_mem_limit = 1024M - - # Full-text field declarations. - # !*: Do not change these. - rt_field = title - rt_field = filename - rt_attr_uint = dummy - rt_attr_string = title - rt_attr_string = filename - rt_attr_string = source - -} - indexer { @@ -171,6 +54,13 @@ searchd listen = 9306:mysql listen = 9308:http + # data directory, where all indexes and binlog files are stored + # optional, default is build-time configured data directory + # !*: This is recommended, it rebuilds indexes if your server crashes. + # !*: http://sphinxsearch.com/docs/current.html#conf-data-dir + # data_dir = # disable logging + data_dir = /var/lib/manticore/data + # log file, searchd run info is logged here # optional, default is 'searchd.log' # !*: Make sure this folder exists and is writable, if you have issues starting sphinx,