mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 00:01:21 +00:00
Add artisan command to populate manticore and elasticsearch indexes
This commit is contained in:
DariusIII
@@ -3,7 +3,7 @@
|
||||
namespace Blacklight;
|
||||
|
||||
use App\Models\Release;
|
||||
use Elastic\Elasticsearch\Exception\ElasticsearchException;
|
||||
use Elasticsearch\Common\Exceptions\ElasticsearchException;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Predb;
|
||||
use App\Models\Release;
|
||||
use Blacklight\ManticoreSearch;
|
||||
use Elasticsearch;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Arr;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class NntmuxPopulateSearchIndexes extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = '
|
||||
nntmux:populate
|
||||
{--manticore : Use ManticoreSearch}
|
||||
{--elastic : Use ElasticSearch}
|
||||
{--releases : Populates the releases_rt index}
|
||||
{--predb : Populates the predb_rt index}
|
||||
{--count=20000 : Sets the chunk size}
|
||||
{--optimize : Optimize ManticoreSearch indexes}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Populate Manticore/Elasticsearch indexes with either releases or predb';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
if ($this->option('releases') && $this->option('manticore')) {
|
||||
$this->manticoreReleases();
|
||||
} elseif ($this->option('predb') && $this->option('manticore')) {
|
||||
$this->manticorePreDB();
|
||||
} elseif ($this->option('optimize')) {
|
||||
$this->optimize();
|
||||
} elseif ($this->option('elastic') && $this->option('releases')) {
|
||||
$this->elasticReleases();
|
||||
} elseif ($this->option('elastic') && $this->option('predb')) {
|
||||
$this->elasticPreDB();
|
||||
} else {
|
||||
$this->error('You need to specify which index to populate! Check the options with the -h parameter.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run releases.
|
||||
*/
|
||||
private function manticoreReleases(): void
|
||||
{
|
||||
$manticore = new ManticoreSearch();
|
||||
$manticore->truncateRTIndex(Arr::wrap('releases_rt'));
|
||||
$total = Release::count();
|
||||
if (! $total) {
|
||||
$this->warn('Could not get database information for releases table.');
|
||||
exit();
|
||||
}
|
||||
$max = $this->option('count');
|
||||
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
|
||||
$this->info("Starting to populate Manticore RT index 'releases_rt' with $total rows and $max inserts per loop.");
|
||||
|
||||
$bar = $this->output->createProgressBar($total);
|
||||
$bar->setOverwrite(true); // Terminal needs to support ANSI Encoding for this?
|
||||
$bar->start();
|
||||
|
||||
DB::table('releases')
|
||||
->orderBy('id')
|
||||
->join('release_files', 'releases.id', '=', 'release_files.releases_id')
|
||||
->select('releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', 'releases.categories_id')
|
||||
->selectRaw('IFNULL(GROUP_CONCAT(release_files.name SEPARATOR " "),"") filename')
|
||||
->groupBy('id')
|
||||
->chunk($max, function ($releases) use ($manticore, $bar) {
|
||||
foreach ($releases as $r) {
|
||||
$manticore->insertRelease([
|
||||
'id' => $r->id,
|
||||
'name' => $r->name,
|
||||
'searchname' => $r->searchname,
|
||||
'fromname' => $r->fromname,
|
||||
'categories_id' => (string) $r->categories_id,
|
||||
'filename' => $r->filename,
|
||||
]);
|
||||
$bar->advance();
|
||||
}
|
||||
});
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run predb.
|
||||
*/
|
||||
private function manticorePreDB(): void
|
||||
{
|
||||
$manticore = new ManticoreSearch();
|
||||
$manticore->truncateRTIndex(['predb_rt']);
|
||||
|
||||
$total = Predb::count();
|
||||
if (! $total) {
|
||||
$this->warn('Could not get database information for predb table.');
|
||||
exit();
|
||||
}
|
||||
$max = $this->option('count');
|
||||
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
|
||||
$this->info("Starting to populate Manticore RT index 'predb_rt' with $total rows and $max inserts per loop.");
|
||||
|
||||
$bar = $this->output->createProgressBar($total);
|
||||
$bar->setOverwrite(true); // Terminal needs to support ANSI Encoding for this?
|
||||
$bar->start();
|
||||
|
||||
DB::table('predb')
|
||||
->select('id', 'title', 'filename', 'source')
|
||||
->groupBy('id')
|
||||
->orderBy('id')
|
||||
->chunk($max, function ($pre) use ($manticore, $bar) {
|
||||
foreach ($pre as $p) {
|
||||
$manticore->insertPredb([
|
||||
'id' => $p->id,
|
||||
'title' => $p->title,
|
||||
'filename' => $p->filename,
|
||||
'source' => $p->source,
|
||||
]);
|
||||
$bar->advance();
|
||||
}
|
||||
});
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return void
|
||||
*/
|
||||
private function elasticReleases()
|
||||
{
|
||||
$max = $this->option('count');
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT r.id, r.name, r.searchname, r.fromname, r.categories_id, r.adddate, r.postdate, IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename
|
||||
FROM releases r
|
||||
LEFT JOIN release_files rf ON r.id = rf.releases_id
|
||||
WHERE r.id > %d
|
||||
GROUP BY r.id
|
||||
ORDER BY r.id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Release::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM releases')->first();
|
||||
if (! $totals) {
|
||||
$this->error('Could not get database information for releases table.');
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
|
||||
$lastId = $minId - 1;
|
||||
$this->info("Starting to populate ElasticSearch index releases with $total releases.");
|
||||
$bar = $this->output->createProgressBar($total);
|
||||
$bar->setOverwrite(true); // Terminal needs to support ANSI Encoding for this?
|
||||
$bar->start();
|
||||
$data = ['body' => []];
|
||||
for ($i = $minId; $i <= ($total + $minId); $i += $max) {
|
||||
$rows = DB::select(sprintf($query, $lastId, $max));
|
||||
if ($rows === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($rows as $row) {
|
||||
if ($row->id > $lastId) {
|
||||
$lastId = $row->id;
|
||||
}
|
||||
$searchName = str_replace(['.', '-'], ' ', $row->searchname);
|
||||
$data['body'][] = [
|
||||
'index' => [
|
||||
'_index' => 'releases',
|
||||
'_id' => $row->id,
|
||||
],
|
||||
];
|
||||
|
||||
$data['body'][] = [
|
||||
'id' => $row->id,
|
||||
'name' => $row->name,
|
||||
'searchname' => $row->searchname,
|
||||
'plainsearchname' => $searchName,
|
||||
'fromname' => $row->fromname,
|
||||
'categories_id' => $row->categories_id,
|
||||
'filename' => $row->filename,
|
||||
'add_date' => $row->adddate,
|
||||
'post_date' => $row->postdate,
|
||||
];
|
||||
}
|
||||
if (! empty($data['body'])) {
|
||||
// Stop and send the bulk request
|
||||
$responses = Elasticsearch::bulk($data);
|
||||
|
||||
// erase the old bulk request
|
||||
$data = ['body' => []];
|
||||
|
||||
// unset the bulk response when you are done to save memory
|
||||
unset($responses);
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
}
|
||||
|
||||
// Send the last batch if it exists
|
||||
if (! empty($data['body'])) {
|
||||
$responses = Elasticsearch::bulk($data);
|
||||
}
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info('Done');
|
||||
}
|
||||
|
||||
/**
|
||||
* @return void
|
||||
*/
|
||||
private function elasticPreDB()
|
||||
{
|
||||
$max = $this->option('count');
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT id, title, filename, source
|
||||
FROM predb
|
||||
WHERE id > %d
|
||||
GROUP BY id
|
||||
ORDER BY id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Predb::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM predb')->first();
|
||||
if (! $totals) {
|
||||
$this->error('Could not get database information for releases table.');
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
|
||||
$lastId = $minId - 1;
|
||||
$this->info("Starting to populate ElasticSearch index predb with $total releases.");
|
||||
$bar = $this->output->createProgressBar($total);
|
||||
$bar->setOverwrite(true); // Terminal needs to support ANSI Encoding for this?
|
||||
$bar->start();
|
||||
$data = ['body' => []];
|
||||
for ($i = $minId; $i <= ($total + $minId); $i += $max) {
|
||||
$rows = DB::select(sprintf($query, $lastId, $max));
|
||||
if ($rows === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($rows as $row) {
|
||||
if ($row->id > $lastId) {
|
||||
$lastId = $row->id;
|
||||
}
|
||||
$data['body'][] = [
|
||||
'index' => [
|
||||
'_index' => 'predb',
|
||||
'_id' => $row->id,
|
||||
],
|
||||
];
|
||||
$data['body'][] = [
|
||||
'id' => $row->id,
|
||||
'title' => $row->title,
|
||||
'filename' => $row->filename,
|
||||
'source' => $row->source,
|
||||
];
|
||||
}
|
||||
if (! empty($data['body'])) {
|
||||
// Stop and send the bulk request
|
||||
$responses = Elasticsearch::bulk($data);
|
||||
|
||||
// erase the old bulk request
|
||||
$data = ['body' => []];
|
||||
|
||||
// unset the bulk response when you are done to save memory
|
||||
unset($responses);
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
}
|
||||
|
||||
// Send the last batch if it exists
|
||||
if (! empty($data['body'])) {
|
||||
$responses = Elasticsearch::bulk($data);
|
||||
}
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info('Done');
|
||||
}
|
||||
|
||||
/**
|
||||
* Run predb.
|
||||
*/
|
||||
private function optimize(): void
|
||||
{
|
||||
$this->info('Optimizing release_rt & predb_rt indexes');
|
||||
try {
|
||||
(new ManticoreSearch())->optimizeRTIndex();
|
||||
} catch (\Exception $e) {
|
||||
echo $e->getMessage();
|
||||
}
|
||||
$this->line('Done!');
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
<?php
|
||||
|
||||
require_once dirname(__DIR__, 2).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
|
||||
|
||||
use App\Models\Predb;
|
||||
use App\Models\Release;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
if (! isset($argv[1])) {
|
||||
exit(
|
||||
"Argument 1 is the index name, releases and predb are the only supported ones currently.\n".
|
||||
"Argument 2 is optional, max number of rows to send to ES at a time, 10,000 is the default if not set.\n"
|
||||
);
|
||||
}
|
||||
|
||||
populate_indexes($argv[1], (isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 10000));
|
||||
|
||||
// Bulk insert releases into ElasticSearch index.
|
||||
function populate_indexes($table, $max)
|
||||
{
|
||||
$allowedIndexes = ['releases', 'predb'];
|
||||
if (\in_array($table, $allowedIndexes, true)) {
|
||||
if ($table === 'releases') {
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT r.id, r.name, r.searchname, r.fromname, r.categories_id, r.adddate, r.postdate, IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename
|
||||
FROM releases r
|
||||
LEFT JOIN release_files rf ON r.id = rf.releases_id
|
||||
WHERE r.id > %d
|
||||
GROUP BY r.id
|
||||
ORDER BY r.id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Release::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM releases')->first();
|
||||
if (! $totals) {
|
||||
exit("Could not get database information for releases table.\n");
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
}
|
||||
|
||||
if ($table === 'predb') {
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT id, title, filename, source
|
||||
FROM predb
|
||||
WHERE id > %d
|
||||
GROUP BY id
|
||||
ORDER BY id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Predb::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM predb')->first();
|
||||
if (! $totals) {
|
||||
exit("Could not get database information for predb table.\n");
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
}
|
||||
|
||||
$lastId = $minId - 1;
|
||||
echo "[Starting to populate ElasticSearch index $table with $total releases.]".PHP_EOL;
|
||||
$data = ['body' => []];
|
||||
for ($i = $minId; $i <= ($total + $max + $minId); $i += $max) {
|
||||
$rows = DB::select(sprintf($query, $lastId, $max));
|
||||
if ($rows === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($rows as $row) {
|
||||
if ($row->id > $lastId) {
|
||||
$lastId = $row->id;
|
||||
}
|
||||
switch ($table) {
|
||||
case 'releases':
|
||||
$searchName = str_replace(['.', '-'], ' ', $row->searchname);
|
||||
$data['body'][] = [
|
||||
'index' => [
|
||||
'_index' => 'releases',
|
||||
'_id' => $row->id,
|
||||
],
|
||||
];
|
||||
|
||||
$data['body'][] = [
|
||||
'id' => $row->id,
|
||||
'name' => $row->name,
|
||||
'searchname' => $row->searchname,
|
||||
'plainsearchname' => $searchName,
|
||||
'fromname' => $row->fromname,
|
||||
'categories_id' => $row->categories_id,
|
||||
'filename' => $row->filename,
|
||||
'add_date' => $row->adddate,
|
||||
'post_date' => $row->postdate,
|
||||
];
|
||||
break;
|
||||
|
||||
case 'predb':
|
||||
$data['body'][] = [
|
||||
'index' => [
|
||||
'_index' => 'predb',
|
||||
'_id' => $row->id,
|
||||
],
|
||||
];
|
||||
$data['body'][] = [
|
||||
'id' => $row->id,
|
||||
'title' => $row->title,
|
||||
'filename' => $row->filename,
|
||||
'source' => $row->source,
|
||||
];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (! empty($data['body'])) {
|
||||
// Stop and send the bulk request
|
||||
$responses = \Elasticsearch::bulk($data);
|
||||
|
||||
// erase the old bulk request
|
||||
$data = ['body' => []];
|
||||
|
||||
// unset the bulk response when you are done to save memory
|
||||
unset($responses);
|
||||
|
||||
echo '.';
|
||||
}
|
||||
}
|
||||
|
||||
// Send the last batch if it exists
|
||||
if (! empty($data['body'])) {
|
||||
$responses = \Elasticsearch::bulk($data);
|
||||
}
|
||||
echo "\n[Done]\n";
|
||||
} else {
|
||||
exit(
|
||||
"Argument 1 is the index name, releases and predb are the only supported ones currently.\n".
|
||||
"Argument 2 is optional, max number of rows to send to ES at a time, 10,000 is the default if not set.\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
<?php
|
||||
|
||||
require_once dirname(__DIR__, 2).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
|
||||
|
||||
use Blacklight\ManticoreSearch;
|
||||
|
||||
try {
|
||||
(new ManticoreSearch())->optimizeRTIndex();
|
||||
} catch (Exception $e) {
|
||||
echo $e->getMessage();
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
<?php
|
||||
|
||||
require_once dirname(__DIR__, 2).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
|
||||
|
||||
use App\Models\Predb;
|
||||
use App\Models\Release;
|
||||
use Blacklight\ManticoreSearch;
|
||||
use Illuminate\Support\Arr;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
if (! isset($argv[1])) {
|
||||
exit(
|
||||
"Argument 1 is the index name, releases_rt and predb_rt are the only supported ones currently.\n".
|
||||
"Argument 2 is optional, max number of rows to send to sphinx at a time, 10,000 is the default if not set.\n".
|
||||
" The value of 10,000 is good for the default sphinx.conf max_packet_size of 8M, if you want\n".
|
||||
" to raise this higher than 10,000, raise the sphinx.conf max_packet_size higher than 8M.\n".
|
||||
" If you have many releases, raise the sphinx.conf max_packet_size to 128M (the maximum), restart sphinx and\n".
|
||||
" and set Argument 2 to 250,000. This will speed up the script tremendously.\n"
|
||||
);
|
||||
}
|
||||
|
||||
populate_rt($argv[1], (isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 10000));
|
||||
|
||||
// Bulk insert releases into Manticore RT index.
|
||||
function populate_rt($table, $max)
|
||||
{
|
||||
$allowedIndexes = ['releases_rt', 'predb_rt'];
|
||||
if (\in_array($table, $allowedIndexes, true)) {
|
||||
$manticore = new ManticoreSearch();
|
||||
$manticore->truncateRTIndex(Arr::wrap($table));
|
||||
if ($table === 'releases_rt') {
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT r.id, r.name, r.searchname, r.fromname, r.categories_id, IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename
|
||||
FROM releases r
|
||||
LEFT JOIN release_files rf ON r.id = rf.releases_id
|
||||
WHERE r.id > %d
|
||||
GROUP BY r.id
|
||||
ORDER BY r.id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Release::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM releases')->first();
|
||||
if (! $totals) {
|
||||
exit("Could not get database information for releases table.\n");
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
}
|
||||
|
||||
if ($table === 'predb_rt') {
|
||||
DB::statement('SET SESSION group_concat_max_len=16384;');
|
||||
$query = 'SELECT id, title, filename, source
|
||||
FROM predb
|
||||
WHERE id > %d
|
||||
GROUP BY id
|
||||
ORDER BY id ASC
|
||||
LIMIT %d';
|
||||
|
||||
$totals = Predb::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM predb')->first();
|
||||
if (! $totals) {
|
||||
exit("Could not get database information for predb table.\n");
|
||||
}
|
||||
$total = $totals->c;
|
||||
$minId = $totals->min;
|
||||
}
|
||||
|
||||
$lastId = $minId - 1;
|
||||
echo "[Starting to populate sphinx RT index $table with $total releases.]".PHP_EOL;
|
||||
for ($i = $minId; $i <= ($total + $max + $minId); $i += $max) {
|
||||
$rows = DB::select(sprintf($query, $lastId, $max));
|
||||
if ($rows === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($rows as $row) {
|
||||
if ($row->id > $lastId) {
|
||||
$lastId = $row->id;
|
||||
}
|
||||
switch ($table) {
|
||||
case 'releases_rt':
|
||||
$manticore->insertRelease([
|
||||
'id' => $row->id,
|
||||
'name' => $row->name,
|
||||
'searchname' => $row->searchname,
|
||||
'fromname' => $row->fromname,
|
||||
'categories_id' => (string) $row->categories_id,
|
||||
'filename' => $row->filename,
|
||||
]);
|
||||
break;
|
||||
|
||||
case 'predb_rt':
|
||||
$manticore->insertPredb([
|
||||
'id' => $row->id,
|
||||
'title' => $row->title,
|
||||
'filename' => $row->filename,
|
||||
'source' => $row->source,
|
||||
]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
echo '.';
|
||||
}
|
||||
echo "\n[Done]\n";
|
||||
} else {
|
||||
exit(
|
||||
"Argument 1 is the index name, releases_rt and predb_rt are the only supported ones currently.\n".
|
||||
"Argument 2 is optional, max number of rows to send to sphinx at a time, 10,000 is the default if not set.\n".
|
||||
" The value of 10,000 is good for the default sphinx.conf max_packet_size of 8M, if you want\n".
|
||||
" to raise this higher than 10,000, raise the sphinx.conf max_packet_size higher than 8M.\n".
|
||||
" If you have many releases, raise the sphinx.conf max_packet_size to 128M (the maximum), restart sphinx and\n".
|
||||
" and set Argument 2 to 250,000. This will speed up the script tremendously.\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user