starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

CS fixes

Browse Source
This commit is contained in:
DariusIII
2025-07-10 23:41:06 +02:00
parent 298113c6d1
commit 23ee522a79
4 changed files with 61 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/NntmuxCheckIndex.php
+13 -10
View File
@@ -2,8 +2,8 @@
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Exception;
use Illuminate\Console\Command;
class NntmuxCheckIndex extends Command
{
@@ -33,8 +33,9 @@ class NntmuxCheckIndex extends Command
$engine = $this->getSelectedEngine();
$index = $this->getSelectedIndex();
if (!$engine || !$index) {
if (! $engine || ! $index) {
$this->error('You must specify both an engine (--manticore or --elastic) and an index (--releases or --predb).');
return Command::FAILURE;
}
@@ -50,6 +51,7 @@ class NntmuxCheckIndex extends Command
return Command::SUCCESS;
} catch (Exception $e) {
$this->error("Error checking index: {$e->getMessage()}");
return Command::FAILURE;
}
}
@@ -63,8 +65,9 @@ class NntmuxCheckIndex extends Command
// Check if index exists
$exists = \Elasticsearch::indices()->exists(['index' => $index]);
if (!$exists) {
if (! $exists) {
$this->error("ElasticSearch index '{$index}' does not exist.");
return;
}
@@ -75,8 +78,8 @@ class NntmuxCheckIndex extends Command
$docCount = $stats['indices'][$index]['total']['docs']['count'] ?? 0;
$storeSize = $stats['indices'][$index]['total']['store']['size_in_bytes'] ?? 0;
$this->info("Document count: " . number_format($docCount));
$this->info("Index size: " . $this->formatBytes($storeSize));
$this->info('Document count: '.number_format($docCount));
$this->info('Index size: '.$this->formatBytes($storeSize));
// Get a sample document
if ($docCount > 0) {
@@ -84,12 +87,12 @@ class NntmuxCheckIndex extends Command
'index' => $index,
'size' => 1,
'body' => [
'query' => ['match_all' => (object)[]]
]
'query' => ['match_all' => (object) []],
],
]);
if (isset($sample['hits']['hits'][0])) {
$this->info("Sample document:");
$this->info('Sample document:');
$this->info(json_encode($sample['hits']['hits'][0]['_source'], JSON_PRETTY_PRINT));
}
}
@@ -109,7 +112,7 @@ class NntmuxCheckIndex extends Command
// This is a basic check - you may need to adjust based on your ManticoreSearch setup
$this->info("Checking ManticoreSearch index '{$indexName}'...");
$this->warn("ManticoreSearch index checking not fully implemented yet.");
$this->warn('ManticoreSearch index checking not fully implemented yet.');
} catch (Exception $e) {
$this->error("Error checking ManticoreSearch index: {$e->getMessage()}");
@@ -128,7 +131,7 @@ class NntmuxCheckIndex extends Command
$bytes /= pow(1024, $pow);
return round($bytes, 2) . ' ' . $units[$pow];
return round($bytes, 2).' '.$units[$pow];
}
/**
app/Console/Commands/NntmuxOffsetPopulate.php
+34 -29
View File
@@ -8,7 +8,6 @@ use Blacklight\ManticoreSearch;
use Exception;
use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Process;
class NntmuxOffsetPopulate extends Command
@@ -36,6 +35,7 @@ class NntmuxOffsetPopulate extends Command
protected $description = 'Populate search indexes using offset-based parallel processing for maximum performance';
private const SUPPORTED_ENGINES = ['manticore', 'elastic'];
private const SUPPORTED_INDEXES = ['releases', 'predb'];
/**
@@ -46,8 +46,9 @@ class NntmuxOffsetPopulate extends Command
$engine = $this->getSelectedEngine();
$index = $this->getSelectedIndex();
if (!$engine || !$index) {
if (! $engine || ! $index) {
$this->error('You must specify both an engine (--manticore or --elastic) and an index (--releases or --predb).');
return Command::FAILURE;
}
@@ -70,8 +71,9 @@ class NntmuxOffsetPopulate extends Command
private function populateIndexParallel(string $engine, string $index): int
{
$total = $this->getTotalRecords($index);
if (!$total) {
if (! $total) {
$this->warn("{$index} table is empty. Nothing to do.");
return Command::SUCCESS;
}
@@ -79,7 +81,7 @@ class NntmuxOffsetPopulate extends Command
$recordsPerProcess = ceil($total / $parallelProcesses);
$this->info(sprintf(
"Processing %s records with %d parallel processes, ~%d records per process",
'Processing %s records with %d parallel processes, ~%d records per process',
number_format($total),
$parallelProcesses,
$recordsPerProcess
@@ -101,7 +103,7 @@ class NntmuxOffsetPopulate extends Command
$processes[] = [
'process' => $process,
'id' => $i,
'range' => $range
'range' => $range,
];
}
@@ -112,6 +114,7 @@ class NntmuxOffsetPopulate extends Command
$this->verifyIndexPopulation($engine, $index, $total);
$this->info('All parallel processes completed successfully!');
return Command::SUCCESS;
}
@@ -131,7 +134,7 @@ class NntmuxOffsetPopulate extends Command
$ranges[] = [
'offset' => $offset,
'limit' => $limit,
'worker_id' => $i
'worker_id' => $i,
];
}
}
@@ -156,7 +159,7 @@ class NntmuxOffsetPopulate extends Command
"--limit={$limit}",
"--worker-id={$workerId}",
"--batch-size={$batchSize}",
"--memory-limit={$memoryLimit}"
"--memory-limit={$memoryLimit}",
];
if ($this->option('disable-keys')) {
@@ -193,7 +196,7 @@ class NntmuxOffsetPopulate extends Command
continue;
}
if (!$process->running()) {
if (! $process->running()) {
$completedProcesses[] = $processId;
$completed++;
$bar->advance();
@@ -212,10 +215,10 @@ class NntmuxOffsetPopulate extends Command
$errorOutput = $result->errorOutput();
if ($output) {
$this->error("Worker {$processId} output: " . substr($output, -500));
$this->error("Worker {$processId} output: ".substr($output, -500));
}
if ($errorOutput) {
$this->error("Worker {$processId} error: " . substr($errorOutput, -500));
$this->error("Worker {$processId} error: ".substr($errorOutput, -500));
}
}
} catch (Exception $e) {
@@ -231,8 +234,8 @@ class NntmuxOffsetPopulate extends Command
$bar->finish();
$this->newLine();
if (!empty($failedProcesses)) {
$this->error("Failed workers: " . implode(', ', $failedProcesses));
if (! empty($failedProcesses)) {
$this->error('Failed workers: '.implode(', ', $failedProcesses));
}
}
@@ -241,7 +244,7 @@ class NntmuxOffsetPopulate extends Command
*/
private function verifyIndexPopulation(string $engine, string $index, int $expectedTotal): void
{
$this->info("Verifying index population...");
$this->info('Verifying index population...');
if ($engine === 'elastic') {
try {
@@ -251,19 +254,19 @@ class NntmuxOffsetPopulate extends Command
$stats = \Elasticsearch::indices()->stats(['index' => $index]);
$actualCount = $stats['indices'][$index]['total']['docs']['count'] ?? 0;
$this->info("Expected: " . number_format($expectedTotal) . " records");
$this->info("Actual: " . number_format($actualCount) . " records");
$this->info('Expected: '.number_format($expectedTotal).' records');
$this->info('Actual: '.number_format($actualCount).' records');
if ($actualCount >= $expectedTotal * 0.95) { // Allow for 5% tolerance
$this->info("✓ Index population successful!");
$this->info('✓ Index population successful!');
} else {
$this->warn("⚠ Index population may be incomplete");
$this->warn('⚠ Index population may be incomplete');
}
} catch (Exception $e) {
$this->warn("Could not verify index population: {$e->getMessage()}");
}
} else {
$this->info("ManticoreSearch index verification not implemented yet");
$this->info('ManticoreSearch index verification not implemented yet');
}
}
@@ -294,8 +297,8 @@ class NntmuxOffsetPopulate extends Command
\Elasticsearch::deleteByQuery([
'index' => $index,
'body' => [
'query' => ['match_all' => (object)[]]
]
'query' => ['match_all' => (object) []],
],
]);
// Force refresh to ensure deletions are visible
@@ -310,7 +313,7 @@ class NntmuxOffsetPopulate extends Command
}
} catch (Exception $e) {
$this->warn("Could not clear ElasticSearch index: {$e->getMessage()}");
$this->info("Attempting to recreate index...");
$this->info('Attempting to recreate index...');
// Fallback: delete and recreate
try {
@@ -338,11 +341,11 @@ class NntmuxOffsetPopulate extends Command
'translog' => [
'durability' => 'async',
'sync_interval' => '30s',
'flush_threshold_size' => '1gb'
]
'flush_threshold_size' => '1gb',
],
],
'mappings' => $this->getIndexMappings($indexName)
]
'mappings' => $this->getIndexMappings($indexName),
],
];
\Elasticsearch::indices()->create($settings);
@@ -364,8 +367,8 @@ class NntmuxOffsetPopulate extends Command
'fromname' => ['type' => 'text', 'analyzer' => 'standard'],
'categories_id' => ['type' => 'integer'],
'filename' => ['type' => 'text', 'analyzer' => 'standard'],
'postdate' => ['type' => 'date']
]
'postdate' => ['type' => 'date'],
],
];
} else {
return [
@@ -373,8 +376,8 @@ class NntmuxOffsetPopulate extends Command
'id' => ['type' => 'long'],
'title' => ['type' => 'text', 'analyzer' => 'standard'],
'filename' => ['type' => 'text', 'analyzer' => 'standard'],
'source' => ['type' => 'keyword']
]
'source' => ['type' => 'keyword'],
],
];
}
}
@@ -397,6 +400,7 @@ class NntmuxOffsetPopulate extends Command
return $engine;
}
}
return null;
}
@@ -410,6 +414,7 @@ class NntmuxOffsetPopulate extends Command
return $index;
}
}
return null;
}
}
app/Console/Commands/NntmuxOffsetWorker.php
+11 -7
View File
@@ -54,8 +54,9 @@ class NntmuxOffsetWorker extends Command
$engine = $this->getSelectedEngine();
$index = $this->getSelectedIndex();
if (!$engine || !$index) {
if (! $engine || ! $index) {
$this->error('Engine and index must be specified');
return Command::FAILURE;
}
@@ -69,6 +70,7 @@ class NntmuxOffsetWorker extends Command
} catch (Exception $e) {
$this->error("Worker {$workerId} failed: {$e->getMessage()}");
return Command::FAILURE;
}
}
@@ -103,7 +105,7 @@ class NntmuxOffsetWorker extends Command
if (count($batchData) >= $batchSize) {
$this->processManticoreBatch($manticore, $indexName, $batchData, $workerId);
$this->info("Worker {$workerId}: Inserted batch of " . count($batchData) . " records");
$this->info("Worker {$workerId}: Inserted batch of ".count($batchData).' records');
$batchData = [];
}
} catch (Exception $e) {
@@ -114,13 +116,13 @@ class NntmuxOffsetWorker extends Command
});
// Process remaining items
if (!empty($batchData)) {
if (! empty($batchData)) {
$this->processManticoreBatch($manticore, $indexName, $batchData, $workerId);
$this->info("Worker {$workerId}: Inserted final batch of " . count($batchData) . " records");
$this->info("Worker {$workerId}: Inserted final batch of ".count($batchData).' records');
}
} else { // ElasticSearch
$query->chunk($batchSize, function ($items) use ($indexName, $transformer, &$processed, &$errors, $batchSize, $workerId) {
$query->chunk($batchSize, function ($items) use ($indexName, $transformer, &$processed, &$errors, $workerId) {
$this->info("Worker {$workerId}: Processing chunk of {$items->count()} items");
$data = ['body' => []];
@@ -143,15 +145,16 @@ class NntmuxOffsetWorker extends Command
}
}
if (!empty($data['body'])) {
if (! empty($data['body'])) {
$this->processElasticBatch($data, $workerId);
$this->info("Worker {$workerId}: Inserted batch of " . (count($data['body']) / 2) . " records");
$this->info("Worker {$workerId}: Inserted batch of ".(count($data['body']) / 2).' records');
}
});
}
$executionTime = round(microtime(true) - $startTime, 2);
$this->info("Worker {$workerId}: Completed processing {$processed} records with {$errors} errors in {$executionTime} seconds");
return Command::SUCCESS;
}
@@ -204,6 +207,7 @@ class NntmuxOffsetWorker extends Command
} else {
return function ($item) {
$searchName = str_replace(['.', '-'], ' ', $item->searchname ?? '');
return [
'id' => $item->id,
'name' => $item->name,
app/Console/Commands/NntmuxPopulateSearchIndexes.php
+3 -2
View File
@@ -292,7 +292,7 @@ class NntmuxPopulateSearchIndexes extends Command
});
// Process remaining items
if (!empty($batchData)) {
if (! empty($batchData)) {
$this->processBatch($manticore, $indexName, $batchData);
}
@@ -458,7 +458,7 @@ class NntmuxPopulateSearchIndexes extends Command
$bar->advance();
}
if (!empty($data['body'])) {
if (! empty($data['body'])) {
$this->processElasticBatch($data, $errorCount);
}
}
@@ -603,6 +603,7 @@ class NntmuxPopulateSearchIndexes extends Command
private function getBatchSize(): int
{
$batchSize = (int) $this->option('batch-size');
return $batchSize > 0 ? $batchSize : self::DEFAULT_BATCH_SIZE;
}