Add missing files

This commit is contained in:
DariusIII
2025-11-01 19:42:03 +01:00
parent 6dd748fec6
commit 31bf2188de
4 changed files with 891 additions and 0 deletions
+101
View File
@@ -0,0 +1,101 @@
<?php
namespace App\Console\Commands;
use App\Models\ShortGroup;
use App\Models\UsenetGroup;
use Blacklight\NNTP;
use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
class GroupsUpdate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'groups:update';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Update first/last article numbers for all active groups';
/**
* Execute the console command.
*/
public function handle(): int
{
$start = now();
// Create NNTP connection
$nntp = new NNTP;
if ($nntp->doConnect() !== true) {
$this->error('❌ Unable to connect to usenet server');
return Command::FAILURE;
}
$this->info('📡 Getting first/last for all active groups...');
try {
$data = $nntp->getGroups();
if ($nntp->isError($data)) {
$this->error('❌ Failed to getGroups() from NNTP server');
return Command::FAILURE;
}
$this->info('🔄 Updating short_groups table...');
// Truncate and rebuild
DB::statement('TRUNCATE TABLE short_groups');
// Get all active groups
$activeGroups = Arr::pluck(
UsenetGroup::query()
->where('active', '=', 1)
->orWhere('backfill', '=', 1)
->get(['name']),
'name'
);
$updated = 0;
$bar = $this->output->createProgressBar(count($data));
$bar->start();
foreach ($data as $newgroup) {
if (\in_array($newgroup['group'], $activeGroups, false)) {
ShortGroup::query()->insert([
'name' => $newgroup['group'],
'first_record' => $newgroup['first'],
'last_record' => $newgroup['last'],
'updated' => now(),
]);
$updated++;
}
$bar->advance();
}
$bar->finish();
$this->newLine(2);
$elapsed = now()->diffInSeconds($start, true);
$this->info("✅ Updated {$updated} groups");
$this->info("⏱️ Running time: {$elapsed} seconds");
return Command::SUCCESS;
} catch (\Exception $e) {
$this->error('❌ Update failed: '.$e->getMessage());
return Command::FAILURE;
}
}
}
@@ -0,0 +1,369 @@
<?php
namespace App\Console\Commands;
use App\Models\Category;
use App\Models\Predb;
use App\Models\Release;
use Blacklight\NameFixer;
use Blacklight\Nfo;
use Blacklight\NNTP;
use Blacklight\NZBContents;
use Blacklight\processing\PostProcess;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class ReleasesFixNamesGroup extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'releases:fix-names-group
{type : Type of fix (standard|predbft)}
{--guid-char= : GUID character to process (for standard type)}
{--limit=1000 : Maximum releases to process}
{--thread=1 : Thread number (for predbft type)}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Fix release names using various methods (group-based processing)';
private NameFixer $nameFixer;
private int $checked = 0;
/**
* Execute the console command.
*/
public function handle(): int
{
$type = $this->argument('type');
$maxPerRun = (int) $this->option('limit');
$this->nameFixer = new NameFixer;
switch ($type) {
case 'standard':
return $this->processStandard($maxPerRun);
case 'predbft':
return $this->processPredbFulltext($maxPerRun);
default:
$this->error("Invalid type: {$type}. Use 'standard' or 'predbft'");
return Command::FAILURE;
}
}
/**
* Process standard name fixing
*/
protected function processStandard(int $maxPerRun): int
{
$guidChar = $this->option('guid-char');
if ($guidChar === null) {
$this->error('--guid-char is required for standard type');
return Command::FAILURE;
}
$this->info("Processing releases with GUID starting with: {$guidChar}");
$this->info("Maximum per run: {$maxPerRun}");
// Allow for larger filename return sets
DB::statement('SET SESSION group_concat_max_len = 65536');
// Find releases to process
$releases = $this->fetchReleases($guidChar, $maxPerRun);
if ($releases->isEmpty()) {
$this->info('No releases to process');
return Command::SUCCESS;
}
$this->info("Found {$releases->count()} releases to process");
$bar = $this->output->createProgressBar($releases->count());
$bar->start();
$nntp = null;
$nzbcontents = null;
foreach ($releases as $release) {
$this->checked++;
$this->nameFixer->reset();
// Process UID
if ((int) $release->proc_uid === NameFixer::PROC_UID_NONE &&
(! empty($release->uid) || ! empty($release->mediainfo))) {
if (! empty($release->uid)) {
$this->nameFixer->checkName($release, true, 'UID, ', 1, true);
}
if (empty($this->nameFixer->matched) && ! empty($release->mediainfo)) {
$this->nameFixer->checkName($release, true, 'Mediainfo, ', 1, true);
}
}
$this->nameFixer->_updateSingleColumn('proc_uid', NameFixer::PROC_UID_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process CRC32
if ((int) $release->proc_crc32 === NameFixer::PROC_CRC_NONE && ! empty($release->crc)) {
$this->nameFixer->reset();
$this->nameFixer->checkName($release, true, 'CRC32, ', 1, true);
}
$this->nameFixer->_updateSingleColumn('proc_crc32', NameFixer::PROC_CRC_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process SRR
if ((int) $release->proc_srr === NameFixer::PROC_SRR_NONE) {
$this->nameFixer->reset();
$this->nameFixer->checkName($release, true, 'SRR, ', 1, true);
}
$this->nameFixer->_updateSingleColumn('proc_srr', NameFixer::PROC_SRR_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process PAR2 hash
if ((int) $release->proc_hash16k === NameFixer::PROC_HASH16K_NONE && ! empty($release->hash)) {
$this->nameFixer->reset();
$this->nameFixer->checkName($release, true, 'PAR2 hash, ', 1, true);
}
$this->nameFixer->_updateSingleColumn('proc_hash16k', NameFixer::PROC_HASH16K_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process NFO
if ((int) $release->nfostatus === Nfo::NFO_FOUND &&
(int) $release->proc_nfo === NameFixer::PROC_NFO_NONE &&
! empty($release->textstring) &&
! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) {
$this->nameFixer->reset();
$this->nameFixer->checkName($release, true, 'NFO, ', 1, true);
}
$this->nameFixer->_updateSingleColumn('proc_nfo', NameFixer::PROC_NFO_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process filenames
if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixer::PROC_FILES_NONE) {
$this->nameFixer->reset();
$fileNames = explode('|', $release->filestring);
if (is_array($fileNames)) {
$releaseFile = $release;
foreach ($fileNames as $fileName) {
if ($this->nameFixer->matched === false) {
$releaseFile->textstring = $fileName;
$this->nameFixer->checkName($releaseFile, true, 'Filenames, ', 1, true);
}
}
}
}
$this->nameFixer->_updateSingleColumn('proc_files', NameFixer::PROC_FILES_DONE, $release->releases_id);
if ($this->nameFixer->matched) {
$bar->advance();
continue;
}
// Process PAR2
if ((int) $release->proc_par2 === NameFixer::PROC_PAR2_NONE) {
// Initialize NZB contents if needed
if (! isset($nzbcontents)) {
$nntp = new NNTP;
$compressedHeaders = config('nntmux_nntp.compressed_headers');
if ((config('nntmux_nntp.use_alternate_nntp_server') === true
? $nntp->doConnect($compressedHeaders, true)
: $nntp->doConnect()) !== true) {
$this->warn('Unable to connect to usenet for PAR2 processing');
} else {
$Nfo = new Nfo;
$nzbcontents = new NZBContents([
'Echo' => false,
'NNTP' => $nntp,
'Nfo' => $Nfo,
'PostProcess' => new PostProcess(['Nfo' => $Nfo, 'NameFixer' => $this->nameFixer]),
]);
}
}
if (isset($nzbcontents)) {
$nzbcontents->checkPAR2($release->guid, $release->releases_id, $release->groups_id, 1, true);
}
}
$this->nameFixer->_updateSingleColumn('proc_par2', NameFixer::PROC_PAR2_DONE, $release->releases_id);
$bar->advance();
}
$bar->finish();
$this->newLine(2);
$this->info("✅ Processed {$this->checked} releases");
$this->info("✅ Fixed {$this->nameFixer->fixed} release names");
return Command::SUCCESS;
}
/**
* Process PreDB fulltext matching
*/
protected function processPredbFulltext(int $maxPerRun): int
{
$thread = (int) $this->option('thread');
$offset = $thread * $maxPerRun - $maxPerRun;
$this->info('Processing PreDB fulltext matching');
$this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}");
$pres = Predb::fromQuery(
sprintf(
'
SELECT p.id AS predb_id, p.title, p.source, p.searched
FROM predb p
WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]"
AND p.searched = 0
AND p.predate < (NOW() - INTERVAL 1 DAY)
ORDER BY p.predate ASC
LIMIT %s
OFFSET %s',
$maxPerRun,
$offset
)
);
if ($pres->isEmpty()) {
$this->info('No PreDB entries to process');
return Command::SUCCESS;
}
$this->info("Found {$pres->count()} PreDB entries to process");
$bar = $this->output->createProgressBar($pres->count());
$bar->start();
foreach ($pres as $pre) {
$this->nameFixer->done = $this->nameFixer->matched = false;
$searched = 0;
$ftmatched = $this->nameFixer->matchPredbFT($pre, true, 1, true);
if ($ftmatched > 0) {
$searched = 1;
} elseif ($ftmatched < 0) {
$searched = -6;
} else {
$searched = $pre['searched'] - 1;
}
Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]);
$this->checked++;
$bar->advance();
}
$bar->finish();
$this->newLine(2);
$this->info("✅ Processed {$this->checked} PreDB entries");
return Command::SUCCESS;
}
/**
* Fetch releases for processing
*/
protected function fetchReleases(string $guidChar, int $maxPerRun)
{
return Release::fromQuery(sprintf("
SELECT
r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo,
r.proc_uid, r.proc_files, r.proc_par2, r.ishashed, r.dehashstatus, r.nfostatus,
r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32,
IFNULL(rf.releases_id, 0) AS fileid, IF(rf.ishashed = 1, rf.name, 0) AS filehash,
IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring,
IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring,
IFNULL(ru.uniqueid, '') AS uid,
IFNULL(ph.hash, 0) AS hash,
IFNULL(rf.crc32, '') AS crc
FROM releases r
LEFT JOIN release_nfos rn ON rn.releases_id = r.id
LEFT JOIN release_files rf ON rf.releases_id = r.id
LEFT JOIN release_unique ru ON ru.releases_id = r.id
LEFT JOIN par_hashes ph ON ph.releases_id = r.id
WHERE r.leftguid = %s
AND r.isrenamed = %d
AND r.predb_id = 0
AND r.passwordstatus >= 0
AND r.nfostatus > %d
AND (
(r.nfostatus = %d AND r.proc_nfo = %d)
OR r.proc_files = %d
OR r.proc_uid = %d
OR r.proc_par2 = %d
OR r.proc_srr = %d
OR r.proc_hash16k = %d
OR r.proc_crc32 = %d
OR (r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0)
)
AND r.categories_id IN (%s)
GROUP BY r.id
ORDER BY r.id DESC
LIMIT %s",
escapeString($guidChar),
NameFixer::IS_RENAMED_NONE,
Nfo::NFO_UNPROC,
Nfo::NFO_FOUND,
NameFixer::PROC_NFO_NONE,
NameFixer::PROC_FILES_NONE,
NameFixer::PROC_UID_NONE,
NameFixer::PROC_PAR2_NONE,
NameFixer::PROC_SRR_NONE,
NameFixer::PROC_HASH16K_NONE,
NameFixer::PROC_CRC_NONE,
Category::getCategoryOthersGroup(),
$maxPerRun
));
}
}
+235
View File
@@ -0,0 +1,235 @@
<?php
namespace App\Console\Commands;
use App\Models\Collection;
use App\Models\Settings;
use App\Services\Tmux\TmuxMonitorService;
use App\Services\Tmux\TmuxSessionManager;
use App\Services\Tmux\TmuxTaskRunner;
use Blacklight\ColorCLI;
use Blacklight\TmuxOutput;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class TmuxMonitor extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'tmux:monitor
{--session= : Tmux session name}
{--reset-collections : Reset old collections before starting}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Monitor and manage tmux processing panes (modernized)';
private TmuxSessionManager $sessionManager;
private TmuxMonitorService $monitor;
private TmuxTaskRunner $taskRunner;
private TmuxOutput $tmuxOutput;
private ColorCLI $colorCli;
/**
* Execute the console command.
*/
public function handle(): int
{
$this->colorCli = new ColorCLI;
try {
// Reset old collections if requested
if ($this->option('reset-collections')) {
$this->resetOldCollections();
}
// Initialize services
$sessionName = $this->option('session')
?? Settings::settingValue('tmux_session')
?? config('tmux.session.default_name', 'nntmux');
$this->sessionManager = new TmuxSessionManager($sessionName);
$this->monitor = new TmuxMonitorService;
$this->taskRunner = new TmuxTaskRunner($sessionName);
$this->tmuxOutput = new TmuxOutput;
// Verify session exists
if (! $this->sessionManager->sessionExists()) {
$this->error("❌ Tmux session '{$sessionName}' does not exist.");
$this->info("💡 Run 'php artisan tmux-ui:start' to create the session first.");
return Command::FAILURE;
}
$this->colorCli->header('Starting Tmux Monitor');
$this->info("📊 Monitoring session: {$sessionName}");
// Initialize monitor
$runVar = $this->monitor->initializeMonitor();
// Main monitoring loop
$iteration = 0;
while ($this->monitor->shouldContinue()) {
$iteration++;
// Collect statistics
$runVar = $this->monitor->collectStatistics();
// Update display
$this->tmuxOutput->updateMonitorPane($runVar);
// Run pane tasks if tmux is running
if ((int) ($runVar['settings']['is_running'] ?? 0) === 1) {
$this->runPaneTasks($runVar);
} else {
if ($iteration % 60 === 0) { // Log every 10 minutes
$this->info('⏸️ Tmux is not running. Waiting...');
}
}
// Increment iteration and sleep
$this->monitor->incrementIteration();
sleep(10);
}
$this->info('🛑 Monitor stopped by exit flag');
return Command::SUCCESS;
} catch (\Exception $e) {
$this->error('❌ Monitor failed: '.$e->getMessage());
logger()->error('Tmux monitor error', [
'message' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
]);
return Command::FAILURE;
}
}
/**
* Reset old collections based on delay time
*/
private function resetOldCollections(): void
{
$delayTime = (int) (Settings::settingValue('delaytime') ?? 2);
$this->colorCli->header('Resetting expired collections...');
try {
DB::transaction(function () use ($delayTime) {
$count = Collection::query()
->where('dateadded', '<', now()->subHours($delayTime))
->update(['dateadded' => now()]);
if ($count > 0) {
$this->info("✅ Reset {$count} collections");
} else {
$this->info('✅ No collections needed resetting');
}
}, 10);
} catch (\Exception $e) {
$this->error('Failed to reset collections: '.$e->getMessage());
}
}
/**
* Run tasks in appropriate panes
*/
private function runPaneTasks(array $runVar): void
{
$sequential = (int) ($runVar['constants']['sequential'] ?? 0);
// Always run IRC scraper
$this->runIRCScraper($runVar);
// Run main tasks based on sequential mode
if ($sequential === 2) {
// Stripped mode - only essential tasks
$this->runSequentialTasks($runVar);
} elseif ($sequential === 1) {
// Basic sequential mode
$this->runBasicTasks($runVar);
} else {
// Full non-sequential mode
$this->runFullTasks($runVar);
}
}
/**
* Run IRC scraper
*/
private function runIRCScraper(array $runVar): void
{
$this->taskRunner->runIRCScraper([
'constants' => $runVar['constants'],
]);
}
/**
* Run full non-sequential tasks
*/
private function runFullTasks(array $runVar): void
{
// Update binaries
$this->taskRunner->runBinariesUpdate($runVar);
// Backfill
$this->taskRunner->runBackfill($runVar);
// Update releases
$this->taskRunner->runReleasesUpdate(array_merge($runVar, ['pane' => '0.3']));
// Post-processing and cleanup tasks
$this->runPostProcessingTasks($runVar);
}
/**
* Run basic sequential tasks
*/
private function runBasicTasks(array $runVar): void
{
// Update releases
$this->taskRunner->runReleasesUpdate(array_merge($runVar, ['pane' => '0.1']));
// Post-processing and cleanup tasks
$this->runPostProcessingTasks($runVar);
}
/**
* Run stripped sequential tasks
*/
private function runSequentialTasks(array $runVar): void
{
// Minimal tasks for complete sequential mode
// Tasks are handled by the sequential script itself
}
/**
* Run post-processing tasks (common to most modes)
*/
private function runPostProcessingTasks(array $runVar): void
{
$sequential = (int) ($runVar['constants']['sequential'] ?? 0);
if ($sequential === 2) {
// Skip post-processing in complete sequential mode
return;
}
// These tasks would be implemented in TmuxTaskRunner
// For now, we're keeping the structure similar to the original
// but using the modern service architecture
}
}
+186
View File
@@ -0,0 +1,186 @@
<?php
namespace App\Console\Commands;
use App\Models\Collection;
use App\Models\Settings;
use App\Services\Tmux\TmuxMonitorService;
use App\Services\Tmux\TmuxTaskRunner;
use Blacklight\ColorCLI;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class TmuxMonitorCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'tmux:monitor
{--session= : Tmux session name}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Monitor and manage tmux processing panes';
private TmuxMonitorService $monitor;
private TmuxTaskRunner $taskRunner;
private ColorCLI $colorCli;
/**
* Execute the console command.
*/
public function handle(): int
{
$this->colorCli = new ColorCLI;
try {
// Reset old collections
$this->resetOldCollections();
$sessionName = $this->option('session') ?? Settings::settingValue('tmux_session') ?? 'newznab';
// Initialize services
$this->monitor = new TmuxMonitorService;
$this->taskRunner = new TmuxTaskRunner($sessionName);
// Initialize monitor
$runVar = $this->monitor->initializeMonitor();
$this->colorCli->header('Starting Tmux Monitor');
$this->info("Monitoring session: {$sessionName}");
// Main monitoring loop
while ($this->monitor->shouldContinue()) {
// Collect statistics
$runVar = $this->monitor->collectStatistics();
// Update display
$this->monitor->updateDisplay();
// Run pane tasks if tmux is running
if ((int) ($runVar['settings']['is_running'] ?? 0) === 1) {
$this->runPaneTasks($runVar);
} else {
$this->info('Tmux is not running. Waiting...');
}
// Increment iteration and sleep
$this->monitor->incrementIteration();
sleep(10);
}
$this->info('Monitor stopped by exit flag');
return Command::SUCCESS;
} catch (\Exception $e) {
$this->error('Monitor failed: '.$e->getMessage());
$this->error($e->getTraceAsString());
return Command::FAILURE;
}
}
/**
* Reset old collections based on delay time
*/
private function resetOldCollections(): void
{
$delayTime = Settings::settingValue('delaytime') ?? 2;
$this->colorCli->header('Resetting expired collections. This may take some time...');
try {
DB::transaction(function () use ($delayTime) {
Collection::query()
->where('dateadded', '<', now()->subHours($delayTime))
->update(['dateadded' => now()]);
}, 10);
$this->info('Collections reset complete');
} catch (\Exception $e) {
$this->error('Failed to reset collections: '.$e->getMessage());
}
}
/**
* Run tasks in panes based on configuration
*/
private function runPaneTasks(array $runVar): void
{
$sequential = (int) ($runVar['constants']['sequential'] ?? 0);
// Define pane tasks based on sequential mode
$paneTasks = $this->getPaneTasks($sequential);
foreach ($paneTasks as $taskName => $config) {
try {
$this->taskRunner->runPaneTask($taskName, $config, $runVar);
} catch (\Exception $e) {
$this->error("Failed to run {$taskName}: ".$e->getMessage());
}
}
}
/**
* Get pane tasks based on sequential mode
*/
private function getPaneTasks(int $sequential): array
{
return match ($sequential) {
1 => $this->getSequentialBasicTasks(),
2 => $this->getSequentialFullTasks(),
default => $this->getStandardTasks(),
};
}
/**
* Get standard (non-sequential) tasks
*/
private function getStandardTasks(): array
{
return [
'main' => ['target' => '0.1'], // Handles binaries, backfill, and releases
'fixnames' => ['target' => '1.0'],
'removecrap' => ['target' => '1.1'],
'ppadditional' => ['target' => '2.0'],
'nonamazon' => ['target' => '2.1'],
'amazon' => ['target' => '2.2'],
'scraper' => ['target' => '3.0'],
];
}
/**
* Get sequential basic tasks
*/
private function getSequentialBasicTasks(): array
{
return [
'main' => ['target' => '0.1'],
'amazon' => ['target' => '2.2'],
'scraper' => ['target' => '3.0'],
'fixnames' => ['target' => '1.0'],
'removecrap' => ['target' => '1.1'],
'ppadditional' => ['target' => '2.0'],
'nonamazon' => ['target' => '2.1'],
];
}
/**
* Get sequential full tasks
*/
private function getSequentialFullTasks(): array
{
return [
'main' => ['target' => '0.1'],
'amazon' => ['target' => '0.2'],
'scraper' => ['target' => '3.0'],
];
}
}