From 31bf2188dee197c30497a42bb03fcf46f564b7da Mon Sep 17 00:00:00 2001 From: DariusIII Date: Sat, 1 Nov 2025 19:42:03 +0100 Subject: [PATCH] Add missing files --- app/Console/Commands/GroupsUpdate.php | 101 +++++ .../Commands/ReleasesFixNamesGroup.php | 369 ++++++++++++++++++ app/Console/Commands/TmuxMonitor.php | 235 +++++++++++ app/Console/Commands/TmuxMonitorCommand.php | 186 +++++++++ 4 files changed, 891 insertions(+) create mode 100644 app/Console/Commands/GroupsUpdate.php create mode 100644 app/Console/Commands/ReleasesFixNamesGroup.php create mode 100644 app/Console/Commands/TmuxMonitor.php create mode 100644 app/Console/Commands/TmuxMonitorCommand.php diff --git a/app/Console/Commands/GroupsUpdate.php b/app/Console/Commands/GroupsUpdate.php new file mode 100644 index 000000000..646e070db --- /dev/null +++ b/app/Console/Commands/GroupsUpdate.php @@ -0,0 +1,101 @@ +doConnect() !== true) { + $this->error('❌ Unable to connect to usenet server'); + + return Command::FAILURE; + } + + $this->info('πŸ“‘ Getting first/last for all active groups...'); + + try { + $data = $nntp->getGroups(); + + if ($nntp->isError($data)) { + $this->error('❌ Failed to getGroups() from NNTP server'); + + return Command::FAILURE; + } + + $this->info('πŸ”„ Updating short_groups table...'); + + // Truncate and rebuild + DB::statement('TRUNCATE TABLE short_groups'); + + // Get all active groups + $activeGroups = Arr::pluck( + UsenetGroup::query() + ->where('active', '=', 1) + ->orWhere('backfill', '=', 1) + ->get(['name']), + 'name' + ); + + $updated = 0; + $bar = $this->output->createProgressBar(count($data)); + $bar->start(); + + foreach ($data as $newgroup) { + if (\in_array($newgroup['group'], $activeGroups, false)) { + ShortGroup::query()->insert([ + 'name' => $newgroup['group'], + 'first_record' => $newgroup['first'], + 'last_record' => $newgroup['last'], + 'updated' => now(), + ]); + + $updated++; + } + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + + $elapsed = now()->diffInSeconds($start, true); + $this->info("βœ… Updated {$updated} groups"); + $this->info("⏱️ Running time: {$elapsed} seconds"); + + return Command::SUCCESS; + + } catch (\Exception $e) { + $this->error('❌ Update failed: '.$e->getMessage()); + + return Command::FAILURE; + } + } +} diff --git a/app/Console/Commands/ReleasesFixNamesGroup.php b/app/Console/Commands/ReleasesFixNamesGroup.php new file mode 100644 index 000000000..712ca7248 --- /dev/null +++ b/app/Console/Commands/ReleasesFixNamesGroup.php @@ -0,0 +1,369 @@ +argument('type'); + $maxPerRun = (int) $this->option('limit'); + + $this->nameFixer = new NameFixer; + + switch ($type) { + case 'standard': + return $this->processStandard($maxPerRun); + + case 'predbft': + return $this->processPredbFulltext($maxPerRun); + + default: + $this->error("Invalid type: {$type}. Use 'standard' or 'predbft'"); + + return Command::FAILURE; + } + } + + /** + * Process standard name fixing + */ + protected function processStandard(int $maxPerRun): int + { + $guidChar = $this->option('guid-char'); + + if ($guidChar === null) { + $this->error('--guid-char is required for standard type'); + + return Command::FAILURE; + } + + $this->info("Processing releases with GUID starting with: {$guidChar}"); + $this->info("Maximum per run: {$maxPerRun}"); + + // Allow for larger filename return sets + DB::statement('SET SESSION group_concat_max_len = 65536'); + + // Find releases to process + $releases = $this->fetchReleases($guidChar, $maxPerRun); + + if ($releases->isEmpty()) { + $this->info('No releases to process'); + + return Command::SUCCESS; + } + + $this->info("Found {$releases->count()} releases to process"); + $bar = $this->output->createProgressBar($releases->count()); + $bar->start(); + + $nntp = null; + $nzbcontents = null; + + foreach ($releases as $release) { + $this->checked++; + $this->nameFixer->reset(); + + // Process UID + if ((int) $release->proc_uid === NameFixer::PROC_UID_NONE && + (! empty($release->uid) || ! empty($release->mediainfo))) { + + if (! empty($release->uid)) { + $this->nameFixer->checkName($release, true, 'UID, ', 1, true); + } + + if (empty($this->nameFixer->matched) && ! empty($release->mediainfo)) { + $this->nameFixer->checkName($release, true, 'Mediainfo, ', 1, true); + } + } + + $this->nameFixer->_updateSingleColumn('proc_uid', NameFixer::PROC_UID_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process CRC32 + if ((int) $release->proc_crc32 === NameFixer::PROC_CRC_NONE && ! empty($release->crc)) { + $this->nameFixer->reset(); + $this->nameFixer->checkName($release, true, 'CRC32, ', 1, true); + } + + $this->nameFixer->_updateSingleColumn('proc_crc32', NameFixer::PROC_CRC_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process SRR + if ((int) $release->proc_srr === NameFixer::PROC_SRR_NONE) { + $this->nameFixer->reset(); + $this->nameFixer->checkName($release, true, 'SRR, ', 1, true); + } + + $this->nameFixer->_updateSingleColumn('proc_srr', NameFixer::PROC_SRR_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process PAR2 hash + if ((int) $release->proc_hash16k === NameFixer::PROC_HASH16K_NONE && ! empty($release->hash)) { + $this->nameFixer->reset(); + $this->nameFixer->checkName($release, true, 'PAR2 hash, ', 1, true); + } + + $this->nameFixer->_updateSingleColumn('proc_hash16k', NameFixer::PROC_HASH16K_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process NFO + if ((int) $release->nfostatus === Nfo::NFO_FOUND && + (int) $release->proc_nfo === NameFixer::PROC_NFO_NONE && + ! empty($release->textstring) && + ! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) { + + $this->nameFixer->reset(); + $this->nameFixer->checkName($release, true, 'NFO, ', 1, true); + } + + $this->nameFixer->_updateSingleColumn('proc_nfo', NameFixer::PROC_NFO_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process filenames + if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixer::PROC_FILES_NONE) { + $this->nameFixer->reset(); + $fileNames = explode('|', $release->filestring); + + if (is_array($fileNames)) { + $releaseFile = $release; + foreach ($fileNames as $fileName) { + if ($this->nameFixer->matched === false) { + $releaseFile->textstring = $fileName; + $this->nameFixer->checkName($releaseFile, true, 'Filenames, ', 1, true); + } + } + } + } + + $this->nameFixer->_updateSingleColumn('proc_files', NameFixer::PROC_FILES_DONE, $release->releases_id); + + if ($this->nameFixer->matched) { + $bar->advance(); + + continue; + } + + // Process PAR2 + if ((int) $release->proc_par2 === NameFixer::PROC_PAR2_NONE) { + // Initialize NZB contents if needed + if (! isset($nzbcontents)) { + $nntp = new NNTP; + $compressedHeaders = config('nntmux_nntp.compressed_headers'); + + if ((config('nntmux_nntp.use_alternate_nntp_server') === true + ? $nntp->doConnect($compressedHeaders, true) + : $nntp->doConnect()) !== true) { + $this->warn('Unable to connect to usenet for PAR2 processing'); + } else { + $Nfo = new Nfo; + $nzbcontents = new NZBContents([ + 'Echo' => false, + 'NNTP' => $nntp, + 'Nfo' => $Nfo, + 'PostProcess' => new PostProcess(['Nfo' => $Nfo, 'NameFixer' => $this->nameFixer]), + ]); + } + } + + if (isset($nzbcontents)) { + $nzbcontents->checkPAR2($release->guid, $release->releases_id, $release->groups_id, 1, true); + } + } + + $this->nameFixer->_updateSingleColumn('proc_par2', NameFixer::PROC_PAR2_DONE, $release->releases_id); + + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + + $this->info("βœ… Processed {$this->checked} releases"); + $this->info("βœ… Fixed {$this->nameFixer->fixed} release names"); + + return Command::SUCCESS; + } + + /** + * Process PreDB fulltext matching + */ + protected function processPredbFulltext(int $maxPerRun): int + { + $thread = (int) $this->option('thread'); + $offset = $thread * $maxPerRun - $maxPerRun; + + $this->info('Processing PreDB fulltext matching'); + $this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}"); + + $pres = Predb::fromQuery( + sprintf( + ' + SELECT p.id AS predb_id, p.title, p.source, p.searched + FROM predb p + WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]" + AND p.searched = 0 + AND p.predate < (NOW() - INTERVAL 1 DAY) + ORDER BY p.predate ASC + LIMIT %s + OFFSET %s', + $maxPerRun, + $offset + ) + ); + + if ($pres->isEmpty()) { + $this->info('No PreDB entries to process'); + + return Command::SUCCESS; + } + + $this->info("Found {$pres->count()} PreDB entries to process"); + $bar = $this->output->createProgressBar($pres->count()); + $bar->start(); + + foreach ($pres as $pre) { + $this->nameFixer->done = $this->nameFixer->matched = false; + $searched = 0; + + $ftmatched = $this->nameFixer->matchPredbFT($pre, true, 1, true); + + if ($ftmatched > 0) { + $searched = 1; + } elseif ($ftmatched < 0) { + $searched = -6; + } else { + $searched = $pre['searched'] - 1; + } + + Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]); + $this->checked++; + + $bar->advance(); + } + + $bar->finish(); + $this->newLine(2); + + $this->info("βœ… Processed {$this->checked} PreDB entries"); + + return Command::SUCCESS; + } + + /** + * Fetch releases for processing + */ + protected function fetchReleases(string $guidChar, int $maxPerRun) + { + return Release::fromQuery(sprintf(" + SELECT + r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo, + r.proc_uid, r.proc_files, r.proc_par2, r.ishashed, r.dehashstatus, r.nfostatus, + r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32, + IFNULL(rf.releases_id, 0) AS fileid, IF(rf.ishashed = 1, rf.name, 0) AS filehash, + IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring, + IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring, + IFNULL(ru.uniqueid, '') AS uid, + IFNULL(ph.hash, 0) AS hash, + IFNULL(rf.crc32, '') AS crc + FROM releases r + LEFT JOIN release_nfos rn ON rn.releases_id = r.id + LEFT JOIN release_files rf ON rf.releases_id = r.id + LEFT JOIN release_unique ru ON ru.releases_id = r.id + LEFT JOIN par_hashes ph ON ph.releases_id = r.id + WHERE r.leftguid = %s + AND r.isrenamed = %d + AND r.predb_id = 0 + AND r.passwordstatus >= 0 + AND r.nfostatus > %d + AND ( + (r.nfostatus = %d AND r.proc_nfo = %d) + OR r.proc_files = %d + OR r.proc_uid = %d + OR r.proc_par2 = %d + OR r.proc_srr = %d + OR r.proc_hash16k = %d + OR r.proc_crc32 = %d + OR (r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0) + ) + AND r.categories_id IN (%s) + GROUP BY r.id + ORDER BY r.id DESC + LIMIT %s", + escapeString($guidChar), + NameFixer::IS_RENAMED_NONE, + Nfo::NFO_UNPROC, + Nfo::NFO_FOUND, + NameFixer::PROC_NFO_NONE, + NameFixer::PROC_FILES_NONE, + NameFixer::PROC_UID_NONE, + NameFixer::PROC_PAR2_NONE, + NameFixer::PROC_SRR_NONE, + NameFixer::PROC_HASH16K_NONE, + NameFixer::PROC_CRC_NONE, + Category::getCategoryOthersGroup(), + $maxPerRun + )); + } +} diff --git a/app/Console/Commands/TmuxMonitor.php b/app/Console/Commands/TmuxMonitor.php new file mode 100644 index 000000000..13ec598cc --- /dev/null +++ b/app/Console/Commands/TmuxMonitor.php @@ -0,0 +1,235 @@ +colorCli = new ColorCLI; + + try { + // Reset old collections if requested + if ($this->option('reset-collections')) { + $this->resetOldCollections(); + } + + // Initialize services + $sessionName = $this->option('session') + ?? Settings::settingValue('tmux_session') + ?? config('tmux.session.default_name', 'nntmux'); + + $this->sessionManager = new TmuxSessionManager($sessionName); + $this->monitor = new TmuxMonitorService; + $this->taskRunner = new TmuxTaskRunner($sessionName); + $this->tmuxOutput = new TmuxOutput; + + // Verify session exists + if (! $this->sessionManager->sessionExists()) { + $this->error("❌ Tmux session '{$sessionName}' does not exist."); + $this->info("πŸ’‘ Run 'php artisan tmux-ui:start' to create the session first."); + + return Command::FAILURE; + } + + $this->colorCli->header('Starting Tmux Monitor'); + $this->info("πŸ“Š Monitoring session: {$sessionName}"); + + // Initialize monitor + $runVar = $this->monitor->initializeMonitor(); + + // Main monitoring loop + $iteration = 0; + while ($this->monitor->shouldContinue()) { + $iteration++; + + // Collect statistics + $runVar = $this->monitor->collectStatistics(); + + // Update display + $this->tmuxOutput->updateMonitorPane($runVar); + + // Run pane tasks if tmux is running + if ((int) ($runVar['settings']['is_running'] ?? 0) === 1) { + $this->runPaneTasks($runVar); + } else { + if ($iteration % 60 === 0) { // Log every 10 minutes + $this->info('⏸️ Tmux is not running. Waiting...'); + } + } + + // Increment iteration and sleep + $this->monitor->incrementIteration(); + sleep(10); + } + + $this->info('πŸ›‘ Monitor stopped by exit flag'); + + return Command::SUCCESS; + + } catch (\Exception $e) { + $this->error('❌ Monitor failed: '.$e->getMessage()); + logger()->error('Tmux monitor error', [ + 'message' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + + return Command::FAILURE; + } + } + + /** + * Reset old collections based on delay time + */ + private function resetOldCollections(): void + { + $delayTime = (int) (Settings::settingValue('delaytime') ?? 2); + + $this->colorCli->header('Resetting expired collections...'); + + try { + DB::transaction(function () use ($delayTime) { + $count = Collection::query() + ->where('dateadded', '<', now()->subHours($delayTime)) + ->update(['dateadded' => now()]); + + if ($count > 0) { + $this->info("βœ… Reset {$count} collections"); + } else { + $this->info('βœ… No collections needed resetting'); + } + }, 10); + + } catch (\Exception $e) { + $this->error('Failed to reset collections: '.$e->getMessage()); + } + } + + /** + * Run tasks in appropriate panes + */ + private function runPaneTasks(array $runVar): void + { + $sequential = (int) ($runVar['constants']['sequential'] ?? 0); + + // Always run IRC scraper + $this->runIRCScraper($runVar); + + // Run main tasks based on sequential mode + if ($sequential === 2) { + // Stripped mode - only essential tasks + $this->runSequentialTasks($runVar); + } elseif ($sequential === 1) { + // Basic sequential mode + $this->runBasicTasks($runVar); + } else { + // Full non-sequential mode + $this->runFullTasks($runVar); + } + } + + /** + * Run IRC scraper + */ + private function runIRCScraper(array $runVar): void + { + $this->taskRunner->runIRCScraper([ + 'constants' => $runVar['constants'], + ]); + } + + /** + * Run full non-sequential tasks + */ + private function runFullTasks(array $runVar): void + { + // Update binaries + $this->taskRunner->runBinariesUpdate($runVar); + + // Backfill + $this->taskRunner->runBackfill($runVar); + + // Update releases + $this->taskRunner->runReleasesUpdate(array_merge($runVar, ['pane' => '0.3'])); + + // Post-processing and cleanup tasks + $this->runPostProcessingTasks($runVar); + } + + /** + * Run basic sequential tasks + */ + private function runBasicTasks(array $runVar): void + { + // Update releases + $this->taskRunner->runReleasesUpdate(array_merge($runVar, ['pane' => '0.1'])); + + // Post-processing and cleanup tasks + $this->runPostProcessingTasks($runVar); + } + + /** + * Run stripped sequential tasks + */ + private function runSequentialTasks(array $runVar): void + { + // Minimal tasks for complete sequential mode + // Tasks are handled by the sequential script itself + } + + /** + * Run post-processing tasks (common to most modes) + */ + private function runPostProcessingTasks(array $runVar): void + { + $sequential = (int) ($runVar['constants']['sequential'] ?? 0); + + if ($sequential === 2) { + // Skip post-processing in complete sequential mode + return; + } + + // These tasks would be implemented in TmuxTaskRunner + // For now, we're keeping the structure similar to the original + // but using the modern service architecture + } +} diff --git a/app/Console/Commands/TmuxMonitorCommand.php b/app/Console/Commands/TmuxMonitorCommand.php new file mode 100644 index 000000000..95f32300f --- /dev/null +++ b/app/Console/Commands/TmuxMonitorCommand.php @@ -0,0 +1,186 @@ +colorCli = new ColorCLI; + + try { + // Reset old collections + $this->resetOldCollections(); + + $sessionName = $this->option('session') ?? Settings::settingValue('tmux_session') ?? 'newznab'; + + // Initialize services + $this->monitor = new TmuxMonitorService; + $this->taskRunner = new TmuxTaskRunner($sessionName); + + // Initialize monitor + $runVar = $this->monitor->initializeMonitor(); + + $this->colorCli->header('Starting Tmux Monitor'); + $this->info("Monitoring session: {$sessionName}"); + + // Main monitoring loop + while ($this->monitor->shouldContinue()) { + // Collect statistics + $runVar = $this->monitor->collectStatistics(); + + // Update display + $this->monitor->updateDisplay(); + + // Run pane tasks if tmux is running + if ((int) ($runVar['settings']['is_running'] ?? 0) === 1) { + $this->runPaneTasks($runVar); + } else { + $this->info('Tmux is not running. Waiting...'); + } + + // Increment iteration and sleep + $this->monitor->incrementIteration(); + sleep(10); + } + + $this->info('Monitor stopped by exit flag'); + + return Command::SUCCESS; + } catch (\Exception $e) { + $this->error('Monitor failed: '.$e->getMessage()); + $this->error($e->getTraceAsString()); + + return Command::FAILURE; + } + } + + /** + * Reset old collections based on delay time + */ + private function resetOldCollections(): void + { + $delayTime = Settings::settingValue('delaytime') ?? 2; + + $this->colorCli->header('Resetting expired collections. This may take some time...'); + + try { + DB::transaction(function () use ($delayTime) { + Collection::query() + ->where('dateadded', '<', now()->subHours($delayTime)) + ->update(['dateadded' => now()]); + }, 10); + + $this->info('Collections reset complete'); + } catch (\Exception $e) { + $this->error('Failed to reset collections: '.$e->getMessage()); + } + } + + /** + * Run tasks in panes based on configuration + */ + private function runPaneTasks(array $runVar): void + { + $sequential = (int) ($runVar['constants']['sequential'] ?? 0); + + // Define pane tasks based on sequential mode + $paneTasks = $this->getPaneTasks($sequential); + + foreach ($paneTasks as $taskName => $config) { + try { + $this->taskRunner->runPaneTask($taskName, $config, $runVar); + } catch (\Exception $e) { + $this->error("Failed to run {$taskName}: ".$e->getMessage()); + } + } + } + + /** + * Get pane tasks based on sequential mode + */ + private function getPaneTasks(int $sequential): array + { + return match ($sequential) { + 1 => $this->getSequentialBasicTasks(), + 2 => $this->getSequentialFullTasks(), + default => $this->getStandardTasks(), + }; + } + + /** + * Get standard (non-sequential) tasks + */ + private function getStandardTasks(): array + { + return [ + 'main' => ['target' => '0.1'], // Handles binaries, backfill, and releases + 'fixnames' => ['target' => '1.0'], + 'removecrap' => ['target' => '1.1'], + 'ppadditional' => ['target' => '2.0'], + 'nonamazon' => ['target' => '2.1'], + 'amazon' => ['target' => '2.2'], + 'scraper' => ['target' => '3.0'], + ]; + } + + /** + * Get sequential basic tasks + */ + private function getSequentialBasicTasks(): array + { + return [ + 'main' => ['target' => '0.1'], + 'amazon' => ['target' => '2.2'], + 'scraper' => ['target' => '3.0'], + 'fixnames' => ['target' => '1.0'], + 'removecrap' => ['target' => '1.1'], + 'ppadditional' => ['target' => '2.0'], + 'nonamazon' => ['target' => '2.1'], + ]; + } + + /** + * Get sequential full tasks + */ + private function getSequentialFullTasks(): array + { + return [ + 'main' => ['target' => '0.1'], + 'amazon' => ['target' => '0.2'], + 'scraper' => ['target' => '3.0'], + ]; + } +}