echoCLI = (bool) config('nntmux.echocli'); $this->nzb = $nzb ?? app(NzbService::class); $this->releaseCleaning = $releaseCleaning ?? new ReleaseCleaningService; $this->releaseManagement = $releaseManagement ?? app(ReleaseManagementService::class); $this->releaseImage = $releaseImage ?? new ReleaseImageService; $this->collectionCleanupService = $collectionCleanupService ?? new CollectionCleanupService; $this->releaseCreationService = $releaseCreationService ?? new ReleaseCreationService( $this->releaseCleaning, $this->collectionCleanupService, app(ReleaseDuplicateFinder::class) ); $this->postProcessService = $postProcessService; $this->binariesConfig = $binariesConfig ?? BinariesConfig::fromSettings(); $this->settings = $this->loadSettings(); $this->validateSettings(); } /** * Load all required settings from database. */ private function loadSettings(): ProcessReleasesSettings { $settingKeys = [ 'delaytime', 'crossposttime', 'maxnzbsprocessed', 'completionpercent', 'collection_timeout', 'maxsizetoformrelease', 'minsizetoformrelease', 'minfilestoformrelease', 'releaseretentiondays', 'deletepasswordedrelease', 'miscotherretentionhours', 'mischashedretentionhours', 'partretentionhours', 'last_run_time', ]; $dbSettings = []; foreach ($settingKeys as $key) { $dbSettings[$key] = Settings::settingValue($key); } return ProcessReleasesSettings::forDatabase($dbSettings); } /** * Validate loaded settings and warn about invalid configurations. */ private function validateSettings(): void { if (! $this->settings->hasValidCompletion()) { cli()->error( PHP_EOL.'Invalid completion setting. Value must be between 0 and 100.' ); } } // ======================================================================== // Public API // ======================================================================== /** * Get the current completion percentage setting. */ public function getCompletion(): int { return $this->settings->completion; } /** * Get the release creation limit. */ public function getReleaseCreationLimit(): int { return $this->settings->releaseCreationLimit; } /** * Get the collection delay time in hours. */ public function getCollectionDelayTime(): int { return $this->settings->collectionDelayTime; } /** * Get the cross-post detection time window in hours. */ public function getCrossPostTime(): int { return $this->settings->crossPostTime; } /** * Check if CLI echo is enabled. */ public function isEchoCLI(): bool { return $this->echoCLI; } /** * Set CLI echo mode. */ public function setEchoCLI(bool $echo): self { $this->echoCLI = $echo; return $this; } /** * Main method for creating releases/NZB files from collections. * * @param int $categorize Categorization type (1=name, 2=searchname) * @param int $postProcess Whether to run post-processing (1=yes) * @param string $groupName Optional group name to filter processing * @param NNTPService $nntp NNTP connection for post-processing * @return int Total number of releases added * * @throws Throwable */ public function processReleases( int $categorize, int $postProcess, string $groupName, NNTPService $nntp ): int { $this->echoCLI = (bool) config('nntmux.echocli'); $overallStartTime = now()->toImmutable(); $this->outputBanner(); if (! $this->validateNzbPath()) { return 0; } $groupID = $this->resolveGroupId($groupName); $normalizedGroupId = $this->normalizeGroupId($groupID); if ($this->echoCLI && $groupName !== '') { $this->outputInfo("Processing group: {$groupName}"); } // Phase 1: Collection processing $this->outputHeader('Phase 1: Collection Processing'); $this->processIncompleteCollections($normalizedGroupId); $this->processCollectionSizes($normalizedGroupId); $this->deleteUnwantedCollections($normalizedGroupId); // Phase 2: Release creation loop $this->outputHeader('Phase 2: Release Creation'); $totals = $this->runReleaseCreationLoop($normalizedGroupId, $categorize, $postProcess, $nntp); // Phase 3: Cleanup $this->outputHeader('Phase 3: Cleanup'); $this->deleteReleases(); $this->outputFinalSummary( $totals['releases'], $totals['nzbs'], $totals['dupes'], $totals['iterations'], $overallStartTime ); return $totals['releases']; } /** * Run the release creation loop. * * @return array{releases: int, nzbs: int, dupes: int, iterations: int} * * @throws Throwable */ private function runReleaseCreationLoop( ?int $normalizedGroupId, int $categorize, int $postProcess, NNTPService $nntp ): array { $totals = ['releases' => 0, 'nzbs' => 0, 'dupes' => 0, 'iterations' => 0]; $limit = $this->settings->releaseCreationLimit; do { $totals['iterations']++; $result = $this->createReleases($normalizedGroupId); $totals['releases'] += $result->added; $totals['dupes'] += $result->dupes; $nzbFilesAdded = $this->createNZBs($normalizedGroupId); $totals['nzbs'] += $nzbFilesAdded; $this->categorizeReleases($categorize, $normalizedGroupId); $this->postProcessReleases($postProcess, $nntp); $this->deleteCollections($normalizedGroupId); $shouldContinue = $result->total() >= $limit || $nzbFilesAdded >= $limit; } while ($shouldContinue); return $totals; } /** * Reset all releases to other->misc category. */ public function resetCategorize(string $where = ''): void { if ($where !== '') { DB::update( 'UPDATE releases SET categories_id = ?, iscategorized = 0 '.$where, [Category::OTHER_MISC] ); } else { Release::query()->update([ 'categories_id' => Category::OTHER_MISC, 'iscategorized' => 0, ]); } ReleaseSearchIndexSync::reindexMatchingWhere($where); } /** * Categorize a release using the specified type. * * @throws \Exception */ public function categorizeRelease(string $type, int|string|null $groupId): int { $categorizer = new CategorizationService; $categorized = 0; $query = Release::query() ->where('categories_id', Category::OTHER_MISC) ->where('iscategorized', 0) ->select(['id', 'fromname', 'groups_id', $type]); if (! empty($groupId)) { $query->where('groups_id', $groupId); } $total = $query->count(); if ($total === 0) { return 0; } $this->outputSubHeader('Categorizing Releases'); $query->chunkById(self::CATEGORIZE_CHUNK_SIZE, function ($releases) use ($categorizer, $type, &$categorized, $total): bool { foreach ($releases as $release) { $categoryResult = $categorizer->determineCategory( $release->groups_id, $release->{$type}, $release->fromname ); Release::query() ->where('id', $release->id) ->update([ 'categories_id' => $categoryResult['categories_id'], 'iscategorized' => 1, ]); ReleaseSearchIndexSync::forIds([(int) $release->id]); $categorized++; $this->outputProgress($categorized, $total, 'Categorizing'); } return true; }); return $categorized; } /** * Process incomplete collections to find complete ones. * * @throws Throwable */ public function processIncompleteCollections(int|string|null $groupID): void { $startTime = now()->toImmutable(); $this->outputSubHeader('Finding Complete Collections'); $normalizedGroupId = $this->normalizeGroupId($groupID); $this->processStuckCollections($normalizedGroupId ?? 0); $this->reconcileIncompleteCollections($normalizedGroupId); $count = $this->countCompleteCollections($normalizedGroupId); $this->outputStat('Complete collections found', $count); $this->outputElapsedTime($startTime); } /** * Calculate sizes for complete collections. * * @throws Throwable */ public function processCollectionSizes(int|string|null $groupID): void { $startTime = now()->toImmutable(); $this->outputSubHeader('Calculating Collection Sizes'); $updated = 0; $lastId = 0; $normalizedGroupId = $this->normalizeGroupId($groupID); do { $query = Collection::query() ->where('id', '>', $lastId) ->where('filecheck', CollectionFileCheckStatus::CompleteParts->value) ->when($normalizedGroupId !== null, static fn ($q) => $q->where('groups_id', $normalizedGroupId)) ->orderBy('id') ->limit($this->binariesConfig->reconcileBatchSize); $ids = $query->pluck('id')->map(static fn ($id): int => (int) $id)->all(); if ($ids === []) { break; } $lastId = (int) end($ids); $updated += Collection::query()->whereIn('id', $ids)->update([ 'filecheck' => CollectionFileCheckStatus::Sized->value, ]); } while (\count($ids) === $this->binariesConfig->reconcileBatchSize); $this->outputStat('Collections sized', $updated); $this->outputElapsedTime($startTime); } /** * Reconcile only a bounded keyset page at a time. Stored parts are the * authority for binary counts/sizes; binary aggregates are then the * authority for collection readiness and filesize. */ private function reconcileIncompleteCollections(?int $groupId): void { $lastId = 0; $statuses = [ CollectionFileCheckStatus::Default->value, CollectionFileCheckStatus::CompleteCollection->value, CollectionFileCheckStatus::CompleteParts->value, CollectionFileCheckStatus::TempComplete->value, CollectionFileCheckStatus::ZeroPart->value, 10, ]; do { $query = Collection::query() ->where('id', '>', $lastId) ->where(function ($query) use ($statuses): void { $query->where('filecheck', CollectionFileCheckStatus::CompleteParts->value); if (Schema::hasColumn('collections', 'last_seen_at')) { $query->orWhere(function ($stale) use ($statuses): void { $stale->whereIn('filecheck', array_values(array_diff( $statuses, [CollectionFileCheckStatus::CompleteParts->value] )))->whereRaw( 'COALESCE(last_seen_at, dateadded, added) < ?', [now()->subHours($this->settings->collectionDelayTime)] ); }); } else { $query->orWhereIn('filecheck', array_values(array_diff( $statuses, [CollectionFileCheckStatus::CompleteParts->value] ))); } }) ->when($groupId !== null, static fn ($q) => $q->where('groups_id', $groupId)) ->orderBy('id') ->limit($this->binariesConfig->reconcileBatchSize); $ids = $query->pluck('id')->map(static fn ($id): int => (int) $id)->all(); if ($ids === []) { break; } $lastId = (int) end($ids); $this->reconcileCollectionIds($ids, $statuses); } while (\count($ids) === $this->binariesConfig->reconcileBatchSize); } /** * @param list $collectionIds * @param list $statuses */ private function reconcileCollectionIds(array $collectionIds, array $statuses): void { if (DB::getDriverName() === 'sqlite') { $this->reconcileCollectionIdsSqlite($collectionIds, $statuses); return; } $idPlaceholders = implode(',', array_fill(0, \count($collectionIds), '?')); $statusPlaceholders = implode(',', array_fill(0, \count($statuses), '?')); DB::transaction(function () use ($collectionIds, $idPlaceholders, $statuses, $statusPlaceholders): void { DB::update( "UPDATE binaries b LEFT JOIN ( SELECT p.binaries_id, COUNT(*) currentparts, COALESCE(SUM(p.size), 0) partsize FROM parts p INNER JOIN binaries selected ON selected.id = p.binaries_id WHERE selected.collections_id IN ({$idPlaceholders}) GROUP BY p.binaries_id ) p ON p.binaries_id = b.id SET b.currentparts = COALESCE(p.currentparts, 0), b.partsize = COALESCE(p.partsize, 0), b.partcheck = CASE WHEN COALESCE(p.currentparts, 0) >= b.totalparts THEN 1 ELSE 0 END WHERE b.collections_id IN ({$idPlaceholders})", [...$collectionIds, ...$collectionIds] ); DB::update( "UPDATE collections c LEFT JOIN ( SELECT b.collections_id, COUNT(*) currentfiles, COALESCE(SUM(CASE WHEN b.partcheck = 1 THEN 1 ELSE 0 END), 0) completefiles, COALESCE(SUM(b.partsize), 0) filesize FROM binaries b WHERE b.collections_id IN ({$idPlaceholders}) GROUP BY b.collections_id ) a ON a.collections_id = c.id SET c.filesize = COALESCE(a.filesize, 0), c.totalfiles = CASE WHEN c.dateadded < DATE_SUB(NOW(), INTERVAL ? HOUR) AND c.filecheck IN (0, 1, 10) THEN COALESCE(a.currentfiles, 0) ELSE c.totalfiles END, c.filecheck = CASE WHEN c.totalfiles > 0 AND COALESCE(a.currentfiles, 0) IN (c.totalfiles, c.totalfiles + 1) AND COALESCE(a.completefiles, 0) >= c.totalfiles THEN ? WHEN c.dateadded < DATE_SUB(NOW(), INTERVAL ? HOUR) AND c.filecheck IN (0, 1, 10) THEN ? ELSE c.filecheck END WHERE c.id IN ({$idPlaceholders}) AND c.filecheck IN ({$statusPlaceholders})", [ ...$collectionIds, $this->settings->collectionDelayTime, CollectionFileCheckStatus::CompleteParts->value, $this->settings->collectionDelayTime, CollectionFileCheckStatus::CompleteParts->value, ...$collectionIds, ...$statuses, ] ); }, self::MAX_RETRIES); } /** * @param list $collectionIds * @param list $statuses */ private function reconcileCollectionIdsSqlite(array $collectionIds, array $statuses): void { DB::transaction(function () use ($collectionIds, $statuses): void { foreach ($collectionIds as $collectionId) { $binaryIds = DB::table('binaries')->where('collections_id', $collectionId)->pluck('id')->all(); foreach ($binaryIds as $binaryId) { $aggregate = DB::selectOne( 'SELECT COUNT(*) currentparts, COALESCE(SUM(size), 0) partsize FROM parts WHERE binaries_id = ?', [$binaryId] ); DB::update( 'UPDATE binaries SET currentparts = ?, partsize = ?, partcheck = CASE WHEN ? >= totalparts THEN 1 ELSE 0 END WHERE id = ?', [(int) $aggregate->currentparts, (int) $aggregate->partsize, (int) $aggregate->currentparts, $binaryId] ); } $collection = DB::table('collections')->where('id', $collectionId)->first(); if ($collection === null || ! \in_array((int) $collection->filecheck, $statuses, true)) { continue; } $aggregate = DB::selectOne( 'SELECT COUNT(*) currentfiles, COALESCE(SUM(CASE WHEN partcheck = 1 THEN 1 ELSE 0 END), 0) completefiles, COALESCE(SUM(partsize), 0) filesize FROM binaries WHERE collections_id = ?', [$collectionId] ); $stale = strtotime((string) $collection->dateadded) < now()->subHours($this->settings->collectionDelayTime)->timestamp && \in_array((int) $collection->filecheck, [0, 1, 10], true); $totalFiles = $stale ? (int) $aggregate->currentfiles : (int) $collection->totalfiles; $ready = $totalFiles > 0 && \in_array((int) $aggregate->currentfiles, [$totalFiles, $totalFiles + 1], true) && (int) $aggregate->completefiles >= $totalFiles; DB::table('collections')->where('id', $collectionId)->update([ 'filesize' => (int) $aggregate->filesize, 'totalfiles' => $totalFiles, 'filecheck' => $ready || $stale ? CollectionFileCheckStatus::CompleteParts->value : (int) $collection->filecheck, ]); } }, self::MAX_RETRIES); } /** * Delete collections that don't meet size/file count requirements. * * @throws Throwable */ public function deleteUnwantedCollections(int|string|null $groupID): void { $startTime = now()->toImmutable(); $this->outputSubHeader('Filtering Collections by Size/File Count'); $normalizedGroupId = $this->normalizeGroupId($groupID); $groupIDs = $normalizedGroupId === null ? UsenetGroup::getActiveIDs() : [['id' => $normalizedGroupId]]; $stats = ['minSize' => 0, 'maxSize' => 0, 'minFiles' => 0, 'par2Only' => 0]; // Delete collections where ALL binaries are par2 files (no actual content) $par2OnlyCollectionIds = DB::table('collections as c') ->join('binaries as b', 'c.id', '=', 'b.collections_id') ->where('c.filecheck', CollectionFileCheckStatus::Sized->value) ->where('c.filesize', '>', 0) ->groupBy('c.id') ->havingRaw("COUNT(b.id) = SUM(CASE WHEN b.name REGEXP '\\\\.(vol[0-9]+\\\\+[0-9]+\\\\.par2|par2)' THEN 1 ELSE 0 END)"); if ($normalizedGroupId !== null) { $par2OnlyCollectionIds->where('c.groups_id', $normalizedGroupId); } $stats['par2Only'] += $this->deleteCollectionQueryInBatches($par2OnlyCollectionIds, 'Par2-only cleanup'); foreach ($groupIDs as $grpID) { $groupSettings = UsenetGroup::getGroupByID($grpID['id']); $groupMinSize = (int) ($groupSettings['minsizetoformrelease'] ?? 0); $groupMinFiles = (int) ($groupSettings['minfilestoformrelease'] ?? 0); if (! $this->hasSizedCollections((int) $grpID['id'])) { continue; } $effectiveMinSize = max($groupMinSize, $this->settings->minSizeToFormRelease); if ($effectiveMinSize > 0) { $ids = Collection::query() ->where('filecheck', CollectionFileCheckStatus::Sized->value) ->where('groups_id', (int) $grpID['id']) ->where('filesize', '>', 0) ->where('filesize', '<', $effectiveMinSize); $stats['minSize'] += $this->deleteCollectionQueryInBatches($ids, 'Min-size cleanup'); } if ($this->settings->maxSizeToFormRelease > 0) { $ids = Collection::query() ->where('filecheck', CollectionFileCheckStatus::Sized->value) ->where('groups_id', (int) $grpID['id']) ->where('filesize', '>', $this->settings->maxSizeToFormRelease); $stats['maxSize'] += $this->deleteCollectionQueryInBatches($ids, 'Max-size cleanup'); } $effectiveMinFiles = max($groupMinFiles, $this->settings->minFilesToFormRelease); if ($effectiveMinFiles > 0) { $ids = Collection::query() ->where('filecheck', CollectionFileCheckStatus::Sized->value) ->where('groups_id', (int) $grpID['id']) ->where('filesize', '>', 0) ->where('totalfiles', '<', $effectiveMinFiles); $stats['minFiles'] += $this->deleteCollectionQueryInBatches($ids, 'Min-files cleanup'); } } $this->outputCollectionDeleteStats($stats, $startTime); } /** * Create releases from complete collections. * * @throws Throwable */ public function createReleases(int|string|null $groupID): ReleaseCreationResult { $result = $this->releaseCreationService->createReleases( $groupID, $this->settings->releaseCreationLimit, $this->echoCLI ); return ReleaseCreationResult::from($result); } /** * Create NZB files from releases that don't have them yet. * * @throws Throwable */ public function createNZBs(int|string|null $groupID): int { $startTime = now()->toImmutable(); $this->outputSubHeader('Creating NZB Files'); $nzbCount = 0; $retryCount = 0; $deletedCount = 0; $claimToken = bin2hex(random_bytes(16)); $limit = max(1, $this->settings->releaseCreationLimit); $total = min(NzbCreationCandidateQuery::baseBuilder($groupID)->count(), $limit); if ($total > 0) { $columns = [ 'id', 'guid', 'name', 'categories_id', 'groups_id', 'postdate', 'nzbstatus', NzbCreationCandidateQuery::ATTEMPTS_COLUMN, NzbCreationCandidateQuery::CLAIM_TOKEN_COLUMN, ]; $processed = 0; $releases = NzbCreationCandidateQuery::claimBatch($groupID, $limit, $claimToken, $columns); foreach ($releases as $release) { try { $result = $this->nzb->createNzbForRelease($release); if ($result->success) { $nzbCount++; continue; } if ($this->shouldDeleteFailedNzbCreation($release, $result)) { $this->deleteFailedNzbCreationRelease($release, $result, $claimToken); $deletedCount++; continue; } $this->recordNzbCreationRetry($release, $result); $retryCount++; } catch (Throwable $e) { $result = NzbCreationResult::transient('Unexpected NZB creation failure: '.$e->getMessage()); if ($this->shouldDeleteFailedNzbCreation($release, $result)) { $this->deleteFailedNzbCreationRelease($release, $result, $claimToken); $deletedCount++; } else { $this->recordNzbCreationRetry($release, $result); $retryCount++; } } finally { NzbCreationCandidateQuery::clearClaim((int) $release->id, $claimToken); $processed++; $this->outputProgress($processed, $total, 'Creating NZBs'); } } } $this->outputStat('NZBs created', $nzbCount); $this->outputStat('NZB creation retries', $retryCount); $this->outputStat('NZB creation failures deleted', $deletedCount); $this->outputElapsedTime($startTime); return $nzbCount; } private function shouldDeleteFailedNzbCreation(Release $release, NzbCreationResult $result): bool { if ($result->isDeterministicFailure()) { return true; } if (! $result->isTransientFailure()) { return false; } return $this->nextNzbCreationAttempt($release) >= self::NZB_CREATION_MAX_ATTEMPTS; } private function nextNzbCreationAttempt(Release $release): int { return ((int) ($release->{NzbCreationCandidateQuery::ATTEMPTS_COLUMN} ?? 0)) + 1; } private function recordNzbCreationRetry(Release $release, NzbCreationResult $result): void { $values = NzbCreationCandidateQuery::failureUpdateValues($result->reason, true); if ($values !== []) { Release::query() ->where('id', $release->id) ->update($values); } Log::channel('nzb_creation')->warning('NZB creation failed; release will be retried', [ 'release_id' => $release->id, 'guid' => $release->guid, 'failure_type' => $result->failureType, 'reason' => $result->reason, 'next_attempt' => $this->nextNzbCreationAttempt($release), 'max_attempts' => self::NZB_CREATION_MAX_ATTEMPTS, ]); } private function deleteFailedNzbCreationRelease(Release $release, NzbCreationResult $result, string $claimToken): void { Log::channel('nzb_creation')->warning('Deleting release after NZB creation failure', [ 'release_id' => $release->id, 'guid' => $release->guid, 'failure_type' => $result->failureType, 'reason' => $result->reason, 'attempt' => $this->nextNzbCreationAttempt($release), 'max_attempts' => self::NZB_CREATION_MAX_ATTEMPTS, ]); try { $collectionIds = $result->collectionIds !== [] ? $result->collectionIds : Collection::query() ->where('releases_id', $release->id) ->pluck('id') ->map(static fn (mixed $id): int => (int) $id) ->all(); if ($collectionIds !== []) { $this->collectionCleanupService->deleteCollectionsAndDescendants( $collectionIds, 'Failed NZB creation cleanup', $this->echoCLI ); } $this->releaseManagement->deleteSingleWithService( ['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage ); } finally { NzbCreationCandidateQuery::clearClaim((int) $release->id, $claimToken); } } /** * Categorize releases based on the specified field. * * @throws \Exception */ public function categorizeReleases(int $categorize, int|string|null $groupID = null): void { $startTime = now()->toImmutable(); $type = match ($categorize) { 2 => 'searchname', default => 'name', }; $count = $this->categorizeRelease($type, $groupID); if ($count > 0) { $this->outputStat('Releases categorized', $count); $this->outputElapsedTime($startTime); } } /** * Run post-processing on releases. * * @throws \Exception */ public function postProcessReleases(int $postProcess, NNTPService $nntp): void { if ($postProcess !== 1) { return; } $this->outputSubHeader('Post-Processing Releases'); $service = $this->postProcessService ?? new PostProcessService; $service->processAll($nntp); } /** * Delete finished and orphaned collections. * * @throws Throwable */ public function deleteCollections(int|string|null $groupID): void { $this->collectionCleanupService->deleteFinishedAndOrphans($this->echoCLI); } /** * Delete unwanted releases based on group-specific settings. * * @throws \Exception */ public function deletedReleasesByGroup(int|string $groupID = ''): void { $startTime = now()->toImmutable(); $stats = ['minSize' => 0, 'maxSize' => 0, 'minFiles' => 0]; if ($this->echoCLI) { cli()->header( 'Process Releases -> Delete releases smaller/larger than minimum size/file count from group/site setting.' ); } $groupIDs = $groupID === '' ? UsenetGroup::getActiveIDs() : [['id' => $groupID]]; foreach ($groupIDs as $grpID) { $this->deleteReleasesUnderMinSize($grpID['id'], $stats); $this->deleteReleasesOverMaxSize($grpID['id'], $stats); $this->deleteReleasesUnderMinFiles($grpID['id'], $stats); } $this->outputReleaseDeleteByGroupStats($stats, $startTime); } /** * Delete releases based on site-wide settings. * * @throws \Exception */ public function deleteReleases(): void { $startTime = now()->toImmutable(); $this->outputSubHeader('Removing Unwanted Releases'); $stats = new ReleaseDeleteStats; $stats = $this->deleteReleasesOverRetention($stats); $stats = $this->deletePasswordedReleases($stats); $stats = $this->deleteCrossPostedReleases($stats); $stats = $this->deleteIncompleteReleases($stats); $stats = $this->deleteDisabledCategoryReleases($stats); $stats = $this->deleteCategoryMinSizeReleases($stats); $stats = $this->deleteDisabledGenreReleases($stats); $stats = $this->deleteMiscReleases($stats); $this->outputReleaseDeleteStats($stats, $startTime); } // ======================================================================== // Private Helper Methods // ======================================================================== private function validateNzbPath(): bool { $nzbPath = config('nntmux_settings.path_to_nzbs'); if (! file_exists($nzbPath)) { if ($this->echoCLI) { cli()->error("Bad or missing NZB directory - {$nzbPath}"); } return false; } return true; } private function resolveGroupId(string $groupName): string { if ($groupName === '') { return ''; } $groupInfo = UsenetGroup::getByName($groupName); return $groupInfo !== null ? (string) $groupInfo['id'] : ''; } private function countCompleteCollections(?int $groupId): int { $query = Collection::query() ->where('filecheck', CollectionFileCheckStatus::CompleteParts->value); if ($groupId !== null) { $query->where('groups_id', $groupId); } return $query->count('id'); } private function hasSizedCollections(int $groupId): bool { return Collection::query() ->where('filecheck', CollectionFileCheckStatus::Sized->value) ->where('groups_id', $groupId) ->where('filesize', '>', 0) ->exists(); } /** @param Builder|\Illuminate\Database\Eloquent\Builder $query */ private function deleteCollectionQueryInBatches( Builder|\Illuminate\Database\Eloquent\Builder $query, string $label, ): int { $deleted = 0; $idColumn = $query instanceof \Illuminate\Database\Eloquent\Builder ? 'id' : 'c.id'; do { $ids = (clone $query)->orderBy($idColumn)->limit(self::BATCH_SIZE)->pluck($idColumn)->all(); if ($ids === []) { break; } $affected = $this->collectionCleanupService->deleteCollectionsAndDescendants( array_map('intval', $ids), $label, $this->echoCLI ); $deleted += $affected; if ($affected < self::BATCH_SIZE) { break; } } while (true); return $deleted; } private function normalizeGroupId(int|string|null $groupID): ?int { if ($groupID === null || $groupID === '') { return null; } if (is_numeric($groupID)) { return (int) $groupID; } $groupInfo = UsenetGroup::getByName($groupID); return $groupInfo !== null ? (int) $groupInfo['id'] : null; } /** * @throws Throwable */ private function processStuckCollections(int $groupID): void { $cutoff = $this->calculateStuckCollectionsCutoff(); $totalDeleted = 0; do { $affected = $this->deleteStuckCollectionBatch($groupID, $cutoff); $totalDeleted += $affected; if ($affected < self::BATCH_SIZE) { break; } usleep(self::BATCH_PAUSE_US); } while (true); if ($this->echoCLI && $totalDeleted > 0) { cli()->primary("Deleted {$totalDeleted} broken/stuck collections.", true); } } private function calculateStuckCollectionsCutoff(): Carbon { $lastRun = $this->settings->lastRunTime; $threshold = null; if ($lastRun !== null) { try { $threshold = Carbon::createFromFormat('Y-m-d H:i:s', $lastRun); } catch (Throwable) { $threshold = null; } } return ($threshold ?? now())->copy()->subHours($this->settings->collectionTimeout); } private function deleteStuckCollectionBatch(int $groupID, Carbon $cutoff): int { $attempt = 0; $affected = 0; do { try { $query = DB::table('collections') ->whereIn('filecheck', [ CollectionFileCheckStatus::Default->value, CollectionFileCheckStatus::CompleteCollection->value, CollectionFileCheckStatus::TempComplete->value, CollectionFileCheckStatus::ZeroPart->value, 10, ]) ->orderBy('id') ->limit(self::BATCH_SIZE); if (Schema::hasColumn('collections', 'last_seen_at')) { $query->whereRaw('COALESCE(last_seen_at, dateadded, added) < ?', [$cutoff]); } else { $query->where('added', '<', $cutoff); } if ($groupID !== 0) { $query->where('groups_id', '=', $groupID); } $ids = $query->pluck('id')->all(); if ($ids === []) { break; } $affected = $this->collectionCleanupService->deleteCollectionsAndDescendants( $ids, 'Stuck collections cleanup', $this->echoCLI ); break; } catch (Throwable $e) { $attempt++; if ($attempt >= self::MAX_RETRIES) { if ($this->echoCLI) { cli()->error( 'Stuck collections delete failed after retries: '.$e->getMessage() ); } break; } usleep(self::RETRY_BASE_DELAY_US * $attempt); } } while (true); return $affected; } // ======================================================================== // Release Deletion Methods // ======================================================================== /** * @param array $stats */ private function deleteReleasesUnderMinSize(int|string $groupId, array &$stats): void { $releases = Release::query() ->where('releases.groups_id', $groupId) ->join('usenet_groups', 'usenet_groups.id', '=', 'releases.groups_id') ->whereRaw( 'GREATEST(IFNULL(usenet_groups.minsizetoformrelease, 0), ?) > 0 '. 'AND releases.size < GREATEST(IFNULL(usenet_groups.minsizetoformrelease, 0), ?)', [$this->settings->minSizeToFormRelease, $this->settings->minSizeToFormRelease] ) ->select(['releases.id', 'releases.guid']) ->get(); foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats['minSize']++; } } /** * @param array $stats */ private function deleteReleasesOverMaxSize(int|string $groupId, array &$stats): void { if ($this->settings->maxSizeToFormRelease <= 0) { return; } $releases = Release::query() ->where('groups_id', $groupId) ->where('size', '>', $this->settings->maxSizeToFormRelease) ->select(['id', 'guid']) ->get(); foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats['maxSize']++; } } /** * @param array $stats */ private function deleteReleasesUnderMinFiles(int|string $groupId, array &$stats): void { if ($this->settings->minFilesToFormRelease <= 0) { return; } $releases = Release::query() ->where('releases.groups_id', $groupId) ->join('usenet_groups', 'usenet_groups.id', '=', 'releases.groups_id') ->whereRaw( 'GREATEST(IFNULL(usenet_groups.minfilestoformrelease, 0), ?) > 0 '. 'AND releases.totalpart < GREATEST(IFNULL(usenet_groups.minfilestoformrelease, 0), ?)', [$this->settings->minFilesToFormRelease, $this->settings->minFilesToFormRelease] ) ->select(['releases.id', 'releases.guid']) ->get(); foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats['minFiles']++; } } private function deleteReleasesOverRetention(ReleaseDeleteStats $stats): ReleaseDeleteStats { if (! $this->settings->hasRetentionCleanup()) { return $stats; } $cutoff = now()->subDays($this->settings->releaseRetentionDays); Release::query() ->where('postdate', '<', $cutoff) ->select(['id', 'guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('retention'); } return true; }); return $stats; } private function deletePasswordedReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { if (! $this->settings->deletePasswordedRelease) { return $stats; } Release::query() ->select(['id', 'guid']) ->where('passwordstatus', '=', ReleaseBrowseService::PASSWD_RAR) ->orWhereIn('id', function ($query): void { $query->select('releases_id') ->from('release_files') ->where('passworded', '=', ReleaseBrowseService::PASSWD_RAR); }) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('password'); } return true; }); return $stats; } private function deleteCrossPostedReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { if (! $this->settings->hasCrossPostDetection()) { return $stats; } $releases = Release::query() ->where('adddate', '>', now()->subHours($this->settings->crossPostTime)) ->groupBy(['name', 'fromname']) ->havingRaw('COUNT(name) > 1 AND COUNT(fromname) > 1') ->select(['id', 'guid']) ->get(); foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('duplicate'); } return $stats; } private function deleteIncompleteReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { if (! $this->settings->hasCompletionCleanup()) { return $stats; } Release::query() ->where('completion', '<', $this->settings->completion) ->where('completion', '>', 0) ->select(['id', 'guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('completion'); } return true; }); return $stats; } private function deleteDisabledCategoryReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { $disabledCategories = Category::getDisabledIDs(); if ($disabledCategories->isEmpty()) { return $stats; } $categoryIds = $disabledCategories->pluck('id')->toArray(); Release::query() ->whereIn('categories_id', $categoryIds) ->select(['id', 'guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('disabledCategory'); } return true; }); return $stats; } private function deleteCategoryMinSizeReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { $categories = Category::query() ->where('minsizetoformrelease', '>', 0) ->select(['id', 'minsizetoformrelease as minsize']) ->get(); foreach ($categories as $category) { Release::query() ->where('categories_id', (int) $category->id) ->where('size', '<', (int) $category->minsize) // @phpstan-ignore property.notFound ->select(['id', 'guid']) ->limit(1000) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('categoryMinSize'); } return true; }); } return $stats; } private function deleteDisabledGenreReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { $genres = new GenreService; $genreList = $genres->getDisabledIDs(); if ($genreList->isEmpty()) { return $stats; } foreach ($genreList as $genre) { $musicInfoQuery = MusicInfo::query() ->where('genre_id', (int) $genre->id) // @phpstan-ignore property.notFound ->select(['id']); Release::query() ->joinSub( $musicInfoQuery, 'mi', static fn ($join) => $join->on('releases.musicinfo_id', '=', 'mi.id') ) ->select(['releases.id', 'releases.guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('disabledGenre'); } return true; }, 'releases.id'); } return $stats; } private function deleteMiscReleases(ReleaseDeleteStats $stats): ReleaseDeleteStats { if ($this->settings->miscOtherRetentionHours > 0) { $cutoff = now()->subHours($this->settings->miscOtherRetentionHours); Release::query() ->where('categories_id', Category::OTHER_MISC) ->where('adddate', '<=', $cutoff) ->select(['id', 'guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('miscOther'); } return true; }); } if ($this->settings->miscHashedRetentionHours > 0) { $cutoff = now()->subHours($this->settings->miscHashedRetentionHours); Release::query() ->where('categories_id', Category::OTHER_HASHED) ->where('adddate', '<=', $cutoff) ->select(['id', 'guid']) ->chunkById(self::BATCH_SIZE, function ($releases) use (&$stats): bool { foreach ($releases as $release) { $this->deleteSingleRelease($release); $stats = $stats->increment('miscHashed'); } return true; }); } return $stats; } private function deleteSingleRelease(object $release): void { $this->releaseManagement->deleteSingle( ['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage ); } // ======================================================================== // Output Helper Methods // ======================================================================== private function outputBanner(): void { if (! $this->echoCLI) { return; } echo PHP_EOL; cli()->header('NNTmux Release Processing'); cli()->info('Started: '.now()->format('Y-m-d H:i:s')); } private function outputHeader(string $title): void { if (! $this->echoCLI) { return; } echo PHP_EOL; cli()->header(strtoupper($title)); cli()->header(str_repeat('-', strlen($title))); } private function outputSubHeader(string $title): void { if (! $this->echoCLI) { return; } cli()->notice(" {$title}"); } /** @phpstan-ignore method.unused */ private function outputSuccess(string $message): void { if (! $this->echoCLI) { return; } cli()->primary(" {$message}"); } private function outputInfo(string $message): void { if (! $this->echoCLI) { return; } cli()->info(" {$message}"); } private function outputStat(string $label, string|int $value, string $suffix = ''): void { if (! $this->echoCLI) { return; } $formattedValue = is_int($value) ? number_format($value) : $value; cli()->primary(" {$label}: {$formattedValue}{$suffix}"); } private function outputElapsedTime(DateTimeInterface $startTime, string $prefix = 'Time'): void { if (! $this->echoCLI) { return; } $elapsed = now()->diffInSeconds($startTime, true); $timeStr = $this->formatElapsedTime($elapsed); cli()->info(" {$prefix}: {$timeStr}"); } private function formatElapsedTime(int|float $seconds): string { if ($seconds < 1) { return sprintf('%dms', (int) ($seconds * 1000)); } if ($seconds < 60) { return sprintf('%.1fs', $seconds); } $minutes = floor($seconds / 60); $remainingSeconds = $seconds % 60; if ($minutes < 60) { return sprintf('%dm %ds', $minutes, (int) $remainingSeconds); } $hours = floor($minutes / 60); $remainingMinutes = $minutes % 60; return sprintf('%dh %dm', $hours, $remainingMinutes); } private function outputProgress(int $current, int $total, string $action): void { if (! $this->echoCLI || $total === 0) { return; } $percent = min(100, (int) (($current / $total) * 100)); echo "\r {$action}: ".number_format($current).'/'.number_format($total)." ({$percent}%) "; if ($current >= $total) { echo PHP_EOL; } } private function outputFinalSummary( int $releasesAdded, int $nzbsCreated, int $dupes, int $iterations, DateTimeInterface $startTime ): void { if (! $this->echoCLI) { return; } $elapsed = now()->diffInSeconds($startTime, true); echo PHP_EOL; cli()->header('SUMMARY'); cli()->header('-------'); cli()->primary(' Releases added: '.number_format($releasesAdded)); cli()->primary(' NZBs created: '.number_format($nzbsCreated)); if ($dupes > 0) { cli()->warning(' Duplicates skipped: '.number_format($dupes)); } cli()->info(' Processing cycles: '.number_format($iterations)); cli()->info(' Total time: '.$this->formatElapsedTime($elapsed)); echo PHP_EOL; } /** * @param array{minSize: int, maxSize: int, minFiles: int} $stats */ private function outputCollectionDeleteStats(array $stats, DateTimeInterface $startTime): void { $totalDeleted = $stats['minSize'] + $stats['maxSize'] + $stats['minFiles'] + ($stats['par2Only'] ?? 0); if ($totalDeleted > 0) { $this->outputStat('Too small', $stats['minSize']); $this->outputStat('Too large', $stats['maxSize']); $this->outputStat('Too few files', $stats['minFiles']); if (($stats['par2Only'] ?? 0) > 0) { $this->outputStat('Par2 only', $stats['par2Only']); } $this->outputStat('Total removed', $totalDeleted); } else { $this->outputInfo('No collections filtered'); } $this->outputElapsedTime($startTime); } /** * @param array{minSize: int, maxSize: int, minFiles: int} $stats */ private function outputReleaseDeleteByGroupStats(array $stats, DateTimeInterface $startTime): void { $total = $stats['minSize'] + $stats['maxSize'] + $stats['minFiles']; if ($total > 0) { $this->outputStat('Too small', $stats['minSize']); $this->outputStat('Too large', $stats['maxSize']); $this->outputStat('Too few files', $stats['minFiles']); } $this->outputElapsedTime($startTime); } private function outputReleaseDeleteStats(ReleaseDeleteStats $stats, DateTimeInterface $startTime): void { if (! $this->echoCLI) { return; } $total = $stats->total(); if ($total > 0) { if ($stats->retention > 0) { $this->outputStat('Past retention', $stats->retention); } if ($stats->password > 0) { $this->outputStat('Passworded', $stats->password); } if ($stats->duplicate > 0) { $this->outputStat('Cross-posted', $stats->duplicate); } if ($stats->completion > 0) { $this->outputStat("Under {$this->settings->completion}% complete", $stats->completion); } if ($stats->disabledCategory > 0) { $this->outputStat('Disabled categories', $stats->disabledCategory); } if ($stats->categoryMinSize > 0) { $this->outputStat('Under category min size', $stats->categoryMinSize); } if ($stats->disabledGenre > 0) { $this->outputStat('Disabled genres', $stats->disabledGenre); } if ($stats->miscOther > 0) { $this->outputStat('Misc->Other expired', $stats->miscOther); } if ($stats->miscHashed > 0) { $this->outputStat('Misc->Hashed expired', $stats->miscHashed); } $this->outputStat('Total releases removed', $total); } else { $this->outputInfo('No releases removed'); } $this->outputElapsedTime($startTime); } }