Refactor ProcessReleases

This commit is contained in:
DariusIII
2025-12-11 12:48:35 +01:00
parent b1a1ed1560
commit b16e77aef8
9 changed files with 1527 additions and 638 deletions
File diff suppressed because it is too large Load Diff
@@ -1,8 +1,9 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Models\Settings;
use Blacklight\processing\ProcessReleases;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
@@ -31,10 +32,10 @@ class ProcessReleasesCommand extends Command
$groupId = $this->argument('groupId');
try {
$releases = new ProcessReleases;
$releases = new ProcessReleases();
if (is_numeric($groupId)) {
$this->processReleasesForGroup($releases, $groupId);
$this->processReleasesForGroup($releases, (string) $groupId);
} else {
$this->processReleasesForGroup($releases, '');
$this->runPostProcessingTasks($releases);
@@ -51,22 +52,24 @@ class ProcessReleasesCommand extends Command
/**
* Create / process releases for a groupID.
*
* Uses the ProcessReleases DTO-based workflow for cleaner code.
*/
private function processReleasesForGroup(ProcessReleases $releases, string $groupID): void
{
$releaseCreationLimit = Settings::settingValue('maxnzbsprocessed') !== ''
? (int) Settings::settingValue('maxnzbsprocessed')
: 1000;
$limit = $releases->getReleaseCreationLimit();
$releases->processIncompleteCollections($groupID);
$releases->processCollectionSizes($groupID);
$releases->deleteUnwantedCollections($groupID);
do {
$releasesCount = $releases->createReleases($groupID);
$result = $releases->createReleases($groupID);
$nzbFilesAdded = $releases->createNZBs($groupID);
} while ($releaseCreationLimit <= $releasesCount['added'] + $releasesCount['dupes']
|| $nzbFilesAdded >= $releaseCreationLimit);
// Continue if we processed up to the limit (more work may be available)
$shouldContinue = $result->total() >= $limit || $nzbFilesAdded >= $limit;
} while ($shouldContinue);
$releases->deleteCollections($groupID);
}
+29 -19
View File
@@ -1,5 +1,7 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Models\Settings;
@@ -36,27 +38,28 @@ class UpdatePerGroup extends Command
{
$groupId = $this->argument('groupId');
if (! is_numeric($groupId)) {
if (!is_numeric($groupId)) {
$this->error('Group ID must be numeric.');
return self::FAILURE;
}
try {
$groupMySQL = UsenetGroup::find($groupId)->toArray();
$group = UsenetGroup::find($groupId);
if ($groupMySQL === null) {
if ($group === null) {
$this->error("Group not found with id {$groupId}");
return self::FAILURE;
}
$groupMySQL = $group->toArray();
$nntp = $this->getNntp();
$backFill = new Backfill;
$backFill = new Backfill();
// Update the group for new binaries
$this->info("Updating binaries for group: {$groupMySQL['name']}");
(new Binaries)->updateGroup($groupMySQL);
(new Binaries())->updateGroup($groupMySQL);
// BackFill the group with 20k articles
$this->info("Backfilling group: {$groupMySQL['name']}");
@@ -64,19 +67,19 @@ class UpdatePerGroup extends Command
// Create releases
$this->info("Processing releases for group: {$groupMySQL['name']}");
$this->processReleases(new ProcessReleases, $groupId);
$this->processReleases(new ProcessReleases(), (string) $groupId);
// Post process the releases
$this->info("Post-processing additional for group: {$groupMySQL['name']}");
(new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp]))->start($groupId);
$this->info("Processing NFO files for group: {$groupMySQL['name']}");
(new Nfo)->processNfoFiles(
(new Nfo())->processNfoFiles(
$nntp,
$groupId,
'',
(int) Settings::settingValue('lookupimdb'),
(int) Settings::settingValue('lookuptv')
(bool)Settings::settingValue('lookupimdb'),
(bool)Settings::settingValue('lookuptv')
);
$this->info("Completed all processing for group: {$groupMySQL['name']}");
@@ -92,37 +95,44 @@ class UpdatePerGroup extends Command
/**
* Create / process releases for a groupID.
*
* Uses the ProcessReleases DTO-based workflow for cleaner code.
*/
private function processReleases(ProcessReleases $releases, string $groupID): void
{
$releaseCreationLimit = Settings::settingValue('maxnzbsprocessed') !== ''
? (int) Settings::settingValue('maxnzbsprocessed')
: 1000;
$limit = $releases->getReleaseCreationLimit();
$releases->processIncompleteCollections($groupID);
$releases->processCollectionSizes($groupID);
$releases->deleteUnwantedCollections($groupID);
do {
$releasesCount = $releases->createReleases($groupID);
$result = $releases->createReleases($groupID);
$nzbFilesAdded = $releases->createNZBs($groupID);
} while ($releaseCreationLimit <= $releasesCount['added'] + $releasesCount['dupes']
|| $nzbFilesAdded >= $releaseCreationLimit);
// Continue if we processed up to the limit (more work may be available)
$shouldContinue = $result->total() >= $limit || $nzbFilesAdded >= $limit;
} while ($shouldContinue);
$releases->deleteCollections($groupID);
}
/**
* Get NNTP connection.
*
* @throws \Exception If unable to connect to usenet
*/
private function getNntp(): NNTP
{
$nntp = new NNTP;
$nntp = new NNTP();
if ((config('nntmux_nntp.use_alternate_nntp_server') === true
$useAlternate = config('nntmux_nntp.use_alternate_nntp_server') === true;
$connected = $useAlternate
? $nntp->doConnect(false, true)
: $nntp->doConnect()) !== true) {
throw new \Exception('Unable to connect to usenet.');
: $nntp->doConnect();
if ($connected !== true) {
throw new \RuntimeException('Unable to connect to usenet.');
}
return $nntp;
+92
View File
@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace App\Enums;
/**
* Collection filecheck status values used during release processing.
*
* These statuses track the progress of collections through the release
* creation pipeline, from initial discovery to NZB file creation.
*/
enum CollectionFileCheckStatus: int
{
/** Default status for newly discovered collections */
case Default = 0;
/** Collection has all expected binary files */
case CompleteCollection = 1;
/** All parts of binaries are complete */
case CompleteParts = 2;
/** Collection size has been calculated */
case Sized = 3;
/** Collection has been converted to a release */
case Inserted = 4;
/** Collection marked for deletion */
case Delete = 5;
/** Temporary complete status during processing */
case TempComplete = 15;
/** Collection has zero-numbered parts (special handling) */
case ZeroPart = 16;
/**
* Get the human-readable description for this status.
*/
public function description(): string
{
return match ($this) {
self::Default => 'Newly discovered',
self::CompleteCollection => 'Complete collection',
self::CompleteParts => 'Complete parts',
self::Sized => 'Size calculated',
self::Inserted => 'Inserted as release',
self::Delete => 'Marked for deletion',
self::TempComplete => 'Temporarily complete',
self::ZeroPart => 'Zero-part collection',
};
}
/**
* Check if this status indicates processing is complete.
*/
public function isProcessingComplete(): bool
{
return match ($this) {
self::Inserted, self::Delete => true,
default => false,
};
}
/**
* Get statuses that indicate a collection is ready for release creation.
*
* @return array<self>
*/
public static function readyForRelease(): array
{
return [self::Sized];
}
/**
* Get statuses that indicate a collection is still being processed.
*
* @return array<self>
*/
public static function inProgress(): array
{
return [
self::Default,
self::CompleteCollection,
self::CompleteParts,
self::TempComplete,
self::ZeroPart,
];
}
}
+37
View File
@@ -0,0 +1,37 @@
<?php
declare(strict_types=1);
namespace App\Enums;
/**
* Binary file completion status during collection processing.
*/
enum FileCompletionStatus: int
{
/** File/binary is still incomplete (missing parts) */
case Incomplete = 0;
/** File/binary has all parts */
case Complete = 1;
/**
* Get human-readable description.
*/
public function description(): string
{
return match ($this) {
self::Incomplete => 'Incomplete',
self::Complete => 'Complete',
};
}
/**
* Check if file is complete.
*/
public function isComplete(): bool
{
return $this === self::Complete;
}
}
+5 -4
View File
@@ -2,6 +2,7 @@
namespace App\Services;
use App\Enums\CollectionFileCheckStatus;
use App\Models\Category;
use App\Models\Collection;
use App\Models\Predb;
@@ -12,10 +13,9 @@ use App\Models\UsenetGroup;
use App\Services\Categorization\CategorizationService;
use Blacklight\ColorCLI;
use Blacklight\NZB;
use Blacklight\processing\ProcessReleases;
use Blacklight\ReleaseCleaning;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str; // for constants
use Illuminate\Support\Str;
class ReleaseCreationService
{
@@ -28,6 +28,7 @@ class ReleaseCreationService
* Create releases from complete collections.
*
* @return array{added:int,dupes:int}
* @throws \Throwable
*/
public function createReleases(int|string|null $groupID, int $limit, bool $echoCLI): array
{
@@ -41,7 +42,7 @@ class ReleaseCreationService
}
$collectionsQuery = Collection::query()
->where('collections.filecheck', ProcessReleases::COLLFC_SIZED)
->where('collections.filecheck', CollectionFileCheckStatus::Sized->value)
->where('collections.filesize', '>', 0);
if (! empty($groupID)) {
$collectionsQuery->where('collections.groups_id', $groupID);
@@ -119,7 +120,7 @@ class ReleaseCreationService
if ($releaseID !== null) {
DB::transaction(static function () use ($collection, $releaseID) {
Collection::query()->where('id', $collection->id)->update([
'filecheck' => ProcessReleases::COLLFC_INSERTED,
'filecheck' => CollectionFileCheckStatus::Inserted->value,
'releases_id' => $releaseID,
]);
}, 10);
@@ -0,0 +1,111 @@
<?php
declare(strict_types=1);
namespace App\Support\DTOs;
/**
* Configuration settings for ProcessReleases operations.
*/
final readonly class ProcessReleasesSettings
{
public function __construct(
public int $collectionDelayTime,
public int $crossPostTime,
public int $releaseCreationLimit,
public int $completion,
public int $collectionTimeout,
public int $maxSizeToFormRelease,
public int $minSizeToFormRelease,
public int $minFilesToFormRelease,
public int $releaseRetentionDays,
public bool $deletePasswordedRelease,
public int $miscOtherRetentionHours,
public int $miscHashedRetentionHours,
public int $partRetentionHours,
public ?string $lastRunTime,
) {}
/**
* Default settings values.
*/
private const DEFAULTS = [
'collectionDelayTime' => 2,
'crossPostTime' => 2,
'releaseCreationLimit' => 1000,
'completion' => 0,
'collectionTimeout' => 48,
'maxSizeToFormRelease' => 0,
'minSizeToFormRelease' => 0,
'minFilesToFormRelease' => 0,
'releaseRetentionDays' => 0,
'deletePasswordedRelease' => false,
'miscOtherRetentionHours' => 0,
'miscHashedRetentionHours' => 0,
'partRetentionHours' => 24,
'lastRunTime' => null,
];
/**
* Create settings from database values.
*
* @param array<string, mixed> $dbSettings
*/
public static function fromDatabase(array $dbSettings): self
{
$getInt = static fn(string $key, int $default): int =>
($dbSettings[$key] ?? '') !== '' ? (int) $dbSettings[$key] : $default;
$completion = min(100, $getInt('completionpercent', self::DEFAULTS['completion']));
return new self(
collectionDelayTime: $getInt('delaytime', self::DEFAULTS['collectionDelayTime']),
crossPostTime: $getInt('crossposttime', self::DEFAULTS['crossPostTime']),
releaseCreationLimit: $getInt('maxnzbsprocessed', self::DEFAULTS['releaseCreationLimit']),
completion: $completion,
collectionTimeout: $getInt('collection_timeout', self::DEFAULTS['collectionTimeout']),
maxSizeToFormRelease: $getInt('maxsizetoformrelease', self::DEFAULTS['maxSizeToFormRelease']),
minSizeToFormRelease: $getInt('minsizetoformrelease', self::DEFAULTS['minSizeToFormRelease']),
minFilesToFormRelease: $getInt('minfilestoformrelease', self::DEFAULTS['minFilesToFormRelease']),
releaseRetentionDays: $getInt('releaseretentiondays', self::DEFAULTS['releaseRetentionDays']),
deletePasswordedRelease: ((int) ($dbSettings['deletepasswordedrelease'] ?? 0)) === 1,
miscOtherRetentionHours: $getInt('miscotherretentionhours', self::DEFAULTS['miscOtherRetentionHours']),
miscHashedRetentionHours: $getInt('mischashedretentionhours', self::DEFAULTS['miscHashedRetentionHours']),
partRetentionHours: $getInt('partretentionhours', self::DEFAULTS['partRetentionHours']),
lastRunTime: !empty($dbSettings['last_run_time']) ? (string) $dbSettings['last_run_time'] : null,
);
}
/**
* Check if completion percentage is valid.
*/
public function hasValidCompletion(): bool
{
return $this->completion >= 0 && $this->completion <= 100;
}
/**
* Check if retention cleanup is enabled.
*/
public function hasRetentionCleanup(): bool
{
return $this->releaseRetentionDays > 0;
}
/**
* Check if cross-post detection is enabled.
*/
public function hasCrossPostDetection(): bool
{
return $this->crossPostTime > 0;
}
/**
* Check if completion-based cleanup is enabled.
*/
public function hasCompletionCleanup(): bool
{
return $this->completion > 0;
}
}
@@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
namespace App\Support\DTOs;
/**
* Data transfer object for release creation results.
*/
final readonly class ReleaseCreationResult
{
public function __construct(
public int $added = 0,
public int $dupes = 0,
) {}
/**
* Get total number of processed collections.
*/
public function total(): int
{
return $this->added + $this->dupes;
}
/**
* Check if any releases were added.
*/
public function hasAddedReleases(): bool
{
return $this->added > 0;
}
/**
* Create from array.
*
* @param array{added?: int, dupes?: int} $data
*/
public static function fromArray(array $data): self
{
return new self(
added: $data['added'] ?? 0,
dupes: $data['dupes'] ?? 0,
);
}
/**
* Convert to array.
*
* @return array{added: int, dupes: int}
*/
public function toArray(): array
{
return [
'added' => $this->added,
'dupes' => $this->dupes,
];
}
}
+106
View File
@@ -0,0 +1,106 @@
<?php
declare(strict_types=1);
namespace App\Support\DTOs;
/**
* Data transfer object for release deletion statistics.
*
* Tracks the number of releases deleted by category during cleanup operations.
*/
final readonly class ReleaseDeleteStats
{
public function __construct(
public int $retention = 0,
public int $password = 0,
public int $duplicate = 0,
public int $completion = 0,
public int $disabledCategory = 0,
public int $categoryMinSize = 0,
public int $disabledGenre = 0,
public int $miscOther = 0,
public int $miscHashed = 0,
) {}
/**
* Create a new instance with an incremented counter.
*/
public function increment(string $field): self
{
$values = [
'retention' => $this->retention,
'password' => $this->password,
'duplicate' => $this->duplicate,
'completion' => $this->completion,
'disabledCategory' => $this->disabledCategory,
'categoryMinSize' => $this->categoryMinSize,
'disabledGenre' => $this->disabledGenre,
'miscOther' => $this->miscOther,
'miscHashed' => $this->miscHashed,
];
if (isset($values[$field])) {
$values[$field]++;
}
return new self(...$values);
}
/**
* Get the total number of deleted releases.
*/
public function total(): int
{
return $this->retention
+ $this->password
+ $this->duplicate
+ $this->completion
+ $this->disabledCategory
+ $this->categoryMinSize
+ $this->disabledGenre
+ $this->miscOther
+ $this->miscHashed;
}
/**
* Convert to array representation.
*
* @return array<string, int>
*/
public function toArray(): array
{
return [
'retention' => $this->retention,
'password' => $this->password,
'duplicate' => $this->duplicate,
'completion' => $this->completion,
'disabledCategory' => $this->disabledCategory,
'categoryMinSize' => $this->categoryMinSize,
'disabledGenre' => $this->disabledGenre,
'miscOther' => $this->miscOther,
'miscHashed' => $this->miscHashed,
];
}
/**
* Create from an array of values.
*
* @param array<string, int> $data
*/
public static function fromArray(array $data): self
{
return new self(
retention: $data['retention'] ?? 0,
password: $data['password'] ?? 0,
duplicate: $data['duplicate'] ?? 0,
completion: $data['completion'] ?? 0,
disabledCategory: $data['disabledCategory'] ?? 0,
categoryMinSize: $data['categoryMinSize'] ?? 0,
disabledGenre: $data['disabledGenre'] ?? 0,
miscOther: $data['miscOther'] ?? 0,
miscHashed: $data['miscHashed'] ?? 0,
);
}
}