From 5ee33502b1c32f998a3a980b68c1b06da48607a2 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Mon, 13 Jul 2026 16:51:10 +0200 Subject: [PATCH] Update nzb creation --- app/Console/Commands/CleanNZB.php | 36 +- app/Models/Content.php | 27 +- app/Models/Genre.php | 15 +- app/Models/Invitation.php | 27 +- app/Models/User.php | 30 +- .../Nzb/NzbCreationCandidateQuery.php | 208 ++++++++ app/Services/Nzb/NzbService.php | 395 +++++++++++---- app/Services/ReleaseProcessingService.php | 152 +++++- app/Support/Data/NzbCreationResult.php | 59 +++ config/logging.php | 9 + ...3_000000_add_nzb_creation_claim_fields.php | 110 +++++ phpstan-baseline.neon | 133 ------ tests/Feature/NzbCreationReliabilityTest.php | 449 ++++++++++++++++++ 13 files changed, 1379 insertions(+), 271 deletions(-) create mode 100644 app/Services/Nzb/NzbCreationCandidateQuery.php create mode 100644 app/Support/Data/NzbCreationResult.php create mode 100644 database/migrations/2026_07_13_000000_add_nzb_creation_claim_fields.php create mode 100644 tests/Feature/NzbCreationReliabilityTest.php diff --git a/app/Console/Commands/CleanNZB.php b/app/Console/Commands/CleanNZB.php index 8dbe099e0..34262c517 100644 --- a/app/Console/Commands/CleanNZB.php +++ b/app/Console/Commands/CleanNZB.php @@ -11,6 +11,7 @@ use App\Services\Releases\ReleaseManagementService; use Illuminate\Console\Command; use Illuminate\Support\Collection; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Log; class CleanNZB extends Command { @@ -22,6 +23,8 @@ class CleanNZB extends Command protected $signature = 'nntmux:nzbclean {--notindb : Delete NZBs that dont exist in database} {--notondisk : Delete release in database that dont have a NZB on disk} + {--temps : Delete stale temporary NZB files left by interrupted creation} + {--temp-age-hours=24 : Minimum temporary NZB age in hours before deletion} {--chunksize=25000 : Chunk size for releases query} {--delete : Pass this argument to actually delete the files. Otherwise it\'s just a dry run.}'; @@ -38,7 +41,7 @@ class CleanNZB extends Command public function handle(): void { // Check if any options are false - if (! $this->option('notindb') && ! $this->option('notondisk')) { + if (! $this->option('notindb') && ! $this->option('notondisk') && ! $this->option('temps')) { $this->error('You must specify at least one option. See: --help'); exit(); } @@ -48,6 +51,9 @@ class CleanNZB extends Command if ($this->option('notondisk')) { $this->GetReleasesWithNoNZBOnDisk($this->option('delete')); } + if ($this->option('temps')) { + $this->cleanTemporaryNzbFiles($this->option('delete')); + } } private function GetNZBsWithNoDatabaseEntry(mixed $delete = false): void @@ -105,4 +111,32 @@ class CleanNZB extends Command }); $this->info("Checked: $checked / Deleted: $deleted"); } + + private function cleanTemporaryNzbFiles(mixed $delete = false): void + { + $nzb = app(NzbService::class); + $ageHours = max(1, (int) $this->option('temp-age-hours')); + $olderThanSeconds = $ageHours * 3600; + + if (! $delete) { + $paths = $nzb->findStaleTemporaryNzbPaths($olderThanSeconds); + foreach ($paths as $path) { + $this->line("Would delete stale temporary NZB: {$path}"); + } + + $this->info('Checked stale temporary NZBs: '.count($paths).' would be deleted.'); + + return; + } + + $deleted = $nzb->cleanupStaleTemporaryNzbs($olderThanSeconds); + if ($deleted > 0) { + Log::channel('nzb_creation')->warning('Deleted stale temporary NZB files', [ + 'count' => $deleted, + 'older_than_hours' => $ageHours, + ]); + } + + $this->info("Deleted stale temporary NZBs: {$deleted}"); + } } diff --git a/app/Models/Content.php b/app/Models/Content.php index a3cb0260c..65ccb52ab 100644 --- a/app/Models/Content.php +++ b/app/Models/Content.php @@ -95,24 +95,33 @@ class Content extends Model /** * Scope: Get only active content. + * + * @param Builder $query + * @return Builder */ - public function scopeActive(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeActive(Builder $query): Builder { return $query->where('status', self::STATUS_ENABLED); } /** * Scope: Get content of a specific type. + * + * @param Builder $query + * @return Builder */ - public function scopeOfType(Builder $query, int $type): Builder // @phpstan-ignore missingType.generics + public function scopeOfType(Builder $query, int $type): Builder { return $query->where('contenttype', $type); } /** * Scope: Get content accessible by a specific role. + * + * @param Builder $query + * @return Builder */ - public function scopeForRole(Builder $query, int $role): Builder // @phpstan-ignore missingType.generics + public function scopeForRole(Builder $query, int $role): Builder { // Admins and moderators can see everything if (\in_array($role, [UserRole::ADMIN->value, UserRole::MODERATOR->value], true)) { @@ -128,8 +137,11 @@ class Content extends Model /** * Scope: Order content by type and ordinal within each type. + * + * @param Builder $query + * @return Builder */ - public function scopeOrdered(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeOrdered(Builder $query): Builder { return $query ->orderBy('contenttype') @@ -138,10 +150,13 @@ class Content extends Model /** * Scope: Get front page content. + * + * @param Builder $query + * @return Builder */ - public function scopeFrontPage(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeFrontPage(Builder $query): Builder { - return $query->active() // @phpstan-ignore method.notFound + return $query->active() ->ofType(self::TYPE_INDEX) ->ordered(); } diff --git a/app/Models/Genre.php b/app/Models/Genre.php index c2c32af42..7cb0e1cc5 100644 --- a/app/Models/Genre.php +++ b/app/Models/Genre.php @@ -73,24 +73,33 @@ class Genre extends Model /** * Scope to filter disabled genres. + * + * @param Builder $query + * @return Builder */ - public function scopeDisabled(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeDisabled(Builder $query): Builder { return $query->where('disabled', '=', 1); } /** * Scope to filter enabled genres. + * + * @param Builder $query + * @return Builder */ - public function scopeEnabled(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeEnabled(Builder $query): Builder { return $query->where('disabled', '=', 0); } /** * Scope to filter by type. + * + * @param Builder $query + * @return Builder */ - public function scopeOfType(Builder $query, string $type): Builder // @phpstan-ignore missingType.generics + public function scopeOfType(Builder $query, string $type): Builder { if (! empty($type)) { return $query->where('type', '=', $type); diff --git a/app/Models/Invitation.php b/app/Models/Invitation.php index 044e99289..5a260236e 100644 --- a/app/Models/Invitation.php +++ b/app/Models/Invitation.php @@ -98,42 +98,57 @@ class Invitation extends Model // @phpstan-ignore missingType.iterableValue /** * Scope to get only active invitations + * + * @param Builder $query + * @return Builder */ - public function scopeActive(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeActive(Builder $query): Builder { return $query->where('is_active', true); } /** * Scope to get valid (active and not expired) invitations + * + * @param Builder $query + * @return Builder */ - public function scopeValid(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeValid(Builder $query): Builder { - return $query->active() // @phpstan-ignore method.notFound + return $query->active() ->where('expires_at', '>', now()) ->whereNull('used_at'); } /** * Scope to get expired invitations + * + * @param Builder $query + * @return Builder */ - public function scopeExpired(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeExpired(Builder $query): Builder { return $query->where('expires_at', '<=', now()); } /** * Scope to get unused invitations + * + * @param Builder $query + * @return Builder */ - public function scopeUnused(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeUnused(Builder $query): Builder { return $query->whereNull('used_at'); } /** * Scope to get used invitations + * + * @param Builder $query + * @return Builder */ - public function scopeUsed(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeUsed(Builder $query): Builder { return $query->whereNotNull('used_at'); } diff --git a/app/Models/User.php b/app/Models/User.php index 97827aadd..17759def9 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -364,16 +364,22 @@ final class User extends Authenticatable implements CanResetPasswordContract, Ha /** * Scope to get active (non-disabled) users. + * + * @param Builder $query + * @return Builder */ - public function scopeActive(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeActive(Builder $query): Builder { return $query->where('roles_id', '!=', UserRole::DISABLED->value); } /** * Scope to get verified users. + * + * @param Builder $query + * @return Builder */ - public function scopeVerified(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeVerified(Builder $query): Builder { return $query->where(function (Builder $verifiedQuery): void { $verifiedQuery @@ -428,8 +434,11 @@ final class User extends Authenticatable implements CanResetPasswordContract, Ha /** * Scope to filter users by role. + * + * @param Builder $query + * @return Builder */ - public function scopeWithRole(Builder $query, int|string $role): Builder // @phpstan-ignore missingType.generics + public function scopeWithRole(Builder $query, int|string $role): Builder { if (is_numeric($role)) { return $query->where('roles_id', (int) $role); @@ -440,8 +449,11 @@ final class User extends Authenticatable implements CanResetPasswordContract, Ha /** * Scope to get users with roles expiring within specified days. + * + * @param Builder $query + * @return Builder */ - public function scopeExpiringSoon(Builder $query, int $days = 7): Builder // @phpstan-ignore missingType.generics + public function scopeExpiringSoon(Builder $query, int $days = 7): Builder { return $query->whereNotNull('rolechangedate') ->whereBetween('rolechangedate', [now(), now()->addDays($days)]); @@ -449,8 +461,11 @@ final class User extends Authenticatable implements CanResetPasswordContract, Ha /** * Scope to get users with expired roles. + * + * @param Builder $query + * @return Builder */ - public function scopeExpired(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeExpired(Builder $query): Builder { return $query->whereNotNull('rolechangedate') ->where('rolechangedate', '<', now()); @@ -458,8 +473,11 @@ final class User extends Authenticatable implements CanResetPasswordContract, Ha /** * Scope to exclude sharing email. + * + * @param Builder $query + * @return Builder */ - public function scopeExcludeSharing(Builder $query): Builder // @phpstan-ignore missingType.generics + public function scopeExcludeSharing(Builder $query): Builder { return $query->where('email', '!=', 'sharing@nZEDb.com'); } diff --git a/app/Services/Nzb/NzbCreationCandidateQuery.php b/app/Services/Nzb/NzbCreationCandidateQuery.php new file mode 100644 index 000000000..d0dcdba50 --- /dev/null +++ b/app/Services/Nzb/NzbCreationCandidateQuery.php @@ -0,0 +1,208 @@ + + */ + public static function baseBuilder(int|string|null $groupID = null, bool $includeClaimed = false): Builder + { + $query = Release::query() + ->from('releases as r') + ->where('r.nzbstatus', NzbService::NZB_NONE); + + if ($groupID !== null && $groupID !== '' && $groupID !== 0 && $groupID !== '0') { + $query->where('r.groups_id', $groupID); + } + + if (! $includeClaimed) { + self::applyClaimWindow($query); + } + + return $query; + } + + /** + * @param list $columns + * @return EloquentCollection + */ + public static function claimBatch( + int|string|null $groupID, + int $limit, + string $token, + array $columns = ['*'], + ): EloquentCollection { + $effectiveLimit = max(1, $limit); + + return DB::transaction(function () use ($groupID, $effectiveLimit, $token, $columns): EloquentCollection { + $supportsClaims = self::supportsClaims(); + $query = self::baseBuilder($groupID) + ->select('r.id') + ->orderByDesc('r.postdate') + ->orderBy('r.id') + ->limit($effectiveLimit); + + if (DB::getDriverName() !== 'sqlite') { + $query->lockForUpdate(); + } + + $ids = $query + ->pluck('r.id') + ->map(static fn (mixed $id): int => (int) $id) + ->all(); + + if ($ids === []) { + return (new Release)->newCollection(); + } + + if ($supportsClaims) { + Release::query() + ->whereIn('id', $ids) + ->update([ + self::CLAIMED_AT_COLUMN => now(), + self::CLAIM_TOKEN_COLUMN => $token, + ]); + } + + return Release::query() + ->whereIn('id', $ids) + ->with('category.parent') + ->select(self::selectableColumns($columns, $supportsClaims)) + ->orderByRaw(self::idOrderExpression($ids)) + ->get(); + }, 3); + } + + public static function clearClaim(int $releaseId, ?string $token = null): void + { + if (! self::supportsClaims()) { + return; + } + + $query = Release::query()->where('id', $releaseId); + if ($token !== null && $token !== '') { + $query->where(self::CLAIM_TOKEN_COLUMN, $token); + } + + $query->update([ + self::CLAIMED_AT_COLUMN => null, + self::CLAIM_TOKEN_COLUMN => null, + ]); + } + + public static function supportsClaims(): bool + { + if (! Schema::hasTable('releases')) { + return false; + } + + return Schema::hasColumn('releases', self::CLAIMED_AT_COLUMN) + && Schema::hasColumn('releases', self::CLAIM_TOKEN_COLUMN) + && Schema::hasColumn('releases', self::ATTEMPTS_COLUMN) + && Schema::hasColumn('releases', self::LAST_ERROR_COLUMN); + } + + /** + * @return array + */ + public static function failureUpdateValues(string $reason, bool $incrementAttempts): array + { + if (! self::supportsClaims()) { + return []; + } + + $values = [ + self::CLAIMED_AT_COLUMN => null, + self::CLAIM_TOKEN_COLUMN => null, + self::LAST_ERROR_COLUMN => mb_substr($reason, 0, 1000), + ]; + + if ($incrementAttempts) { + $values[self::ATTEMPTS_COLUMN] = DB::raw(self::ATTEMPTS_COLUMN.' + 1'); + } + + return $values; + } + + /** + * @param Builder $query + */ + private static function applyClaimWindow(Builder $query): void + { + if (! self::supportsClaims()) { + return; + } + + $staleBefore = now()->subSeconds(self::claimTtlSeconds()); + + $query->where(function (Builder $claimQuery) use ($staleBefore): void { + $claimQuery + ->whereNull('r.'.self::CLAIMED_AT_COLUMN) + ->orWhere('r.'.self::CLAIMED_AT_COLUMN, '<', $staleBefore); + }); + } + + private static function claimTtlSeconds(): int + { + $timeout = (int) (Settings::settingValue('releaseprocessingtimeout') ?: 120); + + return max(300, $timeout * 2); + } + + /** + * @param list $columns + * @return list + */ + private static function selectableColumns(array $columns, bool $supportsClaims): array + { + if ($supportsClaims || $columns === ['*']) { + return $columns; + } + + return array_values(array_filter( + $columns, + static fn (string $column): bool => ! in_array($column, [ + self::CLAIMED_AT_COLUMN, + self::CLAIM_TOKEN_COLUMN, + self::ATTEMPTS_COLUMN, + self::LAST_ERROR_COLUMN, + ], true), + )); + } + + /** + * @param list $ids + */ + private static function idOrderExpression(array $ids): string + { + if (DB::getDriverName() !== 'sqlite') { + return 'FIELD(id, '.implode(',', $ids).')'; + } + + $cases = []; + foreach ($ids as $position => $id) { + $cases[] = 'WHEN '.(int) $id.' THEN '.(int) $position; + } + + return 'CASE id '.implode(' ', $cases).' ELSE '.count($ids).' END'; + } +} diff --git a/app/Services/Nzb/NzbService.php b/app/Services/Nzb/NzbService.php index 9b1cb6d4b..d209ef754 100644 --- a/app/Services/Nzb/NzbService.php +++ b/app/Services/Nzb/NzbService.php @@ -10,11 +10,13 @@ use App\Models\Part; use App\Models\Release; use App\Models\Settings; use App\Services\CollectionCleanupService; +use App\Support\Data\NzbCreationResult; use Illuminate\Database\QueryException; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; +use Throwable; /** * Service for managing NZB files on disk. @@ -83,130 +85,180 @@ class NzbService /** * Write an NZB file for a release. * - * @throws \Throwable + * @throws Throwable */ public function writeNzbForReleaseId(Release $release): bool { - $collections = Collection::whereReleasesId($release->id) - ->join('usenet_groups', 'collections.groups_id', '=', 'usenet_groups.id') - ->select(['collections.*', DB::raw('UNIX_TIMESTAMP(collections.date) AS udate'), 'usenet_groups.name as groupname']) - ->get(); + return $this->createNzbForRelease($release)->success; + } + + public function createNzbForRelease(Release $release): NzbCreationResult + { + try { + $collections = Collection::whereReleasesId($release->id) + ->join('usenet_groups', 'collections.groups_id', '=', 'usenet_groups.id') + ->select(['collections.*', DB::raw('UNIX_TIMESTAMP(collections.date) AS udate'), 'usenet_groups.name as groupname']) + ->get(); + } catch (Throwable $e) { + return NzbCreationResult::transient('Failed to load release collections: '.$e->getMessage()); + } if ($collections->isEmpty()) { - return false; + return NzbCreationResult::deterministic('Release has no collections to write into an NZB.'); } // Pre-load every binary and part for the release in two flat queries // and group them in PHP, instead of issuing one binaries SELECT per // collection and one parts SELECT per binary (1 + N + N*M queries). - $collectionIds = $collections->pluck('id')->all(); - $binariesByCollection = Binary::query() - ->whereIn('collections_id', $collectionIds) - ->orderBy('name') - ->get(['id', 'collections_id', 'name', 'totalparts']) - ->groupBy('collections_id'); + $collectionIds = $collections->pluck('id')->map(static fn (mixed $id): int => (int) $id)->all(); + try { + $binariesByCollection = Binary::query() + ->whereIn('collections_id', $collectionIds) + ->orderBy('name') + ->get(['id', 'collections_id', 'name', 'totalparts']) + ->groupBy('collections_id'); - $allBinaryIds = $binariesByCollection->flatten(1)->pluck('id')->all(); - $partsByBinary = $allBinaryIds === [] - ? collect() - : Part::query() - ->whereIn('binaries_id', $allBinaryIds) - // distinct() preserves the dedup the previous per-binary - // query had, in case two rows share (messageid, size, - // partnumber) for the same binary (defensive). - ->distinct() - ->orderBy('partnumber') - ->get(['binaries_id', 'messageid', 'size', 'partnumber']) - ->groupBy('binaries_id'); + $allBinaryIds = $binariesByCollection->flatten(1)->pluck('id')->all(); + $partsByBinary = $allBinaryIds === [] + ? collect() + : Part::query() + ->whereIn('binaries_id', $allBinaryIds) + // distinct() preserves the dedup the previous per-binary + // query had, in case two rows share (messageid, size, + // partnumber) for the same binary (defensive). + ->distinct() + ->orderBy('partnumber') + ->get(['binaries_id', 'messageid', 'size', 'partnumber']) + ->groupBy('binaries_id'); + } catch (Throwable $e) { + return NzbCreationResult::transient('Failed to load NZB binaries or parts: '.$e->getMessage(), $collectionIds); + } $XMLWriter = new \XMLWriter; $XMLWriter->openMemory(); $XMLWriter->setIndent(true); $XMLWriter->setIndentString(' '); - $XMLWriter->startDocument('1.0', 'UTF-8'); - $XMLWriter->startDtd(self::NZB_DTD_NAME, self::NZB_DTD_PUBLIC, self::NZB_DTD_EXTERNAL); - $XMLWriter->endDtd(); - $XMLWriter->writeComment($this->nzbCommentString); + $path = null; + $tempPath = null; + $gz = null; - $XMLWriter->startElement('nzb'); - $XMLWriter->writeAttribute('xmlns', self::NZB_XML_NS); - $XMLWriter->startElement('head'); - $XMLWriter->startElement('meta'); - $XMLWriter->writeAttribute('type', 'category'); - $XMLWriter->text(! empty($release->category->parent) ? $release->category->parent->title.' >'.$release->category->title : 'Other > Misc'); - $XMLWriter->endElement(); - $XMLWriter->startElement('meta'); - $XMLWriter->writeAttribute('type', 'name'); - $XMLWriter->text($release->name); - $XMLWriter->endElement(); - $XMLWriter->endElement(); // head + try { + $path = ($this->buildNzbPath($release->guid, $this->nzbSplitLevel, true).$release->guid.'.nzb.gz'); + $tempPath = $this->temporaryNzbPath($path); + $gz = $this->openGzipFile($tempPath); - foreach ($collections as $collection) { - $binaries = $binariesByCollection->get($collection->id); - if ($binaries === null || $binaries->isEmpty()) { - return false; + if ($gz === false) { + return NzbCreationResult::transient("Failed to open temporary NZB file for writing: {$tempPath}", $collectionIds, $path); } - $poster = $collection->fromname; + $XMLWriter->startDocument('1.0', 'UTF-8'); + $XMLWriter->startDtd(self::NZB_DTD_NAME, self::NZB_DTD_PUBLIC, self::NZB_DTD_EXTERNAL); + $XMLWriter->endDtd(); + $XMLWriter->writeComment($this->nzbCommentString); - foreach ($binaries as $binary) { - $parts = $partsByBinary->get($binary->id); - if ($parts === null || $parts->isEmpty()) { - return false; + $XMLWriter->startElement('nzb'); + $XMLWriter->writeAttribute('xmlns', self::NZB_XML_NS); + $XMLWriter->startElement('head'); + $XMLWriter->startElement('meta'); + $XMLWriter->writeAttribute('type', 'category'); + $XMLWriter->text(! empty($release->category->parent) ? $release->category->parent->title.' >'.$release->category->title : 'Other > Misc'); + $XMLWriter->endElement(); + $XMLWriter->startElement('meta'); + $XMLWriter->writeAttribute('type', 'name'); + $XMLWriter->text($release->name); + $XMLWriter->endElement(); + $XMLWriter->endElement(); // head + if (! $this->flushXmlWriter($XMLWriter, $gz)) { + return NzbCreationResult::transient("Failed to write NZB header to temporary file: {$tempPath}", $collectionIds, $path); + } + + foreach ($collections as $collection) { + $binaries = $binariesByCollection->get($collection->id); + if ($binaries === null || $binaries->isEmpty()) { + return NzbCreationResult::deterministic("Collection {$collection->id} has no binaries.", $collectionIds, $path); } - $subject = $this->buildBinarySubject($binary->name, $binary->totalparts); - $XMLWriter->startElement('file'); - $XMLWriter->writeAttribute('poster', (string) $poster); - $XMLWriter->writeAttribute('date', (string) $collection->udate); - $XMLWriter->writeAttribute('subject', (string) $subject); - $XMLWriter->startElement('groups'); - if (preg_match_all('#(\S+):\S+#', $collection->xref, $hits)) { - $hits = array_values(array_unique($hits[1])); - foreach ($hits as $group) { + $groups = $this->groupsFromXref((string) $collection->xref); + if ($groups === []) { + return NzbCreationResult::deterministic("Collection {$collection->id} has no valid xref groups.", $collectionIds, $path); + } + + $poster = $collection->fromname; + + foreach ($binaries as $binary) { + $parts = $partsByBinary->get($binary->id); + if ($parts === null || $parts->isEmpty()) { + return NzbCreationResult::deterministic("Binary {$binary->id} has no parts.", $collectionIds, $path); + } + + $subject = $this->buildBinarySubject($binary->name, $binary->totalparts); + $XMLWriter->startElement('file'); + $XMLWriter->writeAttribute('poster', (string) $poster); + $XMLWriter->writeAttribute('date', (string) $collection->udate); + $XMLWriter->writeAttribute('subject', (string) $subject); + $XMLWriter->startElement('groups'); + foreach ($groups as $group) { $XMLWriter->writeElement('group', $group); } - } elseif (preg_match_all('#(\S+)#', $collection->xref, $hits)) { - $hits = array_values(array_unique($hits[1])); - foreach ($hits as $group) { - $XMLWriter->writeElement('group', $group); + $XMLWriter->endElement(); // groups + $XMLWriter->startElement('segments'); + foreach ($parts as $part) { + $messageId = $this->normalizeSegmentMessageId($part->messageid); + if ($messageId === '') { + return NzbCreationResult::deterministic("Part {$part->partnumber} for binary {$binary->id} has an empty message ID.", $collectionIds, $path); + } + + $XMLWriter->startElement('segment'); + $XMLWriter->writeAttribute('bytes', (string) $part->size); + $XMLWriter->writeAttribute('number', (string) $part->partnumber); + $XMLWriter->text($messageId); + $XMLWriter->endElement(); + } + $XMLWriter->endElement(); // segments + $XMLWriter->endElement(); // file + + if (! $this->flushXmlWriter($XMLWriter, $gz)) { + return NzbCreationResult::transient("Failed to write NZB file entry to temporary file: {$tempPath}", $collectionIds, $path); } - } else { - return false; } - $XMLWriter->endElement(); // groups - $XMLWriter->startElement('segments'); - foreach ($parts as $part) { - $messageId = $this->normalizeSegmentMessageId($part->messageid); - $XMLWriter->startElement('segment'); - $XMLWriter->writeAttribute('bytes', (string) $part->size); - $XMLWriter->writeAttribute('number', (string) $part->partnumber); - $XMLWriter->text($messageId); - $XMLWriter->endElement(); - } - $XMLWriter->endElement(); // segments - $XMLWriter->endElement(); // file + } + + $XMLWriter->writeComment($this->siteCommentString); + $XMLWriter->endElement(); // nzb + $XMLWriter->endDocument(); + if (! $this->flushXmlWriter($XMLWriter, $gz)) { + return NzbCreationResult::transient("Failed to write NZB footer to temporary file: {$tempPath}", $collectionIds, $path); + } + + $closed = gzclose($gz); + $gz = null; + if (! $closed) { + return NzbCreationResult::transient("Failed to close temporary NZB file: {$tempPath}", $collectionIds, $path); + } + + if (! $this->moveTemporaryNzbIntoPlace($tempPath, $path)) { + return NzbCreationResult::transient("Failed to move temporary NZB into place: {$tempPath} -> {$path}", $collectionIds, $path); + } + $tempPath = null; + + if (! File::isFile($path) || ! is_readable($path)) { + return NzbCreationResult::transient("Final NZB file is missing or unreadable: {$path}", $collectionIds, $path); + } + + // Mark release as having NZB. + $release->update($this->successfulReleaseUpdateValues()); + } catch (Throwable $e) { + return NzbCreationResult::transient('Failed to write NZB file: '.$e->getMessage(), $collectionIds, $path); + } finally { + unset($XMLWriter); + if (is_resource($gz)) { + gzclose($gz); + } + if ($tempPath !== null && File::isFile($tempPath)) { + File::delete($tempPath); } } - $XMLWriter->writeComment($this->siteCommentString); - $XMLWriter->endElement(); // nzb - $XMLWriter->endDocument(); - $path = ($this->buildNzbPath($release->guid, $this->nzbSplitLevel, true).$release->guid.'.nzb.gz'); - $fp = gzopen($path, 'wb7'); - if (! $fp) { - return false; - } - gzwrite($fp, $XMLWriter->outputMemory()); - gzclose($fp); - unset($XMLWriter); - if (! File::isFile($path)) { - echo "ERROR: $path does not exist.\n"; - - return false; - } - // Mark release as having NZB. - $release->update(['nzbstatus' => self::NZB_ADDED]); // Delete CBP (Collections, Binaries, Parts) for release that has its NZB created. // Use a transaction to ensure cascading deletes complete properly. @@ -216,7 +268,7 @@ class NzbService 'NZB cleanup', false ); - } catch (\Throwable $e) { + } catch (Throwable $e) { // Log the error but don't fail the NZB creation since the file was written successfully Log::warning('Failed to delete collections for release '.$release->id.': '.$e->getMessage()); } @@ -224,7 +276,7 @@ class NzbService // Chmod to fix issues some users have with file permissions. chmod($path, 0777); - return true; + return NzbCreationResult::success($path, $collectionIds); } /** @@ -340,6 +392,82 @@ class NzbService return File::delete($nzbPath); } + /** + * @return list + */ + public function findStaleTemporaryNzbPaths(int $olderThanSeconds = 86400): array + { + $cutoff = time() - max(1, $olderThanSeconds); + $paths = []; + + foreach (array_unique($this->siteNzbPaths) as $basePath) { + if (! File::isDirectory($basePath)) { + continue; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($basePath, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::LEAVES_ONLY + ); + + foreach ($iterator as $file) { + if (! $file->isFile() || $file->isLink()) { + continue; + } + + $path = $file->getPathname(); + if (! $this->isTemporaryNzbPath($path)) { + continue; + } + + try { + if ($file->getMTime() <= $cutoff) { + $paths[] = $path; + } + } catch (Throwable $e) { + Log::channel('nzb_creation')->warning('Failed to inspect temporary NZB file', [ + 'path' => $path, + 'error' => $e->getMessage(), + ]); + } + } + } + + sort($paths); + + return $paths; + } + + public function cleanupStaleTemporaryNzbs(int $olderThanSeconds = 86400): int + { + $deleted = 0; + + foreach ($this->findStaleTemporaryNzbPaths($olderThanSeconds) as $path) { + if (! File::isFile($path)) { + continue; + } + + try { + if (File::delete($path)) { + $deleted++; + + continue; + } + + Log::channel('nzb_creation')->warning('Failed to delete stale temporary NZB file', [ + 'path' => $path, + ]); + } catch (Throwable $e) { + Log::channel('nzb_creation')->warning('Failed to delete stale temporary NZB file', [ + 'path' => $path, + 'error' => $e->getMessage(), + ]); + } + } + + return $deleted; + } + /** * Get the default NZB split level. */ @@ -456,4 +584,79 @@ class NzbService return trim($messageId); } + + /** + * @return list + */ + private function groupsFromXref(string $xref): array + { + if (preg_match_all('#(\S+):\S+#', $xref, $hits)) { + return array_values(array_unique($hits[1])); + } + + if (preg_match_all('#(\S+)#', $xref, $hits)) { + return array_values(array_unique($hits[1])); + } + + return []; + } + + private function temporaryNzbPath(string $path): string + { + return $path.'.tmp.'.getmypid().'.'.bin2hex(random_bytes(6)); + } + + private function isTemporaryNzbPath(string $path): bool + { + return preg_match('/\.nzb\.gz\.tmp\.\d+\.[0-9a-f]{12}$/', basename($path)) === 1; + } + + /** + * @return resource|false + */ + protected function openGzipFile(string $path): mixed + { + return gzopen($path, 'wb7'); + } + + /** + * @param resource $gz + * + * @phpstan-impure + */ + private function flushXmlWriter(\XMLWriter $XMLWriter, mixed $gz): bool + { + $buffer = $XMLWriter->outputMemory(true); + if ($buffer === '') { + return true; + } + + $written = gzwrite($gz, $buffer); + + return $written !== false && $written === \strlen($buffer); + } + + protected function moveTemporaryNzbIntoPlace(string $temporaryPath, string $finalPath): bool + { + return rename($temporaryPath, $finalPath); + } + + /** + * @return array + */ + private function successfulReleaseUpdateValues(): array + { + $values = ['nzbstatus' => self::NZB_ADDED]; + + if (NzbCreationCandidateQuery::supportsClaims()) { + $values += [ + NzbCreationCandidateQuery::ATTEMPTS_COLUMN => 0, + NzbCreationCandidateQuery::LAST_ERROR_COLUMN => null, + NzbCreationCandidateQuery::CLAIMED_AT_COLUMN => null, + NzbCreationCandidateQuery::CLAIM_TOKEN_COLUMN => null, + ]; + } + + return $values; + } } diff --git a/app/Services/ReleaseProcessingService.php b/app/Services/ReleaseProcessingService.php index 535fe0135..8739f4cb5 100644 --- a/app/Services/ReleaseProcessingService.php +++ b/app/Services/ReleaseProcessingService.php @@ -14,10 +14,12 @@ use App\Models\Settings; use App\Models\UsenetGroup; use App\Services\Categorization\CategorizationService; use App\Services\NNTP\NNTPService; +use App\Services\Nzb\NzbCreationCandidateQuery; use App\Services\Nzb\NzbService; use App\Services\Releases\ReleaseBrowseService; use App\Services\Releases\ReleaseDuplicateFinder; use App\Services\Releases\ReleaseManagementService; +use App\Support\Data\NzbCreationResult; use App\Support\Data\ProcessReleasesSettings; use App\Support\Data\ReleaseCreationResult; use App\Support\Data\ReleaseDeleteStats; @@ -26,6 +28,7 @@ use DateTimeInterface; use Illuminate\Database\QueryException; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; use Throwable; /** @@ -51,7 +54,7 @@ final class ReleaseProcessingService private const int CATEGORIZE_CHUNK_SIZE = 1000; - private const int NZB_CHUNK_SIZE = 100; + private const int NZB_CREATION_MAX_ATTEMPTS = 3; private bool $echoCLI; @@ -536,37 +539,146 @@ final class ReleaseProcessingService $startTime = now()->toImmutable(); $this->outputSubHeader('Creating NZB Files'); - $query = Release::query() - ->with('category.parent') - ->where('nzbstatus', '=', NzbService::NZB_NONE) - ->select(['id', 'guid', 'name', 'categories_id']); - - if (! empty($groupID)) { - $query->where('releases.groups_id', $groupID); - } - $nzbCount = 0; - $total = $query->count(); + $retryCount = 0; + $deletedCount = 0; + $claimToken = bin2hex(random_bytes(16)); + $limit = max(1, $this->settings->releaseCreationLimit); + $total = min(NzbCreationCandidateQuery::baseBuilder($groupID)->count(), $limit); if ($total > 0) { - $query->chunkById(self::NZB_CHUNK_SIZE, function ($releases) use (&$nzbCount, $total): bool { - foreach ($releases as $release) { - if ($this->nzb->writeNzbForReleaseId($release)) { - $nzbCount++; - $this->outputProgress($nzbCount, $total, 'Creating NZBs'); - } - } + $columns = [ + 'id', + 'guid', + 'name', + 'categories_id', + 'groups_id', + 'postdate', + 'nzbstatus', + NzbCreationCandidateQuery::ATTEMPTS_COLUMN, + NzbCreationCandidateQuery::CLAIM_TOKEN_COLUMN, + ]; - return true; - }); + $processed = 0; + $releases = NzbCreationCandidateQuery::claimBatch($groupID, $limit, $claimToken, $columns); + foreach ($releases as $release) { + try { + $result = $this->nzb->createNzbForRelease($release); + if ($result->success) { + $nzbCount++; + + continue; + } + + if ($this->shouldDeleteFailedNzbCreation($release, $result)) { + $this->deleteFailedNzbCreationRelease($release, $result, $claimToken); + $deletedCount++; + + continue; + } + + $this->recordNzbCreationRetry($release, $result); + $retryCount++; + } catch (Throwable $e) { + $result = NzbCreationResult::transient('Unexpected NZB creation failure: '.$e->getMessage()); + if ($this->shouldDeleteFailedNzbCreation($release, $result)) { + $this->deleteFailedNzbCreationRelease($release, $result, $claimToken); + $deletedCount++; + } else { + $this->recordNzbCreationRetry($release, $result); + $retryCount++; + } + } finally { + NzbCreationCandidateQuery::clearClaim((int) $release->id, $claimToken); + $processed++; + $this->outputProgress($processed, $total, 'Creating NZBs'); + } + } } $this->outputStat('NZBs created', $nzbCount); + $this->outputStat('NZB creation retries', $retryCount); + $this->outputStat('NZB creation failures deleted', $deletedCount); $this->outputElapsedTime($startTime); return $nzbCount; } + private function shouldDeleteFailedNzbCreation(Release $release, NzbCreationResult $result): bool + { + if ($result->isDeterministicFailure()) { + return true; + } + + if (! $result->isTransientFailure()) { + return false; + } + + return $this->nextNzbCreationAttempt($release) >= self::NZB_CREATION_MAX_ATTEMPTS; + } + + private function nextNzbCreationAttempt(Release $release): int + { + return ((int) ($release->{NzbCreationCandidateQuery::ATTEMPTS_COLUMN} ?? 0)) + 1; + } + + private function recordNzbCreationRetry(Release $release, NzbCreationResult $result): void + { + $values = NzbCreationCandidateQuery::failureUpdateValues($result->reason, true); + if ($values !== []) { + Release::query() + ->where('id', $release->id) + ->update($values); + } + + Log::channel('nzb_creation')->warning('NZB creation failed; release will be retried', [ + 'release_id' => $release->id, + 'guid' => $release->guid, + 'failure_type' => $result->failureType, + 'reason' => $result->reason, + 'next_attempt' => $this->nextNzbCreationAttempt($release), + 'max_attempts' => self::NZB_CREATION_MAX_ATTEMPTS, + ]); + } + + private function deleteFailedNzbCreationRelease(Release $release, NzbCreationResult $result, string $claimToken): void + { + Log::channel('nzb_creation')->warning('Deleting release after NZB creation failure', [ + 'release_id' => $release->id, + 'guid' => $release->guid, + 'failure_type' => $result->failureType, + 'reason' => $result->reason, + 'attempt' => $this->nextNzbCreationAttempt($release), + 'max_attempts' => self::NZB_CREATION_MAX_ATTEMPTS, + ]); + + try { + $collectionIds = $result->collectionIds !== [] + ? $result->collectionIds + : Collection::query() + ->where('releases_id', $release->id) + ->pluck('id') + ->map(static fn (mixed $id): int => (int) $id) + ->all(); + + if ($collectionIds !== []) { + $this->collectionCleanupService->deleteCollectionsAndDescendants( + $collectionIds, + 'Failed NZB creation cleanup', + $this->echoCLI + ); + } + + $this->releaseManagement->deleteSingleWithService( + ['g' => $release->guid, 'i' => $release->id], + $this->nzb, + $this->releaseImage + ); + } finally { + NzbCreationCandidateQuery::clearClaim((int) $release->id, $claimToken); + } + } + /** * Categorize releases based on the specified field. * diff --git a/app/Support/Data/NzbCreationResult.php b/app/Support/Data/NzbCreationResult.php new file mode 100644 index 000000000..2345be334 --- /dev/null +++ b/app/Support/Data/NzbCreationResult.php @@ -0,0 +1,59 @@ + $collectionIds + */ + private function __construct( + public bool $success, + public string $failureType, + public string $reason, + public ?string $path, + public array $collectionIds, + ) {} + + /** + * @param list $collectionIds + */ + public static function success(string $path, array $collectionIds): self + { + return new self(true, self::FAILURE_NONE, '', $path, $collectionIds); + } + + /** + * @param list $collectionIds + */ + public static function deterministic(string $reason, array $collectionIds = [], ?string $path = null): self + { + return new self(false, self::FAILURE_DETERMINISTIC, $reason, $path, $collectionIds); + } + + /** + * @param list $collectionIds + */ + public static function transient(string $reason, array $collectionIds = [], ?string $path = null): self + { + return new self(false, self::FAILURE_TRANSIENT, $reason, $path, $collectionIds); + } + + public function isDeterministicFailure(): bool + { + return $this->failureType === self::FAILURE_DETERMINISTIC; + } + + public function isTransientFailure(): bool + { + return $this->failureType === self::FAILURE_TRANSIENT; + } +} diff --git a/config/logging.php b/config/logging.php index 8e6e514ec..d4ec598f1 100644 --- a/config/logging.php +++ b/config/logging.php @@ -144,6 +144,15 @@ return [ 'permission' => 0775, 'locking' => false, ], + 'nzb_creation' => [ + 'driver' => 'daily', + 'path' => storage_path('logs/nzb_creation.log'), + 'level' => 'debug', + 'days' => 7, + 'bubble' => true, + 'permission' => 0775, + 'locking' => false, + ], 'user_login' => [ 'driver' => 'daily', diff --git a/database/migrations/2026_07_13_000000_add_nzb_creation_claim_fields.php b/database/migrations/2026_07_13_000000_add_nzb_creation_claim_fields.php new file mode 100644 index 000000000..c462a97a6 --- /dev/null +++ b/database/migrations/2026_07_13_000000_add_nzb_creation_claim_fields.php @@ -0,0 +1,110 @@ +timestamp('nzb_creation_claimed_at')->nullable()->after('nzbstatus'); + } + + if (! Schema::hasColumn('releases', 'nzb_creation_claim_token')) { + $table->string('nzb_creation_claim_token', 64)->nullable()->after('nzb_creation_claimed_at'); + } + + if (! Schema::hasColumn('releases', 'nzb_creation_attempts')) { + $table->unsignedSmallInteger('nzb_creation_attempts')->default(0)->after('nzb_creation_claim_token'); + } + + if (! Schema::hasColumn('releases', 'nzb_creation_last_error')) { + $table->text('nzb_creation_last_error')->nullable()->after('nzb_creation_attempts'); + } + }); + + if (! $this->indexExists('releases', self::RELEASES_NZB_CLAIM_INDEX)) { + $this->createNzbCreationClaimIndex(); + } + } + + public function down(): void + { + if (! Schema::hasTable('releases')) { + return; + } + + if ($this->indexExists('releases', self::RELEASES_NZB_CLAIM_INDEX)) { + Schema::table('releases', function (Blueprint $table): void { + $table->dropIndex(self::RELEASES_NZB_CLAIM_INDEX); + }); + } + + Schema::table('releases', function (Blueprint $table): void { + if (Schema::hasColumn('releases', 'nzb_creation_last_error')) { + $table->dropColumn('nzb_creation_last_error'); + } + + if (Schema::hasColumn('releases', 'nzb_creation_attempts')) { + $table->dropColumn('nzb_creation_attempts'); + } + + if (Schema::hasColumn('releases', 'nzb_creation_claim_token')) { + $table->dropColumn('nzb_creation_claim_token'); + } + + if (Schema::hasColumn('releases', 'nzb_creation_claimed_at')) { + $table->dropColumn('nzb_creation_claimed_at'); + } + }); + } + + private function createNzbCreationClaimIndex(): void + { + if (DB::getDriverName() === 'sqlite') { + Schema::table('releases', function (Blueprint $table): void { + $table->index([ + 'nzbstatus', + 'groups_id', + 'leftguid', + 'nzb_creation_claimed_at', + 'postdate', + ], self::RELEASES_NZB_CLAIM_INDEX); + }); + + return; + } + + DB::statement( + 'CREATE INDEX `'.self::RELEASES_NZB_CLAIM_INDEX.'` ON `releases` '. + '(`nzbstatus`, `groups_id`, `leftguid`, `nzb_creation_claimed_at`, `postdate` DESC)' + ); + } + + private function indexExists(string $table, string $indexName): bool + { + if (DB::getDriverName() === 'sqlite') { + $indexes = DB::select("PRAGMA index_list('{$table}')"); + + foreach ($indexes as $index) { + if (($index->name ?? null) === $indexName) { + return true; + } + } + + return false; + } + + return DB::select("SHOW INDEX FROM `{$table}` WHERE Key_name = ?", [$indexName]) !== []; + } +}; diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index bcb38626a..5e1871fce 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -258,12 +258,6 @@ parameters: count: 2 path: app/Http/Controllers/Admin/AdminPageController.php - - - message: '#^Using nullsafe method call on non\-nullable type Carbon\\Carbon\. Use \-\> instead\.$#' - identifier: nullsafe.neverNull - count: 2 - path: app/Http/Controllers/Admin/AdminRegistrationController.php - - message: '#^Access to an undefined property Illuminate\\Database\\Eloquent\\Model\:\:\$id\.$#' identifier: property.notFound @@ -366,12 +360,6 @@ parameters: count: 1 path: app/Http/Controllers/Auth/PasskeyManagementController.php - - - message: '#^Using nullsafe property access on non\-nullable type Spatie\\LaravelPasskeys\\Models\\Passkey\. Use \-\> instead\.$#' - identifier: nullsafe.neverNull - count: 4 - path: app/Http/Controllers/Auth/PasskeyManagementController.php - - message: '#^Method App\\Http\\Controllers\\Auth\\RegisterController\:\:rejectUnavailableRegistration\(\) has parameter \$status with no value type specified in iterable type array\.$#' identifier: missingType.iterableValue @@ -450,84 +438,6 @@ parameters: count: 1 path: app/Models/Category.php - - - message: '#^Method App\\Models\\Content\:\:scopeActive\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Content.php - - - - message: '#^Method App\\Models\\Content\:\:scopeForRole\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Content.php - - - - message: '#^Method App\\Models\\Content\:\:scopeFrontPage\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Content.php - - - - message: '#^Method App\\Models\\Content\:\:scopeOfType\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Content.php - - - - message: '#^Method App\\Models\\Content\:\:scopeOrdered\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Content.php - - - - message: '#^Method App\\Models\\Genre\:\:scopeDisabled\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Genre.php - - - - message: '#^Method App\\Models\\Genre\:\:scopeEnabled\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Genre.php - - - - message: '#^Method App\\Models\\Genre\:\:scopeOfType\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Genre.php - - - - message: '#^Method App\\Models\\Invitation\:\:scopeActive\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Invitation.php - - - - message: '#^Method App\\Models\\Invitation\:\:scopeExpired\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Invitation.php - - - - message: '#^Method App\\Models\\Invitation\:\:scopeUnused\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Invitation.php - - - - message: '#^Method App\\Models\\Invitation\:\:scopeUsed\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Invitation.php - - - - message: '#^Method App\\Models\\Invitation\:\:scopeValid\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/Invitation.php - - message: '#^Call to an undefined method Illuminate\\Database\\Eloquent\\Builder\:\:enabled\(\)\.$#' identifier: method.notFound @@ -576,42 +486,6 @@ parameters: count: 1 path: app/Models/User.php - - - message: '#^Method App\\Models\\User\:\:scopeActive\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - - - message: '#^Method App\\Models\\User\:\:scopeExcludeSharing\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - - - message: '#^Method App\\Models\\User\:\:scopeExpired\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - - - message: '#^Method App\\Models\\User\:\:scopeExpiringSoon\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - - - message: '#^Method App\\Models\\User\:\:scopeVerified\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - - - message: '#^Method App\\Models\\User\:\:scopeWithRole\(\) return type with generic class Illuminate\\Database\\Eloquent\\Builder does not specify its types\: TModel$#' - identifier: missingType.generics - count: 1 - path: app/Models/User.php - - message: '#^Method App\\Models\\UserActivityStat\:\:getDownloadsPerDay\(\) should return array\ but returns list\\>\.$#' identifier: return.type @@ -1021,13 +895,6 @@ parameters: count: 1 path: app/Services/NfoService.php - - - message: '#^Using nullsafe method call on non\-nullable type Carbon\\Carbon\. Use \-\> instead\.$#' - identifier: nullsafe.neverNull - count: 2 - path: app/Services/RegistrationStatusService.php - - - message: '#^Call to function is_int\(\) with string\|null will always evaluate to false\.$#' identifier: function.impossibleType diff --git a/tests/Feature/NzbCreationReliabilityTest.php b/tests/Feature/NzbCreationReliabilityTest.php new file mode 100644 index 000000000..5d8c55e1a --- /dev/null +++ b/tests/Feature/NzbCreationReliabilityTest.php @@ -0,0 +1,449 @@ + + */ + private array $originalEnvironment = []; + + public function createApplication() + { + $this->databasePath = sys_get_temp_dir().'/nntmux-nzb-creation-reliability-test.sqlite'; + + $this->originalEnvironment = [ + 'APP_ENV' => getenv('APP_ENV'), + 'DB_CONNECTION' => getenv('DB_CONNECTION'), + 'DB_DATABASE' => getenv('DB_DATABASE'), + ]; + + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec( + 'INSERT INTO settings (name, value) VALUES '. + "('categorizeforeign', '0'), ". + "('catwebdl', '0'), ". + "('innerfileblacklist', '')" + ); + + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + $this->tempNzbPath = sys_get_temp_dir().'/nntmux-nzb-reliability-'.uniqid('', true); + config(['database.default' => 'sqlite', 'database.connections.sqlite.database' => $this->databasePath]); + config(['nntmux_settings.path_to_nzbs' => $this->tempNzbPath]); + DB::purge(); + DB::reconnect(); + DB::connection()->getPdo()->sqliteCreateFunction('UNIX_TIMESTAMP', static fn (?string $value): int => strtotime((string) $value)); + + $this->createSchema(); + $this->seedSettings(); + } + + protected function tearDown(): void + { + $this->deleteDirectory($this->tempNzbPath); + if ($this->databasePath !== '' && file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + parent::tearDown(); + + foreach ($this->originalEnvironment as $key => $value) { + $this->setEnvironmentValue($key, $value === false ? null : $value); + } + } + + public function test_candidate_query_skips_active_claims_and_recovers_stale_claims(): void + { + $this->insertRelease(1, 'a', claimedAt: now(), postdate: '2026-07-13 10:00:00'); + $this->insertRelease(2, 'b', claimedAt: now()->subSeconds(301), postdate: '2026-07-13 09:00:00'); + $this->insertRelease(3, 'c', postdate: '2026-07-13 08:00:00'); + + $claimed = NzbCreationCandidateQuery::claimBatch(null, 10, 'token-one', ['id']); + + $this->assertSame([2, 3], $claimed->pluck('id')->all()); + $this->assertSame('token-one', DB::table('releases')->where('id', 2)->value('nzb_creation_claim_token')); + $this->assertSame('claimed', DB::table('releases')->where('id', 1)->value('nzb_creation_claim_token')); + } + + public function test_deterministic_creation_failure_deletes_release_and_cbp_rows(): void + { + $this->insertRelease(1, 'a'); + $this->insertCbp(200, 2000, 1); + + $service = $this->releaseProcessingService( + NzbCreationResult::deterministic('Collection has invalid data.', [200]) + ); + + $this->assertSame(0, $service->createNZBs(null)); + $this->assertSame(0, DB::table('releases')->count()); + $this->assertSame(0, DB::table('collections')->count()); + $this->assertSame(0, DB::table('binaries')->count()); + $this->assertSame(0, DB::table('parts')->count()); + } + + public function test_transient_creation_failure_records_retry_before_threshold(): void + { + Log::partialMock(); + $nzbCreationLogger = new RecordingNzbCreationLogger; + Log::shouldReceive('channel') + ->once() + ->with('nzb_creation') + ->andReturn($nzbCreationLogger); + + $this->insertRelease(1, 'a', attempts: 0); + $this->insertCbp(200, 2000, 1); + + $service = $this->releaseProcessingService( + NzbCreationResult::transient('Temporary filesystem failure.', [200]) + ); + + $this->assertSame(0, $service->createNZBs(null)); + $this->assertSame(1, DB::table('releases')->count()); + $this->assertSame(1, (int) DB::table('releases')->where('id', 1)->value('nzb_creation_attempts')); + $this->assertSame('Temporary filesystem failure.', DB::table('releases')->where('id', 1)->value('nzb_creation_last_error')); + $this->assertNull(DB::table('releases')->where('id', 1)->value('nzb_creation_claimed_at')); + $this->assertSame(1, DB::table('collections')->count()); + $this->assertSame('NZB creation failed; release will be retried', $nzbCreationLogger->warnings[0]['message']); + $this->assertSame(1, $nzbCreationLogger->warnings[0]['context']['release_id']); + $this->assertSame(str_repeat('a', 36), $nzbCreationLogger->warnings[0]['context']['guid']); + $this->assertSame(NzbCreationResult::FAILURE_TRANSIENT, $nzbCreationLogger->warnings[0]['context']['failure_type']); + $this->assertSame('Temporary filesystem failure.', $nzbCreationLogger->warnings[0]['context']['reason']); + $this->assertSame(1, $nzbCreationLogger->warnings[0]['context']['next_attempt']); + $this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['max_attempts']); + } + + public function test_third_transient_creation_failure_deletes_release_and_cbp_rows(): void + { + Log::partialMock(); + $nzbCreationLogger = new RecordingNzbCreationLogger; + Log::shouldReceive('channel') + ->once() + ->with('nzb_creation') + ->andReturn($nzbCreationLogger); + + $this->insertRelease(1, 'a', attempts: 2); + $this->insertCbp(200, 2000, 1); + + $service = $this->releaseProcessingService( + NzbCreationResult::transient('Repeated filesystem failure.', [200]) + ); + + $this->assertSame(0, $service->createNZBs(null)); + $this->assertSame(0, DB::table('releases')->count()); + $this->assertSame(0, DB::table('collections')->count()); + $this->assertSame(0, DB::table('binaries')->count()); + $this->assertSame(0, DB::table('parts')->count()); + $this->assertSame('Deleting release after NZB creation failure', $nzbCreationLogger->warnings[0]['message']); + $this->assertSame(1, $nzbCreationLogger->warnings[0]['context']['release_id']); + $this->assertSame(str_repeat('a', 36), $nzbCreationLogger->warnings[0]['context']['guid']); + $this->assertSame(NzbCreationResult::FAILURE_TRANSIENT, $nzbCreationLogger->warnings[0]['context']['failure_type']); + $this->assertSame('Repeated filesystem failure.', $nzbCreationLogger->warnings[0]['context']['reason']); + $this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['attempt']); + $this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['max_attempts']); + } + + public function test_writer_classifies_final_rename_failure_as_transient_without_final_file(): void + { + $this->insertRelease(1, 'd'); + $this->insertWritableCbp(200, 2000, 1); + $release = Release::query()->findOrFail(1); + $release->setRelation('category', (object) ['title' => 'Misc', 'parent' => (object) ['title' => 'Other']]); + + $nzb = new RenameFailingNzbService(app(CollectionCleanupService::class)); + $result = $nzb->createNzbForRelease($release); + + $this->assertFalse($result->success); + $this->assertTrue($result->isTransientFailure()); + $this->assertStringContainsString('move temporary NZB into place', $result->reason); + $this->assertFileDoesNotExist($nzb->getNzbPath($release->guid)); + $this->assertSame(0, (int) DB::table('releases')->where('id', 1)->value('nzbstatus')); + } + + public function test_stale_temporary_nzb_cleanup_deletes_only_old_temp_files(): void + { + $directory = $this->tempNzbPath.'/a'; + mkdir($directory, 0777, true); + + $oldTemporary = $directory.'/'.str_repeat('e', 36).'.nzb.gz.tmp.123.'.str_repeat('a', 12); + $recentTemporary = $directory.'/'.str_repeat('f', 36).'.nzb.gz.tmp.456.'.str_repeat('b', 12); + $finalNzb = $directory.'/'.str_repeat('g', 36).'.nzb.gz'; + + file_put_contents($oldTemporary, 'old'); + file_put_contents($recentTemporary, 'recent'); + file_put_contents($finalNzb, 'final'); + touch($oldTemporary, time() - 7200); + touch($recentTemporary, time() - 60); + touch($finalNzb, time() - 7200); + + $nzb = new NzbService(app(CollectionCleanupService::class)); + + $this->assertSame([$oldTemporary], $nzb->findStaleTemporaryNzbPaths(3600)); + $this->assertSame(1, $nzb->cleanupStaleTemporaryNzbs(3600)); + $this->assertFileDoesNotExist($oldTemporary); + $this->assertFileExists($recentTemporary); + $this->assertFileExists($finalNzb); + } + + private function releaseProcessingService(NzbCreationResult $result): ReleaseProcessingService + { + return (new ReleaseProcessingService( + nzb: new FakeNzbCreationService($result), + releaseManagement: new DatabaseOnlyReleaseManagementService, + collectionCleanupService: app(CollectionCleanupService::class), + ))->setEchoCLI(false); + } + + private function insertRelease( + int $id, + string $leftguid, + int $attempts = 0, + ?\DateTimeInterface $claimedAt = null, + string $postdate = '2026-07-13 00:00:00', + ): void { + DB::table('releases')->insert([ + 'id' => $id, + 'guid' => str_pad($leftguid, 36, $leftguid), + 'leftguid' => $leftguid, + 'name' => 'Release.'.$id, + 'searchname' => 'Release.'.$id, + 'groups_id' => 1, + 'categories_id' => 1, + 'postdate' => $postdate, + 'nzbstatus' => NzbService::NZB_NONE, + 'nzb_creation_claimed_at' => $claimedAt?->format('Y-m-d H:i:s'), + 'nzb_creation_claim_token' => $claimedAt === null ? null : 'claimed', + 'nzb_creation_attempts' => $attempts, + 'nzb_creation_last_error' => null, + ]); + } + + private function insertCbp(int $collectionId, int $binaryId, int $releaseId): void + { + DB::table('collections')->insert([ + 'id' => $collectionId, + 'releases_id' => $releaseId, + ]); + DB::table('binaries')->insert([ + 'id' => $binaryId, + 'collections_id' => $collectionId, + ]); + DB::table('parts')->insert([ + 'binaries_id' => $binaryId, + ]); + } + + private function insertWritableCbp(int $collectionId, int $binaryId, int $releaseId): void + { + DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.test']); + DB::table('collections')->insert([ + 'id' => $collectionId, + 'releases_id' => $releaseId, + 'fromname' => 'poster@example.test', + 'date' => '2026-07-13 10:00:00', + 'xref' => 'alt.test:12345', + 'groups_id' => 1, + ]); + DB::table('binaries')->insert([ + 'id' => $binaryId, + 'collections_id' => $collectionId, + 'name' => 'Example.Release.part01.rar yEnc', + 'totalparts' => 1, + ]); + DB::table('parts')->insert([ + 'binaries_id' => $binaryId, + 'messageid' => '', + 'partnumber' => 1, + 'size' => 100, + ]); + } + + private function seedSettings(): void + { + foreach ([ + 'categorizeforeign' => '0', + 'catwebdl' => '0', + 'releaseprocessingtimeout' => '120', + 'maxnzbsprocessed' => '1000', + 'nzbsplitlevel' => '1', + ] as $name => $value) { + DB::table('settings')->insert(['name' => $name, 'value' => $value]); + } + } + + private function createSchema(): void + { + foreach (['parts', 'binaries', 'collections', 'releases', 'categories', 'root_categories', 'usenet_groups', 'settings'] as $table) { + DB::statement("DROP TABLE IF EXISTS {$table}"); + } + + DB::statement('CREATE TABLE settings (name VARCHAR(255) PRIMARY KEY, value TEXT)'); + DB::statement('CREATE TABLE root_categories (id INTEGER PRIMARY KEY, title VARCHAR(255), status INTEGER DEFAULT 1, disablepreview INTEGER DEFAULT 0)'); + DB::statement('CREATE TABLE categories (id INTEGER PRIMARY KEY, title VARCHAR(255), root_categories_id INTEGER NULL)'); + DB::statement('CREATE TABLE releases ( + id INTEGER PRIMARY KEY, + guid VARCHAR(64), + leftguid VARCHAR(1), + name VARCHAR(255), + searchname VARCHAR(255), + groups_id INTEGER, + categories_id INTEGER, + postdate DATETIME NULL, + nzbstatus INTEGER, + nzb_creation_claimed_at DATETIME NULL, + nzb_creation_claim_token VARCHAR(64) NULL, + nzb_creation_attempts INTEGER DEFAULT 0, + nzb_creation_last_error TEXT NULL + )'); + DB::statement('CREATE TABLE usenet_groups (id INTEGER PRIMARY KEY, name VARCHAR(255))'); + DB::statement('CREATE TABLE collections ( + id INTEGER PRIMARY KEY, + releases_id INTEGER NULL, + fromname VARCHAR(255) NULL, + date DATETIME NULL, + xref TEXT NULL, + groups_id INTEGER NULL + )'); + DB::statement('CREATE TABLE binaries ( + id INTEGER PRIMARY KEY, + collections_id INTEGER, + name VARCHAR(255) NULL, + totalparts INTEGER NULL + )'); + DB::statement('CREATE TABLE parts ( + binaries_id INTEGER, + messageid VARCHAR(255) NULL, + partnumber INTEGER NULL, + size INTEGER NULL + )'); + DB::table('root_categories')->insert(['id' => 1, 'title' => 'Other']); + DB::table('categories')->insert(['id' => 1, 'title' => 'Misc', 'root_categories_id' => 1]); + } + + private function deleteDirectory(string $path): void + { + if ($path === '' || ! is_dir($path)) { + return; + } + + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($path, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST + ); + + foreach ($iterator as $item) { + if ($item->isDir() && ! $item->isLink()) { + rmdir($item->getPathname()); + } else { + unlink($item->getPathname()); + } + } + + rmdir($path); + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv($key.'='.$value); + $_ENV[$key] = $value; + $_SERVER[$key] = $value; + } +} + +class FakeNzbCreationService extends NzbService +{ + public function __construct(private readonly NzbCreationResult $result) + { + parent::__construct(app(CollectionCleanupService::class)); + } + + public function createNzbForRelease(Release $release): NzbCreationResult + { + return $this->result; + } +} + +class DatabaseOnlyReleaseManagementService extends ReleaseManagementService +{ + /** + * @param array $identifiers + */ + public function deleteSingleWithService(array $identifiers, NzbService $nzb, ReleaseImageService $releaseImage): void + { + Release::query()->where('guid', $identifiers['g'])->delete(); + } +} + +class RenameFailingNzbService extends NzbService +{ + protected function moveTemporaryNzbIntoPlace(string $temporaryPath, string $finalPath): bool + { + return false; + } +} + +class RecordingNzbCreationLogger extends AbstractLogger +{ + /** + * @var list}> + */ + public array $warnings = []; + + /** + * @param array $context + */ + public function log($level, string|\Stringable $message, array $context = []): void + { + $this->warnings[] = [ + 'level' => $level, + 'message' => (string) $message, + 'context' => $context, + ]; + } +}