From a4028bef068b79564fbfb823a7cbd44d5443da78 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Fri, 6 Mar 2026 10:27:06 +0100 Subject: [PATCH] Improve nfo processing --- app/Models/ReleaseFile.php | 29 ++ .../AdditionalProcessingOrchestrator.php | 89 ++++++ app/Services/NfoService.php | 280 ++++++++++++++++-- 3 files changed, 374 insertions(+), 24 deletions(-) diff --git a/app/Models/ReleaseFile.php b/app/Models/ReleaseFile.php index 290d1c1f8..4f4f51a38 100644 --- a/app/Models/ReleaseFile.php +++ b/app/Models/ReleaseFile.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace App\Models; use App\Facades\Search; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Log; @@ -58,6 +59,34 @@ class ReleaseFile extends Model return $this->belongsTo(Release::class, 'releases_id'); } + /** + * Scope to filter release files that look like NFO files. + * + * @param Builder $query + * @return Builder + */ + public function scopeNfoFiles(Builder $query): Builder // @phpstan-ignore missingType.generics, return.phpDocType + { + return $query->where(function (Builder $q) { // @phpstan-ignore missingType.generics + $q->where('name', 'like', '%.nfo') + ->orWhere('name', 'like', '%.diz') + ->orWhere('name', 'like', '%.inf') + ->orWhere('name', 'like', '%file\_id.diz') + ->orWhere('name', 'like', '%info.txt'); + }); + } + + /** + * Scope to filter NFO files that have non-zero size (likely contain actual content). + * + * @param Builder $query + * @return Builder + */ + public function scopeNfoFilesWithContent(Builder $query): Builder // @phpstan-ignore missingType.generics, return.phpDocType + { + return $query->nfoFiles()->where('size', '>', 0); + } + /** * Get releasefiles row by id. */ diff --git a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php index 5822ae0d6..80e95e5a5 100644 --- a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php +++ b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php @@ -268,6 +268,11 @@ class AdditionalProcessingOrchestrator if (! $context->foundJPGSample && $this->config->processJPGSample) { $this->processJpgFromReleaseFiles($context); } + + // If still no NFO, try to fetch NFO from release_files entries + if ($context->releaseHasNoNFO) { + $this->processNfoFromReleaseFiles($context); + } } // Finalize @@ -804,6 +809,90 @@ class AdditionalProcessingOrchestrator } } + /** + * Process NFO files from existing release_files entries. + * When archive extraction failed to produce NFO on disk, this falls back to + * checking the release_files table and re-downloading the archive to extract by name. + */ + private function processNfoFromReleaseFiles(ReleaseProcessingContext $context): void + { + $nfoFiles = ReleaseFile::where('releases_id', $context->release->id) + ->nfoFiles() + ->limit(3) + ->get(); + + if ($nfoFiles->isEmpty()) { + return; + } + + if (empty($context->nzbContents)) { + return; + } + + foreach ($context->nzbContents as $nzbFile) { + if (! $context->releaseHasNoNFO) { + break; + } + + $title = $nzbFile['title'] ?? ''; + if (! preg_match( + '/(\\.(part0*1|rar|zip))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$/i', + $title + )) { + continue; + } + + $segments = $nzbFile['segments'] ?? []; + if (empty($segments)) { + continue; + } + + $messageIDs = []; + $segCount = min(count($segments), $this->config->maximumRarSegments); + for ($i = 0; $i < $segCount; $i++) { + $messageIDs[] = (string) $segments[$i]; + } + + if (empty($messageIDs)) { + continue; + } + + $result = $this->downloadService->downloadCompressedFile( + $messageIDs, // @phpstan-ignore argument.type + $context->releaseGroupName, + $context->release->id, + $title + ); + + if (! $result['success'] || empty($result['data'])) { + continue; + } + + foreach ($nfoFiles as $nfoFile) { + $nfoData = $this->archiveService->extractSpecificFile( + $result['data'], + $nfoFile->name, + $context->tmpPath + ); + + if ($nfoData === null || empty($nfoData)) { + continue; + } + + $tempNfoPath = $context->tmpPath.'release_file_'.uniqid('', true).'.nfo'; + File::put($tempNfoPath, $nfoData); + + if ($this->releaseManager->processNfoFile($tempNfoPath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + File::delete($tempNfoPath); + break 2; + } + + File::delete($tempNfoPath); + } + } + } + /** * Process extracted files from archives. */ diff --git a/app/Services/NfoService.php b/app/Services/NfoService.php index 06786b196..5abba54a2 100644 --- a/app/Services/NfoService.php +++ b/app/Services/NfoService.php @@ -5,11 +5,14 @@ declare(strict_types=1); namespace App\Services; use App\Models\Release; +use App\Models\ReleaseFile; use App\Models\ReleaseNfo; use App\Models\Settings; use App\Models\UsenetGroup; use App\Services\NNTP\NNTPService; use App\Services\Nzb\NzbContentsService; +use App\Services\Nzb\NzbParserService; +use App\Services\Nzb\NzbService; use dariusiii\rarinfo\Par2Info; use dariusiii\rarinfo\SfvInfo; use Illuminate\Support\Facades\Cache; @@ -134,8 +137,16 @@ class NfoService */ protected bool $echo; + private string|false $unrarPath; + + private string|false $timeoutPath; + + private int $timeoutSeconds; + public const NFO_FAILED = -9; // We failed to get a NFO after admin set max retries. + public const NFO_FAILED_ARCHIVE = -10; // Archive-based NFO extraction also failed; no more retries. + public const NFO_UNPROC = -1; // Release has not been processed yet. public const NFO_NONFO = 0; // Release has no NFO. @@ -176,6 +187,9 @@ class NfoService }); $this->tmpPath = rtrim((string) config('nntmux.tmp_unrar_path'), '/\\').'/'; + $this->unrarPath = config('nntmux_settings.unrar_path') ?: false; + $this->timeoutPath = config('nntmux_settings.timeout_path') ?: false; + $this->timeoutSeconds = (int) (Settings::settingValue('timeoutseconds') ?: 60); } /** @@ -492,6 +506,165 @@ class NfoService return false; } + /** + * Fallback: attempt to extract an NFO from a compressed archive in the NZB + * when the direct NZB-subject-based download has failed. + * + * Uses the release_files table to know which NFO filename to look for, + * then downloads a RAR/ZIP from the NZB and extracts it with unrar/unzip. + * + * When NFO entries exist in release_files but extraction yields no valid + * content (e.g. 0-byte file), sets nfostatus to NFO_NONFO to stop retries. + * + * @return string|false The NFO content, or false on failure + */ + public function attemptNfoFromArchive(string $guid, int $releaseId, NNTPService $nntp): string|false + { + if ($this->unrarPath === false) { + return false; + } + + $nfoFiles = ReleaseFile::where('releases_id', $releaseId) + ->nfoFiles() + ->limit(3) + ->pluck('name') + ->all(); + + if (empty($nfoFiles)) { + return false; + } + + $hasContentfulNfo = ReleaseFile::where('releases_id', $releaseId) + ->nfoFilesWithContent() + ->exists(); + + if (! $hasContentfulNfo) { + Release::whereId($releaseId)->update(['nfostatus' => self::NFO_NONFO]); + + return false; + } + + $nzbService = app(NzbService::class); + $nzbContents = $nzbService->readNzbContents($guid); + if ($nzbContents === false) { + return false; + } + + $parserService = app(NzbParserService::class); + $fileList = $parserService->parseNzbFileList($nzbContents); + if (empty($fileList)) { + return false; + } + + $alternateNntp = (bool) config('nntmux_nntp.use_alternate_nntp_server'); + $maxSegments = 40; + $triedArchive = false; + + foreach ($fileList as $nzbFile) { + $title = $nzbFile['title'] ?? ''; + if (! preg_match( + '/(\\.(part0*1|rar|zip))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$/i', + $title + )) { + continue; + } + + $segments = $nzbFile['segments'] ?? []; + if (empty($segments)) { + continue; + } + + $messageIDs = array_slice($segments, 0, $maxSegments); + $compressedData = $nntp->getMessagesByMessageID($messageIDs, $alternateNntp); + + if (! is_string($compressedData) || $compressedData === '') { + continue; + } + + $triedArchive = true; + $extracted = $this->extractNfoViaUnrar($compressedData, $nfoFiles, $guid); + if ($extracted !== false) { + return $extracted; + } + + break; + } + + if ($triedArchive) { + Release::whereId($releaseId)->update(['nfostatus' => self::NFO_NONFO]); + } + + return false; + } + + /** + * Extract an NFO file from compressed archive data using the unrar binary. + * + * @param string $compressedData Raw archive bytes + * @param array $nfoFilenames NFO filenames to try extracting + * @param string $guid Release GUID for NFO validation + * @return string|false Extracted NFO content, or false on failure + */ + private function extractNfoViaUnrar(string $compressedData, array $nfoFilenames, string $guid): string|false + { + $uniqueId = uniqid('nfo_', true); + $archiveFile = $this->tmpPath.'archive_'.$uniqueId.'.rar'; + $extractDir = $this->tmpPath.'extract_'.$uniqueId.'/'; + + try { + if (! File::isDirectory($extractDir)) { + File::makeDirectory($extractDir, 0777, true, true); + } + + File::put($archiveFile, $compressedData); + + $killString = $this->getKillString(); + + foreach ($nfoFilenames as $nfoFilename) { + runCmd($killString.$this->unrarPath.'" e -y -c- -inul -p- "'.$archiveFile.'" "'.$nfoFilename.'" "'.$extractDir.'"'); + + $extractedPath = $extractDir.basename($nfoFilename); + if (! File::isFile($extractedPath)) { + $allFiles = File::allFiles($extractDir); + foreach ($allFiles as $file) { + if (strcasecmp($file->getFilename(), basename($nfoFilename)) === 0) { + $extractedPath = $file->getPathname(); + break; + } + } + } + + if (File::isFile($extractedPath) && File::size($extractedPath) > 0) { + $content = File::get($extractedPath); + if ($this->isNFO($content, $guid)) { + return $content; + } + } + } + } catch (Throwable $e) { + Log::debug('NFO unrar extraction failed: '.$e->getMessage()); + } finally { + File::delete($archiveFile); + if (File::isDirectory($extractDir)) { + File::deleteDirectory($extractDir); + } + } + + return false; + } + + /** + * Build the kill/timeout string for wrapping CLI commands. + */ + private function getKillString(): string + { + if ($this->timeoutPath && $this->timeoutSeconds > 0) { + return '"'.$this->timeoutPath.'" --foreground --signal=KILL '.$this->timeoutSeconds.' "'; + } + + return '"'; + } + /** * Attempt to find NFO files inside the NZB's of releases. * @@ -539,10 +712,17 @@ class NfoService $groupName = UsenetGroup::getNameByID($release['groups_id']); $fetchedBinary = $nzbContentsService->getNfoFromNzb($release['guid'], $release['id'], $release['groups_id'], $groupName); + // Fallback: try extracting NFO from a RAR/ZIP in the NZB + if ($fetchedBinary === false) { + $fetchedBinary = $this->attemptNfoFromArchive($release['guid'], $release['id'], $nntp); + if ($fetchedBinary !== false && $this->echo) { + echo 'A'; + } + } + if ($fetchedBinary !== false) { DB::beginTransaction(); try { - // Only insert if not already present $exists = ReleaseNfo::whereReleasesId($release['id'])->exists(); if (! $exists) { ReleaseNfo::query()->insert([ @@ -551,7 +731,6 @@ class NfoService ]); } - // Update status Release::whereId($release['id'])->update(['nfostatus' => self::NFO_FOUND]); DB::commit(); $processedCount++; @@ -570,8 +749,10 @@ class NfoService } } - // Process failed NFO attempts - $this->handleFailedNfoAttempts($groupID, $guidChar); + // Second pass: attempt archive-based extraction for NFO_FAILED (-9) releases + // that have NFO files listed in release_files with non-zero size. + $archiveProcessed = $this->processFailedReleasesViaArchive($nntp, $groupID, $guidChar); + $processedCount += $archiveProcessed; // Output results if ($this->echo) { @@ -651,41 +832,92 @@ class NfoService } /** - * Handle releases that have failed too many NFO fetch attempts + * Process releases at NFO_FAILED (-9) that have NFO files in release_files + * with non-zero size. Attempts archive-based extraction via unrar. + * On failure, sets nfostatus to NFO_FAILED_ARCHIVE (-10) to prevent further retries. */ - private function handleFailedNfoAttempts(string $groupID, string $guidChar): void + private function processFailedReleasesViaArchive(NNTPService $nntp, string $groupID, string $guidChar): int { - $failedQuery = Release::query() - ->where('nfostatus', '<', $this->maxRetries) - ->where('nfostatus', '>', self::NFO_FAILED); + $query = Release::query() + ->where('nfostatus', self::NFO_FAILED) + ->whereExists(function ($sub) { + $sub->select(DB::raw(1)) + ->from('release_files') + ->whereColumn('release_files.releases_id', 'releases.id') + ->where('size', '>', 0) + ->where(function ($q) { + $q->where('name', 'like', '%.nfo') + ->orWhere('name', 'like', '%.diz') + ->orWhere('name', 'like', '%.inf') + ->orWhere('name', 'like', '%file\_id.diz') + ->orWhere('name', 'like', '%info.txt'); + }); + }); if ($guidChar !== '') { - $failedQuery->where('leftguid', $guidChar); + $query->where('leftguid', $guidChar); } if ($groupID !== '') { - $failedQuery->where('groups_id', $groupID); + $query->where('groups_id', $groupID); } - // Process in chunks to avoid memory issues with large result sets - $failedQuery->select(['id'])->chunk(100, function ($releases) { - DB::beginTransaction(); - try { - foreach ($releases as $release) { - // Remove any releasenfo for failed attempts - ReleaseNfo::whereReleasesId($release->id)->delete(); + $releases = $query->limit($this->nzbs) + ->get(['id', 'guid', 'groups_id', 'name']); - // Set release.nfostatus to failed - Release::whereId($release->id)->update(['nfostatus' => self::NFO_FAILED]); + if ($releases->isEmpty()) { + return 0; + } + + if ($this->echo) { + cli()->primary(PHP_EOL.'Attempting archive-based NFO extraction for '.$releases->count().' NFO_FAILED release(s).'); + } + + $processed = 0; + + foreach ($releases as $release) { + try { + $fetchedBinary = $this->attemptNfoFromArchive($release['guid'], $release['id'], $nntp); + + if ($fetchedBinary !== false) { + DB::beginTransaction(); + try { + if (! ReleaseNfo::whereReleasesId($release['id'])->exists()) { + ReleaseNfo::query()->insert([ + 'releases_id' => $release['id'], + 'nfo' => "\x1f\x8b\x08\x00".gzcompress($fetchedBinary), + ]); + } + Release::whereId($release['id'])->update(['nfostatus' => self::NFO_FOUND]); + DB::commit(); + $processed++; + + if ($this->echo) { + echo 'A'; + } + } catch (\Exception $e) { + DB::rollBack(); + if ($this->echo) { + cli()->error("Error saving archive NFO for release {$release['id']}: {$e->getMessage()}"); + } + } + } else { + Release::whereId($release['id'])->update(['nfostatus' => self::NFO_FAILED_ARCHIVE]); + + if ($this->echo) { + echo '-'; + } } - DB::commit(); } catch (\Exception $e) { - DB::rollBack(); + Release::whereId($release['id'])->update(['nfostatus' => self::NFO_FAILED_ARCHIVE]); + if ($this->echo) { - cli()->error("Error handling failed NFO attempts: {$e->getMessage()}"); + cli()->error("Error in archive NFO extraction for release {$release['id']}: {$e->getMessage()}"); } } - }); + } + + return $processed; } /**