diff --git a/app/Console/Commands/PostProcessGuid.php b/app/Console/Commands/PostProcessGuid.php index 1407bb051..e59020d8d 100644 --- a/app/Console/Commands/PostProcessGuid.php +++ b/app/Console/Commands/PostProcessGuid.php @@ -80,7 +80,11 @@ class PostProcessGuid extends Command */ private function processAdditional(string $guid): void { - $this->additionalProcessor->start('', $guid); + try { + $this->additionalProcessor->start('', $guid); + } finally { + $this->additionalProcessor->finish(); + } } /** diff --git a/app/Console/Commands/ProcessAdditionalGuid.php b/app/Console/Commands/ProcessAdditionalGuid.php index d2b7b2da6..0096f4aa1 100644 --- a/app/Console/Commands/ProcessAdditionalGuid.php +++ b/app/Console/Commands/ProcessAdditionalGuid.php @@ -85,7 +85,12 @@ class ProcessAdditionalGuid extends Command } $processor = app(AdditionalProcessingOrchestrator::class); - $ok = $processor->processSingleGuid($guid); + + try { + $ok = $processor->processSingleGuid($guid); + } finally { + $processor->finish(); + } if (! $ok) { $this->error('Processing failed or nothing processed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.'); diff --git a/app/Console/Commands/UpdatePerGroup.php b/app/Console/Commands/UpdatePerGroup.php index d9510a208..bc781e403 100644 --- a/app/Console/Commands/UpdatePerGroup.php +++ b/app/Console/Commands/UpdatePerGroup.php @@ -77,7 +77,12 @@ class UpdatePerGroup extends Command // Post process the releases $this->info("Post-processing additional for group: {$groupMySQL['name']}"); - app(AdditionalProcessingOrchestrator::class)->start($groupId); + $orchestrator = app(AdditionalProcessingOrchestrator::class); + try { + $orchestrator->start($groupId); + } finally { + $orchestrator->finish(); + } $this->info("Processing NFO files for group: {$groupMySQL['name']}"); (new NfoService)->processNfoFiles( diff --git a/app/Providers/AdditionalProcessingServiceProvider.php b/app/Providers/AdditionalProcessingServiceProvider.php index 155148664..abdae6823 100644 --- a/app/Providers/AdditionalProcessingServiceProvider.php +++ b/app/Providers/AdditionalProcessingServiceProvider.php @@ -11,6 +11,8 @@ use App\Services\AdditionalProcessing\ConsoleOutputService; use App\Services\AdditionalProcessing\MediaExtractionService; use App\Services\AdditionalProcessing\NzbContentParser; use App\Services\AdditionalProcessing\ReleaseFileManager; +use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback; +use App\Services\AdditionalProcessing\ReleaseProcessor; use App\Services\AdditionalProcessing\UsenetDownloadService; use App\Services\Categorization\CategorizationService; use App\Services\NameFixing\NameFixingService; @@ -97,6 +99,31 @@ class AdditionalProcessingServiceProvider extends ServiceProvider ); }); + $this->app->singleton(ReleaseFilesArchiveFallback::class, function ($app) { + return new ReleaseFilesArchiveFallback( + $app->make(ProcessingConfiguration::class), + $app->make(ArchiveExtractionService::class), + $app->make(MediaExtractionService::class), + $app->make(UsenetDownloadService::class), + $app->make(ReleaseFileManager::class), + $app->make(ConsoleOutputService::class) + ); + }); + + $this->app->singleton(ReleaseProcessor::class, function ($app) { + return new ReleaseProcessor( + $app->make(ProcessingConfiguration::class), + $app->make(NzbContentParser::class), + $app->make(ArchiveExtractionService::class), + $app->make(MediaExtractionService::class), + $app->make(UsenetDownloadService::class), + $app->make(ReleaseFileManager::class), + $app->make(ReleaseFilesArchiveFallback::class), + $app->make(TempWorkspaceService::class), + $app->make(ConsoleOutputService::class) + ); + }); + // Temp workspace service (might already be registered elsewhere) $this->app->singleton(TempWorkspaceService::class, function () { return new TempWorkspaceService; @@ -106,11 +133,7 @@ class AdditionalProcessingServiceProvider extends ServiceProvider $this->app->singleton(AdditionalProcessingOrchestrator::class, function ($app) { return new AdditionalProcessingOrchestrator( $app->make(ProcessingConfiguration::class), - $app->make(NzbContentParser::class), - $app->make(ArchiveExtractionService::class), - $app->make(MediaExtractionService::class), - $app->make(UsenetDownloadService::class), - $app->make(ReleaseFileManager::class), + $app->make(ReleaseProcessor::class), $app->make(TempWorkspaceService::class), $app->make(ConsoleOutputService::class) ); @@ -122,6 +145,10 @@ class AdditionalProcessingServiceProvider extends ServiceProvider */ public function boot(): void { - // + $this->app->terminating(function (): void { + if ($this->app->bound(AdditionalProcessingOrchestrator::class)) { + $this->app->make(AdditionalProcessingOrchestrator::class)->finish(); + } + }); } } diff --git a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php index 80e95e5a5..495aea384 100644 --- a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php +++ b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php @@ -5,22 +5,16 @@ declare(strict_types=1); namespace App\Services\AdditionalProcessing; use App\Models\Release; -use App\Models\ReleaseFile; -use App\Models\UsenetGroup; use App\Services\AdditionalProcessing\Config\ProcessingConfiguration; use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext; -use App\Services\Releases\ReleaseBrowseService; use App\Services\TempWorkspaceService; use Exception; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Log; /** * Main orchestrator for additional release post-processing. - * Coordinates all processing services to handle NZB parsing, archive extraction, - * media processing, and release updates. + * Coordinates release selection and delegates actual per-release work to ReleaseProcessor. */ class AdditionalProcessingOrchestrator { @@ -35,18 +29,9 @@ class AdditionalProcessingOrchestrator private string $mainTmpPath = ''; - /** - * @var array - */ - private array $triedCompressedMids = []; - public function __construct( private readonly ProcessingConfiguration $config, - private readonly NzbContentParser $nzbParser, - private readonly ArchiveExtractionService $archiveService, - private readonly MediaExtractionService $mediaService, - private readonly UsenetDownloadService $downloadService, - private readonly ReleaseFileManager $releaseManager, + private readonly ReleaseProcessor $processor, private readonly TempWorkspaceService $tempWorkspace, private readonly ConsoleOutputService $output ) {} @@ -58,6 +43,7 @@ class AdditionalProcessingOrchestrator */ public function start(string $groupID = '', string $guidChar = ''): void { + $this->finish(); $this->setupTempPath($guidChar, $groupID); $this->fetchReleases($groupID, $guidChar); @@ -73,6 +59,7 @@ class AdditionalProcessingOrchestrator public function processSingleGuid(string $guid): bool { try { + $this->finish(); $release = Release::where('guid', $guid)->first(); if ($release === null) { $this->output->warning('Release not found for GUID: '.$guid); @@ -100,7 +87,7 @@ class AdditionalProcessingOrchestrator /** * Set up the main temp path. */ - private function setupTempPath(string &$guidChar, string &$groupID): void + private function setupTempPath(string $guidChar, string $groupID): void { $this->mainTmpPath = $this->tempWorkspace->ensureMainTempPath( $this->config->tmpUnrarPath, @@ -115,46 +102,47 @@ class AdditionalProcessingOrchestrator */ private function fetchReleases(int|string $groupID, string $guidChar): void { - $sqlParts = []; - $bindings = []; - - $sqlParts[] = 'SELECT r.id AS id, r.id AS releases_id, r.guid, r.name, r.size, r.groups_id, r.nfostatus, r.fromname,'; - $sqlParts[] = 'r.completion, r.categories_id, r.searchname, r.predb_id, r.pp_timeout_count, c.disablepreview'; - $sqlParts[] = 'FROM releases r'; - $sqlParts[] = 'LEFT JOIN categories c ON c.id = r.categories_id'; - $sqlParts[] = 'WHERE r.passwordstatus = ?'; - $bindings[] = -1; - $sqlParts[] = 'AND r.nzbstatus = ?'; - $bindings[] = 1; - $sqlParts[] = 'AND r.haspreview = ?'; - $bindings[] = -1; - $sqlParts[] = 'AND c.disablepreview = ?'; - $bindings[] = 0; + $query = Release::query() + ->from('releases as r') + ->select([ + 'r.id', + 'r.guid', + 'r.name', + 'r.size', + 'r.groups_id', + 'r.nfostatus', + 'r.fromname', + 'r.completion', + 'r.categories_id', + 'r.searchname', + 'r.predb_id', + 'r.pp_timeout_count', + ]) + ->selectRaw('r.id as releases_id') + ->leftJoin('categories as c', 'c.id', '=', 'r.categories_id') + ->where('r.passwordstatus', -1) + ->where('r.nzbstatus', 1) + ->where('r.haspreview', -1) + ->where('c.disablepreview', 0); if ($this->config->maxSizeGB > 0) { - $sqlParts[] = 'AND r.size < ?'; - $bindings[] = $this->config->maxSizeGB * 1073741824; + $query->where('r.size', '<', $this->config->maxSizeGB * 1073741824); } if ($this->config->minSizeMB > 0) { - $sqlParts[] = 'AND r.size > ?'; - $bindings[] = $this->config->minSizeMB * 1048576; + $query->where('r.size', '>', $this->config->minSizeMB * 1048576); } - if (! empty($groupID)) { - $sqlParts[] = 'AND r.groups_id = ?'; - $bindings[] = $groupID; + if ($groupID !== '') { + $query->where('r.groups_id', $groupID); } - if (! empty($guidChar)) { - $sqlParts[] = 'AND r.leftguid = ?'; - $bindings[] = $guidChar; + if ($guidChar !== '') { + $query->where('r.leftguid', $guidChar); } - $sqlParts[] = 'ORDER BY r.passwordstatus ASC, r.postdate DESC'; - $limit = $this->config->queryLimit > 0 ? $this->config->queryLimit : 25; - $sqlParts[] = 'LIMIT '.$limit; - - $rawResults = DB::select(implode(' ', $sqlParts), $bindings); - $attributeArrays = array_map(static fn ($row) => (array) $row, $rawResults); - $this->releases = Release::hydrate($attributeArrays); + $this->releases = $query + ->orderBy('r.passwordstatus') + ->orderByDesc('r.postdate') + ->limit($this->config->queryLimit > 0 ? $this->config->queryLimit : 25) + ->get(); $this->totalReleases = $this->releases->count(); } @@ -166,917 +154,20 @@ class AdditionalProcessingOrchestrator private function processReleases(): void { foreach ($this->releases as $release) { - $context = new ReleaseProcessingContext($release); - - $this->output->echoReleaseStart($release->id, $release->size); - cli_set_process_title(PHP_BINARY.' '.__DIR__.'/AdditionalProcessingOrchestrator.php ReleaseID: '.$release->id); - - // Create temp folder for this release - try { - $context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($this->mainTmpPath, $release->guid); - } catch (\Throwable $e) { - $this->output->warning('Unable to create directory: '.$e->getMessage()); - - continue; - } - - // Parse NZB contents - $nzbResult = $this->nzbParser->parseNzb($release->guid); - if ($nzbResult['error'] !== null) { - $this->output->warning($nzbResult['error']); - $this->releaseManager->deleteRelease($release); - - continue; - } - $context->nzbContents = $nzbResult['contents']; - - // Check timeout after NZB parsing - if ($this->checkReleaseTimeout($context)) { - continue; - } - - // Initialize context - $this->initializeContext($context); - - // Extract message IDs from NZB - $messageIds = $this->nzbParser->extractMessageIDs( - $context->nzbContents, - $context->releaseGroupName, - $this->config->segmentsToDownload, - $this->config->processThumbnails, - $this->config->processJPGSample, - $this->config->processMediaInfo, - $this->config->processAudioInfo, - $this->config->audioFileRegex, - $this->config->videoFileRegex, - $this->config->supportFileRegex, - $this->config->ignoreBookRegex - ); - - $context->nzbHasCompressedFile = $messageIds['hasCompressedFile']; - $context->sampleMessageIDs = $messageIds['sampleMessageIDs']; - $context->jpgMessageIDs = $messageIds['jpgMessageIDs']; - $context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID']; - $context->audioInfoMessageIDs = $messageIds['audioInfoMessageID']; - $context->audioInfoExtension = $messageIds['audioInfoExtension']; - - // Check for book flood - $bookFlood = $messageIds['bookFileCount'] > 80 - && ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents); - - // Process message ID downloads - if ($this->config->processPasswords || $this->config->processThumbnails - || $this->config->processMediaInfo || $this->config->processAudioInfo - || $this->config->processVideo || $this->config->processJPGSample - ) { - $this->processMessageIDDownloads($context); - - // Check timeout after message ID downloads - if ($this->checkReleaseTimeout($context)) { - continue; - } - - // Process compressed files - if (! $bookFlood && $context->nzbHasCompressedFile) { - $this->processNZBCompressedFiles($context, false); - - // Check timeout after compressed file processing - if ($this->checkReleaseTimeout($context)) { - continue; - } - - if ($this->config->fetchLastFiles) { - $this->processNZBCompressedFiles($context, true); - - // Check timeout after last files processing - if ($this->checkReleaseTimeout($context)) { - continue; - } - } - - if (! $context->releaseHasPassword) { - $this->processExtractedFiles($context); - - // Check timeout after extracted file processing - if ($this->checkReleaseTimeout($context)) { - continue; - } - } - } - - // If still no JPG sample, try to fetch JPG from release_files entries - if (! $context->foundJPGSample && $this->config->processJPGSample) { - $this->processJpgFromReleaseFiles($context); - } - - // If still no NFO, try to fetch NFO from release_files entries - if ($context->releaseHasNoNFO) { - $this->processNfoFromReleaseFiles($context); - } - } - - // Finalize - $this->releaseManager->finalizeRelease($context, $this->config->processPasswords); - - // Cleanup - $this->tempWorkspace->clearDirectory($context->tmpPath, false); + $this->processor->process(new ReleaseProcessingContext($release), $this->mainTmpPath); } $this->output->endOutput(); } - /** - * Check if the current release has exceeded the processing timeout. - * If timed out, handles cleanup, logs the event, and marks the release accordingly. - * - * @return bool True if the release timed out and should be skipped (caller should `continue`) - */ - private function checkReleaseTimeout(ReleaseProcessingContext $context): bool - { - if (! $context->isTimedOut($this->config->releaseProcessingTimeout)) { - return false; - } - - $elapsedSeconds = $context->getElapsedSeconds(); - - // Handle the timeout: increment counter, skip or delete - $deleted = $this->releaseManager->handleReleaseTimeout( - $context->release, - $this->config->maxPpTimeoutCount - ); - - if ($deleted) { - $this->output->echoReleaseTimeoutDeleted( - $context->release->id, - (int) ($context->release->pp_timeout_count ?? 0) + 1 - ); - } else { - $this->output->echoReleaseTimeout($context->release->id, $elapsedSeconds); - } - - // Cleanup temp directory - if (! empty($context->tmpPath)) { - $this->tempWorkspace->clearDirectory($context->tmpPath, false); - } - - return true; - } - - /** - * Initialize the processing context. - */ - private function initializeContext(ReleaseProcessingContext $context): void - { - $context->initializeFromConfig( - $this->config->processVideo, - $this->config->processMediaInfo, - $this->config->processAudioInfo, - $this->config->processAudioSample, - $this->config->processJPGSample, - $this->config->processThumbnails - ); - - $context->passwordStatus = ReleaseBrowseService::PASSWD_NONE; - $context->releaseHasPassword = false; - $context->releaseGroupName = UsenetGroup::getNameByID($context->release->groups_id); - $context->releaseHasNoNFO = (int) $context->release->nfostatus !== 1; - $context->resetMessageIDs(); - $context->resetCounters(); - } - - /** - * Process message ID downloads (sample, media info, audio, JPG). - */ - private function processMessageIDDownloads(ReleaseProcessingContext $context): void - { - // Sample download - if ((! $context->foundSample || ! $context->foundVideo) && ! empty($context->sampleMessageIDs)) { - $result = $this->downloadService->downloadSample( - $context->sampleMessageIDs, - $context->releaseGroupName, - $context->release->id - ); - - if ($result['success'] && $this->downloadService->meetsMinimumSize($result['data'])) { - $this->output->echoSampleDownload(); - $fileLocation = $context->tmpPath.'sample_'.random_int(0, 99999).'.avi'; - File::put($fileLocation, $result['data']); - - if (! $context->foundSample) { - if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundSample = true; - $this->output->echoSampleCreated(); - } - } - if (! $context->foundVideo) { - if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundVideo = true; - $this->output->echoVideoCreated(); - } - } - } elseif (! $result['success']) { - $this->output->echoSampleFailure(); - } - } - - // Media info download - if ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) - && ! empty($context->mediaInfoMessageIDs) - ) { - $result = $this->downloadService->downloadMediaInfo( - $context->mediaInfoMessageIDs, - $context->releaseGroupName, - $context->release->id - ); - - if ($result['success'] && $this->downloadService->meetsMinimumSize($result['data'])) { - $this->output->echoMediaInfoDownload(); - $fileLocation = $context->tmpPath.'media.avi'; - File::put($fileLocation, $result['data']); - - if (! $context->foundMediaInfo) { - if ($this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { - $context->foundMediaInfo = true; - $this->output->echoMediaInfoAdded(); - } - } - if (! $context->foundSample) { - if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundSample = true; - $this->output->echoSampleCreated(); - } - } - if (! $context->foundVideo) { - if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundVideo = true; - $this->output->echoVideoCreated(); - } - } - } elseif (! $result['success']) { - $this->output->echoMediaInfoFailure(); - } - } - - // Audio info download - if ((! $context->foundAudioInfo || ! $context->foundAudioSample) - && ! empty($context->audioInfoMessageIDs) - ) { - $result = $this->downloadService->downloadAudio( - $context->audioInfoMessageIDs, - $context->releaseGroupName, - $context->release->id - ); - - if ($result['success']) { - $this->output->echoAudioDownload(); - $fileLocation = $context->tmpPath.'audio.'.$context->audioInfoExtension; - File::put($fileLocation, $result['data']); - - $audioResult = $this->mediaService->getAudioInfo( - $fileLocation, - $context->audioInfoExtension, - $context, - $context->tmpPath - ); - - if ($audioResult['audioInfo']) { - $this->output->echoAudioInfoAdded(); - } - if ($audioResult['audioSample']) { - $this->output->echoAudioSampleCreated(); - } - } elseif (! $result['success']) { - $this->output->echoAudioFailure(); - } - } - - // JPG download - if (! $context->foundJPGSample && ! empty($context->jpgMessageIDs)) { - $result = $this->downloadService->downloadJPG( - $context->jpgMessageIDs, - $context->releaseGroupName, - $context->release->id - ); - - if ($result['success']) { - $this->output->echoJpgDownload(); - $fileLocation = $context->tmpPath.'samplepicture.jpg'; - File::put($fileLocation, $result['data']); - - if ($this->mediaService->isJpegData($fileLocation)) { - if ($this->mediaService->getJPGSample($fileLocation, $context->release->guid)) { - $context->foundJPGSample = true; - $this->output->echoJpgSaved(); - } - } - File::delete($fileLocation); - } elseif (! $result['success']) { - $this->output->echoJpgFailure(); - } - } - } - - /** - * Process compressed files in the NZB. - */ - private function processNZBCompressedFiles(ReleaseProcessingContext $context, bool $reverse): void - { - if ($context->groupUnavailable) { - return; - } - - $nzbContents = $context->nzbContents; - if ($reverse) { - krsort($nzbContents); - } else { - $this->triedCompressedMids = []; - } - - $failed = $downloaded = 0; - - foreach ($nzbContents as $nzbFile) { - if ($downloaded >= $this->config->maximumRarSegments) { - break; - } - if ($failed >= $this->config->maximumRarPasswordChecks) { - break; - } - if ($context->releaseHasPassword || $context->groupUnavailable) { // @phpstan-ignore booleanOr.rightAlwaysFalse - break; - } - - $title = $nzbFile['title'] ?? ''; - if (! preg_match( - '/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?)(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', - $title - )) { - continue; - } - - // Get message IDs - $segments = $nzbFile['segments'] ?? []; - $segCount = count($segments) - 1; - $messageIDs = []; - - foreach (range(0, $this->config->maximumRarSegments - 1) as $i) { - if ($i > $segCount) { - break; - } - $segment = (string) $segments[$i]; - if (! $reverse) { - $this->triedCompressedMids[] = $segment; - } elseif (in_array($segment, $this->triedCompressedMids, false)) { - continue 2; - } - $messageIDs[] = $segment; - } - - if (empty($messageIDs)) { - continue; - } - - // Download - $result = $this->downloadService->downloadCompressedFile( - $messageIDs, // @phpstan-ignore argument.type - $context->releaseGroupName, - $context->release->id, - $title - ); - - if ($result['groupUnavailable']) { - $context->groupUnavailable = true; - $this->output->echoGroupUnavailable(); - break; - } - - if ($result['success']) { - $this->output->echoCompressedDownload(); - $downloaded++; - - $processed = $this->processCompressedData($result['data'], $context, $reverse); - if ($processed || $context->releaseHasPassword) { // @phpstan-ignore booleanOr.rightAlwaysFalse - break; - } - } else { - $failed++; - $this->output->echoCompressedFailure($failed); - } - } - } - - /** - * Process compressed data. - */ - private function processCompressedData( - string $compressedData, - ReleaseProcessingContext $context, - bool $reverse - ): bool { - $result = $this->archiveService->processCompressedData( - $compressedData, - $context, - $context->tmpPath - ); - - if ($result['hasPassword']) { - $context->releaseHasPassword = true; - $context->passwordStatus = $result['passwordStatus']; - - return false; - } - - // Handle standalone video - if (isset($result['standaloneVideoType'])) { - $this->output->echoInlineVideo(); - $ext = $result['standaloneVideoType']; - $fileLocation = $context->tmpPath.'inline_video_'.uniqid('', true).'.'.$ext; - File::put($fileLocation, $result['standaloneVideoData']); - - if (! $context->foundMediaInfo) { - if ($this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { - $context->foundMediaInfo = true; - $this->output->echoMediaInfoAdded(); - } - } - if (! $context->foundSample) { - if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundSample = true; - $this->output->echoSampleCreated(); - } - } - if (! $context->foundVideo) { - if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { - $context->foundVideo = true; - $this->output->echoVideoCreated(); - } - } - - return $context->foundMediaInfo || $context->foundSample || $context->foundVideo; - } - - if (! $result['success']) { - return false; - } - - // Echo archive marker - if (! empty($result['archiveMarker'])) { - $this->output->echoArchiveMarker($result['archiveMarker']); - } - - // Handle reverse processing for release name extraction - if ($reverse && ! empty($result['dataSummary'])) { - $this->releaseManager->processReleaseNameFromRar($result['dataSummary'], $context); - } - - // Process file list - foreach ($result['files'] as $file) { - if ($context->releaseHasPassword) { - break; - } - - if ($this->releaseManager->addFileInfo($file, $context, $this->config->supportFileRegex)) { - $this->output->echoFileInfoAdded(); - } - } - - if ($context->addedFileInfo > 0) { - $this->releaseManager->updateSearchIndex($context->release->id); - } - - // Try to extract and process JPG files from the archive file list for sample/preview - if (! $context->foundJPGSample && $this->config->processJPGSample) { - $this->processJpgFromArchiveFileList($compressedData, $result['files'], $context); - } - - return $context->totalFileInfo > 0; - } - - /** - * Find and process JPG/PNG files from archive file list to create sample/preview. - * - * @param array $files - */ - private function processJpgFromArchiveFileList( - string $compressedData, - array $files, - ReleaseProcessingContext $context - ): void { - // Find image files (JPG/PNG) in the archive file list - $imageFiles = []; - foreach ($files as $file) { - $name = $file['name'] ?? ''; - if (preg_match('/\.(jpe?g|png)$/i', $name)) { - $imageFiles[] = $file; - } - } - - if (empty($imageFiles)) { - return; - } - - // Sort by size (prefer larger images, likely better quality) - limit to first 3 - usort($imageFiles, fn ($a, $b) => ($b['size'] ?? 0) <=> ($a['size'] ?? 0)); - $imageFiles = array_slice($imageFiles, 0, 3); - - foreach ($imageFiles as $imageFile) { - $imageFilename = $imageFile['name']; - - // Try to extract this specific image from the archive - $imageData = $this->archiveService->extractSpecificFile( - $compressedData, - $imageFilename, - $context->tmpPath - ); - - if ($imageData === null || empty($imageData)) { - continue; - } - - // Save to temp file and validate it's actually a valid image - $ext = strtolower(pathinfo($imageFilename, PATHINFO_EXTENSION)); - $tempImagePath = $context->tmpPath.'extracted_'.uniqid('', true).'.'.$ext; - File::put($tempImagePath, $imageData); - - if ($this->mediaService->isValidImage($tempImagePath)) { - if ($this->mediaService->getJPGSample($tempImagePath, $context->release->guid)) { - $context->foundJPGSample = true; - $this->output->echoJpgSaved(); - File::delete($tempImagePath); - break; - } - } - - File::delete($tempImagePath); - } - } - - /** - * Process JPG/PNG files from existing release_files entries. - * This fetches the archive from NZB and extracts image files that were previously listed. - */ - private function processJpgFromReleaseFiles(ReleaseProcessingContext $context): void - { - // Get JPG/PNG files from release_files table for this release - $imageFiles = ReleaseFile::where('releases_id', $context->release->id) - ->where(function ($query) { - $query->where('name', 'like', '%.jpg') - ->orWhere('name', 'like', '%.jpeg') - ->orWhere('name', 'like', '%.png'); - }) - ->orderByDesc('size') - ->limit(3) - ->get(); - - if ($imageFiles->isEmpty()) { - return; - } - - // We need to download a compressed file from the NZB to extract the JPG - if (empty($context->nzbContents)) { - return; - } - - // Find compressed files in NZB - foreach ($context->nzbContents as $nzbFile) { - if ($context->foundJPGSample) { - break; - } - - $title = $nzbFile['title'] ?? ''; - if (! preg_match( - '/(\\.(part0*1|rar|zip))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$/i', - $title - )) { - continue; - } - - // Get message IDs for first few segments - $segments = $nzbFile['segments'] ?? []; - if (empty($segments)) { - continue; - } - - $messageIDs = []; - $segCount = min(count($segments), $this->config->maximumRarSegments); - for ($i = 0; $i < $segCount; $i++) { - $messageIDs[] = (string) $segments[$i]; - } - - if (empty($messageIDs)) { - continue; - } - - // Download the compressed file - $result = $this->downloadService->downloadCompressedFile( - $messageIDs, // @phpstan-ignore argument.type - $context->releaseGroupName, - $context->release->id, - $title - ); - - if (! $result['success'] || empty($result['data'])) { - continue; - } - - // Try to extract each image file from the archive - foreach ($imageFiles as $imageFile) { - $imageData = $this->archiveService->extractSpecificFile( - $result['data'], - $imageFile->name, - $context->tmpPath - ); - - if ($imageData === null || empty($imageData)) { - continue; - } - - // Save to temp file and validate - $ext = strtolower(pathinfo($imageFile->name, PATHINFO_EXTENSION)); - $tempImagePath = $context->tmpPath.'release_file_'.uniqid('', true).'.'.$ext; - File::put($tempImagePath, $imageData); - - if ($this->mediaService->isValidImage($tempImagePath)) { - if ($this->mediaService->getJPGSample($tempImagePath, $context->release->guid)) { - $context->foundJPGSample = true; - $this->output->echoJpgSaved(); - File::delete($tempImagePath); - break 2; // Exit both loops - } - } - - File::delete($tempImagePath); - } - } - } - - /** - * Process NFO files from existing release_files entries. - * When archive extraction failed to produce NFO on disk, this falls back to - * checking the release_files table and re-downloading the archive to extract by name. - */ - private function processNfoFromReleaseFiles(ReleaseProcessingContext $context): void - { - $nfoFiles = ReleaseFile::where('releases_id', $context->release->id) - ->nfoFiles() - ->limit(3) - ->get(); - - if ($nfoFiles->isEmpty()) { - return; - } - - if (empty($context->nzbContents)) { - return; - } - - foreach ($context->nzbContents as $nzbFile) { - if (! $context->releaseHasNoNFO) { - break; - } - - $title = $nzbFile['title'] ?? ''; - if (! preg_match( - '/(\\.(part0*1|rar|zip))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$/i', - $title - )) { - continue; - } - - $segments = $nzbFile['segments'] ?? []; - if (empty($segments)) { - continue; - } - - $messageIDs = []; - $segCount = min(count($segments), $this->config->maximumRarSegments); - for ($i = 0; $i < $segCount; $i++) { - $messageIDs[] = (string) $segments[$i]; - } - - if (empty($messageIDs)) { - continue; - } - - $result = $this->downloadService->downloadCompressedFile( - $messageIDs, // @phpstan-ignore argument.type - $context->releaseGroupName, - $context->release->id, - $title - ); - - if (! $result['success'] || empty($result['data'])) { - continue; - } - - foreach ($nfoFiles as $nfoFile) { - $nfoData = $this->archiveService->extractSpecificFile( - $result['data'], - $nfoFile->name, - $context->tmpPath - ); - - if ($nfoData === null || empty($nfoData)) { - continue; - } - - $tempNfoPath = $context->tmpPath.'release_file_'.uniqid('', true).'.nfo'; - File::put($tempNfoPath, $nfoData); - - if ($this->releaseManager->processNfoFile($tempNfoPath, $context, $this->downloadService->getNNTP())) { - $this->output->echoNfoFound(); - File::delete($tempNfoPath); - break 2; - } - - File::delete($tempNfoPath); - } - } - } - - /** - * Process extracted files from archives. - */ - private function processExtractedFiles(ReleaseProcessingContext $context): void - { - $nestedLevels = 0; - - // Handle nested archives - while ($nestedLevels < $this->config->maxNestedLevels) { - if ($context->compressedFilesChecked >= self::MAX_COMPRESSED_FILES_TO_CHECK) { - break; - } - - $foundCompressed = false; - $pattern = '/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1)($|[^a-z0-9])/i'; - - try { - $files = $this->tempWorkspace->listFiles($context->tmpPath, $pattern); - } catch (\Throwable) { - break; - } - - foreach ($files as $file) { - $filePath = is_array($file) ? $file[0] : $file->getPathname(); - if (File::isFile($filePath)) { - $rarData = @File::get($filePath); - if (! empty($rarData)) { - $this->processCompressedData($rarData, $context, false); - $foundCompressed = true; - } - File::delete($filePath); - } - } - - if (! $foundCompressed) { - break; - } - $nestedLevels++; - } - - // Process remaining files - try { - $files = $this->tempWorkspace->listFiles($context->tmpPath); - } catch (\Throwable) { - return; - } - - foreach ($files as $file) { - $filePath = is_object($file) ? $file->getPathname() : $file; - - if (preg_match('/[\/\\\\]\.{1,2}$/', $filePath)) { - continue; - } - - if (! File::isFile($filePath)) { - continue; - } - - // PAR2 files - if (! $context->foundPAR2Info && preg_match('/\.par2$/i', $filePath)) { - $this->releaseManager->processPar2File( - $filePath, - $context, - $this->archiveService->getPar2Info() - ); - - continue; - } - - // NFO files - enhanced detection with multiple patterns - if ($context->releaseHasNoNFO) { - // Standard NFO extensions - if (preg_match('/(\.(nfo|inf|ofn|diz)|info\.txt)$/i', $filePath)) { - if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { - $this->output->echoNfoFound(); - } - - continue; - } - // Alternative NFO filenames (file_id.diz, readme.txt, etc.) - elseif ($this->releaseManager->isNfoFilename($filePath)) { - if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { - $this->output->echoNfoFound(); - } - - continue; - } - } - - // Audio files - if ((! $context->foundAudioInfo || ! $context->foundAudioSample) - && preg_match('/(.*)'.$this->config->audioFileRegex.'$/i', $filePath, $fileType) - ) { - $audioPath = $context->tmpPath.'audiofile.'.$fileType[2]; - File::move($filePath, $audioPath); - $audioResult = $this->mediaService->getAudioInfo( - $audioPath, - $fileType[2], - $context, - $context->tmpPath - ); - if ($audioResult['audioInfo']) { - $this->output->echoAudioInfoAdded(); - } - if ($audioResult['audioSample']) { - $this->output->echoAudioSampleCreated(); - } - File::delete($audioPath); - - continue; - } - - // JPG/PNG image files - if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png)$/i', $filePath)) { - if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { - $context->foundJPGSample = true; - $this->output->echoJpgSaved(); - } - File::delete($filePath); - - continue; - } - - // Video files - if ((! $context->foundSample || ! $context->foundVideo || ! $context->foundMediaInfo) - && preg_match('/(.*)'.$this->config->videoFileRegex.'$/i', $filePath) - ) { - $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); - - continue; - } - - // Check file magic - $output = fileInfo($filePath); - if (empty($output)) { - continue; - } - - if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG/i', $output)) { - if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { - $context->foundJPGSample = true; - $this->output->echoJpgSaved(); - } - File::delete($filePath); - } elseif ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) - && preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output) - ) { - $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); - } elseif ((! $context->foundAudioSample || ! $context->foundAudioInfo) - && preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $audioType) - ) { - $ext = match ($audioType[0]) { - 'FLAC' => 'FLAC', - 'layer III' => 'MP3', - 'Vorbis audio' => 'OGG', - default => 'audio', - }; - $audioPath = $context->tmpPath.'audiofile.'.$ext; - File::move($filePath, $audioPath); - $audioResult = $this->mediaService->getAudioInfo($audioPath, $ext, $context, $context->tmpPath); - if ($audioResult['audioInfo']) { - $this->output->echoAudioInfoAdded(); - } - if ($audioResult['audioSample']) { - $this->output->echoAudioSampleCreated(); - } - File::delete($audioPath); - } elseif (! $context->foundPAR2Info && stripos($output, 'Parity') === 0) { - $this->releaseManager->processPar2File( - $filePath, - $context, - $this->archiveService->getPar2Info() - ); - } - } - } - - /** - * Clear the main temp path. - */ - public function __destruct() + public function finish(): void { if ($this->mainTmpPath !== '') { $this->tempWorkspace->clearDirectory($this->mainTmpPath, true); + $this->mainTmpPath = ''; } + + $this->releases = collect(); + $this->totalReleases = 0; } } diff --git a/app/Services/AdditionalProcessing/ArchiveExtractionService.php b/app/Services/AdditionalProcessing/ArchiveExtractionService.php index 47bd5ec62..6d46cc128 100644 --- a/app/Services/AdditionalProcessing/ArchiveExtractionService.php +++ b/app/Services/AdditionalProcessing/ArchiveExtractionService.php @@ -23,10 +23,12 @@ class ArchiveExtractionService private Par2Info $par2Info; public function __construct( - private readonly ProcessingConfiguration $config + private readonly ProcessingConfiguration $config, + ?ArchiveInfo $archiveInfo = null, + ?Par2Info $par2Info = null ) { - $this->archiveInfo = new ArchiveInfo; - $this->par2Info = new Par2Info; + $this->archiveInfo = $archiveInfo ?? new ArchiveInfo; + $this->par2Info = $par2Info ?? new Par2Info; // Configure external clients for ArchiveInfo if ($this->config->unrarPath) { @@ -341,7 +343,14 @@ class ArchiveExtractionService // Try using unrar for RAR files if ($this->config->unrarPath) { - $extracted = $this->extractFileViaUnrar($compressedData, $filename, $tmpPath); + $extracted = $this->extractFileViaExternalTool( + $compressedData, + $filename, + $tmpPath, + 'rar', + fn (string $archiveFile, string $extractDir): string => $this->config->getKillString(). + $this->config->unrarPath.'" e -y -c- -inul -p- "'.$archiveFile.'" "'.$filename.'" "'.$extractDir.'"' + ); if ($extracted !== null) { return $extracted; } @@ -349,7 +358,14 @@ class ArchiveExtractionService // Try using unzip for ZIP files if ($this->config->unzipPath) { - $extracted = $this->extractFileViaUnzip($compressedData, $filename, $tmpPath); + $extracted = $this->extractFileViaExternalTool( + $compressedData, + $filename, + $tmpPath, + 'zip', + fn (string $archiveFile, string $extractDir): string => $this->config->unzipPath. + ' -j "'.$archiveFile.'" "'.$filename.'" -d "'.$extractDir.'"' + ); if ($extracted !== null) { return $extracted; } @@ -361,20 +377,23 @@ class ArchiveExtractionService /** * Extract a specific file using unrar. */ - private function extractFileViaUnrar(string $compressedData, string $filename, string $tmpPath): ?string - { + private function extractFileViaExternalTool( + string $compressedData, + string $filename, + string $tmpPath, + string $archiveExtension, + callable $commandBuilder + ): ?string { try { $extractDir = $tmpPath.'extract_'.uniqid('', true).'/'; if (! File::isDirectory($extractDir)) { File::makeDirectory($extractDir, 0777, true, true); } - $archiveFile = $tmpPath.'archive_'.uniqid('', true).'.rar'; + $archiveFile = $tmpPath.'archive_'.uniqid('', true).'.'.$archiveExtension; File::put($archiveFile, $compressedData); - // Extract specific file using unrar - $killString = $this->config->getKillString(); - runCmd($killString.$this->config->unrarPath.'" e -y -c- -inul -p- "'.$archiveFile.'" "'.$filename.'" "'.$extractDir.'"'); + runCmd($commandBuilder($archiveFile, $extractDir)); File::delete($archiveFile); @@ -401,56 +420,7 @@ class ArchiveExtractionService File::deleteDirectory($extractDir); } catch (\Throwable $e) { if ($this->config->debugMode) { - Log::debug('Unrar extraction failed: '.$e->getMessage()); - } - } - - return null; - } - - /** - * Extract a specific file using unzip. - */ - private function extractFileViaUnzip(string $compressedData, string $filename, string $tmpPath): ?string - { - try { - $extractDir = $tmpPath.'extract_'.uniqid('', true).'/'; - if (! File::isDirectory($extractDir)) { - File::makeDirectory($extractDir, 0777, true, true); - } - - $archiveFile = $tmpPath.'archive_'.uniqid('', true).'.zip'; - File::put($archiveFile, $compressedData); - - // Extract specific file using unzip - runCmd($this->config->unzipPath.' -j "'.$archiveFile.'" "'.$filename.'" -d "'.$extractDir.'"'); - - File::delete($archiveFile); - - // Look for extracted file - $extractedPath = $extractDir.basename($filename); - if (File::isFile($extractedPath)) { - $content = File::get($extractedPath); - File::deleteDirectory($extractDir); - - return $content; - } - - // Try to find it with glob - $files = File::allFiles($extractDir); - foreach ($files as $file) { - if (strtolower($file->getFilename()) === strtolower(basename($filename))) { - $content = File::get($file->getPathname()); - File::deleteDirectory($extractDir); - - return $content; - } - } - - File::deleteDirectory($extractDir); - } catch (\Throwable $e) { - if ($this->config->debugMode) { - Log::debug('Unzip extraction failed: '.$e->getMessage()); + Log::debug('External extraction failed: '.$e->getMessage()); } } diff --git a/app/Services/AdditionalProcessing/ConsoleOutputService.php b/app/Services/AdditionalProcessing/ConsoleOutputService.php index d30593014..9716005c0 100644 --- a/app/Services/AdditionalProcessing/ConsoleOutputService.php +++ b/app/Services/AdditionalProcessing/ConsoleOutputService.php @@ -56,6 +56,13 @@ class ConsoleOutputService $this->echo(PHP_EOL.'['.$releaseId.']['.human_filesize($size, 1).']', 'primaryOver'); } + public function setProcessTitle(int $releaseId): void + { + if (function_exists('cli_set_process_title')) { + @cli_set_process_title(PHP_BINARY.' '.__DIR__.'/AdditionalProcessingOrchestrator.php ReleaseID: '.$releaseId); + } + } + public function echoCompressedDownload(): void { $this->echo('(cB)', 'primaryOver'); diff --git a/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php b/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php index f16b4dc2e..06524d7e3 100644 --- a/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php +++ b/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php @@ -47,33 +47,33 @@ class ReleaseProcessingContext public bool $nzbHasCompressedFile = false; /** - * @var array + * @var list> */ public array $nzbContents = []; // Message IDs for downloading /** - * @var array + * @var list */ public array $sampleMessageIDs = []; /** - * @var array + * @var list */ public array $jpgMessageIDs = []; /** - * @var array + * @var list|string */ public string|array $mediaInfoMessageIDs = []; /** - * @var array + * @var list|string */ public string|array $audioInfoMessageIDs = []; /** - * @var array + * @var list */ public array $rarFileMessageIDs = []; @@ -155,6 +155,23 @@ class ReleaseProcessingContext $this->resetCounters(); } + /** + * Mark a processing artifact as found. + */ + public function markFound(string $type): void + { + match ($type) { + 'video' => $this->foundVideo = true, + 'mediaInfo' => $this->foundMediaInfo = true, + 'audioInfo' => $this->foundAudioInfo = true, + 'audioSample' => $this->foundAudioSample = true, + 'jpgSample' => $this->foundJPGSample = true, + 'sample' => $this->foundSample = true, + 'par2Info' => $this->foundPAR2Info = true, + default => null, + }; + } + /** * Check if more media processing is needed. */ diff --git a/app/Services/AdditionalProcessing/Enums/DownloadKind.php b/app/Services/AdditionalProcessing/Enums/DownloadKind.php new file mode 100644 index 000000000..8ca5d436f --- /dev/null +++ b/app/Services/AdditionalProcessing/Enums/DownloadKind.php @@ -0,0 +1,14 @@ + false, @@ -186,7 +179,7 @@ class NzbContentParser $segments = $file['segments'] ?? []; // Skip support/nfo files - if (preg_match('/(?:'.$supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $title)) { + if (preg_match('/(?:'.$config->supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $title)) { continue; } @@ -199,39 +192,39 @@ class NzbContentParser } // Look for a video sample (not an image) - if ($processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments) + if ($config->processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments) && stripos($title, 'sample') !== false && ! preg_match('/\.jpe?g$/i', $title) ) { - $result['sampleMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload); + $result['sampleMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload); } // Look for a JPG picture (not a CD cover) - if ($processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments) + if ($config->processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments) && ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName) && preg_match('/\.jpe?g[. ")\]]/i', $title) ) { - $result['jpgMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload); + $result['jpgMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload); } // Look for a video file for MediaInfo (sample video) - if ($processMediaInfo && empty($result['mediaInfoMessageID']) && ! empty($segments[0]) + if ($config->processMediaInfo && empty($result['mediaInfoMessageID']) && ! empty($segments[0]) && stripos($title, 'sample') !== false - && preg_match('/'.$videoFileRegex.'[. ")\]]/i', $title) + && preg_match('/'.$config->videoFileRegex.'[. ")\]]/i', $title) ) { $result['mediaInfoMessageID'] = (string) $segments[0]; } // Look for an audio file - if ($processAudioInfo && empty($result['audioInfoMessageID']) && ! empty($segments) - && preg_match('/'.$audioFileRegex.'[. ")\]]/i', $title, $type) + if ($config->processAudioInfo && empty($result['audioInfoMessageID']) && ! empty($segments) + && preg_match('/'.$config->audioFileRegex.'[. ")\]]/i', $title, $type) ) { $result['audioInfoExtension'] = $type[1]; $result['audioInfoMessageID'] = (string) $segments[0]; } // Count book files - if (preg_match($ignoreBookRegex, $title)) { + if (preg_match($config->ignoreBookRegex, $title)) { $result['bookFileCount']++; } } catch (\ErrorException $e) { @@ -264,8 +257,6 @@ class NzbContentParser /** * Get the NZB path for a GUID. - * - * @return list */ public function getNzbPath(string $guid): string|false { diff --git a/app/Services/AdditionalProcessing/ReleaseFileManager.php b/app/Services/AdditionalProcessing/ReleaseFileManager.php index 045f05634..bfea1d56b 100644 --- a/app/Services/AdditionalProcessing/ReleaseFileManager.php +++ b/app/Services/AdditionalProcessing/ReleaseFileManager.php @@ -175,7 +175,7 @@ class ReleaseFileManager // Get file count $releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0; - $passwordStatus = max([$context->passwordStatus]); + $passwordStatus = $context->passwordStatus; // Set to no password if processing is off if (! $processPasswords) { @@ -514,8 +514,8 @@ class ReleaseFileManager if ($extractedName !== null) { $preCheck = Predb::whereTitle($extractedName)->first(); - $context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0; - $candidate = $preCheck->title ?? $extractedName; + $context->release->preid = $preCheck?->id ?? 0; + $candidate = $preCheck?->title ?? $extractedName; $candidate = $this->normalizeCandidateTitle($candidate); if ($this->isPlausibleReleaseTitle($candidate)) { @@ -540,8 +540,8 @@ class ReleaseFileManager if ($extractedName !== null) { $preCheck = Predb::whereTitle($extractedName)->first(); - $context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0; - $candidate = $preCheck->title ?? $extractedName; + $context->release->preid = $preCheck?->id ?? 0; + $candidate = $preCheck?->title ?? $extractedName; $candidate = $this->normalizeCandidateTitle($candidate); if ($this->isPlausibleReleaseTitle($candidate)) { diff --git a/app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php b/app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php new file mode 100644 index 000000000..9b3523c46 --- /dev/null +++ b/app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php @@ -0,0 +1,283 @@ +> $files + */ + public function processJpgFromArchiveFileList( + string $compressedData, + array $files, + ReleaseProcessingContext $context + ): void { + $imageFiles = array_values(array_filter( + $files, + static fn (array $file): bool => preg_match('/\.(jpe?g|png)$/i', (string) ($file['name'] ?? '')) === 1 + )); + + if ($imageFiles === []) { + return; + } + + usort($imageFiles, static fn (array $a, array $b): int => ((int) ($b['size'] ?? 0)) <=> ((int) ($a['size'] ?? 0))); + + $this->processExtractedCandidatesFromData( + $compressedData, + array_slice($imageFiles, 0, 3), + $context, + 'extracted_', + function (string $tempPath, ReleaseProcessingContext $context): bool { + if (! $this->mediaService->isValidImage($tempPath)) { + return false; + } + + if (! $this->mediaService->getJPGSample($tempPath, $context->release->guid)) { + return false; + } + + $context->markFound('jpgSample'); + $this->output->echoJpgSaved(); + + return true; + } + ); + } + + /** + * Re-download archives and extract stored JPG/PNG candidates. + */ + public function processJpgFromReleaseFiles(ReleaseProcessingContext $context): void + { + $imageFiles = ReleaseFile::query() + ->where('releases_id', $context->release->id) + ->where(function ($query) { + $query->where('name', 'like', '%.jpg') + ->orWhere('name', 'like', '%.jpeg') + ->orWhere('name', 'like', '%.png'); + }) + ->orderByDesc('size') + ->limit(3) + ->get(); + + if ($imageFiles->isEmpty()) { + return; + } + + $this->processStoredReleaseFiles( + $context, + $imageFiles, + 'release_file_', + static fn (ReleaseProcessingContext $context): bool => $context->foundJPGSample, + function (string $tempPath, ReleaseProcessingContext $context): bool { + if (! $this->mediaService->isValidImage($tempPath)) { + return false; + } + + if (! $this->mediaService->getJPGSample($tempPath, $context->release->guid)) { + return false; + } + + $context->markFound('jpgSample'); + $this->output->echoJpgSaved(); + + return true; + } + ); + } + + /** + * Re-download archives and extract stored NFO-like candidates. + */ + public function processNfoFromReleaseFiles(ReleaseProcessingContext $context): void + { + $nfoFiles = ReleaseFile::query() + ->where('releases_id', $context->release->id) + ->nfoFiles() + ->limit(3) + ->get(); + + if ($nfoFiles->isEmpty()) { + return; + } + + $nntp = $this->downloadService->getNNTP(); + + $this->processStoredReleaseFiles( + $context, + $nfoFiles, + 'release_file_', + static fn (ReleaseProcessingContext $context): bool => ! $context->releaseHasNoNFO, + function (string $tempPath, ReleaseProcessingContext $context) use ($nntp): bool { + if (! $this->releaseManager->processNfoFile($tempPath, $context, $nntp)) { + return false; + } + + $this->output->echoNfoFound(); + + return true; + } + ); + } + + /** + * @param EloquentCollection $candidates + * @param callable(ReleaseProcessingContext): bool $shouldStop + * @param callable(string, ReleaseProcessingContext): bool $handler + */ + private function processStoredReleaseFiles( + ReleaseProcessingContext $context, + EloquentCollection $candidates, + string $tempPrefix, + callable $shouldStop, + callable $handler + ): void { + if ($candidates->isEmpty() || $context->nzbContents === []) { + return; + } + + foreach ($context->nzbContents as $nzbFile) { + if ($shouldStop($context)) { + break; + } + + $title = (string) ($nzbFile['title'] ?? ''); + if (preg_match( + '/(\.(part0*1|rar|zip))(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i', + $title + ) !== 1) { + continue; + } + + $messageIDs = $this->extractInitialSegments($nzbFile['segments'] ?? []); + if ($messageIDs === []) { + continue; + } + + $result = $this->downloadService->download( + DownloadKind::Compressed, + $messageIDs, + $context->releaseGroupName, + $context->release->id, + $title + ); + + if ($result['groupUnavailable']) { + $context->groupUnavailable = true; + $this->output->echoGroupUnavailable(); + + return; + } + + if (! $result['success'] || empty($result['data'])) { + continue; + } + + foreach ($candidates as $candidate) { + $fileData = $this->archiveService->extractSpecificFile( + $result['data'], + $candidate->name, + $context->tmpPath + ); + + if ($fileData === null || $fileData === '') { + continue; + } + + if ($this->withExtractedTempFile($candidate->name, $fileData, $context, $tempPrefix, $handler)) { + return; + } + } + } + } + + /** + * @param array> $candidates + * @param callable(string, ReleaseProcessingContext): bool $handler + */ + private function processExtractedCandidatesFromData( + string $compressedData, + array $candidates, + ReleaseProcessingContext $context, + string $tempPrefix, + callable $handler + ): void { + foreach ($candidates as $candidate) { + $filename = (string) ($candidate['name'] ?? ''); + if ($filename === '') { + continue; + } + + $fileData = $this->archiveService->extractSpecificFile($compressedData, $filename, $context->tmpPath); + if ($fileData === null || $fileData === '') { + continue; + } + + if ($this->withExtractedTempFile($filename, $fileData, $context, $tempPrefix, $handler)) { + return; + } + } + } + + /** + * @param callable(string, ReleaseProcessingContext): bool $handler + */ + private function withExtractedTempFile( + string $filename, + string $fileData, + ReleaseProcessingContext $context, + string $tempPrefix, + callable $handler + ): bool { + $extension = strtolower(pathinfo($filename, PATHINFO_EXTENSION)); + $suffix = $extension === '' ? '' : '.'.$extension; + $tempPath = $context->tmpPath.$tempPrefix.uniqid('', true).$suffix; + File::put($tempPath, $fileData); + + try { + return $handler($tempPath, $context); + } finally { + File::delete($tempPath); + } + } + + /** + * @param array $segments + * @return list + */ + private function extractInitialSegments(array $segments): array + { + $messageIDs = []; + $segmentCount = min(count($segments), $this->config->maximumRarSegments); + + for ($index = 0; $index < $segmentCount; $index++) { + $messageIDs[] = (string) $segments[$index]; + } + + return $messageIDs; + } +} diff --git a/app/Services/AdditionalProcessing/ReleaseProcessor.php b/app/Services/AdditionalProcessing/ReleaseProcessor.php new file mode 100644 index 000000000..c1579b285 --- /dev/null +++ b/app/Services/AdditionalProcessing/ReleaseProcessor.php @@ -0,0 +1,623 @@ +release; + + $this->output->echoReleaseStart($release->id, $release->size); + $this->output->setProcessTitle((int) $release->id); + + try { + $context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid); + } catch (\Throwable $e) { + $this->output->warning('Unable to create directory: '.$e->getMessage()); + + return; + } + + try { + $nzbResult = $this->nzbParser->parseNzb($release->guid); + if ($nzbResult['error'] !== null) { + $this->output->warning($nzbResult['error']); + $this->releaseManager->deleteRelease($release); + + return; + } + + $context->nzbContents = array_values($nzbResult['contents']); + if ($this->checkReleaseTimeout($context)) { + return; + } + + $this->initializeContext($context); + $messageIds = $this->nzbParser->extractMessageIDs($context->nzbContents, $context->releaseGroupName, $this->config); + + $context->nzbHasCompressedFile = $messageIds['hasCompressedFile']; + $context->sampleMessageIDs = $messageIds['sampleMessageIDs']; + $context->jpgMessageIDs = $messageIds['jpgMessageIDs']; + $context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID']; + $context->audioInfoMessageIDs = $messageIds['audioInfoMessageID']; + $context->audioInfoExtension = $messageIds['audioInfoExtension']; + + $bookFlood = $messageIds['bookFileCount'] > 80 + && ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents); + + if ($this->shouldProcessDownloads()) { + $this->processMessageIdDownloads($context); + if ($this->checkReleaseTimeout($context)) { + return; + } + + if (! $bookFlood && $context->nzbHasCompressedFile) { + $triedCompressedMids = []; + $this->processNzbCompressedFiles($context, false, $triedCompressedMids); + if ($this->checkReleaseTimeout($context)) { + return; + } + + if ($this->config->fetchLastFiles) { + $this->processNzbCompressedFiles($context, true, $triedCompressedMids); + if ($this->checkReleaseTimeout($context)) { + return; + } + } + + if (! $context->releaseHasPassword) { + $this->processExtractedFiles($context); + if ($this->checkReleaseTimeout($context)) { + return; + } + } + } + + if (! $context->foundJPGSample && $this->config->processJPGSample) { + $this->archiveFallback->processJpgFromReleaseFiles($context); + } + + if ($context->releaseHasNoNFO) { + $this->archiveFallback->processNfoFromReleaseFiles($context); + } + } + + $this->releaseManager->finalizeRelease($context, $this->config->processPasswords); + } finally { + if ($context->tmpPath !== '') { + $this->tempWorkspace->clearDirectory($context->tmpPath, false); + } + } + } + + private function shouldProcessDownloads(): bool + { + return $this->config->processPasswords + || $this->config->processThumbnails + || $this->config->processMediaInfo + || $this->config->processAudioInfo + || $this->config->processVideo + || $this->config->processJPGSample; + } + + private function initializeContext(ReleaseProcessingContext $context): void + { + $context->initializeFromConfig( + $this->config->processVideo, + $this->config->processMediaInfo, + $this->config->processAudioInfo, + $this->config->processAudioSample, + $this->config->processJPGSample, + $this->config->processThumbnails + ); + + $context->passwordStatus = ReleaseBrowseService::PASSWD_NONE; + $context->releaseHasPassword = false; + try { + $context->releaseGroupName = UsenetGroup::getNameByID($context->release->groups_id); + } catch (\Throwable) { + $context->releaseGroupName = ''; + } + $context->releaseHasNoNFO = (int) $context->release->nfostatus !== 1; + $context->resetMessageIDs(); + $context->resetCounters(); + } + + private function checkReleaseTimeout(ReleaseProcessingContext $context): bool + { + if (! $context->isTimedOut($this->config->releaseProcessingTimeout)) { + return false; + } + + $deleted = $this->releaseManager->handleReleaseTimeout( + $context->release, + $this->config->maxPpTimeoutCount + ); + + if ($deleted) { + $this->output->echoReleaseTimeoutDeleted( + $context->release->id, + (int) ($context->release->pp_timeout_count ?? 0) + 1 + ); + } else { + $this->output->echoReleaseTimeout($context->release->id, $context->getElapsedSeconds()); + } + + if ($context->tmpPath !== '') { + $this->tempWorkspace->clearDirectory($context->tmpPath, false); + } + + return true; + } + + private function processMessageIdDownloads(ReleaseProcessingContext $context): void + { + if ((! $context->foundSample || ! $context->foundVideo) && $context->sampleMessageIDs !== []) { + $result = $this->downloadService->download( + DownloadKind::Sample, + $context->sampleMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && is_string($result['data']) && $this->downloadService->meetsMinimumSize($result['data'])) { + $this->output->echoSampleDownload(); + $fileLocation = $context->tmpPath.'sample_'.random_int(0, 99999).'.avi'; + File::put($fileLocation, $result['data']); + + if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('sample'); + $this->output->echoSampleCreated(); + } + if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('video'); + $this->output->echoVideoCreated(); + } + } elseif (! $result['success']) { + $this->output->echoSampleFailure(); + } + } + + if ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) + && ! empty($context->mediaInfoMessageIDs) + ) { + $result = $this->downloadService->download( + DownloadKind::MediaInfo, + $context->mediaInfoMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && is_string($result['data']) && $this->downloadService->meetsMinimumSize($result['data'])) { + $this->output->echoMediaInfoDownload(); + $fileLocation = $context->tmpPath.'media.avi'; + File::put($fileLocation, $result['data']); + + if (! $context->foundMediaInfo && $this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { + $context->markFound('mediaInfo'); + $this->output->echoMediaInfoAdded(); + } + if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('sample'); + $this->output->echoSampleCreated(); + } + if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('video'); + $this->output->echoVideoCreated(); + } + } elseif (! $result['success']) { + $this->output->echoMediaInfoFailure(); + } + } + + if ((! $context->foundAudioInfo || ! $context->foundAudioSample) + && ! empty($context->audioInfoMessageIDs) + ) { + $result = $this->downloadService->download( + DownloadKind::Audio, + $context->audioInfoMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && is_string($result['data'])) { + $this->output->echoAudioDownload(); + $fileLocation = $context->tmpPath.'audio.'.$context->audioInfoExtension; + File::put($fileLocation, $result['data']); + + $audioResult = $this->mediaService->getAudioInfo( + $fileLocation, + $context->audioInfoExtension, + $context, + $context->tmpPath + ); + + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + } elseif (! $result['success']) { + $this->output->echoAudioFailure(); + } + } + + if (! $context->foundJPGSample && $context->jpgMessageIDs !== []) { + $result = $this->downloadService->download( + DownloadKind::Jpg, + $context->jpgMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && is_string($result['data'])) { + $this->output->echoJpgDownload(); + $fileLocation = $context->tmpPath.'samplepicture.jpg'; + File::put($fileLocation, $result['data']); + + if ($this->mediaService->isJpegData($fileLocation) + && $this->mediaService->getJPGSample($fileLocation, $context->release->guid) + ) { + $context->markFound('jpgSample'); + $this->output->echoJpgSaved(); + } + + File::delete($fileLocation); + } elseif (! $result['success']) { + $this->output->echoJpgFailure(); + } + } + } + + /** + * @param list $triedCompressedMids + */ + private function processNzbCompressedFiles( + ReleaseProcessingContext $context, + bool $reverse, + array &$triedCompressedMids + ): void { + if ($context->groupUnavailable) { + return; + } + + $nzbContents = $context->nzbContents; + if ($reverse) { + krsort($nzbContents); + } else { + $triedCompressedMids = []; + } + + $failed = $downloaded = 0; + + foreach ($nzbContents as $nzbFile) { + if ($downloaded >= $this->config->maximumRarSegments + || $failed >= $this->config->maximumRarPasswordChecks + || $context->releaseHasPassword + || $context->groupUnavailable + ) { + break; + } + + $title = (string) ($nzbFile['title'] ?? ''); + if (preg_match( + '/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i', + $title + ) !== 1) { + continue; + } + + $segments = $nzbFile['segments'] ?? []; + $messageIDs = []; + $segmentCount = count($segments) - 1; + + foreach (range(0, $this->config->maximumRarSegments - 1) as $index) { + if ($index > $segmentCount) { + break; + } + + $segment = (string) $segments[$index]; + if (! $reverse) { + $triedCompressedMids[] = $segment; + } elseif (in_array($segment, $triedCompressedMids, false)) { + continue 2; + } + + $messageIDs[] = $segment; + } + + if ($messageIDs === []) { + continue; + } + + $result = $this->downloadService->download( + DownloadKind::Compressed, + $messageIDs, + $context->releaseGroupName, + $context->release->id, + $title + ); + + if ($result['groupUnavailable']) { + $context->groupUnavailable = true; + $this->output->echoGroupUnavailable(); + break; + } + + if ($result['success'] && is_string($result['data'])) { + $this->output->echoCompressedDownload(); + $downloaded++; + + $processed = $this->processCompressedData($result['data'], $context, $reverse); + if ($processed || $context->releaseHasPassword) { + break; + } + } else { + $failed++; + $this->output->echoCompressedFailure($failed); + } + } + } + + private function processCompressedData( + string $compressedData, + ReleaseProcessingContext $context, + bool $reverse + ): bool { + $result = $this->archiveService->processCompressedData( + $compressedData, + $context, + $context->tmpPath + ); + + if ($result['hasPassword']) { + $context->releaseHasPassword = true; + $context->passwordStatus = $result['passwordStatus']; + + return false; + } + + if (isset($result['standaloneVideoType'])) { + $this->output->echoInlineVideo(); + $fileLocation = $context->tmpPath.'inline_video_'.uniqid('', true).'.'.$result['standaloneVideoType']; + File::put($fileLocation, $result['standaloneVideoData']); + + if (! $context->foundMediaInfo && $this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { + $context->markFound('mediaInfo'); + $this->output->echoMediaInfoAdded(); + } + if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('sample'); + $this->output->echoSampleCreated(); + } + if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->markFound('video'); + $this->output->echoVideoCreated(); + } + + return $context->foundMediaInfo || $context->foundSample || $context->foundVideo; + } + + if (! $result['success']) { + return false; + } + + if (! empty($result['archiveMarker'])) { + $this->output->echoArchiveMarker($result['archiveMarker']); + } + + if ($reverse && ! empty($result['dataSummary'])) { + $this->releaseManager->processReleaseNameFromRar($result['dataSummary'], $context); + } + + foreach ($result['files'] as $file) { + if ($context->releaseHasPassword) { + break; + } + + if ($this->releaseManager->addFileInfo($file, $context, $this->config->supportFileRegex)) { + $this->output->echoFileInfoAdded(); + } + } + + if ($context->addedFileInfo > 0) { + $this->releaseManager->updateSearchIndex($context->release->id); + } + + if (! $context->foundJPGSample && $this->config->processJPGSample) { + $this->archiveFallback->processJpgFromArchiveFileList($compressedData, $result['files'], $context); + } + + return $context->totalFileInfo > 0; + } + + private function processExtractedFiles(ReleaseProcessingContext $context): void + { + $nestedLevels = 0; + + while ($nestedLevels < $this->config->maxNestedLevels) { + if ($context->compressedFilesChecked >= AdditionalProcessingOrchestrator::MAX_COMPRESSED_FILES_TO_CHECK) { + break; + } + + $foundCompressed = false; + $pattern = '/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1)($|[^a-z0-9])/i'; + + try { + $files = $this->tempWorkspace->listFiles($context->tmpPath, $pattern); + } catch (\Throwable) { + break; + } + + foreach ($files as $file) { + $filePath = is_array($file) ? $file[0] : $file->getPathname(); + if (! File::isFile($filePath)) { + continue; + } + + $rarData = @File::get($filePath); + if (! empty($rarData)) { + $this->processCompressedData($rarData, $context, false); + $foundCompressed = true; + } + File::delete($filePath); + } + + if (! $foundCompressed) { + break; + } + $nestedLevels++; + } + + try { + $files = $this->tempWorkspace->listFiles($context->tmpPath); + } catch (\Throwable) { + return; + } + + foreach ($files as $file) { + $filePath = is_object($file) ? $file->getPathname() : $file; + + if (preg_match('/[\/\\\\]\.{1,2}$/', $filePath) === 1 || ! File::isFile($filePath)) { + continue; + } + + if (! $context->foundPAR2Info && preg_match('/\.par2$/i', $filePath) === 1) { + $this->releaseManager->processPar2File( + $filePath, + $context, + $this->archiveService->getPar2Info() + ); + + continue; + } + + if ($context->releaseHasNoNFO) { + if (preg_match('/(\.(nfo|inf|ofn|diz)|info\.txt)$/i', $filePath) === 1) { + if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + } + + continue; + } + + if ($this->releaseManager->isNfoFilename($filePath)) { + if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + } + + continue; + } + } + + if ((! $context->foundAudioInfo || ! $context->foundAudioSample) + && preg_match('/(.*)'.$this->config->audioFileRegex.'$/i', $filePath, $fileType) === 1 + ) { + $audioPath = $context->tmpPath.'audiofile.'.$fileType[2]; + File::move($filePath, $audioPath); + $audioResult = $this->mediaService->getAudioInfo( + $audioPath, + $fileType[2], + $context, + $context->tmpPath + ); + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + File::delete($audioPath); + + continue; + } + + if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png)$/i', $filePath) === 1) { + if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { + $context->markFound('jpgSample'); + $this->output->echoJpgSaved(); + } + File::delete($filePath); + + continue; + } + + if ((! $context->foundSample || ! $context->foundVideo || ! $context->foundMediaInfo) + && preg_match('/(.*)'.$this->config->videoFileRegex.'$/i', $filePath) === 1 + ) { + $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); + + continue; + } + + $output = fileInfo($filePath); + if ($output === '' || $output === null) { + continue; + } + + if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG/i', $output) === 1) { + if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { + $context->markFound('jpgSample'); + $this->output->echoJpgSaved(); + } + File::delete($filePath); + } elseif ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) + && preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output) === 1 + ) { + $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); + } elseif ((! $context->foundAudioSample || ! $context->foundAudioInfo) + && preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $audioType) === 1 + ) { + $extension = match ($audioType[0]) { + 'FLAC' => 'FLAC', + 'layer III' => 'MP3', + 'Vorbis audio' => 'OGG', + default => 'audio', + }; + $audioPath = $context->tmpPath.'audiofile.'.$extension; + File::move($filePath, $audioPath); + $audioResult = $this->mediaService->getAudioInfo($audioPath, $extension, $context, $context->tmpPath); + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + File::delete($audioPath); + } elseif (! $context->foundPAR2Info && stripos($output, 'Parity') === 0) { + $this->releaseManager->processPar2File( + $filePath, + $context, + $this->archiveService->getPar2Info() + ); + } + } + } +} diff --git a/app/Services/AdditionalProcessing/UsenetDownloadService.php b/app/Services/AdditionalProcessing/UsenetDownloadService.php index be27b77a0..2321bb7c1 100644 --- a/app/Services/AdditionalProcessing/UsenetDownloadService.php +++ b/app/Services/AdditionalProcessing/UsenetDownloadService.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace App\Services\AdditionalProcessing; use App\Services\AdditionalProcessing\Config\ProcessingConfiguration; +use App\Services\AdditionalProcessing\Enums\DownloadKind; use App\Services\NNTP\NNTPService; use Exception; use Illuminate\Support\Facades\Log; @@ -18,9 +19,10 @@ class UsenetDownloadService private NNTPService $nntp; public function __construct( - private readonly ProcessingConfiguration $config + private readonly ProcessingConfiguration $config, + ?NNTPService $nntp = null ) { - $this->nntp = new NNTPService; + $this->nntp = $nntp ?? new NNTPService; } /** @@ -108,80 +110,21 @@ class UsenetDownloadService } /** - * Download sample video content. + * Download content for a specific processing step. * - * @param array $messageIDs - * @return array - */ - public function downloadSample( - array $messageIDs, - string $groupName = '', - ?int $releaseId = null - ): array { - return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); - } - - /** - * Download media info video content. - * - * @param array $messageID - * @return array - */ - public function downloadMediaInfo( - string|array $messageID, - string $groupName = '', - ?int $releaseId = null - ): array { - return $this->downloadByMessageIDs($messageID, $groupName, $releaseId); - } - - /** - * Download audio content. - * - * @param array $messageID - * @return array - */ - public function downloadAudio( - string|array $messageID, - string $groupName = '', - ?int $releaseId = null - ): array { - return $this->downloadByMessageIDs($messageID, $groupName, $releaseId); - } - - /** - * Download JPG content. - * - * @param array $messageIDs - * @return array - */ - public function downloadJPG( - array $messageIDs, - string $groupName = '', - ?int $releaseId = null - ): array { - return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); - } - - /** - * Download compressed file content (RAR, ZIP, etc.). - * - * @param array $messageIDs Message IDs to download - * @param string $groupName Group name for logging - * @param int|null $releaseId Release ID for logging - * @param string|null $fileTitle File title for logging + * @param array|string $messageIDs * @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null} - * - * @throws Exception */ - public function downloadCompressedFile( - array $messageIDs, + public function download( + DownloadKind $kind, + array|string $messageIDs, string $groupName = '', ?int $releaseId = null, ?string $fileTitle = null ): array { if ($this->config->debugMode) { - Log::debug('Attempting compressed fetch', [ + Log::debug('Attempting download', [ + 'kind' => $kind->value, 'release_id' => $releaseId, 'file_title' => $fileTitle, 'message_ids' => $messageIDs, @@ -192,7 +135,8 @@ class UsenetDownloadService $result = $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); if (! $result['success'] && $this->config->debugMode) { - Log::debug('Compressed fetch failed', [ + Log::debug('Download failed', [ + 'kind' => $kind->value, 'release_id' => $releaseId, 'file_title' => $fileTitle, 'message_ids' => $messageIDs, diff --git a/app/Services/PostProcessService.php b/app/Services/PostProcessService.php index 8ed517495..18fdc9c0e 100644 --- a/app/Services/PostProcessService.php +++ b/app/Services/PostProcessService.php @@ -235,7 +235,13 @@ final class PostProcessService */ public function processAdditional(int|string $groupID = '', string $guidChar = ''): void { - app(AdditionalProcessingOrchestrator::class)->start($groupID, $guidChar); + $orchestrator = app(AdditionalProcessingOrchestrator::class); + + try { + $orchestrator->start($groupID, $guidChar); + } finally { + $orchestrator->finish(); + } } /** diff --git a/tests/Unit/AdditionalProcessing/ArchiveExtractionServiceTest.php b/tests/Unit/AdditionalProcessing/ArchiveExtractionServiceTest.php new file mode 100644 index 000000000..00e268a66 --- /dev/null +++ b/tests/Unit/AdditionalProcessing/ArchiveExtractionServiceTest.php @@ -0,0 +1,54 @@ +shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true); + $archiveInfo->shouldReceive('getFileData')->once()->with('cover.jpg')->andReturn('IMAGE-DATA'); + + $service = new ArchiveExtractionService( + $this->makeConfig(), + $archiveInfo, + Mockery::mock(Par2Info::class) + ); + + $this->assertSame('IMAGE-DATA', $service->extractSpecificFile('ARCHIVE', 'cover.jpg', sys_get_temp_dir().'/')); + } + + #[Test] + public function it_detects_common_standalone_video_signatures(): void + { + $service = new ArchiveExtractionService( + $this->makeConfig(), + Mockery::mock(ArchiveInfo::class), + Mockery::mock(Par2Info::class) + ); + + $avi = 'RIFF'.str_repeat("\0", 4).'AVI '.str_repeat("\0", 16); + $mp4 = str_repeat("\0", 4).'ftypisom'.str_repeat("\0", 16); + + $this->assertSame('avi', $service->detectStandaloneVideo($avi)); + $this->assertSame('mp4', $service->detectStandaloneVideo($mp4)); + $this->assertNull($service->detectStandaloneVideo('tiny')); + } +} diff --git a/tests/Unit/AdditionalProcessing/CreatesProcessingConfiguration.php b/tests/Unit/AdditionalProcessing/CreatesProcessingConfiguration.php new file mode 100644 index 000000000..66a1542f3 --- /dev/null +++ b/tests/Unit/AdditionalProcessing/CreatesProcessingConfiguration.php @@ -0,0 +1,70 @@ + $overrides + */ + private function makeConfig(array $overrides = []): ProcessingConfiguration + { + $reflection = new ReflectionClass(ProcessingConfiguration::class); + /** @var ProcessingConfiguration $config */ + $config = $reflection->newInstanceWithoutConstructor(); + + $values = array_merge([ + 'echoCLI' => false, + 'innerFileBlacklist' => false, + 'maxNestedLevels' => 1, + 'extractUsingRarInfo' => true, + 'fetchLastFiles' => false, + 'unrarPath' => false, + 'unzipPath' => false, + 'timeoutPath' => false, + 'timeoutSeconds' => 0, + 'queryLimit' => 25, + 'segmentsToDownload' => 2, + 'maximumRarSegments' => 3, + 'maximumRarPasswordChecks' => 1, + 'maxSizeGB' => 100, + 'minSizeMB' => 0, + 'alternateNNTP' => false, + 'ffmpegDuration' => 5, + 'addPAR2Files' => false, + 'processVideo' => false, + 'processThumbnails' => false, + 'processAudioSample' => false, + 'processJPGSample' => false, + 'processMediaInfo' => false, + 'processAudioInfo' => false, + 'processPasswords' => false, + 'audioSavePath' => sys_get_temp_dir().'/', + 'tmpUnrarPath' => sys_get_temp_dir().'/', + 'debugMode' => false, + 'searchEnabled' => false, + 'searchDriver' => 'manticore', + 'renameMusicMediaInfo' => false, + 'renamePar2' => false, + 'ffmpegPath' => false, + 'mediaInfoPath' => false, + 'releaseProcessingTimeout' => 0, + 'maxPpTimeoutCount' => 3, + 'audioFileRegex' => '\\.(MP3|FLAC|OGG)', + 'ignoreBookRegex' => '/\\b(epub|pdf)\\b/i', + 'supportFileRegex' => '\\.(?:par2|sfv|nzb)', + 'videoFileRegex' => '\\.(AVI|MKV|MP4)', + ], $overrides); + + foreach ($values as $property => $value) { + $prop = new ReflectionProperty(ProcessingConfiguration::class, $property); + $prop->setValue($config, $value); + } + + return $config; + } +} diff --git a/tests/Unit/AdditionalProcessing/ReleaseFilesArchiveFallbackTest.php b/tests/Unit/AdditionalProcessing/ReleaseFilesArchiveFallbackTest.php new file mode 100644 index 000000000..df6326d01 --- /dev/null +++ b/tests/Unit/AdditionalProcessing/ReleaseFilesArchiveFallbackTest.php @@ -0,0 +1,151 @@ +instance('files', new Filesystem); + Facade::setFacadeApplication($container); + + $capsule = new Capsule($container); + $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); + $capsule->setEventDispatcher(new Dispatcher($container)); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $capsule->schema()->create('release_files', function ($table): void { + $table->integer('releases_id'); + $table->string('name'); + $table->integer('size')->default(0); + $table->timestamps(); + }); + + $this->tmpPath = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR.'fallback_'.uniqid('', true).DIRECTORY_SEPARATOR; + (new Filesystem)->makeDirectory($this->tmpPath, 0777, true, true); + } + + protected function tearDown(): void + { + (new Filesystem)->deleteDirectory($this->tmpPath); + Mockery::close(); + parent::tearDown(); + } + + #[Test] + public function it_extracts_jpg_candidates_from_an_archive_file_list(): void + { + $archive = Mockery::mock(ArchiveExtractionService::class); + $archive->shouldReceive('extractSpecificFile')->once()->andReturn('JPEG-DATA'); + + $media = Mockery::mock(MediaExtractionService::class); + $media->shouldReceive('isValidImage')->once()->andReturn(true); + $media->shouldReceive('getJPGSample')->once()->andReturn(true); + + $output = Mockery::mock(ConsoleOutputService::class); + $output->shouldReceive('echoJpgSaved')->once()->andReturnNull(); + + $fallback = new ReleaseFilesArchiveFallback( + $this->makeConfig(['processJPGSample' => true]), + $archive, + $media, + Mockery::mock(UsenetDownloadService::class), + Mockery::mock(ReleaseFileManager::class), + $output + ); + + $context = $this->makeContext(); + $fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.jpg', 'size' => 99]], $context); + + $this->assertTrue($context->foundJPGSample); + } + + #[Test] + public function it_uses_stored_release_files_to_recover_an_nfo(): void + { + ReleaseFile::query()->create([ + 'releases_id' => 12, + 'name' => 'release.nfo', + 'size' => 200, + ]); + + $archive = Mockery::mock(ArchiveExtractionService::class); + $archive->shouldReceive('extractSpecificFile')->once()->andReturn('NFO DATA'); + + $download = Mockery::mock(UsenetDownloadService::class); + $download->shouldReceive('download')->once()->andReturn([ + 'success' => true, + 'data' => 'ARCHIVE', + 'groupUnavailable' => false, + 'error' => null, + ]); + $download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class)); + + $manager = Mockery::mock(ReleaseFileManager::class); + $manager->shouldReceive('processNfoFile')->once()->andReturnUsing(function (string $path, ReleaseProcessingContext $context): bool { + $this->assertFileExists($path); + $context->releaseHasNoNFO = false; + + return true; + }); + + $output = Mockery::mock(ConsoleOutputService::class); + $output->shouldReceive('echoNfoFound')->once()->andReturnNull(); + + $fallback = new ReleaseFilesArchiveFallback( + $this->makeConfig(), + $archive, + Mockery::mock(MediaExtractionService::class), + $download, + $manager, + $output + ); + + $context = $this->makeContext(12); + $context->releaseGroupName = 'alt.binaries.test'; + $context->releaseHasNoNFO = true; + $context->nzbContents = [['title' => 'archive.part01.rar', 'segments' => ['<1>', '<2>']]]; + + $fallback->processNfoFromReleaseFiles($context); + + $this->assertFalse($context->releaseHasNoNFO); + } + + private function makeContext(int $releaseId = 12): ReleaseProcessingContext + { + $context = new ReleaseProcessingContext(new Release([ + 'id' => $releaseId, + 'guid' => 'guid-'.$releaseId, + 'groups_id' => 22, + ])); + $context->tmpPath = $this->tmpPath; + + return $context; + } +} diff --git a/tests/Unit/AdditionalProcessing/ReleaseProcessorTest.php b/tests/Unit/AdditionalProcessing/ReleaseProcessorTest.php new file mode 100644 index 000000000..200282fb3 --- /dev/null +++ b/tests/Unit/AdditionalProcessing/ReleaseProcessorTest.php @@ -0,0 +1,173 @@ +makeProcessor( + nzbParser: Mockery::mock(NzbContentParser::class) + ->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn(['error' => 'broken nzb', 'contents' => []])->getMock(), + releaseManager: Mockery::mock(ReleaseFileManager::class) + ->shouldReceive('deleteRelease')->once()->andReturnNull()->getMock(), + tempWorkspace: Mockery::mock(TempWorkspaceService::class) + ->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/') + ->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull()->getMock(), + output: Mockery::mock(ConsoleOutputService::class) + ->shouldReceive('echoReleaseStart')->once()->andReturnNull() + ->shouldReceive('setProcessTitle')->once()->andReturnNull() + ->shouldReceive('warning')->once()->with('broken nzb')->andReturnNull()->getMock() + ); + + $processor->process($this->makeContext(), '/tmp/main/'); + + $this->assertTrue(true); + } + + #[Test] + public function it_finalizes_a_release_after_basic_successful_processing(): void + { + $config = $this->makeConfig(); + $nzbParser = Mockery::mock(NzbContentParser::class); + $nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([ + 'error' => null, + 'contents' => [['title' => 'file.nzb', 'segments' => []]], + ]); + $nzbParser->shouldReceive('extractMessageIDs')->once()->andReturn([ + 'hasCompressedFile' => false, + 'sampleMessageIDs' => [], + 'jpgMessageIDs' => [], + 'mediaInfoMessageID' => '', + 'audioInfoMessageID' => '', + 'audioInfoExtension' => '', + 'bookFileCount' => 0, + ]); + + $releaseManager = Mockery::mock(ReleaseFileManager::class); + $releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull(); + + $tempWorkspace = Mockery::mock(TempWorkspaceService::class); + $tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/'); + $tempWorkspace->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull(); + + $output = Mockery::mock(ConsoleOutputService::class); + $output->shouldReceive('echoReleaseStart')->once()->andReturnNull(); + $output->shouldReceive('setProcessTitle')->once()->andReturnNull(); + + $processor = new ReleaseProcessor( + $config, + $nzbParser, + Mockery::mock(ArchiveExtractionService::class), + Mockery::mock(MediaExtractionService::class), + Mockery::mock(UsenetDownloadService::class), + $releaseManager, + Mockery::mock(ReleaseFilesArchiveFallback::class), + $tempWorkspace, + $output + ); + + $processor->process($this->makeContext(), '/tmp/main/'); + + $this->assertTrue(true); + } + + #[Test] + public function it_marks_release_timeout_before_full_processing_continues(): void + { + $config = $this->makeConfig(['releaseProcessingTimeout' => 1, 'maxPpTimeoutCount' => 2]); + + $nzbParser = Mockery::mock(NzbContentParser::class); + $nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([ + 'error' => null, + 'contents' => [], + ]); + + $releaseManager = Mockery::mock(ReleaseFileManager::class); + $releaseManager->shouldReceive('handleReleaseTimeout')->once()->andReturn(false); + $releaseManager->shouldNotReceive('finalizeRelease'); + + $tempWorkspace = Mockery::mock(TempWorkspaceService::class); + $tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/'); + $tempWorkspace->shouldReceive('clearDirectory')->twice()->with('/tmp/ap-release/', false)->andReturnNull(); + + $output = Mockery::mock(ConsoleOutputService::class); + $output->shouldReceive('echoReleaseStart')->once()->andReturnNull(); + $output->shouldReceive('setProcessTitle')->once()->andReturnNull(); + $output->shouldReceive('echoReleaseTimeout')->once()->andReturnNull(); + + $processor = new ReleaseProcessor( + $config, + $nzbParser, + Mockery::mock(ArchiveExtractionService::class), + Mockery::mock(MediaExtractionService::class), + Mockery::mock(UsenetDownloadService::class), + $releaseManager, + Mockery::mock(ReleaseFilesArchiveFallback::class), + $tempWorkspace, + $output + ); + + $context = $this->makeContext(); + $context->startTime = hrtime(true) - 2_000_000_000; + + $processor->process($context, '/tmp/main/'); + + $this->assertTrue(true); + } + + private function makeProcessor( + ?NzbContentParser $nzbParser = null, + ?ReleaseFileManager $releaseManager = null, + ?TempWorkspaceService $tempWorkspace = null, + ?ConsoleOutputService $output = null + ): ReleaseProcessor { + return new ReleaseProcessor( + $this->makeConfig(), + $nzbParser ?? Mockery::mock(NzbContentParser::class), + Mockery::mock(ArchiveExtractionService::class), + Mockery::mock(MediaExtractionService::class), + Mockery::mock(UsenetDownloadService::class), + $releaseManager ?? Mockery::mock(ReleaseFileManager::class), + Mockery::mock(ReleaseFilesArchiveFallback::class), + $tempWorkspace ?? Mockery::mock(TempWorkspaceService::class), + $output ?? Mockery::mock(ConsoleOutputService::class) + ); + } + + private function makeContext(): ReleaseProcessingContext + { + return new ReleaseProcessingContext(new Release([ + 'id' => 1, + 'guid' => 'guid-1', + 'size' => 1024, + 'groups_id' => 10, + 'nfostatus' => -1, + 'pp_timeout_count' => 0, + ])); + } +} diff --git a/tests/Unit/AdditionalProcessing/UsenetDownloadServiceTest.php b/tests/Unit/AdditionalProcessing/UsenetDownloadServiceTest.php new file mode 100644 index 000000000..dcea5462c --- /dev/null +++ b/tests/Unit/AdditionalProcessing/UsenetDownloadServiceTest.php @@ -0,0 +1,73 @@ +shouldReceive('getMessagesByMessageID') + ->once() + ->with([''], false) + ->andReturn('BINARY-DATA'); + + $service = new UsenetDownloadService($this->makeConfig(), $nntp); + + $result = $service->download(DownloadKind::Compressed, [''], 'alt.binaries', 55, 'archive.rar'); + + $this->assertTrue($result['success']); + $this->assertSame('BINARY-DATA', $result['data']); + } + + #[Test] + public function it_marks_group_unavailable_errors_explicitly(): void + { + $error = new class + { + public function getMessage(): string + { + return 'No such news group'; + } + }; + + $nntp = Mockery::mock(NNTPService::class); + $nntp->shouldReceive('getMessagesByMessageID') + ->once() + ->with([''], false) + ->andReturn($error); + + $service = new UsenetDownloadService($this->makeConfig(), $nntp); + + $result = $service->download(DownloadKind::Sample, ['']); + + $this->assertFalse($result['success']); + $this->assertTrue($result['groupUnavailable']); + $this->assertStringContainsString('Group unavailable', (string) $result['error']); + } + + #[Test] + public function it_checks_minimum_download_sizes(): void + { + $service = new UsenetDownloadService($this->makeConfig(), Mockery::mock(NNTPService::class)); + + $this->assertFalse($service->meetsMinimumSize(str_repeat('a', 40))); + $this->assertTrue($service->meetsMinimumSize(str_repeat('a', 41))); + } +}