diff --git a/Blacklight/processing/post/ProcessAdditional.php b/Blacklight/processing/post/ProcessAdditional.php index cf9de53e2..167432b6e 100755 --- a/Blacklight/processing/post/ProcessAdditional.php +++ b/Blacklight/processing/post/ProcessAdditional.php @@ -2,3218 +2,98 @@ namespace Blacklight\processing\post; -use App\Models\Category; -use App\Models\MediaInfo as MediaInfoModel; -use App\Models\Predb; -use App\Models\Release; -use App\Models\ReleaseFile; -use App\Models\Settings; -use App\Models\UsenetGroup; +use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator; +use App\Services\AdditionalProcessing\ArchiveExtractionService; +use App\Services\AdditionalProcessing\Config\ProcessingConfiguration; +use App\Services\AdditionalProcessing\ConsoleOutputService; +use App\Services\AdditionalProcessing\MediaExtractionService; +use App\Services\AdditionalProcessing\NzbContentParser; +use App\Services\AdditionalProcessing\ReleaseFileManager; +use App\Services\AdditionalProcessing\UsenetDownloadService; use App\Services\Categorization\CategorizationService; -use Blacklight\ColorCLI; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\TempWorkspaceService; use Blacklight\NameFixer; use Blacklight\Nfo; -use Blacklight\NNTP; use Blacklight\NZB; use Blacklight\ReleaseExtra; use Blacklight\ReleaseImage; -use Blacklight\Releases; -use Blacklight\utility\Utility; -use dariusiii\rarinfo\ArchiveInfo; -use dariusiii\rarinfo\Par2Info; use Exception; -use FFMpeg\Coordinate\Dimension; -use FFMpeg\Coordinate\TimeCode; -use FFMpeg\FFMpeg; -use FFMpeg\FFProbe; -use FFMpeg\Filters\Video\ResizeFilter; -use FFMpeg\Format\Audio\Vorbis; -use FFMpeg\Format\Video\Ogg; -use Illuminate\Contracts\Filesystem\FileNotFoundException; -use Illuminate\Support\Arr; -use Illuminate\Support\Carbon; -use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\File; -use Illuminate\Support\Facades\Log; -use Illuminate\Support\Str; -use Mhor\MediaInfo\MediaInfo; +/** + * @deprecated Use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator instead. + * This class is maintained for backward compatibility only. + * + * Additional release post-processing. + * Handles NZB parsing, archive extraction, media processing, and release updates. + * + * The functionality has been refactored into separate services: + * - ProcessingConfiguration: Configuration management + * - NzbContentParser: NZB file parsing + * - ArchiveExtractionService: Archive handling (RAR, ZIP, 7z, etc.) + * - MediaExtractionService: Video/audio/image processing + * - UsenetDownloadService: NNTP downloads + * - ReleaseFileManager: Database operations + * - ConsoleOutputService: CLI output + * - AdditionalProcessingOrchestrator: Main coordinator + */ class ProcessAdditional { /** * How many compressed (rar/zip) files to check. - * - * @var int */ - public const maxCompressedFilesToCheck = 10; + public const int maxCompressedFilesToCheck = 10; - protected $_releases; - - /** - * Count of releases to work on. - */ - protected int $_totalReleases; - - protected $_release; - - protected NZB $_nzb; - - /** - * List of files with sizes/etc contained in the NZB. - */ - protected array $_nzbContents; - - protected Par2Info $_par2Info; - - protected ArchiveInfo $_archiveInfo; - - /** - * @var bool|null|string - */ - protected mixed $_innerFileBlacklist; - - protected int $_maxNestedLevels; - - /** - * @var null|string - */ - protected mixed $_unrarPath; - - protected mixed $_unzipPath; - - protected mixed $_7zipPath; - - protected string $_killString; - - protected string|bool $_showCLIReleaseID; - - protected int $_queryLimit; - - protected int $_segmentsToDownload; - - protected int $_maximumRarSegments; - - protected int $_maximumRarPasswordChecks; - - protected string $_maxSize; - - protected string $_minSize; - - protected bool $_processThumbnails; - - protected string $_audioSavePath; - - protected string $_supportFileRegex; - - protected bool $_echoCLI; - - protected NNTP $_nntp; - - protected CategorizationService $_categorize; - - protected NameFixer $_nameFixer; - - protected ReleaseExtra $_releaseExtra; - - protected ReleaseImage $_releaseImage; - - protected Nfo $_nfo; - - protected bool $_extractUsingRarInfo; - - protected bool $_alternateNNTP; - - protected int $_ffMPEGDuration; - - protected bool $_addPAR2Files; - - protected bool $_processVideo; - - protected bool $_processJPGSample; - - protected bool $_processAudioSample; - - protected bool $_processMediaInfo; - - protected bool $_processAudioInfo; - - protected bool $_processPasswords; - - protected string $_audioFileRegex; - - protected string $_ignoreBookRegex; - - protected string $_videoFileRegex; - - protected bool $_debugDownloadFailures; // added - - protected bool $_groupUnavailable; // added - - /** - * Have we created a video file for the current release? - */ - protected bool $_foundVideo; - - /** - * Have we found MediaInfo data for a Video for the current release? - */ - protected bool $_foundMediaInfo; - - /** - * Have we found MediaInfo data for a Audio file for the current release? - */ - protected bool $_foundAudioInfo; - - /** - * Have we created a short Audio file sample for the current release? - */ - protected bool $_foundAudioSample; - - /** - * Extension of the found audio file (MP3/FLAC/etc). - */ - protected string $_AudioInfoExtension; - - /** - * Have we downloaded a JPG file for the current release? - */ - protected bool $_foundJPGSample; - - /** - * Have we created a Video JPG image sample for the current release? - */ - protected bool $_foundSample; - - /** - * Have we found PAR2 info on this release? - */ - protected bool $_foundPAR2Info; - - /** - * Message ID's for found content to download. - */ - protected array $_sampleMessageIDs; - - protected $_JPGMessageIDs; - - protected $_MediaInfoMessageIDs; - - protected $_AudioInfoMessageIDs; - - protected $_RARFileMessageIDs; - - /** - * Password status of the current release. - */ - protected int $_passwordStatus; - - /** - * Does the current release have a password? - */ - protected bool $_releaseHasPassword; - - /** - * Does the current release have an NFO file? - */ - protected bool $_releaseHasNoNFO; - - /** - * Name of the current release's usenet group. - */ - protected string $_releaseGroupName; - - /** - * Number of file information added to DB (from rar/zip/par2 contents). - */ - protected int $_addedFileInfo; - - /** - * Number of file information we found from RAR/ZIP. - * (if some of it was already in DB, this count goes up, while the count above does not). - */ - protected int $_totalFileInfo; - - /** - * How many compressed (rar/zip) files have we checked. - */ - protected int $_compressedFilesChecked; - - /** - * Should we download the last rar? - */ - protected bool $_fetchLastFiles; - - /** - * Are we downloading the last rar? - */ - protected bool $_reverse; - - protected ?ManticoreSearch $manticore = null; - - private ?FFMpeg $ffmpeg = null; - - private ?FFProbe $ffprobe = null; - - private ?MediaInfo $mediaInfo = null; - - private ?ElasticSearchSiteSearch $elasticsearch = null; - - private bool $elasticsearchEnabled; + private AdditionalProcessingOrchestrator $orchestrator; /** * ProcessAdditional constructor. * - * * @throws Exception */ public function __construct() { - $this->_echoCLI = config('nntmux.echocli'); + // Build the orchestrator with all dependencies + $config = new ProcessingConfiguration(); - $this->_nntp = new NNTP; - - $this->_nzb = new NZB; - $this->_archiveInfo = new ArchiveInfo; - $this->_categorize = new CategorizationService(); - $this->_nameFixer = new NameFixer; - $this->_releaseExtra = new ReleaseExtra; - $this->_releaseImage = new ReleaseImage; - $this->_par2Info = new Par2Info; - $this->_nfo = new Nfo; - - $this->_innerFileBlacklist = Settings::settingValue('innerfileblacklist') === '' ? false : Settings::settingValue('innerfileblacklist'); - $this->_maxNestedLevels = (int) Settings::settingValue('maxnestedlevels') === 0 ? 3 : (int) Settings::settingValue('maxnestedlevels'); - $this->_extractUsingRarInfo = (int) Settings::settingValue('extractusingrarinfo') !== 0; - $this->_fetchLastFiles = config('nntmux_settings.fetch_last_file'); - $this->_unrarPath = config('nntmux_settings.unrar_path'); - $this->_unzipPath = config('nntmux_settings.unzip_path'); - $this->_7zipPath = config('nntmux_settings.7zip_path'); - - // Pass the binary extractors to ArchiveInfo. - $clients = []; - $clients += [ArchiveInfo::TYPE_RAR => $this->_unrarPath]; - - $this->_archiveInfo->setExternalClients($clients); - - $this->_killString = '"'; - if (config('nntmux_settings.timeout_path') && (int) Settings::settingValue('timeoutseconds') > 0) { - $this->_killString = ( - '"'.config('nntmux_settings.timeout_path'). - '" --foreground --signal=KILL '. - Settings::settingValue('timeoutseconds').' "' - ); - } - - $this->_showCLIReleaseID = (PHP_BINARY.' '.__DIR__.'/ProcessAdditional.php ReleaseID: '); - - // Maximum amount of releases to fetch per run. - $this->_queryLimit = - (Settings::settingValue('maxaddprocessed') !== '') ? (int) Settings::settingValue('maxaddprocessed') : 25; - - // Maximum message ID's to download per file type in the NZB (video, jpg, etc). - $this->_segmentsToDownload = - (Settings::settingValue('segmentstodownload') !== '') ? (int) Settings::settingValue('segmentstodownload') : 2; - - // Maximum message ID's to download for a RAR file. - $this->_maximumRarSegments = - (Settings::settingValue('maxpartsprocessed') !== '') ? (int) Settings::settingValue('maxpartsprocessed') : 3; - - // Maximum RAR files to check for a password before stopping. - $this->_maximumRarPasswordChecks = - (Settings::settingValue('passchkattempts') !== '') ? (int) Settings::settingValue('passchkattempts') : 1; - - $this->_maximumRarPasswordChecks = (max($this->_maximumRarPasswordChecks, 1)); - - // Maximum size of releases in GB. - $this->_maxSize = (Settings::settingValue('maxsizetopostprocess') !== '') ? (int) Settings::settingValue('maxsizetopostprocess') : 100; - // Minimum size of releases in MB. - $this->_minSize = (Settings::settingValue('minsizetopostprocess') !== '') ? (int) Settings::settingValue('minsizetopostprocess') : 100; - - // Use the alternate NNTP provider for downloading Message-ID's ? - $this->_alternateNNTP = config('nntmux_nntp.use_alternate_nntp_server'); - - $this->_ffMPEGDuration = Settings::settingValue('ffmpeg_duration') !== '' ? (int) Settings::settingValue('ffmpeg_duration') : 5; - - $this->_addPAR2Files = config('nntmux_settings.add_par2'); - - if (! config('nntmux_settings.ffmpeg_path')) { - $this->_processAudioSample = $this->_processThumbnails = $this->_processVideo = false; - } else { - $this->_processAudioSample = (int) Settings::settingValue('saveaudiopreview') !== 0; - $this->_processThumbnails = (int) Settings::settingValue('processthumbnails') !== 0; - $this->_processVideo = (int) Settings::settingValue('processvideos') !== 0; - } - - $this->_processJPGSample = (int) Settings::settingValue('processjpg') !== 0; - $this->_processMediaInfo = config('nntmux_settings.mediainfo_path'); - $this->_processAudioInfo = $this->_processMediaInfo; - $this->_processPasswords = config('nntmux_settings.check_passworded_rars') === true && ! empty(config('nntmux_settings.unrar_path')); - - $this->_audioSavePath = config('nntmux_settings.covers_path').'/audiosample/'; - - $this->_audioFileRegex = '\\.(AAC|AIFF|APE|AC3|ASF|DTS|FLAC|MKA|MKS|MP2|MP3|RA|OGG|OGM|W64|WAV|WMA)'; - $this->_ignoreBookRegex = '/\\b(epub|lit|mobi|pdf|sipdf|html)\\b.*\\.rar(?!.{20,})/i'; - $this->_supportFileRegex = '\\.(?:vol\\d{1,3}\\+\\d{1,3}|par2|srs|sfv|nzb)'; - $this->_videoFileRegex = '\\.(AVI|F4V|IFO|M1V|M2V|M4V|MKV|MOV|MP4|MPEG|MPG|MPGV|MPV|OGV|QT|RM|RMVB|TS|VOB|WMV)'; - $this->_debugDownloadFailures = (bool) config('app.debug'); - $this->_groupUnavailable = false; // flag when NNTP group is missing - $this->elasticsearchEnabled = config('nntmux.elasticsearch_enabled') === true; - } - - /** - * Clear out the main temp path when done. - */ - public function __destruct() - { - $this->_clearMainTmpPath(); - } - - private function ffmpeg(): FFMpeg - { - if ($this->ffmpeg === null) { - $this->ffmpeg = FFMpeg::create(['timeout' => Settings::settingValue('timeoutseconds')]); - } - - return $this->ffmpeg; - } - - private function ffprobe(): FFProbe - { - if ($this->ffprobe === null) { - $this->ffprobe = FFProbe::create(); - } - - return $this->ffprobe; - } - - private function mediaInfo(): MediaInfo - { - if ($this->mediaInfo === null) { - $this->mediaInfo = new MediaInfo; - $this->mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true); - $this->mediaInfo->setConfig('command', config('nntmux_settings.mediainfo_path')); - } - - return $this->mediaInfo; - } - - private function manticore(): ManticoreSearch - { - if ($this->manticore === null) { - $this->manticore = new ManticoreSearch; - } - - return $this->manticore; - } - - private function elasticsearch(): ElasticSearchSiteSearch - { - if ($this->elasticsearch === null) { - $this->elasticsearch = new ElasticSearchSiteSearch; - } - - return $this->elasticsearch; + $this->orchestrator = new AdditionalProcessingOrchestrator( + $config, + new NzbContentParser(new NZB(), $config->debugMode, $config->echoCLI), + new ArchiveExtractionService($config), + new MediaExtractionService( + $config, + new ReleaseImage(), + new ReleaseExtra(), + new CategorizationService() + ), + new UsenetDownloadService($config), + new ReleaseFileManager( + $config, + new ReleaseExtra(), + new ReleaseImage(), + new Nfo(), + new NZB(), + new NameFixer() + ), + new TempWorkspaceService(), + new ConsoleOutputService($config->echoCLI) + ); } /** + * Start the additional processing. + * * @throws Exception */ public function start(string $groupID = '', string $guidChar = ''): void { - $this->_setMainTempPath($guidChar, $groupID); - - // Fetch all the releases to work on. - $this->_fetchReleases($groupID, $guidChar); - - // Check if we have releases to work on. - if ($this->_totalReleases > 0) { - // Echo start time and process description. - $this->_echoDescription(); - - $this->_processReleases(); - } + $this->orchestrator->start($groupID, $guidChar); } /** - * @var string Main temp path to work on. - */ - protected string $_mainTmpPath; - - /** - * @var string Temp path for current release. - */ - protected string $tmpPath; - - /** - * @throws \RuntimeException - * @throws Exception - */ - protected function _setMainTempPath(&$guidChar, string &$groupID = ''): void - { - // Set up the temporary files folder location. - $this->_mainTmpPath = config('nntmux.tmp_unrar_path'); - - // Check if it ends with a dir separator. - if (! Str::endsWith($this->_mainTmpPath, '/') && ! Str::endsWith($this->_mainTmpPath, '\\')) { - $this->_mainTmpPath .= '/'; - } - - // If we are doing per group, use the groupID has an inner path, so other scripts don't delete the files we are working on. - if ($groupID !== '') { - $this->_mainTmpPath .= $groupID.'/'; - } elseif ($guidChar !== '') { - $this->_mainTmpPath .= $guidChar.'/'; - } - - if (! File::isDirectory($this->_mainTmpPath)) { - if (! File::makeDirectory($this->_mainTmpPath, 0777, true, true) && ! File::isDirectory($this->_mainTmpPath)) { - throw new \RuntimeException(sprintf('Directory "%s" was not created', $this->_mainTmpPath)); - } - } - - if (! File::isDirectory($this->_mainTmpPath)) { - throw new \RuntimeException('Could not create the tmpunrar folder ('.$this->_mainTmpPath.')'); - } - - $this->_clearMainTmpPath(); - - $this->tmpPath = $this->_mainTmpPath; - } - - /** - * Clear out old folders/files from the main temp folder. - */ - protected function _clearMainTmpPath(): void - { - if ($this->_mainTmpPath !== '') { - $this->_recursivePathDelete($this->_mainTmpPath); - } - } - - /** - * Get all releases that need to be processed. - * - * - * @void - */ - protected function _fetchReleases(int|string $groupID, string &$guidChar): void - { - $sqlParts = []; - $bindings = []; - - $sqlParts[] = 'SELECT r.id AS id, r.id AS releases_id, r.guid, r.name, r.size, r.groups_id, r.nfostatus, r.fromname,'. - ' r.completion, r.categories_id, r.searchname, r.predb_id, c.disablepreview'; - $sqlParts[] = 'FROM releases r'; - $sqlParts[] = 'LEFT JOIN categories c ON c.id = r.categories_id'; - $sqlParts[] = 'WHERE r.passwordstatus = ?'; - $bindings[] = -1; // passwordstatus - $sqlParts[] = 'AND r.nzbstatus = ?'; - $bindings[] = 1; // ensure NZB file exists/ready - $sqlParts[] = 'AND r.haspreview = ?'; - $bindings[] = -1; // haspreview - $sqlParts[] = 'AND c.disablepreview = ?'; - $bindings[] = 0; // categories.disablepreview - - if ($this->_maxSize > 0) { - $sqlParts[] = 'AND r.size < ?'; - $bindings[] = (int) $this->_maxSize * 1073741824; // GB to bytes - } - if ($this->_minSize > 0) { - $sqlParts[] = 'AND r.size > ?'; - $bindings[] = (int) $this->_minSize * 1048576; // MB to bytes - } - if (! empty($groupID)) { - $sqlParts[] = 'AND r.groups_id = ?'; - $bindings[] = $groupID; - } - if (! empty($guidChar)) { - $sqlParts[] = 'AND r.leftguid = ?'; - $bindings[] = $guidChar; - } - - $sqlParts[] = 'ORDER BY r.passwordstatus ASC, r.postdate DESC'; - $limit = $this->_queryLimit; - if ($limit <= 0) { - $limit = 25; // fallback safety - } - $sqlParts[] = 'LIMIT '.$limit; // limit - - $finalSql = implode(' ', $sqlParts); - - $rawResults = DB::select($finalSql, $bindings); - // Hydrate into Release models so later code expecting model (with array access) still works. - $attributeArrays = array_map(static fn ($row) => (array) $row, $rawResults); - $this->_releases = Release::hydrate($attributeArrays); - $this->_totalReleases = $this->_releases->count(); - } - - /** - * Output the description and start time. - * - * @void - */ - protected function _echoDescription(): void - { - if ($this->_totalReleases > 1 && $this->_echoCLI) { - $this->_echo( - PHP_EOL. - 'Additional post-processing, started at: '. - now()->format('D M d, Y G:i a'). - PHP_EOL. - 'Downloaded: (xB)=yEnc article, (cB)=compressed part'. - PHP_EOL. - 'Failures: fC#=Compressed(part #), fS=Sample, fM=Media(video), fA=Audio, fJ=JPEG, G=Missing group'. - PHP_EOL. - 'Processing: r=RAR, z=ZIP, 7z=7zip (names/entries), g=GZIP, b=BZIP2, x=XZ, (vRAW)=Inline video detected'. - PHP_EOL. - 'Added: s=Sample image, j=JPEG image, A=Audio sample, a=Audio MediaInfo, v=Video sample'. - PHP_EOL. - 'Added: m=Video MediaInfo, n=NFO, ^=Inner file details (RAR/ZIP/7z/etc)'. - '', - 'header' - ); - } - } - - /** - * Loop through the releases, processing them 1 at a time. - * - * @throws \RuntimeException - * @throws Exception - */ - protected function _processReleases(): void - { - foreach ($this->_releases as $this->_release) { - $this->_echo( - PHP_EOL.'['.$this->_release->id.']['. - human_filesize($this->_release->size, 1).']', - 'primaryOver' - ); - - cli_set_process_title($this->_showCLIReleaseID.$this->_release->id); - - // Create folder to store temporary files. - if (! $this->_createTempFolder()) { - continue; - } - - // Get NZB contents. - if (! $this->_getNZBContents()) { - continue; - } - - // Reset the current release variables. - $this->_resetReleaseStatus(); - - // Go through the files in the NZB, get the amount of book files. - $totalBooks = $this->_processNZBContents(); - - // Check if this NZB is a large collection of books. - $bookFlood = false; - if ($totalBooks > 80 && ($totalBooks * 2) >= \count($this->_nzbContents)) { - $bookFlood = true; - } - - if ($this->_processPasswords || $this->_processThumbnails || $this->_processMediaInfo || $this->_processAudioInfo || $this->_processVideo - ) { - // Process usenet Message-ID downloads. - $this->_processMessageIDDownloads(); - - // Process compressed (RAR/ZIP) files inside the NZB. - if (! $bookFlood && $this->_NZBHasCompressedFile) { - // Download the RARs/ZIPs, extract the files inside them and insert the file info into the DB. - $this->_processNZBCompressedFiles(); - - // Download rar/zip in reverse order, to get the last rar or zip file. - if ($this->_fetchLastFiles) { - $this->_processNZBCompressedFiles(true); - } - - if (! $this->_releaseHasPassword) { - // Process the extracted files to get video/audio samples/etc. - $this->_processExtractedFiles(); - } - } - } - - // Update the release to say we processed it. - $this->_finalizeRelease(); - - // Delete all files / folders for this release. - $this->_recursivePathDelete($this->tmpPath); - } - if ($this->_echoCLI) { - echo PHP_EOL; - } - } - - protected function _recursivePathDelete(string $path): void - { - if (File::isDirectory($path)) { - foreach (File::allFiles($path) as $file) { - File::delete($file); - } - // Don't delete the main temp folder. - if ($path !== $this->_mainTmpPath) { - File::deleteDirectory($path); - } - } elseif (File::isFile($path)) { - File::delete($path); - } - } - - /** - * Create a temporary storage folder for the current release. - * - * - * - * @throws Exception - */ - protected function _createTempFolder(): bool - { - // Per release defaults. - $this->tmpPath = $this->_mainTmpPath.$this->_release->guid.'/'; - if (! File::isDirectory($this->tmpPath)) { - try { - // Use force + recursive so concurrent processes creating the same path don't error out. - File::makeDirectory($this->tmpPath, 0777, true, true); - } catch (\Throwable $e) { - // If another process created it between the isDirectory check and mkdir call, ignore. - if (! File::isDirectory($this->tmpPath)) { - // Retry once after a tiny delay (filesystem latency / NFS eventual consistency). - usleep(100000); // 100ms - if (! File::isDirectory($this->tmpPath)) { - try { - File::makeDirectory($this->tmpPath, 0777, true, true); - } catch (\Throwable $e2) { - if (config('app.debug') === true) { - Log::warning('Unable to create directory after retry: '.$this->tmpPath.' :: '.$e2->getMessage()); - } - $this->_echo('Unable to create directory: '.$this->tmpPath, 'warning'); - - return false; - } - } - } - } - } - - return File::isDirectory($this->tmpPath); - } - - /** - * Get a list of contents inside a release's NZB file. - * - * - * @throws Exception - */ - protected function _getNZBContents(): bool - { - $nzbPath = $this->_nzb->NZBPath($this->_release->guid); - if ($nzbPath !== false) { - $nzbContents = Utility::unzipGzipFile($nzbPath); - if (! $nzbContents) { - if ($this->_echoCLI) { - $this->_echo('NZB is empty or broken for GUID: '.$this->_release->guid, 'warning'); - } - // Try repair on raw file contents (maybe unzip failed but file readable) - try { - $rawFile = @File::get($nzbPath); - if ($rawFile) { - // If gz, attempt decompress ignoring errors. - if (str_ends_with(strtolower($nzbPath), '.gz')) { - $try = @gzdecode($rawFile); - if ($try !== false) { - $repaired = $this->_repairNZB($try, $nzbPath); - if ($repaired !== null) { - $nzbContents = $repaired; // continue below - } - } - } else { - $repaired = $this->_repairNZB($rawFile, $nzbPath); - if ($repaired !== null) { - $nzbContents = $repaired; - } - } - } - } catch (\Throwable $e) { - // ignore - } - if (! $nzbContents) { - $this->_deleteRelease(); - - return false; - } - } - // Get a list of files in the nzb. - $this->_nzbContents = $this->_nzb->nzbFileList($nzbContents, ['no-file-key' => false, 'strip-count' => true]); - if (\count($this->_nzbContents) === 0) { - // Attempt repair if initial parse yielded no files. - $repaired = $this->_repairNZB($nzbContents, $nzbPath); - if ($repaired !== null) { - $this->_nzbContents = $this->_nzb->nzbFileList($repaired, ['no-file-key' => false, 'strip-count' => true]); - } - if (\count($this->_nzbContents) === 0) { - if ($this->_echoCLI) { - $this->_echo('NZB is potentially broken for GUID: '.$this->_release->guid, 'warning'); - } - $this->_deleteRelease(); - - return false; - } - } - // Sort keys. - ksort($this->_nzbContents, SORT_NATURAL); - - return true; - } - if ($this->_echoCLI) { - $this->_echo('NZB not found for GUID: '.$this->_release->guid, 'warning'); - } - $this->_deleteRelease(); - - return false; - } - - /** - * Attempt to repair a potentially broken NZB XML string. - * Strategies: - * - Strip illegal control characters. - * - Ensure root closing tag exists. - * - Optionally wrap content if root tag missing. - * - Try liberal XML parsing; return repaired content on success. - */ - protected function _repairNZB(string $raw, string $originalPath): ?string - { - $original = $raw; - // Remove common binary / control chars except tab, newline, carriage return. - $fixed = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F]/', '', $raw); - - // If missing opening tag, try to wrap existing segments inside a minimal nzb root. - if (! str_contains(strtolower($fixed), '\n\n".$fixed."\n"; - } else { - // Ensure closing tag. - if (! preg_match('/<\/nzb>\s*$/i', $fixed)) { - $fixed .= "\n"; - } - } - - // Try to parse using libxml recovery. - $opts = LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_COMPACT | LIBXML_NONET | LIBXML_NOCDATA | LIBXML_PARSEHUGE; - $prev = libxml_use_internal_errors(true); - $xml = simplexml_load_string($fixed, 'SimpleXMLElement', $opts); - $errors = libxml_get_errors(); - libxml_clear_errors(); - libxml_use_internal_errors($prev); - - if ($xml === false || empty($xml->file)) { - // Failed; return null so caller can continue normal failure path. - if ($this->_echoCLI) { - $this->_echo('NZB repair failed for GUID: '.$this->_release->guid.' ('.(count($errors)).' XML errors)', 'warning'); - } - - return null; - } - - // If we get here we believe repair succeeded; optionally persist repaired version (gzipped) next to original. - try { - // Only overwrite if we actually changed content. - if ($fixed !== $original) { - // If original was gz (very likely), replace with repaired gz; else plain. - if (str_ends_with(strtolower($originalPath), '.gz')) { - @File::put($originalPath, gzencode($fixed)); - } else { - @File::put($originalPath, $fixed); - } - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::debug('Failed to persist repaired NZB: '.$e->getMessage()); - } - } - - if ($this->_echoCLI) { - $this->_echo('Repaired NZB for GUID: '.$this->_release->guid, 'primaryOver'); - } - - return $fixed; - } - - /** - * The current file we are working on inside a NZB. - */ - protected array $_currentNZBFile; - - /** - * Does the current NZB contain a compressed (RAR/ZIP) file? - */ - protected bool $_NZBHasCompressedFile; - - /** - * Process the files inside the NZB, find Message-ID's to download. - * If we find files with book extensions, return the amount. - */ - protected function _processNZBContents(): int - { - $totalBookFiles = 0; - foreach ($this->_nzbContents as $this->_currentNZBFile) { - try { - // Support / ignore support/nfo files - if (preg_match('/(?:'.$this->_supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $this->_currentNZBFile['title'])) { - continue; - } - // Compressed file detection - if (! $this->_NZBHasCompressedFile && preg_match('/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))("|\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', $this->_currentNZBFile['title'])) { - $this->_NZBHasCompressedFile = true; - } - - // Look for a video sample, make sure it's not an image. - if ($this->_processThumbnails && empty($this->_sampleMessageIDs) && isset($this->_currentNZBFile['segments']) && stripos($this->_currentNZBFile['title'], 'sample') !== false && ! preg_match('/\.jpe?g$/i', $this->_currentNZBFile['title']) - ) { - // Get the amount of segments for this file. - $segCount = (\count($this->_currentNZBFile['segments']) - 1); - // If it's more than 1 try to get up to the site specified value of segments. - for ($i = 0; $i < $this->_segmentsToDownload; $i++) { - if ($i > $segCount) { - break; - } - $this->_sampleMessageIDs[] = (string) $this->_currentNZBFile['segments'][$i]; - } - } - - // Look for a JPG picture, make sure it's not a CD cover. - if ($this->_processJPGSample && empty($this->_JPGMessageIDs) && isset($this->_currentNZBFile['segments']) && ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $this->_releaseGroupName) && preg_match('/\.jpe?g[. ")\]]/i', $this->_currentNZBFile['title']) - ) { - // Get the amount of segments for this file. - $segCount = (\count($this->_currentNZBFile['segments']) - 1); - // If it's more than 1 try to get up to the site specified value of segments. - for ($i = 0; $i < $this->_segmentsToDownload; $i++) { - if ($i > $segCount) { - break; - } - $this->_JPGMessageIDs[] = (string) $this->_currentNZBFile['segments'][$i]; - } - } - - // Look for a video file, make sure it's not a sample, for MediaInfo. - if ($this->_processMediaInfo && empty($this->_MediaInfoMessageIDs) && isset($this->_currentNZBFile['segments'][0]) && stripos($this->_currentNZBFile['title'], 'sample') !== false && preg_match('/'.$this->_videoFileRegex.'[. ")\]]/i', $this->_currentNZBFile['title']) - ) { - $this->_MediaInfoMessageIDs = (string) $this->_currentNZBFile['segments'][0]; - } - - // Look for an audio file. - if ($this->_processAudioInfo && empty($this->_AudioInfoMessageIDs) && isset($this->_currentNZBFile['segments']) && preg_match('/'.$this->_audioFileRegex.'[. ")\]]/i', $this->_currentNZBFile['title'], $type) - ) { - // Get the extension. - $this->_AudioInfoExtension = $type[1]; - $this->_AudioInfoMessageIDs = (string) $this->_currentNZBFile['segments'][0]; - } - - // Some releases contain many books, increment this to ignore them later. - if (preg_match($this->_ignoreBookRegex, $this->_currentNZBFile['title'])) { - $totalBookFiles++; - } - } catch (\ErrorException $e) { - Log::debug($e->getTraceAsString()); - } - } - - return $totalBookFiles; - } - - /** - * List of message-id's we have tried for rar/zip files. - */ - protected array $_triedCompressedMids = []; - - // List entries of a 7z archive using external 7z -slt mode. - private function _listSevenZipEntries(string $compressedData): array - { - if (empty($this->_7zipPath)) { - return []; - } - try { - $tmpFile = $this->tmpPath.uniqid('7zlist_', true).'.7z'; - if (File::put($tmpFile, $compressedData) === false) { - return []; - } - $cmd = [$this->_7zipPath, 'l', '-slt', '-ba', '-bd', $tmpFile]; - $exit = 0; - $out = null; - $err = null; - $ok = $this->_execCommand($cmd, $exit, $out, $err); - if (! $ok || $exit !== 0 || empty($out)) { - // Primary structured listing failed; attempt plain listing fallback. - $exit2 = 0; - $out2 = null; - $err2 = null; - $plainCmd = [$this->_7zipPath, 'l', '-ba', '-bd', $tmpFile]; - $plainOk = $this->_execCommand($plainCmd, $exit2, $out2, $err2); - File::delete($tmpFile); - if (! $plainOk || $exit2 !== 0 || empty($out2)) { - return []; - } - $files = []; - $lines = preg_split('/\r?\n/', trim($out2)); - $seen = 0; - foreach ($lines as $line) { - $line = trim($line); - if ($line === '' || str_starts_with($line, '-----') || str_contains($line, ' Date ') || str_starts_with($line, 'Scanning ') || str_starts_with($line, 'Creating archive')) { - continue; - } - // Typical format: YYYY-MM-DD HH:MM:SS ....A 12345 678 path/filename.ext - if (preg_match('/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\s+\S+\s+\d+\s+\d+\s+(\S.*)$/', $line, $m)) { - $name = $m[1]; - } elseif (preg_match('/^\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2}\s+\S+\s+\d+\s+\d+\s+(\S.*)$/', $line, $m)) { // Locale alt DD.MM.YYYY - $name = $m[1]; - } else { - // Fallback: extract token with extension near end if line contains a dot extension. - if (preg_match('/([A-Za-z0-9_#@()\[\]\-+&., ]+\.[A-Za-z0-9]{2,8})$/', $line, $m2)) { - $name = trim($m2[1]); - } else { - continue; - } - } - $name = trim($name, " \t\r\n"); - if ($name === '' || strlen($name) > 300) { - continue; - } - $files[] = ['name' => $name, 'size' => 0, 'encrypted' => false]; - if (++$seen >= 200) { - break; - } - } - - return $files; - } - // Structured parsing branch. - $blocks = preg_split('/\n\n+/u', trim($out)); - File::delete($tmpFile); - $files = []; - $anyEnc = false; - foreach ($blocks as $blk) { - $lines = preg_split('/\r?\n/', trim($blk)); - $row = []; - foreach ($lines as $line) { - $kv = explode(' = ', $line, 2); - if (count($kv) === 2) { - $row[$kv[0]] = $kv[1]; - } - } - if (empty($row['Path'])) { - continue; - } - $attr = $row['Attributes'] ?? ''; - if (str_contains($attr, 'D')) { - continue; - } // directory - $enc = ($row['Encrypted'] ?? '') === '+'; - if ($enc) { - $anyEnc = true; - } - $size = isset($row['Size']) && ctype_digit($row['Size']) ? (int) $row['Size'] : 0; - $files[] = ['name' => $row['Path'], 'size' => $size, 'encrypted' => $enc]; - if (count($files) >= 200) { - break; - } - } - if ($anyEnc && isset($files[0])) { - $files[0]['__any_encrypted__'] = true; - } - - return $files; - } catch (\Throwable $e) { - if (config('app.debug') === true) { - $this->_debug('Exception listing 7z: '.$e->getMessage()); - } - - return []; - } - } - - /** - * @throws Exception - */ - protected function _processNZBCompressedFiles(bool $reverse = false): void - { - if ($this->_groupUnavailable) { - return; - } - $this->_reverse = $reverse; - if ($this->_reverse) { - if (! krsort($this->_nzbContents)) { - return; - } - } else { - $this->_triedCompressedMids = []; - } - $failed = $downloaded = 0; - // Loop through the files, attempt to find if passworded and files. Starting with what not to process. - foreach ($this->_nzbContents as $nzbFile) { - if ($downloaded >= $this->_maximumRarSegments) { - break; - } - if ($failed >= $this->_maximumRarPasswordChecks) { - break; - } - if ($this->_releaseHasPassword || $this->_groupUnavailable) { - break; - } - // Archive detection - if (! preg_match('/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', $nzbFile['title'])) { - continue; - } - - // Get message-id's for the rar file. - $segCount = (\count($nzbFile['segments']) - 1); - $mID = []; - foreach (range(0, $this->_maximumRarSegments - 1) as $i) { - if ($i > $segCount) { - break; - } - $segment = (string) $nzbFile['segments'][$i]; - if (! $this->_reverse) { - $this->_triedCompressedMids[] = $segment; - } elseif (\in_array($segment, $this->_triedCompressedMids, false)) { - // We already downloaded this file. - continue 2; - } - $mID[] = $segment; - } - // Nothing to download. - if (empty($mID)) { - continue; - } - - // Download the article(s) from usenet. - if ($this->_debugDownloadFailures) { - Log::debug('Attempting compressed fetch', [ - 'release_id' => $this->_release->id ?? null, - 'reverse' => $this->_reverse, - 'file_title' => $nzbFile['title'] ?? '', - 'message_ids' => $mID, - 'group' => $this->_releaseGroupName, - ]); - } - $fetchedBinary = $this->_nntp->getMessagesByMessageID($mID, $this->_alternateNNTP); - // Treat any non-string or blank response as failure. - if (! is_string($fetchedBinary) || $fetchedBinary === '') { - $errorObj = (is_object($fetchedBinary) ? $fetchedBinary : null); - if ($this->_debugDownloadFailures) { - $errMsg = null; - if ($errorObj && method_exists($errorObj, 'getMessage')) { - $errMsg = $errorObj->getMessage(); - } - Log::debug('Compressed fetch failed', [ - 'release_id' => $this->_release->id ?? null, - 'file_title' => $nzbFile['title'] ?? '', - 'message_ids' => $mID, - 'group' => $this->_releaseGroupName, - 'error_object' => $errorObj ? get_class($errorObj) : null, - 'error_message' => $errMsg, - 'raw_type' => gettype($fetchedBinary), - 'length' => is_string($fetchedBinary) ? strlen($fetchedBinary) : 0, - ]); - } - if ($errorObj && method_exists($errorObj, 'getMessage')) { - $msg = $errorObj->getMessage(); - if (stripos($msg, 'No such news group') !== false || stripos($msg, 'Group not found') !== false) { - $this->_groupUnavailable = true; - if ($this->_echoCLI) { - $this->_echo('G', 'warningOver'); // Distinct marker for missing group. - } - break; // Stop processing more compressed files. - } - } - $fetchedBinary = false; - } - - if ($fetchedBinary !== false) { - // Echo we downloaded compressed file. - if ($this->_echoCLI) { - $this->_echo('(cB)', 'primaryOver'); - } - - $downloaded++; - - // Process the compressed file only if it's still a string. - if (is_string($fetchedBinary)) { - $decompressed = $this->_processCompressedData($fetchedBinary); - - if ($decompressed || $this->_releaseHasPassword) { - break; - } - } - } else { - $failed++; - if ($this->_echoCLI) { - $this->_echo('fC'.$failed, 'warningOver'); - } - } - } - } - - /** - * Check if the data is a ZIP / RAR file, extract files, get file info. - * - * - * @throws Exception - */ - protected function _processCompressedData(string &$compressedData): bool - { - $this->_compressedFilesChecked++; - - $earlyType = $this->_detectNonRarZipType($compressedData); - if ($earlyType === '7z') { - $listed = []; - $archiveEncrypted = false; - if (! empty($this->_7zipPath)) { - $listed = $this->_listSevenZipEntries($compressedData); - if (! empty($listed)) { - if (! empty($listed[0]['__any_encrypted__'])) { - $archiveEncrypted = true; - } - if ($archiveEncrypted) { - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - - return false; // don't add filenames when encrypted - } - $now = time(); - foreach ($listed as $entry) { - if (! empty($entry['__any_encrypted__'])) { - continue; - } - $nm = $entry['name']; - if ($nm === '' || strlen($nm) > 300) { - continue; - } - $info = [ - 'name' => $nm, - 'size' => $entry['size'] ?? 0, - 'date' => $now, - 'pass' => 0, - 'crc32' => '', - 'source' => '7z-list', - ]; - // New filtering rule: only consider specific playable/text/image extensions - // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). - $name = $info['name'] ?? ''; - $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); - $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; - if (! in_array($ext, $allowedExt, true)) { - continue; // Skip disallowed extensions (also excludes gz, xz, etc.) - } - $base = pathinfo($name, PATHINFO_FILENAME); - $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); - if ($letterCount <= 5) { - continue; // Require >5 letter characters in base name - } - - $this->_addFileInfo($info); - if ($this->_totalFileInfo >= 50) { - break; - } - } - if ($this->_totalFileInfo > 0) { - if ($this->_echoCLI) { - $this->_echo('7z', 'primaryOver'); - } - - return true; - } - } - } - $addedFromHeader = false; - try { - if (class_exists(__NAMESPACE__.'\\SevenZipPartialParser')) { - $parser = new SevenZipPartialParser($compressedData); - if ($parser->isEncrypted()) { - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - - return false; - } - $hdrNames = $parser->getFileNames(); - if (config('app.debug') === true && ! empty($hdrNames)) { - $this->_debug('7z header names: '.substr(implode(', ', $hdrNames), 0, 300)); - } - if ($parser->hasEncodedHeader() && empty($listed) && config('app.debug') === true) { - $this->_debug('7z encoded header detected; external list unavailable or returned no entries.'); - } - if (! empty($hdrNames) && empty($listed)) { - $now = time(); - $count = 0; - foreach ($hdrNames as $nm) { - if ($nm === '' || strlen($nm) > 300) { - continue; - } - $info = [ - 'name' => $nm, - 'size' => 0, - 'date' => $now, - 'pass' => 0, - 'crc32' => '', - 'source' => '7z-header', - ]; - // New filtering rule: only consider specific playable/text/image extensions - // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). - $name = $info['name'] ?? ''; - $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); - $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; - if (! in_array($ext, $allowedExt, true)) { - continue; // Skip disallowed extensions (also excludes gz, xz, etc.) - } - $base = pathinfo($name, PATHINFO_FILENAME); - $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); - if ($letterCount <= 5) { - continue; // Require >5 letter characters in base name - } - - $this->_addFileInfo($info); - $count++; - if ($count >= 50) { - break; - } - } - if ($this->_totalFileInfo > 0) { - $addedFromHeader = true; - if ($this->_echoCLI) { - $this->_echo('7z', 'primaryOver'); - } - - return true; - } - } - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - $this->_debug('7z header parse failed: '.$e->getMessage()); - } - } - if (! $addedFromHeader && empty($listed)) { - $names = $this->_scanSevenZipFilenames($compressedData); - if (! empty($names)) { - $now = time(); - foreach ($names as $nm) { - if ($nm === '' || strlen($nm) > 300) { - continue; - } - $info = [ - 'name' => $nm, - 'size' => 0, - 'date' => $now, - 'pass' => 0, - 'crc32' => '', - 'source' => '7z-scan', - ]; - // New filtering rule: only consider specific playable/text/image extensions - // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). - $name = $info['name'] ?? ''; - $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); - $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; - if (! in_array($ext, $allowedExt, true)) { - continue; // Skip disallowed extensions (also excludes gz, xz, etc.) - } - $base = pathinfo($name, PATHINFO_FILENAME); - $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); - if ($letterCount <= 5) { - continue; // Require >5 letter characters in base name - } - - $this->_addFileInfo($info); - if ($this->_totalFileInfo >= 50) { - break; - } - } - if ($this->_totalFileInfo > 0) { - if ($this->_echoCLI) { - $this->_echo('7z', 'primaryOver'); - } - - return true; - } - } - } - // continue to extraction attempt below if still nothing - } - if (in_array($earlyType, ['7z', 'gzip', 'bzip2', 'xz'], true)) { - if (! empty($this->_7zipPath)) { - $extMap = ['7z' => '7z', 'gzip' => 'gz', 'bzip2' => 'bz2', 'xz' => 'xz']; - $markerMap = ['7z' => '7z', 'gzip' => 'g', 'bzip2' => 'b', 'xz' => 'x']; - $ext = $extMap[$earlyType]; - $marker = $markerMap[$earlyType]; - if ($this->_extractViaSevenZip($compressedData, $ext, $earlyType, $marker)) { - return true; - } - } - } - - if (! $this->_archiveInfo->setData($compressedData, true)) { - $otherType = $earlyType ?? $this->_detectNonRarZipType($compressedData); - if (in_array($otherType, ['7z', 'gzip', 'bzip2', 'xz'], true) && ! empty($this->_7zipPath)) { - $extMap = [ - '7z' => '7z', - 'gzip' => 'gz', - 'bzip2' => 'bz2', - 'xz' => 'xz', - ]; - $markerMap = [ - '7z' => '7z', - 'gzip' => 'g', - 'bzip2' => 'b', - 'xz' => 'x', - ]; - $ext = $extMap[$otherType]; - $marker = $markerMap[$otherType]; - if ($this->_extractViaSevenZip($compressedData, $ext, $otherType, $marker)) { - return true; - } - } - // Standalone video (raw first segments) detection when not recognized as archive. - $videoType = $this->_detectStandaloneVideo($compressedData); - if ($videoType !== null) { - try { - $extMap = ['avi' => 'avi', 'mkv' => 'mkv', 'mpg' => 'mpg', 'mp4' => 'mp4']; - $ext = $extMap[$videoType] ?? 'dat'; - $fileLocation = $this->tmpPath.'inline_video_'.uniqid('', true).'.'.$ext; - if (File::put($fileLocation, $compressedData) !== false) { - if ($this->_echoCLI) { - $this->_echo('(vRAW)', 'primaryOver'); - } - if (! $this->_foundMediaInfo) { - $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); - } - if (! $this->_foundSample) { - $this->_foundSample = $this->_getSample($fileLocation); - } - if (! $this->_foundVideo) { - $this->_foundVideo = $this->_getVideo($fileLocation); - } - - return $this->_foundMediaInfo || $this->_foundSample || $this->_foundVideo; - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::debug('Standalone video handling failed: '.$e->getMessage()); - } - } - } - if (config('app.debug') === true) { - $alt = $otherType !== null ? 'Probable alternate type: '.$otherType.'.' : ''; - $sevenZip = ! empty($this->_7zipPath) ? '' : '7zip binary NOT configured; extended formats disabled.'; - $this->_debug(sprintf( - 'Unrecognized compressed data. %s %s', - $alt, - $sevenZip - )); - } - - return false; - } - - if ($this->_archiveInfo->error !== '') { - if (config('app.debug') === true) { - $this->_debug('ArchiveInfo Error: '.$this->_archiveInfo->error); - } - - return false; - } - - try { - $dataSummary = $this->_archiveInfo->getSummary(true); - } catch (Exception $exception) { - if (config('app.debug') === true) { - Log::warning($exception->getTraceAsString()); - } - - return false; - } - - if (! $this->_extractUsingRarInfo) { - try { - if ($this->_unrarPath !== false) { - $unrarDir = $this->tmpPath.'unrar/'; - if (! File::isDirectory($unrarDir)) { - File::makeDirectory($unrarDir, 0777, true, true); - } - } - $unzipDir = $this->tmpPath.'unzip/'; - if (! File::isDirectory($unzipDir)) { - File::makeDirectory($unzipDir, 0777, true, true); - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::warning('Failed ensuring extraction subdirectories: '.$e->getMessage()); - } - } - } - - if (! empty($this->_archiveInfo->isEncrypted) || (isset($dataSummary['is_encrypted']) && (int) $dataSummary['is_encrypted'] !== 0)) { - if (config('app.debug') === true) { - $this->_debug('ArchiveInfo: Compressed file has a password.'); - } - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - - return false; - } - - if ($this->_reverse) { - $fileData = $dataSummary['file_list'] ?? []; - if (! empty($fileData)) { - $rarFileName = Arr::pluck($fileData, 'name'); - $extractedName = $this->extractReleaseNameFromFile($rarFileName[0]); - if ($extractedName !== null) { - $preCheck = Predb::whereTitle($extractedName)->first(); - $this->_release->preid = $preCheck !== null ? $preCheck->value('id') : 0; - $candidate = $preCheck->title ?? $extractedName; - $candidate = $this->normalizeCandidateTitle($candidate); - if ($this->isPlausibleReleaseTitle($candidate)) { - (new NameFixer)->updateRelease($this->_release, $candidate, 'RarInfo FileName Match', true, 'Filenames, ', true, true, $this->_release->preid); - } elseif (config('app.debug') === true) { - $this->_debug('RarInfo: Ignored low-quality candidate "'.$candidate.'" from inner file name.'); - } - } elseif (! empty($dataSummary['archives']) && ! empty($dataSummary['archives'][$rarFileName[0]]['file_list'])) { - $archiveData = $dataSummary['archives'][$rarFileName[0]]['file_list']; - $archiveFileName = Arr::pluck($archiveData, 'name'); - $extractedName = $this->extractReleaseNameFromFile($archiveFileName[0]); - if ($extractedName !== null) { - $preCheck = Predb::whereTitle($extractedName)->first(); - $this->_release->preid = $preCheck !== null ? $preCheck->value('id') : 0; - $candidate = $preCheck->title ?? $extractedName; - $candidate = $this->normalizeCandidateTitle($candidate); - if ($this->isPlausibleReleaseTitle($candidate)) { - (new NameFixer)->updateRelease($this->_release, $candidate, 'RarInfo FileName Match', true, 'Filenames, ', true, true, $this->_release->preid); - } elseif (config('app.debug') === true) { - $this->_debug('RarInfo: Ignored low-quality candidate "'.$candidate.'" from nested archive file name.'); - } - } - } - } - } - - switch ($dataSummary['main_type']) { - case ArchiveInfo::TYPE_RAR: - if ($this->_echoCLI) { - $this->_echo('r', 'primaryOver'); - } - if (! $this->_extractUsingRarInfo && $this->_unrarPath !== false) { - $fileName = $this->tmpPath.uniqid('', true).'.rar'; - File::put($fileName, $compressedData); - runCmd($this->_killString.$this->_unrarPath.'" e -ai -ep -c- -id -inul -kb -or -p- -r -y "'.$fileName.'" "'.$this->tmpPath.'unrar/"'); - File::delete($fileName); - } - break; - case ArchiveInfo::TYPE_ZIP: - if ($this->_echoCLI) { - $this->_echo('z', 'primaryOver'); - } - if (! $this->_extractUsingRarInfo) { - $fileName = $this->tmpPath.uniqid('', true).'.zip'; - File::put($fileName, $compressedData); - runCmd($this->_unzipPath.' -o "'.$fileName.'" -d "'.$this->tmpPath.'unzip/"'); - File::delete($fileName); - } - break; - default: - return false; - } - - return $this->_processCompressedFileList(); - } - - /** - * Get a list of files inside the Compressed file. - * - * - * @throws Exception - */ - protected function _processCompressedFileList(): bool - { - // Get a list of files inside the Compressed file. - $files = $this->_archiveInfo->getArchiveFileList(); - if (! \is_array($files) || \count($files) === 0) { - return false; - } - - // Loop through the files. - foreach ($files as $file) { - if ($this->_releaseHasPassword) { - break; - } - - if (isset($file['name'])) { - if (isset($file['error'])) { - if (config('app.debug') === true) { - $this->_debug("Error: {$file['error']} (in: {$file['source']})"); - } - - continue; - } - - if (isset($file['pass']) && $file['pass'] === true) { - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - break; - } - - if ($this->_innerFileBlacklist !== false && preg_match($this->_innerFileBlacklist, $file['name'])) { - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - break; - } - - $fileName = []; - if (preg_match('/[^\/\\\\]*\.[a-zA-Z0-9]*$/', $file['name'], $fileName)) { - $fileName = $fileName[0]; - } else { - $fileName = ''; - } - - if ($this->_extractUsingRarInfo) { - // Extract files from the rar. - if (isset($file['compressed']) && (int) $file['compressed'] === 0) { - File::put( - $this->tmpPath.random_int(10, 999999).'_'.$fileName, - $this->_archiveInfo->getFileData($file['name'], $file['source']) - ); - } // If the files are compressed, use a binary extractor. - else { - $this->_archiveInfo->extractFile($file['name'], $this->tmpPath.random_int(10, 999999).'_'.$fileName); - } - } - } - $this->_addFileInfo($file); - } - if ($this->_addedFileInfo > 0) { - if ($this->elasticsearchEnabled) { - $this->elasticsearch()->updateRelease($this->_release->id); - } else { - $this->manticore()->updateRelease($this->_release->id); - } - } - - return $this->_totalFileInfo > 0; - } - - /** - * @throws Exception - */ - protected function _addFileInfo(&$file): void - { - if (! isset($file['error']) && isset($file['source']) && - ! preg_match('/(?:'.$this->_supportFileRegex.'|part\d+|[rz]\d{1,3}|zipr\d{2,3}|\d{2,3}|zipx?|zip|rar|7z|gz|bz2|xz)(\s*\.rar)?$/i', $file['name']) - ) { - // Cache the amount of files we find in the RAR or ZIP, return this to say we did find RAR or ZIP content. - // This is so we don't download more RAR or ZIP files for no reason. - $this->_totalFileInfo++; - - /* Check if we already have the file or not. - * Also make sure we don't add too many files, some releases have 100's of files, like PS3 releases. - */ - if ($this->_addedFileInfo < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name'], 'size' => $file['size']])->first() === null) { - $addReleaseFiles = ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $file['date'], $file['pass'], '', $file['crc32'] ?? ''); - if (! empty($addReleaseFiles)) { - $this->_addedFileInfo++; - - if ($this->_echoCLI) { - $this->_echo('^', 'primaryOver'); - } - - // Check for "codec spam" - // Previous pattern '/[\/\\]Codec[\/\\]Setup\.exe$/i' caused a runtime compilation error in some PCRE builds. - if (preg_match('#(?:^|[\\/])Codec[\\/]Setup\.exe$#i', $file['name'])) { - if (config('app.debug') === true) { - $this->_debug('Codec spam found, setting release to potentially passworded.'); - } - $this->_releaseHasPassword = true; - $this->_passwordStatus = Releases::PASSWD_RAR; - } // Run a PreDB filename check on insert to try and match the release - elseif ($file['name'] !== '' && ! str_starts_with($file['name'], '.')) { - $this->_release['filename'] = $file['name']; - $this->_release['releases_id'] = $this->_release->id; - $this->_nameFixer->matchPreDbFiles($this->_release, 1, 1, true); - } - } - } - } - } - - /** - * Go through all the extracted files in the temp folder and process them. - * - * @throws Exception - */ - protected function _processExtractedFiles(): void - { - $nestedLevels = 0; - while ($nestedLevels < $this->_maxNestedLevels) { - if ($this->_compressedFilesChecked >= self::maxCompressedFilesToCheck) { - break; - } - $foundCompressedFile = false; - $files = $this->_getTempDirectoryContents('/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1|7z(?:\.\d{3})?|(?:tar\.)?(?:gz|bz2|xz))($|[^a-z0-9])/i'); - if (! empty($files)) { - foreach ($files as $file) { - // Check if the file exists. - if (File::isFile($file[0])) { - $rarData = @File::get($file[0]); - if (! empty($rarData)) { - $this->_processCompressedData($rarData); - $foundCompressedFile = true; - } - File::delete($file[0]); - } - } - } - - // If we found no compressed files, break out. - if (! $foundCompressedFile) { - break; - } - - $nestedLevels++; - } - - $fileType = []; - - // Get all the remaining files in the temp dir. - $files = $this->_getTempDirectoryContents(); - if (! empty($files)) { - foreach ($files as $file) { - $file = $file->getPathname(); - - // Skip /. and /.. - if (preg_match('/[\/\\\\]\.{1,2}$/', $file)) { - continue; - } - - if (File::isFile($file)) { - // Process PAR2 files. - if (! $this->_foundPAR2Info && preg_match('/\.par2$/', $file)) { - $this->_siftPAR2Info($file); - } // Process NFO files. - elseif ($this->_releaseHasNoNFO && preg_match('/(\.(nfo|inf|ofn)|info\.txt)$/i', $file)) { - $this->_processNfoFile($file); - } // Process audio files. - elseif ((! $this->_foundAudioInfo || ! $this->_foundAudioSample) && preg_match('/(.*)'.$this->_audioFileRegex.'$/i', $file, $fileType)) { - // Try to get audio sample/audio media info. - File::move($file, $this->tmpPath.'audiofile.'.$fileType[2]); - $this->_getAudioInfo($this->tmpPath.'audiofile.'.$fileType[2], $fileType[2]); - File::delete($this->tmpPath.'audiofile.'.$fileType[2]); - } // Process JPG files. - elseif (! $this->_foundJPGSample && preg_match('/\.jpe?g$/i', $file)) { - $this->_getJPGSample($file); - File::delete($file); - } // Video sample // video clip // video media info. - elseif ((! $this->_foundSample || ! $this->_foundVideo || ! $this->_foundMediaInfo) && preg_match('/(.*)'.$this->_videoFileRegex.'$/i', $file)) { - $this->_processVideoFile($file); - } // Check file's magic info. - else { - $output = Utility::fileInfo($file); - if (! empty($output)) { - switch (true) { - case ! $this->_foundJPGSample && preg_match('/^JPE?G/i', $output): - $this->_getJPGSample($file); - File::delete($file); - break; - - case (! $this->_foundMediaInfo || ! $this->_foundSample || ! $this->_foundVideo) && preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output): - $this->_processVideoFile($file); - break; - - case (! $this->_foundAudioSample || ! $this->_foundAudioInfo) && preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $fileType): - switch ($fileType[0]) { - case 'FLAC': - $fileType = 'FLAC'; - break; - case 'layer III': - $fileType = 'MP3'; - break; - case 'Vorbis audio': - $fileType = 'OGG'; - break; - } - File::move($file, $this->tmpPath.'audiofile.'.$fileType); - $this->_getAudioInfo($this->tmpPath.'audiofile.'.$fileType, $fileType); - File::delete($this->tmpPath.'audiofile.'.$fileType); - break; - - case ! $this->_foundPAR2Info && stripos($output, 'Parity') === 0: - $this->_siftPAR2Info($file); - break; - } - } - } - } - } - } - } - - /** - * Download all binaries from usenet and form samples / get media info / etc from them. - * - * @void - * - * @throws Exception - */ - protected function _processMessageIDDownloads(): void - { - $this->_processSampleMessageIDs(); - $this->_processMediaInfoMessageIDs(); - $this->_processAudioInfoMessageIDs(); - $this->_processJPGMessageIDs(); - } - - /** - * Download and process binaries for sample videos. - * - * @void - * - * @throws Exception - */ - protected function _processSampleMessageIDs(): void - { - // Download and process sample image. - if (! $this->_foundSample || ! $this->_foundVideo) { - if (! empty($this->_sampleMessageIDs)) { - // Download it from usenet. - $sampleBinary = $this->_nntp->getMessagesByMessageID($this->_sampleMessageIDs, $this->_alternateNNTP); - if (! is_string($sampleBinary) || $sampleBinary === '') { - if ($this->_debugDownloadFailures) { - $errMsg = null; - if (is_object($sampleBinary) && method_exists($sampleBinary, 'getMessage')) { - $errMsg = $sampleBinary->getMessage(); - } - Log::debug('Sample fetch failed', [ - 'release_id' => $this->_release->id ?? null, - 'message_ids' => $this->_sampleMessageIDs, - 'group' => $this->_releaseGroupName, - 'error_object' => is_object($sampleBinary) ? get_class($sampleBinary) : null, - 'error_message' => $errMsg, - 'raw_type' => gettype($sampleBinary), - 'length' => is_string($sampleBinary) ? strlen($sampleBinary) : 0, - ]); - } - $sampleBinary = false; - } - if ($sampleBinary !== false) { - if ($this->_echoCLI) { - $this->_echo('(sB)', 'primaryOver'); - } - - // Check if it's more than 40 bytes. - if (is_string($sampleBinary) && \strlen($sampleBinary) > 40) { - $fileLocation = $this->tmpPath.'sample_'.random_int(0, 99999).'.avi'; - // Try to create the file. - File::put($fileLocation, $sampleBinary); - - // Try to get a sample picture. - if (! $this->_foundSample) { - $this->_foundSample = $this->_getSample($fileLocation); - } - - // Try to get a sample video. - if (! $this->_foundVideo) { - $this->_foundVideo = $this->_getVideo($fileLocation); - } - } - } elseif ($this->_echoCLI) { - $this->_echo('fS', 'warningOver'); - } - } - } - } - - /** - * Download and process binaries for media info from videos. - * - * @void - * - * @throws Exception - */ - protected function _processMediaInfoMessageIDs(): void - { - // Download and process mediainfo. Also try to get a sample if we didn't get one yet. - if (! $this->_foundMediaInfo || ! $this->_foundSample || ! $this->_foundVideo) { - if (! empty($this->_MediaInfoMessageIDs)) { - // Try to download it from usenet. - $mediaBinary = $this->_nntp->getMessagesByMessageID($this->_MediaInfoMessageIDs, $this->_alternateNNTP); - if (! is_string($mediaBinary) || $mediaBinary === '') { - if ($this->_debugDownloadFailures) { - $errMsg = null; - if (is_object($mediaBinary) && method_exists($mediaBinary, 'getMessage')) { - $errMsg = $mediaBinary->getMessage(); - } - Log::debug('MediaInfo fetch failed', [ - 'release_id' => $this->_release->id ?? null, - 'message_id' => $this->_MediaInfoMessageIDs, - 'group' => $this->_releaseGroupName, - 'error_object' => is_object($mediaBinary) ? get_class($mediaBinary) : null, - 'error_message' => $errMsg, - 'raw_type' => gettype($mediaBinary), - ]); - } - $mediaBinary = false; - } - if ($mediaBinary !== false) { - if ($this->_echoCLI) { - $this->_echo('(mB)', 'primaryOver'); - } - - // If it's more than 40 bytes... - if (is_string($mediaBinary) && \strlen($mediaBinary) > 40) { - $fileLocation = $this->tmpPath.'media.avi'; - // Create a file on the disk with it. - File::put($fileLocation, $mediaBinary); - - // Try to get media info. - if (! $this->_foundMediaInfo) { - $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); - } - - // Try to get a sample picture. - if (! $this->_foundSample) { - $this->_foundSample = $this->_getSample($fileLocation); - } - - // Try to get a sample video. - if (! $this->_foundVideo) { - $this->_foundVideo = $this->_getVideo($fileLocation); - } - } - } elseif ($this->_echoCLI) { - $this->_echo('fM', 'warningOver'); - } - } - } - } - - /** - * Download and process binaries for media info from songs. - * - * @void - * - * @throws Exception - */ - protected function _processAudioInfoMessageIDs(): void - { - // Download an audio file, use media info to try to get the artist / album. - if (! $this->_foundAudioInfo || ! $this->_foundAudioSample) { - if (! empty($this->_AudioInfoMessageIDs)) { - // Try to download it from usenet. - $audioBinary = $this->_nntp->getMessagesByMessageID($this->_AudioInfoMessageIDs, $this->_alternateNNTP); - if (! is_string($audioBinary) || $audioBinary === '') { - if ($this->_debugDownloadFailures) { - $errMsg = null; - if (is_object($audioBinary) && method_exists($audioBinary, 'getMessage')) { - $errMsg = $audioBinary->getMessage(); - } - Log::debug('Audio fetch failed', [ - 'release_id' => $this->_release->id ?? null, - 'message_id' => $this->_AudioInfoMessageIDs, - 'group' => $this->_releaseGroupName, - 'error_object' => is_object($audioBinary) ? get_class($audioBinary) : null, - 'error_message' => $errMsg, - 'raw_type' => gettype($audioBinary), - ]); - } - $audioBinary = false; - } - if ($audioBinary !== false) { - if ($this->_echoCLI) { - $this->_echo('(aB)', 'primaryOver'); - } - - $fileLocation = $this->tmpPath.'audio.'.$this->_AudioInfoExtension; - // Create a file with it. - File::put($fileLocation, $audioBinary); - - // Try to get media info / sample of the audio file. - $this->_getAudioInfo($fileLocation, $this->_AudioInfoExtension); - } elseif ($this->_echoCLI) { - $this->_echo('fA', 'warningOver'); - } - } - } - } - - /** - * Download and process binaries for JPG pictures. - * - * @void - * - * @throws Exception - */ - protected function _processJPGMessageIDs(): void - { - // Download JPG file. - if (! $this->_foundJPGSample && ! empty($this->_JPGMessageIDs)) { - // Try to download it. - $jpgBinary = $this->_nntp->getMessagesByMessageID($this->_JPGMessageIDs, $this->_alternateNNTP); - if (! is_string($jpgBinary) || $jpgBinary === '') { - if ($this->_debugDownloadFailures) { - $errMsg = null; - if (is_object($jpgBinary) && method_exists($jpgBinary, 'getMessage')) { - $errMsg = $jpgBinary->getMessage(); - } - Log::debug('JPG fetch failed', [ - 'release_id' => $this->_release->id ?? null, - 'message_ids' => $this->_JPGMessageIDs, - 'group' => $this->_releaseGroupName, - 'error_object' => is_object($jpgBinary) ? get_class($jpgBinary) : null, - 'error_message' => $errMsg, - 'raw_type' => gettype($jpgBinary), - 'length' => is_string($jpgBinary) ? strlen($jpgBinary) : 0, - ]); - } - $jpgBinary = false; - } - - if ($jpgBinary !== false) { - if ($this->_echoCLI) { - $this->_echo('(jB)', 'primaryOver'); - } - - // Try to create a file with it. - File::put($this->tmpPath.'samplepicture.jpg', $jpgBinary); - - // Check if image exif data is jpeg - if (exif_imagetype($this->tmpPath.'samplepicture.jpg') === IMAGETYPE_JPEG) { - // Try to resize and move it. - $this->_foundJPGSample = $this->_releaseImage->saveImage($this->_release->guid.'_thumb', - $this->tmpPath.'samplepicture.jpg', $this->_releaseImage->jpgSavePath, 650, 650) === 1; - - if ($this->_foundJPGSample) { - // Update the DB to say we got it. - Release::query()->where('id', $this->_release->id)->update(['jpgstatus' => 1]); - - if ($this->_echoCLI) { - $this->_echo('j', 'primaryOver'); - } - } - - File::delete($this->tmpPath.'samplepicture.jpg'); - } - } elseif ($this->_echoCLI) { - $this->_echo('fJ', 'warningOver'); - } - } - } - - /** - * Update the release to say we processed it. - */ - protected function _finalizeRelease(): void - { - $updateRows = ['haspreview' => 0]; - - // If samples exist from previous runs, set flags. - if (File::isFile($this->_releaseImage->imgSavePath.$this->_release->guid.'_thumb.jpg')) { - $updateRows = ['haspreview' => 1]; - } - - if (File::isFile($this->_releaseImage->vidSavePath.$this->_release->guid.'.ogv')) { - $updateRows += ['videostatus' => 1]; - } - - if (File::isFile($this->_releaseImage->jpgSavePath.$this->_release->guid.'_thumb.jpg')) { - $updateRows += ['jpgstatus' => 1]; - } - - // Get the number of files we found inside the RAR/ZIPs. - $releaseFilesCount = ReleaseFile::whereReleasesId($this->_release->id)->count('releases_id'); - - if ($releaseFilesCount === null) { - $releaseFilesCount = 0; - } - - $this->_passwordStatus = max([$this->_passwordStatus]); - - // Set the release to no password if password processing is off. - if (! $this->_processPasswords) { - $this->_releaseHasPassword = false; - } - - // If we failed to get anything from the RAR/ZIPs update the release with what we have, if the rar/zip has no password. - if (! $this->_releaseHasPassword && $this->_NZBHasCompressedFile && $releaseFilesCount === 0) { - $release = Release::query()->where('id', $this->_release->id); - $release->update( - $updateRows - ); - } // Else update the release with the password status (if the admin enabled the setting). - else { - $updateRows += ['passwordstatus' => $this->_processPasswords ? $this->_passwordStatus : Releases::PASSWD_NONE, - 'rarinnerfilecount' => $releaseFilesCount, ]; - Release::query()->where('id', $this->_release->id)->update( - $updateRows - ); - } - } - - /** - * Get the contents of a directory. - */ - protected function _getTempDirectoryContents(string $pattern = '', string $path = ''): array - { - $path = $path ?: $this->tmpPath; - - // Attempt to (re)create the directory if it vanished (external cleanup, race condition, etc). - if ($path !== '' && ! File::isDirectory($path)) { - try { - File::makeDirectory($path, 0777, true, true); - } catch (\Throwable $e) { - // Ignore here; will be handled below when trying to read. - } - } - - try { - $files = File::allFiles($path); - } catch (\Throwable $e) { - // Second chance: directory might have disappeared between existence check and allFiles call. - if ($path !== '' && ! File::isDirectory($path)) { - try { - File::makeDirectory($path, 0777, true, true); - $files = File::allFiles($path); // retry once - } catch (\Throwable $e2) { - if (config('app.debug') === true) { - Log::error($e2->getTraceAsString()); - } - throw new \RuntimeException('ERROR: Could not open temp dir: '.$e2->getMessage()); - } - } else { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - throw new \RuntimeException('ERROR: Could not open temp dir: '.$e->getMessage()); - } - } - - if ($pattern !== '') { - $filteredFiles = []; - foreach ($files as $file) { - if (preg_match($pattern, $file->getRelativePathname())) { - $filteredFiles[] = $file->getRelativePathname(); - } - } - - return $filteredFiles; - } - - return $files; - } - - /** - * @throws Exception - */ - protected function _getAudioInfo($fileLocation, $fileExtension): bool - { - // Return values. - $retVal = $audVal = false; - - // Check if audio sample fetching is on. - if (! $this->_processAudioSample) { - $audVal = true; - } - - // Check if media info fetching is on. - if (! $this->_processAudioInfo) { - $retVal = true; - } - - $rQuery = Release::query()->where('proc_pp', '=', 0)->where('id', $this->_release->id)->select(['searchname', 'fromname', 'categories_id'])->first(); - - $musicParent = (string) Category::MUSIC_ROOT; - if ($rQuery === null || ! preg_match( - sprintf( - '/%d\d{3}|%d|%d|%d/', - $musicParent[0], - Category::OTHER_MISC, - Category::MOVIE_OTHER, - Category::TV_OTHER - ), - $rQuery->id - ) - ) { - return false; - } - - if (File::isFile($fileLocation)) { - // Check if media info is enabled. - if (! $retVal) { - // Get the media info for the file. - try { - $xmlArray = $this->mediaInfo()->getInfo($fileLocation, false); - - if ($xmlArray !== null) { - foreach ($xmlArray->getAudios() as $track) { - if ($track->get('album') !== null && $track->get('performer') !== null) { - if ((int) $this->_release->predb_id === 0 && config('nntmux.rename_music_mediainfo')) { - // Make the extension upper case. - $ext = strtoupper($fileExtension); - - // Form a new search name. - if (! empty($track->get('recorded_date')) && preg_match('/(?:19|20)\d\d/', $track->get('recorded_date')->getFullname, $Year)) { - $newName = $track->get('performer')->getFullName().' - '.$track->get('album')->getFullName().' ('.$Year[0].') '.$ext; - } else { - $newName = $track->get('performer')->getFullName().' - '.$track->get('album')->getFullName().' '.$ext; - } - - // Get the category or try to determine it. - if ($ext === 'MP3') { - $newCat = Category::MUSIC_MP3; - } elseif ($ext === 'FLAC') { - $newCat = Category::MUSIC_LOSSLESS; - } else { - $newCat = $this->_categorize->determineCategory($rQuery->groups_id, $newName, $rQuery->fromname); - } - - $newTitle = escapeString(substr($newName, 0, 255)); - // Update the search name. - $release = Release::whereId($this->_release->id); - $release->update(['searchname' => $newTitle, 'categories_id' => $newCat['categories_id'], 'iscategorized' => 1, 'isrenamed' => 1, 'proc_pp' => 1]); - - if ($this->elasticsearchEnabled) { - $this->elasticsearch()->updateRelease($this->_release->id); - } else { - $this->manticore()->updateRelease($this->_release->id); - } - - // Echo the changed name. - if ($this->_echoCLI) { - NameFixer::echoChangedReleaseName( - [ - 'new_name' => $newTitle, - 'old_name' => $rQuery->searchname, - 'new_category' => $newCat, - 'old_category' => $rQuery->id, - 'group' => $rQuery->groups_id, - 'releases_id' => $this->_release->id, - 'method' => 'ProcessAdditional->_getAudioInfo', - ] - ); - } - } - - // Add the media info. - $this->_releaseExtra->addFromXml($this->_release->id, $xmlArray); - - $retVal = true; - $this->_foundAudioInfo = true; - if ($this->_echoCLI) { - $this->_echo('a', 'primaryOver'); - } - break; - } - } - } - } catch (\RuntimeException $e) { - Log::debug($e->getMessage()); - } catch (\TypeError $e) { - Log::debug($e->getMessage()); - } - } - - // Check if creating audio samples is enabled. - if (! $audVal) { - // File name to store an audio file. - $audioFileName = ($this->_release->guid.'.ogg'); - - // Create an audio sample. - if ($this->ffprobe()->isValid($fileLocation)) { - try { - $audioSample = $this->ffmpeg()->open($fileLocation); - $format = new Vorbis; - $audioSample->clip(TimeCode::fromSeconds(30), TimeCode::fromSeconds(30)); - $audioSample->save($format, $this->tmpPath.$audioFileName); - } catch (\InvalidArgumentException $e) { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) - } - } - - // Check if the new file was created. - if (File::isFile($this->tmpPath.$audioFileName)) { - // Try to move the temp audio file. - $renamed = File::move($this->tmpPath.$audioFileName, $this->_audioSavePath.$audioFileName); - - if (! $renamed) { - // Try to copy it if it fails. - $copied = File::copy($this->tmpPath.$audioFileName, $this->_audioSavePath.$audioFileName); - - // Delete the old file. - File::delete($this->tmpPath.$audioFileName); - - // If it didn't copy continue. - if (! $copied) { - return false; - } - } - - // Try to set the file perms. - @chmod($this->_audioSavePath.$audioFileName, 0764); - - // Update DB to said we got a audio sample. - Release::query()->where('id', $this->_release->id)->update(['audiostatus' => 1]); - - $audVal = $this->_foundAudioSample = true; - - if ($this->_echoCLI) { - $this->_echo('A', 'primaryOver'); - } - } - } - } - - return $retVal && $audVal; - } - - /** - * Try to get a JPG picture, resize it and store it on disk. - */ - protected function _getJPGSample(string $fileLocation): void - { - // Try to resize/move the image. - $this->_foundJPGSample = ( - $this->_releaseImage->saveImage( - $this->_release->guid.'_thumb', - $fileLocation, - $this->_releaseImage->jpgSavePath, - 650, - 650 - ) === 1 - ); - - // If it's successful, tell the DB. - if ($this->_foundJPGSample) { - Release::query()->where('id', $this->_release->id)->update(['jpgstatus' => 1]); - } - } - - private function getVideoTime(string $videoLocation): string - { - // Get the real duration of the file. - if ($this->ffprobe()->isValid($videoLocation)) { - $time = $this->ffprobe()->format($videoLocation)->get('duration'); - } - - if (empty($time) || ! preg_match('/time=(\d{1,2}:\d{1,2}:)?(\d{1,2})\.(\d{1,2})\s*bitrate=/i', $time, $numbers)) { - return ''; - } - - // Reduce the last number by 1; this is to make sure we don't ask avconv/ffmpeg for non-existing data. - if ($numbers[3] > 0) { - $numbers[3]--; - } elseif ($numbers[1] > 0) { - $numbers[2]--; - $numbers[3] = '99'; - } - - // Manually pad the numbers in case they are 1 number. to get 02, for example, instead of 2. - return '00:00:'.str_pad($numbers[2], 2, '0', STR_PAD_LEFT).'.'.str_pad($numbers[3], 2, '0', STR_PAD_LEFT); - } - - /** - * @throws Exception - */ - protected function _getSample(string $fileLocation): bool - { - if (! $this->_processThumbnails) { - return false; - } - - if (File::isFile($fileLocation)) { - // Create a path to a temp file. - $fileName = ($this->tmpPath.'zzzz'.random_int(5, 12).random_int(5, 12).'.jpg'); - - $time = $this->getVideoTime($fileLocation); - - // Create the image. - if ($this->ffprobe()->isValid($fileLocation)) { - try { - $this->ffmpeg()->open($fileLocation)->frame(TimeCode::fromString($time === '' ? '00:00:03:00' : $time))->save($fileName); - } catch (\RuntimeException $runtimeException) { - if (config('app.debug') === true) { - Log::error($runtimeException->getTraceAsString()); - } - // We show no error we just log it, we failed to save the frame and move on - } catch (\InvalidArgumentException $e) { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - // Again we do nothing, we just want to catch the error - } - } - - // Check if the file exists. - if (File::isFile($fileName)) { - // Try to resize/move the image. - $saved = $this->_releaseImage->saveImage( - $this->_release->guid.'_thumb', - $fileName, - $this->_releaseImage->imgSavePath, - 800, - 600 - ); - - // Delete the temp file we created. - File::delete($fileName); - - // Check if it saved. - if ($saved === 1) { - if ($this->_echoCLI) { - $this->_echo('s', 'primaryOver'); - } - - return true; - } - } - } - - return false; - } - - /** - * @throws Exception - */ - protected function _getVideo(string $fileLocation): bool - { - if (! $this->_processVideo) { - return false; - } - - // Try to find an avi file. - if (File::isFile($fileLocation)) { - // Create a filename to store the temp file. - $fileName = ($this->tmpPath.'zzzz'.$this->_release->guid.'.ogv'); - - $newMethod = false; - // If the wanted sample length is less than 60, try to get a sample from the end of the video. - if ($this->_ffMPEGDuration < 60) { - // Get the real duration of the file. - $time = $this->getVideoTime($fileLocation); - - if ($time !== '' && preg_match('/(\d{2}).(\d{2})/', $time, $numbers)) { - $newMethod = true; - // Get the lowest time we can start making the video at based on how many seconds the admin wants the video to be. - if ($numbers[1] <= $this->_ffMPEGDuration) { - // If the clip is shorter than the length we want. - // The lowest we want is 0. - $lowestLength = '00:00:00.00'; - } else { - // If the clip is longer than the length we want. - // The lowest we want is the difference between the max video length and our wanted total time. - $lowestLength = ($numbers[1] - $this->_ffMPEGDuration); - // Form the time string. - $end = '.'.$numbers[2]; - $lowestLength = match (\strlen($lowestLength)) { - 1 => ('00:00:0'.$lowestLength.$end), - 2 => ('00:00:'.$lowestLength.$end), - default => '00:00:60.00', - }; - } - - // Try to get the sample (from the end instead of the start). - if ($this->ffprobe()->isValid($fileLocation)) { - try { - $video = $this->ffmpeg()->open($fileLocation); - $videoSample = $video->clip(TimeCode::fromString($lowestLength), TimeCode::fromSeconds($this->_ffMPEGDuration)); - $format = new Ogg; - $format->setAudioCodec('libvorbis'); - $videoSample->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); - $videoSample->save($format, $fileName); - } catch (\InvalidArgumentException $e) { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) - } - } - } - } - - // If longer than 60 or we could not get the video length, run the old way. - if (! $newMethod && $this->ffprobe()->isValid($fileLocation)) { - try { - $video = $this->ffmpeg()->open($fileLocation); - $videoSample = $video->clip(TimeCode::fromSeconds(0), TimeCode::fromSeconds($this->_ffMPEGDuration)); - $format = new Ogg; - $format->setAudioCodec('libvorbis'); - $videoSample->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); - $videoSample->save($format, $fileName); - } catch (\InvalidArgumentException $e) { - if (config('app.debug') === true) { - Log::error($e->getTraceAsString()); - } - // We do nothing, just prevent displaying errors because the file cannot be open (a corrupted or incomplete file) - } - } - - // Until we find the video file. - if (File::isFile($fileName)) { - // Create a path to where the file should be moved. - $newFile = ($this->_releaseImage->vidSavePath.$this->_release->guid.'.ogv'); - - // Try to move the file to the new path. - // If we couldn't rename it, try to copy it. - if (! @File::move($fileName, $newFile)) { - $copied = @File::copy($fileName, $newFile); - - // Delete the old file. - File::delete($fileName); - - // If it didn't copy, continue. - if (! $copied) { - return false; - } - } - - // Change the permissions. - @chmod($newFile, 0764); - - // Update query to say we got the video. - Release::query()->where('guid', $this->_release->guid)->update(['videostatus' => 1]); - if ($this->_echoCLI) { - $this->_echo('v', 'primaryOver'); - } - - return true; - } - } - - return false; - } - - /** - * @throws Exception - */ - protected function _getMediaInfo($fileLocation): bool - { - if (! $this->_processMediaInfo) { - return false; - } - - // Look for the video file. - if (File::isFile($fileLocation)) { - try { - $xmlArray = $this->mediaInfo()->getInfo($fileLocation, true); - - // Insert it into the DB. - \App\Models\MediaInfo::addData($this->_release->id, $xmlArray); - $this->_releaseExtra->addFromXml($this->_release->id, $xmlArray); - - if ($this->_echoCLI) { - $this->_echo('m', 'primaryOver'); - } - - return true; - } catch (\RuntimeException $e) { - Log::debug($e->getMessage()); - - return false; - } catch (\TypeError $e) { - Log::debug($e->getMessage()); - - return false; - } catch (\ErrorException $e) { - Log::debug($e->getMessage()); - - return false; - } - } - - return false; - } - - /** - * @throws Exception - */ - protected function _siftPAR2Info($fileLocation): void - { - $this->_par2Info->open($fileLocation); - - if ($this->_par2Info->error) { - return; - } - $releaseInfo = Release::query()->where('id', $this->_release->id)->select(['postdate', 'proc_pp'])->first(); - - if ($releaseInfo === null) { - return; - } - - $postDate = Carbon::createFromFormat('Y-m-d H:i:s', $releaseInfo->postdate)->getTimestamp(); - - // Only get a new name if the category is OTHER. - $foundName = true; - if ((int) $releaseInfo->proc_pp === 0 && config('nntmux.rename_par2') && - \in_array( - (int) $this->_release->categories_id, - Category::OTHERS_GROUP, - false - ) - ) { - $foundName = false; - } - - $filesAdded = 0; - - foreach ($this->_par2Info->getFileList() as $file) { - if (! isset($file['name'])) { - continue; - } - - // If we found a name and added 10 files, stop. - if ($foundName && $filesAdded > 10) { - break; - } - - // Add to release files. - if ($this->_addPAR2Files) { - if ($filesAdded < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name']])->first() === null - ) { - // Try to add the files to the DB. - if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $postDate, 0, $file['hash_16K'])) { - $filesAdded++; - } - } - } else { - $filesAdded++; - } - - // Try to get a new name. - if (! $foundName) { - $this->_release->textstring = $file['name']; - $this->_release->releases_id = $this->_release->id; - if ($this->_nameFixer->checkName($this->_release, $this->_echoCLI, 'PAR2, ', 1, 1)) { - $foundName = true; - } - } - } - // Update the file count with the new file count + old file count. - Release::query()->where('id', $this->_release->id)->increment('rarinnerfilecount', $filesAdded); - $this->_foundPAR2Info = true; - } - - /** - * @throws Exception Potentially thrown by addAlternateNfo - */ - protected function _processNfoFile(string $fileLocation): void - { - try { - $data = File::get($fileLocation); - if ($this->_nfo->isNFO($data, $this->_release->guid) && $this->_nfo->addAlternateNfo($data, $this->_release, $this->_nntp)) { - $this->_releaseHasNoNFO = false; - } - } catch (FileNotFoundException $e) { - // Log or handle the case where the file doesn't exist or isn't readable - Log::warning("Could not read potential NFO file: {$fileLocation} - {$e->getMessage()}"); - } - } - - /** - * @throws Exception - */ - protected function _processVideoFile($fileLocation): void - { - // Try to get a sample with it. - if (! $this->_foundSample) { - $this->_foundSample = $this->_getSample($fileLocation); - } - - /* Try to get a video with it. - * Don't get it here if _sampleMessageIDs is empty - * or has 1 message-id (Saves downloading another part). - */ - if (! $this->_foundVideo && \count($this->_sampleMessageIDs) < 2) { - $this->_foundVideo = $this->_getVideo($fileLocation); - } - - // Try to get media info with it. - if (! $this->_foundMediaInfo) { - $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); - } - } - - /** - * Comparison function for uSort, for sorting NZB files. - */ - protected function _sortNZB(array|string|null $a, array|string|null $b): int - { - $pos = 0; - $af = $bf = false; - $a = preg_replace('/\d+[ ._-]?(\/|\||[o0]f)[ ._-]?\d+?(?![ ._-]\d)/i', ' ', $a['title']); - $b = preg_replace('/\d+[ ._-]?(\/|\||[o0]f)[ ._-]?\d+?(?![ ._-]\d)/i', ' ', $b['title']); - if (preg_match('/\\.(part\\d+|[rz]\\d+)(\\s*\\.rar)*($|[ ")\]-])/i', $a)) { - $af = true; - } - if (preg_match('/\\.(part\\d+|[rz]\\d+)(\\s*\\.rar)*($|[ ")\]-])/i', $b)) { - $bf = true; - } - if (! $af && preg_match('/\.rar($|[ ")\]-])/i', $a)) { - $a = preg_replace('/\.rar(?:$|[ ")\]-])/i', '.*rar', $a); - $af = true; - } - if (! $bf && preg_match('/\.rar($|[ ")\]-])/i', $b)) { - $b = preg_replace('/\.rar(?:$|[ ")\]-])/i', '.*rar', $b); - $bf = true; - } - if (! $af && ! $bf) { - return strnatcasecmp($a, $b); - } - if (! $bf) { - return -1; - } - if (! $af) { - return 1; - } - if ($af && $bf) { - return strnatcasecmp($a, $b); - } - if ($af) { - return -1; - } - if ($bf) { - return 1; - } - - return $pos; - } - - /** - * Reset some variables for the current release. - */ - protected function _resetReleaseStatus(): void - { - // Only process for samples, previews and images if not disabled. - $this->_foundVideo = ! $this->_processVideo; - $this->_foundMediaInfo = ! $this->_processMediaInfo; - $this->_foundAudioInfo = ! $this->_processAudioInfo; - $this->_foundAudioSample = ! $this->_processAudioSample; - $this->_foundJPGSample = ! $this->_processJPGSample; - $this->_foundSample = ! $this->_processThumbnails; - $this->_foundPAR2Info = false; - - $this->_passwordStatus = Releases::PASSWD_NONE; - $this->_releaseHasPassword = false; - - $this->_releaseGroupName = UsenetGroup::getNameByID($this->_release->groups_id); - - $this->_releaseHasNoNFO = false; - // Make sure we don't already have a nfo. - if ((int) $this->_release->nfostatus !== 1) { - $this->_releaseHasNoNFO = true; - } - - $this->_NZBHasCompressedFile = false; - - $this->_sampleMessageIDs = $this->_JPGMessageIDs = $this->_MediaInfoMessageIDs = []; - $this->_AudioInfoMessageIDs = $this->_RARFileMessageIDs = []; - $this->_AudioInfoExtension = ''; - - $this->_addedFileInfo = 0; - $this->_totalFileInfo = 0; - $this->_compressedFilesChecked = 0; - $this->_groupUnavailable = false; - } - - /** - * Echo a string to CLI. - * - * @param string $string String to echo. - * @param string $type Method type. - * - * @void - */ - protected function _echo(string $string, string $type): void - { - if ($this->_echoCLI) { - (new ColorCLI)->$type($string); - } - } - - /** - * Echo a string to CLI. For debugging. - * - * - * @void - */ - protected function _debug(string $string): void - { - if (config('app.env') === 'local' && config('app.debug') === true) { - $this->_echo('DEBUG: '.$string, 'debug'); - } - } - - /** - * Delete the current release (used when NZB is irreparably broken). - * Removes DB row, related ancillary rows, search index documents, NZB and preview assets. - */ - protected function _deleteRelease(): void - { - try { - if (! isset($this->_release) || empty($this->_release->id)) { - return; - } - $id = (int) $this->_release->id; - $guid = $this->_release->guid ?? ''; - - // Delete related files (NZB + media samples/images if present) - try { - $nzbPath = $this->_nzb->NZBPath($guid); - if ($nzbPath && File::exists($nzbPath)) { - File::delete($nzbPath); - } - } catch (\Throwable $e) { - // ignore - } - // Remove preview assets - try { - $img = $this->_releaseImage->imgSavePath.$guid.'_thumb.jpg'; - $jpg = $this->_releaseImage->jpgSavePath.$guid.'_thumb.jpg'; - $vid = $this->_releaseImage->vidSavePath.$guid.'.ogv'; - foreach ([$img, $jpg, $vid] as $f) { - if ($f && File::exists($f)) { - File::delete($f); - } - } - } catch (\Throwable $e) { - // ignore - } - - // Delete ancillary rows. - try { - ReleaseFile::where('releases_id', $id)->delete(); - } catch (\Throwable $e) { - } - try { - MediaInfoModel::where('releases_id', $id)->delete(); - } catch (\Throwable $e) { - } - - // Delete search index document. - try { - if ($this->elasticsearchEnabled) { - $this->elasticsearch()->deleteRelease($id); - } else { - $this->manticore()->deleteRelease(['i' => $id, 'g' => $guid]); - } - } catch (\Throwable $e) { - // ignore index delete failures - } - - // Finally delete release row. - try { - Release::where('id', $id)->delete(); - } catch (\Throwable $e) { - } - - if ($this->_echoCLI) { - $this->_echo('Deleted broken release ID '.$id, 'warningOver'); - } - } catch (\Throwable $e) { - // Last resort: swallow any exception. - } - } - - /** - * Detect non-RAR/ZIP archive container types we can optionally extract with 7zip. - * Only returns values used by _processCompressedData's earlyType handling. - */ - protected function _detectNonRarZipType(string $data): ?string - { - $head6 = substr($data, 0, 6); - $head4 = substr($data, 0, 4); - // 7z signature 37 7A BC AF 27 1C with additional sanity checks. - if ($head6 === "\x37\x7A\xBC\xAF\x27\x1C" && $this->_isLikely7z($data)) { - return '7z'; - } - // GZIP 1F 8B 08 (check first 3 bytes) - if (strncmp($head4, "\x1F\x8B\x08", 3) === 0) { - return 'gzip'; - } - // BZip2 BZh - if (strncmp($head4, 'BZh', 3) === 0) { - return 'bzip2'; - } - // XZ FD 37 7A 58 5A 00 - if ($head6 === "\xFD7zXZ\x00") { - return 'xz'; - } - // (Optionally detect PDF so we can skip repeated processing, but we don't extract it.) - if ($head4 === '%PDF') { - return 'pdf'; - } - - return null; - } - - /** - * Heuristic validation to reduce false positives on 7z signature. - * 7z structure: signature (6 bytes) + version (2 bytes) + start header CRC (4 bytes) + next 8 bytes start header. - */ - private function _isLikely7z(string $data): bool - { - if (strlen($data) < 32) { - return false; // unrealistically small for valid 7z - } - // Version bytes: major=0x00, minor commonly 0x02-0x06 (allow up to 0x09 future-proof) - $verMajor = ord($data[6]); - $verMinor = ord($data[7]); - if ($verMajor !== 0x00 || $verMinor < 0x02 || $verMinor > 0x09) { - return false; - } - // Start header CRC (bytes 8-11) should not be all 0x00 or all 0xFF - $crc = substr($data, 8, 4); - if ($crc === "\x00\x00\x00\x00" || $crc === "\xFF\xFF\xFF\xFF") { - return false; - } - - return true; - } - - /** - * Attempt to recover (probable) filenames from a partial 7z archive without full parsing. - * Strategy: - * - Limit to first few MB to keep performance reasonable. - * - Collapse UTF-16LE (ASCII interleaved with 0x00) to plain ASCII. - * - Regex match typical filename patterns (printable chars + common extensions). - * - Return unique list (case-insensitive) capped at 50. - */ - private function _scanSevenZipFilenames(string $data): array - { - $slice = substr($data, 0, 8 * 1024 * 1024); - $converted = preg_replace('/([\x20-\x7E])\x00/', '$1', $slice); - if ($converted === null) { - $converted = $slice; - } - $converted = str_replace("\x00", ' ', $converted); - $exts = '7z|rar|zip|gz|bz2|xz|tar|tgz|mp4|mkv|avi|mpg|mpeg|mov|ts|wmv|flv|m4v|mp3|flac|ogg|wav|aac|aiff|ape|mka|nfo|txt|diz|pdf|epub|mobi|jpg|jpeg|png|gif|sfv|par2|exe|dll|srt|sub|idx|iso|bin|cue|mds|mdf'; - // Fixed safe pattern (no dynamic delimiter conflicts). - // Optional path (0-120 allowed chars + slash) then filename (2-160 chars) + .ext - $regex = '~(?:[A-Za-z0-9 _.+&@#,()!-]{0,120}[\\/])?([A-Za-z0-9 _.+&@#,()!-]{2,160}\.(?:'.$exts.'))~i'; - preg_match_all($regex, $converted, $m, PREG_SET_ORDER); - if (empty($m)) { - return []; - } - $names = []; - foreach ($m as $match) { - $candidate = $match[1]; - $candidate = preg_replace('/ {2,}/', ' ', $candidate); - if ($candidate === '' || strlen($candidate) < 5) { - continue; - } - if (substr_count($candidate, '.') > 10) { - continue; - } - $candidate = trim($candidate, " .-\t\n\r"); - $lower = strtolower($candidate); - if (! isset($names[$lower])) { - $names[$lower] = $candidate; - if (count($names) >= 80) { - break; - } - } - } - - return array_values($names); - } - - /** - * Execute an external command capturing stdout/stderr and exit code. - */ - private function _execCommand(string|array $cmd, ?int &$exitCode = null, ?string &$stdout = null, ?string &$stderr = null): bool - { - $descriptorSpec = [ - 1 => ['pipe', 'w'], // stdout - 2 => ['pipe', 'w'], // stderr - ]; - // If we have an array, pass directly to proc_open to avoid shell parsing. - $process = @proc_open($cmd, $descriptorSpec, $pipes, null, null, ['bypass_shell' => true]); - if (! \is_resource($process)) { - $exitCode = -1; - - return false; - } - $stdout = stream_get_contents($pipes[1]); - fclose($pipes[1]); - $stderr = stream_get_contents($pipes[2]); - fclose($pipes[2]); - $exitCode = proc_close($process); - - return $exitCode === 0; - } - - private function _extractViaSevenZip(string $compressedData, string $ext, string $type, string $marker): bool - { - try { - if (! $this->_extractUsingRarInfo && ! empty($this->_7zipPath)) { - $extractDir = $this->tmpPath.'un7z/'.uniqid('', true).'/'; - if (! File::isDirectory($extractDir)) { - File::makeDirectory($extractDir, 0777, true, true); - } - $fileName = $this->tmpPath.uniqid('', true).'.'.$ext; - File::put($fileName, $compressedData); - - $added = false; - $now = time(); - - // Build command parts (array form avoids quoting issues). - $cmdParts = [ - $this->_7zipPath, - 'e', - '-y', - '-bd', - '-o'.$extractDir, - $fileName, - ]; - $exit = 0; - $out = null; - $err = null; - $ok = $this->_execCommand($cmdParts, $exit, $out, $err); - - if (File::isDirectory($extractDir)) { - foreach (File::allFiles($extractDir) as $f) { - $info = [ - 'name' => $f->getFilename(), - 'size' => $f->getSize(), - 'date' => $now, - 'pass' => 0, - 'crc32' => '', - 'source' => $type, - ]; - // New filtering rule: only consider specific playable/text/image extensions - // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). - $name = $info['name'] ?? ''; - $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); - $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; - if (! in_array($ext, $allowedExt, true)) { - continue; // Skip disallowed extensions (also excludes gz, xz, etc.) - } - $base = pathinfo($name, PATHINFO_FILENAME); - $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); - if ($letterCount <= 5) { - continue; // Require >5 letter characters in base name - } - - $this->_addFileInfo($info); - $added = true; - } - } - // Diagnostics when extraction fails or produces no files. - if ((! $ok || ! $added) && config('app.debug') === true) { - $context = [ - 'release_id' => $this->_release->id ?? null, - 'tmp_file_exists' => File::exists($fileName), - 'tmp_file_size' => File::exists($fileName) ? File::size($fileName) : 0, - 'extract_dir_exists' => File::isDirectory($extractDir), - 'exit_code' => $exit, - 'stderr_snip' => substr((string) $err, 0, 400), - 'stdout_snip' => substr((string) $out, 0, 200), - 'cmd_parts' => $cmdParts, - 'type' => $type, - ]; - Log::debug('7zip extraction produced no files', $context); - } - - // Cleanup input file always after logging. - File::delete($fileName); - - if ($added && $this->_echoCLI) { - $this->_echo($marker, 'primaryOver'); - } - - return $added; - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::warning(strtoupper($type).' extraction exception: '.$e->getMessage()); - } - } - - return false; - } - - /** - * Detect if the binary blob appears to be a standalone video (not inside an archive). - * Used to opportunistically derive MediaInfo / previews from first-part inline uploads. - */ - private function _detectStandaloneVideo(string $data): ?string - { - $len = strlen($data); - if ($len < 16) { - return null; - } - // AVI (RIFF + 'AVI ') - if (strncmp($data, 'RIFF', 4) === 0 && substr($data, 8, 4) === 'AVI ') { - return 'avi'; - } - // Matroska / WebM (EBML header) - if (strncmp($data, "\x1A\x45\xDF\xA3", 4) === 0) { - return 'mkv'; - } - // MPEG Program Stream (pack / sequence headers) - $sig4 = substr($data, 0, 4); - if ($sig4 === "\x00\x00\x01\xBA" || $sig4 === "\x00\x00\x01\xB3") { - return 'mpg'; - } - // MPEG Transport Stream (0x47 sync every 188 bytes for first 5 packets) - if ($len >= 188 * 5) { - $isTs = true; - for ($i = 0; $i < 5; $i++) { - $pos = 188 * $i; - if (! isset($data[$pos]) || $data[$pos] !== "\x47") { - $isTs = false; - break; - } - } - if ($isTs) { - return 'mpg'; - } - } - // MP4 / MOV ftyp brands - if ($len >= 12 && substr($data, 4, 4) === 'ftyp') { - $brand = substr($data, 8, 4); - $brands = ['isom', 'iso2', 'avc1', 'mp41', 'mp42', 'dash', 'MSNV', 'qt ', 'M4V ', 'M4P ', 'M4B ', 'M4A ']; - if (in_array($brand, $brands, true)) { - return 'mp4'; - } - } - - return null; - } - - /** - * Process only a single release by GUID (utility method). + * Process only a single release by GUID. */ public function processSingleGuid(string $guid): bool { - try { - $release = Release::where('guid', $guid)->first(); - if ($release === null) { - if ($this->_echoCLI) { - $this->_echo('Release not found for GUID: '.$guid, 'warning'); - } - - return false; - } - $this->_releases = collect([$release]); - $this->_totalReleases = 1; - $guidChar = $release->leftguid ?? substr($release->guid, 0, 1); - $groupID = ''; - $this->_setMainTempPath($guidChar, $groupID); - $this->_processReleases(); - - return true; - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('processSingleGuid failed: '.$e->getMessage()); - } - - return false; - } - } - - /** - * Extract release name from a filename, handling file extensions and using an improved pattern. - * Returns null if no valid release name found. - */ - private function extractReleaseNameFromFile(string $filename): ?string - { - // Strip directory path if present (e.g., "Sample/file.mkv" -> "file.mkv") - $basename = basename($filename); - - // First, strip common file extensions (video, archive, metadata, package, and executable files) - $cleaned = preg_replace('/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi)$/i', '', $basename); - - // Try improved regex that better handles audio/video codecs with dots (DD.5.1, H.264, etc.) - // Match from start to a separator (hyphen or dot) followed by a group name (2+ word chars) - // This preserves everything including complex codec patterns like DD.5.1 and H.264 - if (preg_match('/^(.+[-.][A-Za-z0-9_]{2,})$/i', $cleaned, $match)) { - // Matched: full string ending with -GROUP or .GROUP - // Capitalize first letter of each word (using . as delimiter for release names) - return ucwords($match[1], '.-_ '); - } - - // Fallback to standard PREDB_REGEX - if (preg_match(NameFixer::PREDB_REGEX, $cleaned, $hit)) { - return ucwords($hit[0], '.'); - } - - // If neither pattern matched, return null - return null; - } - - /** - * Normalize a candidate title by stripping trailing part/vol/rNN tokens and trivial suffixes. - */ - private function normalizeCandidateTitle(string $title): string - { - $t = trim($title); - // Remove common video file extensions - $t = preg_replace('/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx)$/i', '', $t) ?? $t; - // Remove archive and metadata file extensions - $t = preg_replace('/\.(par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi|jpe?g|png|gif|bmp)$/i', '', $t) ?? $t; - // Remove common trailing segment markers like .part01, .vol12+3, -r12, r12 - $t = preg_replace('/[.\-_ ](?:part|vol|r)\d+(?:\+\d+)?$/i', '', $t) ?? $t; - // Collapse multiple spaces/underscores - $t = preg_replace('/[\s_]+/', ' ', $t) ?? $t; - - // Trim stray punctuation - return trim($t, " .-_\t\r\n"); - } - - /** - * Heuristic filter to avoid poor RarInfo-based renames (e.g., short or generic names). - */ - private function isPlausibleReleaseTitle(string $title): bool - { - $t = trim($title); - if ($t === '') { - return false; - } - // Minimum length and token count - if (strlen($t) < 12) { - return false; - } - $wordCount = preg_match_all('/[A-Za-z0-9]{3,}/', $t); - if ($wordCount < 2) { - return false; - } - // Reject if it still ends with a segment marker - if (preg_match('/(?:^|[.\-_ ])(?:part|vol|r)\d+(?:\+\d+)?$/i', $t)) { - return false; - } - // Reject generic installer/setup filenames - if (preg_match('/^(setup|install|installer|patch|update|crack|keygen)\d*[\s._-]/i', $t)) { - return false; - } - // Acceptance criteria - $hasGroupSuffix = (bool) preg_match('/[-.][A-Za-z0-9]{2,}$/', $t); - $hasYear = (bool) preg_match('/\b(19|20)\d{2}\b/', $t); - $hasQuality = (bool) preg_match('/\b(480p|720p|1080p|2160p|4k|webrip|web[ .-]?dl|bluray|bdrip|dvdrip|hdtv|hdrip|xvid|x264|x265|hevc|h\.?264|ts|cam|r5|proper|repack)\b/i', $t); - $hasTV = (bool) preg_match('/\bS\d{1,2}[Eex]\d{1,3}\b/i', $t); - $hasXXX = (bool) preg_match('/\bXXX\b/i', $t); - - // Accept if: - // - Has a group suffix (e.g., -ETHEL or .NBQ) - // - Has TV episode identifier with quality indicator (e.g., S03E14 + 720p) - // - Has year with quality or TV identifier - // - Has XXX tag - if ($hasGroupSuffix || ($hasTV && $hasQuality) || ($hasYear && ($hasQuality || $hasTV)) || $hasXXX) { - return true; - } - - return false; + return $this->orchestrator->processSingleGuid($guid); } } diff --git a/app/Providers/AdditionalProcessingServiceProvider.php b/app/Providers/AdditionalProcessingServiceProvider.php new file mode 100644 index 000000000..e6be636ef --- /dev/null +++ b/app/Providers/AdditionalProcessingServiceProvider.php @@ -0,0 +1,118 @@ +app->singleton(ProcessingConfiguration::class, function () { + return new ProcessingConfiguration(); + }); + + // Console output service + $this->app->singleton(ConsoleOutputService::class, function ($app) { + $config = $app->make(ProcessingConfiguration::class); + return new ConsoleOutputService($config->echoCLI); + }); + + // NZB content parser + $this->app->singleton(NzbContentParser::class, function ($app) { + $config = $app->make(ProcessingConfiguration::class); + return new NzbContentParser( + new NZB(), + $config->debugMode, + $config->echoCLI + ); + }); + + // Archive extraction service + $this->app->singleton(ArchiveExtractionService::class, function ($app) { + return new ArchiveExtractionService( + $app->make(ProcessingConfiguration::class) + ); + }); + + // Usenet download service + $this->app->singleton(UsenetDownloadService::class, function ($app) { + return new UsenetDownloadService( + $app->make(ProcessingConfiguration::class) + ); + }); + + // Release file manager + $this->app->singleton(ReleaseFileManager::class, function ($app) { + return new ReleaseFileManager( + $app->make(ProcessingConfiguration::class), + new ReleaseExtra(), + new ReleaseImage(), + new Nfo(), + new NZB(), + new NameFixer() + ); + }); + + // Media extraction service + $this->app->singleton(MediaExtractionService::class, function ($app) { + return new MediaExtractionService( + $app->make(ProcessingConfiguration::class), + new ReleaseImage(), + new ReleaseExtra(), + new CategorizationService() + ); + }); + + // Temp workspace service (might already be registered elsewhere) + $this->app->singleton(TempWorkspaceService::class, function () { + return new TempWorkspaceService(); + }); + + // Main orchestrator + $this->app->singleton(AdditionalProcessingOrchestrator::class, function ($app) { + return new AdditionalProcessingOrchestrator( + $app->make(ProcessingConfiguration::class), + $app->make(NzbContentParser::class), + $app->make(ArchiveExtractionService::class), + $app->make(MediaExtractionService::class), + $app->make(UsenetDownloadService::class), + $app->make(ReleaseFileManager::class), + $app->make(TempWorkspaceService::class), + $app->make(ConsoleOutputService::class) + ); + }); + } + + /** + * Bootstrap services. + */ + public function boot(): void + { + // + } +} + diff --git a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php new file mode 100644 index 000000000..9936d4ae0 --- /dev/null +++ b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php @@ -0,0 +1,732 @@ +setupTempPath($guidChar, $groupID); + $this->fetchReleases($groupID, $guidChar); + + if ($this->totalReleases > 0) { + $this->output->echoDescription($this->totalReleases); + $this->processReleases(); + } + } + + /** + * Process a single release by GUID. + */ + public function processSingleGuid(string $guid): bool + { + try { + $release = Release::where('guid', $guid)->first(); + if ($release === null) { + $this->output->warning('Release not found for GUID: '.$guid); + return false; + } + + $this->releases = collect([$release]); + $this->totalReleases = 1; + $guidChar = $release->leftguid ?? substr($release->guid, 0, 1); + $groupID = ''; + $this->setupTempPath($guidChar, $groupID); + $this->processReleases(); + + return true; + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::error('processSingleGuid failed: '.$e->getMessage()); + } + return false; + } + } + + /** + * Set up the main temp path. + */ + private function setupTempPath(string &$guidChar, string &$groupID): void + { + $this->mainTmpPath = $this->tempWorkspace->ensureMainTempPath( + $this->config->tmpUnrarPath, + $guidChar, + $groupID + ); + $this->tempWorkspace->clearDirectory($this->mainTmpPath, true); + } + + /** + * Fetch releases for processing. + */ + private function fetchReleases(int|string $groupID, string $guidChar): void + { + $sqlParts = []; + $bindings = []; + + $sqlParts[] = 'SELECT r.id AS id, r.id AS releases_id, r.guid, r.name, r.size, r.groups_id, r.nfostatus, r.fromname,'; + $sqlParts[] = 'r.completion, r.categories_id, r.searchname, r.predb_id, c.disablepreview'; + $sqlParts[] = 'FROM releases r'; + $sqlParts[] = 'LEFT JOIN categories c ON c.id = r.categories_id'; + $sqlParts[] = 'WHERE r.passwordstatus = ?'; + $bindings[] = -1; + $sqlParts[] = 'AND r.nzbstatus = ?'; + $bindings[] = 1; + $sqlParts[] = 'AND r.haspreview = ?'; + $bindings[] = -1; + $sqlParts[] = 'AND c.disablepreview = ?'; + $bindings[] = 0; + + if ($this->config->maxSizeGB > 0) { + $sqlParts[] = 'AND r.size < ?'; + $bindings[] = $this->config->maxSizeGB * 1073741824; + } + if ($this->config->minSizeMB > 0) { + $sqlParts[] = 'AND r.size > ?'; + $bindings[] = $this->config->minSizeMB * 1048576; + } + if (! empty($groupID)) { + $sqlParts[] = 'AND r.groups_id = ?'; + $bindings[] = $groupID; + } + if (! empty($guidChar)) { + $sqlParts[] = 'AND r.leftguid = ?'; + $bindings[] = $guidChar; + } + + $sqlParts[] = 'ORDER BY r.passwordstatus ASC, r.postdate DESC'; + $limit = $this->config->queryLimit > 0 ? $this->config->queryLimit : 25; + $sqlParts[] = 'LIMIT '.$limit; + + $rawResults = DB::select(implode(' ', $sqlParts), $bindings); + $attributeArrays = array_map(static fn ($row) => (array) $row, $rawResults); + $this->releases = Release::hydrate($attributeArrays); + $this->totalReleases = $this->releases->count(); + } + + /** + * Process all fetched releases. + * + * @throws Exception + */ + private function processReleases(): void + { + foreach ($this->releases as $release) { + $context = new ReleaseProcessingContext($release); + + $this->output->echoReleaseStart($release->id, $release->size); + cli_set_process_title(PHP_BINARY.' '.__DIR__.'/AdditionalProcessingOrchestrator.php ReleaseID: '.$release->id); + + // Create temp folder for this release + try { + $context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($this->mainTmpPath, $release->guid); + } catch (\Throwable $e) { + $this->output->warning('Unable to create directory: '.$e->getMessage()); + continue; + } + + // Parse NZB contents + $nzbResult = $this->nzbParser->parseNzb($release->guid); + if ($nzbResult['error'] !== null) { + $this->output->warning($nzbResult['error']); + $this->releaseManager->deleteRelease($release); + continue; + } + $context->nzbContents = $nzbResult['contents']; + + // Initialize context + $this->initializeContext($context); + + // Extract message IDs from NZB + $messageIds = $this->nzbParser->extractMessageIDs( + $context->nzbContents, + $context->releaseGroupName, + $this->config->segmentsToDownload, + $this->config->processThumbnails, + $this->config->processJPGSample, + $this->config->processMediaInfo, + $this->config->processAudioInfo, + $this->config->audioFileRegex, + $this->config->videoFileRegex, + $this->config->supportFileRegex, + $this->config->ignoreBookRegex + ); + + $context->nzbHasCompressedFile = $messageIds['hasCompressedFile']; + $context->sampleMessageIDs = $messageIds['sampleMessageIDs']; + $context->jpgMessageIDs = $messageIds['jpgMessageIDs']; + $context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID']; + $context->audioInfoMessageIDs = $messageIds['audioInfoMessageID']; + $context->audioInfoExtension = $messageIds['audioInfoExtension']; + + // Check for book flood + $bookFlood = $messageIds['bookFileCount'] > 80 + && ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents); + + // Process message ID downloads + if ($this->config->processPasswords || $this->config->processThumbnails + || $this->config->processMediaInfo || $this->config->processAudioInfo + || $this->config->processVideo + ) { + $this->processMessageIDDownloads($context); + + // Process compressed files + if (! $bookFlood && $context->nzbHasCompressedFile) { + $this->processNZBCompressedFiles($context, false); + + if ($this->config->fetchLastFiles) { + $this->processNZBCompressedFiles($context, true); + } + + if (! $context->releaseHasPassword) { + $this->processExtractedFiles($context); + } + } + } + + // Finalize + $this->releaseManager->finalizeRelease($context, $this->config->processPasswords); + + // Cleanup + $this->tempWorkspace->clearDirectory($context->tmpPath, false); + } + + $this->output->endOutput(); + } + + /** + * Initialize the processing context. + */ + private function initializeContext(ReleaseProcessingContext $context): void + { + $context->initializeFromConfig( + $this->config->processVideo, + $this->config->processMediaInfo, + $this->config->processAudioInfo, + $this->config->processAudioSample, + $this->config->processJPGSample, + $this->config->processThumbnails + ); + + $context->passwordStatus = Releases::PASSWD_NONE; + $context->releaseHasPassword = false; + $context->releaseGroupName = UsenetGroup::getNameByID($context->release->groups_id); + $context->releaseHasNoNFO = (int) $context->release->nfostatus !== 1; + $context->resetMessageIDs(); + $context->resetCounters(); + } + + /** + * Process message ID downloads (sample, media info, audio, JPG). + */ + private function processMessageIDDownloads(ReleaseProcessingContext $context): void + { + // Sample download + if ((! $context->foundSample || ! $context->foundVideo) && ! empty($context->sampleMessageIDs)) { + $result = $this->downloadService->downloadSample( + $context->sampleMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && $this->downloadService->meetsMinimumSize($result['data'])) { + $this->output->echoSampleDownload(); + $fileLocation = $context->tmpPath.'sample_'.random_int(0, 99999).'.avi'; + File::put($fileLocation, $result['data']); + + if (! $context->foundSample) { + if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundSample = true; + $this->output->echoSampleCreated(); + } + } + if (! $context->foundVideo) { + if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundVideo = true; + $this->output->echoVideoCreated(); + } + } + } elseif (! $result['success']) { + $this->output->echoSampleFailure(); + } + } + + // Media info download + if ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) + && ! empty($context->mediaInfoMessageIDs) + ) { + $result = $this->downloadService->downloadMediaInfo( + $context->mediaInfoMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success'] && $this->downloadService->meetsMinimumSize($result['data'])) { + $this->output->echoMediaInfoDownload(); + $fileLocation = $context->tmpPath.'media.avi'; + File::put($fileLocation, $result['data']); + + if (! $context->foundMediaInfo) { + if ($this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { + $context->foundMediaInfo = true; + $this->output->echoMediaInfoAdded(); + } + } + if (! $context->foundSample) { + if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundSample = true; + $this->output->echoSampleCreated(); + } + } + if (! $context->foundVideo) { + if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundVideo = true; + $this->output->echoVideoCreated(); + } + } + } elseif (! $result['success']) { + $this->output->echoMediaInfoFailure(); + } + } + + // Audio info download + if ((! $context->foundAudioInfo || ! $context->foundAudioSample) + && ! empty($context->audioInfoMessageIDs) + ) { + $result = $this->downloadService->downloadAudio( + $context->audioInfoMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success']) { + $this->output->echoAudioDownload(); + $fileLocation = $context->tmpPath.'audio.'.$context->audioInfoExtension; + File::put($fileLocation, $result['data']); + + $audioResult = $this->mediaService->getAudioInfo( + $fileLocation, + $context->audioInfoExtension, + $context, + $context->tmpPath + ); + + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + } elseif (! $result['success']) { + $this->output->echoAudioFailure(); + } + } + + // JPG download + if (! $context->foundJPGSample && ! empty($context->jpgMessageIDs)) { + $result = $this->downloadService->downloadJPG( + $context->jpgMessageIDs, + $context->releaseGroupName, + $context->release->id + ); + + if ($result['success']) { + $this->output->echoJpgDownload(); + $fileLocation = $context->tmpPath.'samplepicture.jpg'; + File::put($fileLocation, $result['data']); + + if ($this->mediaService->isJpegData($fileLocation)) { + if ($this->mediaService->getJPGSample($fileLocation, $context->release->guid)) { + $context->foundJPGSample = true; + $this->output->echoJpgSaved(); + } + } + File::delete($fileLocation); + } elseif (! $result['success']) { + $this->output->echoJpgFailure(); + } + } + } + + /** + * Process compressed files in the NZB. + */ + private function processNZBCompressedFiles(ReleaseProcessingContext $context, bool $reverse): void + { + if ($context->groupUnavailable) { + return; + } + + $nzbContents = $context->nzbContents; + if ($reverse) { + krsort($nzbContents); + } else { + $this->triedCompressedMids = []; + } + + $failed = $downloaded = 0; + + foreach ($nzbContents as $nzbFile) { + if ($downloaded >= $this->config->maximumRarSegments) { + break; + } + if ($failed >= $this->config->maximumRarPasswordChecks) { + break; + } + if ($context->releaseHasPassword || $context->groupUnavailable) { + break; + } + + $title = $nzbFile['title'] ?? ''; + if (! preg_match( + '/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', + $title + )) { + continue; + } + + // Get message IDs + $segments = $nzbFile['segments'] ?? []; + $segCount = count($segments) - 1; + $messageIDs = []; + + foreach (range(0, $this->config->maximumRarSegments - 1) as $i) { + if ($i > $segCount) { + break; + } + $segment = (string) $segments[$i]; + if (! $reverse) { + $this->triedCompressedMids[] = $segment; + } elseif (in_array($segment, $this->triedCompressedMids, false)) { + continue 2; + } + $messageIDs[] = $segment; + } + + if (empty($messageIDs)) { + continue; + } + + // Download + $result = $this->downloadService->downloadCompressedFile( + $messageIDs, + $context->releaseGroupName, + $context->release->id, + $title + ); + + if ($result['groupUnavailable']) { + $context->groupUnavailable = true; + $this->output->echoGroupUnavailable(); + break; + } + + if ($result['success']) { + $this->output->echoCompressedDownload(); + $downloaded++; + + $processed = $this->processCompressedData($result['data'], $context, $reverse); + if ($processed || $context->releaseHasPassword) { + break; + } + } else { + $failed++; + $this->output->echoCompressedFailure($failed); + } + } + } + + /** + * Process compressed data. + */ + private function processCompressedData( + string $compressedData, + ReleaseProcessingContext $context, + bool $reverse + ): bool { + $result = $this->archiveService->processCompressedData( + $compressedData, + $context, + $context->tmpPath + ); + + if ($result['hasPassword']) { + $context->releaseHasPassword = true; + $context->passwordStatus = $result['passwordStatus']; + return false; + } + + // Handle standalone video + if (isset($result['standaloneVideoType'])) { + $this->output->echoInlineVideo(); + $ext = $result['standaloneVideoType']; + $fileLocation = $context->tmpPath.'inline_video_'.uniqid('', true).'.'.$ext; + File::put($fileLocation, $result['standaloneVideoData']); + + if (! $context->foundMediaInfo) { + if ($this->mediaService->getMediaInfo($fileLocation, $context->release->id)) { + $context->foundMediaInfo = true; + $this->output->echoMediaInfoAdded(); + } + } + if (! $context->foundSample) { + if ($this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundSample = true; + $this->output->echoSampleCreated(); + } + } + if (! $context->foundVideo) { + if ($this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) { + $context->foundVideo = true; + $this->output->echoVideoCreated(); + } + } + + return $context->foundMediaInfo || $context->foundSample || $context->foundVideo; + } + + if (! $result['success']) { + return false; + } + + // Echo archive marker + if (! empty($result['archiveMarker'])) { + $this->output->echoArchiveMarker($result['archiveMarker']); + } + + // Handle reverse processing for release name extraction + if ($reverse && ! empty($result['dataSummary'])) { + $this->releaseManager->processReleaseNameFromRar($result['dataSummary'], $context); + } + + // Process file list + foreach ($result['files'] as $file) { + if ($context->releaseHasPassword) { + break; + } + + if ($this->releaseManager->addFileInfo($file, $context, $this->config->supportFileRegex)) { + $this->output->echoFileInfoAdded(); + } + } + + if ($context->addedFileInfo > 0) { + $this->releaseManager->updateSearchIndex($context->release->id); + } + + return $context->totalFileInfo > 0; + } + + /** + * Process extracted files from archives. + */ + private function processExtractedFiles(ReleaseProcessingContext $context): void + { + $nestedLevels = 0; + + // Handle nested archives + while ($nestedLevels < $this->config->maxNestedLevels) { + if ($context->compressedFilesChecked >= self::MAX_COMPRESSED_FILES_TO_CHECK) { + break; + } + + $foundCompressed = false; + $pattern = '/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1|7z(?:\.\d{3})?|(?:tar\.)?(?:gz|bz2|xz))($|[^a-z0-9])/i'; + + try { + $files = $this->tempWorkspace->listFiles($context->tmpPath, $pattern); + } catch (\Throwable) { + break; + } + + foreach ($files as $file) { + $filePath = is_array($file) ? $file[0] : $file->getPathname(); + if (File::isFile($filePath)) { + $rarData = @File::get($filePath); + if (! empty($rarData)) { + $this->processCompressedData($rarData, $context, false); + $foundCompressed = true; + } + File::delete($filePath); + } + } + + if (! $foundCompressed) { + break; + } + $nestedLevels++; + } + + // Process remaining files + try { + $files = $this->tempWorkspace->listFiles($context->tmpPath); + } catch (\Throwable) { + return; + } + + foreach ($files as $file) { + $filePath = is_object($file) ? $file->getPathname() : $file; + + if (preg_match('/[\/\\\\]\.{1,2}$/', $filePath)) { + continue; + } + + if (! File::isFile($filePath)) { + continue; + } + + // PAR2 files + if (! $context->foundPAR2Info && preg_match('/\.par2$/i', $filePath)) { + $this->releaseManager->processPar2File( + $filePath, + $context, + $this->archiveService->getPar2Info() + ); + continue; + } + + // NFO files + if ($context->releaseHasNoNFO && preg_match('/(\.(nfo|inf|ofn)|info\.txt)$/i', $filePath)) { + if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + } + continue; + } + + // Audio files + if ((! $context->foundAudioInfo || ! $context->foundAudioSample) + && preg_match('/(.*)'.$this->config->audioFileRegex.'$/i', $filePath, $fileType) + ) { + $audioPath = $context->tmpPath.'audiofile.'.$fileType[2]; + File::move($filePath, $audioPath); + $audioResult = $this->mediaService->getAudioInfo( + $audioPath, + $fileType[2], + $context, + $context->tmpPath + ); + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + File::delete($audioPath); + continue; + } + + // JPG files + if (! $context->foundJPGSample && preg_match('/\.jpe?g$/i', $filePath)) { + if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { + $context->foundJPGSample = true; + $this->output->echoJpgSaved(); + } + File::delete($filePath); + continue; + } + + // Video files + if ((! $context->foundSample || ! $context->foundVideo || ! $context->foundMediaInfo) + && preg_match('/(.*)'.$this->config->videoFileRegex.'$/i', $filePath) + ) { + $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); + continue; + } + + // Check file magic + $output = Utility::fileInfo($filePath); + if (empty($output)) { + continue; + } + + if (! $context->foundJPGSample && preg_match('/^JPE?G/i', $output)) { + if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) { + $context->foundJPGSample = true; + $this->output->echoJpgSaved(); + } + File::delete($filePath); + } elseif ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo) + && preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output) + ) { + $this->mediaService->processVideoFile($filePath, $context, $context->tmpPath); + } elseif ((! $context->foundAudioSample || ! $context->foundAudioInfo) + && preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $audioType) + ) { + $ext = match ($audioType[0]) { + 'FLAC' => 'FLAC', + 'layer III' => 'MP3', + 'Vorbis audio' => 'OGG', + default => 'audio', + }; + $audioPath = $context->tmpPath.'audiofile.'.$ext; + File::move($filePath, $audioPath); + $audioResult = $this->mediaService->getAudioInfo($audioPath, $ext, $context, $context->tmpPath); + if ($audioResult['audioInfo']) { + $this->output->echoAudioInfoAdded(); + } + if ($audioResult['audioSample']) { + $this->output->echoAudioSampleCreated(); + } + File::delete($audioPath); + } elseif (! $context->foundPAR2Info && stripos($output, 'Parity') === 0) { + $this->releaseManager->processPar2File( + $filePath, + $context, + $this->archiveService->getPar2Info() + ); + } + } + } + + /** + * Clear the main temp path. + */ + public function __destruct() + { + if ($this->mainTmpPath !== '') { + $this->tempWorkspace->clearDirectory($this->mainTmpPath, true); + } + } +} + diff --git a/app/Services/AdditionalProcessing/ArchiveExtractionService.php b/app/Services/AdditionalProcessing/ArchiveExtractionService.php new file mode 100644 index 000000000..dda9f5701 --- /dev/null +++ b/app/Services/AdditionalProcessing/ArchiveExtractionService.php @@ -0,0 +1,726 @@ +archiveInfo = new ArchiveInfo(); + $this->par2Info = new Par2Info(); + + // Configure external clients for ArchiveInfo + if ($this->config->unrarPath) { + $this->archiveInfo->setExternalClients([ArchiveInfo::TYPE_RAR => $this->config->unrarPath]); + } + } + + /** + * Process compressed data and extract file information. + * + * @return array{success: bool, files: array, hasPassword: bool, passwordStatus: int} + */ + public function processCompressedData( + string $compressedData, + ReleaseProcessingContext $context, + string $tmpPath + ): array { + $result = [ + 'success' => false, + 'files' => [], + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + ]; + + $context->compressedFilesChecked++; + + // Detect archive type early + $archiveType = $this->detectArchiveType($compressedData); + + // Handle 7z, gzip, bzip2, xz with external 7zip binary + if (in_array($archiveType, ['7z', 'gzip', 'bzip2', 'xz'], true)) { + if ($archiveType === '7z') { + $sevenZipResult = $this->processSevenZipArchive($compressedData, $context, $tmpPath); + if ($sevenZipResult['success'] || $sevenZipResult['hasPassword']) { + return $sevenZipResult; + } + } + + if ($this->config->sevenZipPath) { + $extractResult = $this->extractViaSevenZip($compressedData, $archiveType, $tmpPath); + if ($extractResult['success']) { + return $extractResult; + } + } + } + + // Try ArchiveInfo for RAR/ZIP + if (! $this->archiveInfo->setData($compressedData, true)) { + // Handle standalone video detection + $videoType = $this->detectStandaloneVideo($compressedData); + if ($videoType !== null) { + return [ + 'success' => false, + 'files' => [], + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + 'standaloneVideoType' => $videoType, + 'standaloneVideoData' => $compressedData, + ]; + } + return $result; + } + + if ($this->archiveInfo->error !== '') { + if ($this->config->debugMode) { + Log::debug('ArchiveInfo Error: '.$this->archiveInfo->error); + } + return $result; + } + + try { + $dataSummary = $this->archiveInfo->getSummary(true); + } catch (\Exception $e) { + if ($this->config->debugMode) { + Log::warning($e->getTraceAsString()); + } + return $result; + } + + // Check for encryption + if (! empty($this->archiveInfo->isEncrypted) + || (isset($dataSummary['is_encrypted']) && (int) $dataSummary['is_encrypted'] !== 0) + ) { + if ($this->config->debugMode) { + Log::debug('ArchiveInfo: Compressed file has a password.'); + } + return [ + 'success' => false, + 'files' => [], + 'hasPassword' => true, + 'passwordStatus' => Releases::PASSWD_RAR, + ]; + } + + // Prepare extraction directories + $this->prepareExtractionDirectories($tmpPath); + + // Process based on archive type + $archiveMarker = $this->extractArchive($compressedData, $dataSummary, $tmpPath); + + // Get file list + $files = $this->archiveInfo->getArchiveFileList(); + if (! is_array($files) || count($files) === 0) { + return $result; + } + + return [ + 'success' => true, + 'files' => $files, + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + 'archiveMarker' => $archiveMarker, + 'dataSummary' => $dataSummary, + ]; + } + + /** + * Detect the archive type from binary signature. + */ + public function detectArchiveType(string $data): ?string + { + $head6 = substr($data, 0, 6); + $head4 = substr($data, 0, 4); + + // 7z signature + if ($head6 === "\x37\x7A\xBC\xAF\x27\x1C" && $this->isLikely7z($data)) { + return '7z'; + } + // GZIP + if (strncmp($head4, "\x1F\x8B\x08", 3) === 0) { + return 'gzip'; + } + // BZip2 + if (strncmp($head4, 'BZh', 3) === 0) { + return 'bzip2'; + } + // XZ + if ($head6 === "\xFD7zXZ\x00") { + return 'xz'; + } + // PDF (skip) + if ($head4 === '%PDF') { + return 'pdf'; + } + + return null; + } + + /** + * Heuristic validation for 7z signature. + */ + private function isLikely7z(string $data): bool + { + if (strlen($data) < 32) { + return false; + } + $verMajor = ord($data[6]); + $verMinor = ord($data[7]); + if ($verMajor !== 0x00 || $verMinor < 0x02 || $verMinor > 0x09) { + return false; + } + $crc = substr($data, 8, 4); + if ($crc === "\x00\x00\x00\x00" || $crc === "\xFF\xFF\xFF\xFF") { + return false; + } + return true; + } + + /** + * Process a 7z archive using external binary and internal header parsing. + */ + private function processSevenZipArchive( + string $compressedData, + ReleaseProcessingContext $context, + string $tmpPath + ): array { + $result = [ + 'success' => false, + 'files' => [], + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + ]; + + if (! $this->config->sevenZipPath) { + return $result; + } + + // Try listing with external 7z binary + $listed = $this->listSevenZipEntries($compressedData, $tmpPath); + if (! empty($listed)) { + if (! empty($listed[0]['__any_encrypted__'])) { + return [ + 'success' => false, + 'files' => [], + 'hasPassword' => true, + 'passwordStatus' => Releases::PASSWD_RAR, + ]; + } + + $files = $this->filterSevenZipFiles($listed); + if (! empty($files)) { + return [ + 'success' => true, + 'files' => $files, + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + 'archiveMarker' => '7z', + ]; + } + } + + // Fallback: scan for filenames in raw data + $scannedNames = $this->scanSevenZipFilenames($compressedData); + if (! empty($scannedNames)) { + $files = array_map(fn($name) => [ + 'name' => $name, + 'size' => 0, + 'date' => time(), + 'pass' => 0, + 'crc32' => '', + 'source' => '7z-scan', + ], $scannedNames); + + return [ + 'success' => true, + 'files' => $files, + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + 'archiveMarker' => '7z', + ]; + } + + return $result; + } + + /** + * List entries of a 7z archive using external 7z binary. + */ + public function listSevenZipEntries(string $compressedData, string $tmpPath): array + { + if (! $this->config->sevenZipPath) { + return []; + } + + try { + $tmpFile = $tmpPath.uniqid('7zlist_', true).'.7z'; + if (File::put($tmpFile, $compressedData) === false) { + return []; + } + + $cmd = [$this->config->sevenZipPath, 'l', '-slt', '-ba', '-bd', $tmpFile]; + $exitCode = 0; + $stdout = null; + $stderr = null; + $ok = $this->execCommand($cmd, $exitCode, $stdout, $stderr); + + if (! $ok || $exitCode !== 0 || empty($stdout)) { + // Try plain listing fallback + $plainResult = $this->listSevenZipPlain($tmpFile); + File::delete($tmpFile); + return $plainResult; + } + + File::delete($tmpFile); + return $this->parseSevenZipStructuredOutput($stdout); + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::debug('Exception listing 7z: '.$e->getMessage()); + } + return []; + } + } + + /** + * Plain 7z listing fallback. + */ + private function listSevenZipPlain(string $tmpFile): array + { + $cmd = [$this->config->sevenZipPath, 'l', '-ba', '-bd', $tmpFile]; + $exitCode = 0; + $stdout = null; + $stderr = null; + $ok = $this->execCommand($cmd, $exitCode, $stdout, $stderr); + + if (! $ok || $exitCode !== 0 || empty($stdout)) { + return []; + } + + $files = []; + $lines = preg_split('/\r?\n/', trim($stdout)); + foreach ($lines as $line) { + $line = trim($line); + if ($line === '' || str_starts_with($line, '-----') + || str_contains($line, ' Date ') + || str_starts_with($line, 'Scanning ') + ) { + continue; + } + + $name = null; + if (preg_match('/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\s+\S+\s+\d+\s+\d+\s+(\S.*)$/', $line, $m)) { + $name = $m[1]; + } elseif (preg_match('/([A-Za-z0-9_#@()\[\]\-+&., ]+\.[A-Za-z0-9]{2,8})$/', $line, $m2)) { + $name = trim($m2[1]); + } + + if ($name && strlen($name) <= 300) { + $files[] = ['name' => trim($name), 'size' => 0, 'encrypted' => false]; + if (count($files) >= 200) { + break; + } + } + } + + return $files; + } + + /** + * Parse structured 7z output. + */ + private function parseSevenZipStructuredOutput(string $output): array + { + $blocks = preg_split('/\n\n+/u', trim($output)); + $files = []; + $anyEncrypted = false; + + foreach ($blocks as $block) { + $lines = preg_split('/\r?\n/', trim($block)); + $row = []; + foreach ($lines as $line) { + $kv = explode(' = ', $line, 2); + if (count($kv) === 2) { + $row[$kv[0]] = $kv[1]; + } + } + + if (empty($row['Path'])) { + continue; + } + + $attr = $row['Attributes'] ?? ''; + if (str_contains($attr, 'D')) { + continue; // directory + } + + $encrypted = ($row['Encrypted'] ?? '') === '+'; + if ($encrypted) { + $anyEncrypted = true; + } + + $size = isset($row['Size']) && ctype_digit($row['Size']) ? (int) $row['Size'] : 0; + $files[] = ['name' => $row['Path'], 'size' => $size, 'encrypted' => $encrypted]; + + if (count($files) >= 200) { + break; + } + } + + if ($anyEncrypted && isset($files[0])) { + $files[0]['__any_encrypted__'] = true; + } + + return $files; + } + + /** + * Filter 7z files using extension whitelist. + */ + private function filterSevenZipFiles(array $files): array + { + $allowedExtensions = $this->getAllowedExtensions(); + $filtered = []; + + foreach ($files as $entry) { + if (! empty($entry['__any_encrypted__'])) { + continue; + } + + $name = $entry['name'] ?? ''; + if ($name === '' || strlen($name) > 300) { + continue; + } + + $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); + if (! in_array($ext, $allowedExtensions, true)) { + continue; + } + + $base = pathinfo($name, PATHINFO_FILENAME); + $letterCount = preg_match_all('/[a-z]/i', $base); + if ($letterCount <= 5) { + continue; + } + + $filtered[] = [ + 'name' => $name, + 'size' => $entry['size'] ?? 0, + 'date' => time(), + 'pass' => 0, + 'crc32' => '', + 'source' => '7z-list', + ]; + + if (count($filtered) >= 50) { + break; + } + } + + return $filtered; + } + + /** + * Scan for filenames in 7z raw data. + */ + private function scanSevenZipFilenames(string $data): array + { + $slice = substr($data, 0, 8 * 1024 * 1024); + $converted = preg_replace('/([\x20-\x7E])\x00/', '$1', $slice) ?? $slice; + $converted = str_replace("\x00", ' ', $converted); + + $exts = '7z|rar|zip|gz|bz2|xz|tar|tgz|mp4|mkv|avi|mpg|mpeg|mov|ts|wmv|flv|m4v|mp3|flac|ogg|wav|aac|aiff|ape|mka|nfo|txt|diz|pdf|epub|mobi|jpg|jpeg|png|gif|sfv|par2|exe|dll|srt|sub|idx|iso|bin|cue|mds|mdf'; + $regex = '~(?:[A-Za-z0-9 _.+&@#,()!-]{0,120}[\\/])?([A-Za-z0-9 _.+&@#,()!-]{2,160}\.(?:'.$exts.'))~i'; + preg_match_all($regex, $converted, $matches, PREG_SET_ORDER); + + if (empty($matches)) { + return []; + } + + $names = []; + foreach ($matches as $match) { + $candidate = preg_replace('/ {2,}/', ' ', $match[1]); + if ($candidate === '' || strlen($candidate) < 5 || substr_count($candidate, '.') > 10) { + continue; + } + $candidate = trim($candidate, " .-\t\n\r"); + $lower = strtolower($candidate); + if (! isset($names[$lower])) { + $names[$lower] = $candidate; + if (count($names) >= 80) { + break; + } + } + } + + return array_values($names); + } + + /** + * Extract using 7zip binary. + */ + public function extractViaSevenZip(string $compressedData, string $type, string $tmpPath): array + { + $result = [ + 'success' => false, + 'files' => [], + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + ]; + + if ($this->config->extractUsingRarInfo || ! $this->config->sevenZipPath) { + return $result; + } + + try { + $extMap = ['7z' => '7z', 'gzip' => 'gz', 'bzip2' => 'bz2', 'xz' => 'xz']; + $markerMap = ['7z' => '7z', 'gzip' => 'g', 'bzip2' => 'b', 'xz' => 'x']; + $ext = $extMap[$type] ?? 'dat'; + $marker = $markerMap[$type] ?? $type; + + $extractDir = $tmpPath.'un7z/'.uniqid('', true).'/'; + if (! File::isDirectory($extractDir)) { + File::makeDirectory($extractDir, 0777, true, true); + } + + $fileName = $tmpPath.uniqid('', true).'.'.$ext; + File::put($fileName, $compressedData); + + $cmd = [$this->config->sevenZipPath, 'e', '-y', '-bd', '-o'.$extractDir, $fileName]; + $exitCode = 0; + $stdout = null; + $stderr = null; + $this->execCommand($cmd, $exitCode, $stdout, $stderr); + + $files = []; + if (File::isDirectory($extractDir)) { + foreach (File::allFiles($extractDir) as $f) { + $files[] = [ + 'name' => $f->getFilename(), + 'size' => $f->getSize(), + 'date' => time(), + 'pass' => 0, + 'crc32' => '', + 'source' => $type, + ]; + } + } + + File::delete($fileName); + + if (! empty($files)) { + return [ + 'success' => true, + 'files' => $this->filterExtractedFiles($files), + 'hasPassword' => false, + 'passwordStatus' => Releases::PASSWD_NONE, + 'archiveMarker' => $marker, + ]; + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::warning(strtoupper($type).' extraction exception: '.$e->getMessage()); + } + } + + return $result; + } + + /** + * Filter extracted files by allowed extensions. + */ + private function filterExtractedFiles(array $files): array + { + $allowedExtensions = $this->getAllowedExtensions(); + $filtered = []; + + foreach ($files as $file) { + $name = $file['name'] ?? ''; + $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); + + if (! in_array($ext, $allowedExtensions, true)) { + continue; + } + + $base = pathinfo($name, PATHINFO_FILENAME); + $letterCount = preg_match_all('/[a-z]/i', $base); + if ($letterCount <= 5) { + continue; + } + + $filtered[] = $file; + } + + return $filtered; + } + + /** + * Get list of allowed file extensions. + */ + private function getAllowedExtensions(): array + { + return ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', + 'flac', 'png', 'epub', 'cbz', 'cbr', 'djvu']; + } + + /** + * Prepare extraction directories. + */ + private function prepareExtractionDirectories(string $tmpPath): void + { + if ($this->config->extractUsingRarInfo) { + return; + } + + try { + if ($this->config->unrarPath) { + $unrarDir = $tmpPath.'unrar/'; + if (! File::isDirectory($unrarDir)) { + File::makeDirectory($unrarDir, 0777, true, true); + } + } + $unzipDir = $tmpPath.'unzip/'; + if (! File::isDirectory($unzipDir)) { + File::makeDirectory($unzipDir, 0777, true, true); + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::warning('Failed ensuring extraction subdirectories: '.$e->getMessage()); + } + } + } + + /** + * Extract archive based on type. + */ + private function extractArchive(string $compressedData, array $dataSummary, string $tmpPath): string + { + $killString = $this->config->getKillString(); + + switch ($dataSummary['main_type']) { + case ArchiveInfo::TYPE_RAR: + if (! $this->config->extractUsingRarInfo && $this->config->unrarPath) { + $fileName = $tmpPath.uniqid('', true).'.rar'; + File::put($fileName, $compressedData); + runCmd($killString.$this->config->unrarPath.'" e -ai -ep -c- -id -inul -kb -or -p- -r -y "'.$fileName.'" "'.$tmpPath.'unrar/"'); + File::delete($fileName); + } + return 'r'; + + case ArchiveInfo::TYPE_ZIP: + if (! $this->config->extractUsingRarInfo && $this->config->unzipPath) { + $fileName = $tmpPath.uniqid('', true).'.zip'; + File::put($fileName, $compressedData); + runCmd($this->config->unzipPath.' -o "'.$fileName.'" -d "'.$tmpPath.'unzip/"'); + File::delete($fileName); + } + return 'z'; + } + + return ''; + } + + /** + * Detect standalone video from binary data. + */ + public function detectStandaloneVideo(string $data): ?string + { + $len = strlen($data); + if ($len < 16) { + return null; + } + + // AVI + if (strncmp($data, 'RIFF', 4) === 0 && substr($data, 8, 4) === 'AVI ') { + return 'avi'; + } + // Matroska / WebM + if (strncmp($data, "\x1A\x45\xDF\xA3", 4) === 0) { + return 'mkv'; + } + // MPEG + $sig4 = substr($data, 0, 4); + if ($sig4 === "\x00\x00\x01\xBA" || $sig4 === "\x00\x00\x01\xB3") { + return 'mpg'; + } + // Transport Stream + if ($len >= 188 * 5) { + $isTs = true; + for ($i = 0; $i < 5; $i++) { + if (! isset($data[188 * $i]) || $data[188 * $i] !== "\x47") { + $isTs = false; + break; + } + } + if ($isTs) { + return 'mpg'; + } + } + // MP4/MOV + if ($len >= 12 && substr($data, 4, 4) === 'ftyp') { + $brands = ['isom', 'iso2', 'avc1', 'mp41', 'mp42', 'dash', 'MSNV', 'qt ', 'M4V ', 'M4P ', 'M4B ', 'M4A ']; + if (in_array(substr($data, 8, 4), $brands, true)) { + return 'mp4'; + } + } + + return null; + } + + /** + * Get PAR2 info parser. + */ + public function getPar2Info(): Par2Info + { + return $this->par2Info; + } + + /** + * Get archive info handler. + */ + public function getArchiveInfo(): ArchiveInfo + { + return $this->archiveInfo; + } + + /** + * Execute a command with output capture. + */ + private function execCommand(array $cmd, ?int &$exitCode, ?string &$stdout, ?string &$stderr): bool + { + $descriptorSpec = [ + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + + $process = @proc_open($cmd, $descriptorSpec, $pipes, null, null, ['bypass_shell' => true]); + if (! is_resource($process)) { + $exitCode = -1; + return false; + } + + $stdout = stream_get_contents($pipes[1]); + fclose($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[2]); + $exitCode = proc_close($process); + + return $exitCode === 0; + } +} + diff --git a/app/Services/AdditionalProcessing/Config/ProcessingConfiguration.php b/app/Services/AdditionalProcessing/Config/ProcessingConfiguration.php new file mode 100644 index 000000000..1bbe00ad9 --- /dev/null +++ b/app/Services/AdditionalProcessing/Config/ProcessingConfiguration.php @@ -0,0 +1,110 @@ +echoCLI = (bool) config('nntmux.echocli'); + $this->innerFileBlacklist = Settings::settingValue('innerfileblacklist') === '' + ? false + : Settings::settingValue('innerfileblacklist'); + $this->maxNestedLevels = (int) Settings::settingValue('maxnestedlevels') ?: 3; + $this->extractUsingRarInfo = (int) Settings::settingValue('extractusingrarinfo') !== 0; + $this->fetchLastFiles = (bool) config('nntmux_settings.fetch_last_file'); + $this->unrarPath = config('nntmux_settings.unrar_path') ?: false; + $this->unzipPath = config('nntmux_settings.unzip_path') ?: false; + $this->sevenZipPath = config('nntmux_settings.7zip_path') ?: false; + $this->timeoutPath = config('nntmux_settings.timeout_path') ?: false; + $this->timeoutSeconds = (int) Settings::settingValue('timeoutseconds'); + $this->queryLimit = (int) (Settings::settingValue('maxaddprocessed') ?: 25); + $this->segmentsToDownload = (int) (Settings::settingValue('segmentstodownload') ?: 2); + $this->maximumRarSegments = (int) (Settings::settingValue('maxpartsprocessed') ?: 3); + $this->maximumRarPasswordChecks = max((int) (Settings::settingValue('passchkattempts') ?: 1), 1); + $this->maxSizeGB = (int) (Settings::settingValue('maxsizetopostprocess') ?: 100); + $this->minSizeMB = (int) (Settings::settingValue('minsizetopostprocess') ?: 100); + $this->alternateNNTP = (bool) config('nntmux_nntp.use_alternate_nntp_server'); + $this->ffmpegDuration = (int) (Settings::settingValue('ffmpeg_duration') ?: 5); + $this->addPAR2Files = (bool) config('nntmux_settings.add_par2'); + $this->ffmpegPath = config('nntmux_settings.ffmpeg_path') ?: false; + $this->mediaInfoPath = config('nntmux_settings.mediainfo_path') ?: false; + if (! $this->ffmpegPath) { + $this->processAudioSample = false; + $this->processThumbnails = false; + $this->processVideo = false; + } else { + $this->processAudioSample = (int) Settings::settingValue('saveaudiopreview') !== 0; + $this->processThumbnails = (int) Settings::settingValue('processthumbnails') !== 0; + $this->processVideo = (int) Settings::settingValue('processvideos') !== 0; + } + $this->processJPGSample = (int) Settings::settingValue('processjpg') !== 0; + $this->processMediaInfo = (bool) $this->mediaInfoPath; + $this->processAudioInfo = $this->processMediaInfo; + $this->processPasswords = config('nntmux_settings.check_passworded_rars') === true + && ! empty(config('nntmux_settings.unrar_path')); + $this->audioSavePath = config('nntmux_settings.covers_path').'/audiosample/'; + $this->tmpUnrarPath = config('nntmux.tmp_unrar_path'); + $this->debugMode = (bool) config('app.debug'); + $this->elasticsearchEnabled = config('nntmux.elasticsearch_enabled') === true; + $this->renameMusicMediaInfo = (bool) config('nntmux.rename_music_mediainfo'); + $this->renamePar2 = (bool) config('nntmux.rename_par2'); + // Regex patterns + $this->audioFileRegex = '\\.(AAC|AIFF|APE|AC3|ASF|DTS|FLAC|MKA|MKS|MP2|MP3|RA|OGG|OGM|W64|WAV|WMA)'; + $this->ignoreBookRegex = '/\\b(epub|lit|mobi|pdf|sipdf|html)\\b.*\\.rar(?!.{20,})/i'; + $this->supportFileRegex = '\\.(?:vol\\d{1,3}\\+\\d{1,3}|par2|srs|sfv|nzb)'; + $this->videoFileRegex = '\\.(AVI|F4V|IFO|M1V|M2V|M4V|MKV|MOV|MP4|MPEG|MPG|MPGV|MPV|OGV|QT|RM|RMVB|TS|VOB|WMV)'; + } + /** + * Build the kill string for timeout command wrapper. + */ + public function getKillString(): string + { + if ($this->timeoutPath && $this->timeoutSeconds > 0) { + return '"'.$this->timeoutPath.'" --foreground --signal=KILL '.$this->timeoutSeconds.' "'; + } + return '"'; + } +} diff --git a/app/Services/AdditionalProcessing/ConsoleOutputService.php b/app/Services/AdditionalProcessing/ConsoleOutputService.php new file mode 100644 index 000000000..aa5e38707 --- /dev/null +++ b/app/Services/AdditionalProcessing/ConsoleOutputService.php @@ -0,0 +1,87 @@ +colorCLI = new ColorCLI(); + } + public function echo(string $message, string $type = 'primary'): void + { + if ($this->echoCLI) { + $this->colorCLI->$type($message); + } + } + public function debug(string $message): void + { + if ($this->echoCLI && config('app.env') === 'local' && config('app.debug') === true) { + $this->colorCLI->debug('DEBUG: '.$message); + } + } + public function echoDescription(int $totalReleases): void + { + if ($totalReleases > 1 && $this->echoCLI) { + $this->echo( + PHP_EOL. + 'Additional post-processing, started at: '. + now()->format('D M d, Y G:i a'). + PHP_EOL. + 'Downloaded: (xB)=yEnc article, (cB)=compressed part'. + PHP_EOL. + 'Failures: fC#=Compressed(part #), fS=Sample, fM=Media(video), fA=Audio, fJ=JPEG, G=Missing group'. + PHP_EOL. + 'Processing: r=RAR, z=ZIP, 7z=7zip (names/entries), g=GZIP, b=BZIP2, x=XZ, (vRAW)=Inline video detected'. + PHP_EOL. + 'Added: s=Sample image, j=JPEG image, A=Audio sample, a=Audio MediaInfo, v=Video sample'. + PHP_EOL. + 'Added: m=Video MediaInfo, n=NFO, ^=Inner file details (RAR/ZIP/7z/etc)'. + '', + 'header' + ); + } + } + public function echoReleaseStart(int $releaseId, int $size): void + { + $this->echo(PHP_EOL.'['.$releaseId.']['.human_filesize($size, 1).']', 'primaryOver'); + } + public function echoCompressedDownload(): void { $this->echo('(cB)', 'primaryOver'); } + public function echoCompressedFailure(int $failCount): void { $this->echo('fC'.$failCount, 'warningOver'); } + public function echoGroupUnavailable(): void { $this->echo('G', 'warningOver'); } + public function echoSampleDownload(): void { $this->echo('(sB)', 'primaryOver'); } + public function echoSampleFailure(): void { $this->echo('fS', 'warningOver'); } + public function echoMediaInfoDownload(): void { $this->echo('(mB)', 'primaryOver'); } + public function echoMediaInfoFailure(): void { $this->echo('fM', 'warningOver'); } + public function echoAudioDownload(): void { $this->echo('(aB)', 'primaryOver'); } + public function echoAudioFailure(): void { $this->echo('fA', 'warningOver'); } + public function echoJpgDownload(): void { $this->echo('(jB)', 'primaryOver'); } + public function echoJpgFailure(): void { $this->echo('fJ', 'warningOver'); } + public function echoArchiveMarker(string $marker): void { $this->echo($marker, 'primaryOver'); } + public function echoFileInfoAdded(): void { $this->echo('^', 'primaryOver'); } + public function echoSampleCreated(): void { $this->echo('s', 'primaryOver'); } + public function echoJpgSaved(): void { $this->echo('j', 'primaryOver'); } + public function echoAudioSampleCreated(): void { $this->echo('A', 'primaryOver'); } + public function echoAudioInfoAdded(): void { $this->echo('a', 'primaryOver'); } + public function echoVideoCreated(): void { $this->echo('v', 'primaryOver'); } + public function echoMediaInfoAdded(): void { $this->echo('m', 'primaryOver'); } + public function echoNfoFound(): void { $this->echo('n', 'primaryOver'); } + public function echoInlineVideo(): void { $this->echo('(vRAW)', 'primaryOver'); } + public function warning(string $message): void { $this->echo($message, 'warning'); } + public function echoReleaseDeleted(int $releaseId): void { $this->echo('Deleted broken release ID '.$releaseId, 'warningOver'); } + public function endOutput(): void + { + if ($this->echoCLI) { + echo PHP_EOL; + } + } + public function isEnabled(): bool + { + return $this->echoCLI; + } +} diff --git a/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php b/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php new file mode 100644 index 000000000..8d1910baa --- /dev/null +++ b/app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php @@ -0,0 +1,118 @@ +release = $release; + } + /** + * Initialize processing state based on configuration flags. + * Sets "found" flags to true if processing is disabled (to skip those steps). + */ + public function initializeFromConfig( + bool $processVideo, + bool $processMediaInfo, + bool $processAudioInfo, + bool $processAudioSample, + bool $processJPGSample, + bool $processThumbnails + ): void { + $this->foundVideo = ! $processVideo; + $this->foundMediaInfo = ! $processMediaInfo; + $this->foundAudioInfo = ! $processAudioInfo; + $this->foundAudioSample = ! $processAudioSample; + $this->foundJPGSample = ! $processJPGSample; + $this->foundSample = ! $processThumbnails; + $this->foundPAR2Info = false; + } + /** + * Reset message ID arrays for a fresh processing run. + */ + public function resetMessageIDs(): void + { + $this->sampleMessageIDs = []; + $this->jpgMessageIDs = []; + $this->mediaInfoMessageIDs = []; + $this->audioInfoMessageIDs = []; + $this->rarFileMessageIDs = []; + $this->audioInfoExtension = ''; + } + /** + * Reset file counters. + */ + public function resetCounters(): void + { + $this->addedFileInfo = 0; + $this->totalFileInfo = 0; + $this->compressedFilesChecked = 0; + } + /** + * Full reset for processing a new release. + */ + public function reset(): void + { + $this->passwordStatus = 0; + $this->releaseHasPassword = false; + $this->nzbHasCompressedFile = false; + $this->groupUnavailable = false; + $this->resetMessageIDs(); + $this->resetCounters(); + } + /** + * Check if more media processing is needed. + */ + public function needsMediaProcessing(): bool + { + return ! $this->foundVideo + || ! $this->foundMediaInfo + || ! $this->foundAudioInfo + || ! $this->foundAudioSample; + } + /** + * Check if any sample is still needed. + */ + public function needsSample(): bool + { + return ! $this->foundSample || ! $this->foundJPGSample; + } +} diff --git a/app/Services/AdditionalProcessing/MediaExtractionService.php b/app/Services/AdditionalProcessing/MediaExtractionService.php new file mode 100644 index 000000000..02801e75f --- /dev/null +++ b/app/Services/AdditionalProcessing/MediaExtractionService.php @@ -0,0 +1,490 @@ +ffprobe()->isValid($videoLocation)) { + return ''; + } + $time = $this->ffprobe()->format($videoLocation)->get('duration'); + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::debug($e->getMessage()); + } + return ''; + } + + if (empty($time) || ! preg_match('/time=(\d{1,2}:\d{1,2}:)?(\d{1,2})\.(\d{1,2})\s*bitrate=/i', $time, $numbers)) { + return ''; + } + + if ($numbers[3] > 0) { + $numbers[3]--; + } elseif ($numbers[1] > 0) { + $numbers[2]--; + $numbers[3] = '99'; + } + + return '00:00:'.str_pad($numbers[2], 2, '0', STR_PAD_LEFT).'.'.str_pad($numbers[3], 2, '0', STR_PAD_LEFT); + } + + /** + * Extract a sample image from a video file. + */ + public function getSample(string $fileLocation, string $tmpPath, string $guid): bool + { + if (! $this->config->processThumbnails || ! File::isFile($fileLocation)) { + return false; + } + + $fileName = $tmpPath.'zzzz'.random_int(5, 12).random_int(5, 12).'.jpg'; + $time = $this->getVideoTime($fileLocation); + + try { + if ($this->ffprobe()->isValid($fileLocation)) { + $this->ffmpeg()->open($fileLocation) + ->frame(TimeCode::fromString($time === '' ? '00:00:03:00' : $time)) + ->save($fileName); + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::error($e->getTraceAsString()); + } + return false; + } + + if (! File::isFile($fileName)) { + return false; + } + + $saved = $this->releaseImage->saveImage( + $guid.'_thumb', + $fileName, + $this->releaseImage->imgSavePath, + 800, + 600 + ); + + File::delete($fileName); + + return $saved === 1; + } + + /** + * Create a video sample clip. + */ + public function getVideo(string $fileLocation, string $tmpPath, string $guid): bool + { + if (! $this->config->processVideo || ! File::isFile($fileLocation)) { + return false; + } + + $fileName = $tmpPath.'zzzz'.$guid.'.ogv'; + $newMethod = false; + + // Try to get sample from end of video if duration is short + if ($this->config->ffmpegDuration < 60) { + $time = $this->getVideoTime($fileLocation); + if ($time !== '' && preg_match('/(\d{2}).(\d{2})/', $time, $numbers)) { + $newMethod = true; + if ($numbers[1] <= $this->config->ffmpegDuration) { + $lowestLength = '00:00:00.00'; + } else { + $lowestLength = ($numbers[1] - $this->config->ffmpegDuration); + $end = '.'.$numbers[2]; + $lowestLength = match (strlen($lowestLength)) { + 1 => '00:00:0'.$lowestLength.$end, + 2 => '00:00:'.$lowestLength.$end, + default => '00:00:60.00', + }; + } + + try { + if ($this->ffprobe()->isValid($fileLocation)) { + $video = $this->ffmpeg()->open($fileLocation); + $clip = $video->clip( + TimeCode::fromString($lowestLength), + TimeCode::fromSeconds($this->config->ffmpegDuration) + ); + $format = new Ogg(); + $format->setAudioCodec('libvorbis'); + $clip->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); + $clip->save($format, $fileName); + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::error($e->getTraceAsString()); + } + } + } + } + + // Fallback: use start of video + if (! $newMethod) { + try { + if ($this->ffprobe()->isValid($fileLocation)) { + $video = $this->ffmpeg()->open($fileLocation); + $clip = $video->clip( + TimeCode::fromSeconds(0), + TimeCode::fromSeconds($this->config->ffmpegDuration) + ); + $format = new Ogg(); + $format->setAudioCodec('libvorbis'); + $clip->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); + $clip->save($format, $fileName); + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::error($e->getTraceAsString()); + } + } + } + + if (! File::isFile($fileName)) { + return false; + } + + $newFile = $this->releaseImage->vidSavePath.$guid.'.ogv'; + + if (! @File::move($fileName, $newFile)) { + $copied = @File::copy($fileName, $newFile); + File::delete($fileName); + if (! $copied) { + return false; + } + } + + @chmod($newFile, 0764); + Release::query()->where('guid', $guid)->update(['videostatus' => 1]); + + return true; + } + + /** + * Extract media info from a video file. + */ + public function getMediaInfo(string $fileLocation, int $releaseId): bool + { + if (! $this->config->processMediaInfo || ! File::isFile($fileLocation)) { + return false; + } + + try { + $xmlArray = $this->mediaInfo()->getInfo($fileLocation, true); + \App\Models\MediaInfo::addData($releaseId, $xmlArray); + $this->releaseExtra->addFromXml($releaseId, $xmlArray); + return true; + } catch (\Throwable $e) { + Log::debug($e->getMessage()); + return false; + } + } + + /** + * Process a JPG sample image. + */ + public function getJPGSample(string $fileLocation, string $guid): bool + { + $saved = $this->releaseImage->saveImage( + $guid.'_thumb', + $fileLocation, + $this->releaseImage->jpgSavePath, + 650, + 650 + ); + + if ($saved === 1) { + Release::query()->where('guid', $guid)->update(['jpgstatus' => 1]); + return true; + } + + return false; + } + + /** + * Process audio file for media info and sample. + * + * @return array{audioInfo: bool, audioSample: bool} + */ + public function getAudioInfo( + string $fileLocation, + string $fileExtension, + ReleaseProcessingContext $context, + string $tmpPath + ): array { + $result = ['audioInfo' => false, 'audioSample' => false]; + + if (! $this->config->processAudioSample) { + $result['audioSample'] = true; + } + if (! $this->config->processAudioInfo) { + $result['audioInfo'] = true; + } + + $rQuery = Release::query() + ->where('proc_pp', '=', 0) + ->where('id', $context->release->id) + ->select(['searchname', 'fromname', 'categories_id', 'groups_id']) + ->first(); + + $musicParent = (string) Category::MUSIC_ROOT; + if ($rQuery === null || ! preg_match( + sprintf( + '/%d\d{3}|%d|%d|%d/', + $musicParent[0], + Category::OTHER_MISC, + Category::MOVIE_OTHER, + Category::TV_OTHER + ), + $rQuery->categories_id + )) { + return $result; + } + + if (! File::isFile($fileLocation)) { + return $result; + } + + // Get media info + if (! $result['audioInfo']) { + try { + $xmlArray = $this->mediaInfo()->getInfo($fileLocation, false); + if ($xmlArray !== null) { + foreach ($xmlArray->getAudios() as $track) { + if ($track->get('album') !== null && $track->get('performer') !== null) { + if ((int) $context->release->predb_id === 0 && $this->config->renameMusicMediaInfo) { + $ext = strtoupper($fileExtension); + + $newName = $track->get('performer')->getFullName().' - '.$track->get('album')->getFullName(); + if (! empty($track->get('recorded_date')) + && preg_match('/(?:19|20)\d\d/', $track->get('recorded_date')->getFullname, $Year) + ) { + $newName .= ' ('.$Year[0].') '.$ext; + } else { + $newName .= ' '.$ext; + } + + $newCat = match ($ext) { + 'MP3' => Category::MUSIC_MP3, + 'FLAC' => Category::MUSIC_LOSSLESS, + default => $this->categorize->determineCategory($rQuery->groups_id, $newName, $rQuery->fromname), + }; + + $newTitle = escapeString(substr($newName, 0, 255)); + Release::whereId($context->release->id)->update([ + 'searchname' => $newTitle, + 'categories_id' => is_array($newCat) ? $newCat['categories_id'] : $newCat, + 'iscategorized' => 1, + 'isrenamed' => 1, + 'proc_pp' => 1, + ]); + + if ($this->config->elasticsearchEnabled) { + $this->elasticsearch()->updateRelease($context->release->id); + } else { + $this->manticore()->updateRelease($context->release->id); + } + + if ($this->config->echoCLI) { + NameFixer::echoChangedReleaseName([ + 'new_name' => $newTitle, + 'old_name' => $rQuery->searchname, + 'new_category' => $newCat, + 'old_category' => $rQuery->categories_id, + 'group' => $rQuery->groups_id, + 'releases_id' => $context->release->id, + 'method' => 'MediaExtractionService->getAudioInfo', + ]); + } + } + + $this->releaseExtra->addFromXml($context->release->id, $xmlArray); + $result['audioInfo'] = true; + $context->foundAudioInfo = true; + break; + } + } + } + } catch (\Throwable $e) { + Log::debug($e->getMessage()); + } + } + + // Create audio sample + if (! $result['audioSample']) { + $audioFileName = $context->release->guid.'.ogg'; + + try { + if ($this->ffprobe()->isValid($fileLocation)) { + $audioSample = $this->ffmpeg()->open($fileLocation); + $format = new Vorbis(); + $audioSample->clip(TimeCode::fromSeconds(30), TimeCode::fromSeconds(30)); + $audioSample->save($format, $tmpPath.$audioFileName); + } + } catch (\Throwable $e) { + if ($this->config->debugMode) { + Log::error($e->getTraceAsString()); + } + } + + if (File::isFile($tmpPath.$audioFileName)) { + $renamed = File::move($tmpPath.$audioFileName, $this->config->audioSavePath.$audioFileName); + if (! $renamed) { + $copied = File::copy($tmpPath.$audioFileName, $this->config->audioSavePath.$audioFileName); + File::delete($tmpPath.$audioFileName); + if (! $copied) { + return $result; + } + } + + @chmod($this->config->audioSavePath.$audioFileName, 0764); + Release::query()->where('id', $context->release->id)->update(['audiostatus' => 1]); + $result['audioSample'] = true; + $context->foundAudioSample = true; + } + } + + return $result; + } + + /** + * Process a video file for sample, video clip, and media info. + */ + public function processVideoFile( + string $fileLocation, + ReleaseProcessingContext $context, + string $tmpPath + ): array { + $result = [ + 'sample' => false, + 'video' => false, + 'mediaInfo' => false, + ]; + + if (! $context->foundSample) { + $result['sample'] = $this->getSample($fileLocation, $tmpPath, $context->release->guid); + if ($result['sample']) { + $context->foundSample = true; + } + } + + // Only get video if sampleMessageIDs count is less than 2 + if (! $context->foundVideo && count($context->sampleMessageIDs) < 2) { + $result['video'] = $this->getVideo($fileLocation, $tmpPath, $context->release->guid); + if ($result['video']) { + $context->foundVideo = true; + } + } + + if (! $context->foundMediaInfo) { + $result['mediaInfo'] = $this->getMediaInfo($fileLocation, $context->release->id); + if ($result['mediaInfo']) { + $context->foundMediaInfo = true; + } + } + + return $result; + } + + /** + * Check if data appears to be a JPEG image. + */ + public function isJpegData(string $filePath): bool + { + if (! File::isFile($filePath)) { + return false; + } + return exif_imagetype($filePath) === IMAGETYPE_JPEG; + } + + private function ffmpeg(): FFMpeg + { + if ($this->ffmpeg === null) { + $timeout = $this->config->timeoutSeconds > 0 ? $this->config->timeoutSeconds : 60; + $this->ffmpeg = FFMpeg::create(['timeout' => $timeout]); + } + return $this->ffmpeg; + } + + private function ffprobe(): FFProbe + { + if ($this->ffprobe === null) { + $this->ffprobe = FFProbe::create(); + } + return $this->ffprobe; + } + + private function mediaInfo(): MediaInfo + { + if ($this->mediaInfo === null) { + $this->mediaInfo = new MediaInfo(); + $this->mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true); + if ($this->config->mediaInfoPath) { + $this->mediaInfo->setConfig('command', $this->config->mediaInfoPath); + } + } + return $this->mediaInfo; + } + + private function manticore(): ManticoreSearch + { + if ($this->manticore === null) { + $this->manticore = new ManticoreSearch(); + } + return $this->manticore; + } + + private function elasticsearch(): ElasticSearchSiteSearch + { + if ($this->elasticsearch === null) { + $this->elasticsearch = new ElasticSearchSiteSearch(); + } + return $this->elasticsearch; + } +} + diff --git a/app/Services/AdditionalProcessing/NzbContentParser.php b/app/Services/AdditionalProcessing/NzbContentParser.php new file mode 100644 index 000000000..74d9d5246 --- /dev/null +++ b/app/Services/AdditionalProcessing/NzbContentParser.php @@ -0,0 +1,264 @@ +nzb->NZBPath($guid); + if ($nzbPath === false) { + return ['contents' => [], 'error' => 'NZB not found for GUID: '.$guid]; + } + + $nzbContents = Utility::unzipGzipFile($nzbPath); + if (! $nzbContents) { + // Try repair on raw file contents + $nzbContents = $this->attemptRawRepair($nzbPath); + if (! $nzbContents) { + return ['contents' => [], 'error' => 'NZB is empty or broken for GUID: '.$guid]; + } + } + + // Get a list of files in the NZB + $fileList = $this->nzb->nzbFileList($nzbContents, ['no-file-key' => false, 'strip-count' => true]); + if (count($fileList) === 0) { + // Attempt repair if initial parse yielded no files + $repaired = $this->repairNzb($nzbContents, $nzbPath, $guid); + if ($repaired !== null) { + $fileList = $this->nzb->nzbFileList($repaired, ['no-file-key' => false, 'strip-count' => true]); + } + if (count($fileList) === 0) { + return ['contents' => [], 'error' => 'NZB is potentially broken for GUID: '.$guid]; + } + } + + // Sort keys naturally + ksort($fileList, SORT_NATURAL); + + return ['contents' => $fileList, 'error' => null]; + } + + /** + * Attempt to repair raw file contents before XML parsing. + */ + private function attemptRawRepair(string $nzbPath): ?string + { + try { + $rawFile = @File::get($nzbPath); + if (! $rawFile) { + return null; + } + + // If gzipped, attempt decompress + if (str_ends_with(strtolower($nzbPath), '.gz')) { + $decompressed = @gzdecode($rawFile); + if ($decompressed !== false) { + return $this->repairNzb($decompressed, $nzbPath, ''); + } + } else { + return $this->repairNzb($rawFile, $nzbPath, ''); + } + } catch (\Throwable) { + // Ignore + } + + return null; + } + + /** + * Attempt to repair a potentially broken NZB XML string. + */ + public function repairNzb(string $raw, string $originalPath, string $guid): ?string + { + // Remove common binary / control chars except tab, newline, carriage return + $fixed = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F]/', '', $raw); + + // If missing opening tag, wrap content + if (! str_contains(strtolower($fixed), '\n\n".$fixed."\n"; + } else { + // Ensure closing tag + if (! preg_match('/<\/nzb>\s*$/i', $fixed)) { + $fixed .= "\n"; + } + } + + // Try to parse using libxml recovery + $opts = LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_COMPACT | LIBXML_NONET | LIBXML_NOCDATA | LIBXML_PARSEHUGE; + $prev = libxml_use_internal_errors(true); + $xml = simplexml_load_string($fixed, 'SimpleXMLElement', $opts); + $errors = libxml_get_errors(); + libxml_clear_errors(); + libxml_use_internal_errors($prev); + + if ($xml === false || empty($xml->file)) { + if ($this->debugMode && $this->echoCLI) { + $guidInfo = $guid ? ' for GUID: '.$guid : ''; + echo 'NZB repair failed'.$guidInfo.' ('.count($errors).' XML errors)'.PHP_EOL; + } + return null; + } + + // Persist a repaired version if content changed + try { + if ($fixed !== $raw) { + if (str_ends_with(strtolower($originalPath), '.gz')) { + @File::put($originalPath, gzencode($fixed)); + } else { + @File::put($originalPath, $fixed); + } + } + } catch (\Throwable $e) { + if ($this->debugMode) { + Log::debug('Failed to persist repaired NZB: '.$e->getMessage()); + } + } + + return $fixed; + } + + /** + * Process NZB contents to extract message IDs for different file types. + * + * @return array{ + * hasCompressedFile: bool, + * sampleMessageIDs: array, + * jpgMessageIDs: array, + * mediaInfoMessageID: string, + * audioInfoMessageID: string, + * audioInfoExtension: string, + * bookFileCount: int + * } + */ + public function extractMessageIDs( + array $nzbContents, + string $groupName, + int $segmentsToDownload, + bool $processThumbnails, + bool $processJPGSample, + bool $processMediaInfo, + bool $processAudioInfo, + string $audioFileRegex, + string $videoFileRegex, + string $supportFileRegex, + string $ignoreBookRegex + ): array { + $result = [ + 'hasCompressedFile' => false, + 'sampleMessageIDs' => [], + 'jpgMessageIDs' => [], + 'mediaInfoMessageID' => '', + 'audioInfoMessageID' => '', + 'audioInfoExtension' => '', + 'bookFileCount' => 0, + ]; + + foreach ($nzbContents as $file) { + try { + $title = $file['title'] ?? ''; + $segments = $file['segments'] ?? []; + + // Skip support/nfo files + if (preg_match('/(?:'.$supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $title)) { + continue; + } + + // Compressed file detection + if (! $result['hasCompressedFile'] && preg_match( + '/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))("|\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', + $title + )) { + $result['hasCompressedFile'] = true; + } + + // Look for a video sample (not an image) + if ($processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments) + && stripos($title, 'sample') !== false + && ! preg_match('/\.jpe?g$/i', $title) + ) { + $result['sampleMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload); + } + + // Look for a JPG picture (not a CD cover) + if ($processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments) + && ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName) + && preg_match('/\.jpe?g[. ")\]]/i', $title) + ) { + $result['jpgMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload); + } + + // Look for a video file for MediaInfo (sample video) + if ($processMediaInfo && empty($result['mediaInfoMessageID']) && ! empty($segments[0]) + && stripos($title, 'sample') !== false + && preg_match('/'.$videoFileRegex.'[. ")\]]/i', $title) + ) { + $result['mediaInfoMessageID'] = (string) $segments[0]; + } + + // Look for an audio file + if ($processAudioInfo && empty($result['audioInfoMessageID']) && ! empty($segments) + && preg_match('/'.$audioFileRegex.'[. ")\]]/i', $title, $type) + ) { + $result['audioInfoExtension'] = $type[1]; + $result['audioInfoMessageID'] = (string) $segments[0]; + } + + // Count book files + if (preg_match($ignoreBookRegex, $title)) { + $result['bookFileCount']++; + } + } catch (\ErrorException $e) { + Log::debug($e->getTraceAsString()); + } + } + + return $result; + } + + /** + * Extract segment message IDs up to a limit. + */ + private function extractSegments(array $segments, int $limit): array + { + $ids = []; + $segCount = count($segments) - 1; + for ($i = 0; $i < $limit; $i++) { + if ($i > $segCount) { + break; + } + $ids[] = (string) $segments[$i]; + } + return $ids; + } + + /** + * Get the NZB path for a GUID. + */ + public function getNzbPath(string $guid): string|false + { + return $this->nzb->NZBPath($guid); + } +} + diff --git a/app/Services/AdditionalProcessing/ReleaseFileManager.php b/app/Services/AdditionalProcessing/ReleaseFileManager.php new file mode 100644 index 000000000..960f6b588 --- /dev/null +++ b/app/Services/AdditionalProcessing/ReleaseFileManager.php @@ -0,0 +1,521 @@ +config->debugMode) { + Log::debug("Error: {$file['error']} (in: {$file['source']})"); + } + return false; + } + + if (! isset($file['name'])) { + return false; + } + + // Check for password + if (isset($file['pass']) && $file['pass'] === true) { + $context->releaseHasPassword = true; + $context->passwordStatus = Releases::PASSWD_RAR; + return false; + } + + // Check inner file blacklist + if ($this->config->innerFileBlacklist !== false + && preg_match($this->config->innerFileBlacklist, $file['name']) + ) { + $context->releaseHasPassword = true; + $context->passwordStatus = Releases::PASSWD_RAR; + return false; + } + + // Skip support files + if (preg_match( + '/(?:'.$supportFileRegex.'|part\d+|[rz]\d{1,3}|zipr\d{2,3}|\d{2,3}|zipx?|zip|rar|7z|gz|bz2|xz)(\s*\.rar)?$/i', + $file['name'] + )) { + return false; + } + + // Increment total file info count + $context->totalFileInfo++; + + // Don't add too many files + if ($context->addedFileInfo >= 11) { + return false; + } + + // Check if a file already exists + $exists = ReleaseFile::query() + ->where([ + 'releases_id' => $context->release->id, + 'name' => $file['name'], + 'size' => $file['size'], + ]) + ->first(); + + if ($exists !== null) { + return false; + } + + // Add the file + $added = ReleaseFile::addReleaseFiles( + $context->release->id, + $file['name'], + $file['size'], + $file['date'], + $file['pass'] ?? 0, + '', + $file['crc32'] ?? '' + ); + + if (! empty($added)) { + $context->addedFileInfo++; + + // Check for codec spam + if (preg_match('#(?:^|[/\\\\])Codec[/\\\\]Setup\.exe$#i', $file['name'])) { + if ($this->config->debugMode) { + Log::debug('Codec spam found, setting release to potentially passworded.'); + } + $context->releaseHasPassword = true; + $context->passwordStatus = Releases::PASSWD_RAR; + } elseif ($file['name'] !== '' && ! str_starts_with($file['name'], '.')) { + // Run PreDB filename check + $context->release['filename'] = $file['name']; + $context->release['releases_id'] = $context->release->id; + $this->nameFixer->matchPreDbFiles($context->release, 1, 1, true); + } + + return true; + } + + return false; + } + + /** + * Update search indexes after adding file info. + */ + public function updateSearchIndex(int $releaseId): void + { + if ($this->config->elasticsearchEnabled) { + $this->elasticsearch()->updateRelease($releaseId); + } else { + $this->manticore()->updateRelease($releaseId); + } + } + + /** + * Finalize release processing with status updates. + */ + public function finalizeRelease(ReleaseProcessingContext $context, bool $processPasswords): void + { + $updateRows = ['haspreview' => 0]; + + // Check for existing samples + if (File::isFile($this->releaseImage->imgSavePath.$context->release->guid.'_thumb.jpg')) { + $updateRows = ['haspreview' => 1]; + } + + if (File::isFile($this->releaseImage->vidSavePath.$context->release->guid.'.ogv')) { + $updateRows['videostatus'] = 1; + } + + if (File::isFile($this->releaseImage->jpgSavePath.$context->release->guid.'_thumb.jpg')) { + $updateRows['jpgstatus'] = 1; + } + + // Get file count + $releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0; + + $passwordStatus = max([$context->passwordStatus]); + + // Set to no password if processing is off + if (! $processPasswords) { + $context->releaseHasPassword = false; + } + + // Update based on conditions + if (! $context->releaseHasPassword && $context->nzbHasCompressedFile && $releaseFilesCount === 0) { + Release::query()->where('id', $context->release->id)->update($updateRows); + } else { + $updateRows['passwordstatus'] = $processPasswords ? $passwordStatus : Releases::PASSWD_NONE; + $updateRows['rarinnerfilecount'] = $releaseFilesCount; + Release::query()->where('id', $context->release->id)->update($updateRows); + } + } + + /** + * Delete a broken release completely. + */ + public function deleteRelease(Release $release): void + { + try { + if (empty($release->id)) { + return; + } + + $id = (int) $release->id; + $guid = $release->guid ?? ''; + + // Delete NZB file + try { + $nzbPath = $this->nzb->NZBPath($guid); + if ($nzbPath && File::exists($nzbPath)) { + File::delete($nzbPath); + } + } catch (\Throwable) { + // Ignore + } + + // Delete preview assets + try { + $files = [ + $this->releaseImage->imgSavePath.$guid.'_thumb.jpg', + $this->releaseImage->jpgSavePath.$guid.'_thumb.jpg', + $this->releaseImage->vidSavePath.$guid.'.ogv', + ]; + foreach ($files as $file) { + if ($file && File::exists($file)) { + File::delete($file); + } + } + } catch (\Throwable) { + // Ignore + } + + // Delete related database rows + try { + ReleaseFile::where('releases_id', $id)->delete(); + } catch (\Throwable) { + } + + try { + MediaInfoModel::where('releases_id', $id)->delete(); + } catch (\Throwable) { + } + + // Delete from search index + try { + if ($this->config->elasticsearchEnabled) { + $this->elasticsearch()->deleteRelease($id); + } else { + $this->manticore()->deleteRelease(['i' => $id, 'g' => $guid]); + } + } catch (\Throwable) { + // Ignore + } + + // Delete release row + try { + Release::where('id', $id)->delete(); + } catch (\Throwable) { + } + } catch (\Throwable) { + // Last resort: swallow any exception + } + } + + /** + * Process PAR2 file for file info and release name matching. + */ + public function processPar2File( + string $fileLocation, + ReleaseProcessingContext $context, + \dariusiii\rarinfo\Par2Info $par2Info + ): bool { + $par2Info->open($fileLocation); + + if ($par2Info->error) { + return false; + } + + $releaseInfo = Release::query() + ->where('id', $context->release->id) + ->select(['postdate', 'proc_pp']) + ->first(); + + if ($releaseInfo === null) { + return false; + } + + $postDate = Carbon::createFromFormat('Y-m-d H:i:s', $releaseInfo->postdate)->getTimestamp(); + + // Only get new name if category is OTHER + $foundName = true; + if ((int) $releaseInfo->proc_pp === 0 && $this->config->renamePar2 + && in_array((int) $context->release->categories_id, \App\Models\Category::OTHERS_GROUP, false) + ) { + $foundName = false; + } + + $filesAdded = 0; + + foreach ($par2Info->getFileList() as $file) { + if (! isset($file['name'])) { + continue; + } + + if ($foundName && $filesAdded > 10) { + break; + } + + // Add to release files + if ($this->config->addPAR2Files) { + if ($filesAdded < 11 + && ReleaseFile::query() + ->where(['releases_id' => $context->release->id, 'name' => $file['name']]) + ->first() === null + ) { + if (ReleaseFile::addReleaseFiles( + $context->release->id, + $file['name'], + $file['size'], + $postDate, + 0, + $file['hash_16K'] + )) { + $filesAdded++; + } + } + } else { + $filesAdded++; + } + + // Try to get a new name + if (! $foundName) { + $context->release->textstring = $file['name']; + $context->release->releases_id = $context->release->id; + if ($this->nameFixer->checkName($context->release, $this->config->echoCLI, 'PAR2, ', 1, 1)) { + $foundName = true; + } + } + } + + // Update file count + Release::query()->where('id', $context->release->id)->increment('rarinnerfilecount', $filesAdded); + $context->foundPAR2Info = true; + + return true; + } + + /** + * Process NFO file. + */ + public function processNfoFile( + string $fileLocation, + ReleaseProcessingContext $context, + \Blacklight\NNTP $nntp + ): bool { + try { + $data = File::get($fileLocation); + if ($this->nfo->isNFO($data, $context->release->guid) + && $this->nfo->addAlternateNfo($data, $context->release, $nntp) + ) { + $context->releaseHasNoNFO = false; + return true; + } + } catch (FileNotFoundException $e) { + Log::warning("Could not read potential NFO file: {$fileLocation} - {$e->getMessage()}"); + } + + return false; + } + + /** + * Handle release name extraction from RAR file content. + */ + public function processReleaseNameFromRar( + array $dataSummary, + ReleaseProcessingContext $context + ): void { + $fileData = $dataSummary['file_list'] ?? []; + if (empty($fileData)) { + return; + } + + $rarFileName = array_column($fileData, 'name'); + if (empty($rarFileName[0])) { + return; + } + + $extractedName = $this->extractReleaseNameFromFile($rarFileName[0]); + + if ($extractedName !== null) { + $preCheck = Predb::whereTitle($extractedName)->first(); + $context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0; + $candidate = $preCheck->title ?? $extractedName; + $candidate = $this->normalizeCandidateTitle($candidate); + + if ($this->isPlausibleReleaseTitle($candidate)) { + (new NameFixer())->updateRelease( + $context->release, + $candidate, + 'RarInfo FileName Match', + true, + 'Filenames, ', + true, + true, + $context->release->preid + ); + } elseif ($this->config->debugMode) { + Log::debug('RarInfo: Ignored low-quality candidate "'.$candidate.'" from inner file name.'); + } + } elseif (! empty($dataSummary['archives'][$rarFileName[0]]['file_list'])) { + // Try nested archive + $archiveData = $dataSummary['archives'][$rarFileName[0]]['file_list']; + $archiveFileName = array_column($archiveData, 'name'); + $extractedName = $this->extractReleaseNameFromFile($archiveFileName[0] ?? ''); + + if ($extractedName !== null) { + $preCheck = Predb::whereTitle($extractedName)->first(); + $context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0; + $candidate = $preCheck->title ?? $extractedName; + $candidate = $this->normalizeCandidateTitle($candidate); + + if ($this->isPlausibleReleaseTitle($candidate)) { + (new NameFixer())->updateRelease( + $context->release, + $candidate, + 'RarInfo FileName Match', + true, + 'Filenames, ', + true, + true, + $context->release->preid + ); + } + } + } + } + + /** + * Extract the release name from a filename. + */ + private function extractReleaseNameFromFile(string $filename): ?string + { + $basename = basename($filename); + $cleaned = preg_replace( + '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi)$/i', + '', + $basename + ); + + if (preg_match('/^(.+[-.][A-Za-z0-9_]{2,})$/i', $cleaned, $match)) { + return ucwords($match[1], '.-_ '); + } + + if (preg_match(NameFixer::PREDB_REGEX, $cleaned, $hit)) { + return ucwords($hit[0], '.'); + } + + return null; + } + + /** + * Normalize a candidate title. + */ + private function normalizeCandidateTitle(string $title): string + { + $t = trim($title); + $t = preg_replace('/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx)$/i', '', $t) ?? $t; + $t = preg_replace('/\.(par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi|jpe?g|png|gif|bmp)$/i', '', $t) ?? $t; + $t = preg_replace('/[.\-_ ](?:part|vol|r)\d+(?:\+\d+)?$/i', '', $t) ?? $t; + $t = preg_replace('/[\s_]+/', ' ', $t) ?? $t; + return trim($t, " .-_\t\r\n"); + } + + /** + * Check if a title is plausible for release naming. + */ + private function isPlausibleReleaseTitle(string $title): bool + { + $t = trim($title); + if ($t === '' || strlen($t) < 12) { + return false; + } + + $wordCount = preg_match_all('/[A-Za-z0-9]{3,}/', $t); + if ($wordCount < 2) { + return false; + } + + if (preg_match('/(?:^|[.\-_ ])(?:part|vol|r)\d+(?:\+\d+)?$/i', $t)) { + return false; + } + + if (preg_match('/^(setup|install|installer|patch|update|crack|keygen)\d*[\s._-]/i', $t)) { + return false; + } + + $hasGroupSuffix = (bool) preg_match('/[-.][A-Za-z0-9]{2,}$/', $t); + $hasYear = (bool) preg_match('/\b(19|20)\d{2}\b/', $t); + $hasQuality = (bool) preg_match('/\b(480p|720p|1080p|2160p|4k|webrip|web[ .-]?dl|bluray|bdrip|dvdrip|hdtv|hdrip|xvid|x264|x265|hevc|h\.?264|ts|cam|r5|proper|repack)\b/i', $t); + $hasTV = (bool) preg_match('/\bS\d{1,2}[Eex]\d{1,3}\b/i', $t); + $hasXXX = (bool) preg_match('/\bXXX\b/i', $t); + + return $hasGroupSuffix || ($hasTV && $hasQuality) || ($hasYear && ($hasQuality || $hasTV)) || $hasXXX; + } + + private function manticore(): ManticoreSearch + { + if ($this->manticore === null) { + $this->manticore = new ManticoreSearch(); + } + return $this->manticore; + } + + private function elasticsearch(): ElasticSearchSiteSearch + { + if ($this->elasticsearch === null) { + $this->elasticsearch = new ElasticSearchSiteSearch(); + } + return $this->elasticsearch; + } +} + diff --git a/app/Services/AdditionalProcessing/UsenetDownloadService.php b/app/Services/AdditionalProcessing/UsenetDownloadService.php new file mode 100644 index 000000000..fa7a6a600 --- /dev/null +++ b/app/Services/AdditionalProcessing/UsenetDownloadService.php @@ -0,0 +1,203 @@ +nntp = new NNTP(); + } + + /** + * Download binary content from usenet using message IDs. + * + * @param array|string $messageIDs Single or array of message IDs + * @param string $groupName Group name for logging + * @param int|null $releaseId Release ID for logging + * @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null} + * @throws Exception + */ + public function downloadByMessageIDs( + array|string $messageIDs, + string $groupName = '', + ?int $releaseId = null + ): array { + $result = [ + 'success' => false, + 'data' => null, + 'groupUnavailable' => false, + 'error' => null, + ]; + + if (empty($messageIDs)) { + $result['error'] = 'No message IDs provided'; + return $result; + } + + // Ensure array format + if (is_string($messageIDs)) { + $messageIDs = [$messageIDs]; + } + + if ($this->config->debugMode) { + Log::debug('Attempting NNTP fetch', [ + 'release_id' => $releaseId, + 'message_ids' => $messageIDs, + 'group' => $groupName, + ]); + } + + $binary = $this->nntp->getMessagesByMessageID($messageIDs, $this->config->alternateNNTP); + + // Handle non-string or empty response as failure + if (! is_string($binary) || $binary === '') { + $errorMessage = null; + + if (is_object($binary) && method_exists($binary, 'getMessage')) { + $errorMessage = $binary->getMessage(); + + // Check for group unavailability + if (stripos($errorMessage, 'No such news group') !== false + || stripos($errorMessage, 'Group not found') !== false + ) { + $result['groupUnavailable'] = true; + $result['error'] = 'Group unavailable: '.$errorMessage; + return $result; + } + } + + if ($this->config->debugMode) { + Log::debug('NNTP fetch failed', [ + 'release_id' => $releaseId, + 'message_ids' => $messageIDs, + 'group' => $groupName, + 'error_object' => is_object($binary) ? get_class($binary) : null, + 'error_message' => $errorMessage, + 'raw_type' => gettype($binary), + 'length' => is_string($binary) ? strlen($binary) : 0, + ]); + } + + $result['error'] = $errorMessage ?? 'Download failed'; + return $result; + } + + $result['success'] = true; + $result['data'] = $binary; + return $result; + } + + /** + * Download sample video content. + */ + public function downloadSample( + array $messageIDs, + string $groupName = '', + ?int $releaseId = null + ): array { + return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); + } + + /** + * Download media info video content. + */ + public function downloadMediaInfo( + string|array $messageID, + string $groupName = '', + ?int $releaseId = null + ): array { + return $this->downloadByMessageIDs($messageID, $groupName, $releaseId); + } + + /** + * Download audio content. + */ + public function downloadAudio( + string|array $messageID, + string $groupName = '', + ?int $releaseId = null + ): array { + return $this->downloadByMessageIDs($messageID, $groupName, $releaseId); + } + + /** + * Download JPG content. + */ + public function downloadJPG( + array $messageIDs, + string $groupName = '', + ?int $releaseId = null + ): array { + return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); + } + + /** + * Download compressed file content (RAR, ZIP, etc.). + * + * @param array $messageIDs Message IDs to download + * @param string $groupName Group name for logging + * @param int|null $releaseId Release ID for logging + * @param string|null $fileTitle File title for logging + * @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null} + * @throws Exception + */ + public function downloadCompressedFile( + array $messageIDs, + string $groupName = '', + ?int $releaseId = null, + ?string $fileTitle = null + ): array { + if ($this->config->debugMode) { + Log::debug('Attempting compressed fetch', [ + 'release_id' => $releaseId, + 'file_title' => $fileTitle, + 'message_ids' => $messageIDs, + 'group' => $groupName, + ]); + } + + $result = $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId); + + if (! $result['success'] && $this->config->debugMode) { + Log::debug('Compressed fetch failed', [ + 'release_id' => $releaseId, + 'file_title' => $fileTitle, + 'message_ids' => $messageIDs, + 'group' => $groupName, + 'error' => $result['error'], + ]); + } + + return $result; + } + + /** + * Get the NNTP client instance. + */ + public function getNNTP(): NNTP + { + return $this->nntp; + } + + /** + * Check if the minimum content size requirement is met. + */ + public function meetsMinimumSize(string $data, int $minimumBytes = 40): bool + { + return strlen($data) > $minimumBytes; + } +} + diff --git a/bootstrap/providers.php b/bootstrap/providers.php index 5e723264a..6a9d98cd0 100644 --- a/bootstrap/providers.php +++ b/bootstrap/providers.php @@ -1,6 +1,7 @@