_echoCLI = config('nntmux.echocli'); $this->_nntp = new NNTP; $this->_nzb = new NZB; $this->_archiveInfo = new ArchiveInfo; $this->_categorize = new Categorize; $this->_nameFixer = new NameFixer; $this->_releaseExtra = new ReleaseExtra; $this->_releaseImage = new ReleaseImage; $this->_par2Info = new Par2Info; $this->_nfo = new Nfo; $this->_innerFileBlacklist = Settings::settingValue('innerfileblacklist') === '' ? false : Settings::settingValue('innerfileblacklist'); $this->_maxNestedLevels = (int) Settings::settingValue('maxnestedlevels') === 0 ? 3 : (int) Settings::settingValue('maxnestedlevels'); $this->_extractUsingRarInfo = (int) Settings::settingValue('extractusingrarinfo') !== 0; $this->_fetchLastFiles = config('nntmux_settings.fetch_last_file'); $this->_unrarPath = config('nntmux_settings.unrar_path'); $this->_unzipPath = config('nntmux_settings.unzip_path'); $this->_7zipPath = config('nntmux_settings.7zip_path'); // Pass the binary extractors to ArchiveInfo. $clients = []; $clients += [ArchiveInfo::TYPE_RAR => $this->_unrarPath]; $this->_archiveInfo->setExternalClients($clients); $this->_killString = '"'; if (config('nntmux_settings.timeout_path') && (int) Settings::settingValue('timeoutseconds') > 0) { $this->_killString = ( '"'.config('nntmux_settings.timeout_path'). '" --foreground --signal=KILL '. Settings::settingValue('timeoutseconds').' "' ); } $this->_showCLIReleaseID = (PHP_BINARY.' '.__DIR__.'/ProcessAdditional.php ReleaseID: '); // Maximum amount of releases to fetch per run. $this->_queryLimit = (Settings::settingValue('maxaddprocessed') !== '') ? (int) Settings::settingValue('maxaddprocessed') : 25; // Maximum message ID's to download per file type in the NZB (video, jpg, etc). $this->_segmentsToDownload = (Settings::settingValue('segmentstodownload') !== '') ? (int) Settings::settingValue('segmentstodownload') : 2; // Maximum message ID's to download for a RAR file. $this->_maximumRarSegments = (Settings::settingValue('maxpartsprocessed') !== '') ? (int) Settings::settingValue('maxpartsprocessed') : 3; // Maximum RAR files to check for a password before stopping. $this->_maximumRarPasswordChecks = (Settings::settingValue('passchkattempts') !== '') ? (int) Settings::settingValue('passchkattempts') : 1; $this->_maximumRarPasswordChecks = (max($this->_maximumRarPasswordChecks, 1)); // Maximum size of releases in GB. $this->_maxSize = (Settings::settingValue('maxsizetopostprocess') !== '') ? (int) Settings::settingValue('maxsizetopostprocess') : 100; // Minimum size of releases in MB. $this->_minSize = (Settings::settingValue('minsizetopostprocess') !== '') ? (int) Settings::settingValue('minsizetopostprocess') : 100; // Use the alternate NNTP provider for downloading Message-ID's ? $this->_alternateNNTP = config('nntmux_nntp.use_alternate_nntp_server'); $this->_ffMPEGDuration = Settings::settingValue('ffmpeg_duration') !== '' ? (int) Settings::settingValue('ffmpeg_duration') : 5; $this->_addPAR2Files = config('nntmux_settings.add_par2'); if (! config('nntmux_settings.ffmpeg_path')) { $this->_processAudioSample = $this->_processThumbnails = $this->_processVideo = false; } else { $this->_processAudioSample = (int) Settings::settingValue('saveaudiopreview') !== 0; $this->_processThumbnails = (int) Settings::settingValue('processthumbnails') !== 0; $this->_processVideo = (int) Settings::settingValue('processvideos') !== 0; } $this->_processJPGSample = (int) Settings::settingValue('processjpg') !== 0; $this->_processMediaInfo = config('nntmux_settings.mediainfo_path'); $this->_processAudioInfo = $this->_processMediaInfo; $this->_processPasswords = config('nntmux_settings.check_passworded_rars') === true && ! empty(config('nntmux_settings.unrar_path')); $this->_audioSavePath = config('nntmux_settings.covers_path').'/audiosample/'; $this->_audioFileRegex = '\\.(AAC|AIFF|APE|AC3|ASF|DTS|FLAC|MKA|MKS|MP2|MP3|RA|OGG|OGM|W64|WAV|WMA)'; $this->_ignoreBookRegex = '/\\b(epub|lit|mobi|pdf|sipdf|html)\\b.*\\.rar(?!.{20,})/i'; $this->_supportFileRegex = '\\.(?:vol\\d{1,3}\\+\\d{1,3}|par2|srs|sfv|nzb)'; $this->_videoFileRegex = '\\.(AVI|F4V|IFO|M1V|M2V|M4V|MKV|MOV|MP4|MPEG|MPG|MPGV|MPV|OGV|QT|RM|RMVB|TS|VOB|WMV)'; $this->_debugDownloadFailures = (bool) config('app.debug'); $this->_groupUnavailable = false; // flag when NNTP group is missing $this->elasticsearchEnabled = config('nntmux.elasticsearch_enabled') === true; } /** * Clear out the main temp path when done. */ public function __destruct() { $this->_clearMainTmpPath(); } private function ffmpeg(): FFMpeg { if ($this->ffmpeg === null) { $this->ffmpeg = FFMpeg::create(['timeout' => Settings::settingValue('timeoutseconds')]); } return $this->ffmpeg; } private function ffprobe(): FFProbe { if ($this->ffprobe === null) { $this->ffprobe = FFProbe::create(); } return $this->ffprobe; } private function mediaInfo(): MediaInfo { if ($this->mediaInfo === null) { $this->mediaInfo = new MediaInfo; $this->mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true); $this->mediaInfo->setConfig('command', config('nntmux_settings.mediainfo_path')); } return $this->mediaInfo; } private function manticore(): ManticoreSearch { if ($this->manticore === null) { $this->manticore = new ManticoreSearch; } return $this->manticore; } private function elasticsearch(): ElasticSearchSiteSearch { if ($this->elasticsearch === null) { $this->elasticsearch = new ElasticSearchSiteSearch; } return $this->elasticsearch; } /** * @throws Exception */ public function start(string $groupID = '', string $guidChar = ''): void { $this->_setMainTempPath($guidChar, $groupID); // Fetch all the releases to work on. $this->_fetchReleases($groupID, $guidChar); // Check if we have releases to work on. if ($this->_totalReleases > 0) { // Echo start time and process description. $this->_echoDescription(); $this->_processReleases(); } } /** * @var string Main temp path to work on. */ protected string $_mainTmpPath; /** * @var string Temp path for current release. */ protected string $tmpPath; /** * @throws \RuntimeException * @throws Exception */ protected function _setMainTempPath(&$guidChar, string &$groupID = ''): void { // Set up the temporary files folder location. $this->_mainTmpPath = config('nntmux.tmp_unrar_path'); // Check if it ends with a dir separator. if (! Str::endsWith($this->_mainTmpPath, '/') && ! Str::endsWith($this->_mainTmpPath, '\\')) { $this->_mainTmpPath .= '/'; } // If we are doing per group, use the groupID has an inner path, so other scripts don't delete the files we are working on. if ($groupID !== '') { $this->_mainTmpPath .= $groupID.'/'; } elseif ($guidChar !== '') { $this->_mainTmpPath .= $guidChar.'/'; } if (! File::isDirectory($this->_mainTmpPath)) { if (! File::makeDirectory($this->_mainTmpPath, 0777, true, true) && ! File::isDirectory($this->_mainTmpPath)) { throw new \RuntimeException(sprintf('Directory "%s" was not created', $this->_mainTmpPath)); } } if (! File::isDirectory($this->_mainTmpPath)) { throw new \RuntimeException('Could not create the tmpunrar folder ('.$this->_mainTmpPath.')'); } $this->_clearMainTmpPath(); $this->tmpPath = $this->_mainTmpPath; } /** * Clear out old folders/files from the main temp folder. */ protected function _clearMainTmpPath(): void { if ($this->_mainTmpPath !== '') { $this->_recursivePathDelete($this->_mainTmpPath); } } /** * Get all releases that need to be processed. * * * @void */ protected function _fetchReleases(int|string $groupID, string &$guidChar): void { $sqlParts = []; $bindings = []; $sqlParts[] = 'SELECT r.id AS id, r.id AS releases_id, r.guid, r.name, r.size, r.groups_id, r.nfostatus, r.fromname,'. ' r.completion, r.categories_id, r.searchname, r.predb_id, c.disablepreview'; $sqlParts[] = 'FROM releases r'; $sqlParts[] = 'LEFT JOIN categories c ON c.id = r.categories_id'; $sqlParts[] = 'WHERE r.passwordstatus = ?'; $bindings[] = -1; // passwordstatus $sqlParts[] = 'AND r.nzbstatus = ?'; $bindings[] = 1; // ensure NZB file exists/ready $sqlParts[] = 'AND r.haspreview = ?'; $bindings[] = -1; // haspreview $sqlParts[] = 'AND c.disablepreview = ?'; $bindings[] = 0; // categories.disablepreview if ($this->_maxSize > 0) { $sqlParts[] = 'AND r.size < ?'; $bindings[] = (int) $this->_maxSize * 1073741824; // GB to bytes } if ($this->_minSize > 0) { $sqlParts[] = 'AND r.size > ?'; $bindings[] = (int) $this->_minSize * 1048576; // MB to bytes } if (! empty($groupID)) { $sqlParts[] = 'AND r.groups_id = ?'; $bindings[] = $groupID; } if (! empty($guidChar)) { $sqlParts[] = 'AND r.leftguid = ?'; $bindings[] = $guidChar; } $sqlParts[] = 'ORDER BY r.passwordstatus ASC, r.postdate DESC'; $limit = $this->_queryLimit; if ($limit <= 0) { $limit = 25; // fallback safety } $sqlParts[] = 'LIMIT '.$limit; // limit $finalSql = implode(' ', $sqlParts); $rawResults = DB::select($finalSql, $bindings); // Hydrate into Release models so later code expecting model (with array access) still works. $attributeArrays = array_map(static fn ($row) => (array) $row, $rawResults); $this->_releases = Release::hydrate($attributeArrays); $this->_totalReleases = $this->_releases->count(); } /** * Output the description and start time. * * @void */ protected function _echoDescription(): void { if ($this->_totalReleases > 1 && $this->_echoCLI) { $this->_echo( PHP_EOL. 'Additional post-processing, started at: '. now()->format('D M d, Y G:i a'). PHP_EOL. 'Downloaded: (xB)=yEnc article, (cB)=compressed part'. PHP_EOL. 'Failures: fC#=Compressed(part #), fS=Sample, fM=Media(video), fA=Audio, fJ=JPEG, G=Missing group'. PHP_EOL. 'Processing: r=RAR, z=ZIP, 7z=7zip (names/entries), g=GZIP, b=BZIP2, x=XZ, (vRAW)=Inline video detected'. PHP_EOL. 'Added: s=Sample image, j=JPEG image, A=Audio sample, a=Audio MediaInfo, v=Video sample'. PHP_EOL. 'Added: m=Video MediaInfo, n=NFO, ^=Inner file details (RAR/ZIP/7z/etc)'. '', 'header' ); } } /** * Loop through the releases, processing them 1 at a time. * * @throws \RuntimeException * @throws Exception */ protected function _processReleases(): void { foreach ($this->_releases as $this->_release) { $this->_echo( PHP_EOL.'['.$this->_release->id.']['. human_filesize($this->_release->size, 1).']', 'primaryOver' ); cli_set_process_title($this->_showCLIReleaseID.$this->_release->id); // Create folder to store temporary files. if (! $this->_createTempFolder()) { continue; } // Get NZB contents. if (! $this->_getNZBContents()) { continue; } // Reset the current release variables. $this->_resetReleaseStatus(); // Go through the files in the NZB, get the amount of book files. $totalBooks = $this->_processNZBContents(); // Check if this NZB is a large collection of books. $bookFlood = false; if ($totalBooks > 80 && ($totalBooks * 2) >= \count($this->_nzbContents)) { $bookFlood = true; } if ($this->_processPasswords || $this->_processThumbnails || $this->_processMediaInfo || $this->_processAudioInfo || $this->_processVideo ) { // Process usenet Message-ID downloads. $this->_processMessageIDDownloads(); // Process compressed (RAR/ZIP) files inside the NZB. if (! $bookFlood && $this->_NZBHasCompressedFile) { // Download the RARs/ZIPs, extract the files inside them and insert the file info into the DB. $this->_processNZBCompressedFiles(); // Download rar/zip in reverse order, to get the last rar or zip file. if ($this->_fetchLastFiles) { $this->_processNZBCompressedFiles(true); } if (! $this->_releaseHasPassword) { // Process the extracted files to get video/audio samples/etc. $this->_processExtractedFiles(); } } } // Update the release to say we processed it. $this->_finalizeRelease(); // Delete all files / folders for this release. $this->_recursivePathDelete($this->tmpPath); } if ($this->_echoCLI) { echo PHP_EOL; } } protected function _recursivePathDelete(string $path): void { if (File::isDirectory($path)) { foreach (File::allFiles($path) as $file) { File::delete($file); } // Don't delete the main temp folder. if ($path !== $this->_mainTmpPath) { File::deleteDirectory($path); } } elseif (File::isFile($path)) { File::delete($path); } } /** * Create a temporary storage folder for the current release. * * * * @throws Exception */ protected function _createTempFolder(): bool { // Per release defaults. $this->tmpPath = $this->_mainTmpPath.$this->_release->guid.'/'; if (! File::isDirectory($this->tmpPath)) { try { // Use force + recursive so concurrent processes creating the same path don't error out. File::makeDirectory($this->tmpPath, 0777, true, true); } catch (\Throwable $e) { // If another process created it between the isDirectory check and mkdir call, ignore. if (! File::isDirectory($this->tmpPath)) { // Retry once after a tiny delay (filesystem latency / NFS eventual consistency). usleep(100000); // 100ms if (! File::isDirectory($this->tmpPath)) { try { File::makeDirectory($this->tmpPath, 0777, true, true); } catch (\Throwable $e2) { if (config('app.debug') === true) { Log::warning('Unable to create directory after retry: '.$this->tmpPath.' :: '.$e2->getMessage()); } $this->_echo('Unable to create directory: '.$this->tmpPath, 'warning'); return false; } } } } } return File::isDirectory($this->tmpPath); } /** * Get a list of contents inside a release's NZB file. * * * @throws Exception */ protected function _getNZBContents(): bool { $nzbPath = $this->_nzb->NZBPath($this->_release->guid); if ($nzbPath !== false) { $nzbContents = Utility::unzipGzipFile($nzbPath); if (! $nzbContents) { if ($this->_echoCLI) { $this->_echo('NZB is empty or broken for GUID: '.$this->_release->guid, 'warning'); } // Try repair on raw file contents (maybe unzip failed but file readable) try { $rawFile = @File::get($nzbPath); if ($rawFile) { // If gz, attempt decompress ignoring errors. if (str_ends_with(strtolower($nzbPath), '.gz')) { $try = @gzdecode($rawFile); if ($try !== false) { $repaired = $this->_repairNZB($try, $nzbPath); if ($repaired !== null) { $nzbContents = $repaired; // continue below } } } else { $repaired = $this->_repairNZB($rawFile, $nzbPath); if ($repaired !== null) { $nzbContents = $repaired; } } } } catch (\Throwable $e) { // ignore } if (! $nzbContents) { $this->_deleteRelease(); return false; } } // Get a list of files in the nzb. $this->_nzbContents = $this->_nzb->nzbFileList($nzbContents, ['no-file-key' => false, 'strip-count' => true]); if (\count($this->_nzbContents) === 0) { // Attempt repair if initial parse yielded no files. $repaired = $this->_repairNZB($nzbContents, $nzbPath); if ($repaired !== null) { $this->_nzbContents = $this->_nzb->nzbFileList($repaired, ['no-file-key' => false, 'strip-count' => true]); } if (\count($this->_nzbContents) === 0) { if ($this->_echoCLI) { $this->_echo('NZB is potentially broken for GUID: '.$this->_release->guid, 'warning'); } $this->_deleteRelease(); return false; } } // Sort keys. ksort($this->_nzbContents, SORT_NATURAL); return true; } if ($this->_echoCLI) { $this->_echo('NZB not found for GUID: '.$this->_release->guid, 'warning'); } $this->_deleteRelease(); return false; } /** * Attempt to repair a potentially broken NZB XML string. * Strategies: * - Strip illegal control characters. * - Ensure root closing tag exists. * - Optionally wrap content if root tag missing. * - Try liberal XML parsing; return repaired content on success. */ protected function _repairNZB(string $raw, string $originalPath): ?string { $original = $raw; // Remove common binary / control chars except tab, newline, carriage return. $fixed = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F]/', '', $raw); // If missing opening tag, try to wrap existing segments inside a minimal nzb root. if (! str_contains(strtolower($fixed), '\n\n".$fixed."\n"; } else { // Ensure closing tag. if (! preg_match('/<\/nzb>\s*$/i', $fixed)) { $fixed .= "\n"; } } // Try to parse using libxml recovery. $opts = LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_COMPACT | LIBXML_NONET | LIBXML_NOCDATA | LIBXML_PARSEHUGE; $prev = libxml_use_internal_errors(true); $xml = simplexml_load_string($fixed, 'SimpleXMLElement', $opts); $errors = libxml_get_errors(); libxml_clear_errors(); libxml_use_internal_errors($prev); if ($xml === false || empty($xml->file)) { // Failed; return null so caller can continue normal failure path. if ($this->_echoCLI) { $this->_echo('NZB repair failed for GUID: '.$this->_release->guid.' ('.(count($errors)).' XML errors)', 'warning'); } return null; } // If we get here we believe repair succeeded; optionally persist repaired version (gzipped) next to original. try { // Only overwrite if we actually changed content. if ($fixed !== $original) { // If original was gz (very likely), replace with repaired gz; else plain. if (str_ends_with(strtolower($originalPath), '.gz')) { @File::put($originalPath, gzencode($fixed)); } else { @File::put($originalPath, $fixed); } } } catch (\Throwable $e) { if (config('app.debug') === true) { Log::debug('Failed to persist repaired NZB: '.$e->getMessage()); } } if ($this->_echoCLI) { $this->_echo('Repaired NZB for GUID: '.$this->_release->guid, 'primaryOver'); } return $fixed; } /** * The current file we are working on inside a NZB. */ protected array $_currentNZBFile; /** * Does the current NZB contain a compressed (RAR/ZIP) file? */ protected bool $_NZBHasCompressedFile; /** * Process the files inside the NZB, find Message-ID's to download. * If we find files with book extensions, return the amount. */ protected function _processNZBContents(): int { $totalBookFiles = 0; foreach ($this->_nzbContents as $this->_currentNZBFile) { try { // Support / ignore support/nfo files if (preg_match('/(?:'.$this->_supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $this->_currentNZBFile['title'])) { continue; } // Compressed file detection if (! $this->_NZBHasCompressedFile && preg_match('/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))("|\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', $this->_currentNZBFile['title'])) { $this->_NZBHasCompressedFile = true; } // Look for a video sample, make sure it's not an image. if ($this->_processThumbnails && empty($this->_sampleMessageIDs) && isset($this->_currentNZBFile['segments']) && stripos($this->_currentNZBFile['title'], 'sample') !== false && ! preg_match('/\.jpe?g$/i', $this->_currentNZBFile['title']) ) { // Get the amount of segments for this file. $segCount = (\count($this->_currentNZBFile['segments']) - 1); // If it's more than 1 try to get up to the site specified value of segments. for ($i = 0; $i < $this->_segmentsToDownload; $i++) { if ($i > $segCount) { break; } $this->_sampleMessageIDs[] = (string) $this->_currentNZBFile['segments'][$i]; } } // Look for a JPG picture, make sure it's not a CD cover. if ($this->_processJPGSample && empty($this->_JPGMessageIDs) && isset($this->_currentNZBFile['segments']) && ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $this->_releaseGroupName) && preg_match('/\.jpe?g[. ")\]]/i', $this->_currentNZBFile['title']) ) { // Get the amount of segments for this file. $segCount = (\count($this->_currentNZBFile['segments']) - 1); // If it's more than 1 try to get up to the site specified value of segments. for ($i = 0; $i < $this->_segmentsToDownload; $i++) { if ($i > $segCount) { break; } $this->_JPGMessageIDs[] = (string) $this->_currentNZBFile['segments'][$i]; } } // Look for a video file, make sure it's not a sample, for MediaInfo. if ($this->_processMediaInfo && empty($this->_MediaInfoMessageIDs) && isset($this->_currentNZBFile['segments'][0]) && stripos($this->_currentNZBFile['title'], 'sample') !== false && preg_match('/'.$this->_videoFileRegex.'[. ")\]]/i', $this->_currentNZBFile['title']) ) { $this->_MediaInfoMessageIDs = (string) $this->_currentNZBFile['segments'][0]; } // Look for an audio file. if ($this->_processAudioInfo && empty($this->_AudioInfoMessageIDs) && isset($this->_currentNZBFile['segments']) && preg_match('/'.$this->_audioFileRegex.'[. ")\]]/i', $this->_currentNZBFile['title'], $type) ) { // Get the extension. $this->_AudioInfoExtension = $type[1]; $this->_AudioInfoMessageIDs = (string) $this->_currentNZBFile['segments'][0]; } // Some releases contain many books, increment this to ignore them later. if (preg_match($this->_ignoreBookRegex, $this->_currentNZBFile['title'])) { $totalBookFiles++; } } catch (\ErrorException $e) { Log::debug($e->getTraceAsString()); } } return $totalBookFiles; } /** * List of message-id's we have tried for rar/zip files. */ protected array $_triedCompressedMids = []; // List entries of a 7z archive using external 7z -slt mode. private function _listSevenZipEntries(string $compressedData): array { if (empty($this->_7zipPath)) { return []; } try { $tmpFile = $this->tmpPath.uniqid('7zlist_', true).'.7z'; if (File::put($tmpFile, $compressedData) === false) { return []; } $cmd = [$this->_7zipPath, 'l', '-slt', '-ba', '-bd', $tmpFile]; $exit = 0; $out = null; $err = null; $ok = $this->_execCommand($cmd, $exit, $out, $err); if (! $ok || $exit !== 0 || empty($out)) { // Primary structured listing failed; attempt plain listing fallback. $exit2 = 0; $out2 = null; $err2 = null; $plainCmd = [$this->_7zipPath, 'l', '-ba', '-bd', $tmpFile]; $plainOk = $this->_execCommand($plainCmd, $exit2, $out2, $err2); File::delete($tmpFile); if (! $plainOk || $exit2 !== 0 || empty($out2)) { return []; } $files = []; $lines = preg_split('/\r?\n/', trim($out2)); $seen = 0; foreach ($lines as $line) { $line = trim($line); if ($line === '' || str_starts_with($line, '-----') || str_contains($line, ' Date ') || str_starts_with($line, 'Scanning ') || str_starts_with($line, 'Creating archive')) { continue; } // Typical format: YYYY-MM-DD HH:MM:SS ....A 12345 678 path/filename.ext if (preg_match('/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\s+\S+\s+\d+\s+\d+\s+(\S.*)$/', $line, $m)) { $name = $m[1]; } elseif (preg_match('/^\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2}\s+\S+\s+\d+\s+\d+\s+(\S.*)$/', $line, $m)) { // Locale alt DD.MM.YYYY $name = $m[1]; } else { // Fallback: extract token with extension near end if line contains a dot extension. if (preg_match('/([A-Za-z0-9_#@()\[\]\-+&., ]+\.[A-Za-z0-9]{2,8})$/', $line, $m2)) { $name = trim($m2[1]); } else { continue; } } $name = trim($name, " \t\r\n"); if ($name === '' || strlen($name) > 300) { continue; } $files[] = ['name' => $name, 'size' => 0, 'encrypted' => false]; if (++$seen >= 200) { break; } } return $files; } // Structured parsing branch. $blocks = preg_split('/\n\n+/u', trim($out)); File::delete($tmpFile); $files = []; $anyEnc = false; foreach ($blocks as $blk) { $lines = preg_split('/\r?\n/', trim($blk)); $row = []; foreach ($lines as $line) { $kv = explode(' = ', $line, 2); if (count($kv) === 2) { $row[$kv[0]] = $kv[1]; } } if (empty($row['Path'])) { continue; } $attr = $row['Attributes'] ?? ''; if (str_contains($attr, 'D')) { continue; } // directory $enc = ($row['Encrypted'] ?? '') === '+'; if ($enc) { $anyEnc = true; } $size = isset($row['Size']) && ctype_digit($row['Size']) ? (int) $row['Size'] : 0; $files[] = ['name' => $row['Path'], 'size' => $size, 'encrypted' => $enc]; if (count($files) >= 200) { break; } } if ($anyEnc && isset($files[0])) { $files[0]['__any_encrypted__'] = true; } return $files; } catch (\Throwable $e) { if (config('app.debug') === true) { $this->_debug('Exception listing 7z: '.$e->getMessage()); } return []; } } /** * @throws Exception */ protected function _processNZBCompressedFiles(bool $reverse = false): void { if ($this->_groupUnavailable) { return; } $this->_reverse = $reverse; if ($this->_reverse) { if (! krsort($this->_nzbContents)) { return; } } else { $this->_triedCompressedMids = []; } $failed = $downloaded = 0; // Loop through the files, attempt to find if passworded and files. Starting with what not to process. foreach ($this->_nzbContents as $nzbFile) { if ($downloaded >= $this->_maximumRarSegments) { break; } if ($failed >= $this->_maximumRarPasswordChecks) { break; } if ($this->_releaseHasPassword || $this->_groupUnavailable) { break; } // Archive detection if (! preg_match('/(\\.(part\\d+|[rz]\\d+|rar|0+|0*10?|zipr\\d{2,3}|zipx?|7z(?:\\.\\d{3})?|(?:tar\\.)?(?:gz|bz2|xz))(\\s*\\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\\.[1-9]\\d{1,2}".*\\(\\d+\\/\\d{2,}\\)$)/i', $nzbFile['title'])) { continue; } // Get message-id's for the rar file. $segCount = (\count($nzbFile['segments']) - 1); $mID = []; foreach (range(0, $this->_maximumRarSegments - 1) as $i) { if ($i > $segCount) { break; } $segment = (string) $nzbFile['segments'][$i]; if (! $this->_reverse) { $this->_triedCompressedMids[] = $segment; } elseif (\in_array($segment, $this->_triedCompressedMids, false)) { // We already downloaded this file. continue 2; } $mID[] = $segment; } // Nothing to download. if (empty($mID)) { continue; } // Download the article(s) from usenet. if ($this->_debugDownloadFailures) { Log::debug('Attempting compressed fetch', [ 'release_id' => $this->_release->id ?? null, 'reverse' => $this->_reverse, 'file_title' => $nzbFile['title'] ?? '', 'message_ids' => $mID, 'group' => $this->_releaseGroupName, ]); } $fetchedBinary = $this->_nntp->getMessagesByMessageID($mID, $this->_alternateNNTP); // Treat any non-string or blank response as failure. if (! is_string($fetchedBinary) || $fetchedBinary === '') { $errorObj = (is_object($fetchedBinary) ? $fetchedBinary : null); if ($this->_debugDownloadFailures) { $errMsg = null; if ($errorObj && method_exists($errorObj, 'getMessage')) { $errMsg = $errorObj->getMessage(); } Log::debug('Compressed fetch failed', [ 'release_id' => $this->_release->id ?? null, 'file_title' => $nzbFile['title'] ?? '', 'message_ids' => $mID, 'group' => $this->_releaseGroupName, 'error_object' => $errorObj ? get_class($errorObj) : null, 'error_message' => $errMsg, 'raw_type' => gettype($fetchedBinary), 'length' => is_string($fetchedBinary) ? strlen($fetchedBinary) : 0, ]); } if ($errorObj && method_exists($errorObj, 'getMessage')) { $msg = $errorObj->getMessage(); if (stripos($msg, 'No such news group') !== false || stripos($msg, 'Group not found') !== false) { $this->_groupUnavailable = true; if ($this->_echoCLI) { $this->_echo('G', 'warningOver'); // Distinct marker for missing group. } break; // Stop processing more compressed files. } } $fetchedBinary = false; } if ($fetchedBinary !== false) { // Echo we downloaded compressed file. if ($this->_echoCLI) { $this->_echo('(cB)', 'primaryOver'); } $downloaded++; // Process the compressed file only if it's still a string. if (is_string($fetchedBinary)) { $decompressed = $this->_processCompressedData($fetchedBinary); if ($decompressed || $this->_releaseHasPassword) { break; } } } else { $failed++; if ($this->_echoCLI) { $this->_echo('fC'.$failed, 'warningOver'); } } } } /** * Check if the data is a ZIP / RAR file, extract files, get file info. * * * @throws Exception */ protected function _processCompressedData(string &$compressedData): bool { $this->_compressedFilesChecked++; $earlyType = $this->_detectNonRarZipType($compressedData); if ($earlyType === '7z') { $listed = []; $archiveEncrypted = false; if (! empty($this->_7zipPath)) { $listed = $this->_listSevenZipEntries($compressedData); if (! empty($listed)) { if (! empty($listed[0]['__any_encrypted__'])) { $archiveEncrypted = true; } if ($archiveEncrypted) { $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; return false; // don't add filenames when encrypted } $now = time(); foreach ($listed as $entry) { if (! empty($entry['__any_encrypted__'])) { continue; } $nm = $entry['name']; if ($nm === '' || strlen($nm) > 300) { continue; } $info = [ 'name' => $nm, 'size' => $entry['size'] ?? 0, 'date' => $now, 'pass' => 0, 'crc32' => '', 'source' => '7z-list', ]; // New filtering rule: only consider specific playable/text/image extensions // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). $name = $info['name'] ?? ''; $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; if (! in_array($ext, $allowedExt, true)) { continue; // Skip disallowed extensions (also excludes gz, xz, etc.) } $base = pathinfo($name, PATHINFO_FILENAME); $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); if ($letterCount <= 5) { continue; // Require >5 letter characters in base name } $this->_addFileInfo($info); if ($this->_totalFileInfo >= 50) { break; } } if ($this->_totalFileInfo > 0) { if ($this->_echoCLI) { $this->_echo('7z', 'primaryOver'); } return true; } } } $addedFromHeader = false; try { if (class_exists(__NAMESPACE__.'\\SevenZipPartialParser')) { $parser = new SevenZipPartialParser($compressedData); if ($parser->isEncrypted()) { $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; return false; } $hdrNames = $parser->getFileNames(); if (config('app.debug') === true && ! empty($hdrNames)) { $this->_debug('7z header names: '.substr(implode(', ', $hdrNames), 0, 300)); } if ($parser->hasEncodedHeader() && empty($listed) && config('app.debug') === true) { $this->_debug('7z encoded header detected; external list unavailable or returned no entries.'); } if (! empty($hdrNames) && empty($listed)) { $now = time(); $count = 0; foreach ($hdrNames as $nm) { if ($nm === '' || strlen($nm) > 300) { continue; } $info = [ 'name' => $nm, 'size' => 0, 'date' => $now, 'pass' => 0, 'crc32' => '', 'source' => '7z-header', ]; // New filtering rule: only consider specific playable/text/image extensions // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). $name = $info['name'] ?? ''; $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; if (! in_array($ext, $allowedExt, true)) { continue; // Skip disallowed extensions (also excludes gz, xz, etc.) } $base = pathinfo($name, PATHINFO_FILENAME); $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); if ($letterCount <= 5) { continue; // Require >5 letter characters in base name } $this->_addFileInfo($info); $count++; if ($count >= 50) { break; } } if ($this->_totalFileInfo > 0) { $addedFromHeader = true; if ($this->_echoCLI) { $this->_echo('7z', 'primaryOver'); } return true; } } } } catch (\Throwable $e) { if (config('app.debug') === true) { $this->_debug('7z header parse failed: '.$e->getMessage()); } } if (! $addedFromHeader && empty($listed)) { $names = $this->_scanSevenZipFilenames($compressedData); if (! empty($names)) { $now = time(); foreach ($names as $nm) { if ($nm === '' || strlen($nm) > 300) { continue; } $info = [ 'name' => $nm, 'size' => 0, 'date' => $now, 'pass' => 0, 'crc32' => '', 'source' => '7z-scan', ]; // New filtering rule: only consider specific playable/text/image extensions // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). $name = $info['name'] ?? ''; $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; if (! in_array($ext, $allowedExt, true)) { continue; // Skip disallowed extensions (also excludes gz, xz, etc.) } $base = pathinfo($name, PATHINFO_FILENAME); $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); if ($letterCount <= 5) { continue; // Require >5 letter characters in base name } $this->_addFileInfo($info); if ($this->_totalFileInfo >= 50) { break; } } if ($this->_totalFileInfo > 0) { if ($this->_echoCLI) { $this->_echo('7z', 'primaryOver'); } return true; } } } // continue to extraction attempt below if still nothing } if (in_array($earlyType, ['7z', 'gzip', 'bzip2', 'xz'], true)) { if (! empty($this->_7zipPath)) { $extMap = ['7z' => '7z', 'gzip' => 'gz', 'bzip2' => 'bz2', 'xz' => 'xz']; $markerMap = ['7z' => '7z', 'gzip' => 'g', 'bzip2' => 'b', 'xz' => 'x']; $ext = $extMap[$earlyType]; $marker = $markerMap[$earlyType]; if ($this->_extractViaSevenZip($compressedData, $ext, $earlyType, $marker)) { return true; } } } if (! $this->_archiveInfo->setData($compressedData, true)) { $otherType = $earlyType ?? $this->_detectNonRarZipType($compressedData); if (in_array($otherType, ['7z', 'gzip', 'bzip2', 'xz'], true) && ! empty($this->_7zipPath)) { $extMap = [ '7z' => '7z', 'gzip' => 'gz', 'bzip2' => 'bz2', 'xz' => 'xz', ]; $markerMap = [ '7z' => '7z', 'gzip' => 'g', 'bzip2' => 'b', 'xz' => 'x', ]; $ext = $extMap[$otherType]; $marker = $markerMap[$otherType]; if ($this->_extractViaSevenZip($compressedData, $ext, $otherType, $marker)) { return true; } } // Standalone video (raw first segments) detection when not recognized as archive. $videoType = $this->_detectStandaloneVideo($compressedData); if ($videoType !== null) { try { $extMap = ['avi' => 'avi', 'mkv' => 'mkv', 'mpg' => 'mpg', 'mp4' => 'mp4']; $ext = $extMap[$videoType] ?? 'dat'; $fileLocation = $this->tmpPath.'inline_video_'.uniqid('', true).'.'.$ext; if (File::put($fileLocation, $compressedData) !== false) { if ($this->_echoCLI) { $this->_echo('(vRAW)', 'primaryOver'); } if (! $this->_foundMediaInfo) { $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); } if (! $this->_foundSample) { $this->_foundSample = $this->_getSample($fileLocation); } if (! $this->_foundVideo) { $this->_foundVideo = $this->_getVideo($fileLocation); } return $this->_foundMediaInfo || $this->_foundSample || $this->_foundVideo; } } catch (\Throwable $e) { if (config('app.debug') === true) { Log::debug('Standalone video handling failed: '.$e->getMessage()); } } } if (config('app.debug') === true) { $alt = $otherType !== null ? 'Probable alternate type: '.$otherType.'.' : ''; $sevenZip = ! empty($this->_7zipPath) ? '' : '7zip binary NOT configured; extended formats disabled.'; $this->_debug(sprintf( 'Unrecognized compressed data. %s %s', $alt, $sevenZip )); } return false; } if ($this->_archiveInfo->error !== '') { if (config('app.debug') === true) { $this->_debug('ArchiveInfo Error: '.$this->_archiveInfo->error); } return false; } try { $dataSummary = $this->_archiveInfo->getSummary(true); } catch (Exception $exception) { if (config('app.debug') === true) { Log::warning($exception->getTraceAsString()); } return false; } if (! $this->_extractUsingRarInfo) { try { if ($this->_unrarPath !== false) { $unrarDir = $this->tmpPath.'unrar/'; if (! File::isDirectory($unrarDir)) { File::makeDirectory($unrarDir, 0777, true, true); } } $unzipDir = $this->tmpPath.'unzip/'; if (! File::isDirectory($unzipDir)) { File::makeDirectory($unzipDir, 0777, true, true); } } catch (\Throwable $e) { if (config('app.debug') === true) { Log::warning('Failed ensuring extraction subdirectories: '.$e->getMessage()); } } } if (! empty($this->_archiveInfo->isEncrypted) || (isset($dataSummary['is_encrypted']) && (int) $dataSummary['is_encrypted'] !== 0)) { if (config('app.debug') === true) { $this->_debug('ArchiveInfo: Compressed file has a password.'); } $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; return false; } if ($this->_reverse) { $fileData = $dataSummary['file_list'] ?? []; if (! empty($fileData)) { $rarFileName = Arr::pluck($fileData, 'name'); $extractedName = $this->extractReleaseNameFromFile($rarFileName[0]); if ($extractedName !== null) { $preCheck = Predb::whereTitle($extractedName)->first(); $this->_release->preid = $preCheck !== null ? $preCheck->value('id') : 0; $candidate = $preCheck->title ?? $extractedName; $candidate = $this->normalizeCandidateTitle($candidate); if ($this->isPlausibleReleaseTitle($candidate)) { (new NameFixer)->updateRelease($this->_release, $candidate, 'RarInfo FileName Match', true, 'Filenames, ', true, true, $this->_release->preid); } elseif (config('app.debug') === true) { $this->_debug('RarInfo: Ignored low-quality candidate "'.$candidate.'" from inner file name.'); } } elseif (! empty($dataSummary['archives']) && ! empty($dataSummary['archives'][$rarFileName[0]]['file_list'])) { $archiveData = $dataSummary['archives'][$rarFileName[0]]['file_list']; $archiveFileName = Arr::pluck($archiveData, 'name'); $extractedName = $this->extractReleaseNameFromFile($archiveFileName[0]); if ($extractedName !== null) { $preCheck = Predb::whereTitle($extractedName)->first(); $this->_release->preid = $preCheck !== null ? $preCheck->value('id') : 0; $candidate = $preCheck->title ?? $extractedName; $candidate = $this->normalizeCandidateTitle($candidate); if ($this->isPlausibleReleaseTitle($candidate)) { (new NameFixer)->updateRelease($this->_release, $candidate, 'RarInfo FileName Match', true, 'Filenames, ', true, true, $this->_release->preid); } elseif (config('app.debug') === true) { $this->_debug('RarInfo: Ignored low-quality candidate "'.$candidate.'" from nested archive file name.'); } } } } } switch ($dataSummary['main_type']) { case ArchiveInfo::TYPE_RAR: if ($this->_echoCLI) { $this->_echo('r', 'primaryOver'); } if (! $this->_extractUsingRarInfo && $this->_unrarPath !== false) { $fileName = $this->tmpPath.uniqid('', true).'.rar'; File::put($fileName, $compressedData); runCmd($this->_killString.$this->_unrarPath.'" e -ai -ep -c- -id -inul -kb -or -p- -r -y "'.$fileName.'" "'.$this->tmpPath.'unrar/"'); File::delete($fileName); } break; case ArchiveInfo::TYPE_ZIP: if ($this->_echoCLI) { $this->_echo('z', 'primaryOver'); } if (! $this->_extractUsingRarInfo) { $fileName = $this->tmpPath.uniqid('', true).'.zip'; File::put($fileName, $compressedData); runCmd($this->_unzipPath.' -o "'.$fileName.'" -d "'.$this->tmpPath.'unzip/"'); File::delete($fileName); } break; default: return false; } return $this->_processCompressedFileList(); } /** * Get a list of files inside the Compressed file. * * * @throws Exception */ protected function _processCompressedFileList(): bool { // Get a list of files inside the Compressed file. $files = $this->_archiveInfo->getArchiveFileList(); if (! \is_array($files) || \count($files) === 0) { return false; } // Loop through the files. foreach ($files as $file) { if ($this->_releaseHasPassword) { break; } if (isset($file['name'])) { if (isset($file['error'])) { if (config('app.debug') === true) { $this->_debug("Error: {$file['error']} (in: {$file['source']})"); } continue; } if (isset($file['pass']) && $file['pass'] === true) { $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; break; } if ($this->_innerFileBlacklist !== false && preg_match($this->_innerFileBlacklist, $file['name'])) { $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; break; } $fileName = []; if (preg_match('/[^\/\\\\]*\.[a-zA-Z0-9]*$/', $file['name'], $fileName)) { $fileName = $fileName[0]; } else { $fileName = ''; } if ($this->_extractUsingRarInfo) { // Extract files from the rar. if (isset($file['compressed']) && (int) $file['compressed'] === 0) { File::put( $this->tmpPath.random_int(10, 999999).'_'.$fileName, $this->_archiveInfo->getFileData($file['name'], $file['source']) ); } // If the files are compressed, use a binary extractor. else { $this->_archiveInfo->extractFile($file['name'], $this->tmpPath.random_int(10, 999999).'_'.$fileName); } } } $this->_addFileInfo($file); } if ($this->_addedFileInfo > 0) { if ($this->elasticsearchEnabled) { $this->elasticsearch()->updateRelease($this->_release->id); } else { $this->manticore()->updateRelease($this->_release->id); } } return $this->_totalFileInfo > 0; } /** * @throws Exception */ protected function _addFileInfo(&$file): void { if (! isset($file['error']) && isset($file['source']) && ! preg_match('/(?:'.$this->_supportFileRegex.'|part\d+|[rz]\d{1,3}|zipr\d{2,3}|\d{2,3}|zipx?|zip|rar|7z|gz|bz2|xz)(\s*\.rar)?$/i', $file['name']) ) { // Cache the amount of files we find in the RAR or ZIP, return this to say we did find RAR or ZIP content. // This is so we don't download more RAR or ZIP files for no reason. $this->_totalFileInfo++; /* Check if we already have the file or not. * Also make sure we don't add too many files, some releases have 100's of files, like PS3 releases. */ if ($this->_addedFileInfo < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name'], 'size' => $file['size']])->first() === null) { $addReleaseFiles = ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $file['date'], $file['pass'], '', $file['crc32'] ?? ''); if (! empty($addReleaseFiles)) { $this->_addedFileInfo++; if ($this->_echoCLI) { $this->_echo('^', 'primaryOver'); } // Check for "codec spam" // Previous pattern '/[\/\\]Codec[\/\\]Setup\.exe$/i' caused a runtime compilation error in some PCRE builds. if (preg_match('#(?:^|[\\/])Codec[\\/]Setup\.exe$#i', $file['name'])) { if (config('app.debug') === true) { $this->_debug('Codec spam found, setting release to potentially passworded.'); } $this->_releaseHasPassword = true; $this->_passwordStatus = Releases::PASSWD_RAR; } // Run a PreDB filename check on insert to try and match the release elseif ($file['name'] !== '' && ! str_starts_with($file['name'], '.')) { $this->_release['filename'] = $file['name']; $this->_release['releases_id'] = $this->_release->id; $this->_nameFixer->matchPreDbFiles($this->_release, 1, 1, true); } } } } } /** * Go through all the extracted files in the temp folder and process them. * * @throws Exception */ protected function _processExtractedFiles(): void { $nestedLevels = 0; while ($nestedLevels < $this->_maxNestedLevels) { if ($this->_compressedFilesChecked >= self::maxCompressedFilesToCheck) { break; } $foundCompressedFile = false; $files = $this->_getTempDirectoryContents('/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1|7z(?:\.\d{3})?|(?:tar\.)?(?:gz|bz2|xz))($|[^a-z0-9])/i'); if (! empty($files)) { foreach ($files as $file) { // Check if the file exists. if (File::isFile($file[0])) { $rarData = @File::get($file[0]); if (! empty($rarData)) { $this->_processCompressedData($rarData); $foundCompressedFile = true; } File::delete($file[0]); } } } // If we found no compressed files, break out. if (! $foundCompressedFile) { break; } $nestedLevels++; } $fileType = []; // Get all the remaining files in the temp dir. $files = $this->_getTempDirectoryContents(); if (! empty($files)) { foreach ($files as $file) { $file = $file->getPathname(); // Skip /. and /.. if (preg_match('/[\/\\\\]\.{1,2}$/', $file)) { continue; } if (File::isFile($file)) { // Process PAR2 files. if (! $this->_foundPAR2Info && preg_match('/\.par2$/', $file)) { $this->_siftPAR2Info($file); } // Process NFO files. elseif ($this->_releaseHasNoNFO && preg_match('/(\.(nfo|inf|ofn)|info\.txt)$/i', $file)) { $this->_processNfoFile($file); } // Process audio files. elseif ((! $this->_foundAudioInfo || ! $this->_foundAudioSample) && preg_match('/(.*)'.$this->_audioFileRegex.'$/i', $file, $fileType)) { // Try to get audio sample/audio media info. File::move($file, $this->tmpPath.'audiofile.'.$fileType[2]); $this->_getAudioInfo($this->tmpPath.'audiofile.'.$fileType[2], $fileType[2]); File::delete($this->tmpPath.'audiofile.'.$fileType[2]); } // Process JPG files. elseif (! $this->_foundJPGSample && preg_match('/\.jpe?g$/i', $file)) { $this->_getJPGSample($file); File::delete($file); } // Video sample // video clip // video media info. elseif ((! $this->_foundSample || ! $this->_foundVideo || ! $this->_foundMediaInfo) && preg_match('/(.*)'.$this->_videoFileRegex.'$/i', $file)) { $this->_processVideoFile($file); } // Check file's magic info. else { $output = Utility::fileInfo($file); if (! empty($output)) { switch (true) { case ! $this->_foundJPGSample && preg_match('/^JPE?G/i', $output): $this->_getJPGSample($file); File::delete($file); break; case (! $this->_foundMediaInfo || ! $this->_foundSample || ! $this->_foundVideo) && preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output): $this->_processVideoFile($file); break; case (! $this->_foundAudioSample || ! $this->_foundAudioInfo) && preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $fileType): switch ($fileType[0]) { case 'FLAC': $fileType = 'FLAC'; break; case 'layer III': $fileType = 'MP3'; break; case 'Vorbis audio': $fileType = 'OGG'; break; } File::move($file, $this->tmpPath.'audiofile.'.$fileType); $this->_getAudioInfo($this->tmpPath.'audiofile.'.$fileType, $fileType); File::delete($this->tmpPath.'audiofile.'.$fileType); break; case ! $this->_foundPAR2Info && stripos($output, 'Parity') === 0: $this->_siftPAR2Info($file); break; } } } } } } } /** * Download all binaries from usenet and form samples / get media info / etc from them. * * @void * * @throws Exception */ protected function _processMessageIDDownloads(): void { $this->_processSampleMessageIDs(); $this->_processMediaInfoMessageIDs(); $this->_processAudioInfoMessageIDs(); $this->_processJPGMessageIDs(); } /** * Download and process binaries for sample videos. * * @void * * @throws Exception */ protected function _processSampleMessageIDs(): void { // Download and process sample image. if (! $this->_foundSample || ! $this->_foundVideo) { if (! empty($this->_sampleMessageIDs)) { // Download it from usenet. $sampleBinary = $this->_nntp->getMessagesByMessageID($this->_sampleMessageIDs, $this->_alternateNNTP); if (! is_string($sampleBinary) || $sampleBinary === '') { if ($this->_debugDownloadFailures) { $errMsg = null; if (is_object($sampleBinary) && method_exists($sampleBinary, 'getMessage')) { $errMsg = $sampleBinary->getMessage(); } Log::debug('Sample fetch failed', [ 'release_id' => $this->_release->id ?? null, 'message_ids' => $this->_sampleMessageIDs, 'group' => $this->_releaseGroupName, 'error_object' => is_object($sampleBinary) ? get_class($sampleBinary) : null, 'error_message' => $errMsg, 'raw_type' => gettype($sampleBinary), 'length' => is_string($sampleBinary) ? strlen($sampleBinary) : 0, ]); } $sampleBinary = false; } if ($sampleBinary !== false) { if ($this->_echoCLI) { $this->_echo('(sB)', 'primaryOver'); } // Check if it's more than 40 bytes. if (is_string($sampleBinary) && \strlen($sampleBinary) > 40) { $fileLocation = $this->tmpPath.'sample_'.random_int(0, 99999).'.avi'; // Try to create the file. File::put($fileLocation, $sampleBinary); // Try to get a sample picture. if (! $this->_foundSample) { $this->_foundSample = $this->_getSample($fileLocation); } // Try to get a sample video. if (! $this->_foundVideo) { $this->_foundVideo = $this->_getVideo($fileLocation); } } } elseif ($this->_echoCLI) { $this->_echo('fS', 'warningOver'); } } } } /** * Download and process binaries for media info from videos. * * @void * * @throws Exception */ protected function _processMediaInfoMessageIDs(): void { // Download and process mediainfo. Also try to get a sample if we didn't get one yet. if (! $this->_foundMediaInfo || ! $this->_foundSample || ! $this->_foundVideo) { if (! empty($this->_MediaInfoMessageIDs)) { // Try to download it from usenet. $mediaBinary = $this->_nntp->getMessagesByMessageID($this->_MediaInfoMessageIDs, $this->_alternateNNTP); if (! is_string($mediaBinary) || $mediaBinary === '') { if ($this->_debugDownloadFailures) { $errMsg = null; if (is_object($mediaBinary) && method_exists($mediaBinary, 'getMessage')) { $errMsg = $mediaBinary->getMessage(); } Log::debug('MediaInfo fetch failed', [ 'release_id' => $this->_release->id ?? null, 'message_id' => $this->_MediaInfoMessageIDs, 'group' => $this->_releaseGroupName, 'error_object' => is_object($mediaBinary) ? get_class($mediaBinary) : null, 'error_message' => $errMsg, 'raw_type' => gettype($mediaBinary), ]); } $mediaBinary = false; } if ($mediaBinary !== false) { if ($this->_echoCLI) { $this->_echo('(mB)', 'primaryOver'); } // If it's more than 40 bytes... if (is_string($mediaBinary) && \strlen($mediaBinary) > 40) { $fileLocation = $this->tmpPath.'media.avi'; // Create a file on the disk with it. File::put($fileLocation, $mediaBinary); // Try to get media info. if (! $this->_foundMediaInfo) { $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); } // Try to get a sample picture. if (! $this->_foundSample) { $this->_foundSample = $this->_getSample($fileLocation); } // Try to get a sample video. if (! $this->_foundVideo) { $this->_foundVideo = $this->_getVideo($fileLocation); } } } elseif ($this->_echoCLI) { $this->_echo('fM', 'warningOver'); } } } } /** * Download and process binaries for media info from songs. * * @void * * @throws Exception */ protected function _processAudioInfoMessageIDs(): void { // Download an audio file, use media info to try to get the artist / album. if (! $this->_foundAudioInfo || ! $this->_foundAudioSample) { if (! empty($this->_AudioInfoMessageIDs)) { // Try to download it from usenet. $audioBinary = $this->_nntp->getMessagesByMessageID($this->_AudioInfoMessageIDs, $this->_alternateNNTP); if (! is_string($audioBinary) || $audioBinary === '') { if ($this->_debugDownloadFailures) { $errMsg = null; if (is_object($audioBinary) && method_exists($audioBinary, 'getMessage')) { $errMsg = $audioBinary->getMessage(); } Log::debug('Audio fetch failed', [ 'release_id' => $this->_release->id ?? null, 'message_id' => $this->_AudioInfoMessageIDs, 'group' => $this->_releaseGroupName, 'error_object' => is_object($audioBinary) ? get_class($audioBinary) : null, 'error_message' => $errMsg, 'raw_type' => gettype($audioBinary), ]); } $audioBinary = false; } if ($audioBinary !== false) { if ($this->_echoCLI) { $this->_echo('(aB)', 'primaryOver'); } $fileLocation = $this->tmpPath.'audio.'.$this->_AudioInfoExtension; // Create a file with it. File::put($fileLocation, $audioBinary); // Try to get media info / sample of the audio file. $this->_getAudioInfo($fileLocation, $this->_AudioInfoExtension); } elseif ($this->_echoCLI) { $this->_echo('fA', 'warningOver'); } } } } /** * Download and process binaries for JPG pictures. * * @void * * @throws Exception */ protected function _processJPGMessageIDs(): void { // Download JPG file. if (! $this->_foundJPGSample && ! empty($this->_JPGMessageIDs)) { // Try to download it. $jpgBinary = $this->_nntp->getMessagesByMessageID($this->_JPGMessageIDs, $this->_alternateNNTP); if (! is_string($jpgBinary) || $jpgBinary === '') { if ($this->_debugDownloadFailures) { $errMsg = null; if (is_object($jpgBinary) && method_exists($jpgBinary, 'getMessage')) { $errMsg = $jpgBinary->getMessage(); } Log::debug('JPG fetch failed', [ 'release_id' => $this->_release->id ?? null, 'message_ids' => $this->_JPGMessageIDs, 'group' => $this->_releaseGroupName, 'error_object' => is_object($jpgBinary) ? get_class($jpgBinary) : null, 'error_message' => $errMsg, 'raw_type' => gettype($jpgBinary), 'length' => is_string($jpgBinary) ? strlen($jpgBinary) : 0, ]); } $jpgBinary = false; } if ($jpgBinary !== false) { if ($this->_echoCLI) { $this->_echo('(jB)', 'primaryOver'); } // Try to create a file with it. File::put($this->tmpPath.'samplepicture.jpg', $jpgBinary); // Check if image exif data is jpeg if (exif_imagetype($this->tmpPath.'samplepicture.jpg') === IMAGETYPE_JPEG) { // Try to resize and move it. $this->_foundJPGSample = $this->_releaseImage->saveImage($this->_release->guid.'_thumb', $this->tmpPath.'samplepicture.jpg', $this->_releaseImage->jpgSavePath, 650, 650) === 1; if ($this->_foundJPGSample) { // Update the DB to say we got it. Release::query()->where('id', $this->_release->id)->update(['jpgstatus' => 1]); if ($this->_echoCLI) { $this->_echo('j', 'primaryOver'); } } File::delete($this->tmpPath.'samplepicture.jpg'); } } elseif ($this->_echoCLI) { $this->_echo('fJ', 'warningOver'); } } } /** * Update the release to say we processed it. */ protected function _finalizeRelease(): void { $updateRows = ['haspreview' => 0]; // If samples exist from previous runs, set flags. if (File::isFile($this->_releaseImage->imgSavePath.$this->_release->guid.'_thumb.jpg')) { $updateRows = ['haspreview' => 1]; } if (File::isFile($this->_releaseImage->vidSavePath.$this->_release->guid.'.ogv')) { $updateRows += ['videostatus' => 1]; } if (File::isFile($this->_releaseImage->jpgSavePath.$this->_release->guid.'_thumb.jpg')) { $updateRows += ['jpgstatus' => 1]; } // Get the number of files we found inside the RAR/ZIPs. $releaseFilesCount = ReleaseFile::whereReleasesId($this->_release->id)->count('releases_id'); if ($releaseFilesCount === null) { $releaseFilesCount = 0; } $this->_passwordStatus = max([$this->_passwordStatus]); // Set the release to no password if password processing is off. if (! $this->_processPasswords) { $this->_releaseHasPassword = false; } // If we failed to get anything from the RAR/ZIPs update the release with what we have, if the rar/zip has no password. if (! $this->_releaseHasPassword && $this->_NZBHasCompressedFile && $releaseFilesCount === 0) { $release = Release::query()->where('id', $this->_release->id); $release->update( $updateRows ); } // Else update the release with the password status (if the admin enabled the setting). else { $updateRows += ['passwordstatus' => $this->_processPasswords ? $this->_passwordStatus : Releases::PASSWD_NONE, 'rarinnerfilecount' => $releaseFilesCount, ]; Release::query()->where('id', $this->_release->id)->update( $updateRows ); } } /** * Get the contents of a directory. */ protected function _getTempDirectoryContents(string $pattern = '', string $path = ''): array { $path = $path ?: $this->tmpPath; // Attempt to (re)create the directory if it vanished (external cleanup, race condition, etc). if ($path !== '' && ! File::isDirectory($path)) { try { File::makeDirectory($path, 0777, true, true); } catch (\Throwable $e) { // Ignore here; will be handled below when trying to read. } } try { $files = File::allFiles($path); } catch (\Throwable $e) { // Second chance: directory might have disappeared between existence check and allFiles call. if ($path !== '' && ! File::isDirectory($path)) { try { File::makeDirectory($path, 0777, true, true); $files = File::allFiles($path); // retry once } catch (\Throwable $e2) { if (config('app.debug') === true) { Log::error($e2->getTraceAsString()); } throw new \RuntimeException('ERROR: Could not open temp dir: '.$e2->getMessage()); } } else { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } throw new \RuntimeException('ERROR: Could not open temp dir: '.$e->getMessage()); } } if ($pattern !== '') { $filteredFiles = []; foreach ($files as $file) { if (preg_match($pattern, $file->getRelativePathname())) { $filteredFiles[] = $file->getRelativePathname(); } } return $filteredFiles; } return $files; } /** * @throws Exception */ protected function _getAudioInfo($fileLocation, $fileExtension): bool { // Return values. $retVal = $audVal = false; // Check if audio sample fetching is on. if (! $this->_processAudioSample) { $audVal = true; } // Check if media info fetching is on. if (! $this->_processAudioInfo) { $retVal = true; } $rQuery = Release::query()->where('proc_pp', '=', 0)->where('id', $this->_release->id)->select(['searchname', 'fromname', 'categories_id'])->first(); $musicParent = (string) Category::MUSIC_ROOT; if ($rQuery === null || ! preg_match( sprintf( '/%d\d{3}|%d|%d|%d/', $musicParent[0], Category::OTHER_MISC, Category::MOVIE_OTHER, Category::TV_OTHER ), $rQuery->id ) ) { return false; } if (File::isFile($fileLocation)) { // Check if media info is enabled. if (! $retVal) { // Get the media info for the file. try { $xmlArray = $this->mediaInfo()->getInfo($fileLocation, false); if ($xmlArray !== null) { foreach ($xmlArray->getAudios() as $track) { if ($track->get('album') !== null && $track->get('performer') !== null) { if ((int) $this->_release->predb_id === 0 && config('nntmux.rename_music_mediainfo')) { // Make the extension upper case. $ext = strtoupper($fileExtension); // Form a new search name. if (! empty($track->get('recorded_date')) && preg_match('/(?:19|20)\d\d/', $track->get('recorded_date')->getFullname, $Year)) { $newName = $track->get('performer')->getFullName().' - '.$track->get('album')->getFullName().' ('.$Year[0].') '.$ext; } else { $newName = $track->get('performer')->getFullName().' - '.$track->get('album')->getFullName().' '.$ext; } // Get the category or try to determine it. if ($ext === 'MP3') { $newCat = Category::MUSIC_MP3; } elseif ($ext === 'FLAC') { $newCat = Category::MUSIC_LOSSLESS; } else { $newCat = $this->_categorize->determineCategory($rQuery->groups_id, $newName, $rQuery->fromname); } $newTitle = escapeString(substr($newName, 0, 255)); // Update the search name. $release = Release::whereId($this->_release->id); $release->update(['searchname' => $newTitle, 'categories_id' => $newCat['categories_id'], 'iscategorized' => 1, 'isrenamed' => 1, 'proc_pp' => 1]); if ($this->elasticsearchEnabled) { $this->elasticsearch()->updateRelease($this->_release->id); } else { $this->manticore()->updateRelease($this->_release->id); } // Echo the changed name. if ($this->_echoCLI) { NameFixer::echoChangedReleaseName( [ 'new_name' => $newTitle, 'old_name' => $rQuery->searchname, 'new_category' => $newCat, 'old_category' => $rQuery->id, 'group' => $rQuery->groups_id, 'releases_id' => $this->_release->id, 'method' => 'ProcessAdditional->_getAudioInfo', ] ); } } // Add the media info. $this->_releaseExtra->addFromXml($this->_release->id, $xmlArray); $retVal = true; $this->_foundAudioInfo = true; if ($this->_echoCLI) { $this->_echo('a', 'primaryOver'); } break; } } } } catch (\RuntimeException $e) { Log::debug($e->getMessage()); } catch (\TypeError $e) { Log::debug($e->getMessage()); } } // Check if creating audio samples is enabled. if (! $audVal) { // File name to store an audio file. $audioFileName = ($this->_release->guid.'.ogg'); // Create an audio sample. if ($this->ffprobe()->isValid($fileLocation)) { try { $audioSample = $this->ffmpeg()->open($fileLocation); $format = new Vorbis; $audioSample->clip(TimeCode::fromSeconds(30), TimeCode::fromSeconds(30)); $audioSample->save($format, $this->tmpPath.$audioFileName); } catch (\InvalidArgumentException $e) { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) } } // Check if the new file was created. if (File::isFile($this->tmpPath.$audioFileName)) { // Try to move the temp audio file. $renamed = File::move($this->tmpPath.$audioFileName, $this->_audioSavePath.$audioFileName); if (! $renamed) { // Try to copy it if it fails. $copied = File::copy($this->tmpPath.$audioFileName, $this->_audioSavePath.$audioFileName); // Delete the old file. File::delete($this->tmpPath.$audioFileName); // If it didn't copy continue. if (! $copied) { return false; } } // Try to set the file perms. @chmod($this->_audioSavePath.$audioFileName, 0764); // Update DB to said we got a audio sample. Release::query()->where('id', $this->_release->id)->update(['audiostatus' => 1]); $audVal = $this->_foundAudioSample = true; if ($this->_echoCLI) { $this->_echo('A', 'primaryOver'); } } } } return $retVal && $audVal; } /** * Try to get a JPG picture, resize it and store it on disk. */ protected function _getJPGSample(string $fileLocation): void { // Try to resize/move the image. $this->_foundJPGSample = ( $this->_releaseImage->saveImage( $this->_release->guid.'_thumb', $fileLocation, $this->_releaseImage->jpgSavePath, 650, 650 ) === 1 ); // If it's successful, tell the DB. if ($this->_foundJPGSample) { Release::query()->where('id', $this->_release->id)->update(['jpgstatus' => 1]); } } private function getVideoTime(string $videoLocation): string { // Get the real duration of the file. if ($this->ffprobe()->isValid($videoLocation)) { $time = $this->ffprobe()->format($videoLocation)->get('duration'); } if (empty($time) || ! preg_match('/time=(\d{1,2}:\d{1,2}:)?(\d{1,2})\.(\d{1,2})\s*bitrate=/i', $time, $numbers)) { return ''; } // Reduce the last number by 1; this is to make sure we don't ask avconv/ffmpeg for non-existing data. if ($numbers[3] > 0) { $numbers[3]--; } elseif ($numbers[1] > 0) { $numbers[2]--; $numbers[3] = '99'; } // Manually pad the numbers in case they are 1 number. to get 02, for example, instead of 2. return '00:00:'.str_pad($numbers[2], 2, '0', STR_PAD_LEFT).'.'.str_pad($numbers[3], 2, '0', STR_PAD_LEFT); } /** * @throws Exception */ protected function _getSample(string $fileLocation): bool { if (! $this->_processThumbnails) { return false; } if (File::isFile($fileLocation)) { // Create a path to a temp file. $fileName = ($this->tmpPath.'zzzz'.random_int(5, 12).random_int(5, 12).'.jpg'); $time = $this->getVideoTime($fileLocation); // Create the image. if ($this->ffprobe()->isValid($fileLocation)) { try { $this->ffmpeg()->open($fileLocation)->frame(TimeCode::fromString($time === '' ? '00:00:03:00' : $time))->save($fileName); } catch (\RuntimeException $runtimeException) { if (config('app.debug') === true) { Log::error($runtimeException->getTraceAsString()); } // We show no error we just log it, we failed to save the frame and move on } catch (\InvalidArgumentException $e) { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) } catch (\Throwable $e) { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } // Again we do nothing, we just want to catch the error } } // Check if the file exists. if (File::isFile($fileName)) { // Try to resize/move the image. $saved = $this->_releaseImage->saveImage( $this->_release->guid.'_thumb', $fileName, $this->_releaseImage->imgSavePath, 800, 600 ); // Delete the temp file we created. File::delete($fileName); // Check if it saved. if ($saved === 1) { if ($this->_echoCLI) { $this->_echo('s', 'primaryOver'); } return true; } } } return false; } /** * @throws Exception */ protected function _getVideo(string $fileLocation): bool { if (! $this->_processVideo) { return false; } // Try to find an avi file. if (File::isFile($fileLocation)) { // Create a filename to store the temp file. $fileName = ($this->tmpPath.'zzzz'.$this->_release->guid.'.ogv'); $newMethod = false; // If the wanted sample length is less than 60, try to get a sample from the end of the video. if ($this->_ffMPEGDuration < 60) { // Get the real duration of the file. $time = $this->getVideoTime($fileLocation); if ($time !== '' && preg_match('/(\d{2}).(\d{2})/', $time, $numbers)) { $newMethod = true; // Get the lowest time we can start making the video at based on how many seconds the admin wants the video to be. if ($numbers[1] <= $this->_ffMPEGDuration) { // If the clip is shorter than the length we want. // The lowest we want is 0. $lowestLength = '00:00:00.00'; } else { // If the clip is longer than the length we want. // The lowest we want is the difference between the max video length and our wanted total time. $lowestLength = ($numbers[1] - $this->_ffMPEGDuration); // Form the time string. $end = '.'.$numbers[2]; $lowestLength = match (\strlen($lowestLength)) { 1 => ('00:00:0'.$lowestLength.$end), 2 => ('00:00:'.$lowestLength.$end), default => '00:00:60.00', }; } // Try to get the sample (from the end instead of the start). if ($this->ffprobe()->isValid($fileLocation)) { try { $video = $this->ffmpeg()->open($fileLocation); $videoSample = $video->clip(TimeCode::fromString($lowestLength), TimeCode::fromSeconds($this->_ffMPEGDuration)); $format = new Ogg; $format->setAudioCodec('libvorbis'); $videoSample->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); $videoSample->save($format, $fileName); } catch (\InvalidArgumentException $e) { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } // We do nothing, just prevent displaying errors because the file cannot be open(corrupted or incomplete file) } } } } // If longer than 60 or we could not get the video length, run the old way. if (! $newMethod && $this->ffprobe()->isValid($fileLocation)) { try { $video = $this->ffmpeg()->open($fileLocation); $videoSample = $video->clip(TimeCode::fromSeconds(0), TimeCode::fromSeconds($this->_ffMPEGDuration)); $format = new Ogg; $format->setAudioCodec('libvorbis'); $videoSample->filters()->resize(new Dimension(320, -1), ResizeFilter::RESIZEMODE_SCALE_HEIGHT); $videoSample->save($format, $fileName); } catch (\InvalidArgumentException $e) { if (config('app.debug') === true) { Log::error($e->getTraceAsString()); } // We do nothing, just prevent displaying errors because the file cannot be open (a corrupted or incomplete file) } } // Until we find the video file. if (File::isFile($fileName)) { // Create a path to where the file should be moved. $newFile = ($this->_releaseImage->vidSavePath.$this->_release->guid.'.ogv'); // Try to move the file to the new path. // If we couldn't rename it, try to copy it. if (! @File::move($fileName, $newFile)) { $copied = @File::copy($fileName, $newFile); // Delete the old file. File::delete($fileName); // If it didn't copy, continue. if (! $copied) { return false; } } // Change the permissions. @chmod($newFile, 0764); // Update query to say we got the video. Release::query()->where('guid', $this->_release->guid)->update(['videostatus' => 1]); if ($this->_echoCLI) { $this->_echo('v', 'primaryOver'); } return true; } } return false; } /** * @throws Exception */ protected function _getMediaInfo($fileLocation): bool { if (! $this->_processMediaInfo) { return false; } // Look for the video file. if (File::isFile($fileLocation)) { try { $xmlArray = $this->mediaInfo()->getInfo($fileLocation, true); // Insert it into the DB. \App\Models\MediaInfo::addData($this->_release->id, $xmlArray); $this->_releaseExtra->addFromXml($this->_release->id, $xmlArray); if ($this->_echoCLI) { $this->_echo('m', 'primaryOver'); } return true; } catch (\RuntimeException $e) { Log::debug($e->getMessage()); return false; } catch (\TypeError $e) { Log::debug($e->getMessage()); return false; } catch (\ErrorException $e) { Log::debug($e->getMessage()); return false; } } return false; } /** * @throws Exception */ protected function _siftPAR2Info($fileLocation): void { $this->_par2Info->open($fileLocation); if ($this->_par2Info->error) { return; } $releaseInfo = Release::query()->where('id', $this->_release->id)->select(['postdate', 'proc_pp'])->first(); if ($releaseInfo === null) { return; } $postDate = Carbon::createFromFormat('Y-m-d H:i:s', $releaseInfo->postdate)->getTimestamp(); // Only get a new name if the category is OTHER. $foundName = true; if ((int) $releaseInfo->proc_pp === 0 && config('nntmux.rename_par2') && \in_array( (int) $this->_release->categories_id, Category::OTHERS_GROUP, false ) ) { $foundName = false; } $filesAdded = 0; foreach ($this->_par2Info->getFileList() as $file) { if (! isset($file['name'])) { continue; } // If we found a name and added 10 files, stop. if ($foundName && $filesAdded > 10) { break; } // Add to release files. if ($this->_addPAR2Files) { if ($filesAdded < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name']])->first() === null ) { // Try to add the files to the DB. if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $postDate, 0, $file['hash_16K'])) { $filesAdded++; } } } else { $filesAdded++; } // Try to get a new name. if (! $foundName) { $this->_release->textstring = $file['name']; $this->_release->releases_id = $this->_release->id; if ($this->_nameFixer->checkName($this->_release, $this->_echoCLI, 'PAR2, ', 1, 1)) { $foundName = true; } } } // Update the file count with the new file count + old file count. Release::query()->where('id', $this->_release->id)->increment('rarinnerfilecount', $filesAdded); $this->_foundPAR2Info = true; } /** * @throws Exception Potentially thrown by addAlternateNfo */ protected function _processNfoFile(string $fileLocation): void { try { $data = File::get($fileLocation); if ($this->_nfo->isNFO($data, $this->_release->guid) && $this->_nfo->addAlternateNfo($data, $this->_release, $this->_nntp)) { $this->_releaseHasNoNFO = false; } } catch (FileNotFoundException $e) { // Log or handle the case where the file doesn't exist or isn't readable Log::warning("Could not read potential NFO file: {$fileLocation} - {$e->getMessage()}"); } } /** * @throws Exception */ protected function _processVideoFile($fileLocation): void { // Try to get a sample with it. if (! $this->_foundSample) { $this->_foundSample = $this->_getSample($fileLocation); } /* Try to get a video with it. * Don't get it here if _sampleMessageIDs is empty * or has 1 message-id (Saves downloading another part). */ if (! $this->_foundVideo && \count($this->_sampleMessageIDs) < 2) { $this->_foundVideo = $this->_getVideo($fileLocation); } // Try to get media info with it. if (! $this->_foundMediaInfo) { $this->_foundMediaInfo = $this->_getMediaInfo($fileLocation); } } /** * Comparison function for uSort, for sorting NZB files. */ protected function _sortNZB(array|string|null $a, array|string|null $b): int { $pos = 0; $af = $bf = false; $a = preg_replace('/\d+[ ._-]?(\/|\||[o0]f)[ ._-]?\d+?(?![ ._-]\d)/i', ' ', $a['title']); $b = preg_replace('/\d+[ ._-]?(\/|\||[o0]f)[ ._-]?\d+?(?![ ._-]\d)/i', ' ', $b['title']); if (preg_match('/\\.(part\\d+|[rz]\\d+)(\\s*\\.rar)*($|[ ")\]-])/i', $a)) { $af = true; } if (preg_match('/\\.(part\\d+|[rz]\\d+)(\\s*\\.rar)*($|[ ")\]-])/i', $b)) { $bf = true; } if (! $af && preg_match('/\.rar($|[ ")\]-])/i', $a)) { $a = preg_replace('/\.rar(?:$|[ ")\]-])/i', '.*rar', $a); $af = true; } if (! $bf && preg_match('/\.rar($|[ ")\]-])/i', $b)) { $b = preg_replace('/\.rar(?:$|[ ")\]-])/i', '.*rar', $b); $bf = true; } if (! $af && ! $bf) { return strnatcasecmp($a, $b); } if (! $bf) { return -1; } if (! $af) { return 1; } if ($af && $bf) { return strnatcasecmp($a, $b); } if ($af) { return -1; } if ($bf) { return 1; } return $pos; } /** * Reset some variables for the current release. */ protected function _resetReleaseStatus(): void { // Only process for samples, previews and images if not disabled. $this->_foundVideo = ! $this->_processVideo; $this->_foundMediaInfo = ! $this->_processMediaInfo; $this->_foundAudioInfo = ! $this->_processAudioInfo; $this->_foundAudioSample = ! $this->_processAudioSample; $this->_foundJPGSample = ! $this->_processJPGSample; $this->_foundSample = ! $this->_processThumbnails; $this->_foundPAR2Info = false; $this->_passwordStatus = Releases::PASSWD_NONE; $this->_releaseHasPassword = false; $this->_releaseGroupName = UsenetGroup::getNameByID($this->_release->groups_id); $this->_releaseHasNoNFO = false; // Make sure we don't already have a nfo. if ((int) $this->_release->nfostatus !== 1) { $this->_releaseHasNoNFO = true; } $this->_NZBHasCompressedFile = false; $this->_sampleMessageIDs = $this->_JPGMessageIDs = $this->_MediaInfoMessageIDs = []; $this->_AudioInfoMessageIDs = $this->_RARFileMessageIDs = []; $this->_AudioInfoExtension = ''; $this->_addedFileInfo = 0; $this->_totalFileInfo = 0; $this->_compressedFilesChecked = 0; $this->_groupUnavailable = false; } /** * Echo a string to CLI. * * @param string $string String to echo. * @param string $type Method type. * * @void */ protected function _echo(string $string, string $type): void { if ($this->_echoCLI) { (new ColorCLI)->$type($string); } } /** * Echo a string to CLI. For debugging. * * * @void */ protected function _debug(string $string): void { if (config('app.env') === 'local' && config('app.debug') === true) { $this->_echo('DEBUG: '.$string, 'debug'); } } /** * Delete the current release (used when NZB is irreparably broken). * Removes DB row, related ancillary rows, search index documents, NZB and preview assets. */ protected function _deleteRelease(): void { try { if (! isset($this->_release) || empty($this->_release->id)) { return; } $id = (int) $this->_release->id; $guid = $this->_release->guid ?? ''; // Delete related files (NZB + media samples/images if present) try { $nzbPath = $this->_nzb->NZBPath($guid); if ($nzbPath && File::exists($nzbPath)) { File::delete($nzbPath); } } catch (\Throwable $e) { // ignore } // Remove preview assets try { $img = $this->_releaseImage->imgSavePath.$guid.'_thumb.jpg'; $jpg = $this->_releaseImage->jpgSavePath.$guid.'_thumb.jpg'; $vid = $this->_releaseImage->vidSavePath.$guid.'.ogv'; foreach ([$img, $jpg, $vid] as $f) { if ($f && File::exists($f)) { File::delete($f); } } } catch (\Throwable $e) { // ignore } // Delete ancillary rows. try { ReleaseFile::where('releases_id', $id)->delete(); } catch (\Throwable $e) { } try { MediaInfoModel::where('releases_id', $id)->delete(); } catch (\Throwable $e) { } // Delete search index document. try { if ($this->elasticsearchEnabled) { $this->elasticsearch()->deleteRelease($id); } else { $this->manticore()->deleteRelease(['i' => $id, 'g' => $guid]); } } catch (\Throwable $e) { // ignore index delete failures } // Finally delete release row. try { Release::where('id', $id)->delete(); } catch (\Throwable $e) { } if ($this->_echoCLI) { $this->_echo('Deleted broken release ID '.$id, 'warningOver'); } } catch (\Throwable $e) { // Last resort: swallow any exception. } } /** * Detect non-RAR/ZIP archive container types we can optionally extract with 7zip. * Only returns values used by _processCompressedData's earlyType handling. */ protected function _detectNonRarZipType(string $data): ?string { $head6 = substr($data, 0, 6); $head4 = substr($data, 0, 4); // 7z signature 37 7A BC AF 27 1C with additional sanity checks. if ($head6 === "\x37\x7A\xBC\xAF\x27\x1C" && $this->_isLikely7z($data)) { return '7z'; } // GZIP 1F 8B 08 (check first 3 bytes) if (strncmp($head4, "\x1F\x8B\x08", 3) === 0) { return 'gzip'; } // BZip2 BZh if (strncmp($head4, 'BZh', 3) === 0) { return 'bzip2'; } // XZ FD 37 7A 58 5A 00 if ($head6 === "\xFD7zXZ\x00") { return 'xz'; } // (Optionally detect PDF so we can skip repeated processing, but we don't extract it.) if ($head4 === '%PDF') { return 'pdf'; } return null; } /** * Heuristic validation to reduce false positives on 7z signature. * 7z structure: signature (6 bytes) + version (2 bytes) + start header CRC (4 bytes) + next 8 bytes start header. */ private function _isLikely7z(string $data): bool { if (strlen($data) < 32) { return false; // unrealistically small for valid 7z } // Version bytes: major=0x00, minor commonly 0x02-0x06 (allow up to 0x09 future-proof) $verMajor = ord($data[6]); $verMinor = ord($data[7]); if ($verMajor !== 0x00 || $verMinor < 0x02 || $verMinor > 0x09) { return false; } // Start header CRC (bytes 8-11) should not be all 0x00 or all 0xFF $crc = substr($data, 8, 4); if ($crc === "\x00\x00\x00\x00" || $crc === "\xFF\xFF\xFF\xFF") { return false; } return true; } /** * Attempt to recover (probable) filenames from a partial 7z archive without full parsing. * Strategy: * - Limit to first few MB to keep performance reasonable. * - Collapse UTF-16LE (ASCII interleaved with 0x00) to plain ASCII. * - Regex match typical filename patterns (printable chars + common extensions). * - Return unique list (case-insensitive) capped at 50. */ private function _scanSevenZipFilenames(string $data): array { $slice = substr($data, 0, 8 * 1024 * 1024); $converted = preg_replace('/([\x20-\x7E])\x00/', '$1', $slice); if ($converted === null) { $converted = $slice; } $converted = str_replace("\x00", ' ', $converted); $exts = '7z|rar|zip|gz|bz2|xz|tar|tgz|mp4|mkv|avi|mpg|mpeg|mov|ts|wmv|flv|m4v|mp3|flac|ogg|wav|aac|aiff|ape|mka|nfo|txt|diz|pdf|epub|mobi|jpg|jpeg|png|gif|sfv|par2|exe|dll|srt|sub|idx|iso|bin|cue|mds|mdf'; // Fixed safe pattern (no dynamic delimiter conflicts). // Optional path (0-120 allowed chars + slash) then filename (2-160 chars) + .ext $regex = '~(?:[A-Za-z0-9 _.+&@#,()!-]{0,120}[\\/])?([A-Za-z0-9 _.+&@#,()!-]{2,160}\.(?:'.$exts.'))~i'; preg_match_all($regex, $converted, $m, PREG_SET_ORDER); if (empty($m)) { return []; } $names = []; foreach ($m as $match) { $candidate = $match[1]; $candidate = preg_replace('/ {2,}/', ' ', $candidate); if ($candidate === '' || strlen($candidate) < 5) { continue; } if (substr_count($candidate, '.') > 10) { continue; } $candidate = trim($candidate, " .-\t\n\r"); $lower = strtolower($candidate); if (! isset($names[$lower])) { $names[$lower] = $candidate; if (count($names) >= 80) { break; } } } return array_values($names); } /** * Execute an external command capturing stdout/stderr and exit code. */ private function _execCommand(string|array $cmd, ?int &$exitCode = null, ?string &$stdout = null, ?string &$stderr = null): bool { $descriptorSpec = [ 1 => ['pipe', 'w'], // stdout 2 => ['pipe', 'w'], // stderr ]; // If we have an array, pass directly to proc_open to avoid shell parsing. $process = @proc_open($cmd, $descriptorSpec, $pipes, null, null, ['bypass_shell' => true]); if (! \is_resource($process)) { $exitCode = -1; return false; } $stdout = stream_get_contents($pipes[1]); fclose($pipes[1]); $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); $exitCode = proc_close($process); return $exitCode === 0; } private function _extractViaSevenZip(string $compressedData, string $ext, string $type, string $marker): bool { try { if (! $this->_extractUsingRarInfo && ! empty($this->_7zipPath)) { $extractDir = $this->tmpPath.'un7z/'.uniqid('', true).'/'; if (! File::isDirectory($extractDir)) { File::makeDirectory($extractDir, 0777, true, true); } $fileName = $this->tmpPath.uniqid('', true).'.'.$ext; File::put($fileName, $compressedData); $added = false; $now = time(); // Build command parts (array form avoids quoting issues). $cmdParts = [ $this->_7zipPath, 'e', '-y', '-bd', '-o'.$extractDir, $fileName, ]; $exit = 0; $out = null; $err = null; $ok = $this->_execCommand($cmdParts, $exit, $out, $err); if (File::isDirectory($extractDir)) { foreach (File::allFiles($extractDir) as $f) { $info = [ 'name' => $f->getFilename(), 'size' => $f->getSize(), 'date' => $now, 'pass' => 0, 'crc32' => '', 'source' => $type, ]; // New filtering rule: only consider specific playable/text/image extensions // and require more than 5 letter characters in the base name (letters only, ignoring digits/punctuation). $name = $info['name'] ?? ''; $ext = strtolower(pathinfo($name, PATHINFO_EXTENSION)); $allowedExt = ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', 'flac', 'txt', 'png', 'gif', 'pdf', 'doc', 'docx', 'epub', 'cbz', 'cbr', 'djvu']; if (! in_array($ext, $allowedExt, true)) { continue; // Skip disallowed extensions (also excludes gz, xz, etc.) } $base = pathinfo($name, PATHINFO_FILENAME); $letterCount = preg_match_all('/[a-z]/i', $base, $mDummy); if ($letterCount <= 5) { continue; // Require >5 letter characters in base name } $this->_addFileInfo($info); $added = true; } } // Diagnostics when extraction fails or produces no files. if ((! $ok || ! $added) && config('app.debug') === true) { $context = [ 'release_id' => $this->_release->id ?? null, 'tmp_file_exists' => File::exists($fileName), 'tmp_file_size' => File::exists($fileName) ? File::size($fileName) : 0, 'extract_dir_exists' => File::isDirectory($extractDir), 'exit_code' => $exit, 'stderr_snip' => substr((string) $err, 0, 400), 'stdout_snip' => substr((string) $out, 0, 200), 'cmd_parts' => $cmdParts, 'type' => $type, ]; Log::debug('7zip extraction produced no files', $context); } // Cleanup input file always after logging. File::delete($fileName); if ($added && $this->_echoCLI) { $this->_echo($marker, 'primaryOver'); } return $added; } } catch (\Throwable $e) { if (config('app.debug') === true) { Log::warning(strtoupper($type).' extraction exception: '.$e->getMessage()); } } return false; } /** * Detect if the binary blob appears to be a standalone video (not inside an archive). * Used to opportunistically derive MediaInfo / previews from first-part inline uploads. */ private function _detectStandaloneVideo(string $data): ?string { $len = strlen($data); if ($len < 16) { return null; } // AVI (RIFF + 'AVI ') if (strncmp($data, 'RIFF', 4) === 0 && substr($data, 8, 4) === 'AVI ') { return 'avi'; } // Matroska / WebM (EBML header) if (strncmp($data, "\x1A\x45\xDF\xA3", 4) === 0) { return 'mkv'; } // MPEG Program Stream (pack / sequence headers) $sig4 = substr($data, 0, 4); if ($sig4 === "\x00\x00\x01\xBA" || $sig4 === "\x00\x00\x01\xB3") { return 'mpg'; } // MPEG Transport Stream (0x47 sync every 188 bytes for first 5 packets) if ($len >= 188 * 5) { $isTs = true; for ($i = 0; $i < 5; $i++) { $pos = 188 * $i; if (! isset($data[$pos]) || $data[$pos] !== "\x47") { $isTs = false; break; } } if ($isTs) { return 'mpg'; } } // MP4 / MOV ftyp brands if ($len >= 12 && substr($data, 4, 4) === 'ftyp') { $brand = substr($data, 8, 4); $brands = ['isom', 'iso2', 'avc1', 'mp41', 'mp42', 'dash', 'MSNV', 'qt ', 'M4V ', 'M4P ', 'M4B ', 'M4A ']; if (in_array($brand, $brands, true)) { return 'mp4'; } } return null; } /** * Process only a single release by GUID (utility method). */ public function processSingleGuid(string $guid): bool { try { $release = Release::where('guid', $guid)->first(); if ($release === null) { if ($this->_echoCLI) { $this->_echo('Release not found for GUID: '.$guid, 'warning'); } return false; } $this->_releases = collect([$release]); $this->_totalReleases = 1; $guidChar = $release->leftguid ?? substr($release->guid, 0, 1); $groupID = ''; $this->_setMainTempPath($guidChar, $groupID); $this->_processReleases(); return true; } catch (\Throwable $e) { if (config('app.debug') === true) { Log::error('processSingleGuid failed: '.$e->getMessage()); } return false; } } /** * Extract release name from a filename, handling file extensions and using an improved pattern. * Returns null if no valid release name found. */ private function extractReleaseNameFromFile(string $filename): ?string { // Strip directory path if present (e.g., "Sample/file.mkv" -> "file.mkv") $basename = basename($filename); // First, strip common file extensions (video, archive, metadata, package, and executable files) $cleaned = preg_replace('/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi)$/i', '', $basename); // Try improved regex that better handles audio/video codecs with dots (DD.5.1, H.264, etc.) // Match from start to a separator (hyphen or dot) followed by a group name (2+ word chars) // This preserves everything including complex codec patterns like DD.5.1 and H.264 if (preg_match('/^(.+[-.][A-Za-z0-9_]{2,})$/i', $cleaned, $match)) { // Matched: full string ending with -GROUP or .GROUP // Capitalize first letter of each word (using . as delimiter for release names) return ucwords($match[1], '.-_ '); } // Fallback to standard PREDB_REGEX if (preg_match(NameFixer::PREDB_REGEX, $cleaned, $hit)) { return ucwords($hit[0], '.'); } // If neither pattern matched, return null return null; } /** * Normalize a candidate title by stripping trailing part/vol/rNN tokens and trivial suffixes. */ private function normalizeCandidateTitle(string $title): string { $t = trim($title); // Remove common video file extensions $t = preg_replace('/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx)$/i', '', $t) ?? $t; // Remove archive and metadata file extensions $t = preg_replace('/\.(par2?|nfo|sfv|nzb|rar|zip|r\d{2,3}|pkg|exe|msi|jpe?g|png|gif|bmp)$/i', '', $t) ?? $t; // Remove common trailing segment markers like .part01, .vol12+3, -r12, r12 $t = preg_replace('/[.\-_ ](?:part|vol|r)\d+(?:\+\d+)?$/i', '', $t) ?? $t; // Collapse multiple spaces/underscores $t = preg_replace('/[\s_]+/', ' ', $t) ?? $t; // Trim stray punctuation return trim($t, " .-_\t\r\n"); } /** * Heuristic filter to avoid poor RarInfo-based renames (e.g., short or generic names). */ private function isPlausibleReleaseTitle(string $title): bool { $t = trim($title); if ($t === '') { return false; } // Minimum length and token count if (strlen($t) < 12) { return false; } $wordCount = preg_match_all('/[A-Za-z0-9]{3,}/', $t); if ($wordCount < 2) { return false; } // Reject if it still ends with a segment marker if (preg_match('/(?:^|[.\-_ ])(?:part|vol|r)\d+(?:\+\d+)?$/i', $t)) { return false; } // Reject generic installer/setup filenames if (preg_match('/^(setup|install|installer|patch|update|crack|keygen)\d*[\s._-]/i', $t)) { return false; } // Acceptance criteria $hasGroupSuffix = (bool) preg_match('/[-.][A-Za-z0-9]{2,}$/', $t); $hasYear = (bool) preg_match('/\b(19|20)\d{2}\b/', $t); $hasQuality = (bool) preg_match('/\b(480p|720p|1080p|2160p|4k|webrip|web[ .-]?dl|bluray|bdrip|dvdrip|hdtv|hdrip|xvid|x264|x265|hevc|h\.?264|ts|cam|r5|proper|repack)\b/i', $t); $hasTV = (bool) preg_match('/\bS\d{1,2}[Eex]\d{1,3}\b/i', $t); $hasXXX = (bool) preg_match('/\bXXX\b/i', $t); // Accept if: // - Has a group suffix (e.g., -ETHEL or .NBQ) // - Has TV episode identifier with quality indicator (e.g., S03E14 + 720p) // - Has year with quality or TV identifier // - Has XXX tag if ($hasGroupSuffix || ($hasTV && $hasQuality) || ($hasYear && ($hasQuality || $hasTV)) || $hasXXX) { return true; } return false; } }