diff --git a/Blacklight/Binaries.php b/Blacklight/Binaries.php index 536c5f641..15627567e 100644 --- a/Blacklight/Binaries.php +++ b/Blacklight/Binaries.php @@ -1,202 +1,123 @@ startUpdate = now(); - $this->timeCleaning = 0; + // Handle legacy array options for backward compatibility + if (\is_array($options)) { + $this->config = BinariesConfig::fromSettings(); - $this->_echoCLI = config('nntmux.echocli'); + // Use provided NNTP if available + if (isset($options['NNTP']) && $options['NNTP'] instanceof NNTP) { + $this->nntp = $options['NNTP']; + } else { + $this->nntp = new NNTP; + } + } else { + $this->config = $options ?? BinariesConfig::fromSettings(); + $this->nntp = new NNTP; + } - $this->_pdo = DB::connection()->getPdo(); $this->colorCli = new ColorCLI; - $this->_nntp = new NNTP; - $this->_collectionsCleaning = new CollectionsCleaning; - $this->xrefService = new XrefService; - $this->blacklistService = new BlacklistService; + $this->headerParser = new HeaderParser; + $this->headerStorage = new HeaderStorageService(config: $this->config); + $this->missedPartHandler = new MissedPartHandler( + $this->config->partRepairLimit, + $this->config->partRepairMaxTries + ); - $this->messageBuffer = Settings::settingValue('maxmssgs') !== '' ? - (int) Settings::settingValue('maxmssgs') : 20000; - $this->_compressedHeaders = config('nntmux_nntp.compressed_headers'); - $this->_partRepair = (int) Settings::settingValue('partrepair') === 1; - $this->_newGroupScanByDays = (int) Settings::settingValue('newgroupscanmethod') === 1; - $this->_newGroupMessagesToScan = Settings::settingValue('newgroupmsgstoscan') !== '' ? (int) Settings::settingValue('newgroupmsgstoscan') : 50000; - $this->_newGroupDaysToScan = Settings::settingValue('newgroupdaystoscan') !== '' ? (int) Settings::settingValue('newgroupdaystoscan') : 3; - $this->_partRepairLimit = Settings::settingValue('maxpartrepair') !== '' ? (int) Settings::settingValue('maxpartrepair') : 15000; - $this->_partRepairMaxTries = (Settings::settingValue('partrepairmaxtries') !== '' ? (int) Settings::settingValue('partrepairmaxtries') : 3); + $this->startUpdate = now(); - $this->blackList = $this->whiteList = []; + // Initialize legacy BC properties + $this->messageBuffer = $this->config->messageBuffer; + $this->blackList = []; + $this->whiteList = []; } /** @@ -210,55 +131,53 @@ class Binaries public function updateAllGroups(int $maxHeaders = 100000): void { $groups = UsenetGroup::getActive()->toArray(); - $groupCount = \count($groups); - if ($groupCount > 0) { - $counter = 1; - $allTime = now(); - $this->log( - 'Updating: '.$groupCount.' group(s) - Using compression? '.($this->_compressedHeaders ? 'Yes' : 'No'), - __FUNCTION__, - 'header' - ); - - // Loop through groups. - foreach ($groups as $group) { - $this->log( - 'Starting group '.$counter.' of '.$groupCount, - __FUNCTION__, - 'header' - ); - try { - $this->updateGroup($group, $maxHeaders); - } catch (\Throwable $e) { - if ($this->_echoCLI) { - $this->colorCli->error('Error updating group '.$group['name'].': '.$e->getMessage()); - } - if (config('app.debug')) { - Log::error('updateGroup failed for '.$group['name'].': '.$e->getMessage()); - } - } - $counter++; - } - - $endTime = now()->diffInSeconds($allTime, true); - $this->log( - 'Updating completed in '.$endTime.Str::plural(' second', $endTime), - __FUNCTION__, - 'primary' - ); - } else { + if ($groupCount === 0) { $this->log( 'No groups specified. Ensure groups are added to NNTmux\'s database for updating.', __FUNCTION__, 'warning' ); + + return; } + + $counter = 1; + $allTime = now(); + + $this->log( + 'Updating: '.$groupCount.' group(s) - Using compression? '.($this->config->compressedHeaders ? 'Yes' : 'No'), + __FUNCTION__, + 'header' + ); + + foreach ($groups as $group) { + $this->log( + 'Starting group '.$counter.' of '.$groupCount, + __FUNCTION__, + 'header' + ); + + try { + $this->updateGroup($group, $maxHeaders); + } catch (\Throwable $e) { + $this->logError('Error updating group '.$group['name'].': '.$e->getMessage()); + } + + $counter++; + } + + $endTime = now()->diffInSeconds($allTime, true); + $this->log( + 'Updating completed in '.$endTime.Str::plural(' second', $endTime), + __FUNCTION__, + 'primary' + ); } /** - * When the indexer is started, log the date/time. + * Log the indexer start time. */ public function logIndexerStart(): void { @@ -279,213 +198,50 @@ class Binaries $startGroup = now(); $this->logIndexerStart(); - // Select the group on the NNTP server, gets the latest info on it. - $groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']); - if ($this->_nntp::isError($groupNNTP)) { - $groupNNTP = $this->_nntp->dataError($this->_nntp, $groupMySQL['name']); - - if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) { - UsenetGroup::disableIfNotExist($groupMySQL['id']); - } - if ($this->_nntp::isError($groupNNTP)) { - return; - } + // Select the group on the NNTP server + $groupNNTP = $this->selectNntpGroup($groupMySQL); + if ($groupNNTP === null) { + return; } - if ($this->_echoCLI) { + if ($this->config->echoCli) { $this->colorCli->primary('Processing '.$groupMySQL['name']); } - // Attempt to repair any missing parts before grabbing new ones. - if ((int) $groupMySQL['last_record'] !== 0) { - if ($this->_partRepair) { - if ($this->_echoCLI) { - $this->colorCli->primary('Part repair enabled. Checking for missing parts.'); - } - $this->partRepair($groupMySQL); - } elseif ($this->_echoCLI) { - $this->colorCli->primary('Part repair disabled by user.'); + // Attempt to repair any missing parts before grabbing new ones + if ((int) $groupMySQL['last_record'] !== 0 && $this->config->partRepair) { + if ($this->config->echoCli) { + $this->colorCli->primary('Part repair enabled. Checking for missing parts.'); } + $this->partRepair($groupMySQL); + } elseif ($this->config->echoCli && (int) $groupMySQL['last_record'] !== 0) { + $this->colorCli->primary('Part repair disabled by user.'); } - // Generate postdate for first record, for those that upgraded. + // Generate postdate for first record, for those that upgraded if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] !== 0) { $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); - UsenetGroup::query()->where('id', $groupMySQL['id'])->update(['first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get())]); + UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ + 'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()), + ]); } - // Get first article we want aka the oldest. - if ((int) $groupMySQL['last_record'] === 0) { - if ($this->_newGroupScanByDays) { - // For new newsgroups - determine here how far we want to go back using date. - $first = $this->daytopost($this->_newGroupDaysToScan, $groupNNTP); - } elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer))) { - // If what we want is lower than the groups first article, set the wanted first to the first. - $first = $groupNNTP['first']; - } else { - // Or else, use the newest article minus how much we should get for new groups. - $first = (string) ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)); - } + // Calculate article range + $range = $this->calculateArticleRange($groupMySQL, $groupNNTP, $maxHeaders); - // We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet) - $leaveOver = $this->messageBuffer; + if ($range['total'] <= 0) { + $this->outputNoNewArticles($groupMySQL, $groupNNTP, $range); - // If this is not a new group, go from our newest to the servers newest. - } else { - // Set our oldest wanted to our newest local article. - $first = $groupMySQL['last_record']; - - // This is how many articles we will grab. (the servers newest minus our newest). - $totalCount = (string) ($groupNNTP['last'] - $first); - - // Check if the server has more articles than our loop limit x 2. - if ($totalCount > ($this->messageBuffer * 2)) { - // Get the remainder of $totalCount / $this->message buffer - $leaveOver = round($totalCount % $this->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->messageBuffer; - } else { - // Else get half of the available. - $leaveOver = round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); - } + return; } - // The last article we want, aka the newest. - $last = $groupLast = (string) ($groupNNTP['last'] - $leaveOver); + $this->outputNewArticlesInfo($groupMySQL, $groupNNTP, $range); + $this->processArticleRange($groupMySQL, $groupNNTP, $range); - // If the newest we want is older than the oldest we want somehow.. set them equal. - if ($last < $first) { - $last = $groupLast = $first; - } - - // This is how many articles we are going to get. - $total = (string) ($groupLast - $first); - // This is how many articles are available (without $leaveOver). - $realTotal = (string) ($groupNNTP['last'] - $first); - - // Check if we should limit the amount of fetched new headers. - if ($maxHeaders > 0) { - if ($maxHeaders < ($groupLast - $first)) { - $groupLast = $last = (string) ($first + $maxHeaders); - } - $total = (string) ($groupLast - $first); - } - - // If total is bigger than 0 it means we have new parts in the newsgroup. - if ($total > 0) { - if ($this->_echoCLI) { - $this->colorCli->primary( - ( - (int) $groupMySQL['last_record'] === 0 - ? 'New group '.$groupNNTP['group'].' starting with '. - ( - $this->_newGroupScanByDays - ? $this->_newGroupDaysToScan.' days' - : number_format($this->_newGroupMessagesToScan).' messages' - ).' worth.' - : 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.' - ). - ' Leaving '.number_format($leaveOver). - " for next pass.\nServer oldest: ".number_format($groupNNTP['first']). - ' Server newest: '.number_format($groupNNTP['last']). - ' Local newest: '.number_format($groupMySQL['last_record']) - ); - } - - $done = false; - // Get all the parts (in portions of $this->messageBuffer to not use too much memory). - while (! $done) { - // Increment last until we reach $groupLast (group newest article). - if ($total > $this->messageBuffer) { - if ((string) ($first + $this->messageBuffer) > $groupLast) { - $last = $groupLast; - } else { - $last = (string) ($first + $this->messageBuffer); - } - } - // Increment first so we don't get an article we already had. - $first++; - - if ($this->_echoCLI) { - $this->colorCli->header( - PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first). - ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. - number_format($groupLast - $last).' articles in queue).' - ); - } - - // Get article headers from newsgroup. - $scanSummary = $this->scan($groupMySQL, $first, $last); - - // Check if we fetched headers. - if (! empty($scanSummary)) { - // If new group, update first record & postdate - if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) { - $groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; - - if (isset($scanSummary['firstArticleDate'])) { - $groupMySQL['first_record_postdate'] = strtotime($scanSummary['firstArticleDate']); - } else { - $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); - } - - UsenetGroup::query() - ->where('id', $groupMySQL['id']) - ->update( - [ - 'first_record' => $scanSummary['firstArticleNumber'], - 'first_record_postdate' => Carbon::createFromTimestamp( - $groupMySQL['first_record_postdate'], - date_default_timezone_get() - ), - ] - ); - } - - $scanSummary['lastArticleDate'] = (isset($scanSummary['lastArticleDate']) ? strtotime($scanSummary['lastArticleDate']) : false); - if (! is_numeric($scanSummary['lastArticleDate'])) { - $scanSummary['lastArticleDate'] = $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); - } - - UsenetGroup::query() - ->where('id', $groupMySQL['id']) - ->update( - [ - 'last_record' => $scanSummary['lastArticleNumber'], - 'last_record_postdate' => Carbon::createFromTimestamp($scanSummary['lastArticleDate'], date_default_timezone_get()), - 'last_updated' => now(), - ] - ); - } else { - // If we didn't fetch headers, update the record still. - UsenetGroup::query() - ->where('id', $groupMySQL['id']) - ->update( - [ - 'last_record' => $last, - 'last_updated' => now(), - ] - ); - } - - if ((int) $last === (int) $groupLast) { - $done = true; - } else { - $first = $last; - } - } - - if ($this->_echoCLI) { - $endGroup = now()->diffInSeconds($startGroup, true); - $this->colorCli->primary( - PHP_EOL.'Group '.$groupMySQL['name'].' processed in '. - $endGroup.Str::plural(' second', $endGroup) - ); - } - } elseif ($this->_echoCLI) { + if ($this->config->echoCli) { + $endGroup = now()->diffInSeconds($startGroup, true); $this->colorCli->primary( - 'No new articles for '.$groupMySQL['name'].' (first '.number_format($first). - ', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']). - ', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']). - ' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record']) + PHP_EOL.'Group '.$groupMySQL['name'].' processed in '.$endGroup.Str::plural(' second', $endGroup) ); } } @@ -505,194 +261,80 @@ class Binaries */ public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', ?array $missingParts = null): array { - // Start time of scan method and of fetching headers. $this->startLoop = now(); $this->groupMySQL = $groupMySQL; $this->last = $last; $this->first = $first; - $this->notYEnc = $this->headersBlackListed = 0; + $this->headersReceived = []; - $returnArray = $stdHeaders = []; - + $returnArray = []; $partRepair = ($type === 'partrepair'); - $this->addToPartRepair = ($type === 'update' && $this->_partRepair); + $addToPartRepair = ($type === 'update' && $this->config->partRepair); - // Download the headers. - if ($partRepair) { - // This is slower but possibly is better with missing headers. - $headers = $this->_nntp->getOverview($this->first.'-'.$this->last, true, false); - } else { - $headers = $this->_nntp->getXOVER($this->first.'-'.$this->last); - } - - // If there was an error, try to reconnect. - if ($this->_nntp::isError($headers)) { - // Increment if part repair and return false. + // Download headers from NNTP + $headers = $this->downloadHeaders($partRepair); + if ($headers === null) { if ($partRepair) { - MissedPart::query()->where('groups_id', $this->groupMySQL['id'])->where('numberid', ((int) $this->first === (int) $this->last ? '= '.$this->first : 'IN ('.implode(',', range($this->first, $this->last)).')'))->increment('attempts', 1); - - return $returnArray; + $this->missedPartHandler->incrementRangeAttempts($groupMySQL['id'], $first, $last); } - // This is usually a compression error, so try disabling compression. - $this->_nntp->doQuit(); - if ($this->_nntp->doConnect(false) !== true) { - return $returnArray; - } - - // Re-select group, download headers again without compression and re-enable compression. - $this->_nntp->selectGroup($this->groupMySQL['name']); - $headers = $this->_nntp->getXOVER($this->first.'-'.$this->last); - $this->_nntp->enableCompression(); - - // Check if the non-compression headers have an error. - if ($this->_nntp::isError($headers)) { - $message = ((int) $headers->code === 0 ? 'Unknown error' : $headers->message); - $this->log( - "Code {$headers->code}: $message\nSkipping group: {$this->groupMySQL['name']}", - __FUNCTION__, - 'error' - ); - - return $returnArray; - } + return $returnArray; } - // Start of processing headers. $this->startCleaning = now(); - - // End of the getting data from usenet. $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true); - // Check if we got headers. $msgCount = \count($headers); - if ($msgCount < 1) { return $returnArray; } - $this->getHighLowArticleInfo($returnArray, $headers, $msgCount); + // Extract article range info + $returnArray = $this->headerParser->getArticleRange($headers); - $headersRepaired = $rangeNotReceived = $this->headersReceived = $this->headersNotInserted = []; + // Parse and filter headers + $this->headerParser->reset(); + $parseResult = $this->headerParser->parse($headers, $groupMySQL['name'], $partRepair, $missingParts); - foreach ($headers as $header) { - // Check if we got the article or not. - if (isset($header['Number'])) { - $this->headersReceived[] = $header['Number']; - } else { - if ($this->addToPartRepair) { - $rangeNotReceived[] = $header['Number']; - } + $this->headersReceived = array_column($headers, 'Number'); + $this->headersReceived = array_filter($this->headersReceived); + $this->notYEnc = $parseResult['notYEnc']; + $this->headersBlackListed = $parseResult['blacklisted']; - continue; - } + // Update blacklist last_activity + $this->headerParser->flushBlacklistUpdates(); - // If set we are running in partRepair mode. - if ($partRepair && $missingParts !== null) { - if (! \in_array($header['Number'], $missingParts, false)) { - // If article isn't one that is missing skip it. - continue; - } - // We got the part this time. Remove article from part repair. - $headersRepaired[] = $header['Number']; - } + unset($headers); - // Parse subject to get base name and part/total like "(12/45)"; normalize to include yEnc if missing. - if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $header['matches'])) { - if (stripos($header['Subject'], 'yEnc') === false) { - $header['matches'][1] .= ' yEnc'; - } - } else { - $this->notYEnc++; - - continue; - } - - // Filter subject based on black/white list. - if ($this->blacklistService->isBlackListed($header, $this->groupMySQL['name'])) { - $this->headersBlackListed++; - - continue; - } - - if (empty($header['Bytes'])) { - $header['Bytes'] = (isset($this->header[':bytes']) ? $header[':bytes'] : 0); - } - - $stdHeaders[] = $header; - } - - unset($headers); // Reclaim memory now that headers are split. - - // Update blacklist last_activity for matched rules. - $ids = $this->blacklistService->getAndClearIdsToUpdate(); - if (! empty($ids)) { - $this->blacklistService->updateBlacklistUsage($ids); - } - - if ($this->_echoCLI && ! $partRepair) { + if ($this->config->echoCli && ! $partRepair) { $this->outputHeaderInitial(); } - if (! empty($stdHeaders)) { + // Store headers + $this->startUpdate = now(); // Reset before storage begins + $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true); + + $headersNotInserted = []; + if (! empty($parseResult['headers'])) { try { - $this->storeHeaders($stdHeaders); + $headersNotInserted = $this->headerStorage->store($parseResult['headers'], $groupMySQL, $addToPartRepair); } catch (\Throwable $e) { - if ($this->_echoCLI) { - $this->colorCli->error('storeHeaders failed: '.$e->getMessage()); - } - if (config('app.debug')) { - Log::error('storeHeaders failed: '.$e->getMessage()); - } + $this->logError('storeHeaders failed: '.$e->getMessage()); } } - unset($stdHeaders); - // Start of part repair. $this->startPR = now(); - - // End of inserting. $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true); - if ($partRepair && \count($headersRepaired) > 0) { - $this->removeRepairedParts($headersRepaired, $this->groupMySQL['id']); + // Handle repaired parts + if ($partRepair && ! empty($parseResult['repaired'])) { + $this->missedPartHandler->removeRepairedParts($parseResult['repaired'], $groupMySQL['id']); } - unset($headersRepaired); - if ($this->addToPartRepair) { - $notInsertedCount = \count($this->headersNotInserted); - if ($notInsertedCount > 0) { - $this->addMissingParts($this->headersNotInserted, $this->groupMySQL['id']); - - $this->log( - $notInsertedCount.' articles failed to insert!', - __FUNCTION__, - 'warning' - ); - - if (config('app.debug') === true) { - Log::warning($notInsertedCount.' articles failed to insert!'); - } - } - unset($this->headersNotInserted); - - // Check if we have any missing headers. - if (($this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1) > \count($this->headersReceived)) { - $rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($this->first, $this->last), $this->headersReceived)); - } - $notReceivedCount = \count($rangeNotReceived); - if ($notReceivedCount > 0) { - $this->addMissingParts($rangeNotReceived, $this->groupMySQL['id']); - - if ($this->_echoCLI) { - $this->colorCli->alternate( - 'Server did not return '.$notReceivedCount. - ' articles from '.$this->groupMySQL['name'].'.' - ); - } - } - unset($rangeNotReceived); + // Handle part repair tracking + if ($addToPartRepair) { + $this->handlePartRepairTracking($headersNotInserted, $parseResult['headers']); } $this->outputHeaderDuration(); @@ -700,531 +342,6 @@ class Binaries return $returnArray; } - /** - * Parse headers into collections/binaries and store header data as parts. - * - * - * - * @throws \Exception - * @throws \Throwable - */ - protected function storeHeaders(array $headers = []): void - { - // Refactored (Option A + improvements): - // - Single transaction for entire header batch (unchanged approach) - // - Parameterized queries instead of sprintf + manual escaping - // - Store raw message-id (including < >) without mangling; rely on binding - // - Chunk very large multi-row inserts to mitigate max_allowed_packet issues - // - Preserve original rollback semantics when any collection/binary insert fails mid-loop - $binariesUpdate = $collectionIDs = $articles = []; - $parts = []; - $insertedCollectionIds = []; - $insertedBinaryIds = []; - $insertedPartNumbers = []; - $batchCollectionHashes = []; - - // Defensive defaults when called directly in tests/harness. - if (! isset($this->headersNotInserted)) { - $this->headersNotInserted = []; - } - if (! isset($this->headersReceived)) { - $this->headersReceived = []; - } - - // Generate a batch marker to enable targeted cleanup on rollback. - $batchNoise = bin2hex(random_bytes(8)); - - DB::beginTransaction(); - $hadErrors = false; - - // Reasonable default chunk size (can be overridden via config nntmux.parts_chunk_size) - $partsChunkSize = (int) (config('nntmux.parts_chunk_size') ?? 5000); - if ($partsChunkSize < 100) { // guard against absurdly small values - $partsChunkSize = 100; - } - - foreach ($headers as $this->header) { - // Prepare meta for inserts. - if (! isset($articles[$this->header['matches'][1]])) { - $fileCount = $this->getFileCount($this->header['matches'][1]); - if ($fileCount[1] === 0 && $fileCount[3] === 0) { - $fileCount = $this->getFileCount($this->header['matches'][0]); - } - - $collMatch = $this->_collectionsCleaning->collectionsCleaner( - $this->header['matches'][1], - $this->groupMySQL['name'] - ); - - $this->header['CollectionKey'] = $collMatch['name'].$fileCount[3]; - - if (! isset($collectionIDs[$this->header['CollectionKey']])) { - $this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date'])); - $now = now()->timestamp; - - $existingXref = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref'); - $headerTokens = $this->xrefService->extractTokens($this->header['Xref'] ?? ''); - $newTokens = $this->xrefService->diffNewTokens($existingXref, $this->header['Xref'] ?? ''); - $finalXrefAppend = implode(' ', $newTokens); // tokens to append on duplicate - - $date = $this->header['Date'] > $now ? $now : $this->header['Date']; - $unixtime = is_numeric($this->header['Date']) ? $date : $now; - $random = sodium_bin2hex(random_bytes(16)); - - $collectionHash = sha1($this->header['CollectionKey']); - $driver = DB::getDriverName(); - $batchCollectionHashes[$collectionHash] = true; - - try { - if ($driver === 'sqlite') { - // Basic INSERT OR IGNORE then optional xref append update. - DB::statement('INSERT OR IGNORE INTO collections (subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) VALUES (?, ?, datetime(? , "unixepoch"), ?, ?, ?, ?, ?, datetime("now"), ?)', [ - substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255), - mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings()), - $unixtime, - implode(' ', $headerTokens), - $this->groupMySQL['id'], - $fileCount[3], - $collectionHash, - $collMatch['id'], - $batchNoise, - ]); - } else { - // MySQL / MariaDB path - $insertSql = 'INSERT INTO collections ' - .'(subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) ' - .'VALUES (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, NOW(), ?) ' - .'ON DUPLICATE KEY UPDATE dateadded = NOW()'; - $bindings = [ - substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255), - mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings()), - $unixtime, - implode(' ', $headerTokens), - $this->groupMySQL['id'], - $fileCount[3], - $collectionHash, - $collMatch['id'], - $batchNoise, - ]; - if ($finalXrefAppend !== '') { - $insertSql .= ', xref = CONCAT(xref, "\\n", ?)'; - $bindings[] = $finalXrefAppend; - } - DB::statement($insertSql, $bindings); - } - $lastId = (int) $this->_pdo->lastInsertId(); - if ($lastId > 0) { - $collectionID = $lastId; - $insertedCollectionIds[$collectionID] = true; // mark for cleanup on rollback - } else { - $collectionID = (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0); - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('Collection insert failed: '.$e->getMessage()); - } - if ($this->addToPartRepair) { - $this->headersNotInserted[] = $this->header['Number']; - } - $hadErrors = true; - - continue; // Skip to next header - } - - if (! $collectionID) { - if ($this->addToPartRepair) { - $this->headersNotInserted[] = $this->header['Number']; - } - $hadErrors = true; - - continue; - } - $collectionIDs[$this->header['CollectionKey']] = $collectionID; - } else { - $collectionID = $collectionIDs[$this->header['CollectionKey']]; - } - - // Binary insert (unique by binaryhash + collections_id) - parameterized with sqlite fallback. - $hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']); - $driver = DB::getDriverName(); - try { - if ($driver === 'sqlite') { - DB::statement('INSERT OR IGNORE INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) VALUES (?, ?, ?, ?, 1, ?, ?)', [ - $hash, - mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), - $collectionID, - $this->header['matches'][3], - $fileCount[1], - $this->header['Bytes'], - ]); - // Note: Do not update here if row existed; aggregated update handles extra parts. - } else { - $binarySql = 'INSERT INTO binaries ' - .'(binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) ' - .'VALUES (UNHEX(?), ?, ?, ?, 1, ?, ?) ' - .'ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + VALUES(partsize)'; - DB::statement($binarySql, [ - $hash, - mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), - $collectionID, - $this->header['matches'][3], - $fileCount[1], - $this->header['Bytes'], - ]); - } - - $binaryID = (int) $this->_pdo->lastInsertId(); - if ($binaryID === 0) { - $bin = DB::selectOne('SELECT id FROM binaries WHERE binaryhash '.($driver === 'sqlite' ? '= ?' : '= UNHEX(?)').' AND collections_id = ? LIMIT 1', $driver === 'sqlite' ? [$hash, $collectionID] : [$hash, $collectionID]); - $binaryID = (int) ($bin->id ?? 0); - } else { - $insertedBinaryIds[$binaryID] = true; // created in this batch - } - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('Binary insert failed: '.$e->getMessage()); - } - if ($this->addToPartRepair) { - $this->headersNotInserted[] = $this->header['Number']; - } - $hadErrors = true; - - continue; // Skip - } - - if (! $binaryID) { - if ($this->addToPartRepair) { - $this->headersNotInserted[] = $this->header['Number']; - } - $hadErrors = true; - - continue; - } - - $binariesUpdate[$binaryID]['Size'] = 0; - $binariesUpdate[$binaryID]['Parts'] = 0; - $articles[$this->header['matches'][1]]['CollectionID'] = $collectionID; - $articles[$this->header['matches'][1]]['BinaryID'] = $binaryID; - } else { - $binaryID = $articles[$this->header['matches'][1]]['BinaryID']; - $binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; - $binariesUpdate[$binaryID]['Parts']++; - } - - $parts[] = [ - 'binaries_id' => $binaryID, - 'number' => $this->header['Number'], - 'messageid' => $this->header['Message-ID'], - 'partnumber' => $this->header['matches'][2], - 'size' => $this->header['Bytes'], - ]; - - // Flush parts in chunks to avoid oversized packets / memory spikes - if (\count($parts) >= $partsChunkSize) { - if (! $this->flushPartsChunk($parts)) { - $hadErrors = true; - break; - } - // Successful flush: track part numbers inserted in this chunk - foreach ($parts as $r) { - $insertedPartNumbers[] = $r['number']; - } - $parts = []; - } - } - - unset($headers); // free memory - - // Flush any remaining parts. - if (! empty($parts) && ! $hadErrors) { - if (! $this->flushPartsChunk($parts)) { - $hadErrors = true; - } else { - foreach ($parts as $r) { - $insertedPartNumbers[] = $r['number']; - } - } - } - - // Start of inserting into SQL. - $this->startUpdate = now(); - $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true); - - // Batch update binaries aggregated size/parts (post-first part) using chunking as well. - if (! $hadErrors && ! empty($binariesUpdate)) { - $binaryRows = []; - foreach ($binariesUpdate as $binaryID => $binary) { - $extraSize = $binary['Size'] ?? 0; - $extraParts = $binary['Parts'] ?? 0; - if ($extraSize > 0 || $extraParts > 0) { - $binaryRows[] = [ - 'id' => $binaryID, - 'partsize' => $extraSize, - 'currentparts' => $extraParts, - ]; - } - } - if (! empty($binaryRows)) { - $driver = DB::getDriverName(); - if ($driver === 'sqlite') { - // Perform individual updates for sqlite. - foreach ($binaryRows as $row) { - try { - DB::statement('UPDATE binaries SET partsize = partsize + ?, currentparts = currentparts + ? WHERE id = ?', [ - $row['partsize'], $row['currentparts'], $row['id'], - ]); - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('Binaries aggregate sqlite update failed: '.$e->getMessage()); - } - $hadErrors = true; - break; - } - } - } else { - $updateChunk = (int) (config('nntmux.binaries_update_chunk_size') ?? 1000); - if ($updateChunk < 100) { - $updateChunk = 100; - } - $chunked = array_chunk($binaryRows, $updateChunk); - foreach ($chunked as $chunk) { - $placeholders = []; - $bindings = []; - foreach ($chunk as $row) { - $placeholders[] = '(?,?,?)'; - $bindings[] = $row['id']; - $bindings[] = $row['partsize']; - $bindings[] = $row['currentparts']; - } - $sql = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '.implode(',', $placeholders) - .' ON DUPLICATE KEY UPDATE partsize = partsize + VALUES(partsize), currentparts = currentparts + VALUES(currentparts)'; - try { - DB::statement($sql, $bindings); - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('Binaries aggregate update failed: '.$e->getMessage()); - } - $hadErrors = true; - break; - } - } - } - } - } - - try { - if ($hadErrors) { - DB::rollBack(); - // Safety cleanup: remove any rows created for this batch in case rollback did not apply (e.g., driver quirks) - try { - if (! empty($insertedPartNumbers)) { - $nums = $insertedPartNumbers; - $ph = implode(',', array_fill(0, count($nums), '?')); - DB::statement('DELETE FROM parts WHERE number IN ('.$ph.')', $nums); - } - if (! empty($insertedBinaryIds)) { - $ids = array_keys($insertedBinaryIds); - $phb = implode(',', array_fill(0, count($ids), '?')); - DB::statement('DELETE FROM binaries WHERE id IN ('.$phb.')', $ids); - } - $allCollectionIds = array_values(array_unique(array_map('intval', $collectionIDs))); - if (! empty($insertedCollectionIds) || ! empty($allCollectionIds)) { - $ids = ! empty($insertedCollectionIds) ? array_keys($insertedCollectionIds) : $allCollectionIds; - $phc = implode(',', array_fill(0, count($ids), '?')); - // Remove parts and binaries referencing these collections, then the collections - DB::statement('DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN ('.$phc.'))', $ids); - DB::statement('DELETE FROM binaries WHERE collections_id IN ('.$phc.')', $ids); - DB::statement('DELETE FROM collections WHERE id IN ('.$phc.')', $ids); - } elseif (! empty($batchCollectionHashes)) { - $hashes = array_keys($batchCollectionHashes); - $phh = implode(',', array_fill(0, count($hashes), '?')); - DB::statement('DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ('.$phh.')))', $hashes); - DB::statement('DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ('.$phh.'))', $hashes); - DB::statement('DELETE FROM collections WHERE collectionhash IN ('.$phh.')', $hashes); - } else { - // Fallback by noise marker - DB::statement('DELETE FROM parts WHERE binaries_id IN ( - SELECT b.id FROM binaries b WHERE b.collections_id IN ( - SELECT c.id FROM collections c WHERE c.noise = ? - ) - )', [$batchNoise]); - DB::statement('DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE noise = ?)', [$batchNoise]); - DB::statement('DELETE FROM collections WHERE noise = ?', [$batchNoise]); - } - // Final guard for sqlite tests: nuke any leftovers by group id - if (DB::getDriverName() === 'sqlite') { - DB::statement('DELETE FROM parts'); - DB::statement('DELETE FROM binaries'); - DB::statement('DELETE FROM collections'); - } - } catch (\Throwable $cleanupE) { - if (config('app.debug') === true) { - Log::warning('Post-rollback cleanup failed: '.$cleanupE->getMessage()); - } - } - if ($this->addToPartRepair) { - $this->headersNotInserted = array_unique(array_merge($this->headersNotInserted, $this->headersReceived)); - } - } else { - DB::commit(); - } - } catch (\Throwable $e) { - DB::rollBack(); - try { - if (! empty($insertedPartNumbers)) { - $nums = $insertedPartNumbers; - $ph = implode(',', array_fill(0, count($nums), '?')); - DB::statement('DELETE FROM parts WHERE number IN ('.$ph.')', $nums); - } - if (! empty($insertedBinaryIds)) { - $ids = array_keys($insertedBinaryIds); - $phb = implode(',', array_fill(0, count($ids), '?')); - DB::statement('DELETE FROM binaries WHERE id IN ('.$phb.')', $ids); - } - $allCollectionIds = array_values(array_unique(array_map('intval', $collectionIDs))); - if (! empty($insertedCollectionIds) || ! empty($allCollectionIds)) { - $ids = ! empty($insertedCollectionIds) ? array_keys($insertedCollectionIds) : $allCollectionIds; - $phc = implode(',', array_fill(0, count($ids), '?')); - DB::statement('DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN ('.$phc.'))', $ids); - DB::statement('DELETE FROM binaries WHERE collections_id IN ('.$phc.')', $ids); - DB::statement('DELETE FROM collections WHERE id IN ('.$phc.')', $ids); - } elseif (! empty($batchCollectionHashes)) { - $hashes = array_keys($batchCollectionHashes); - $phh = implode(',', array_fill(0, count($hashes), '?')); - DB::statement('DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ('.$phh.')))', $hashes); - DB::statement('DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ('.$phh.'))', $hashes); - DB::statement('DELETE FROM collections WHERE collectionhash IN ('.$phh.')', $hashes); - } else { - DB::statement('DELETE FROM parts WHERE binaries_id IN ( - SELECT b.id FROM binaries b WHERE b.collections_id IN ( - SELECT c.id FROM collections c WHERE c.noise = ? - ) - )', [$batchNoise]); - DB::statement('DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE noise = ?)', [$batchNoise]); - DB::statement('DELETE FROM collections WHERE noise = ?', [$batchNoise]); - } - if (DB::getDriverName() === 'sqlite') { - DB::statement('DELETE FROM parts'); - DB::statement('DELETE FROM binaries'); - DB::statement('DELETE FROM collections'); - } - } catch (\Throwable $cleanupE) { - if (config('app.debug') === true) { - Log::warning('Post-rollback cleanup (exception path) failed: '.$cleanupE->getMessage()); - } - } - if ($this->addToPartRepair) { - $this->headersNotInserted = array_unique(array_merge($this->headersNotInserted, $this->headersReceived)); - } - if (config('app.debug') === true) { - Log::error('storeHeaders final stage failed: '.$e->getMessage()); - } - } - } - - // Flush a chunk of part rows using parameter binding; returns bool success. - protected function flushPartsChunk(array $parts): bool - { - if (empty($parts)) { - return true; - } - $placeholders = []; - $bindings = []; - $driver = DB::getDriverName(); - foreach ($parts as $row) { - $placeholders[] = '(?,?,?,?,?)'; - $bindings[] = $row['binaries_id']; - $bindings[] = $row['number']; - $bindings[] = $row['messageid']; - $bindings[] = $row['partnumber']; - $bindings[] = $row['size']; - } - if ($driver === 'sqlite') { - $sql = 'INSERT OR IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders); - } else { - $sql = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders); - } - try { - DB::statement($sql, $bindings); - - return true; - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error('Parts chunk insert failed: '.$e->getMessage()); - } - if ($this->addToPartRepair) { - foreach ($parts as $row) { - $this->headersNotInserted[] = $row['number']; - } - } - } - - return false; - } - - /** - * Gets the First and Last Article Number and Date for the received headers. - */ - protected function getHighLowArticleInfo(array &$returnArray, array $headers, int $msgCount): void - { - // Get highest and lowest article numbers/dates. - $iterator1 = 0; - $iterator2 = $msgCount - 1; - while (true) { - if (! isset($returnArray['firstArticleNumber']) && isset($headers[$iterator1]['Number'])) { - $returnArray['firstArticleNumber'] = $headers[$iterator1]['Number']; - $returnArray['firstArticleDate'] = $headers[$iterator1]['Date']; - } - - if (! isset($returnArray['lastArticleNumber']) && isset($headers[$iterator2]['Number'])) { - $returnArray['lastArticleNumber'] = $headers[$iterator2]['Number']; - $returnArray['lastArticleDate'] = $headers[$iterator2]['Date']; - } - - // Break if we found non empty articles. - if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) { - break; - } - - // Break out if we couldn't find anything. - if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) { - break; - } - } - } - - /** - * Outputs the initial header scan results after yEnc check and blacklist routines. - */ - protected function outputHeaderInitial(): void - { - $this->colorCli->primary( - 'Received '.\count($this->headersReceived). - ' articles of '.number_format($this->last - $this->first + 1).' requested, '. - $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' - ); - } - - /** - * Outputs speed metrics of the scan function to CLI. - */ - protected function outputHeaderDuration(): void - { - $currentMicroTime = now(); - if ($this->_echoCLI) { - $this->colorCli->alternateOver(number_format($this->timeHeaders, 2).'s'). - $this->colorCli->primaryOver(' to download articles, '). - $this->colorCli->alternateOver(number_format($this->timeCleaning, 2).'s'). - $this->colorCli->primaryOver(' to process collections, '). - $this->colorCli->alternateOver(number_format($this->timeInsert, 2).'s'). - $this->colorCli->primaryOver(' to insert binaries/parts, '). - $this->colorCli->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startPR, true), 2).'s'). - $this->colorCli->primaryOver(' for part repair, '). - $this->colorCli->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startLoop, true), 2).'s'). - $this->colorCli->primary(' total.'); - } - } - /** * Attempt to get missing article headers. * @@ -1235,121 +352,47 @@ class Binaries */ public function partRepair(array $groupArr): void { - // Get all parts in part repair table. - $missingParts = []; - try { - $missingParts = DB::select(sprintf(' - SELECT * FROM missed_parts - WHERE groups_id = %d AND attempts < %d - ORDER BY numberid ASC LIMIT %d', $groupArr['id'], $this->_partRepairMaxTries, $this->_partRepairLimit)); - } catch (\PDOException $e) { - if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') { - $this->colorCli->notice('Deadlock occurred'); - DB::rollBack(); - } - } - + $missingParts = $this->missedPartHandler->getMissingParts($groupArr['id']); $missingCount = \count($missingParts); - if ($missingCount > 0) { - if ($this->_echoCLI) { - $this->colorCli->primary( - 'Attempting to repair '. - number_format($missingCount). - ' parts.' - ); - } - // Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4. - $ranges = $partList = []; - $firstPart = $lastNum = $missingParts[0]->numberid; + if ($missingCount === 0) { + $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']); - foreach ($missingParts as $part) { - if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) { - $ranges[] = [ - 'partfrom' => $firstPart, - 'partto' => $lastNum, - 'partlist' => $partList, - ]; - - $firstPart = $part->numberid; - $partList = []; - } - $partList[] = $part->numberid; - $lastNum = $part->numberid; - } - - $ranges[] = [ - 'partfrom' => $firstPart, - 'partto' => $lastNum, - 'partlist' => $partList, - ]; - - // Download missing parts in ranges. - foreach ($ranges as $range) { - $partFrom = $range['partfrom']; - $partTo = $range['partto']; - $partList = $range['partlist']; - - if ($this->_echoCLI) { - echo \chr(random_int(45, 46)).PHP_EOL; - } - - // Get article headers from newsgroup. - $this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList); - } - - // Calculate parts repaired - $result = DB::select( - sprintf( - ' - SELECT COUNT(id) AS num - FROM missed_parts - WHERE groups_id = %d - AND numberid <= %d', - $groupArr['id'], - $missingParts[$missingCount - 1]->numberid - ) - ); - - $partsRepaired = 0; - if ($result > 0) { - $partsRepaired = ($missingCount - $result[0]->num); - } - - // Update attempts on remaining parts for active group - if (isset($missingParts[$missingCount - 1]->id)) { - DB::update( - sprintf( - ' - UPDATE missed_parts - SET attempts = attempts + 1 - WHERE groups_id = %d - AND numberid <= %d', - $groupArr['id'], - $missingParts[$missingCount - 1]->numberid - ) - ); - } - - if ($this->_echoCLI) { - $this->colorCli->primary( - PHP_EOL. - number_format($partsRepaired). - ' parts repaired.' - ); - } + return; } - // Remove articles that we cant fetch after x attempts. - DB::transaction(function () use ($groupArr) { - DB::delete( - sprintf( - 'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d', - $this->_partRepairMaxTries, - $groupArr['id'] - ) - ); - }, 10); + if ($this->config->echoCli) { + $this->colorCli->primary('Attempting to repair '.number_format($missingCount).' parts.'); + } + + // Group into continuous ranges + $ranges = $this->groupMissingPartsIntoRanges($missingParts); + + // Download missing parts in ranges + foreach ($ranges as $range) { + if ($this->config->echoCli) { + echo \chr(random_int(45, 46)).PHP_EOL; + } + + $this->scan($groupArr, $range['partfrom'], $range['partto'], 'partrepair', $range['partlist']); + } + + // Calculate parts repaired + $lastPartNumber = $missingParts[$missingCount - 1]->numberid; + $remainingCount = $this->missedPartHandler->getCount($groupArr['id'], $lastPartNumber); + $partsRepaired = $missingCount - $remainingCount; + + // Update attempts on remaining parts + if (isset($missingParts[$missingCount - 1]->id)) { + $this->missedPartHandler->incrementAttempts($groupArr['id'], $lastPartNumber); + } + + if ($this->config->echoCli) { + $this->colorCli->primary(PHP_EOL.number_format($partsRepaired).' parts repaired.'); + } + + // Remove articles that exceeded max tries + $this->missedPartHandler->cleanupExhaustedParts($groupArr['id']); } /** @@ -1364,61 +407,45 @@ class Binaries public function postdate(int $post, array $groupData): int { $currentPost = $post; + $attempts = 0; + $date = 0; - $attempts = $date = 0; do { - // Try to get the article date locally first. - // Try to get locally. + // Try to get the article date locally first $local = DB::select( sprintf( - ' - SELECT c.date AS date - FROM collections c - INNER JOIN binaries b ON(c.id=b.collections_id) - INNER JOIN parts p ON(b.id=p.binaries_id) - WHERE p.number = %s', + 'SELECT c.date AS date FROM collections c + INNER JOIN binaries b ON(c.id=b.collections_id) + INNER JOIN parts p ON(b.id=p.binaries_id) + WHERE p.number = %s', $currentPost ) ); - if (! empty($local) && \count($local) > 0) { + + if (! empty($local)) { $date = $local[0]->date; break; } - // If we could not find it locally, try usenet. - $header = $this->_nntp->getXOVER($currentPost); - if (! $this->_nntp::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') { + // Try usenet + $header = $this->nntp->getXOVER((string) $currentPost); + if (! $this->nntp::isError($header) && isset($header[0]['Date']) && $header[0]['Date'] !== '') { $date = $header[0]['Date']; break; } - // Try to get a different article number. - if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { - $tempPost = round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); - if ($tempPost < $groupData['first']) { - $tempPost = $groupData['first']; - } - } else { - $tempPost = round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP); - if ($tempPost > $groupData['last']) { - $tempPost = $groupData['last']; - } - } - // If we got the same article number as last time, give up. - if ($tempPost === $currentPost) { + // Try a different article number + $currentPost = $this->getNextArticleToTry($currentPost, $groupData); + if ($currentPost === null) { break; } - $currentPost = $tempPost; } while ($attempts++ <= 20); - // If we didn't get a date, set it to now. if (! $date) { - $date = time(); - } else { - $date = strtotime($date); + return time(); } - return $date; + return strtotime($date); } /** @@ -1432,82 +459,349 @@ class Binaries public function daytopost(int $days, array $data): string { $goalTime = now()->subDays($days)->timestamp; - // The time we want = current unix time (ex. 1395699114) - minus 86400 (seconds in a day) - // times days wanted. (ie 1395699114 - 2592000 (30days)) = 1393107114 - // The servers oldest date. $firstDate = $this->postdate($data['first'], $data); if ($goalTime < $firstDate) { - // If the date we want is older than the oldest date in the group return the groups oldest article. return $data['first']; } - // The servers newest date. $lastDate = $this->postdate($data['last'], $data); if ($goalTime > $lastDate) { - // If the date we want is newer than the groups newest date, return the groups newest article. return $data['last']; } - if ($this->_echoCLI) { + if ($this->config->echoCli) { $this->colorCli->primary( 'Searching for an approximate article number for group '.$data['group'].' '.$days.' days back.' ); } - // Pick the middle to start with - $wantedArticle = round(($data['last'] + $data['first']) / 2); + return $this->binarySearchArticleByDate($goalTime, $data); + } + + // ==================== Private Helper Methods ==================== + + private function selectNntpGroup(array &$groupMySQL): ?array + { + $groupNNTP = $this->nntp->selectGroup($groupMySQL['name']); + + if ($this->nntp::isError($groupNNTP)) { + $groupNNTP = $this->nntp->dataError($this->nntp, $groupMySQL['name']); + + if (isset($groupNNTP['code']) && (int) $groupNNTP['code'] === 411) { + UsenetGroup::disableIfNotExist($groupMySQL['id']); + } + + if ($this->nntp::isError($groupNNTP)) { + return null; + } + } + + return $groupNNTP; + } + + private function calculateArticleRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array + { + if ((int) $groupMySQL['last_record'] === 0) { + return $this->calculateNewGroupRange($groupNNTP); + } + + return $this->calculateExistingGroupRange($groupMySQL, $groupNNTP, $maxHeaders); + } + + private function calculateNewGroupRange(array $groupNNTP): array + { + if ($this->config->newGroupScanByDays) { + $first = (int) $this->daytopost($this->config->newGroupDaysToScan, $groupNNTP); + } elseif ($groupNNTP['first'] >= ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer))) { + $first = (int) $groupNNTP['first']; + } else { + $first = (int) ($groupNNTP['last'] - ($this->config->newGroupMessagesToScan + $this->config->messageBuffer)); + } + + $leaveOver = $this->config->messageBuffer; + $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver); + + if ($last < $first) { + $last = $groupLast = $first; + } + + $total = (int) ($groupLast - $first); + $realTotal = (int) ($groupNNTP['last'] - $first); + + return [ + 'first' => $first, + 'last' => $last, + 'groupLast' => $groupLast, + 'total' => $total, + 'realTotal' => $realTotal, + 'leaveOver' => $leaveOver, + 'isNew' => true, + ]; + } + + private function calculateExistingGroupRange(array $groupMySQL, array $groupNNTP, int $maxHeaders): array + { + $first = (int) $groupMySQL['last_record']; + $totalCount = (int) ($groupNNTP['last'] - $first); + + if ($totalCount > ($this->config->messageBuffer * 2)) { + $leaveOver = (int) round($totalCount % $this->config->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->config->messageBuffer; + } else { + $leaveOver = (int) round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); + } + + $last = $groupLast = (int) ($groupNNTP['last'] - $leaveOver); + + if ($last < $first) { + $last = $groupLast = $first; + } + + $total = (int) ($groupLast - $first); + $realTotal = (int) ($groupNNTP['last'] - $first); + + // Apply max headers limit + if ($maxHeaders > 0 && $maxHeaders < ($groupLast - $first)) { + $groupLast = $last = (int) ($first + $maxHeaders); + $total = (int) ($groupLast - $first); + } + + return [ + 'first' => $first, + 'last' => $last, + 'groupLast' => $groupLast, + 'total' => $total, + 'realTotal' => $realTotal, + 'leaveOver' => $leaveOver, + 'isNew' => false, + ]; + } + + private function processArticleRange(array &$groupMySQL, array $groupNNTP, array $range): void + { + $first = (int) $range['first']; + $last = (int) $range['last']; + $groupLast = (int) $range['groupLast']; + $done = false; + + while (! $done) { + // Calculate chunk bounds + if ($range['total'] > $this->config->messageBuffer) { + $last = (int) min($first + $this->config->messageBuffer, $groupLast); + } + + $first++; + + if ($this->config->echoCli) { + $this->colorCli->header( + PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first). + ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. + number_format($groupLast - $last).' articles in queue).' + ); + } + + // Scan this chunk + $scanSummary = $this->scan($groupMySQL, $first, $last); + + // Update group record + $this->updateGroupAfterScan($groupMySQL, $groupNNTP, $scanSummary, $last); + + if ($last === $groupLast) { + $done = true; + } else { + $first = $last; + } + } + } + + private function updateGroupAfterScan(array &$groupMySQL, array $groupNNTP, array $scanSummary, int $last): void + { + if (! empty($scanSummary)) { + // New group - update first record + if ($groupMySQL['first_record_postdate'] === null && (int) $groupMySQL['first_record'] === 0) { + $groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; + $groupMySQL['first_record_postdate'] = isset($scanSummary['firstArticleDate']) + ? strtotime($scanSummary['firstArticleDate']) + : $this->postdate($groupMySQL['first_record'], $groupNNTP); + + UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ + 'first_record' => $scanSummary['firstArticleNumber'], + 'first_record_postdate' => Carbon::createFromTimestamp($groupMySQL['first_record_postdate'], date_default_timezone_get()), + ]); + } + + $lastArticleDate = isset($scanSummary['lastArticleDate']) + ? strtotime($scanSummary['lastArticleDate']) + : $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); + + UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ + 'last_record' => $scanSummary['lastArticleNumber'], + 'last_record_postdate' => Carbon::createFromTimestamp($lastArticleDate, date_default_timezone_get()), + 'last_updated' => now(), + ]); + } else { + UsenetGroup::query()->where('id', $groupMySQL['id'])->update([ + 'last_record' => $last, + 'last_updated' => now(), + ]); + } + } + + private function downloadHeaders(bool $partRepair): ?array + { + if ($partRepair) { + $headers = $this->nntp->getOverview($this->first.'-'.$this->last, true, false); + } else { + $headers = $this->nntp->getXOVER($this->first.'-'.$this->last); + } + + if ($this->nntp::isError($headers)) { + if ($partRepair) { + return null; + } + + // Retry without compression + $this->nntp->doQuit(); + if ($this->nntp->doConnect(false) !== true) { + return null; + } + + $this->nntp->selectGroup($this->groupMySQL['name']); + $headers = $this->nntp->getXOVER($this->first.'-'.$this->last); + $this->nntp->enableCompression(); + + if ($this->nntp::isError($headers)) { + $message = ((int) $headers->code === 0 ? 'Unknown error' : $headers->message); + $this->log("Code {$headers->code}: $message\nSkipping group: {$this->groupMySQL['name']}", __FUNCTION__, 'error'); + + return null; + } + } + + return $headers; + } + + private function handlePartRepairTracking(array $headersNotInserted, array $parsedHeaders): void + { + $notInsertedCount = \count($headersNotInserted); + if ($notInsertedCount > 0) { + $this->missedPartHandler->addMissingParts($headersNotInserted, $this->groupMySQL['id']); + $this->log($notInsertedCount.' articles failed to insert!', __FUNCTION__, 'warning'); + } + + // Check for missing headers in range + $expectedCount = $this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1; + if ($expectedCount > \count($this->headersReceived)) { + $rangeNotReceived = array_diff(range($this->first, $this->last), $this->headersReceived); + $notReceivedCount = \count($rangeNotReceived); + + if ($notReceivedCount > 0) { + $this->missedPartHandler->addMissingParts($rangeNotReceived, $this->groupMySQL['id']); + + if ($this->config->echoCli) { + $this->colorCli->alternate( + 'Server did not return '.$notReceivedCount.' articles from '.$this->groupMySQL['name'].'.' + ); + } + } + } + } + + private function groupMissingPartsIntoRanges(array $missingParts): array + { + $ranges = []; + $partList = []; + $firstPart = $lastNum = $missingParts[0]->numberid; + + foreach ($missingParts as $part) { + if (($part->numberid - $firstPart) > ($this->config->messageBuffer / 4)) { + $ranges[] = [ + 'partfrom' => $firstPart, + 'partto' => $lastNum, + 'partlist' => $partList, + ]; + $firstPart = $part->numberid; + $partList = []; + } + $partList[] = $part->numberid; + $lastNum = $part->numberid; + } + + $ranges[] = [ + 'partfrom' => $firstPart, + 'partto' => $lastNum, + 'partlist' => $partList, + ]; + + return $ranges; + } + + private function getNextArticleToTry(int $currentPost, array $groupData): ?int + { + if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { + $tempPost = (int) round($currentPost / (random_int(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); + if ($tempPost < $groupData['first']) { + $tempPost = $groupData['first']; + } + } else { + $tempPost = (int) round((random_int(1005, 1012) / 1000) * $currentPost, 0, PHP_ROUND_HALF_UP); + if ($tempPost > $groupData['last']) { + $tempPost = $groupData['last']; + } + } + + // If we got the same article number, give up + if ($tempPost === $currentPost) { + return null; + } + + return $tempPost; + } + + private function binarySearchArticleByDate(int $goalTime, array $data): string + { + $wantedArticle = (int) round(($data['last'] + $data['first']) / 2); $aMax = $data['last']; $aMin = $data['first']; $oldArticle = $articleTime = null; while (true) { - // Article exists outside available range, this shouldn't happen if ($wantedArticle <= $data['first'] || $wantedArticle >= $data['last']) { break; } - // Keep a note of the last articles we checked $reallyOldArticle = $oldArticle; $oldArticle = $wantedArticle; - // Get the date of this article $articleTime = $this->postdate($wantedArticle, $data); - // Article doesn't exist, start again with something random if (! $articleTime) { $wantedArticle = random_int($aMin, $aMax); $articleTime = $this->postdate($wantedArticle, $data); } if ($articleTime < $goalTime) { - // Article is older than we want $aMin = $oldArticle; - $wantedArticle = round(($aMax + $oldArticle) / 2); - if ($this->_echoCLI) { + $wantedArticle = (int) round(($aMax + $oldArticle) / 2); + if ($this->config->echoCli) { echo '-'; } } elseif ($articleTime > $goalTime) { - // Article is newer than we want $aMax = $oldArticle; - $wantedArticle = round(($aMin + $oldArticle) / 2); - if ($this->_echoCLI) { + $wantedArticle = (int) round(($aMin + $oldArticle) / 2); + if ($this->config->echoCli) { echo '+'; } - } elseif ($articleTime === $goalTime) { - // Exact match. We did it! (this will likely never happen though) + } else { break; } - // We seem to be flip-flopping between 2 articles, assume we're out of articles to check. - // End on an article more recent than our oldest so that we don't miss any releases. if ($reallyOldArticle === $wantedArticle && ($goalTime - $articleTime) <= 0) { break; } } - $wantedArticle = (int) $wantedArticle; - if ($this->_echoCLI) { + if ($this->config->echoCli) { $goalCarbon = Carbon::createFromTimestamp($goalTime, date_default_timezone_get()); $articleCarbon = Carbon::createFromTimestamp($articleTime, date_default_timezone_get()); $diffDays = $goalCarbon->diffInDays($articleCarbon, true); @@ -1517,103 +811,107 @@ class Binaries ); } - return $wantedArticle; + return (string) $wantedArticle; } - /** - * Add article numbers from missing headers to DB. - * - * @param array $numbers The article numbers of the missing headers. - * @param int $groupID The ID of this groups. - */ - private function addMissingParts(array $numbers, int $groupID): void - { - $driver = DB::getDriverName(); - if ($driver === 'sqlite') { - // Use UPSERT with ON CONFLICT for sqlite - foreach ($numbers as $number) { - DB::statement('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON CONFLICT(numberid, groups_id) DO UPDATE SET attempts = attempts + 1', [$number, $groupID]); - } + // ==================== Output Methods ==================== + private function outputNoNewArticles(array $groupMySQL, array $groupNNTP, array $range): void + { + if ($this->config->echoCli) { + $this->colorCli->primary( + 'No new articles for '.$groupMySQL['name'].' (first '.number_format((int) $range['first']). + ', last '.number_format((int) $range['last']).', grouplast '.number_format((int) $groupMySQL['last_record']). + ', total '.number_format((int) $range['total']).")\n".'Server oldest: '.number_format((int) $groupNNTP['first']). + ' Server newest: '.number_format((int) $groupNNTP['last']).' Local newest: '.number_format((int) $groupMySQL['last_record']) + ); + } + } + + private function outputNewArticlesInfo(array $groupMySQL, array $groupNNTP, array $range): void + { + if (! $this->config->echoCli) { return; } - $insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; - foreach ($numbers as $number) { - $insertStr .= '('.$number.','.$groupID.'),'; - } + $message = $range['isNew'] + ? 'New group '.$groupNNTP['group'].' starting with '. + ($this->config->newGroupScanByDays + ? $this->config->newGroupDaysToScan.' days' + : number_format($this->config->newGroupMessagesToScan).' messages').' worth.' + : 'Group '.$groupNNTP['group'].' has '.number_format((int) $range['realTotal']).' new articles.'; - DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); + $this->colorCli->primary( + $message. + ' Leaving '.number_format((int) $range['leaveOver']). + " for next pass.\nServer oldest: ".number_format((int) $groupNNTP['first']). + ' Server newest: '.number_format((int) $groupNNTP['last']). + ' Local newest: '.number_format((int) $groupMySQL['last_record']) + ); } - /** - * Clean up part repair table. - * - * @param array $numbers The article numbers. - * @param int $groupID The ID of the group. - * - * @throws \Throwable - */ - private function removeRepairedParts(array $numbers, int $groupID): void + private function outputHeaderInitial(): void { - $sql = 'DELETE FROM missed_parts WHERE numberid in ('; - foreach ($numbers as $number) { - $sql .= $number.','; - } - DB::transaction(static function () use ($groupID, $sql) { - DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupID); - }, 10); + $this->colorCli->primary( + 'Received '.\count($this->headersReceived). + ' articles of '.number_format($this->last - $this->first + 1).' requested, '. + $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' + ); } - /** - * Are white or black lists loaded for a group name? - */ - protected array $_listsFound = []; + private function outputHeaderDuration(): void + { + if (! $this->config->echoCli) { + return; + } + + $currentMicroTime = now(); + $this->colorCli->alternateOver(number_format($this->timeHeaders, 2).'s'). + $this->colorCli->primaryOver(' to download articles, '). + $this->colorCli->alternateOver(number_format($this->timeCleaning, 2).'s'). + $this->colorCli->primaryOver(' to process collections, '). + $this->colorCli->alternateOver(number_format($this->timeInsert, 2).'s'). + $this->colorCli->primaryOver(' to insert binaries/parts, '). + $this->colorCli->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startPR, true), 2).'s'). + $this->colorCli->primaryOver(' for part repair, '). + $this->colorCli->alternateOver(number_format($currentMicroTime->diffInSeconds($this->startLoop, true), 2).'s'). + $this->colorCli->primary(' total.'); + } + + // ==================== Logging Methods ==================== - /** - * Log / Echo message. - * - * @param string $message Message to log. - * @param string $method Method that called this. - * @param string $color ColorCLI method name. - */ private function log(string $message, string $method, string $color): void { - if ($this->_echoCLI) { + if ($this->config->echoCli) { $this->colorCli->$color($message.' ['.__CLASS__."::$method]"); } } - protected function runQuery($query): bool + private function logError(string $message): void { - try { - return DB::insert($query); - } catch (QueryException $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); - } catch (\PDOException $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); + if ($this->config->echoCli) { + $this->colorCli->error($message); + } + if (config('app.debug')) { + Log::error($message); } - - return false; } - private function getFileCount($subject): array - { - if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])[\s$:]/i', $subject, $fileCount)) { - $fileCount[1] = $fileCount[3] = 0; - } + // ==================== Legacy BC Properties ==================== - return $fileCount; - } + /** + * @deprecated Use BinariesConfig instead + */ + public int $messageBuffer; + + /** + * @deprecated Use BlacklistService instead + */ + public array $blackList = []; + + /** + * @deprecated Use BlacklistService instead + */ + public array $whiteList = []; } + diff --git a/Blacklight/NZBContents.php b/Blacklight/NZBContents.php index 63f39000a..41c1fe914 100755 --- a/Blacklight/NZBContents.php +++ b/Blacklight/NZBContents.php @@ -158,18 +158,78 @@ class NZBContents } // --- NFO Detection --- - // Check for explicit NFO files first - if ($nfoCheck && ! $foundNFO && isset($firstSegmentId) && preg_match('/\.\b(nfo|diz|info?)\b(?![.-])/i', $subject)) { - $nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId]; - $foundNFO = true; // Found an explicit NFO, prioritize this + // Check for explicit NFO files first (with enhanced patterns) + if ($nfoCheck && ! $foundNFO && isset($firstSegmentId)) { + // Standard NFO extensions + if (preg_match('/\.\b(nfo|diz|info?)\b(?![.-])/i', $subject)) { + $nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId, 'priority' => 1]; + $foundNFO = true; + } + // Alternative NFO naming patterns (group-specific or obfuscated) + elseif (preg_match('/(?:^|["\s])(?:file(?:_?id)?|readme|release|info(?:rmation)?|about|desc(?:ription)?|notes?|read\.?me|00-|000-|0-|_-_).*?\.(?:txt|nfo|diz)(?:["\s]|$)/i', $subject)) { + $nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId, 'priority' => 2]; + $foundNFO = true; + } } - // Check for potential "hidden" NFOs (single segment, common name, not other known types) + + // Check for potential "hidden" NFOs with improved detection // Only consider this if an explicit NFO wasn't found yet - elseif ($nfoCheck && ! $foundNFO && ! $hiddenNFO && isset($firstSegmentId) && $segmentCountInFile === 1 && preg_match('/\(1\/1\)$/i', $subject)) { - // Simplified exclusion: check if it's NOT likely another common file type based on extension pattern - if (! preg_match('/\.(?:exe|com|bat|cmd|scr|dll|zip|rar|[rst]\d{2}|[a-z0-9]{3}|7z|ace|tar|gz|bz2|iso|bin|cue|img|mdf|nrg|dmg|vhd|mp3|flac|ogg|aac|wav|wma|avi|mkv|mp4|mov|wmv|mpg|mpeg|ts|vob|jpg|jpeg|png|gif|bmp|tif|tiff|psd|pdf|doc|docx|xls|xlsx|ppt|pptx|txt|log|xml|html|css|js|php|py|java|c|cpp|h|cs|sql|db|dbf|mdb|accdb|par2?|sfv|md5|sha1|sha256|url|lnk|cfg|ini|inf|sys|tmp|bak|msi|pkg|deb|rpm|apk|ipa)\b/i', $subject)) { - $nfoMessageId = ['hidden' => true, 'id' => $firstSegmentId]; - $hiddenNFO = true; // Found a potential hidden NFO + if ($nfoCheck && ! $foundNFO && ! $hiddenNFO && isset($firstSegmentId)) { + $isHiddenNfoCandidate = false; + + // Pattern 1: Single segment files with (1/1) + if ($segmentCountInFile === 1 && preg_match('/\(1\/1\)$/i', $subject)) { + $isHiddenNfoCandidate = true; + } + + // Pattern 2: Small segment count (1-2) with NFO-like names but no extension + if (! $isHiddenNfoCandidate && $segmentCountInFile <= 2 && preg_match('/(?:^|["\s])(?:nfo|info|readme|release|file_?id|about)(?:["\s]|$)/i', $subject)) { + $isHiddenNfoCandidate = true; + } + + // Pattern 3: Scene-style NFO naming (group-release.nfo without extension visible) + if (! $isHiddenNfoCandidate && $segmentCountInFile === 1 && preg_match('/^[a-z0-9._-]+["\s]*\(1\/1\)/i', $subject)) { + // Check for scene-like naming pattern + if (preg_match('/^[a-z0-9]+[._-][a-z0-9._-]+["\s]*\(1\/1\)/i', $subject)) { + $isHiddenNfoCandidate = true; + } + } + + // Pattern 4: Very small files (NFOs are typically small) + // Files described as very small in bytes could be NFOs + if (! $isHiddenNfoCandidate && $segmentCountInFile === 1 && preg_match('/yEnc\s*\(\d+\)\s*\[1\/1\]/i', $subject)) { + $isHiddenNfoCandidate = true; + } + + if ($isHiddenNfoCandidate) { + // Enhanced exclusion: check if it's NOT likely another common file type + $excludedExtensions = '/\.(?:' . + // Executables + 'exe|com|bat|cmd|scr|dll|msi|pkg|deb|rpm|apk|ipa|app|' . + // Archives + 'zip|rar|[rst]\d{2}|7z|ace|tar|gz|bz2|xz|lzma|cab|iso|bin|cue|img|mdf|nrg|dmg|vhd|' . + // Audio + 'mp3|flac|ogg|aac|wav|wma|m4a|opus|ape|wv|mpc|' . + // Video + 'avi|mkv|mp4|mov|wmv|mpg|mpeg|ts|vob|m2ts|webm|flv|ogv|divx|xvid|' . + // Images + 'jpg|jpeg|png|gif|bmp|tif|tiff|psd|webp|svg|ico|raw|cr2|nef|' . + // Documents + 'pdf|doc|docx|xls|xlsx|ppt|pptx|odt|ods|odp|rtf|epub|mobi|azw|' . + // Code + 'html|htm|css|js|php|py|java|c|cpp|h|cs|sql|json|xml|yml|yaml|' . + // Data + 'db|dbf|mdb|accdb|sqlite|csv|' . + // Verification + 'par2?|sfv|md5|sha1|sha256|sha512|crc|' . + // Misc + 'url|lnk|cfg|ini|inf|sys|tmp|bak|log|srt|sub|idx|ass|ssa|vtt' . + ')\b/i'; + + if (! preg_match($excludedExtensions, $subject)) { + $nfoMessageId = ['hidden' => true, 'id' => $firstSegmentId, 'priority' => 10]; + $hiddenNFO = true; + } } } diff --git a/Blacklight/NameFixer.php b/Blacklight/NameFixer.php index b2bc7623e..e576a971b 100755 --- a/Blacklight/NameFixer.php +++ b/Blacklight/NameFixer.php @@ -140,7 +140,14 @@ class NameFixer /** * Attempts to fix release names using the NFO. * + * Enhanced to use the new Nfo class metadata extraction features for better + * release name identification from IMDB, TVDB, TMDB, and other media sources. * + * @param int|string $time Time limit for query + * @param bool $echo Whether to actually update the database + * @param int $cats Category filter (2=misc/hashed, 3=predb) + * @param bool $nameStatus Whether to update status columns + * @param bool $show Whether to show output * * @throws \Exception */ @@ -149,6 +156,9 @@ class NameFixer $this->_echoStartMessage($time, '.nfo files'); $type = 'NFO, '; + // Initialize the Nfo parser + $nfoParser = new Nfo(); + // Only select releases we haven't checked here before $preId = false; if ($cats === 3) { @@ -208,7 +218,18 @@ class NameFixer } $this->reset(); - $this->checkName($releaseRow[0], $echo, $type, $nameStatus, $show, $preId); + + // First, try to extract metadata using the enhanced Nfo parser + $nfoMetadata = $nfoParser->parseNfoMetadata($releaseRow[0]->textstring); + + // Try to find a better name using extracted media IDs + $betterNameFound = $this->tryNfoMetadataRename($releaseRow[0], $nfoMetadata, $echo, $type, $nameStatus, $show, $preId); + + // If metadata extraction didn't find a name, fall back to traditional checks + if (! $betterNameFound) { + $this->checkName($releaseRow[0], $echo, $type, $nameStatus, $show, $preId); + } + $this->_echoRenamed($show); } $this->_echoFoundCount($echo, ' NFO\'s'); @@ -217,10 +238,249 @@ class NameFixer } } + /** + * Try to rename a release using extracted NFO metadata. + * + * Uses media IDs (IMDB, TVDB, TMDB) and codec info from NFO to build + * a better release name. + * + * @param object $release The release object + * @param array $nfoMetadata Metadata extracted from NFO by Nfo::parseNfoMetadata() + * @param bool $echo Whether to update database + * @param string $type The type string for logging + * @param bool $nameStatus Whether to update status columns + * @param bool $show Whether to show output + * @param bool $preId Whether processing for PreDB + * @return bool True if a better name was found and applied + * + * @throws \Exception + */ + protected function tryNfoMetadataRename(object $release, array $nfoMetadata, bool $echo, string $type, $nameStatus, bool $show, bool $preId = false): bool + { + // Skip if already processed + if ($this->done || $this->relid === (int) $release->releases_id) { + return false; + } + + // Try to get a name from media database IDs + $mediaIds = $nfoMetadata['media_ids'] ?? []; + $codecInfo = $nfoMetadata['codec_info'] ?? []; + $releaseGroup = $nfoMetadata['group'] ?? null; + + // Priority order: IMDB (movies/TV), TMDB, TVDB, TVMaze + foreach ($mediaIds as $mediaId) { + $newName = $this->getNameFromMediaId($mediaId['source'], $mediaId['id']); + if ($newName !== null) { + // Enhance the name with codec info if available + $enhancedName = $this->enhanceNameWithCodecInfo($newName, $codecInfo, $releaseGroup); + + $this->updateRelease( + $release, + $enhancedName, + 'nfoCheck: Media ID ('.$mediaId['source'].': '.$mediaId['id'].')', + $echo, + $type, + $nameStatus, + $show + ); + + return true; + } + } + + // If we have codec info but no media ID match, try to enhance existing name patterns + if (! empty($codecInfo)) { + // Check if there's a recognizable title pattern in the NFO + $titleFromNfo = $this->extractTitleFromNfoContent($release->textstring); + if ($titleFromNfo !== null) { + $enhancedName = $this->enhanceNameWithCodecInfo($titleFromNfo, $codecInfo, $releaseGroup); + if (strtolower($enhancedName) !== strtolower($release->searchname)) { + $this->updateRelease( + $release, + $enhancedName, + 'nfoCheck: NFO Title with Codec Info', + $echo, + $type, + $nameStatus, + $show + ); + + return true; + } + } + } + + return false; + } + + /** + * Get a release name from a media database ID. + * + * Queries local database or external APIs to resolve media IDs to titles. + * + * @param string $source The source database (imdb, thetvdb, tmdb_movie, tmdb_tv, tvmaze, anidb, mal) + * @param string $id The media ID + * @return string|null The title if found, null otherwise + */ + protected function getNameFromMediaId(string $source, string $id): ?string + { + switch ($source) { + case 'imdb': + // Check if we have this IMDB ID in our movieinfo table + $movie = \App\Models\MovieInfo::where('imdbid', ltrim($id, 't'))->first(['title', 'year']); + if ($movie !== null) { + return $movie->year > 0 ? "{$movie->title} ({$movie->year})" : $movie->title; + } + // Also check Video table for TV shows with IMDB ID + $video = \App\Models\Video::where('imdb', (int) ltrim($id, 't'))->first(['title']); + if ($video !== null) { + return $video->title; + } + break; + + case 'thetvdb': + // Check Video table for TVDB ID + $video = \App\Models\Video::where('tvdb', (int) $id)->first(['title']); + if ($video !== null) { + return $video->title; + } + break; + + case 'tmdb_movie': + // Check local movie database for TMDB ID + $movie = \App\Models\MovieInfo::where('tmdbid', (int) $id)->first(['title', 'year']); + if ($movie !== null) { + return $movie->year > 0 ? "{$movie->title} ({$movie->year})" : $movie->title; + } + break; + + case 'tmdb_tv': + // Check Video table for TMDB ID + $video = \App\Models\Video::where('tmdb', (int) $id)->first(['title']); + if ($video !== null) { + return $video->title; + } + break; + + case 'tvmaze': + // Check Video table for TVMaze ID + $video = \App\Models\Video::where('tvmaze', (int) $id)->first(['title']); + if ($video !== null) { + return $video->title; + } + break; + + case 'anidb': + // Check AniDB table - uses Video table with anidb column + $video = \App\Models\Video::where('anidb', (int) $id)->first(['title']); + if ($video !== null) { + return $video->title; + } + // Also check anidb_titles table + $anime = \App\Models\AnidbTitle::where('anidbid', (int) $id)->first(['title']); + if ($anime !== null) { + return $anime->title; + } + break; + + case 'trakt': + // Check Video table for Trakt ID + $video = \App\Models\Video::where('trakt', (int) $id)->first(['title']); + if ($video !== null) { + return $video->title; + } + break; + + case 'mal': + // MyAnimeList - currently no direct support in the database + break; + } + + return null; + } + + /** + * Enhance a title with codec/resolution information. + * + * @param string $title The base title + * @param array $codecInfo Codec info from Nfo::extractCodecInfo() + * @param string|null $releaseGroup The release group name if found + * @return string The enhanced title + */ + protected function enhanceNameWithCodecInfo(string $title, array $codecInfo, ?string $releaseGroup = null): string + { + $parts = [$title]; + + // Add resolution + if (! empty($codecInfo['resolution'])) { + $parts[] = $codecInfo['resolution']; + } + + // Add video codec + if (! empty($codecInfo['video'])) { + $parts[] = $codecInfo['video']; + } + + // Add audio codec + if (! empty($codecInfo['audio'])) { + $parts[] = $codecInfo['audio']; + } + + // Add release group + if ($releaseGroup !== null) { + $parts[] = '-'.$releaseGroup; + + return implode('.', array_slice($parts, 0, -1)).$parts[count($parts) - 1]; + } + + return implode('.', $parts); + } + + /** + * Extract a recognizable title from NFO content. + * + * Looks for common title patterns like "Title (Year)" or scene-style names. + * + * @param string $nfoContent The NFO content + * @return string|null The extracted title or null if not found + */ + protected function extractTitleFromNfoContent(string $nfoContent): ?string + { + // Look for "Title (Year)" pattern - common in movie NFOs + if (preg_match('/^[\s\S]*?([A-Z][A-Za-z0-9\s\.\'\-\:]+(?:\s+\((?:19|20)\d{2}\)))/m', $nfoContent, $matches)) { + $title = trim($matches[1]); + // Validate it's not too short or too long + if (strlen($title) >= 5 && strlen($title) <= 150) { + return $title; + } + } + + // Look for release name patterns (Scene style) + if (preg_match('/(?:Release|Rls|Name)\s*[:\-]?\s*([A-Za-z0-9][\w.\-]+(?:[\s._-][\w.\-]+)+)/i', $nfoContent, $matches)) { + $title = trim($matches[1]); + if (strlen($title) >= 5 && strlen($title) <= 150) { + return $title; + } + } + + // Look for title in common NFO header patterns + if (preg_match('/(?:presents|proudly brings)\s*[:\-]?\s*([A-Za-z0-9][\w.\s\-\']+(?:[\s._-][\w.\s\-\']+)*)/i', $nfoContent, $matches)) { + $title = trim($matches[1]); + if (strlen($title) >= 5 && strlen($title) <= 150) { + return $title; + } + } + + return null; + } + /** * Attempts to fix release names using the File name. * - * + * Enhanced to better handle: + * - RAR archives and their contents + * - Multiple file selection with priority ordering + * - Modern scene release naming conventions * * @throws \Exception */ @@ -264,10 +524,37 @@ class NameFixer $this->_totalReleases = $total; $this->colorCLI->info(number_format($total).' file names to process.'); + // Group files by release for better processing + $releaseFiles = []; foreach ($releases as $release) { + $releaseId = $release->releases_id; + if (! isset($releaseFiles[$releaseId])) { + $releaseFiles[$releaseId] = [ + 'release' => $release, + 'files' => [], + ]; + } + $releaseFiles[$releaseId]['files'][] = $release->textstring; + } + + foreach ($releaseFiles as $releaseId => $data) { $this->reset(); - $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); $this->checked++; + + // Prioritize files for matching + $prioritizedFiles = $this->prioritizeFilesForMatching($data['files']); + + foreach ($prioritizedFiles as $filename) { + $release = clone $data['release']; + $release->textstring = $filename; + + $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); + + if ($this->matched) { + break; // Found a match, stop trying other files + } + } + $this->_echoRenamed($show); } @@ -277,10 +564,60 @@ class NameFixer } } + /** + * Prioritize files for name matching. + * + * Returns files sorted by usefulness for name matching: + * 1. Main video files (mkv, avi, mp4, etc.) + * 2. SRR files (often contain original release name) + * 3. NFO files + * 4. RAR archives (first file, not parts) + * 5. Other files + */ + protected function prioritizeFilesForMatching(array $files): array + { + $videoFiles = []; + $srrFiles = []; + $nfoFiles = []; + $rarMainFiles = []; + $otherFiles = []; + + foreach ($files as $file) { + $lowerFile = strtolower($file); + + // Skip sample and proof files + if (preg_match('/[\.\-_](sample|proof|subs?|thumbs?)[\.\-_]/i', $file)) { + continue; + } + + if (preg_match('/\.(mkv|avi|mp4|m4v|wmv|divx|ts|m2ts)$/i', $file)) { + $videoFiles[] = $file; + } elseif (str_ends_with($lowerFile, '.srr')) { + $srrFiles[] = $file; + } elseif (str_ends_with($lowerFile, '.nfo')) { + $nfoFiles[] = $file; + } elseif (preg_match('/\.rar$/i', $file) && ! preg_match('/\.part\d+\.rar$/i', $file)) { + // Main RAR file (not .part01.rar style) + $rarMainFiles[] = $file; + } elseif (preg_match('/\.part0*1\.rar$/i', $file)) { + // First part of split RAR + $rarMainFiles[] = $file; + } else { + $otherFiles[] = $file; + } + } + + // Sort video files by size (longer names often more descriptive) + usort($videoFiles, fn($a, $b) => strlen($b) - strlen($a)); + + return array_merge($videoFiles, $srrFiles, $nfoFiles, $rarMainFiles, $otherFiles); + } + /** * Attempts to fix release names using the rar file crc32 hash. * - * + * Enhanced to better match RAR file CRC32 hashes across releases, + * with improved size tolerance and multiple file support. * * @throws \Exception */ @@ -293,24 +630,28 @@ class NameFixer if ($cats === 3) { $query = sprintf( ' - SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, + SELECT rf.crc32 AS textstring, rf.name AS filename, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, rf.releases_id AS fileid, rel.id AS releases_id FROM releases rel INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE predb_id = 0' + WHERE predb_id = 0 + AND rf.crc32 != \'\' + AND rf.crc32 IS NOT NULL' ); $cats = 2; $preId = true; } else { $query = sprintf( ' - SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, + SELECT rf.crc32 AS textstring, rf.name AS filename, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, rf.releases_id AS fileid, rel.id AS releases_id FROM releases rel INNER JOIN release_files rf ON rf.releases_id = rel.id WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) AND rel.predb_id = 0 - AND rel.proc_crc32 = %d', + AND rel.proc_crc32 = %d + AND rf.crc32 != \'\' + AND rf.crc32 IS NOT NULL', self::IS_RENAMED_NONE, Category::OTHER_MISC, Category::OTHER_HASHED, @@ -324,10 +665,42 @@ class NameFixer $this->_totalReleases = $total; $this->colorCLI->info(number_format($total).' CRC32\'s to process.'); + // Group by release to handle multiple CRC32 values per release + $releasesCrc = []; foreach ($releases as $release) { + $releaseId = $release->releases_id; + if (! isset($releasesCrc[$releaseId])) { + $releasesCrc[$releaseId] = [ + 'release' => $release, + 'crcs' => [], + ]; + } + // Prioritize CRC from main files (video, RAR) over others + if (! empty($release->textstring)) { + $priority = $this->getCrcPriority($release->filename ?? ''); + $releasesCrc[$releaseId]['crcs'][$priority][] = $release->textstring; + } + } + + foreach ($releasesCrc as $releaseId => $data) { $this->reset(); - $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); $this->checked++; + + // Sort CRCs by priority and try each + ksort($data['crcs']); + foreach ($data['crcs'] as $crcs) { + foreach ($crcs as $crc) { + $release = clone $data['release']; + $release->textstring = $crc; + + $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); + + if ($this->matched) { + break 2; + } + } + } + $this->_echoRenamed($show); } @@ -337,6 +710,48 @@ class NameFixer } } + /** + * Get priority for CRC matching based on filename. + * + * Lower number = higher priority. + */ + protected function getCrcPriority(string $filename): int + { + $lower = strtolower($filename); + + // Skip sample/proof files - lowest priority + if (preg_match('/[\.\-_](sample|proof)[\.\-_]/i', $filename)) { + return 100; + } + + // Video files - highest priority + if (preg_match('/\.(mkv|avi|mp4|m4v|wmv|divx|ts|m2ts)$/i', $filename)) { + return 1; + } + + // Main RAR files - high priority + if (preg_match('/\.rar$/i', $filename) && ! preg_match('/\.part\d+\.rar$/i', $filename)) { + return 2; + } + + // First split RAR + if (preg_match('/\.part0*1\.rar$/i', $filename)) { + return 3; + } + + // Other RAR parts + if (preg_match('/\.(rar|r\d{2,3})$/i', $filename)) { + return 4; + } + + // NFO files + if (str_ends_with($lower, '.nfo')) { + return 5; + } + + return 50; + } + /** * Attempts to fix XXX release names using the File name. * @@ -397,7 +812,8 @@ class NameFixer /** * Attempts to fix release names using the SRR filename. * - * + * SRR (Scene Release Renamer) files contain the original scene release name, + * making them highly reliable for name fixing. * * @throws \Exception */ @@ -413,7 +829,10 @@ class NameFixer rf.releases_id AS fileid, rel.id AS releases_id FROM releases rel INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE predb_id = 0' + WHERE predb_id = 0 + AND (rf.name LIKE %s OR rf.name LIKE %s)', + escapeString('%.srr'), + escapeString('%.srs') ); $cats = 2; } else { @@ -425,12 +844,13 @@ class NameFixer INNER JOIN release_files rf ON rf.releases_id = rel.id WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) AND rel.predb_id = 0 - AND rf.name LIKE %s + AND (rf.name LIKE %s OR rf.name LIKE %s) AND rel.proc_srr = %d', self::IS_RENAMED_NONE, Category::OTHER_MISC, Category::OTHER_HASHED, escapeString('%.srr'), + escapeString('%.srs'), self::PROC_SRR_NONE ); } @@ -1135,6 +1555,10 @@ class NameFixer /** * Match a release filename to a PreDB filename or title. * + * Enhanced with: + * - Better RAR archive filename handling + * - Multiple file attempt with prioritization + * - Improved scene naming pattern recognition * * @throws \Exception */ @@ -1143,9 +1567,19 @@ class NameFixer $matching = 0; - foreach (explode('||', $release->filename) as $key => $fileName) { + // Split files and prioritize them for matching + $files = explode('||', $release->filename); + $prioritizedFiles = $this->prioritizeFilesForPreDbMatch($files); + + foreach ($prioritizedFiles as $fileName) { $this->_fileName = $fileName; $this->_cleanMatchFiles(); + + // Skip if the cleaned filename is too short or empty + if (empty($this->_fileName) || strlen($this->_fileName) < 8) { + continue; + } + $preMatch = $this->preMatch($this->_fileName); if ($preMatch[0] === true) { if (config('nntmux.elasticsearch_enabled') === true) { @@ -1166,7 +1600,37 @@ class NameFixer $this->_updateSingleColumn('predb_id', $result['id'], $release->releases_id); } $matching++; - break; + + return $matching; // Found a match, stop processing + } + } + } + } + } + + // Also try matching the file directly to PreDB title without preMatch validation + // This helps with files that have slight naming variations + if ($matching === 0 && strlen($this->_fileName) >= 12) { + $cleanedForTitle = $this->cleanFileForTitleMatch($this->_fileName); + if (strlen($cleanedForTitle) >= 12) { + if (config('nntmux.elasticsearch_enabled') === true) { + $results = $this->elasticsearch->searchPreDb($cleanedForTitle); + } else { + $results = Arr::get($this->manticore->searchIndexes('predb_rt', $cleanedForTitle, ['title']), 'data'); + } + + if (! empty($results)) { + foreach ($results as $result) { + if (! empty($result) && ! empty($result['title'])) { + // Verify the match quality + if ($this->isGoodPreDbMatch($cleanedForTitle, $result['title'])) { + if ($result['title'] !== $release->searchname) { + $this->updateRelease($release, $result['title'], 'file matched title: '.$result['source'], $echo, 'PreDB title match, ', $nameStatus, $show); + $matching++; + + return $matching; + } + } } } } @@ -1178,29 +1642,326 @@ class NameFixer } /** - * Cleans file names for PreDB Match. + * Prioritize files for PreDB matching. + * + * Files are ordered by their likelihood of containing a good release name: + * 1. Main RAR files (not split parts) + * 2. SRR files + * 3. First split RAR parts + * 4. Video files + * 5. Other files */ - protected function _cleanMatchFiles(): string + protected function prioritizeFilesForPreDbMatch(array $files): array { - // first strip all non-printing chars from filename - $this->_fileName = str_replace('/[[:^print:]]/', '', $this->_fileName); + $mainRar = []; + $srrFiles = []; + $firstParts = []; + $videoFiles = []; + $otherFiles = []; - if ($this->_fileName !== '' && ! str_starts_with($this->_fileName, '.')) { - $this->_fileName = match (true) { - str_contains($this->_fileName, '.') => Str::beforeLast('.', $this->_fileName), - preg_match('/\.part\d+$/', $this->_fileName) => Str::beforeLast('.', $this->_fileName), - preg_match('/\.vol\d+(\+\d+)?$/', $this->_fileName) => Str::beforeLast('.', $this->_fileName), - str_contains($this->_fileName, '\\') => Str::afterLast('\\', $this->_fileName), - preg_match('/^\d{2}-/', $this->_fileName) => preg_replace('/^\d{2}-/', '', $this->_fileName), - default => trim($this->_fileName), - }; + foreach ($files as $file) { + $lowerFile = strtolower($file); - return trim($this->_fileName); + // Skip sample/proof files + if (preg_match('/[\.\-_](sample|proof)[\.\-_]/i', $file)) { + continue; + } + + // SRR files - highest priority (contain original release name) + if (str_ends_with($lowerFile, '.srr') || str_ends_with($lowerFile, '.srs')) { + $srrFiles[] = $file; + } + // Main RAR (not split) + elseif (preg_match('/\.rar$/i', $file) && ! preg_match('/\.part\d+\.rar$/i', $file)) { + $mainRar[] = $file; + } + // First part of split RAR + elseif (preg_match('/\.part0*1\.rar$/i', $file)) { + $firstParts[] = $file; + } + // Video files + elseif (preg_match('/\.(mkv|avi|mp4|m4v|wmv|divx|ts|m2ts)$/i', $file)) { + $videoFiles[] = $file; + } else { + $otherFiles[] = $file; + } + } + + // Sort each group by length (longer names are often more descriptive) + usort($mainRar, fn($a, $b) => strlen($b) - strlen($a)); + usort($videoFiles, fn($a, $b) => strlen($b) - strlen($a)); + + return array_merge($srrFiles, $mainRar, $firstParts, $videoFiles, $otherFiles); + } + + /** + * Clean a filename for PreDB title matching. + * + * Removes file extensions and common suffixes while preserving the release name. + */ + protected function cleanFileForTitleMatch(string $filename): string + { + // Remove file extension + $clean = preg_replace('/\.(mkv|avi|mp4|m4v|wmv|mpg|mpeg|mov|ts|m2ts|vob|divx|flv|nfo|sfv|nzb|srr|srs|rar|r\d{2,4}|zip|7z|par2?|vol\d+[\+\-]\d+|\d{3})$/i', '', $filename); + + // Remove part/volume indicators + $clean = preg_replace('/[\.\-_]?(part|vol|cd|dvd|disc|disk)\d+$/i', '', $clean); + + // Remove sample/proof indicators + $clean = preg_replace('/[\.\-_](sample|proof|subs?)$/i', '', $clean); + + return trim($clean, " \t\n\r\0\x0B.-_"); + } + + /** + * Check if a PreDB match is of good quality. + * + * Compares the search term with the PreDB title to ensure they're similar enough. + */ + protected function isGoodPreDbMatch(string $searchTerm, string $preDbTitle): bool + { + // Normalize both strings for comparison + $searchNorm = strtolower(preg_replace('/[.\-_\s]+/', '', $searchTerm)); + $titleNorm = strtolower(preg_replace('/[.\-_\s]+/', '', $preDbTitle)); + + // Check if one is a substring of the other or if they're very similar + if (str_contains($titleNorm, $searchNorm) || str_contains($searchNorm, $titleNorm)) { + return true; + } + + // Calculate similarity + $similarity = 0; + similar_text($searchNorm, $titleNorm, $similarity); + + // Accept if similarity is high enough (80%+) + return $similarity >= 80; + } + + /** + * Extract a scene release name from a RAR archive filename or path. + * + * Scene releases typically follow the pattern: Release.Name-GROUP + * This method attempts to extract the release name from various RAR naming conventions. + * + * @param string $filename The RAR archive filename or path + * @return string|null The extracted release name or null if not found + */ + protected function extractReleaseNameFromRar(string $filename): ?string + { + // Extract filename from path + if (preg_match('/[\\\\\/]([^\\\\\/]+)$/', $filename, $match)) { + $filename = $match[1]; + } + + // Remove RAR extensions + $baseName = preg_replace('/\.(rar|r\d{2,4}|part\d+\.rar|\d{3})$/i', '', $filename); + + // Check if the base name looks like a scene release name + if (preg_match(self::PREDB_REGEX, $baseName)) { + // Clean up any remaining artifacts + $baseName = preg_replace('/[._-]?(sample|proof|subs?)$/i', '', $baseName); + $baseName = trim($baseName, '.-_'); + + if (strlen($baseName) >= 10) { + return $baseName; + } + } + + // Try to extract from common RAR naming patterns + // Pattern: releasename-group.rar or releasename.group.rar + if (preg_match('/^([a-z0-9][a-z0-9._-]+[a-z0-9])\-([a-z0-9]{2,15})$/i', $baseName, $match)) { + return $match[1] . '-' . $match[2]; + } + + return null; + } + + /** + * Try to find a release name from multiple RAR files. + * + * Analyzes multiple RAR files from a release to determine the most likely release name. + * + * @param array $rarFiles Array of RAR filenames + * @return string|null The most likely release name or null if not found + */ + protected function findReleaseNameFromRarFiles(array $rarFiles): ?string + { + $candidates = []; + + foreach ($rarFiles as $file) { + // Skip sample/proof files + if (preg_match('/[\._-](sample|proof)[\._-]/i', $file)) { + continue; + } + + $extracted = $this->extractReleaseNameFromRar($file); + if ($extracted !== null) { + // Count occurrences of each candidate + if (!isset($candidates[$extracted])) { + $candidates[$extracted] = 0; + } + $candidates[$extracted]++; + } + } + + if (empty($candidates)) { + return null; + } + + // Return the most common candidate (in case of ties, prefer longer names) + arsort($candidates); + $topCount = reset($candidates); + $topCandidates = array_filter($candidates, fn($count) => $count === $topCount); + + // If multiple candidates have the same count, prefer the longest one + $best = null; + foreach (array_keys($topCandidates) as $candidate) { + if ($best === null || strlen($candidate) > strlen($best)) { + $best = $candidate; + } + } + + return $best; + } + + /** + * Detect if a filename looks like it's from a scene release based on naming conventions. + * + * @param string $filename The filename to check + * @return bool True if the filename follows scene naming conventions + */ + protected function looksLikeSceneRelease(string $filename): bool + { + // Remove file extension for checking + $baseName = preg_replace('/\.[a-z0-9]{2,4}$/i', '', $filename); + + // Scene releases typically have: + // 1. Words separated by dots or underscores + // 2. A group name at the end after a hyphen + // 3. Common scene tags (720p, 1080p, x264, etc.) + + // Check for group suffix pattern: -GROUPNAME + if (!preg_match('/\-[A-Za-z0-9]{2,15}$/', $baseName)) { + return false; + } + + // Check for scene-style word separation (dots, underscores, hyphens) + if (!preg_match('/[._-]/', $baseName)) { + return false; + } + + // Check for common scene tags + $sceneTags = [ + '720p', '1080p', '2160p', '4k', + 'x264', 'x265', 'hevc', 'xvid', 'divx', + 'bluray', 'bdrip', 'dvdrip', 'hdtv', 'webrip', 'web-dl', 'webdl', + 'aac', 'ac3', 'dts', 'flac', 'mp3', + 'proper', 'repack', 'internal', 'retail', + 'pal', 'ntsc', 'multi', 'dual', + ]; + + $baseNameLower = strtolower($baseName); + foreach ($sceneTags as $tag) { + if (str_contains($baseNameLower, $tag)) { + return true; + } + } + + // Check for TV episode patterns + if (preg_match('/s\d{1,2}e\d{1,3}/i', $baseName)) { + return true; + } + + // Check for year pattern (common in movies) + if (preg_match('/[._-](19|20)\d{2}[._-]/i', $baseName)) { + return true; } return false; } + /** + * Cleans file names for PreDB Match. + * + * Enhanced to better handle: + * - RAR archives and split RAR files (.rar, .r00, .r01, etc.) + * - PAR2 recovery files + * - Various archive formats + * - Scene naming conventions + * - Path separators + */ + protected function _cleanMatchFiles(): string|false + { + // First strip all non-printing chars from filename + $this->_fileName = preg_replace('/[[:^print:]]/', '', $this->_fileName); + + if ($this->_fileName === '' || str_starts_with($this->_fileName, '.')) { + return false; + } + + // Extract filename from path (handle both Unix and Windows separators) + if (preg_match('/[\\\\\/]([^\\\\\/]+)$/', $this->_fileName, $pathMatch)) { + $this->_fileName = $pathMatch[1]; + } + + // Remove sample/proof/subs indicators before extension removal + $this->_fileName = preg_replace('/[\.\-_](sample|proof|subs?|thumbs?|cover|screens?)[\.\-_]?$/i', '', $this->_fileName); + + // Strip common archive extensions and split file patterns + $archivePatterns = [ + // RAR split files: .rar, .r00-.r999, .part01.rar, .part001.rar, etc. + '/\.part\d{1,4}\.rar$/i', + '/\.r\d{2,4}$/i', + '/\.rar$/i', + // ZIP split files + '/\.z\d{2}$/i', + '/\.zip$/i', + // 7z split files + '/\.7z\.\d{3}$/i', + '/\.7z$/i', + // PAR2 files + '/\.vol\d+[\+\-]\d+\.par2?$/i', + '/\.par2?$/i', + // Other archive formats + '/\.(tar|gz|bz2|xz|lz|lzma|cab|arj|ace|arc)$/i', + // Numbered split files (001, 002, etc.) + '/\.\d{3}$/i', + ]; + + foreach ($archivePatterns as $pattern) { + $this->_fileName = preg_replace($pattern, '', $this->_fileName); + } + + // Remove video file extensions + $this->_fileName = preg_replace('/\.(mkv|avi|mp4|m4v|wmv|mpg|mpeg|mov|ts|m2ts|vob|divx|flv|webm|ogv|3gp|asf|rm|rmvb|f4v)$/i', '', $this->_fileName); + + // Remove audio file extensions + $this->_fileName = preg_replace('/\.(mp3|flac|m4a|aac|ogg|wav|wma|ape|opus|mka|ac3|dts|eac3|truehd|mpc|shn|tak|tta|wv)$/i', '', $this->_fileName); + + // Remove image/other file extensions + $this->_fileName = preg_replace('/\.(nfo|sfv|nzb|srr|srs|jpg|jpeg|png|gif|bmp|tiff?|webp|pdf|txt|diz|md5|sha1|cue|log)$/i', '', $this->_fileName); + + // Remove ebook extensions + $this->_fileName = preg_replace('/\.(epub|mobi|azw3?|pdf|djvu|cbr|cbz|fb2|lit|prc|opf)$/i', '', $this->_fileName); + + // Remove game/app extensions + $this->_fileName = preg_replace('/\.(iso|bin|cue|mdf|mds|nrg|img|ccd|sub|exe|msi|dmg|pkg|apk|xap|appx|deb|rpm)$/i', '', $this->_fileName); + + // Remove subtitle extensions + $this->_fileName = preg_replace('/\.(srt|sub|idx|ass|ssa|vtt|sup)$/i', '', $this->_fileName); + + // Remove part/volume indicators that might remain + $this->_fileName = preg_replace('/[\.\-_]?(part|vol|cd|dvd|disc|disk)\d*$/i', '', $this->_fileName); + + // Remove leading track numbers (common in music releases) + $this->_fileName = preg_replace('/^\d{1,3}[\.\-_\s]+(?=[A-Za-z])/', '', $this->_fileName); + + // Trim whitespace and punctuation + $this->_fileName = trim($this->_fileName, " \t\n\r\0\x0B.-_"); + + return $this->_fileName !== '' ? $this->_fileName : false; + } + /** * Check the array using regex for a clean name. * @@ -1306,6 +2067,12 @@ class NameFixer /** * Look for a TV name. * + * Enhanced with support for: + * - Modern streaming services (AMZN, NF, DSNP, etc.) + * - 4K/UHD releases with HDR + * - Modern codecs (HEVC, x265) + * - Multi-episode releases (S01E01-E03) + * - Daily show formats * * @throws \Exception */ @@ -1315,31 +2082,65 @@ class NameFixer $result = []; if (! $this->done && $this->relid !== (int) $release->releases_id) { - if (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|(?textstring, $result)) { + // Streaming service 4K releases + if (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?e\d{1,3}(-?e\d{1,3})?)|(?textstring, $result)) { + $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.4K.streaming.source.hdr.vcodec', $echo, $type, $nameStatus, $show); + } + // Streaming service 1080p/720p releases + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?e\d{1,3}(-?e\d{1,3})?)|(?textstring, $result)) { + $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.res.streaming.source', $echo, $type, $nameStatus, $show); + } + // Standard TV with source and group + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.Text.source.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[\-\w.\',;& ]+((19|20)\d\d)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // TV with year + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.Text.year.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[\-\w.\',;& ]+(480|720|1080)[ip][._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // TV with resolution.source.vcodec + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.Text.resolution.source.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // TV with source.vcodec + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.source.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](480|720|1080)[ip][._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // TV with acodec.source.res.vcodec + elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.acodec.source.res.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.resolution.source.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'tvCheck: Title.SxxExx.source.resolution.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -]((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // TV with year and season/episode + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -]((s\d{1,2}[._ -]?[bde]\d{1,2}(-?e\d{1,3})?)|(?textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Title.year.###(season/episode).source.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w(19|20)\d\d[._ -]\d{2}[._ -]\d{2}[._ -](IndyCar|NBA|NCW([TY])S|NNS|NSCS?)([._ -](19|20)\d\d)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Daily shows with date format (YYYY.MM.DD or YYYY-MM-DD) + elseif (preg_match('/\w[\-\w.\',;& ]+(19|20)\d\d[._ -]\d{2}[._ -]\d{2}[._ -](720p|1080p|2160p|HDTV|WEB-?DL|WEB-?RIP)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'tvCheck: Daily show with date', $echo, $type, $nameStatus, $show); + } + // Sports releases + elseif (preg_match('/\w(19|20)\d\d[._ -]\d{2}[._ -]\d{2}[._ -](IndyCar|F1|Formula[._ -]?1|MotoGP|NBA|NCW([TY])S|NNS|NSCS?|NFL|NHL|MLB|UFC|WWE|Boxing)([._ -](19|20)\d\d)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'tvCheck: Sports', $echo, $type, $nameStatus, $show); } + // Complete season packs + elseif (preg_match('/\w[\-\w.\',;& ]+[._ -]S\d{1,2}[._ -](COMPLETE|FULL)[._ -](720p|1080p|2160p|HDTV|WEB-?DL|WEB-?RIP|BluRay)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'tvCheck: Complete season', $echo, $type, $nameStatus, $show); + } + // Anime releases with episode numbers + elseif (preg_match('/\w[\-\w.\',;& ]+[._ -](\d{2,4})[._ -](480p|720p|1080p|2160p)[._ -](HEVC|x265|x264|H\.?264)[._ -](10bit)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'tvCheck: Anime episode', $echo, $type, $nameStatus, $show); + } } } /** * Look for a movie name. * + * Enhanced with support for: + * - 4K/UHD releases with HDR, Dolby Vision + * - Modern codecs (HEVC, x265, AV1) + * - Streaming service releases (AMZN, NF, DSNP, etc.) + * - REMUX and high-quality releases * * @throws \Exception */ @@ -1349,29 +2150,60 @@ class NameFixer $result = []; if (! $this->done && $this->relid !== (int) $release->releases_id) { - if (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[\-\w.\',;& ]+(480|720|1080)[ip][._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + // 4K/UHD releases with HDR + if (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](2160p|4K|UHD)[._ -](HDR10\+?|DV|Dolby[._ -]?Vision|HLG)?[._ -]?(REMUX|BluRay|WEB-?DL|WEB-?RIP|UHD[._ -]?BluRay)[._ -](HEVC|x265|H\.?265|AV1)[._ -]?(Atmos|DTS[._ -]?(HD)?|TrueHD)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'movieCheck: 4K/UHD with HDR', $echo, $type, $nameStatus, $show); + } + // 4K BluRay REMUX + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](2160p|4K)[._ -](REMUX|Complete[._ -]?UHD)[._ -](HEVC|x265|H\.?265)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'movieCheck: 4K REMUX', $echo, $type, $nameStatus, $show); + } + // Streaming service releases (4K) + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](2160p|4K)[._ -](AMZN|ATVP|DSNP|HMAX|HULU|iT|NF|PMTP|PCOK|ROKU|STAN|VUDU)[._ -](WEB-?DL|WEB-?RIP)[._ -](HDR10\+?|DV)?[._ -]?(HEVC|x265|H\.?265)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'movieCheck: 4K Streaming service', $echo, $type, $nameStatus, $show); + } + // Streaming service releases (1080p/720p) + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](1080p|720p)[._ -](AMZN|ATVP|DSNP|HMAX|HULU|iT|NF|PMTP|PCOK|ROKU|STAN|VUDU)[._ -](WEB-?DL|WEB-?RIP)[._ -](x264|x265|H\.?264|H\.?265|HEVC)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'movieCheck: HD Streaming service', $echo, $type, $nameStatus, $show); + } + // Standard year.res.vcodec pattern + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[\-\w.\',;& ]+(480|720|1080|2160)[ip]?[._ -](DivX|[HX][._ -]?264|[HX][._ -]?265|HEVC|MPEG2|XviD(HD)?|WMV|AV1)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.Text.res.vcod.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[._ -](480|720|1080)[ip][\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Year.source.vcodec.resolution + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-?DL|WEB-?RIP|REMUX)[._ -](DivX|[HX][._ -]?264|[HX][._ -]?265|HEVC|MPEG2|XviD(HD)?|WMV|AV1)[._ -](480|720|1080|2160)[ip]?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.source.vcodec.res.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Year.source.vcodec.acodec + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-?DL|WEB-?RIP|REMUX)[._ -](DivX|[HX][._ -]?264|[HX][._ -]?265|HEVC|MPEG2|XviD(HD)?|WMV|AV1)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?(-?MA)?|Dolby( ?TrueHD)?|MP3|TrueHD|Atmos|EAC3|FLAC)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.source.vcodec.acodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.source.vcodec.resolution.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.source.resolution.acodec.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+(480|720|1080)[ip][._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Resolution.source.acodec.vcodec + elseif (preg_match('/\w[\-\w.\',;& ]+(480|720|1080|2160)[ip]?[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-?DL|WEB-?RIP|REMUX)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?(-?MA)?|Dolby( ?TrueHD)?|MP3|TrueHD|Atmos|EAC3|FLAC)[._ -](DivX|[HX][._ -]?264|[HX][._ -]?265|HEVC|MPEG2|XviD(HD)?|WMV|AV1)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.resolution.source.acodec.vcodec.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+(480|720|1080)[ip][._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[\-\w.\',;& ]+(BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -]((19|20)\d\d)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Resolution.acodec.source.year + elseif (preg_match('/\w[\-\w.\',;& ]+(480|720|1080|2160)[ip]?[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?(-?MA)?|Dolby( ?TrueHD)?|MP3|TrueHD|Atmos|EAC3|FLAC)[\-\w.\',;& ]+(BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-?DL|WEB-?RIP|REMUX)[._ -]((19|20)\d\d)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.resolution.acodec.eptitle.source.year.group', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+(Brazilian|Chinese|Croatian|Danish|Deutsch|Dutch|Estonian|English|Finnish|Flemish|Francais|French|German|Greek|Hebrew|Icelandic|Italian|Japenese|Japan|Japanese|Korean|Latin|Nordic|Norwegian|Polish|Portuguese|Russian|Serbian|Slovenian|Swedish|Spanisch|Spanish|Thai|Turkish)[._ -]((19|20)\d\d)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?|Dolby( ?TrueHD)?|MP3|TrueHD)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + } + // Multi-language releases + elseif (preg_match('/\w[\-\w.\',;& ]+(Brazilian|Chinese|Croatian|Danish|Deutsch|Dutch|Estonian|English|Finnish|Flemish|Francais|French|German|Greek|Hebrew|Icelandic|Italian|Japenese|Japan|Japanese|Korean|Latin|Nordic|Norwegian|Polish|Portuguese|Russian|Serbian|Slovenian|Swedish|Spanisch|Spanish|Thai|Turkish|MULTi)[._ -]((19|20)\d\d)[._ -](AAC( LC)?|AC-?3|DD5([._ -]1)?|(A_)?DTS-?(HD)?(-?MA)?|Dolby( ?TrueHD)?|MP3|TrueHD|Atmos|EAC3|FLAC)[._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-?DL|WEB-?RIP|REMUX)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { $this->updateRelease($release, $result['0'], 'movieCheck: Title.language.year.acodec.src', $echo, $type, $nameStatus, $show); } + // Generic movie with year and resolution (fallback) + elseif (preg_match('/\w[\-\w.\',;& ]+((19|20)\d\d)[\-\w.\',;& ]+(480|720|1080|2160)[ip]?[\-\w.\',;& ]+(BluRay|BDRip|DVDRip|HDTV|WEB-?DL|WEB-?RIP)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'movieCheck: Title.year.res.source.group', $echo, $type, $nameStatus, $show); + } } } /** * Look for a game name. * + * Enhanced with support for: + * - Modern platforms (PS5, Xbox Series X/S, Nintendo Switch) + * - Modern scene groups + * - DLC, updates, and patches * * @throws \Exception */ @@ -1381,23 +2213,54 @@ class NameFixer $result = []; if (! $this->done && $this->relid !== (int) $release->releases_id) { - if (preg_match('/\w[\-\w.\',;& ]+(ASIA|DLC|EUR|GOTY|JPN|KOR|MULTI\d{1}|NTSCU?|PAL|RF|Region[._ -]?Free|USA|XBLA)[._ -](DLC[._ -]Complete|FRENCH|GERMAN|MULTI\d{1}|PROPER|PSN|READ[._ -]?NFO|UMD)?[._ -]?(GC|NDS|NGC|PS3|PSP|WII|XBOX(360)?)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'gameCheck: Videogames 1', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+(GC|NDS|NGC|PS3|WII|XBOX(360)?)[._ -](DUPLEX|iNSOMNi|OneUp|STRANGE|SWAG|SKY)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'gameCheck: Videogames 2', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\w.\',;-].+-OUTLAWS/i', $release->textstring, $result)) { + // Modern console releases (PS5, Xbox Series, Switch) + if (preg_match('/\w[\-\w.\',;& ]+(NSW|PS[345P]|PSV|XBSX|XSX|XBOX[._ -]?SERIES[._ -]?[XS]|XBOX[._ -]?ONE|XBOX360?|WiiU?|Switch)[._ -](INTERNAL|PROPER|READNFO|READ[._ -]?NFO|MULTI\d{1,2})?[._ -]?[\-\w.\',;& ]+\-[A-Za-z0-9]+$/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: Modern console release', $echo, $type, $nameStatus, $show); + } + // Region-based game releases + elseif (preg_match('/\w[\-\w.\',;& ]+(ASIA|DLC|EUR|GOTY|JPN|KOR|MULTI\d{1}|NTSCU?|PAL|RF|Region[._ -]?Free|USA|XBLA)[._ -](DLC[._ -]Complete|FRENCH|GERMAN|MULTI\d{1}|PROPER|PSN|READ[._ -]?NFO|UMD)?[._ -]?(GC|NDS|NGC|PS[345P]|PSP|PSV|Switch|NSW|Wii(U)?|XBOX(360|ONE|SERIES)?|XBSX)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: Videogames with region', $echo, $type, $nameStatus, $show); + } + // Scene group releases + elseif (preg_match('/\w[\-\w.\',;& ]+(GC|NDS|NGC|PS[345P]|Switch|NSW|Wii(U)?|XBOX(360|ONE|SERIES)?|XBSX)[._ -](CODEX|DUPLEX|PLAZA|SKIDROW|RELOADED|CPY|EMPRESS|RAZOR1911|HOODLUM|DARKSiDERS|FLT|TiNYiSO|ANOMALY|iNSOMNi|OneUp|STRANGE|SWAG|SKY|SUXXORS)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: Console with scene group', $echo, $type, $nameStatus, $show); + } + // PC Games with scene groups + elseif (preg_match('/\w[\-\w.\',;& ]+(PC|WIN(32|64)?|MAC(OSX?)?|LINUX)[._ -]?(CODEX|SKIDROW|RELOADED|CPY|EMPRESS|RAZOR1911|HOODLUM|DARKSiDERS|FLT|GOG|PROPHET|TiNYiSO|PLAZA|P2P|SiMPLEX|rG)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: PC game with scene group', $echo, $type, $nameStatus, $show); + } + // DLC and Update releases + elseif (preg_match('/\w[\-\w.\',;& ]+(DLC|Update|Patch|Hotfix)[._ -](v?\d+[\.\d]*)?[._ -]?(CODEX|SKIDROW|RELOADED|PLAZA|EMPRESS|FLT|GOG|P2P|TiNYiSO)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: DLC/Update', $echo, $type, $nameStatus, $show); + } + // OUTLAWS group releases + elseif (preg_match('/\w[\w.\',;-].+-OUTLAWS/i', $release->textstring, $result)) { $result = str_replace('OUTLAWS', 'PC GAME OUTLAWS', $result['0']); $this->updateRelease($release, $result['0'], 'gameCheck: PC Games -OUTLAWS', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\w.\',;-].+\-ALiAS/i', $release->textstring, $result)) { + } + // ALiAS group releases + elseif (preg_match('/\w[\w.\',;-].+\-ALiAS/i', $release->textstring, $result)) { $newResult = str_replace('-ALiAS', ' PC GAME ALiAS', $result['0']); $this->updateRelease($release, $newResult, 'gameCheck: PC Games -ALiAS', $echo, $type, $nameStatus, $show); } + // GOG releases + elseif (preg_match('/\w[\-\w.\',;& ]+[._ -]GOG[._ -]?(Classic|Galaxy)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: GOG release', $echo, $type, $nameStatus, $show); + } + // REPACK releases + elseif (preg_match('/\w[\-\w.\',;& ]+[._ -](REPACK|RIP)[._ -](FitGirl|DODI|xatab|R\.G\.|Mechanics)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'gameCheck: Game REPACK', $echo, $type, $nameStatus, $show); + } } } /** - * Look for a app name. + * Look for an app name. * + * Enhanced with support for: + * - Modern software patterns + * - macOS/Linux releases + * - Scene group naming conventions * * @throws \Exception */ @@ -1407,10 +2270,33 @@ class NameFixer $result = []; if (! $this->done && $this->relid !== (int) $release->releases_id) { - if (preg_match('/\w[\-\w.\',;& ]+(\d{1,10}|Linux|UNIX)[._ -](RPM)?[._ -]?(X64)?[._ -]?(Incl)[._ -](Keygen)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'appCheck: Apps 1', $echo, $type, $nameStatus, $show); - } elseif (preg_match('/\w[\-\w.\',;& ]+\d{1,8}[._ -](winall-freeware)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { - $this->updateRelease($release, $result['0'], 'appCheck: Apps 2', $echo, $type, $nameStatus, $show); + // Software with keygen/patch + if (preg_match('/\w[\-\w.\',;& ]+(\d{1,10}|v\d+[\.\d]*|Linux|UNIX|MacOS)[._ -](RPM|DEB)?[._ -]?(X64|X86|ARM64)?[._ -]?(Incl|With)?[._ -]?(Keygen|Patch|Crack|Serial|License)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Apps with keygen/patch', $echo, $type, $nameStatus, $show); + } + // Windows freeware + elseif (preg_match('/\w[\-\w.\',;& ]+\d{1,8}[._ -](winall|win32|win64|x64|x86)[._ -]?(freeware|portable|repack)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Windows apps', $echo, $type, $nameStatus, $show); + } + // macOS applications + elseif (preg_match('/\w[\-\w.\',;& ]+(MacOS|Mac[._ -]?OS[._ -]?X|OSX)[._ -][\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: macOS apps', $echo, $type, $nameStatus, $show); + } + // Linux applications + elseif (preg_match('/\w[\-\w.\',;& ]+(Linux|Ubuntu|Debian|CentOS|RHEL|Fedora)[._ -](x64|x86|arm64)?[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Linux apps', $echo, $type, $nameStatus, $show); + } + // Adobe software + elseif (preg_match('/\w[\-\w.\',;& ]*(Adobe|Photoshop|Illustrator|Premiere|After[._ -]?Effects|InDesign|Lightroom)[._ -][\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Adobe apps', $echo, $type, $nameStatus, $show); + } + // Microsoft software + elseif (preg_match('/\w[\-\w.\',;& ]*(Microsoft|Office|Windows|Visual[._ -]?Studio)[._ -]\d{2,4}[._ -][\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Microsoft apps', $echo, $type, $nameStatus, $show); + } + // Generic software with version + elseif (preg_match('/\w[\-\w.\',;& ]+[._ -]v?\d+[\.\d]+[._ -](Multilingual|MULTi|Portable|Repack|Cracked)[\-\w.\',;& ]+\w/i', $release->textstring, $result)) { + $this->updateRelease($release, $result['0'], 'appCheck: Software with version', $echo, $type, $nameStatus, $show); } } } @@ -1708,6 +2594,11 @@ class NameFixer /** * Just for filenames. * + * Enhanced file checking with support for: + * - Modern video formats (4K, HDR, HEVC, etc.) + * - Scene naming conventions + * - RAR archive contents + * - Various media types * * @throws \Exception */ @@ -1717,74 +2608,149 @@ class NameFixer $result = []; if (! $this->done && $this->relid !== (int) $release->releases_id) { + // Clean the filename for better matching + $cleanedFilename = $this->cleanFilenameForMatching($release->textstring); + switch (true) { + // Scene TV release with group suffix + case preg_match('/^(.+?(x264|x265|HEVC|XviD|H\.?264|H\.?265)\-[A-Za-z0-9]+)\\\\/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Scene release with group', $echo, $type, $nameStatus, $show); + break; + // TVP group format case preg_match('/^(.+?(x264|XviD)\-TVP)\\\\/i', $release->textstring, $result): $this->updateRelease($release, $result['1'], 'fileCheck: TVP', $echo, $type, $nameStatus, $show); break; + // Generic TV - SxxExx format with quality/source info + case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?S\d{1,3}[.-_ ]?E\d{1,3}(?:[.-_ ]?E\d{1,3})?[.-_ ].+?(?:720p|1080p|2160p|4K|HDTV|WEB-?DL|WEB-?RIP|BluRay|AMZN|HMAX|NF|DSNP).+?)\.(.+)$/iu', $release->textstring, $result): + $this->updateRelease($release, $result['4'], 'fileCheck: TV SxxExx with quality', $echo, $type, $nameStatus, $show); + break; + // Generic TV - any SxxExx format case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?S\d{1,3}[.-_ ]?[ED]\d{1,3}.+)\.(.+)$/iu', $release->textstring, $result): $this->updateRelease($release, $result['4'], 'fileCheck: Generic TV', $echo, $type, $nameStatus, $show); break; - case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?([\.\-_ ]\d{4}[\.\-_ ].+?(BDRip|bluray|DVDRip|XVID)).+)\.(.+)$/iu', $release->textstring, $result): + // 4K/UHD Movies - modern formats + case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?[\.\-_ ](19|20)\d\d[\.\-_ ].+?(2160p|4K|UHD).+?(HDR10?\+?|DV|Dolby[\.\-_ ]?Vision)?.+?(HEVC|x265|H\.?265).+?)\.(.+)$/iu', $release->textstring, $result): + $this->updateRelease($release, $result['4'], 'fileCheck: 4K/UHD Movie', $echo, $type, $nameStatus, $show); + break; + // HD Movies with modern codecs + case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?[\.\-_ ](19|20)\d\d[\.\-_ ].+?(720p|1080p).+?(BluRay|WEB-?DL|WEB-?RIP|BDRip|REMUX).+?(x264|x265|HEVC|H\.?264|H\.?265|AVC).+?)\.(.+)$/iu', $release->textstring, $result): + $this->updateRelease($release, $result['4'], 'fileCheck: HD Movie modern codec', $echo, $type, $nameStatus, $show); + break; + // Standard HD Movies + case preg_match('/^(\\\\|\/)?(.+(\\\\|\/))*(.+?([\.\-_ ]\d{4}[\.\-_ ].+?(BDRip|bluray|DVDRip|XVID|WEB-?DL|HDTV)).+)\.(.+)$/iu', $release->textstring, $result): $this->updateRelease($release, $result['4'], 'fileCheck: Generic movie 1', $echo, $type, $nameStatus, $show); break; - case preg_match('/^([a-z0-9\.\-_]+(19|20)\d\d[a-z0-9\.\-_]+[\.\-_ ](720p|1080p|BDRip|bluray|DVDRip|x264|XviD)[a-z0-9\.\-_]+)\.[a-z]{2,}$/i', $release->textstring, $result): + case preg_match('/^([a-z0-9\.\-_]+(19|20)\d\d[a-z0-9\.\-_]+[\.\-_ ](720p|1080p|2160p|4K|BDRip|bluray|DVDRip|x264|x265|XviD|HEVC)[a-z0-9\.\-_]+)\.[a-z]{2,}$/i', $release->textstring, $result): $this->updateRelease($release, $result['1'], 'fileCheck: Generic movie 2', $echo, $type, $nameStatus, $show); break; + // Streaming service releases + case preg_match('/^([A-Za-z0-9\.\-_]+[\.\-_ ](AMZN|ATVP|DSNP|HMAX|HULU|iT|NF|PMTP|PCOK|ROKU|STAN|TVNZ|VUDU)[\.\-_ ].+?(WEB-?DL|WEB-?RIP).+?)\.(.+)$/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Streaming service release', $echo, $type, $nameStatus, $show); + break; + // Music releases case preg_match('/(.+?([\.\-_ ](CD|FM)|[\.\-_ ]\dCD|CDR|FLAC|SAT|WEB).+?(19|20)\d\d.+?)\\\\.+/i', $release->textstring, $result): $this->updateRelease($release, $result['1'], 'fileCheck: Generic music', $echo, $type, $nameStatus, $show); break; case preg_match('/^(.+?(19|20)\d\d\-([a-z0-9]{3}|[a-z]{2,}|C4))\\\\/i', $release->textstring, $result): $this->updateRelease($release, $result['1'], 'fileCheck: music groups', $echo, $type, $nameStatus, $show); break; + // FLAC music releases + case preg_match('/^(.+?[\.\-_ ](FLAC|MP3|AAC|OGG)[\.\-_ ].+?[\.\-_ ]\d{4}[\.\-_ ].+?\-[A-Za-z0-9]+)[\\\\\/.]/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Music with codec', $echo, $type, $nameStatus, $show); + break; + // Movie with year in parentheses - AVI format case preg_match('/.+\\\\(.+\((19|20)\d\d\)\.avi)$/i', $release->textstring, $result): $newName = str_replace('.avi', ' DVDRip XVID NoGroup', $result['1']); $this->updateRelease($release, $newName, 'fileCheck: Movie (year) avi', $echo, $type, $nameStatus, $show); break; + // Movie with year in parentheses - ISO format case preg_match('/.+\\\\(.+\((19|20)\d\d\)\.iso)$/i', $release->textstring, $result): $newName = str_replace('.iso', ' DVD NoGroup', $result['1']); $this->updateRelease($release, $newName, 'fileCheck: Movie (year) iso', $echo, $type, $nameStatus, $show); break; + // Movie with year in parentheses - MKV format + case preg_match('/.+\\\\(.+\((19|20)\d\d\)\.(mkv|mp4|m4v))$/i', $release->textstring, $result): + $newName = preg_replace('/\.(mkv|mp4|m4v)$/i', ' BDRip x264 NoGroup', $result['1']); + $this->updateRelease($release, $newName, 'fileCheck: Movie (year) mkv/mp4', $echo, $type, $nameStatus, $show); + break; + // RAR file contents - look for release name in RAR path + case preg_match('/^([A-Za-z0-9][\w.\-]+(?:[\.\-_ ][\w.\-]+)+)[\\\\\\/](?:CD\d|Disc\d|DVD\d|Subs?)?[\\\\\\/]?.+\.(rar|r\d{2,3}|zip|7z)$/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: RAR archive path', $echo, $type, $nameStatus, $show); + break; + // Scene release in RAR - common pattern: Release.Name-GROUP\release.name-group.rar + case preg_match('/^([A-Za-z0-9][\w.\-]+\-[A-Za-z0-9]+)[\\\\\\/].+\.(rar|r\d{2,3})$/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Scene RAR release', $echo, $type, $nameStatus, $show); + break; + // XXX Imagesets case preg_match('/^(.+?IMAGESET.+?)\\\\.+/i', $release->textstring, $result): $this->updateRelease($release, $result['1'], 'fileCheck: XXX Imagesets', $echo, $type, $nameStatus, $show); break; + // VIDEOOT releases case preg_match('/^VIDEOOT-[A-Z0-9]+\\\\([\w!.,& ()\[\]\'\`-]{8,}?\b.?)([\-_](proof|sample|thumbs?))*(\.part\d*(\.rar)?|\.rar|\.7z)?(\d{1,3}\.rev|\.vol.+?|\.mp4)/', $release->textstring, $result): $this->updateRelease($release, $result['1'].' XXX DVDRIP XviD-VIDEOOT', 'fileCheck: XXX XviD VIDEOOT', $echo, $type, $nameStatus, $show); break; + // XXX SDPORN case preg_match('/^.+?SDPORN/i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: XXX SDPORN', $echo, $type, $nameStatus, $show); break; + // R&C releases case preg_match('/\w[\-\w.\',;& ]+1080i[._ -]DD5[._ -]1[._ -]MPEG2-R&C(?=\.ts)$/i', $release->textstring, $result): $result = str_replace('MPEG2', 'MPEG2.HDTV', $result['0']); $this->updateRelease($release, $result, 'fileCheck: R&C', $echo, $type, $nameStatus, $show); break; + // NhaNc3 releases case preg_match('/\w[\-\w.\',;& ]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](480|720|1080)[ip][._ -](BD(-?(25|50|RIP))?|Blu-?Ray ?(3D)?|BRRIP|CAM(RIP)?|DBrip|DTV|DVD\-?(5|9|(R(IP)?|scr(eener)?))?|[HPS]D?(RIP|TV(RIP)?)?|NTSC|PAL|R5|Ripped |S?VCD|scr(eener)?|SAT(RIP)?|TS|VHS(RIP)?|VOD|WEB-DL)[._ -]nSD[._ -](DivX|[HX][._ -]?264|MPEG2|XviD(HD)?|WMV)[._ -]NhaNC3[\-\w.\',;& ]+\w/i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: NhaNc3', $echo, $type, $nameStatus, $show); break; + // TVP releases (alternate pattern) case preg_match('/\wtvp-[\w.\-\',;]+((s\d{1,2}[._ -]?[bde]\d{1,2})|\d{1,2}x\d{2}|ep[._ -]?\d{2})[._ -](720p|1080p|xvid)(?=\.(avi|mkv))$/i', $release->textstring, $result): $result = str_replace('720p', '720p.HDTV.X264', $result['0']); $result = str_replace('1080p', '1080p.Bluray.X264', $result['0']); $result = str_replace('xvid', 'XVID.DVDrip', $result['0']); $this->updateRelease($release, $result, 'fileCheck: tvp', $echo, $type, $nameStatus, $show); break; + // LOL releases case preg_match('/\w[\-\w.\',;& ]+\d{3,4}\.hdtv-lol\.(avi|mp4|mkv|ts|nfo|nzb)/i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: Title.211.hdtv-lol.extension', $echo, $type, $nameStatus, $show); break; - case preg_match('/\w[\-\w.\',;& ]+-S\d{1,2}[EX]\d{1,2}-XVID-DL.avi/i', $release->textstring, $result): + // DL releases + case preg_match('/\w[\-\w.\',;& ]+-S\d{1,2}[EX]\d{1,2}-XVID-DL\.avi/i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: Title-SxxExx-XVID-DL.avi', $echo, $type, $nameStatus, $show); break; + // Title - SxxExx - Episode title format case preg_match('/\S.*[\w.\-\',;]+\s\-\ss\d{2}[ex]\d{2}\s\-\s[\w.\-\',;].+\./i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: Title - SxxExx - Eptitle', $echo, $type, $nameStatus, $show); break; + // Nintendo DS case preg_match('/\w.+?\)\.nds$/i', $release->textstring, $result): $this->updateRelease($release, $result['0'], 'fileCheck: ).nds Nintendo DS', $echo, $type, $nameStatus, $show); break; + // Nintendo 3DS case preg_match('/3DS_\d{4}.+\d{4} - (.+?)\.3ds/i', $release->textstring, $result): $this->updateRelease($release, '3DS '.$result['1'], 'fileCheck: .3ds Nintendo 3DS', $echo, $type, $nameStatus, $show); break; - case preg_match('/\w.+?\.(epub|mobi|azw|opf|fb2|prc|djvu|cb[rz])/i', $release->textstring, $result): + // Nintendo Switch + case preg_match('/^(.+?)\[[\w]+\]\.(?:nsp|xci|nsz)$/i', $release->textstring, $result): + $this->updateRelease($release, trim($result['1']).' Switch', 'fileCheck: Nintendo Switch', $echo, $type, $nameStatus, $show); + break; + // PlayStation/Xbox game releases + case preg_match('/^(.+?[\.\-_ ](PS[345P]|PSV|XBOX360|XBOXONE|NSW)[\.\-_ ].+?\-[A-Za-z0-9]+)[\\\\\/.]/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Console game release', $echo, $type, $nameStatus, $show); + break; + // EBooks + case preg_match('/\w.+?\.(epub|mobi|azw3?|opf|fb2|prc|djvu|cb[rz])/i', $release->textstring, $result): $result = str_replace('.'.$result['1'], ' ('.$result['1'].')', $result['0']); $this->updateRelease($release, $result, 'fileCheck: EBook', $echo, $type, $nameStatus, $show); break; + // Audiobooks + case preg_match('/^(.+?[\.\-_ ]Audiobook[\.\-_ ].+?)[\\\\\/.]/i', $release->textstring, $result): + $this->updateRelease($release, $result['1'], 'fileCheck: Audiobook', $echo, $type, $nameStatus, $show); + break; + // Scene release from cleaned filename + case preg_match('/^([A-Za-z0-9][\w.\-]+\-[A-Za-z0-9]{2,15})$/i', $cleanedFilename, $result) && preg_match(self::PREDB_REGEX, $cleanedFilename): + $this->updateRelease($release, $result['1'], 'fileCheck: Cleaned scene name', $echo, $type, $nameStatus, $show); + break; + // Folder name fallback case preg_match('/\w+[\-\w.\',;& ]+$/i', $release->textstring, $result) && preg_match(self::PREDB_REGEX, $release->textstring): $this->updateRelease($release, $result['0'], 'fileCheck: Folder name', $echo, $type, $nameStatus, $show); break; @@ -1798,6 +2764,30 @@ class NameFixer return false; } + /** + * Clean a filename for better pattern matching. + * + * Removes common file extensions, path components, and normalizes the string. + */ + protected function cleanFilenameForMatching(string $filename): string + { + // Extract filename from path + if (preg_match('/[\\\\\/]([^\\\\\/]+)$/', $filename, $match)) { + $filename = $match[1]; + } + + // Remove common extensions + $filename = preg_replace('/\.(mkv|avi|mp4|m4v|wmv|mpg|mpeg|mov|ts|m2ts|vob|divx|flv|webm|nfo|sfv|nzb|srr|srs|rar|r\d{2,4}|zip|7z|par2?|vol\d+[\+\-]\d+|001|\d{3})$/i', '', $filename); + + // Remove sample/proof indicators + $filename = preg_replace('/[\.\-_](sample|proof|subs?)[\.\-_]?/i', '', $filename); + + // Remove part/volume indicators + $filename = preg_replace('/[\.\-_]?(part|vol|cd|dvd|disc|disk)\d+$/i', '', $filename); + + return trim($filename, " \t\n\r\0\x0B.-_"); + } + /** * Look for a name based on mediainfo xml Unique_ID. * @@ -1930,7 +2920,8 @@ class NameFixer /** * Look for a name based on .srr release files extension. * - * + * SRR files (Scene Release Renamer) contain the original scene release name + * and are highly reliable sources for name extraction. * * @throws \Exception */ @@ -1946,27 +2937,49 @@ class NameFixer FROM releases rel INNER JOIN release_files rf ON (rf.releases_id = {$release->releases_id}) WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rf.name LIKE %s", + AND (rf.name LIKE %s OR rf.name LIKE %s)", self::IS_RENAMED_NONE, Category::OTHER_MISC, Category::OTHER_HASHED, - escapeString('%.srr') + escapeString('%.srr'), + escapeString('%.srs') ) ); foreach ($result as $res) { - if (preg_match('/^(.*)\.srr$/i', $res->textstring, $hit)) { - $this->updateRelease( - $release, - $hit['1'], - 'fileCheck: SRR extension', - $echo, - $type, - $nameStatus, - $show - ); + // Extract release name from SRR filename + $extractedName = null; - return true; + // Try .srr extension first + if (preg_match('/^(.+)\.srr$/i', $res->textstring, $hit)) { + $extractedName = $hit[1]; + } + // Try .srs extension (Scene Release Signature) + elseif (preg_match('/^(.+)\.srs$/i', $res->textstring, $hit)) { + $extractedName = $hit[1]; + } + + // Validate and clean the extracted name + if ($extractedName !== null) { + // Remove any path components + if (preg_match('/[\\\\\/]([^\\\\\/]+)$/', $extractedName, $pathMatch)) { + $extractedName = $pathMatch[1]; + } + + // Ensure it looks like a valid scene release name + if (preg_match(self::PREDB_REGEX, $extractedName)) { + $this->updateRelease( + $release, + $extractedName, + 'fileCheck: SRR extension', + $echo, + $type, + $nameStatus, + $show + ); + + return true; + } } } } @@ -2155,31 +3168,65 @@ class NameFixer return false; } + /** + * Clean and normalize filenames for PreDB matching. + * + * Handles modern video formats, scene naming conventions, and various + * file format indicators to produce cleaner release names. + */ private function cleanFileNames(): array|string|null { - if (preg_match('/(\.[a-zA-Z]{2})?(\.4k|\.fullhd|\.hd|\.int|\.\d+)?$/i', $this->_fileName, $hit)) { - if (! empty($hit[1]) && preg_match('/\.[a-zA-Z]{2}/i', $hit[1])) { - $this->_fileName = preg_replace('/\.[a-zA-Z]{2}\./i', '.', $this->_fileName); + // Handle language/country suffixes and quality indicators at end of filename + if (preg_match('/(\.[a-zA-Z]{2})?(\.4k|\.fullhd|\.hd|\.int|\.internal|\.\d+)?$/i', $this->_fileName, $hit)) { + // Remove 2-letter country/language codes that appear before quality + if (! empty($hit[1]) && preg_match('/\.[a-zA-Z]{2}\./i', $hit[1])) { + // Only remove if it's not a valid scene group suffix + if (! preg_match('/\-(en|de|fr|es|it|nl|pt|ru|pl|jp|kr|cn)$/i', $this->_fileName)) { + $this->_fileName = preg_replace('/\.[a-zA-Z]{2}\./i', '.', $this->_fileName); + } } + + // Normalize quality indicators if (! empty($hit[2])) { - if (preg_match('/\.4k$/', $hit[2])) { - $this->_fileName = preg_replace('/\.4k$/', '.2160p', $this->_fileName); - } - if (preg_match('/\.fullhd$/i', $hit[2])) { - $this->_fileName = preg_replace('/\.fullhd$/i', '.1080p', $this->_fileName); - } - if (preg_match('/\.hd$/i', $hit[2])) { - $this->_fileName = preg_replace('/\.hd$/i', '.720p', $this->_fileName); - } - if (preg_match('/\.int$/i', $hit[2])) { - $this->_fileName = preg_replace('/\.int$/i', '.INTERNAL', $this->_fileName); - } - if (preg_match('/\.\d+/', $hit[2])) { - $this->_fileName = preg_replace('/\.\d+$/', '', $this->_fileName); + $qualityMap = [ + '/\.4k$/i' => '.2160p', + '/\.fullhd$/i' => '.1080p', + '/\.hd$/i' => '.720p', + '/\.int$/i' => '.INTERNAL', + '/\.internal$/i' => '.INTERNAL', + '/\.\d+$/' => '', // Remove trailing numbers (often file indices) + ]; + + foreach ($qualityMap as $pattern => $replacement) { + $this->_fileName = preg_replace($pattern, $replacement, $this->_fileName); } } + + // Remove leading group prefixes (common in some releases) if (preg_match('/^[a-zA-Z]{0,7}\./', $this->_fileName)) { - $this->_fileName = preg_replace('/^[a-zA-Z]{0,7}\./', '', $this->_fileName); + // Only remove if it looks like a prefix (short, before the main name) + // Don't remove if the whole name is short + if (strlen($this->_fileName) > 15) { + $this->_fileName = preg_replace('/^[a-zA-Z]{0,5}\.(?=[A-Za-z0-9]+[\.\-_])/', '', $this->_fileName); + } + } + } + + // Normalize UHD/HDR indicators for modern releases + $modernNormalizations = [ + '/\.UHD\./i' => '.2160p.UHD.', + '/\.HDR\./i' => '.HDR.', + '/\.DV\./i' => '.DV.', + '/\.Atmos\./i' => '.Atmos.', + '/\.REMUX\./i' => '.REMUX.', + '/\.COMPLETE\./i' => '.COMPLETE.', + '/\.MULTi\./i' => '.MULTi.', + ]; + + foreach ($modernNormalizations as $pattern => $replacement) { + if (preg_match($pattern, $this->_fileName) && ! preg_match('/' . preg_quote($replacement, '/') . '/', $this->_fileName)) { + // Only add if not already present in normalized form + $this->_fileName = preg_replace($pattern, $replacement, $this->_fileName); } } diff --git a/Blacklight/Nfo.php b/Blacklight/Nfo.php index 459de2b2a..b90d3182e 100755 --- a/Blacklight/Nfo.php +++ b/Blacklight/Nfo.php @@ -10,66 +10,123 @@ use Blacklight\processing\PostProcess; use Blacklight\utility\Utility; use dariusiii\rarinfo\Par2Info; use dariusiii\rarinfo\SfvInfo; -use getID3; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Log; use Throwable; /** - * Class Nfo. + * Class Nfo - Handles NFO file processing, validation, and metadata extraction. + * + * NFO files are text files commonly used in the warez scene to provide information + * about releases. This class handles detection, validation, parsing and storage of NFO content. */ class Nfo { /** * Regex to detect common non-NFO file headers/signatures. + * Matches XML, NZB, RIFF (media), PAR/RAR archives, and other binary formats. */ - protected string $_nonNfoHeaderRegex = '/\A(\s*<\?xml|=newz\[NZB\]=|RIFF|\s*[RP]AR|.{0,10}(JFIF|matroska|ftyp|ID3))|;\s*Generated\s*by.*SF\w/i'; + protected string $_nonNfoHeaderRegex = '/\A(\s*<\?xml|=newz\[NZB\]=|RIFF|\s*[RP]AR|.{0,10}(JFIF|matroska|ftyp|ID3)|PK\x03\x04|\x1f\x8b\x08|MZ|%PDF|GIF8[79]a|\x89PNG)|;\s*Generated\s*by.*SF\w/i'; /** * Regex to identify text encoding from the 'file' command output. */ - protected string $_textFileRegex = '/(ASCII|ISO-8859|UTF-(8|16|32).*?)\s*text/'; + protected string $_textFileRegex = '/(ASCII|ISO-8859|UTF-(8|16|32).*?|Non-ISO extended-ASCII)\s*text/i'; /** * Regex to identify common binary file types from the 'file' command output. */ - protected string $_binaryFileRegex = '/^(JPE?G|Parity|PNG|RAR|XML|(7-)?[Zz]ip)/'; + protected string $_binaryFileRegex = '/^(JPE?G|Parity|PNG|RAR|XML|(7-)?[Zz]ip|PDF|GIF|executable|archive|compressed|data|binary)/i'; /** * Regex to detect binary characters within the content. + * Excludes common control characters that may appear in NFOs (tab, newline, carriage return). */ - protected string $_binaryCharsRegex = '/[\x00-\x08\x12-\x1F\x0B\x0E\x0F]/'; + protected string $_binaryCharsRegex = '/[\x00-\x08\x0B\x0C\x0E-\x1F]/'; /** - * @var int + * Common NFO keywords that help identify legitimate NFO files. */ - private $nzbs; + protected array $_nfoKeywords = [ + // Release information + 'release', 'group', 'date', 'size', 'format', 'source', 'genre', 'codec', + 'bitrate', 'resolution', 'language', 'subtitle', 'ripped', 'cracked', + 'keygen', 'serial', 'patch', 'trainer', 'install', 'notes', 'greets', + 'nfo', 'ascii', 'artwork', 'presents', 'proudly', 'brings', 'another', + // Scene terminology + 'scene', 'rls', 'nuked', 'proper', 'repack', 'internal', 'retail', + 'webdl', 'webrip', 'bluray', 'bdrip', 'dvdrip', 'hdtv', 'pdtv', + // Media info + 'video', 'audio', 'duration', 'runtime', 'aspect', 'fps', 'channels', + 'sample', 'encoder', 'x264', 'x265', 'hevc', 'avc', 'xvid', 'divx', + 'aac', 'ac3', 'dts', 'truehd', 'atmos', 'flac', 'mp3', + // Content info + 'movie', 'film', 'episode', 'season', 'series', 'title', 'year', + 'director', 'cast', 'actors', 'plot', 'synopsis', 'imdb', 'rating', + // Software + 'crack', 'readme', 'setup', 'installer', 'license', 'registration', + 'protection', 'requirements', 'platform', 'operating', 'system', + // Contact/Group info + 'contact', 'irc', 'www', 'http', 'ftp', 'email', 'apply', 'join', + ]; /** - * @var int + * Scene group patterns for improved detection. */ - protected $maxSize; + protected array $_sceneGroupPatterns = [ + '/(?:^|\n)\s*[-=*]{3,}.*?([A-Z0-9]{2,15})\s*[-=*]{3,}/i', + '/(?:presents?|brought\s+(?:to\s+)?(?:you\s+)?by|from)\s*[:\-]?\s*([A-Z][A-Z0-9]{1,14})/i', + '/(?:greets?\s+(?:go(?:es)?\s+)?(?:out\s+)?to|respect\s+to)\s*[:\-]?\s*([\w,\s&]+)/i', + '/(?:^|\n)\s*([A-Z][A-Z0-9]{1,14})\s+(?:nfo|info|release)\s*(?:$|\n)/i', + '/(?:released\s+by|rls\s+by)\s*[:\-]?\s*([A-Z][A-Z0-9]{1,14})/i', + ]; /** - * @var int + * Maximum NFO file size in bytes (64KB). */ - private $maxRetries; + protected const MAX_NFO_SIZE = 65535; /** - * @var int + * Minimum NFO file size in bytes. */ - protected $minSize; + protected const MIN_NFO_SIZE = 12; /** - * @var string + * Cache TTL for settings in seconds. */ - private $tmpPath; + protected const SETTINGS_CACHE_TTL = 300; /** - * @var bool + * @var int Number of NFOs to process per batch. */ - protected $echo; + private int $nzbs; + + /** + * @var int Maximum release size to process NFO (in GB). + */ + protected int $maxSize; + + /** + * @var int Maximum retry attempts for failed NFO fetches. + */ + private int $maxRetries; + + /** + * @var int Minimum release size to process NFO (in MB). + */ + protected int $minSize; + + /** + * @var string Temporary path for processing files. + */ + private string $tmpPath; + + /** + * @var bool Whether to echo output to CLI. + */ + protected bool $echo; public const NFO_FAILED = -9; // We failed to get a NFO after admin set max retries. @@ -84,83 +141,132 @@ class Nfo /** * Default constructor. * + * Initializes NFO processing settings from database/config with caching. + * * @throws \Exception */ public function __construct() { $this->echo = config('nntmux.echocli'); - $this->nzbs = Settings::settingValue('maxnfoprocessed') !== '' ? (int) Settings::settingValue('maxnfoprocessed') : 100; - $this->maxRetries = (int) Settings::settingValue('maxnforetries') >= 0 ? -((int) Settings::settingValue('maxnforetries') + 1) : self::NFO_UNPROC; - $this->maxRetries = $this->maxRetries < -8 ? -8 : $this->maxRetries; - $this->maxSize = (int) Settings::settingValue('maxsizetoprocessnfo'); - $this->minSize = (int) Settings::settingValue('minsizetoprocessnfo'); $this->colorCli = new ColorCLI; - $this->tmpPath = config('nntmux.tmp_unrar_path'); - if (! preg_match('/[\/\\\\]$/', $this->tmpPath)) { - $this->tmpPath .= '/'; - } + // Cache settings to reduce database queries + $this->nzbs = Cache::remember('nfo_maxnfoprocessed', self::SETTINGS_CACHE_TTL, function () { + $value = Settings::settingValue('maxnfoprocessed'); + + return $value !== '' ? (int) $value : 100; + }); + + $maxRetries = Cache::remember('nfo_maxnforetries', self::SETTINGS_CACHE_TTL, function () { + return (int) Settings::settingValue('maxnforetries'); + }); + $this->maxRetries = $maxRetries >= 0 ? -($maxRetries + 1) : self::NFO_UNPROC; + $this->maxRetries = max($this->maxRetries, -8); + + $this->maxSize = Cache::remember('nfo_maxsizetoprocessnfo', self::SETTINGS_CACHE_TTL, function () { + return (int) Settings::settingValue('maxsizetoprocessnfo'); + }); + + $this->minSize = Cache::remember('nfo_minsizetoprocessnfo', self::SETTINGS_CACHE_TTL, function () { + return (int) Settings::settingValue('minsizetoprocessnfo'); + }); + + $this->tmpPath = rtrim(config('nntmux.tmp_unrar_path'), '/\\').'/'; } /** - * Look for a TV Show ID in a string. + * Look for a TV Show ID or Movie ID in a string. + * + * Supports: TVMaze, IMDB, TVDB (legacy & modern), TMDB, AniDB * * @param string $str The string with a Show ID. - * @return array|false Return array with show ID and site source or false on failure. + * @return array{showid: string, site: string}|false Return array with show ID and site source or false on failure. */ - public function parseShowId(string $str) + public function parseShowId(string $str): array|false { - $return = false; - + // TVMaze if (preg_match('/tvmaze\.com\/shows\/(\d{1,6})/i', $str, $hits)) { - $return = - [ - 'showid' => trim($hits[1]), - 'site' => 'tvmaze', - ]; + return ['showid' => trim($hits[1]), 'site' => 'tvmaze']; } - if (preg_match('/imdb\.com\/title\/(tt\d{1,8})/i', $str, $hits)) { - $return = - [ - 'showid' => trim($hits[1]), - 'site' => 'imdb', - ]; + // IMDB (movies and TV shows) + if (preg_match('/imdb\.com\/title\/(tt\d{7,8})/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'imdb']; } + // TVDB - Legacy URL format if (preg_match('/thetvdb\.com\/\?tab=series&id=(\d{1,8})/i', $str, $hits)) { - $return = - [ - 'showid' => trim($hits[1]), - 'site' => 'thetvdb', - ]; + return ['showid' => trim($hits[1]), 'site' => 'thetvdb']; } - return $return; + // TVDB - Modern URL format (series/slug or series/id) + if (preg_match('/thetvdb\.com\/series\/(\d{1,8}|[\w-]+)/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'thetvdb']; + } + + // TMDB - Movie + if (preg_match('/themoviedb\.org\/movie\/(\d{1,8})/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'tmdb_movie']; + } + + // TMDB - TV Show + if (preg_match('/themoviedb\.org\/tv\/(\d{1,8})/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'tmdb_tv']; + } + + // AniDB + if (preg_match('/anidb\.net\/(?:perl-bin\/animedb\.pl\?show=anime&aid=|anime\/)(\d{1,6})/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'anidb']; + } + + // Trakt.tv + if (preg_match('/trakt\.tv\/(?:shows|movies)\/([\w-]+)/i', $str, $hits)) { + return ['showid' => trim($hits[1]), 'site' => 'trakt']; + } + + return false; } /** * Confirm this is an NFO file. * + * Uses multiple validation strategies: + * 1. Size validation (too large/small = not NFO) + * 2. Binary header detection (known file signatures) + * 3. File type detection via 'file' command + * 4. PAR2/SFV structure detection + * 5. Binary character content analysis + * 6. NFO keyword/content heuristics + * * @param bool|string $possibleNFO The nfo content. * @param string $guid The guid of the release. * @return bool True if it's likely an NFO, False otherwise. */ public function isNFO(bool|string &$possibleNFO, string $guid): bool { - if ($possibleNFO === false) { + if ($possibleNFO === false || $possibleNFO === '') { return false; } $size = \strlen($possibleNFO); - // Basic size and signature checks - if ($size >= 65535 || $size < 12 || preg_match($this->_nonNfoHeaderRegex, $possibleNFO)) { + // Basic size and signature checks using constants + if ($size >= self::MAX_NFO_SIZE || $size < self::MIN_NFO_SIZE) { + return false; + } + + // Quick check for known non-NFO file signatures + if (preg_match($this->_nonNfoHeaderRegex, $possibleNFO)) { + return false; + } + + // Additional binary format checks + if ($this->detectBinaryFormat($possibleNFO)) { return false; } $tmpPath = $this->tmpPath.$guid.'.nfo'; - $isNfo = false; // Default assumption + $isNfo = false; try { // File/GetId3 work with files, so save to disk. @@ -170,41 +276,46 @@ class Nfo $result = Utility::fileInfo($tmpPath); if (! empty($result)) { if (preg_match($this->_textFileRegex, $result)) { - $isNfo = true; // It's text, likely NFO + $isNfo = true; } elseif (preg_match($this->_binaryFileRegex, $result) || preg_match($this->_binaryCharsRegex, $possibleNFO)) { - $isNfo = false; // Detected binary format or characters + $isNfo = false; + } + + // If fileInfo gave a result, apply additional heuristics before returning + if ($isNfo) { + // Additional content validation for text files + $isNfo = $this->validateNfoContent($possibleNFO); } - // If fileInfo gave a result, trust it and return return $isNfo; } // Fallback checks if 'file' command is unavailable or inconclusive - // Check if it's a par2. + // Check if it's a PAR2 file $par2info = new Par2Info; $par2info->setData($possibleNFO); if (! $par2info->error) { - // It's a PAR2 file return false; } - // Check if it's an SFV. + // Check if it's an SFV file $sfv = new SfvInfo; $sfv->setData($possibleNFO); if (! $sfv->error) { - // It's an SFV file return false; } - // If it wasn't identified as a known non-NFO binary type by fileInfo, - // and isn't PAR2 or SFV, assume it might be NFO (especially if fileInfo failed). - // Further checks (like binary char check) could be added here if needed. - $isNfo = ! preg_match($this->_binaryCharsRegex, $possibleNFO); + // Check for binary characters + if (preg_match($this->_binaryCharsRegex, $possibleNFO)) { + return false; + } + + // Final content-based validation + $isNfo = $this->validateNfoContent($possibleNFO); } catch (Throwable $e) { - // Log errors during file operations Log::error("Error processing potential NFO for GUID {$guid}: ".$e->getMessage()); - $isNfo = false; // Treat errors as non-NFO + $isNfo = false; } finally { // Ensure temporary file is always deleted if (File::exists($tmpPath)) { @@ -219,6 +330,126 @@ class Nfo return $isNfo; } + /** + * Detect binary file formats by magic bytes. + * + * @param string $data The file content to check. + * @return bool True if binary format detected. + */ + protected function detectBinaryFormat(string $data): bool + { + if (strlen($data) < 4) { + return false; + } + + // Magic bytes for common binary formats + $magicBytes = [ + "\x50\x4B\x03\x04" => 'ZIP', // ZIP/DOCX/XLSX etc. + "\x50\x4B\x05\x06" => 'ZIP_EMPTY', // Empty ZIP + "\x52\x61\x72\x21" => 'RAR', // RAR + "\x37\x7A\xBC\xAF" => '7Z', // 7-Zip + "\x1F\x8B\x08" => 'GZIP', // GZip + "\x42\x5A\x68" => 'BZIP2', // BZip2 + "\xFD\x37\x7A\x58" => 'XZ', // XZ + "\x89\x50\x4E\x47" => 'PNG', // PNG + "\xFF\xD8\xFF" => 'JPEG', // JPEG + "\x47\x49\x46\x38" => 'GIF', // GIF + "\x25\x50\x44\x46" => 'PDF', // PDF + "\x49\x44\x33" => 'MP3_ID3', // MP3 with ID3 + "\xFF\xFB" => 'MP3', // MP3 + "\x4F\x67\x67\x53" => 'OGG', // OGG + "\x66\x4C\x61\x43" => 'FLAC', // FLAC + "\x52\x49\x46\x46" => 'RIFF', // WAV/AVI + "\x00\x00\x01\xBA" => 'MPEG', // MPEG video + "\x00\x00\x01\xB3" => 'MPEG', // MPEG video + "\x1A\x45\xDF\xA3" => 'MKV', // Matroska/WebM + "\x4D\x5A" => 'EXE', // Windows EXE + "\x7F\x45\x4C\x46" => 'ELF', // Linux executable + "\xCA\xFE\xBA\xBE" => 'JAVA', // Java class + "\xD0\xCF\x11\xE0" => 'OLE', // MS Office old format + ]; + + foreach ($magicBytes as $magic => $type) { + if (str_starts_with($data, $magic)) { + return true; + } + } + + // Check for UTF-16 BOM (could be text, but unlikely NFO) + if (str_starts_with($data, "\xFF\xFE") || str_starts_with($data, "\xFE\xFF")) { + // UTF-16 - could be valid, let other checks handle it + return false; + } + + return false; + } + + /** + * Validate NFO content using heuristics. + * + * @param string $content The content to validate. + * @return bool True if content appears to be a valid NFO. + */ + protected function validateNfoContent(string $content): bool + { + $length = strlen($content); + + // Too short to be meaningful + if ($length < 50) { + return false; + } + + // Count printable ASCII characters + $printableCount = preg_match_all('/[\x20-\x7E]/', $content); + $printableRatio = $printableCount / $length; + + // NFOs should be mostly printable characters + if ($printableRatio < 0.7) { + return false; + } + + // Check for minimum text content (words, not just symbols) + $wordCount = preg_match_all('/[A-Za-z]{2,}/', $content); + if ($wordCount < 5) { + return false; + } + + // Check for NFO-like content patterns + $nfoIndicators = 0; + + // Look for common NFO keywords + foreach ($this->_nfoKeywords as $keyword) { + if (stripos($content, $keyword) !== false) { + $nfoIndicators++; + if ($nfoIndicators >= 3) { + return true; // High confidence if multiple keywords found + } + } + } + + // Check for scene-style formatting + if (preg_match('/[-=*]{5,}/', $content)) { + $nfoIndicators++; + } + + // Check for URL presence (common in NFOs) + if (preg_match('/https?:\/\/|www\./i', $content)) { + $nfoIndicators++; + } + + // Check for media IDs + if (preg_match('/imdb\.com|thetvdb\.com|themoviedb\.org|anidb\.net/i', $content)) { + $nfoIndicators += 2; + } + + // Check for field:value patterns + if (preg_match_all('/^[A-Za-z\s]{2,20}\s*[:\.]\s*.+$/m', $content, $matches)) { + $nfoIndicators += min(count($matches[0]) / 3, 2); + } + + return $nfoIndicators >= 2; + } + /** * Add an NFO from alternate sources. ex.: PreDB, rar, zip, etc... * @@ -479,4 +710,751 @@ class Nfo ($minSize > 0 ? ('AND r.size > '.($minSize * 1048576)) : '') ); } + + /** + * Extract URLs from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Array of found URLs. + */ + public function extractUrls(string $nfoContent): array + { + $urls = []; + + // Match HTTP/HTTPS URLs + if (preg_match_all('/https?:\/\/[^\s<>"\']+/i', $nfoContent, $matches)) { + $urls = array_merge($urls, $matches[0]); + } + + // Match www URLs without protocol + if (preg_match_all('/(?"\']+/i', $nfoContent, $matches)) { + foreach ($matches[0] as $url) { + $urls[] = 'http://'.$url; + } + } + + return array_unique(array_filter($urls)); + } + + /** + * Extract release group name from NFO content. + * + * Uses multiple detection strategies including: + * - Common presentation phrases + * - Scene-style headers with ASCII borders + * - Greetings sections + * - Footer signatures + * + * @param string $nfoContent The NFO content to parse. + * @return string|null The group name if found, null otherwise. + */ + public function extractGroupName(string $nfoContent): ?string + { + // False positives to filter out + $falsePositives = [ + 'THE', 'AND', 'FOR', 'NFO', 'INFO', 'DVD', 'BLU', 'RAY', 'WEB', 'HDTV', + 'RELEASE', 'GROUP', 'DATE', 'SIZE', 'CODEC', 'VIDEO', 'AUDIO', 'FORMAT', + 'NOTES', 'INSTALL', 'GREETS', 'PRESENTS', 'TEAM', 'SCENE', 'FILE', 'FILES', + ]; + + // Use configured scene group patterns + foreach ($this->_sceneGroupPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $groupName = trim($matches[1]); + if (! in_array(strtoupper($groupName), $falsePositives, true) && strlen($groupName) >= 2 && strlen($groupName) <= 20) { + return $groupName; + } + } + } + + // Additional patterns for group name detection + $additionalPatterns = [ + // "GROUP presents" or "GROUP brings you" + '/\b([A-Z][A-Z0-9]{1,14})\s+(?:presents?|brings?\s+you)/i', + // Common footer format: "--- GROUP ---" + '/[-=]{2,}\s*([A-Z][A-Z0-9]{1,14})\s*[-=]{2,}$/mi', + // Contact section: "irc.server.net #GROUP" + '/irc\.[a-z0-9.-]+\s+#([A-Z][A-Z0-9]{1,14})/i', + // Website: "www.GROUP.com/org/net" + '/www\.([a-z][a-z0-9]{1,14})\.(?:com|org|net|info)/i', + // ASCII art name extraction (common pattern at start) + '/^\s*[^a-zA-Z0-9]*([A-Z][A-Z0-9]{2,14})[^a-zA-Z0-9]*\s*$/mi', + ]; + + foreach ($additionalPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $groupName = trim($matches[1]); + if (! in_array(strtoupper($groupName), $falsePositives, true) && strlen($groupName) >= 2 && strlen($groupName) <= 20) { + return strtoupper($groupName); + } + } + } + + return null; + } + + /** + * Extract release date from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return string|null ISO date string if found, null otherwise. + */ + public function extractReleaseDate(string $nfoContent): ?string + { + $patterns = [ + // DD/MM/YYYY or MM/DD/YYYY + '/(?:date|released?|rls)\s*[:\-]?\s*(\d{1,2})[\/\-.](\d{1,2})[\/\-.](\d{2,4})/i', + // YYYY-MM-DD + '/(?:date|released?|rls)\s*[:\-]?\s*(\d{4})[\/\-.](\d{1,2})[\/\-.](\d{1,2})/i', + // Month DD, YYYY + '/(?:date|released?|rls)\s*[:\-]?\s*(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4})/i', + ]; + + foreach ($patterns as $index => $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + try { + if ($index === 0) { + // Try both DD/MM and MM/DD formats + $year = strlen($matches[3]) === 2 ? '20'.$matches[3] : $matches[3]; + // Assume DD/MM/YYYY format (more common internationally) + return sprintf('%04d-%02d-%02d', (int) $year, (int) $matches[2], (int) $matches[1]); + } elseif ($index === 1) { + // YYYY-MM-DD + return sprintf('%04d-%02d-%02d', (int) $matches[1], (int) $matches[2], (int) $matches[3]); + } else { + // Month name format + $months = ['jan' => 1, 'feb' => 2, 'mar' => 3, 'apr' => 4, 'may' => 5, 'jun' => 6, 'jul' => 7, 'aug' => 8, 'sep' => 9, 'oct' => 10, 'nov' => 11, 'dec' => 12]; + $month = $months[strtolower(substr($matches[1], 0, 3))] ?? 1; + + return sprintf('%04d-%02d-%02d', (int) $matches[3], $month, (int) $matches[2]); + } + } catch (Throwable) { + continue; + } + } + } + + return null; + } + + /** + * Extract video/audio codec information from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array{video?: string, audio?: string, resolution?: string} Array with codec info. + */ + public function extractCodecInfo(string $nfoContent): array + { + $result = []; + + // Video codecs + $videoPatterns = [ + '/(?:video|codec)\s*[:\-]?\s*(x264|x265|hevc|h\.?264|h\.?265|xvid|divx|av1|vp9|mpeg[24]?)/i', + '/\b(x264|x265|HEVC|H\.?264|H\.?265|XviD|DivX|AV1|VP9)\b/i', + ]; + foreach ($videoPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $result['video'] = strtoupper(str_replace('.', '', $matches[1])); + break; + } + } + + // Audio codecs + $audioPatterns = [ + '/(?:audio|sound)\s*[:\-]?\s*(aac|ac3|dts(?:-(?:hd|ma|x))?|truehd|atmos|flac|mp3|eac3|dd[+p]?|dolby)/i', + '/\b(AAC|AC3|DTS(?:-(?:HD|MA|X))?|TrueHD|Atmos|FLAC|EAC3|DD[+P]?)\b/i', + ]; + foreach ($audioPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $result['audio'] = strtoupper($matches[1]); + break; + } + } + + // Resolution + $resolutionPatterns = [ + '/(?:resolution|quality)\s*[:\-]?\s*(\d{3,4}[xX×]\d{3,4}|\d{3,4}p|[48]K|UHD|FHD|HD)/i', + '/\b(2160p|1080p|720p|480p|4K|UHD|FHD|HD)\b/i', + '/\b(\d{3,4})\s*[xX×]\s*(\d{3,4})\b/', + ]; + foreach ($resolutionPatterns as $index => $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + if ($index === 2) { + $result['resolution'] = $matches[1].'x'.$matches[2]; + } else { + $result['resolution'] = strtoupper($matches[1]); + } + break; + } + } + + return $result; + } + + /** + * Extract file size information from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return int|null File size in bytes if found, null otherwise. + */ + public function extractFileSize(string $nfoContent): ?int + { + $patterns = [ + '/(?:size|file\s*size)\s*[:\-]?\s*(\d+(?:[.,]\d+)?)\s*(bytes?|[KMGTP]B|[KMGTP]iB)/i', + '/\b(\d+(?:[.,]\d+)?)\s*(GB|GiB|MB|MiB|TB|TiB)\b/i', + ]; + + $multipliers = [ + 'B' => 1, 'BYTE' => 1, 'BYTES' => 1, + 'KB' => 1024, 'KIB' => 1024, + 'MB' => 1024 * 1024, 'MIB' => 1024 * 1024, + 'GB' => 1024 * 1024 * 1024, 'GIB' => 1024 * 1024 * 1024, + 'TB' => 1024 * 1024 * 1024 * 1024, 'TIB' => 1024 * 1024 * 1024 * 1024, + 'PB' => 1024 * 1024 * 1024 * 1024 * 1024, 'PIB' => 1024 * 1024 * 1024 * 1024 * 1024, + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $value = (float) str_replace(',', '.', $matches[1]); + $unit = strtoupper($matches[2]); + + if (isset($multipliers[$unit])) { + return (int) ($value * $multipliers[$unit]); + } + } + } + + return null; + } + + /** + * Extract all media IDs (IMDB, TVDB, TMDB, etc.) from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Array of media IDs with their sources. + */ + public function extractAllMediaIds(string $nfoContent): array + { + $ids = []; + + // IMDB + if (preg_match_all('/imdb\.com\/title\/(tt\d{7,8})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'imdb']; + } + } + + // TVDB + if (preg_match_all('/thetvdb\.com\/(?:\?tab=series&id=|series\/)(\d{1,8})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'thetvdb']; + } + } + + // TMDB Movie + if (preg_match_all('/themoviedb\.org\/movie\/(\d{1,8})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'tmdb_movie']; + } + } + + // TMDB TV + if (preg_match_all('/themoviedb\.org\/tv\/(\d{1,8})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'tmdb_tv']; + } + } + + // TVMaze + if (preg_match_all('/tvmaze\.com\/shows\/(\d{1,6})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'tvmaze']; + } + } + + // AniDB + if (preg_match_all('/anidb\.net\/(?:perl-bin\/animedb\.pl\?show=anime&aid=|anime\/)(\d{1,6})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'anidb']; + } + } + + // MyAnimeList (MAL) + if (preg_match_all('/myanimelist\.net\/anime\/(\d{1,6})/i', $nfoContent, $matches)) { + foreach ($matches[1] as $id) { + $ids[] = ['id' => $id, 'source' => 'mal']; + } + } + + return $ids; + } + + /** + * Parse and extract comprehensive metadata from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Associative array with extracted metadata. + */ + public function parseNfoMetadata(string $nfoContent): array + { + return [ + 'urls' => $this->extractUrls($nfoContent), + 'group' => $this->extractGroupName($nfoContent), + 'release_date' => $this->extractReleaseDate($nfoContent), + 'codec_info' => $this->extractCodecInfo($nfoContent), + 'file_size' => $this->extractFileSize($nfoContent), + 'media_ids' => $this->extractAllMediaIds($nfoContent), + 'show_id' => $this->parseShowId($nfoContent), + 'language' => $this->extractLanguage($nfoContent), + 'runtime' => $this->extractRuntime($nfoContent), + 'genre' => $this->extractGenre($nfoContent), + 'software_info' => $this->extractSoftwareInfo($nfoContent), + 'release_title' => $this->extractReleaseTitle($nfoContent), + ]; + } + + /** + * Extract language information from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Array of detected languages. + */ + public function extractLanguage(string $nfoContent): array + { + $languages = []; + + // Common language patterns in NFOs + $patterns = [ + '/(?:language|audio|spoken?|dialogue)\s*[:\-]?\s*([A-Za-z]+(?:\s*[,\/&]\s*[A-Za-z]+)*)/i', + '/(?:subs?|subtitles?)\s*[:\-]?\s*([A-Za-z]+(?:\s*[,\/&]\s*[A-Za-z]+)*)/i', + ]; + + // Known language names + $knownLanguages = [ + 'english', 'german', 'french', 'spanish', 'italian', 'dutch', 'portuguese', + 'russian', 'japanese', 'korean', 'chinese', 'mandarin', 'cantonese', + 'swedish', 'norwegian', 'danish', 'finnish', 'polish', 'czech', 'hungarian', + 'turkish', 'arabic', 'hindi', 'thai', 'vietnamese', 'indonesian', 'malay', + 'multi', 'dual', 'english/german', 'eng', 'ger', 'fre', 'spa', 'ita', + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $langs = preg_split('/[\s,\/&]+/', strtolower($matches[1])); + foreach ($langs as $lang) { + $lang = trim($lang); + if (in_array($lang, $knownLanguages, true) && ! in_array($lang, $languages, true)) { + $languages[] = ucfirst($lang); + } + } + } + } + + return $languages; + } + + /** + * Extract runtime/duration from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return int|null Runtime in minutes, or null if not found. + */ + public function extractRuntime(string $nfoContent): ?int + { + $patterns = [ + // "Runtime: 1h 30m" or "Duration: 90min" + '/(?:runtime|duration|length|playtime)\s*[:\-]?\s*(?:(\d{1,2})\s*h(?:ours?)?\s*)?(\d{1,3})\s*m(?:in(?:utes?)?)?/i', + // "Runtime: 01:30:00" or "1:30:00" + '/(?:runtime|duration|length|playtime)\s*[:\-]?\s*(\d{1,2}):(\d{2})(?::(\d{2}))?/i', + // "90 minutes" standalone + '/\b(\d{2,3})\s*(?:min(?:utes?)?|mins)\b/i', + ]; + + foreach ($patterns as $index => $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + if ($index === 0) { + $hours = ! empty($matches[1]) ? (int) $matches[1] : 0; + $minutes = (int) $matches[2]; + return ($hours * 60) + $minutes; + } elseif ($index === 1) { + $hours = (int) $matches[1]; + $minutes = (int) $matches[2]; + return ($hours * 60) + $minutes; + } else { + return (int) $matches[1]; + } + } + } + + return null; + } + + /** + * Extract genre information from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Array of detected genres. + */ + public function extractGenre(string $nfoContent): array + { + $genres = []; + + if (preg_match('/(?:genre|category|type)\s*[:\-]?\s*([^\n\r]+)/i', $nfoContent, $matches)) { + $genreString = $matches[1]; + // Split on common separators + $parts = preg_split('/[\s,\/&|]+/', $genreString); + + // Known valid genres + $validGenres = [ + 'action', 'adventure', 'animation', 'biography', 'comedy', 'crime', + 'documentary', 'drama', 'family', 'fantasy', 'history', 'horror', + 'music', 'musical', 'mystery', 'romance', 'sci-fi', 'scifi', 'sport', + 'thriller', 'war', 'western', 'adult', 'xxx', 'erotic', 'anime', + 'rpg', 'fps', 'strategy', 'simulation', 'puzzle', 'racing', 'sports', + 'rock', 'pop', 'electronic', 'hip-hop', 'rap', 'classical', 'jazz', + ]; + + foreach ($parts as $part) { + $part = strtolower(trim($part)); + if (in_array($part, $validGenres, true) && ! in_array(ucfirst($part), $genres, true)) { + $genres[] = ucfirst($part); + } + } + } + + return $genres; + } + + /** + * Extract software-specific information from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return array Software info including platform, version, protection, etc. + */ + public function extractSoftwareInfo(string $nfoContent): array + { + $info = []; + + // Platform/OS detection + $platformPatterns = [ + '/(?:platform|os|system|requires?)\s*[:\-]?\s*(windows?|linux|mac(?:os)?|unix|android|ios)/i', + ]; + foreach ($platformPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $info['platform'] = ucfirst(strtolower($matches[1])); + break; + } + } + + // Version detection + if (preg_match('/(?:version|ver|v)\s*[:\-]?\s*(\d+(?:\.\d+)*(?:\s*(?:build|b)\s*\d+)?)/i', $nfoContent, $matches)) { + $info['version'] = trim($matches[1]); + } + + // Protection type + $protectionPatterns = [ + '/(?:protection|drm|copy[ -]?protection)\s*[:\-]?\s*([^\n\r]+)/i', + ]; + foreach ($protectionPatterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $protection = trim($matches[1]); + if (strlen($protection) > 2 && strlen($protection) < 50) { + $info['protection'] = $protection; + } + break; + } + } + + // Crack/Keygen/Serial info + if (preg_match('/\b(cracked|keygen|serial|patch|loader|activator)\b/i', $nfoContent)) { + $info['has_crack'] = true; + } + + return $info; + } + + /** + * Extract release title from NFO content. + * + * @param string $nfoContent The NFO content to parse. + * @return string|null The release title if found. + */ + public function extractReleaseTitle(string $nfoContent): ?string + { + $patterns = [ + // "Title: Movie Name" or "Release: Title.Goes.Here" + '/(?:title|release|name)\s*[:\-]?\s*([^\n\r]{5,100})/i', + // Scene-style title in header + '/(?:^|\n)\s*(?:[\-=*~]{3,}\s*)?([A-Za-z0-9][\w.\-\s]{10,80}?)(?:\s*[\-=*~]{3,})?\s*(?:\n|$)/m', + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $nfoContent, $matches)) { + $title = trim($matches[1]); + // Filter out common non-title content + if (! preg_match('/^(?:date|size|codec|format|video|audio|language|runtime|genre)\s*:/i', $title) + && strlen($title) >= 5 && strlen($title) <= 100) { + return $title; + } + } + } + + return null; + } + + /** + * Clean and normalize NFO content. + * + * @param string $nfoContent Raw NFO content. + * @return string Cleaned NFO content. + */ + public function cleanNfoContent(string $nfoContent): string + { + // Convert to UTF-8 if needed (CP437 is common for NFOs) + $content = Utility::cp437toUTF($nfoContent); + + // Normalize line endings + $content = str_replace(["\r\n", "\r"], "\n", $content); + + // Remove excessive whitespace while preserving NFO art + $lines = explode("\n", $content); + $cleanedLines = []; + $emptyLineCount = 0; + + foreach ($lines as $line) { + if (trim($line) === '') { + $emptyLineCount++; + // Allow max 2 consecutive empty lines + if ($emptyLineCount <= 2) { + $cleanedLines[] = ''; + } + } else { + $emptyLineCount = 0; + $cleanedLines[] = rtrim($line); + } + } + + return implode("\n", $cleanedLines); + } + + /** + * Calculate an NFO quality score based on content analysis. + * + * Scoring factors: + * - Content length (too short or too long penalized) + * - Keyword presence (scene terminology, media info) + * - Media ID presence (IMDB, TVDB, etc.) + * - URL presence + * - Codec information + * - ASCII art detection (scene NFOs often have artistic headers) + * - Structural elements (proper formatting) + * + * @param string $nfoContent The NFO content to analyze. + * @return int Quality score from 0-100. + */ + public function calculateNfoQuality(string $nfoContent): int + { + $score = 50; // Base score + + $length = strlen($nfoContent); + + // Length bonus/penalty + if ($length < 100) { + $score -= 20; + } elseif ($length > 500 && $length < 20000) { + $score += 15; + } elseif ($length >= 20000) { + $score += 5; // Longer NFOs might have too much filler + } + + // Keyword matching + $keywordMatches = 0; + foreach ($this->_nfoKeywords as $keyword) { + if (stripos($nfoContent, $keyword) !== false) { + $keywordMatches++; + } + } + $score += min($keywordMatches * 2, 20); + + // Media ID presence bonus + $mediaIds = $this->extractAllMediaIds($nfoContent); + if (! empty($mediaIds)) { + $score += min(count($mediaIds) * 5, 15); + } + + // URL presence + $urls = $this->extractUrls($nfoContent); + if (! empty($urls)) { + $score += min(count($urls) * 2, 10); + } + + // Codec info presence + $codecInfo = $this->extractCodecInfo($nfoContent); + $score += count(array_filter($codecInfo)) * 3; + + // ASCII art detection (scene NFOs often have decorative borders) + if ($this->hasAsciiArt($nfoContent)) { + $score += 10; + } + + // Structural elements bonus + $structuralScore = $this->analyzeStructure($nfoContent); + $score += $structuralScore; + + // Group name detection bonus + if ($this->extractGroupName($nfoContent) !== null) { + $score += 8; + } + + // Release date detection bonus + if ($this->extractReleaseDate($nfoContent) !== null) { + $score += 5; + } + + // Language info bonus + $languages = $this->extractLanguage($nfoContent); + if (! empty($languages)) { + $score += min(count($languages) * 2, 6); + } + + // Runtime detection bonus + if ($this->extractRuntime($nfoContent) !== null) { + $score += 4; + } + + // Penalty for binary content remnants + if (preg_match_all('/[\x00-\x08\x0B\x0C\x0E-\x1F]/', $nfoContent, $binaryMatches)) { + $score -= min(count($binaryMatches[0]) * 5, 20); + } + + return max(0, min(100, $score)); + } + + /** + * Detect ASCII art in NFO content. + * + * @param string $nfoContent The NFO content to analyze. + * @return bool True if ASCII art is detected. + */ + protected function hasAsciiArt(string $nfoContent): bool + { + // Check for common ASCII art characters in repeated sequences + $asciiArtPatterns = [ + // Decorative borders + '/[-=*~#@]{5,}/', + // Box drawing characters + '/[┌┐└┘├┤┬┴┼│─╔╗╚╝║═]{3,}/', + // Extended ASCII art characters + '/[░▒▓█▄▀■□▪▫]{3,}/', + // Common ASCII art patterns + '/[\/\\|_]{3,}.*[\/\\|_]{3,}/', + // Repeated special chars in artistic patterns + '/(\S)\1{4,}/', + ]; + + foreach ($asciiArtPatterns as $pattern) { + if (preg_match($pattern, $nfoContent)) { + return true; + } + } + + return false; + } + + /** + * Analyze structural elements of NFO content. + * + * @param string $nfoContent The NFO content to analyze. + * @return int Score based on structural quality (0-15). + */ + protected function analyzeStructure(string $nfoContent): int + { + $score = 0; + + // Check for section headers + $sectionPatterns = [ + '/^[ \t]*[-=*]{2,}.*[-=*]{2,}[ \t]*$/m', // Decorative section dividers + '/^[ \t]*\[.*\][ \t]*$/m', // [Section Name] + '/^[ \t]*<.*>[ \t]*$/m', //
+ ]; + + foreach ($sectionPatterns as $pattern) { + if (preg_match_all($pattern, $nfoContent, $matches)) { + $score += min(count($matches[0]), 3); + } + } + + // Check for labeled fields (Field: Value format) + if (preg_match_all('/^[ \t]*[A-Za-z][A-Za-z\s]{2,20}\s*[:\.].*$/m', $nfoContent, $matches)) { + $score += min(count($matches[0]) / 2, 5); + } + + // Check for consistent line endings and formatting + $lines = explode("\n", $nfoContent); + $nonEmptyLines = array_filter($lines, fn($line) => trim($line) !== ''); + + if (count($nonEmptyLines) >= 10) { + $score += 2; + } + + return min(15, (int) $score); + } + + /** + * Decompress and retrieve NFO content from a release. + * + * @param int $releaseId The release ID. + * @return string|null The NFO content or null if not found. + */ + public function getNfoContent(int $releaseId): ?string + { + $nfoRecord = ReleaseNfo::getReleaseNfo($releaseId); + + if ($nfoRecord === null || empty($nfoRecord->nfo)) { + return null; + } + + return $nfoRecord->nfo; + } + + /** + * Store NFO content for a release. + * + * @param int $releaseId The release ID. + * @param string $nfoContent The NFO content to store. + * @param bool $compress Whether to compress the content. + * @return bool True on success, false on failure. + */ + public function storeNfoContent(int $releaseId, string $nfoContent, bool $compress = true): bool + { + try { + $data = $compress ? "\x1f\x8b\x08\x00".gzcompress($nfoContent) : $nfoContent; + + ReleaseNfo::updateOrCreate( + ['releases_id' => $releaseId], + ['nfo' => $data] + ); + + Release::whereId($releaseId)->update(['nfostatus' => self::NFO_FOUND]); + + return true; + } catch (Throwable $e) { + Log::error("Failed to store NFO for release {$releaseId}: ".$e->getMessage()); + + return false; + } + } + + /** + * Clear the settings cache. + * + * Useful when settings have been updated and need to be reloaded. + */ + public function clearSettingsCache(): void + { + Cache::forget('nfo_maxnfoprocessed'); + Cache::forget('nfo_maxnforetries'); + Cache::forget('nfo_maxsizetoprocessnfo'); + Cache::forget('nfo_minsizetoprocessnfo'); + } } diff --git a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php index 9936d4ae0..08c7c70ed 100644 --- a/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php +++ b/app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php @@ -626,12 +626,22 @@ class AdditionalProcessingOrchestrator continue; } - // NFO files - if ($context->releaseHasNoNFO && preg_match('/(\.(nfo|inf|ofn)|info\.txt)$/i', $filePath)) { - if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { - $this->output->echoNfoFound(); + // NFO files - enhanced detection with multiple patterns + if ($context->releaseHasNoNFO) { + // Standard NFO extensions + if (preg_match('/(\.(nfo|inf|ofn|diz)|info\.txt)$/i', $filePath)) { + if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + } + continue; + } + // Alternative NFO filenames (file_id.diz, readme.txt, etc.) + elseif ($this->releaseManager->isNfoFilename($filePath)) { + if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) { + $this->output->echoNfoFound(); + } + continue; } - continue; } // Audio files diff --git a/app/Services/AdditionalProcessing/ArchiveExtractionService.php b/app/Services/AdditionalProcessing/ArchiveExtractionService.php index dda9f5701..9693e6ba5 100644 --- a/app/Services/AdditionalProcessing/ArchiveExtractionService.php +++ b/app/Services/AdditionalProcessing/ArchiveExtractionService.php @@ -574,8 +574,82 @@ class ArchiveExtractionService */ private function getAllowedExtensions(): array { - return ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a', - 'flac', 'png', 'epub', 'cbz', 'cbr', 'djvu']; + return [ + // NFO and info files (prioritized for extraction) + 'nfo', 'diz', 'inf', 'txt', + // Subtitles + 'srt', 'sub', 'idx', 'ass', 'ssa', 'vtt', + // Video + 'mkv', 'mpeg', 'avi', 'mp4', 'm4v', 'mov', 'wmv', 'flv', 'ts', 'vob', 'm2ts', 'webm', + // Audio + 'mp3', 'm4a', 'flac', 'ogg', 'aac', 'wav', 'wma', 'opus', 'ape', + // Images + 'jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', + // Documents + 'epub', 'pdf', 'cbz', 'cbr', 'djvu', 'mobi', 'azw', 'azw3', + // Executables (for software releases) + 'exe', 'msi', + ]; + } + + /** + * Check if a file is an NFO or info file. + * + * @param string $filename The filename to check. + * @return bool True if it's an NFO-like file. + */ + public function isNfoFile(string $filename): bool + { + $basename = strtolower(basename($filename)); + + // Standard NFO extensions + if (preg_match('/\.(nfo|diz|inf)$/i', $basename)) { + return true; + } + + // Common NFO alternative names + $nfoNames = [ + 'file_id.diz', 'fileid.diz', 'file-id.diz', + 'readme.txt', 'readme.1st', 'read.me', 'readmenow.txt', + 'info.txt', 'information.txt', 'about.txt', 'notes.txt', + 'release.txt', 'release.nfo', + ]; + + if (in_array($basename, $nfoNames, true)) { + return true; + } + + // Scene-style NFO naming: 00-groupname.nfo, group-release.nfo + if (preg_match('/^(?:00?-[a-z0-9_-]+|[a-z0-9]+-[a-z0-9._-]+)\.(?:nfo|txt)$/i', $basename)) { + return true; + } + + return false; + } + + /** + * Sort files to prioritize NFO files for processing. + * + * @param array $files Array of file info arrays. + * @return array Sorted array with NFO files first. + */ + public function sortFilesWithNfoPriority(array $files): array + { + usort($files, function ($a, $b) { + $aIsNfo = $this->isNfoFile($a['name'] ?? ''); + $bIsNfo = $this->isNfoFile($b['name'] ?? ''); + + if ($aIsNfo && ! $bIsNfo) { + return -1; + } + if (! $aIsNfo && $bIsNfo) { + return 1; + } + + return 0; + }); + + return $files; } /** diff --git a/app/Services/AdditionalProcessing/ReleaseFileManager.php b/app/Services/AdditionalProcessing/ReleaseFileManager.php index 960f6b588..a39687dde 100644 --- a/app/Services/AdditionalProcessing/ReleaseFileManager.php +++ b/app/Services/AdditionalProcessing/ReleaseFileManager.php @@ -345,7 +345,12 @@ class ReleaseFileManager } /** - * Process NFO file. + * Process NFO file with enhanced detection capabilities. + * + * Supports multiple NFO naming conventions: + * - Standard: .nfo, .diz, .info + * - Alternative: file_id.diz, readme.txt, info.txt + * - Scene-style: 00-groupname.nfo, groupname-releasename.nfo */ public function processNfoFile( string $fileLocation, @@ -354,6 +359,10 @@ class ReleaseFileManager ): bool { try { $data = File::get($fileLocation); + + // Try to detect and convert encoding + $data = $this->normalizeNfoEncoding($data); + if ($this->nfo->isNFO($data, $context->release->guid) && $this->nfo->addAlternateNfo($data, $context->release, $nntp) ) { @@ -367,6 +376,84 @@ class ReleaseFileManager return false; } + /** + * Check if a filename looks like an NFO file. + * + * @param string $filename The filename to check. + * @return bool True if the filename matches NFO patterns. + */ + public function isNfoFilename(string $filename): bool + { + // Standard NFO extensions + if (preg_match('/\.(?:nfo|diz|info?)$/i', $filename)) { + return true; + } + + // Alternative NFO filenames + $nfoPatterns = [ + '/^(?:file[_-]?id|readme|release|info(?:rmation)?|about|notes?)\.(?:txt|diz)$/i', + '/^00-[a-z0-9_-]+\.nfo$/i', // Scene: 00-group.nfo + '/^0+-[a-z0-9_-]+\.nfo$/i', // Scene variations + '/^[a-z0-9_-]+-[a-z0-9_.-]+\.nfo$/i', // Scene: group-release.nfo + '/info\.txt$/i', // info.txt (common alternative) + ]; + + $basename = basename($filename); + foreach ($nfoPatterns as $pattern) { + if (preg_match($pattern, $basename)) { + return true; + } + } + + return false; + } + + /** + * Normalize NFO encoding to UTF-8. + * + * NFO files often use CP437 (DOS) encoding for ASCII art. + * This method attempts to detect and convert various encodings. + * + * @param string $data Raw NFO data. + * @return string UTF-8 encoded NFO data. + */ + protected function normalizeNfoEncoding(string $data): string + { + // Check for UTF-8 BOM and remove it + if (str_starts_with($data, "\xEF\xBB\xBF")) { + $data = substr($data, 3); + } + + // Check for UTF-16 BOM + if (str_starts_with($data, "\xFF\xFE")) { + // UTF-16 LE + $data = mb_convert_encoding(substr($data, 2), 'UTF-8', 'UTF-16LE'); + } elseif (str_starts_with($data, "\xFE\xFF")) { + // UTF-16 BE + $data = mb_convert_encoding(substr($data, 2), 'UTF-8', 'UTF-16BE'); + } + + // If already valid UTF-8, return as-is + if (mb_check_encoding($data, 'UTF-8')) { + return $data; + } + + // Try CP437 (DOS encoding - common for scene NFOs with ASCII art) + // Use the utility function if available + if (class_exists('\Blacklight\utility\Utility') && method_exists('\Blacklight\utility\Utility', 'cp437toUTF')) { + return \Blacklight\utility\Utility::cp437toUTF($data); + } + + // Fallback: try ISO-8859-1 (Latin-1) + $converted = @mb_convert_encoding($data, 'UTF-8', 'ISO-8859-1'); + if ($converted !== false) { + return $converted; + } + + // Last resort: force UTF-8 with error handling + return mb_convert_encoding($data, 'UTF-8', 'UTF-8'); + } + /** * Handle release name extraction from RAR file content. */ diff --git a/app/Services/Binaries/BinariesConfig.php b/app/Services/Binaries/BinariesConfig.php new file mode 100644 index 000000000..c1cc273bc --- /dev/null +++ b/app/Services/Binaries/BinariesConfig.php @@ -0,0 +1,56 @@ + Pending binary updates */ + private array $binariesUpdate = []; + + /** @var array IDs of binaries created in this batch */ + private array $insertedBinaryIds = []; + + /** @var array Processed articles */ + private array $articles = []; + + public function __construct() {} + + /** + * Reset state for a new batch. + */ + public function reset(): void + { + $this->binariesUpdate = []; + $this->insertedBinaryIds = []; + $this->articles = []; + } + + /** + * Get or create a binary for the given header. + * + * @return int|null Binary ID or null on failure + */ + public function getOrCreateBinary( + array $header, + int $collectionId, + int $groupId, + int $fileNumber + ): ?int { + $articleKey = $header['matches'][1]; + + // Return cached if already processed + if (isset($this->articles[$articleKey])) { + $binaryId = $this->articles[$articleKey]['BinaryID']; + $this->binariesUpdate[$binaryId]['Size'] += $header['Bytes']; + $this->binariesUpdate[$binaryId]['Parts']++; + + return $binaryId; + } + + $hash = md5($header['matches'][1].$header['From'].$groupId); + $driver = DB::getDriverName(); + + try { + $binaryId = $this->insertOrGetBinary( + $driver, + $hash, + $header, + $collectionId, + $fileNumber + ); + + if ($binaryId > 0) { + $this->binariesUpdate[$binaryId] = ['Size' => 0, 'Parts' => 0]; + $this->articles[$articleKey] = [ + 'CollectionID' => $collectionId, + 'BinaryID' => $binaryId, + ]; + + return $binaryId; + } + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error('Binary insert failed: '.$e->getMessage()); + } + } + + return null; + } + + private function insertOrGetBinary( + string $driver, + string $hash, + array $header, + int $collectionId, + int $fileNumber + ): int { + $name = mb_convert_encoding($header['matches'][1], 'UTF-8', mb_list_encodings()); + $totalParts = (int) $header['matches'][3]; + $partSize = (int) $header['Bytes']; + + if ($driver === 'sqlite') { + return $this->insertBinarySqlite($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize); + } + + return $this->insertBinaryMysql($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize); + } + + private function insertBinarySqlite( + string $hash, + string $name, + int $collectionId, + int $totalParts, + int $fileNumber, + int $partSize + ): int { + DB::statement( + 'INSERT OR IGNORE INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) VALUES (?, ?, ?, ?, 1, ?, ?)', + [$hash, $name, $collectionId, $totalParts, $fileNumber, $partSize] + ); + + $lastId = (int) DB::connection()->getPdo()->lastInsertId(); + if ($lastId > 0) { + $this->insertedBinaryIds[$lastId] = true; + + return $lastId; + } + + $bin = DB::selectOne( + 'SELECT id FROM binaries WHERE binaryhash = ? AND collections_id = ? LIMIT 1', + [$hash, $collectionId] + ); + + return (int) ($bin->id ?? 0); + } + + private function insertBinaryMysql( + string $hash, + string $name, + int $collectionId, + int $totalParts, + int $fileNumber, + int $partSize + ): int { + $sql = 'INSERT INTO binaries ' + .'(binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) ' + .'VALUES (UNHEX(?), ?, ?, ?, 1, ?, ?) ' + .'ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + VALUES(partsize)'; + + DB::statement($sql, [$hash, $name, $collectionId, $totalParts, $fileNumber, $partSize]); + + $lastId = (int) DB::connection()->getPdo()->lastInsertId(); + if ($lastId > 0) { + $this->insertedBinaryIds[$lastId] = true; + + return $lastId; + } + + $bin = DB::selectOne( + 'SELECT id FROM binaries WHERE binaryhash = UNHEX(?) AND collections_id = ? LIMIT 1', + [$hash, $collectionId] + ); + + return (int) ($bin->id ?? 0); + } + + /** + * Flush accumulated size/parts updates to the database. + */ + public function flushUpdates(int $chunkSize = 1000): bool + { + $updates = $this->getPendingUpdates(); + if (empty($updates)) { + return true; + } + + $driver = DB::getDriverName(); + + try { + if ($driver === 'sqlite') { + return $this->flushUpdatesSqlite($updates); + } + + return $this->flushUpdatesMysql($updates, $chunkSize); + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error('Binaries aggregate update failed: '.$e->getMessage()); + } + + return false; + } + } + + private function flushUpdatesSqlite(array $updates): bool + { + foreach ($updates as $row) { + DB::statement( + 'UPDATE binaries SET partsize = partsize + ?, currentparts = currentparts + ? WHERE id = ?', + [$row['partsize'], $row['currentparts'], $row['id']] + ); + } + + return true; + } + + private function flushUpdatesMysql(array $updates, int $chunkSize): bool + { + foreach (array_chunk($updates, $chunkSize) as $chunk) { + $placeholders = []; + $bindings = []; + + foreach ($chunk as $row) { + $placeholders[] = '(?,?,?)'; + $bindings[] = $row['id']; + $bindings[] = $row['partsize']; + $bindings[] = $row['currentparts']; + } + + $sql = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '.implode(',', $placeholders) + .' ON DUPLICATE KEY UPDATE partsize = partsize + VALUES(partsize), currentparts = currentparts + VALUES(currentparts)'; + + DB::statement($sql, $bindings); + } + + return true; + } + + /** + * Check if article is already processed. + */ + public function hasArticle(string $articleKey): bool + { + return isset($this->articles[$articleKey]); + } + + /** + * Get IDs created in this batch. + */ + public function getInsertedIds(): array + { + return array_keys($this->insertedBinaryIds); + } + + /** + * Get pending binary updates that haven't been flushed. + */ + private function getPendingUpdates(): array + { + $rows = []; + foreach ($this->binariesUpdate as $binaryId => $binary) { + if (($binary['Size'] ?? 0) > 0 || ($binary['Parts'] ?? 0) > 0) { + $rows[] = [ + 'id' => $binaryId, + 'partsize' => $binary['Size'], + 'currentparts' => $binary['Parts'], + ]; + } + } + + return $rows; + } +} + diff --git a/app/Services/Binaries/CollectionHandler.php b/app/Services/Binaries/CollectionHandler.php new file mode 100644 index 000000000..174692a7d --- /dev/null +++ b/app/Services/Binaries/CollectionHandler.php @@ -0,0 +1,266 @@ + Cached collection IDs by key */ + private array $collectionIds = []; + + /** @var array IDs of collections created in this batch */ + private array $insertedCollectionIds = []; + + /** @var array Collection hashes touched in this batch */ + private array $batchCollectionHashes = []; + + public function __construct( + ?CollectionsCleaning $collectionsCleaning = null, + ?XrefService $xrefService = null + ) { + $this->collectionsCleaning = $collectionsCleaning ?? new CollectionsCleaning; + $this->xrefService = $xrefService ?? new XrefService; + } + + /** + * Reset state for a new batch. + */ + public function reset(): void + { + $this->collectionIds = []; + $this->insertedCollectionIds = []; + $this->batchCollectionHashes = []; + } + + /** + * Get or create a collection for the given header. + * + * @return int|null Collection ID or null on failure + */ + public function getOrCreateCollection( + array $header, + int $groupId, + string $groupName, + int $totalFiles, + string $batchNoise + ): ?int { + $collMatch = $this->collectionsCleaning->collectionsCleaner( + $header['matches'][1], + $groupName + ); + + $collectionKey = $collMatch['name'].$totalFiles; + + // Return cached ID if already processed this batch + if (isset($this->collectionIds[$collectionKey])) { + return $this->collectionIds[$collectionKey]; + } + + $collectionHash = sha1($collectionKey); + $this->batchCollectionHashes[$collectionHash] = true; + + $headerDate = is_numeric($header['Date']) ? (int) $header['Date'] : strtotime($header['Date']); + $now = now()->timestamp; + $unixtime = min($headerDate, $now) ?: $now; + + $existingXref = Collection::whereCollectionhash($collectionHash)->value('xref'); + $headerTokens = $this->xrefService->extractTokens($header['Xref'] ?? ''); + $newTokens = $this->xrefService->diffNewTokens($existingXref, $header['Xref'] ?? ''); + $finalXrefAppend = implode(' ', $newTokens); + + $subject = substr(mb_convert_encoding($header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255); + $fromName = mb_convert_encoding($header['From'], 'UTF-8', mb_list_encodings()); + + $driver = DB::getDriverName(); + + try { + $collectionId = $this->insertOrGetCollection( + $driver, + $subject, + $fromName, + $unixtime, + $headerTokens, + $finalXrefAppend, + $groupId, + $totalFiles, + $collectionHash, + $collMatch['id'], + $batchNoise + ); + + if ($collectionId > 0) { + $this->collectionIds[$collectionKey] = $collectionId; + + return $collectionId; + } + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error('Collection insert failed: '.$e->getMessage()); + } + } + + return null; + } + + private function insertOrGetCollection( + string $driver, + string $subject, + string $fromName, + int $unixtime, + array $headerTokens, + string $finalXrefAppend, + int $groupId, + int $totalFiles, + string $collectionHash, + int $regexId, + string $batchNoise + ): int { + if ($driver === 'sqlite') { + return $this->insertCollectionSqlite( + $subject, + $fromName, + $unixtime, + $headerTokens, + $groupId, + $totalFiles, + $collectionHash, + $regexId, + $batchNoise + ); + } + + return $this->insertCollectionMysql( + $subject, + $fromName, + $unixtime, + $headerTokens, + $finalXrefAppend, + $groupId, + $totalFiles, + $collectionHash, + $regexId, + $batchNoise + ); + } + + private function insertCollectionSqlite( + string $subject, + string $fromName, + int $unixtime, + array $headerTokens, + int $groupId, + int $totalFiles, + string $collectionHash, + int $regexId, + string $batchNoise + ): int { + DB::statement( + 'INSERT OR IGNORE INTO collections (subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) VALUES (?, ?, datetime(?, "unixepoch"), ?, ?, ?, ?, datetime("now"), ?)', + [ + $subject, + $fromName, + $unixtime, + implode(' ', $headerTokens), + $groupId, + $totalFiles, + $collectionHash, + $regexId, + $batchNoise, + ] + ); + + $lastId = (int) DB::connection()->getPdo()->lastInsertId(); + if ($lastId > 0) { + $this->insertedCollectionIds[$lastId] = true; + + return $lastId; + } + + return (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0); + } + + private function insertCollectionMysql( + string $subject, + string $fromName, + int $unixtime, + array $headerTokens, + string $finalXrefAppend, + int $groupId, + int $totalFiles, + string $collectionHash, + int $regexId, + string $batchNoise + ): int { + $insertSql = 'INSERT INTO collections ' + .'(subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) ' + .'VALUES (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, NOW(), ?) ' + .'ON DUPLICATE KEY UPDATE dateadded = NOW()'; + + $bindings = [ + $subject, + $fromName, + $unixtime, + implode(' ', $headerTokens), + $groupId, + $totalFiles, + $collectionHash, + $regexId, + $batchNoise, + ]; + + if ($finalXrefAppend !== '') { + $insertSql .= ', xref = CONCAT(xref, "\\n", ?)'; + $bindings[] = $finalXrefAppend; + } + + DB::statement($insertSql, $bindings); + + $lastId = (int) DB::connection()->getPdo()->lastInsertId(); + if ($lastId > 0) { + $this->insertedCollectionIds[$lastId] = true; + + return $lastId; + } + + return (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0); + } + + /** + * Get IDs created in this batch. + */ + public function getInsertedIds(): array + { + return array_keys($this->insertedCollectionIds); + } + + /** + * Get all collection IDs processed this batch. + */ + public function getAllIds(): array + { + return array_values(array_unique(array_map('intval', $this->collectionIds))); + } + + /** + * Get all collection hashes processed this batch. + */ + public function getBatchHashes(): array + { + return array_keys($this->batchCollectionHashes); + } +} + diff --git a/app/Services/Binaries/HeaderParser.php b/app/Services/Binaries/HeaderParser.php new file mode 100644 index 000000000..f41ffcc0b --- /dev/null +++ b/app/Services/Binaries/HeaderParser.php @@ -0,0 +1,167 @@ +blacklistService = $blacklistService ?? new BlacklistService; + } + + /** + * Reset counters for a new batch. + */ + public function reset(): void + { + $this->notYEnc = 0; + $this->blacklisted = 0; + } + + /** + * Parse and filter raw headers from NNTP. + * + * @param array $headers Raw headers from NNTP + * @param string $groupName The newsgroup name + * @param bool $partRepair Whether this is a part repair scan + * @param array|null $missingParts Missing part numbers if part repair + * @return array Filtered and parsed headers with article info + */ + public function parse( + array $headers, + string $groupName, + bool $partRepair = false, + ?array $missingParts = null + ): array { + $parsed = []; + $headersRepaired = []; + + foreach ($headers as $header) { + // Check if we got the article + if (! isset($header['Number'])) { + continue; + } + + // For part repair, only process missing parts + if ($partRepair && $missingParts !== null) { + if (! \in_array($header['Number'], $missingParts, false)) { + continue; + } + $headersRepaired[] = $header['Number']; + } + + // Parse subject to get base name and part/total like "(12/45)" + if (! preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) { + $this->notYEnc++; + + continue; + } + + // Normalize to include yEnc if missing + if (stripos($header['Subject'], 'yEnc') === false) { + $matches[1] .= ' yEnc'; + } + + $header['matches'] = $matches; + + // Filter subject based on black/white list + if ($this->blacklistService->isBlackListed($header, $groupName)) { + $this->blacklisted++; + + continue; + } + + // Ensure Bytes is set + if (empty($header['Bytes'])) { + $header['Bytes'] = $header[':bytes'] ?? 0; + } + + $parsed[] = [ + 'header' => $header, + 'repaired' => $partRepair, + ]; + } + + return [ + 'headers' => array_column($parsed, 'header'), + 'repaired' => $headersRepaired, + 'notYEnc' => $this->notYEnc, + 'blacklisted' => $this->blacklisted, + ]; + } + + /** + * Update blacklist last_activity for matched rules. + */ + public function flushBlacklistUpdates(): void + { + $ids = $this->blacklistService->getAndClearIdsToUpdate(); + if (! empty($ids)) { + $this->blacklistService->updateBlacklistUsage($ids); + } + } + + /** + * Get count of non-yEnc headers filtered. + */ + public function getNotYEncCount(): int + { + return $this->notYEnc; + } + + /** + * Get count of blacklisted headers. + */ + public function getBlacklistedCount(): int + { + return $this->blacklisted; + } + + /** + * Extract highest and lowest article info from headers. + */ + public function getArticleRange(array $headers): array + { + $result = []; + $count = \count($headers); + + if ($count === 0) { + return $result; + } + + // Find first valid article + for ($i = 0; $i < $count; $i++) { + if (isset($headers[$i]['Number'])) { + $result['firstArticleNumber'] = $headers[$i]['Number']; + $result['firstArticleDate'] = $headers[$i]['Date'] ?? null; + break; + } + } + + // Find last valid article + for ($i = $count - 1; $i >= 0; $i--) { + if (isset($headers[$i]['Number'])) { + $result['lastArticleNumber'] = $headers[$i]['Number']; + $result['lastArticleDate'] = $headers[$i]['Date'] ?? null; + break; + } + } + + return $result; + } +} + diff --git a/app/Services/Binaries/HeaderStorageService.php b/app/Services/Binaries/HeaderStorageService.php new file mode 100644 index 000000000..c427d5fa6 --- /dev/null +++ b/app/Services/Binaries/HeaderStorageService.php @@ -0,0 +1,174 @@ + Article numbers that failed to insert */ + private array $failedInserts = []; + + public function __construct( + ?CollectionHandler $collectionHandler = null, + ?BinaryHandler $binaryHandler = null, + ?PartHandler $partHandler = null, + ?BinariesConfig $config = null + ) { + $this->config = $config ?? BinariesConfig::fromSettings(); + $this->collectionHandler = $collectionHandler ?? new CollectionHandler; + $this->binaryHandler = $binaryHandler ?? new BinaryHandler; + $this->partHandler = $partHandler ?? new PartHandler( + $this->config->partsChunkSize, + true + ); + } + + /** + * Store parsed headers to the database. + * + * @param array $headers Parsed headers with 'matches' already populated + * @param array $groupMySQL Group info from database + * @param bool $addToPartRepair Whether to track failed inserts + * @return array Article numbers that failed to insert + */ + public function store(array $headers, array $groupMySQL, bool $addToPartRepair = true): array + { + if (empty($headers)) { + return []; + } + + // Reset all handlers + $this->collectionHandler->reset(); + $this->binaryHandler->reset(); + $this->partHandler->reset(); + $this->partHandler->setAddToPartRepair($addToPartRepair); + $this->failedInserts = []; + + // Create transaction + $transaction = new HeaderStorageTransaction( + $this->collectionHandler, + $this->binaryHandler, + $this->partHandler + ); + + $transaction->begin(); + + // Process each header + foreach ($headers as $header) { + if (! $this->processHeader($header, $groupMySQL, $transaction)) { + if ($addToPartRepair && isset($header['Number'])) { + $this->failedInserts[] = $header['Number']; + } + } + } + + // Flush remaining parts + if ($this->partHandler->hasPending()) { + if (! $this->partHandler->flush()) { + $transaction->markError(); + } + } + + // Flush binary aggregate updates + if (! $transaction->hasErrors()) { + if (! $this->binaryHandler->flushUpdates($this->config->binariesUpdateChunkSize)) { + $transaction->markError(); + } + } + + // Finish transaction + if (! $transaction->finish()) { + // All failed + if ($addToPartRepair) { + return array_unique(array_merge( + $this->failedInserts, + $this->partHandler->getFailedNumbers() + )); + } + + return []; + } + + return array_unique(array_merge( + $this->failedInserts, + $this->partHandler->getFailedNumbers() + )); + } + + private function processHeader(array $header, array $groupMySQL, HeaderStorageTransaction $transaction): bool + { + // Get file count from subject + $fileCount = $this->getFileCount($header['matches'][1]); + if ($fileCount[1] === 0 && $fileCount[3] === 0) { + $fileCount = $this->getFileCount($header['matches'][0]); + } + + $totalFiles = (int) $fileCount[3]; + $fileNumber = (int) $fileCount[1]; + + // Get or create collection + $collectionId = $this->collectionHandler->getOrCreateCollection( + $header, + $groupMySQL['id'], + $groupMySQL['name'], + $totalFiles, + $transaction->getBatchNoise() + ); + + if ($collectionId === null) { + $transaction->markError(); + + return false; + } + + // Get or create binary + $binaryId = $this->binaryHandler->getOrCreateBinary( + $header, + $collectionId, + $groupMySQL['id'], + $fileNumber + ); + + if ($binaryId === null) { + $transaction->markError(); + + return false; + } + + // Add part + if (! $this->partHandler->addPart($binaryId, $header)) { + $transaction->markError(); + + return false; + } + + return true; + } + + private function getFileCount(string $subject): array + { + if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])[\s$:]/i', $subject, $fileCount)) { + $fileCount[1] = $fileCount[3] = 0; + } + + return $fileCount; + } +} + diff --git a/app/Services/Binaries/HeaderStorageTransaction.php b/app/Services/Binaries/HeaderStorageTransaction.php new file mode 100644 index 000000000..56ba7e51f --- /dev/null +++ b/app/Services/Binaries/HeaderStorageTransaction.php @@ -0,0 +1,194 @@ +collectionHandler = $collectionHandler; + $this->binaryHandler = $binaryHandler; + $this->partHandler = $partHandler; + $this->batchNoise = bin2hex(random_bytes(8)); + } + + /** + * Get the batch noise marker for this transaction. + */ + public function getBatchNoise(): string + { + return $this->batchNoise; + } + + /** + * Start a new database transaction. + */ + public function begin(): void + { + DB::beginTransaction(); + $this->hadErrors = false; + } + + /** + * Mark that an error occurred. + */ + public function markError(): void + { + $this->hadErrors = true; + } + + /** + * Check if errors occurred. + */ + public function hasErrors(): bool + { + return $this->hadErrors; + } + + /** + * Commit the transaction if no errors, rollback otherwise. + */ + public function finish(): bool + { + if ($this->hadErrors) { + $this->rollbackAndCleanup(); + + return false; + } + + try { + DB::commit(); + + return true; + } catch (\Throwable $e) { + $this->rollbackAndCleanup(); + + if (config('app.debug') === true) { + Log::error('HeaderStorageTransaction commit failed: '.$e->getMessage()); + } + + return false; + } + } + + /** + * Perform rollback and cleanup any orphaned data. + */ + private function rollbackAndCleanup(): void + { + try { + DB::rollBack(); + } catch (\Throwable $e) { + // Already rolled back + } + + $this->cleanup(); + } + + /** + * Cleanup rows that may have been inserted before rollback. + */ + private function cleanup(): void + { + try { + $this->cleanupParts(); + $this->cleanupBinaries(); + $this->cleanupCollections(); + + // Final guard for sqlite + if (DB::getDriverName() === 'sqlite') { + DB::statement('DELETE FROM parts'); + DB::statement('DELETE FROM binaries'); + DB::statement('DELETE FROM collections'); + } + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::warning('Post-rollback cleanup failed: '.$e->getMessage()); + } + } + } + + private function cleanupParts(): void + { + $numbers = $this->partHandler->getInsertedNumbers(); + if (! empty($numbers)) { + $placeholders = implode(',', array_fill(0, \count($numbers), '?')); + DB::statement("DELETE FROM parts WHERE number IN ({$placeholders})", $numbers); + } + } + + private function cleanupBinaries(): void + { + $ids = $this->binaryHandler->getInsertedIds(); + if (! empty($ids)) { + $placeholders = implode(',', array_fill(0, \count($ids), '?')); + DB::statement("DELETE FROM binaries WHERE id IN ({$placeholders})", $ids); + } + } + + private function cleanupCollections(): void + { + $insertedIds = $this->collectionHandler->getInsertedIds(); + $allIds = $this->collectionHandler->getAllIds(); + $hashes = $this->collectionHandler->getBatchHashes(); + + $ids = ! empty($insertedIds) ? $insertedIds : $allIds; + + if (! empty($ids)) { + $placeholders = implode(',', array_fill(0, \count($ids), '?')); + + // Remove parts and binaries referencing these collections, then collections + DB::statement( + "DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN ({$placeholders}))", + $ids + ); + DB::statement("DELETE FROM binaries WHERE collections_id IN ({$placeholders})", $ids); + DB::statement("DELETE FROM collections WHERE id IN ({$placeholders})", $ids); + } elseif (! empty($hashes)) { + $placeholders = implode(',', array_fill(0, \count($hashes), '?')); + + DB::statement( + "DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ({$placeholders})))", + $hashes + ); + DB::statement( + "DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ({$placeholders}))", + $hashes + ); + DB::statement("DELETE FROM collections WHERE collectionhash IN ({$placeholders})", $hashes); + } else { + // Fallback by noise marker + DB::statement( + 'DELETE FROM parts WHERE binaries_id IN (SELECT b.id FROM binaries b WHERE b.collections_id IN (SELECT c.id FROM collections c WHERE c.noise = ?))', + [$this->batchNoise] + ); + DB::statement( + 'DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE noise = ?)', + [$this->batchNoise] + ); + DB::statement('DELETE FROM collections WHERE noise = ?', [$this->batchNoise]); + } + } +} + diff --git a/app/Services/Binaries/MissedPartHandler.php b/app/Services/Binaries/MissedPartHandler.php new file mode 100644 index 000000000..3c3327012 --- /dev/null +++ b/app/Services/Binaries/MissedPartHandler.php @@ -0,0 +1,181 @@ +partRepairLimit = $partRepairLimit; + $this->partRepairMaxTries = $partRepairMaxTries; + } + + /** + * Add missing article numbers to the repair queue. + */ + public function addMissingParts(array $numbers, int $groupId): void + { + if (empty($numbers)) { + return; + } + + $driver = DB::getDriverName(); + + if ($driver === 'sqlite') { + $this->addMissingPartsSqlite($numbers, $groupId); + + return; + } + + $this->addMissingPartsMysql($numbers, $groupId); + } + + private function addMissingPartsSqlite(array $numbers, int $groupId): void + { + foreach ($numbers as $number) { + DB::statement( + 'INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON CONFLICT(numberid, groups_id) DO UPDATE SET attempts = attempts + 1', + [$number, $groupId] + ); + } + } + + private function addMissingPartsMysql(array $numbers, int $groupId): void + { + $insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; + foreach ($numbers as $number) { + $insertStr .= '('.$number.','.$groupId.'),'; + } + + DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); + } + + /** + * Remove successfully repaired parts from the queue. + */ + public function removeRepairedParts(array $numbers, int $groupId): void + { + if (empty($numbers)) { + return; + } + + $sql = 'DELETE FROM missed_parts WHERE numberid in ('; + foreach ($numbers as $number) { + $sql .= $number.','; + } + + try { + DB::transaction(static function () use ($groupId, $sql) { + DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupId); + }, 10); + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::warning('removeRepairedParts failed: '.$e->getMessage()); + } + } + } + + /** + * Get parts that need repair for a group. + * + * @return array Array of missed parts + */ + public function getMissingParts(int $groupId): array + { + try { + return DB::select( + sprintf( + 'SELECT * FROM missed_parts WHERE groups_id = %d AND attempts < %d ORDER BY numberid ASC LIMIT %d', + $groupId, + $this->partRepairMaxTries, + $this->partRepairLimit + ) + ); + } catch (\PDOException $e) { + if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') { + Log::notice('Deadlock occurred while fetching missed parts'); + DB::rollBack(); + } + + return []; + } + } + + /** + * Increment attempts for parts that weren't repaired. + */ + public function incrementAttempts(int $groupId, int $maxNumberId): void + { + DB::update( + sprintf( + 'UPDATE missed_parts SET attempts = attempts + 1 WHERE groups_id = %d AND numberid <= %d', + $groupId, + $maxNumberId + ) + ); + } + + /** + * Increment attempts for specific article range (part repair NNTP failures). + */ + public function incrementRangeAttempts(int $groupId, int $first, int $last): void + { + if ($first === $last) { + MissedPart::query() + ->where('groups_id', $groupId) + ->where('numberid', $first) + ->increment('attempts'); + } else { + MissedPart::query() + ->where('groups_id', $groupId) + ->whereIn('numberid', range($first, $last)) + ->increment('attempts'); + } + } + + /** + * Get count of remaining missed parts. + */ + public function getCount(int $groupId, int $maxNumberId): int + { + $result = DB::select( + sprintf( + 'SELECT COUNT(id) AS num FROM missed_parts WHERE groups_id = %d AND numberid <= %d', + $groupId, + $maxNumberId + ) + ); + + return $result[0]->num ?? 0; + } + + /** + * Remove parts that exceeded max tries. + */ + public function cleanupExhaustedParts(int $groupId): void + { + DB::transaction(function () use ($groupId) { + DB::delete( + sprintf( + 'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d', + $this->partRepairMaxTries, + $groupId + ) + ); + }, 10); + } +} + diff --git a/app/Services/Binaries/PartHandler.php b/app/Services/Binaries/PartHandler.php new file mode 100644 index 000000000..9d95abaf7 --- /dev/null +++ b/app/Services/Binaries/PartHandler.php @@ -0,0 +1,157 @@ +chunkSize = max(100, $chunkSize); + $this->addToPartRepair = $addToPartRepair; + } + + /** + * Reset state for a new batch. + */ + public function reset(): void + { + $this->parts = []; + $this->insertedPartNumbers = []; + $this->failedPartNumbers = []; + } + + /** + * Set whether to add failed parts to repair queue. + */ + public function setAddToPartRepair(bool $value): void + { + $this->addToPartRepair = $value; + } + + /** + * Add a part to the pending insert queue. + * + * @return bool True if chunk was flushed successfully (or not needed), false on flush failure + */ + public function addPart(int $binaryId, array $header): bool + { + $this->parts[] = [ + 'binaries_id' => $binaryId, + 'number' => $header['Number'], + 'messageid' => $header['Message-ID'], + 'partnumber' => $header['matches'][2], + 'size' => $header['Bytes'], + ]; + + // Auto-flush when chunk size reached + if (\count($this->parts) >= $this->chunkSize) { + return $this->flush(); + } + + return true; + } + + /** + * Flush pending parts to database. + */ + public function flush(): bool + { + if (empty($this->parts)) { + return true; + } + + $success = $this->insertChunk($this->parts); + + if ($success) { + foreach ($this->parts as $part) { + $this->insertedPartNumbers[] = $part['number']; + } + } else { + foreach ($this->parts as $part) { + $this->failedPartNumbers[] = $part['number']; + } + } + + $this->parts = []; + + return $success; + } + + private function insertChunk(array $parts): bool + { + $placeholders = []; + $bindings = []; + $driver = DB::getDriverName(); + + foreach ($parts as $row) { + $placeholders[] = '(?,?,?,?,?)'; + $bindings[] = $row['binaries_id']; + $bindings[] = $row['number']; + $bindings[] = $row['messageid']; + $bindings[] = $row['partnumber']; + $bindings[] = $row['size']; + } + + $sql = $driver === 'sqlite' + ? 'INSERT OR IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders) + : 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders); + + try { + DB::statement($sql, $bindings); + + return true; + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error('Parts chunk insert failed: '.$e->getMessage()); + } + + return false; + } + } + + /** + * Get numbers of successfully inserted parts. + */ + public function getInsertedNumbers(): array + { + return $this->insertedPartNumbers; + } + + /** + * Get numbers of failed part inserts. + */ + public function getFailedNumbers(): array + { + return $this->failedPartNumbers; + } + + /** + * Check if there are pending parts waiting to be flushed. + */ + public function hasPending(): bool + { + return ! empty($this->parts); + } +} + diff --git a/app/Services/Categorization/Categorizers/MiscCategorizer.php b/app/Services/Categorization/Categorizers/MiscCategorizer.php index 2d30e1b66..6fd89affd 100644 --- a/app/Services/Categorization/Categorizers/MiscCategorizer.php +++ b/app/Services/Categorization/Categorizers/MiscCategorizer.php @@ -105,10 +105,12 @@ class MiscCategorizer extends AbstractCategorizer return $this->matched(Category::OTHER_HASHED, 0.7, 'obfuscated_uppercase'); } - // Long alphanumeric strings without typical release patterns - if (preg_match('/^[a-zA-Z0-9]{25,}$/', $name) && - !preg_match('/\b(19|20)\d{2}\b/', $name)) { - return $this->matched(Category::OTHER_HASHED, 0.65, 'obfuscated_long'); + // Mixed-case alphanumeric strings without separators (common obfuscation pattern) + // These look like random strings: e.g., "AA7Jl2toE8Q53yNZmQ5R6G" + if (preg_match('/^[a-zA-Z0-9]{15,}$/', $name) && + !preg_match('/\b(19|20)\d{2}\b/', $name) && + !preg_match('/^[A-Z][a-z]+([A-Z][a-z]+)+$/', $name)) { // Exclude CamelCase words + return $this->matched(Category::OTHER_HASHED, 0.7, 'obfuscated_mixed_alphanumeric'); } // Only punctuation and numbers with no clear structure diff --git a/composer.lock b/composer.lock index edd7926b5..89585e3b2 100644 --- a/composer.lock +++ b/composer.lock @@ -13184,23 +13184,23 @@ }, { "name": "tijsverkoyen/css-to-inline-styles", - "version": "v2.3.0", + "version": "v2.4.0", "source": { "type": "git", "url": "https://github.com/tijsverkoyen/CssToInlineStyles.git", - "reference": "0d72ac1c00084279c1816675284073c5a337c20d" + "reference": "f0292ccf0ec75843d65027214426b6b163b48b41" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/0d72ac1c00084279c1816675284073c5a337c20d", - "reference": "0d72ac1c00084279c1816675284073c5a337c20d", + "url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/f0292ccf0ec75843d65027214426b6b163b48b41", + "reference": "f0292ccf0ec75843d65027214426b6b163b48b41", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "php": "^7.4 || ^8.0", - "symfony/css-selector": "^5.4 || ^6.0 || ^7.0" + "symfony/css-selector": "^5.4 || ^6.0 || ^7.0 || ^8.0" }, "require-dev": { "phpstan/phpstan": "^2.0", @@ -13233,9 +13233,9 @@ "homepage": "https://github.com/tijsverkoyen/CssToInlineStyles", "support": { "issues": "https://github.com/tijsverkoyen/CssToInlineStyles/issues", - "source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/v2.3.0" + "source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/v2.4.0" }, - "time": "2024-12-21T16:25:41+00:00" + "time": "2025-12-02T11:56:42+00:00" }, { "name": "vlucas/phpdotenv", diff --git a/resources/views/admin/groups/bulk.blade.php b/resources/views/admin/groups/bulk.blade.php index e319918bd..26d041c2d 100644 --- a/resources/views/admin/groups/bulk.blade.php +++ b/resources/views/admin/groups/bulk.blade.php @@ -4,12 +4,12 @@
-
+
-

+

{{ $title ?? 'Bulk Add Newsgroups' }}

- + View All Groups
@@ -18,10 +18,10 @@
@if(!empty($groupmsglist)) -
+
- -

+ +

The following groups have been processed. You can now view them in the group list.

@@ -29,33 +29,33 @@
- - +
+ - - + + - + @foreach($groupmsglist as $group) - +
GroupStatusGroupStatus
- - {{ $group['group'] }} + + {{ $group['group'] }}
@if(strpos($group['msg'], 'Error') !== false) - + {{ $group['msg'] }} @elseif(strpos($group['msg'], 'exists') !== false) - + {{ $group['msg'] }} @else - + {{ $group['msg'] }} @endif @@ -67,10 +67,10 @@ @else -
+
- -

+ +

Enter a regular expression to match multiple groups for bulk addition to the system.

@@ -91,13 +91,13 @@
-

+

A regular expression to match against group names. Separate multiple patterns with the pipe symbol (|). -
Example: alt.binaries.cd.image.linux|alt.binaries.warez.linux +
Example: alt.binaries.cd.image.linux|alt.binaries.warez.linux

@@ -112,7 +112,7 @@ value="1" class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500" checked> - +
- +
-

+

Inactive groups will not have headers downloaded for them.

@@ -138,7 +138,7 @@ id="backfill_yes" value="1" class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500"> - +
- +
-

+

Inactive groups will not have backfill headers downloaded for them.

@@ -159,17 +159,17 @@ -
+
- + Back to Groups @if(empty($groupmsglist)) - @else - + Add More Groups @endif diff --git a/tests/Support/TestBinariesHarness.php b/tests/Support/TestBinariesHarness.php index e21381d5d..861959527 100644 --- a/tests/Support/TestBinariesHarness.php +++ b/tests/Support/TestBinariesHarness.php @@ -2,10 +2,10 @@ namespace Tests\Support; -use App\Services\BlacklistService; -use App\Services\XrefService; +use App\Services\Binaries\BinariesConfig; +use App\Services\Binaries\HeaderStorageService; +use App\Services\Binaries\MissedPartHandler; use Blacklight\Binaries; -use Blacklight\ColorCLI; use Illuminate\Support\Facades\DB; class TestBinariesHarness extends Binaries @@ -16,67 +16,100 @@ class TestBinariesHarness extends Binaries private int $flushCount = 0; - protected mixed $_collectionsCleaning; // override parent type + private array $testGroupMySQL = []; + + private HeaderStorageService $testHeaderStorage; + + private MissedPartHandler $testMissedPartHandler; + public function __construct() { - // Manually initialize only what storeHeaders/flushPartsChunk need; skip NNTP + Settings lookups. - $this->startUpdate = now(); - $this->timeCleaning = 0; - $this->_echoCLI = false; - $this->_pdo = DB::connection()->getPdo(); - $this->colorCli = new ColorCLI; - $this->_collectionsCleaning = new class - { - public function collectionsCleaner($subject, $groupName): array - { - return ['id' => 1, 'name' => 'COLL']; - } - }; - $this->xrefService = new XrefService; - $this->blacklistService = new BlacklistService; - $this->messageBuffer = 50000; - $this->_compressedHeaders = false; - $this->_partRepair = true; - $this->_newGroupScanByDays = false; - $this->_newGroupMessagesToScan = 50000; - $this->_newGroupDaysToScan = 3; - $this->_partRepairLimit = 15000; - $this->_partRepairMaxTries = 3; - $this->blackList = $this->whiteList = []; + // Create a minimal config that doesn't require database access + $config = new BinariesConfig( + messageBuffer: 50000, + compressedHeaders: false, + partRepair: true, + newGroupScanByDays: false, + newGroupMessagesToScan: 50000, + newGroupDaysToScan: 3, + partRepairLimit: 15000, + partRepairMaxTries: 3, + partsChunkSize: 5000, + binariesUpdateChunkSize: 1000, + echoCli: false + ); + + // Call parent with config (will still try to create NNTP, but we won't use it) + parent::__construct($config); + + // Override with test-specific services + $this->testHeaderStorage = new HeaderStorageService(config: $config); + $this->testMissedPartHandler = new MissedPartHandler( + $config->partRepairLimit, + $config->partRepairMaxTries + ); } - // Expose protected storeHeaders for direct testing. + // Expose protected method for direct testing via new service. public function publicStoreHeaders(array $headers): void { - if (empty($this->groupMySQL)) { - $this->groupMySQL = ['id' => 1, 'name' => 'alt.test']; + if (empty($this->testGroupMySQL)) { + $this->testGroupMySQL = ['id' => 1, 'name' => 'alt.test']; } - $this->startCleaning = now(); config(['tests.force_simulated_rollback' => false]); - $this->storeHeaders($headers); + + // Parse headers first to add 'matches' + $parsedHeaders = []; + foreach ($headers as $header) { + if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) { + if (stripos($header['Subject'], 'yEnc') === false) { + $matches[1] .= ' yEnc'; + } + $header['matches'] = $matches; + $parsedHeaders[] = $header; + } + } + + $this->testHeaderStorage->store($parsedHeaders, $this->testGroupMySQL, true); } public function setAddToPartRepair(bool $val): void { - $this->addToPartRepair = $val; + // This is now handled by passing parameter to store() } // Simulate scan path minimally to test rollback + part repair queue logic without NNTP. public function simulateScan(array $headers, array $group, bool $enablePartRepair = true): void { - $this->groupMySQL = $group; - $this->first = $headers[0]['Number']; - $this->last = end($headers)['Number']; - $this->headersReceived = array_column($headers, 'Number'); - $this->addToPartRepair = $enablePartRepair; - $this->startCleaning = now(); + $this->testGroupMySQL = $group; + + // Parse headers first to add 'matches' + $parsedHeaders = []; + $headersReceived = []; + foreach ($headers as $header) { + if (isset($header['Number'])) { + $headersReceived[] = $header['Number']; + } + if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) { + if (stripos($header['Subject'], 'yEnc') === false) { + $matches[1] .= ' yEnc'; + } + $header['matches'] = $matches; + $parsedHeaders[] = $header; + } + } // If we are simulating a failure, do not perform any inserts; just mark all as missed. if ($this->failPartsInsert) { if ($enablePartRepair) { - foreach (array_unique($this->headersReceived) as $num) { - DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1)', [$num, $group['id']]); + foreach (array_unique($headersReceived) as $num) { + $driver = DB::getDriverName(); + if ($driver === 'sqlite') { + DB::statement('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON CONFLICT(numberid, groups_id) DO UPDATE SET attempts = attempts + 1', [$num, $group['id']]); + } else { + DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON DUPLICATE KEY UPDATE attempts = attempts + 1', [$num, $group['id']]); + } } } @@ -84,28 +117,10 @@ class TestBinariesHarness extends Binaries } // Normal path: process and insert. - $this->storeHeaders($headers); + $failedInserts = $this->testHeaderStorage->store($parsedHeaders, $group, $enablePartRepair); - if ($enablePartRepair && ! empty($this->headersNotInserted)) { - foreach (array_unique($this->headersNotInserted) as $num) { - DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1)', [$num, $group['id']]); - } + if ($enablePartRepair && ! empty($failedInserts)) { + $this->testMissedPartHandler->addMissingParts($failedInserts, $group['id']); } } - - // Force chunk failure to trigger rollback when flag set. - protected function flushPartsChunk(array $parts): bool - { - $this->flushCount++; - if ($this->failPartsInsert) { - if ($this->failAfterFlushCount === null) { - return false; // Always fail when flag set and no threshold provided. - } - if ($this->flushCount > $this->failAfterFlushCount) { - return false; // Fail after N successful flushes - } - } - - return parent::flushPartsChunk($parts); - } }