diff --git a/Blacklight/Backfill.php b/Blacklight/Backfill.php deleted file mode 100755 index 9b42d31a4..000000000 --- a/Blacklight/Backfill.php +++ /dev/null @@ -1,307 +0,0 @@ -_echoCLI = config('nntmux.echocli'); - - $this->_nntp = new NNTP; - $this->_binaries = new BinariesService; - - $this->colorCli = new ColorCLI; - - $this->_compressedHeaders = config('nntmux_nntp.compressed_headers'); - $this->_safeBackFillDate = Settings::settingValue('safebackfilldate') !== '' ? (string) Settings::settingValue('safebackfilldate') : '2012-08-14'; - $this->_safePartRepair = (int) Settings::settingValue('safepartrepair') === 1 ? 'update' : 'backfill'; - $this->_disableBackfillGroup = (int) Settings::settingValue('disablebackfillgroup') === 1; - } - - /** - * @throws \Throwable - */ - public function backfillAllGroups(string $groupName = '', int|string $articles = '', string $type = ''): void - { - if ($groupName !== '') { - $grp[] = UsenetGroup::getByName($groupName); - } else { - $grp = UsenetGroup::getActiveBackfill($type); - } - - $groupCount = \count($grp); - if ($groupCount > 0) { - $counter = 1; - $allTime = now(); - $dMessage = ( - 'Backfilling: '. - $groupCount. - ' group(s) - Using compression? '. - ($this->_compressedHeaders ? 'Yes' : 'No') - ); - - if ($this->_echoCLI) { - $this->colorCli->header($dMessage); - } - - if ($articles !== '' && ! is_numeric($articles)) { - $articles = 20000; - } - - // Loop through groups. - foreach ($grp as $groupArr) { - if ($groupName === '') { - $dMessage = 'Starting group '.$counter.' of '.$groupCount; - - if ($this->_echoCLI) { - $this->colorCli->header($dMessage); - } - } - $this->backfillGroup($groupArr->toArray(), $groupCount - $counter, $articles); - $counter++; - } - - $dMessage = 'Backfilling completed in '.now()->diffInSeconds($allTime, true).' seconds.'; - - if ($this->_echoCLI) { - $this->colorCli->primary($dMessage); - } - } else { - $dMessage = 'No groups specified. Ensure groups are added to database for updating.'; - - if ($this->_echoCLI) { - $this->colorCli->warning($dMessage); - } - } - } - - /** - * Backfill single group. - * - * - * @throws \Throwable - */ - public function backfillGroup(array $groupArr, int $left, int|string $articles = ''): void - { - // Start time for this group. - $startGroup = now()->timestamp; - - $this->_binaries->logIndexerStart(); - - $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']); - - // If our local oldest article 0, it means we never ran update_binaries on the group. - if ($groupArr['first_record'] <= 0) { - $dMessage = - 'You need to run update_binaries on '. - $groupName. - '. Otherwise the group is dead, you must disable it.'; - - if ($this->_echoCLI) { - $this->colorCli->error($dMessage); - } - - return; - } - - // Select group, here, only once - $data = $this->_nntp->selectGroup($groupArr['name']); - if ($this->_nntp->isError($data)) { - $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']); - if ($this->_nntp->isError($data)) { - return; - } - } - - if ($this->_echoCLI) { - $this->colorCli->primary('Processing '.$groupName); - } - - // Check if this is days or post backfill. - $postCheck = $articles !== ''; - - // Get target post based on date or user specified number. - $targetpost = (string) ( - $postCheck - ? - round($groupArr['first_record'] - $articles) - : - $this->_binaries->daytopost($groupArr['backfill_target'], $data) - ); - - // Check if target post is smaller than server's oldest, set it to oldest if so. - if ($targetpost < $data['first']) { - $targetpost = $data['first']; - } - - // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest. - if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) { - $dMessage = - 'We have hit the maximum we can backfill for '. - $groupName. - ($this->_disableBackfillGroup ? ', disabling backfill on it.' : - ', skipping it, consider disabling backfill on it.'); - - if ($this->_disableBackfillGroup) { - UsenetGroup::updateGroupStatus($groupArr['id'], 'backfill', 0); - } - - if ($this->_echoCLI) { - $this->colorCli->notice($dMessage); - } - - return; - } - - if ($this->_echoCLI) { - $this->colorCli->primary( - 'Group '. - $groupName. - "'s oldest article is ". - number_format($data['first']). - ', newest is '. - number_format($data['last']). - '.Our target article is '. - number_format($targetpost). - '. Our oldest article is article '. - number_format($groupArr['first_record']). - '.' - ); - } - - // Set first and last, moving the window by max messages. - $last = ($groupArr['first_record'] - 1); - // Set the initial "chunk". - $first = ($last - $this->_binaries->getMessageBuffer() + 1); - - // Just in case this is the last chunk we needed. - if ($targetpost > $first) { - $first = $targetpost; - } - - $done = false; - while ($done === false) { - if ($this->_echoCLI) { - $this->colorCli->header('Getting '. - number_format($last - $first + 1). - ' articles from '. - $groupName. - ', '. - $left. - ' group(s) left. ('. - number_format($first - $targetpost). - ' articles in queue'); - } - - flush(); - $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair); - - // Get the oldest date. - if (isset($lastMsg['firstArticleDate'])) { - // Try to get it from the oldest pulled article. - $newdate = strtotime($lastMsg['firstArticleDate']); - } else { - // If above failed, try to get it with postdate method. - $newdate = $this->_binaries->postdate($first, $data); - } - - DB::update( - sprintf( - ' - UPDATE usenet_groups - SET first_record_postdate = FROM_UNIXTIME(%s), first_record = %s, last_updated = NOW() - WHERE id = %d', - $newdate, - escapeString($first), - $groupArr['id'] - ) - ); - if ($first === $targetpost) { - $done = true; - } else { - // Keep going: set new last, new first, check for last chunk. - $last = ($first - 1); - $first = ($last - $this->_binaries->getMessageBuffer() + 1); - if ($targetpost > $first) { - $first = $targetpost; - } - } - } - - if ($this->_echoCLI) { - $this->colorCli->primary( - PHP_EOL. - 'Group '. - $groupName. - ' processed in '. - number_format(now()->timestamp - $startGroup, 2). - ' seconds.' - ); - } - } - - /** - * @throws \Throwable - */ - public function safeBackfill(int|string $articles = ''): void - { - $groupName = UsenetGroup::query() - ->whereBetween('first_record_postdate', [Carbon::createFromDate($this->_safeBackFillDate), now()]) - ->where('backfill', '=', 1) - ->select(['name']) - ->orderBy('name') - ->first(); - - if ($groupName === null) { - $dMessage = - 'No groups to backfill, they are all at the target date '. - $this->_safeBackFillDate. - ', or you have not enabled them to be backfilled in the groups page.'.PHP_EOL; - exit($dMessage); - } - $this->backfillAllGroups($groupName['name'], $articles); - } -} diff --git a/app/Console/Commands/BackfillGroup.php b/app/Console/Commands/BackfillGroup.php index 28a4d393e..e6cb40211 100644 --- a/app/Console/Commands/BackfillGroup.php +++ b/app/Console/Commands/BackfillGroup.php @@ -3,7 +3,7 @@ namespace App\Console\Commands; use App\Models\Settings; -use Blacklight\Backfill; +use App\Services\Backfill\BackfillService; use Illuminate\Console\Command; use Illuminate\Support\Facades\Log; @@ -50,7 +50,7 @@ class BackfillGroup extends Command } $this->info("Backfilling group: {$group}"); - (new Backfill)->backfillAllGroups($group, $quantity); + (new BackfillService(nntp: $nntp))->backfillAllGroups($group, $quantity); return self::SUCCESS; } catch (\Throwable $e) { diff --git a/app/Console/Commands/UpdateBackfill.php b/app/Console/Commands/UpdateBackfill.php index 41e6542ff..fe93556c9 100644 --- a/app/Console/Commands/UpdateBackfill.php +++ b/app/Console/Commands/UpdateBackfill.php @@ -2,7 +2,7 @@ namespace App\Console\Commands; -use Blacklight\Backfill; +use App\Services\Backfill\BackfillService; use Blacklight\NNTP; use Illuminate\Console\Command; @@ -34,7 +34,7 @@ class UpdateBackfill extends Command try { $nntp = $this->getNntp(); - $backfill = new Backfill(['NNTP' => $nntp]); + $backfill = new BackfillService(nntp: $nntp); match (true) { $mode === 'all' && ! isset($quantity) => $backfill->backfillAllGroups(), diff --git a/app/Console/Commands/UpdatePerGroup.php b/app/Console/Commands/UpdatePerGroup.php index 0c2e40203..6b23e93eb 100644 --- a/app/Console/Commands/UpdatePerGroup.php +++ b/app/Console/Commands/UpdatePerGroup.php @@ -6,8 +6,8 @@ namespace App\Console\Commands; use App\Models\Settings; use App\Models\UsenetGroup; +use App\Services\Backfill\BackfillService; use App\Services\Binaries\BinariesService; -use Blacklight\Backfill; use Blacklight\Nfo; use Blacklight\NNTP; use Blacklight\processing\post\ProcessAdditional; @@ -55,7 +55,7 @@ class UpdatePerGroup extends Command $groupMySQL = $group->toArray(); $nntp = $this->getNntp(); - $backFill = new Backfill(); + $backfillService = new BackfillService(nntp: $nntp); // Update the group for new binaries $this->info("Updating binaries for group: {$groupMySQL['name']}"); @@ -63,7 +63,7 @@ class UpdatePerGroup extends Command // BackFill the group with 20k articles $this->info("Backfilling group: {$groupMySQL['name']}"); - $backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal'); + $backfillService->backfillAllGroups($groupMySQL['name'], 20000, 'normal'); // Create releases $this->info("Processing releases for group: {$groupMySQL['name']}"); diff --git a/app/Services/Backfill/BackfillConfig.php b/app/Services/Backfill/BackfillConfig.php new file mode 100644 index 000000000..c7fae12bd --- /dev/null +++ b/app/Services/Backfill/BackfillConfig.php @@ -0,0 +1,51 @@ +config = $config ?? BackfillConfig::fromSettings(); + $this->binaries = $binaries ?? new BinariesService; + $this->nntp = $nntp ?? new NNTP; + $this->colorCli = $colorCli ?? new ColorCLI; + } + + /** + * Backfill all groups or a specific group. + * + * @param string $groupName Optional specific group to backfill + * @param int|string $articles Number of articles to backfill, or empty for date-based + * @param string $type Backfill type filter + * + * @throws \Throwable + */ + public function backfillAllGroups(string $groupName = '', int|string $articles = '', string $type = ''): void + { + $groups = $this->getGroupsToBackfill($groupName, $type); + + if ($groups === []) { + $this->log('No groups specified. Ensure groups are added to database for updating.', 'warning'); + + return; + } + + $groupCount = \count($groups); + $this->logBackfillStart($groupCount); + + $articles = $this->normalizeArticleCount($articles); + $startTime = now(); + + foreach ($groups as $index => $group) { + $this->logGroupProgress($groupName, $index + 1, $groupCount); + $this->backfillGroup($group->toArray(), $groupCount - $index - 1, $articles); + } + + $this->logBackfillComplete($startTime); + } + + /** + * Backfill a single group. + * + * @param array $groupArr Group data array + * @param int $remainingGroups Number of groups remaining after this one + * @param int|string $articles Number of articles to backfill, or empty for date-based + * + * @throws \Throwable + */ + public function backfillGroup(array $groupArr, int $remainingGroups, int|string $articles = ''): void + { + $startTime = now(); + $this->binaries->logIndexerStart(); + + $shortGroupName = $this->getShortGroupName($groupArr['name']); + + if (! $this->validateGroupState($groupArr, $shortGroupName)) { + return; + } + + $serverData = $this->selectNntpGroup($groupArr['name']); + if ($serverData === null) { + return; + } + + $this->log("Processing {$shortGroupName}", 'primary'); + + $targetPost = $this->calculateTargetPost($groupArr, $articles, $serverData); + + if (! $this->validateTargetPost($groupArr, $targetPost, $serverData, $shortGroupName)) { + return; + } + + $this->logGroupInfo($groupArr, $serverData, $targetPost, $shortGroupName); + + $this->processBackfillChunks($groupArr, $targetPost, $remainingGroups, $shortGroupName); + + $this->logGroupComplete($shortGroupName, $startTime); + } + + /** + * Safe backfill - backfill groups that haven't reached the safe backfill date. + * + * @param int|string $articles Number of articles to backfill + * + * @throws \Throwable + */ + public function safeBackfill(int|string $articles = ''): void + { + $group = UsenetGroup::query() + ->whereBetween('first_record_postdate', [Carbon::createFromDate($this->config->safeBackFillDate), now()]) + ->where('backfill', '=', 1) + ->select(['name']) + ->orderBy('name') + ->first(); + + if ($group === null) { + $message = sprintf( + 'No groups to backfill, they are all at the target date %s, or you have not enabled them to be backfilled in the groups page.', + $this->config->safeBackFillDate + ); + exit($message.PHP_EOL); + } + + $this->backfillAllGroups($group->name, $articles); + } + + /** + * Get groups to backfill based on criteria. + */ + private function getGroupsToBackfill(string $groupName, string $type): array + { + if ($groupName !== '') { + $group = UsenetGroup::getByName($groupName); + + return $group ? [$group] : []; + } + + return UsenetGroup::getActiveBackfill($type)->all(); + } + + /** + * Normalize article count parameter. + */ + private function normalizeArticleCount(int|string $articles): int|string + { + if ($articles !== '' && ! is_numeric($articles)) { + return self::DEFAULT_ARTICLE_COUNT; + } + + return $articles; + } + + /** + * Get shortened group name for display. + */ + private function getShortGroupName(string $groupName): string + { + return str_replace('alt.binaries', 'a.b', $groupName); + } + + /** + * Validate that group is in a valid state for backfilling. + */ + private function validateGroupState(array $groupArr, string $shortGroupName): bool + { + if ($groupArr['first_record'] <= 0) { + $this->log( + "You need to run update_binaries on {$shortGroupName}. Otherwise the group is dead, you must disable it.", + 'error' + ); + + return false; + } + + return true; + } + + /** + * Select NNTP group and return server data. + */ + private function selectNntpGroup(string $groupName): ?array + { + $data = $this->nntp->selectGroup($groupName); + + if ($this->nntp->isError($data)) { + $data = $this->nntp->dataError($this->nntp, $groupName); + if ($this->nntp->isError($data)) { + return null; + } + } + + return $data; + } + + /** + * Calculate target post number based on articles count or date. + */ + private function calculateTargetPost(array $groupArr, int|string $articles, array $serverData): int + { + $isArticleBased = $articles !== ''; + + $targetPost = $isArticleBased + ? (int) round($groupArr['first_record'] - (int) $articles) + : $this->binaries->daytopost($groupArr['backfill_target'], $serverData); + + // Ensure target is not below server's oldest article + return max($targetPost, $serverData['first']); + } + + /** + * Validate that target post is achievable. + */ + private function validateTargetPost(array $groupArr, int $targetPost, array $serverData, string $shortGroupName): bool + { + if ($targetPost >= $groupArr['first_record'] || $groupArr['first_record'] <= $serverData['first']) { + $message = "We have hit the maximum we can backfill for {$shortGroupName}"; + $message .= $this->config->disableBackfillGroup + ? ', disabling backfill on it.' + : ', skipping it, consider disabling backfill on it.'; + + if ($this->config->disableBackfillGroup) { + UsenetGroup::updateGroupStatus($groupArr['id'], 'backfill', 0); + } + + $this->log($message, 'notice'); + + return false; + } + + return true; + } + + /** + * Process backfill in chunks. + */ + private function processBackfillChunks(array $groupArr, int $targetPost, int $remainingGroups, string $shortGroupName): void + { + $messageBuffer = $this->binaries->getMessageBuffer(); + $last = $groupArr['first_record'] - 1; + $first = max($last - $messageBuffer + 1, $targetPost); + + while (true) { + $this->logChunkProgress($first, $last, $shortGroupName, $remainingGroups, $targetPost); + + flush(); + $scanResult = $this->binaries->scan($groupArr, $first, $last, $this->config->safePartRepair); + + $this->updateGroupRecord($groupArr, $first, $scanResult); + + if ($first === $targetPost) { + break; + } + + // Move to next chunk + $last = $first - 1; + $first = max($last - $messageBuffer + 1, $targetPost); + } + } + + /** + * Update group record with new first_record and postdate. + */ + private function updateGroupRecord(array $groupArr, int $first, ?array $scanResult): void + { + $newDate = isset($scanResult['firstArticleDate']) + ? strtotime($scanResult['firstArticleDate']) + : $this->binaries->postdate($first, $this->nntp->selectGroup($groupArr['name'])); + + DB::update( + 'UPDATE usenet_groups SET first_record_postdate = FROM_UNIXTIME(?), first_record = ?, last_updated = NOW() WHERE id = ?', + [$newDate, $first, $groupArr['id']] + ); + } + + /** + * Log message with appropriate styling. + */ + private function log(string $message, string $type = 'primary'): void + { + if (! $this->config->echoCli) { + return; + } + + match ($type) { + 'header' => $this->colorCli->header($message), + 'warning' => $this->colorCli->warning($message), + 'error' => $this->colorCli->error($message), + 'notice' => $this->colorCli->notice($message), + default => $this->colorCli->primary($message), + }; + } + + /** + * Log backfill start information. + */ + private function logBackfillStart(int $groupCount): void + { + $compressionStatus = $this->config->compressedHeaders ? 'Yes' : 'No'; + $this->log("Backfilling: {$groupCount} group(s) - Using compression? {$compressionStatus}", 'header'); + } + + /** + * Log group progress. + */ + private function logGroupProgress(string $groupName, int $current, int $total): void + { + if ($groupName === '') { + $this->log("Starting group {$current} of {$total}", 'header'); + } + } + + /** + * Log backfill completion. + */ + private function logBackfillComplete(\Illuminate\Support\Carbon $startTime): void + { + $duration = now()->diffInSeconds($startTime, true); + $this->log("Backfilling completed in {$duration} seconds."); + } + + /** + * Log group info before processing. + */ + private function logGroupInfo(array $groupArr, array $serverData, int $targetPost, string $shortGroupName): void + { + $this->log(sprintf( + "Group %s's oldest article is %s, newest is %s. Our target article is %s. Our oldest article is article %s.", + $shortGroupName, + number_format($serverData['first']), + number_format($serverData['last']), + number_format($targetPost), + number_format($groupArr['first_record']) + )); + } + + /** + * Log chunk progress. + */ + private function logChunkProgress(int $first, int $last, string $shortGroupName, int $remainingGroups, int $targetPost): void + { + $this->log(sprintf( + 'Getting %s articles from %s, %d group(s) left. (%s articles in queue)', + number_format($last - $first + 1), + $shortGroupName, + $remainingGroups, + number_format($first - $targetPost) + ), 'header'); + } + + /** + * Log group completion. + */ + private function logGroupComplete(string $shortGroupName, \Illuminate\Support\Carbon $startTime): void + { + $duration = number_format(now()->timestamp - $startTime->timestamp, 2); + $this->log(PHP_EOL."Group {$shortGroupName} processed in {$duration} seconds."); + } +} +