mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Backfill is a service now
This commit is contained in:
DariusIII
@@ -1,307 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Blacklight;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\Binaries\BinariesService;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
/**
|
||||
* Class Backfill.
|
||||
*/
|
||||
class Backfill
|
||||
{
|
||||
protected BinariesService $_binaries;
|
||||
|
||||
protected NNTP $_nntp;
|
||||
|
||||
/**
|
||||
* Should we use compression for headers?
|
||||
*/
|
||||
protected bool $_compressedHeaders;
|
||||
|
||||
/**
|
||||
* Log and or echo debug.
|
||||
*/
|
||||
protected bool $_debug = false;
|
||||
|
||||
/**
|
||||
* Echo to cli?
|
||||
*/
|
||||
protected bool $_echoCLI;
|
||||
|
||||
/**
|
||||
* How far back should we go on safe back fill?
|
||||
*/
|
||||
protected string $_safeBackFillDate;
|
||||
|
||||
protected string $_safePartRepair;
|
||||
|
||||
/**
|
||||
* Should we disable the group if we have backfilled far enough?
|
||||
*/
|
||||
protected bool $_disableBackfillGroup;
|
||||
|
||||
protected ColorCLI $colorCli;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->_echoCLI = config('nntmux.echocli');
|
||||
|
||||
$this->_nntp = new NNTP;
|
||||
$this->_binaries = new BinariesService;
|
||||
|
||||
$this->colorCli = new ColorCLI;
|
||||
|
||||
$this->_compressedHeaders = config('nntmux_nntp.compressed_headers');
|
||||
$this->_safeBackFillDate = Settings::settingValue('safebackfilldate') !== '' ? (string) Settings::settingValue('safebackfilldate') : '2012-08-14';
|
||||
$this->_safePartRepair = (int) Settings::settingValue('safepartrepair') === 1 ? 'update' : 'backfill';
|
||||
$this->_disableBackfillGroup = (int) Settings::settingValue('disablebackfillgroup') === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function backfillAllGroups(string $groupName = '', int|string $articles = '', string $type = ''): void
|
||||
{
|
||||
if ($groupName !== '') {
|
||||
$grp[] = UsenetGroup::getByName($groupName);
|
||||
} else {
|
||||
$grp = UsenetGroup::getActiveBackfill($type);
|
||||
}
|
||||
|
||||
$groupCount = \count($grp);
|
||||
if ($groupCount > 0) {
|
||||
$counter = 1;
|
||||
$allTime = now();
|
||||
$dMessage = (
|
||||
'Backfilling: '.
|
||||
$groupCount.
|
||||
' group(s) - Using compression? '.
|
||||
($this->_compressedHeaders ? 'Yes' : 'No')
|
||||
);
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->header($dMessage);
|
||||
}
|
||||
|
||||
if ($articles !== '' && ! is_numeric($articles)) {
|
||||
$articles = 20000;
|
||||
}
|
||||
|
||||
// Loop through groups.
|
||||
foreach ($grp as $groupArr) {
|
||||
if ($groupName === '') {
|
||||
$dMessage = 'Starting group '.$counter.' of '.$groupCount;
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->header($dMessage);
|
||||
}
|
||||
}
|
||||
$this->backfillGroup($groupArr->toArray(), $groupCount - $counter, $articles);
|
||||
$counter++;
|
||||
}
|
||||
|
||||
$dMessage = 'Backfilling completed in '.now()->diffInSeconds($allTime, true).' seconds.';
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary($dMessage);
|
||||
}
|
||||
} else {
|
||||
$dMessage = 'No groups specified. Ensure groups are added to database for updating.';
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->warning($dMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill single group.
|
||||
*
|
||||
*
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function backfillGroup(array $groupArr, int $left, int|string $articles = ''): void
|
||||
{
|
||||
// Start time for this group.
|
||||
$startGroup = now()->timestamp;
|
||||
|
||||
$this->_binaries->logIndexerStart();
|
||||
|
||||
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
|
||||
|
||||
// If our local oldest article 0, it means we never ran update_binaries on the group.
|
||||
if ($groupArr['first_record'] <= 0) {
|
||||
$dMessage =
|
||||
'You need to run update_binaries on '.
|
||||
$groupName.
|
||||
'. Otherwise the group is dead, you must disable it.';
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->error($dMessage);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Select group, here, only once
|
||||
$data = $this->_nntp->selectGroup($groupArr['name']);
|
||||
if ($this->_nntp->isError($data)) {
|
||||
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
|
||||
if ($this->_nntp->isError($data)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary('Processing '.$groupName);
|
||||
}
|
||||
|
||||
// Check if this is days or post backfill.
|
||||
$postCheck = $articles !== '';
|
||||
|
||||
// Get target post based on date or user specified number.
|
||||
$targetpost = (string) (
|
||||
$postCheck
|
||||
?
|
||||
round($groupArr['first_record'] - $articles)
|
||||
:
|
||||
$this->_binaries->daytopost($groupArr['backfill_target'], $data)
|
||||
);
|
||||
|
||||
// Check if target post is smaller than server's oldest, set it to oldest if so.
|
||||
if ($targetpost < $data['first']) {
|
||||
$targetpost = $data['first'];
|
||||
}
|
||||
|
||||
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
|
||||
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
|
||||
$dMessage =
|
||||
'We have hit the maximum we can backfill for '.
|
||||
$groupName.
|
||||
($this->_disableBackfillGroup ? ', disabling backfill on it.' :
|
||||
', skipping it, consider disabling backfill on it.');
|
||||
|
||||
if ($this->_disableBackfillGroup) {
|
||||
UsenetGroup::updateGroupStatus($groupArr['id'], 'backfill', 0);
|
||||
}
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->notice($dMessage);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
'Group '.
|
||||
$groupName.
|
||||
"'s oldest article is ".
|
||||
number_format($data['first']).
|
||||
', newest is '.
|
||||
number_format($data['last']).
|
||||
'.Our target article is '.
|
||||
number_format($targetpost).
|
||||
'. Our oldest article is article '.
|
||||
number_format($groupArr['first_record']).
|
||||
'.'
|
||||
);
|
||||
}
|
||||
|
||||
// Set first and last, moving the window by max messages.
|
||||
$last = ($groupArr['first_record'] - 1);
|
||||
// Set the initial "chunk".
|
||||
$first = ($last - $this->_binaries->getMessageBuffer() + 1);
|
||||
|
||||
// Just in case this is the last chunk we needed.
|
||||
if ($targetpost > $first) {
|
||||
$first = $targetpost;
|
||||
}
|
||||
|
||||
$done = false;
|
||||
while ($done === false) {
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->header('Getting '.
|
||||
number_format($last - $first + 1).
|
||||
' articles from '.
|
||||
$groupName.
|
||||
', '.
|
||||
$left.
|
||||
' group(s) left. ('.
|
||||
number_format($first - $targetpost).
|
||||
' articles in queue');
|
||||
}
|
||||
|
||||
flush();
|
||||
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
|
||||
|
||||
// Get the oldest date.
|
||||
if (isset($lastMsg['firstArticleDate'])) {
|
||||
// Try to get it from the oldest pulled article.
|
||||
$newdate = strtotime($lastMsg['firstArticleDate']);
|
||||
} else {
|
||||
// If above failed, try to get it with postdate method.
|
||||
$newdate = $this->_binaries->postdate($first, $data);
|
||||
}
|
||||
|
||||
DB::update(
|
||||
sprintf(
|
||||
'
|
||||
UPDATE usenet_groups
|
||||
SET first_record_postdate = FROM_UNIXTIME(%s), first_record = %s, last_updated = NOW()
|
||||
WHERE id = %d',
|
||||
$newdate,
|
||||
escapeString($first),
|
||||
$groupArr['id']
|
||||
)
|
||||
);
|
||||
if ($first === $targetpost) {
|
||||
$done = true;
|
||||
} else {
|
||||
// Keep going: set new last, new first, check for last chunk.
|
||||
$last = ($first - 1);
|
||||
$first = ($last - $this->_binaries->getMessageBuffer() + 1);
|
||||
if ($targetpost > $first) {
|
||||
$first = $targetpost;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
PHP_EOL.
|
||||
'Group '.
|
||||
$groupName.
|
||||
' processed in '.
|
||||
number_format(now()->timestamp - $startGroup, 2).
|
||||
' seconds.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function safeBackfill(int|string $articles = ''): void
|
||||
{
|
||||
$groupName = UsenetGroup::query()
|
||||
->whereBetween('first_record_postdate', [Carbon::createFromDate($this->_safeBackFillDate), now()])
|
||||
->where('backfill', '=', 1)
|
||||
->select(['name'])
|
||||
->orderBy('name')
|
||||
->first();
|
||||
|
||||
if ($groupName === null) {
|
||||
$dMessage =
|
||||
'No groups to backfill, they are all at the target date '.
|
||||
$this->_safeBackFillDate.
|
||||
', or you have not enabled them to be backfilled in the groups page.'.PHP_EOL;
|
||||
exit($dMessage);
|
||||
}
|
||||
$this->backfillAllGroups($groupName['name'], $articles);
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Settings;
|
||||
use Blacklight\Backfill;
|
||||
use App\Services\Backfill\BackfillService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
@@ -50,7 +50,7 @@ class BackfillGroup extends Command
|
||||
}
|
||||
|
||||
$this->info("Backfilling group: {$group}");
|
||||
(new Backfill)->backfillAllGroups($group, $quantity);
|
||||
(new BackfillService(nntp: $nntp))->backfillAllGroups($group, $quantity);
|
||||
|
||||
return self::SUCCESS;
|
||||
} catch (\Throwable $e) {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Blacklight\Backfill;
|
||||
use App\Services\Backfill\BackfillService;
|
||||
use Blacklight\NNTP;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
@@ -34,7 +34,7 @@ class UpdateBackfill extends Command
|
||||
|
||||
try {
|
||||
$nntp = $this->getNntp();
|
||||
$backfill = new Backfill(['NNTP' => $nntp]);
|
||||
$backfill = new BackfillService(nntp: $nntp);
|
||||
|
||||
match (true) {
|
||||
$mode === 'all' && ! isset($quantity) => $backfill->backfillAllGroups(),
|
||||
|
||||
@@ -6,8 +6,8 @@ namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\Backfill\BackfillService;
|
||||
use App\Services\Binaries\BinariesService;
|
||||
use Blacklight\Backfill;
|
||||
use Blacklight\Nfo;
|
||||
use Blacklight\NNTP;
|
||||
use Blacklight\processing\post\ProcessAdditional;
|
||||
@@ -55,7 +55,7 @@ class UpdatePerGroup extends Command
|
||||
|
||||
$groupMySQL = $group->toArray();
|
||||
$nntp = $this->getNntp();
|
||||
$backFill = new Backfill();
|
||||
$backfillService = new BackfillService(nntp: $nntp);
|
||||
|
||||
// Update the group for new binaries
|
||||
$this->info("Updating binaries for group: {$groupMySQL['name']}");
|
||||
@@ -63,7 +63,7 @@ class UpdatePerGroup extends Command
|
||||
|
||||
// BackFill the group with 20k articles
|
||||
$this->info("Backfilling group: {$groupMySQL['name']}");
|
||||
$backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal');
|
||||
$backfillService->backfillAllGroups($groupMySQL['name'], 20000, 'normal');
|
||||
|
||||
// Create releases
|
||||
$this->info("Processing releases for group: {$groupMySQL['name']}");
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Backfill;
|
||||
|
||||
use App\Models\Settings;
|
||||
|
||||
/**
|
||||
* Configuration DTO for Backfill processing.
|
||||
* Encapsulates all settings in an immutable object for easier testing and injection.
|
||||
*/
|
||||
final readonly class BackfillConfig
|
||||
{
|
||||
public function __construct(
|
||||
public bool $compressedHeaders = true,
|
||||
public bool $echoCli = false,
|
||||
public string $safeBackFillDate = '2012-08-14',
|
||||
public string $safePartRepair = 'backfill',
|
||||
public bool $disableBackfillGroup = false,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Create configuration from application settings.
|
||||
*/
|
||||
public static function fromSettings(): self
|
||||
{
|
||||
return new self(
|
||||
compressedHeaders: (bool) config('nntmux_nntp.compressed_headers'),
|
||||
echoCli: (bool) config('nntmux.echocli'),
|
||||
safeBackFillDate: self::getSettingString('safebackfilldate', '2012-08-14'),
|
||||
safePartRepair: self::getSettingInt('safepartrepair', 0) === 1 ? 'update' : 'backfill',
|
||||
disableBackfillGroup: self::getSettingInt('disablebackfillgroup', 0) === 1,
|
||||
);
|
||||
}
|
||||
|
||||
private static function getSettingString(string $key, string $default): string
|
||||
{
|
||||
$value = Settings::settingValue($key);
|
||||
|
||||
return $value !== '' ? (string) $value : $default;
|
||||
}
|
||||
|
||||
private static function getSettingInt(string $key, int $default): int
|
||||
{
|
||||
$value = Settings::settingValue($key);
|
||||
|
||||
return $value !== '' ? (int) $value : $default;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Backfill;
|
||||
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\Binaries\BinariesService;
|
||||
use Blacklight\ColorCLI;
|
||||
use Blacklight\NNTP;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
/**
|
||||
* Service for backfilling Usenet groups with historical articles.
|
||||
*
|
||||
* This service handles downloading older articles from Usenet groups
|
||||
* to fill in historical data. It supports:
|
||||
* - Backfilling by article count or target date
|
||||
* - Safe backfill with date-based targeting
|
||||
* - Automatic group disable when backfill limit is reached
|
||||
*/
|
||||
final class BackfillService
|
||||
{
|
||||
private const DEFAULT_ARTICLE_COUNT = 20000;
|
||||
|
||||
private BackfillConfig $config;
|
||||
|
||||
private BinariesService $binaries;
|
||||
|
||||
private NNTP $nntp;
|
||||
|
||||
private ColorCLI $colorCli;
|
||||
|
||||
public function __construct(
|
||||
?BackfillConfig $config = null,
|
||||
?BinariesService $binaries = null,
|
||||
?NNTP $nntp = null,
|
||||
?ColorCLI $colorCli = null,
|
||||
) {
|
||||
$this->config = $config ?? BackfillConfig::fromSettings();
|
||||
$this->binaries = $binaries ?? new BinariesService;
|
||||
$this->nntp = $nntp ?? new NNTP;
|
||||
$this->colorCli = $colorCli ?? new ColorCLI;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill all groups or a specific group.
|
||||
*
|
||||
* @param string $groupName Optional specific group to backfill
|
||||
* @param int|string $articles Number of articles to backfill, or empty for date-based
|
||||
* @param string $type Backfill type filter
|
||||
*
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function backfillAllGroups(string $groupName = '', int|string $articles = '', string $type = ''): void
|
||||
{
|
||||
$groups = $this->getGroupsToBackfill($groupName, $type);
|
||||
|
||||
if ($groups === []) {
|
||||
$this->log('No groups specified. Ensure groups are added to database for updating.', 'warning');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$groupCount = \count($groups);
|
||||
$this->logBackfillStart($groupCount);
|
||||
|
||||
$articles = $this->normalizeArticleCount($articles);
|
||||
$startTime = now();
|
||||
|
||||
foreach ($groups as $index => $group) {
|
||||
$this->logGroupProgress($groupName, $index + 1, $groupCount);
|
||||
$this->backfillGroup($group->toArray(), $groupCount - $index - 1, $articles);
|
||||
}
|
||||
|
||||
$this->logBackfillComplete($startTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill a single group.
|
||||
*
|
||||
* @param array $groupArr Group data array
|
||||
* @param int $remainingGroups Number of groups remaining after this one
|
||||
* @param int|string $articles Number of articles to backfill, or empty for date-based
|
||||
*
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function backfillGroup(array $groupArr, int $remainingGroups, int|string $articles = ''): void
|
||||
{
|
||||
$startTime = now();
|
||||
$this->binaries->logIndexerStart();
|
||||
|
||||
$shortGroupName = $this->getShortGroupName($groupArr['name']);
|
||||
|
||||
if (! $this->validateGroupState($groupArr, $shortGroupName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$serverData = $this->selectNntpGroup($groupArr['name']);
|
||||
if ($serverData === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->log("Processing {$shortGroupName}", 'primary');
|
||||
|
||||
$targetPost = $this->calculateTargetPost($groupArr, $articles, $serverData);
|
||||
|
||||
if (! $this->validateTargetPost($groupArr, $targetPost, $serverData, $shortGroupName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->logGroupInfo($groupArr, $serverData, $targetPost, $shortGroupName);
|
||||
|
||||
$this->processBackfillChunks($groupArr, $targetPost, $remainingGroups, $shortGroupName);
|
||||
|
||||
$this->logGroupComplete($shortGroupName, $startTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Safe backfill - backfill groups that haven't reached the safe backfill date.
|
||||
*
|
||||
* @param int|string $articles Number of articles to backfill
|
||||
*
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function safeBackfill(int|string $articles = ''): void
|
||||
{
|
||||
$group = UsenetGroup::query()
|
||||
->whereBetween('first_record_postdate', [Carbon::createFromDate($this->config->safeBackFillDate), now()])
|
||||
->where('backfill', '=', 1)
|
||||
->select(['name'])
|
||||
->orderBy('name')
|
||||
->first();
|
||||
|
||||
if ($group === null) {
|
||||
$message = sprintf(
|
||||
'No groups to backfill, they are all at the target date %s, or you have not enabled them to be backfilled in the groups page.',
|
||||
$this->config->safeBackFillDate
|
||||
);
|
||||
exit($message.PHP_EOL);
|
||||
}
|
||||
|
||||
$this->backfillAllGroups($group->name, $articles);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get groups to backfill based on criteria.
|
||||
*/
|
||||
private function getGroupsToBackfill(string $groupName, string $type): array
|
||||
{
|
||||
if ($groupName !== '') {
|
||||
$group = UsenetGroup::getByName($groupName);
|
||||
|
||||
return $group ? [$group] : [];
|
||||
}
|
||||
|
||||
return UsenetGroup::getActiveBackfill($type)->all();
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize article count parameter.
|
||||
*/
|
||||
private function normalizeArticleCount(int|string $articles): int|string
|
||||
{
|
||||
if ($articles !== '' && ! is_numeric($articles)) {
|
||||
return self::DEFAULT_ARTICLE_COUNT;
|
||||
}
|
||||
|
||||
return $articles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get shortened group name for display.
|
||||
*/
|
||||
private function getShortGroupName(string $groupName): string
|
||||
{
|
||||
return str_replace('alt.binaries', 'a.b', $groupName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that group is in a valid state for backfilling.
|
||||
*/
|
||||
private function validateGroupState(array $groupArr, string $shortGroupName): bool
|
||||
{
|
||||
if ($groupArr['first_record'] <= 0) {
|
||||
$this->log(
|
||||
"You need to run update_binaries on {$shortGroupName}. Otherwise the group is dead, you must disable it.",
|
||||
'error'
|
||||
);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select NNTP group and return server data.
|
||||
*/
|
||||
private function selectNntpGroup(string $groupName): ?array
|
||||
{
|
||||
$data = $this->nntp->selectGroup($groupName);
|
||||
|
||||
if ($this->nntp->isError($data)) {
|
||||
$data = $this->nntp->dataError($this->nntp, $groupName);
|
||||
if ($this->nntp->isError($data)) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate target post number based on articles count or date.
|
||||
*/
|
||||
private function calculateTargetPost(array $groupArr, int|string $articles, array $serverData): int
|
||||
{
|
||||
$isArticleBased = $articles !== '';
|
||||
|
||||
$targetPost = $isArticleBased
|
||||
? (int) round($groupArr['first_record'] - (int) $articles)
|
||||
: $this->binaries->daytopost($groupArr['backfill_target'], $serverData);
|
||||
|
||||
// Ensure target is not below server's oldest article
|
||||
return max($targetPost, $serverData['first']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that target post is achievable.
|
||||
*/
|
||||
private function validateTargetPost(array $groupArr, int $targetPost, array $serverData, string $shortGroupName): bool
|
||||
{
|
||||
if ($targetPost >= $groupArr['first_record'] || $groupArr['first_record'] <= $serverData['first']) {
|
||||
$message = "We have hit the maximum we can backfill for {$shortGroupName}";
|
||||
$message .= $this->config->disableBackfillGroup
|
||||
? ', disabling backfill on it.'
|
||||
: ', skipping it, consider disabling backfill on it.';
|
||||
|
||||
if ($this->config->disableBackfillGroup) {
|
||||
UsenetGroup::updateGroupStatus($groupArr['id'], 'backfill', 0);
|
||||
}
|
||||
|
||||
$this->log($message, 'notice');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process backfill in chunks.
|
||||
*/
|
||||
private function processBackfillChunks(array $groupArr, int $targetPost, int $remainingGroups, string $shortGroupName): void
|
||||
{
|
||||
$messageBuffer = $this->binaries->getMessageBuffer();
|
||||
$last = $groupArr['first_record'] - 1;
|
||||
$first = max($last - $messageBuffer + 1, $targetPost);
|
||||
|
||||
while (true) {
|
||||
$this->logChunkProgress($first, $last, $shortGroupName, $remainingGroups, $targetPost);
|
||||
|
||||
flush();
|
||||
$scanResult = $this->binaries->scan($groupArr, $first, $last, $this->config->safePartRepair);
|
||||
|
||||
$this->updateGroupRecord($groupArr, $first, $scanResult);
|
||||
|
||||
if ($first === $targetPost) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Move to next chunk
|
||||
$last = $first - 1;
|
||||
$first = max($last - $messageBuffer + 1, $targetPost);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update group record with new first_record and postdate.
|
||||
*/
|
||||
private function updateGroupRecord(array $groupArr, int $first, ?array $scanResult): void
|
||||
{
|
||||
$newDate = isset($scanResult['firstArticleDate'])
|
||||
? strtotime($scanResult['firstArticleDate'])
|
||||
: $this->binaries->postdate($first, $this->nntp->selectGroup($groupArr['name']));
|
||||
|
||||
DB::update(
|
||||
'UPDATE usenet_groups SET first_record_postdate = FROM_UNIXTIME(?), first_record = ?, last_updated = NOW() WHERE id = ?',
|
||||
[$newDate, $first, $groupArr['id']]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log message with appropriate styling.
|
||||
*/
|
||||
private function log(string $message, string $type = 'primary'): void
|
||||
{
|
||||
if (! $this->config->echoCli) {
|
||||
return;
|
||||
}
|
||||
|
||||
match ($type) {
|
||||
'header' => $this->colorCli->header($message),
|
||||
'warning' => $this->colorCli->warning($message),
|
||||
'error' => $this->colorCli->error($message),
|
||||
'notice' => $this->colorCli->notice($message),
|
||||
default => $this->colorCli->primary($message),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Log backfill start information.
|
||||
*/
|
||||
private function logBackfillStart(int $groupCount): void
|
||||
{
|
||||
$compressionStatus = $this->config->compressedHeaders ? 'Yes' : 'No';
|
||||
$this->log("Backfilling: {$groupCount} group(s) - Using compression? {$compressionStatus}", 'header');
|
||||
}
|
||||
|
||||
/**
|
||||
* Log group progress.
|
||||
*/
|
||||
private function logGroupProgress(string $groupName, int $current, int $total): void
|
||||
{
|
||||
if ($groupName === '') {
|
||||
$this->log("Starting group {$current} of {$total}", 'header');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log backfill completion.
|
||||
*/
|
||||
private function logBackfillComplete(\Illuminate\Support\Carbon $startTime): void
|
||||
{
|
||||
$duration = now()->diffInSeconds($startTime, true);
|
||||
$this->log("Backfilling completed in {$duration} seconds.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Log group info before processing.
|
||||
*/
|
||||
private function logGroupInfo(array $groupArr, array $serverData, int $targetPost, string $shortGroupName): void
|
||||
{
|
||||
$this->log(sprintf(
|
||||
"Group %s's oldest article is %s, newest is %s. Our target article is %s. Our oldest article is article %s.",
|
||||
$shortGroupName,
|
||||
number_format($serverData['first']),
|
||||
number_format($serverData['last']),
|
||||
number_format($targetPost),
|
||||
number_format($groupArr['first_record'])
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Log chunk progress.
|
||||
*/
|
||||
private function logChunkProgress(int $first, int $last, string $shortGroupName, int $remainingGroups, int $targetPost): void
|
||||
{
|
||||
$this->log(sprintf(
|
||||
'Getting %s articles from %s, %d group(s) left. (%s articles in queue)',
|
||||
number_format($last - $first + 1),
|
||||
$shortGroupName,
|
||||
$remainingGroups,
|
||||
number_format($first - $targetPost)
|
||||
), 'header');
|
||||
}
|
||||
|
||||
/**
|
||||
* Log group completion.
|
||||
*/
|
||||
private function logGroupComplete(string $shortGroupName, \Illuminate\Support\Carbon $startTime): void
|
||||
{
|
||||
$duration = number_format(now()->timestamp - $startTime->timestamp, 2);
|
||||
$this->log(PHP_EOL."Group {$shortGroupName} processed in {$duration} seconds.");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user