headerNone(); return; } // Streaming mode if ((bool) config('nntmux.stream_fork_output', false) === true) { $commands = []; foreach ($work as $group) { $commands[] = PHP_BINARY.' misc/update/backfill.php '.$group->name.(isset($group->max) ? (' '.$group->max) : ''); } $this->runStreamingCommands($commands, $maxProcesses, 'backfill'); return; } $pool = $this->createPool($maxProcesses); $this->headerStart('backfill', $count, $maxProcesses); $taskNum = $count; foreach ($work as $group) { $pool->add(function () use ($group) { return $this->executeCommand(PHP_BINARY.' misc/update/backfill.php '.$group->name.(isset($group->max) ? (' '.$group->max) : '')); }, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($group, &$taskNum) { echo $output; $this->colorCli->primary('Task #'.$taskNum.' Backfilled group '.$group->name); $taskNum--; })->catch(function (\Throwable $exception) { echo $exception->getMessage(); })->catch(static function (SerializableException $serializableException) { // swallow }); } $pool->wait(); } public function safeBackfill(): void { // make sure short_groups is up-to-date $this->executeCommand(PHP_BINARY.' misc/update/tmux/bin/update_groups.php'); $backfill_qty = (int) Settings::settingValue('backfill_qty'); $backfill_order = (int) Settings::settingValue('backfill_order'); $backfill_days = (int) Settings::settingValue('backfill_days'); $maxMessages = (int) Settings::settingValue('maxmssgs'); $threads = (int) Settings::settingValue('backfillthreads'); $orderby = 'ORDER BY a.last_record ASC'; switch ($backfill_order) { case 1: $orderby = 'ORDER BY first_record_postdate DESC'; break; case 2: $orderby = 'ORDER BY first_record_postdate ASC'; break; case 3: $orderby = 'ORDER BY name ASC'; break; case 4: $orderby = 'ORDER BY name DESC'; break; case 5: $orderby = 'ORDER BY a.last_record DESC'; break; } $backfilldays = '0'; if ($backfill_days === 1) { $backfilldays = 'g.backfill_target'; } elseif ($backfill_days === 2) { $backfilldays = (string) now()->diffInDays(Carbon::createFromFormat('Y-m-d', Settings::settingValue('safebackfilldate')), true); } $sql = 'SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS their_first, MAX(a.last_record) AS their_last FROM usenet_groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '.$backfilldays.' DAY ) < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record '.$orderby.' LIMIT 1'; $data = DB::select($sql); $groupName = ''; $count = 0; if (! empty($data) && isset($data[0]->name)) { $groupName = $data[0]->name; $count = ($data[0]->our_first - $data[0]->their_first); } if ($count <= 0) { $this->headerNone(); if (config('nntmux.echocli') && $groupName !== '') { $this->colorCli->primary('No backfill needed for group '.$groupName); } return; } $getEach = ($count > ($backfill_qty * $threads)) ? (int) ceil(($backfill_qty * $threads) / $maxMessages) : (int) ceil($count / $maxMessages); $queues = []; for ($i = 0; $i <= $getEach - 1; $i++) { $queues[$i] = sprintf('get_range backfill %s %s %s %s', $groupName, $data[0]->our_first - $i * $maxMessages - $maxMessages, $data[0]->our_first - $i * $maxMessages - 1, $i + 1); } // Streaming mode if ((bool) config('nntmux.stream_fork_output', false) === true) { $commands = []; foreach ($queues as $queue) { $commands[] = $this->buildDnrCommand($queue); } $this->runStreamingCommands($commands, $threads, 'safe_backfill'); return; } $pool = $this->createPool($threads); $this->headerStart('safe_backfill', count($queues), $threads); foreach ($queues as $queue) { $pool->add(function () use ($queue) { return $this->executeCommand($this->buildDnrCommand($queue)); }, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($groupName) { echo $output; $this->colorCli->primary('Backfilled group '.$groupName); })->catch(function (\Throwable $exception) { echo $exception->getMessage(); })->catch(static function (SerializableException $serializableException) { // swallow }); } $pool->wait(); } }