This commit is contained in:
DariusIII
2025-08-15 11:45:51 +02:00
parent 607229a1b1
commit 2a8282f6ac
8 changed files with 48 additions and 22 deletions
+17 -7
View File
@@ -3,19 +3,14 @@
namespace Blacklight\libraries;
use App\Models\Settings;
use App\Models\UsenetGroup;
use Blacklight\ColorCLI;
use Blacklight\Nfo;
use Blacklight\NZB;
use Blacklight\libraries\Runners\BackfillRunner;
use Blacklight\libraries\Runners\BinariesRunner;
use Blacklight\libraries\Runners\PostProcessRunner;
use Blacklight\libraries\Runners\ReleasesRunner;
use Blacklight\Nfo;
use Blacklight\processing\PostProcess;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Spatie\Async\Output\SerializableException;
use Spatie\Async\Pool;
use Symfony\Component\Process\Process;
@@ -94,8 +89,11 @@ class Forking
/** Runners **/
private BackfillRunner $backfillRunner;
private BinariesRunner $binariesRunner;
private ReleasesRunner $releasesRunner;
private PostProcessRunner $postProcessRunner;
/**
@@ -164,6 +162,7 @@ class Forking
private function createPool(int $concurrency): Pool
{
$concurrency = max(1, $concurrency);
return Pool::create()
->concurrency($concurrency)
->timeout(config('nntmux.multiprocessing_max_child_time'));
@@ -268,7 +267,7 @@ class Forking
foreach ($groups as $g) {
try {
$q = DB::select(sprintf('SELECT id FROM collections WHERE groups_id = %d LIMIT 1', $g->id));
if (!empty($q)) {
if (! empty($q)) {
$count++;
}
} catch (\PDOException $e) {
@@ -277,6 +276,7 @@ class Forking
}
}
}
return $count;
}
@@ -284,17 +284,27 @@ class Forking
// but are not used when processWorkType delegates to runner classes.
private function backfill(): void {}
private function safeBackfill(): void {}
private function binaries(): void {}
private function safeBinaries(): void {}
private function fixRelNames(): void {}
private function releases(): void {}
public function postProcess(array $releases, int $maxProcess): void {}
private function postProcessAdd(): void {}
private function postProcessNfo(): void {}
private function postProcessMov(): void {}
private function postProcessTv(): void {}
private function processSingle(): void
{
$postProcess = new PostProcess;
@@ -59,11 +59,16 @@ class BackfillRunner extends BaseRunner
$orderby = 'ORDER BY a.last_record ASC';
switch ($backfill_order) {
case 1: $orderby = 'ORDER BY first_record_postdate DESC'; break;
case 2: $orderby = 'ORDER BY first_record_postdate ASC'; break;
case 3: $orderby = 'ORDER BY name ASC'; break;
case 4: $orderby = 'ORDER BY name DESC'; break;
case 5: $orderby = 'ORDER BY a.last_record DESC'; break;
case 1: $orderby = 'ORDER BY first_record_postdate DESC';
break;
case 2: $orderby = 'ORDER BY first_record_postdate ASC';
break;
case 3: $orderby = 'ORDER BY name ASC';
break;
case 4: $orderby = 'ORDER BY name DESC';
break;
case 5: $orderby = 'ORDER BY a.last_record DESC';
break;
}
$backfilldays = '0';
@@ -90,7 +95,7 @@ class BackfillRunner extends BaseRunner
$groupName = '';
$count = 0;
if (!empty($data) && isset($data[0]->name)) {
if (! empty($data) && isset($data[0]->name)) {
$groupName = $data[0]->name;
$count = ($data[0]->our_first - $data[0]->their_first);
}
@@ -100,6 +105,7 @@ class BackfillRunner extends BaseRunner
if (config('nntmux.echocli') && $groupName !== '') {
$this->colorCli->primary('No backfill needed for group '.$groupName);
}
return;
}
+4 -1
View File
@@ -14,12 +14,13 @@ abstract class BaseRunner
public function __construct(?ColorCLI $colorCli = null)
{
$this->colorCli = $colorCli ?? new ColorCLI();
$this->colorCli = $colorCli ?? new ColorCLI;
}
protected function createPool(int $concurrency): Pool
{
$concurrency = max(1, $concurrency);
return Pool::create()
->concurrency($concurrency)
->timeout(config('nntmux.multiprocessing_max_child_time'));
@@ -28,6 +29,7 @@ abstract class BaseRunner
protected function buildDnrCommand(string $args): string
{
$dnr_path = PHP_BINARY.' misc/update/multiprocessing/.do_not_run/switch.php "php ';
return $dnr_path.$args.'"';
}
@@ -40,6 +42,7 @@ abstract class BaseRunner
echo $buffer;
}
});
return $process->getOutput();
}
@@ -65,6 +65,7 @@ class BinariesRunner extends BaseRunner
if (empty($groups)) {
$this->headerNone();
return;
}
@@ -106,7 +107,7 @@ class BinariesRunner extends BaseRunner
$pool->add(function () use ($queue) {
return $this->executeCommand($this->buildDnrCommand($queue));
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($hit) {
if (!empty($hit)) {
if (! empty($hit)) {
echo $output;
$this->colorCli->primary('Updated group '.$hit[0]);
}
@@ -4,8 +4,8 @@ namespace Blacklight\libraries\Runners;
use App\Models\Settings;
use Blacklight\NZB;
use Spatie\Async\Output\SerializableException;
use Illuminate\Support\Facades\DB;
use Spatie\Async\Output\SerializableException;
class PostProcessRunner extends BaseRunner
{
@@ -13,6 +13,7 @@ class PostProcessRunner extends BaseRunner
{
if (empty($releases)) {
$this->headerNone();
return;
}
@@ -67,6 +68,7 @@ class PostProcessRunner extends BaseRunner
{
if ((int) Settings::settingValue('lookupnfo') !== 1) {
$this->headerNone();
return;
}
@@ -75,6 +77,7 @@ class PostProcessRunner extends BaseRunner
$checkSql = 'SELECT r.id FROM releases r WHERE 1=1 '.$nfoQuery.' LIMIT 1';
if (count(DB::select($checkSql)) === 0) {
$this->headerNone();
return;
}
@@ -94,6 +97,7 @@ class PostProcessRunner extends BaseRunner
{
if ((int) Settings::settingValue('lookupimdb') <= 0) {
$this->headerNone();
return;
}
@@ -110,6 +114,7 @@ class PostProcessRunner extends BaseRunner
LIMIT 1';
if (count(DB::select($checkSql)) === 0) {
$this->headerNone();
return;
}
@@ -133,6 +138,7 @@ class PostProcessRunner extends BaseRunner
{
if ((int) Settings::settingValue('lookuptv') <= 0) {
$this->headerNone();
return;
}
@@ -149,6 +155,7 @@ class PostProcessRunner extends BaseRunner
'.$condLookup.' '.$condRenamedOnly;
if (count(DB::select($checkSql)) === 0) {
$this->headerNone();
return;
}
@@ -19,7 +19,7 @@ class ReleasesRunner extends BaseRunner
foreach ($groups as $group) {
try {
$query = DB::select(sprintf('SELECT id FROM collections WHERE groups_id = %d LIMIT 1', $group->id));
if (!empty($query)) {
if (! empty($query)) {
$uGroups[] = ['id' => $group->id, 'name' => $group->name];
}
} catch (\PDOException $e) {
@@ -85,13 +85,13 @@ class ReleasesRunner extends BaseRunner
{
$maxThreads = max(1, min(16, $maxThreads));
$leftGuids = ['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'];
$leftGuids = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
if ($mode === 'predbft') {
$preCount = DB::select(
"SELECT COUNT(p.id) AS num FROM predb p WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP '[\"\<\> ]' AND p.searched = 0 AND p.predate < (NOW() - INTERVAL 1 DAY)"
);
if (!empty($preCount) && (int) $preCount[0]->num > 0 && $maxPerRun > 0) {
if (! empty($preCount) && (int) $preCount[0]->num > 0 && $maxPerRun > 0) {
$leftGuids = \array_slice($leftGuids, 0, (int) ceil($preCount[0]->num / $maxPerRun));
} else {
$leftGuids = [];
@@ -18,7 +18,6 @@ use Blacklight\NZB;
use Blacklight\ReleaseCleaning;
use Blacklight\ReleaseImage;
use Blacklight\Releases;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str;
@@ -990,7 +989,6 @@ class ProcessReleases
break;
}
// Yield briefly to reduce contention in busy systems.
usleep(10000);
} while (true);
@@ -4,7 +4,8 @@ use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration {
return new class extends Migration
{
public function up(): void
{
Schema::table('collections', function (Blueprint $table) {