mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-28 17:01:16 +00:00
Use concurrency and drop spatie/async package
This commit is contained in:
@@ -11,7 +11,6 @@ use Blacklight\libraries\Runners\ReleasesRunner;
|
||||
use Blacklight\Nfo;
|
||||
use Blacklight\processing\PostProcess;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Spatie\Async\Pool;
|
||||
use Symfony\Component\Process\Process;
|
||||
|
||||
/**
|
||||
@@ -146,17 +145,6 @@ class Forking
|
||||
*/
|
||||
private bool $ppRenamedOnly;
|
||||
|
||||
/**
|
||||
* Helper to ensure concurrency is at least 1 and with a common timeout.
|
||||
*/
|
||||
private function createPool(int $concurrency): Pool
|
||||
{
|
||||
$concurrency = max(1, $concurrency);
|
||||
|
||||
return Pool::create()
|
||||
->concurrency($concurrency)
|
||||
->timeout(config('nntmux.multiprocessing_max_child_time'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to build artisan command from legacy command format.
|
||||
|
||||
@@ -4,8 +4,9 @@ namespace Blacklight\libraries\Runners;
|
||||
|
||||
use App\Models\Settings;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Spatie\Async\Output\SerializableException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class BackfillRunner extends BaseRunner
|
||||
{
|
||||
@@ -38,26 +39,30 @@ class BackfillRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
|
||||
$this->headerStart('backfill', $count, $maxProcesses);
|
||||
|
||||
$taskNum = $count;
|
||||
foreach ($work as $group) {
|
||||
$pool->add(function () use ($group) {
|
||||
return $this->executeCommand(PHP_BINARY.' artisan update:backfill '.$group->name.(isset($group->max) ? (' '.$group->max) : ''));
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($group, &$taskNum) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Task #'.$taskNum.' Backfilled group '.$group->name);
|
||||
$taskNum--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($work, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $group) {
|
||||
$command = PHP_BINARY.' artisan update:backfill '.$group->name.(isset($group->max) ? (' '.$group->max) : '');
|
||||
$tasks[$group->name] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $groupName => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Backfilled group '.$groupName);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Backfill batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function safeBackfill(): void
|
||||
@@ -143,23 +148,29 @@ class BackfillRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($threads);
|
||||
|
||||
$this->headerStart('safe_backfill', count($queues), $threads);
|
||||
|
||||
foreach ($queues as $queue) {
|
||||
$pool->add(function () use ($queue) {
|
||||
return $this->executeCommand($this->buildDnrCommand($queue));
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($groupName) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Backfilled group '.$groupName);
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($queues, max(1, $threads), true);
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $queue) {
|
||||
$command = $this->buildDnrCommand($queue);
|
||||
$tasks[$idx] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $idx => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Backfilled group '.$groupName);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Safe backfill batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,28 +3,17 @@
|
||||
namespace Blacklight\libraries\Runners;
|
||||
|
||||
use Blacklight\ColorCLI;
|
||||
use Spatie\Async\Pool;
|
||||
use Symfony\Component\Process\Process;
|
||||
|
||||
abstract class BaseRunner
|
||||
{
|
||||
protected ColorCLI $colorCli;
|
||||
|
||||
protected const ASYNC_BUFFER_SIZE = 2000000;
|
||||
|
||||
public function __construct(?ColorCLI $colorCli = null)
|
||||
{
|
||||
$this->colorCli = $colorCli ?? new ColorCLI;
|
||||
}
|
||||
|
||||
protected function createPool(int $concurrency): Pool
|
||||
{
|
||||
$concurrency = max(1, $concurrency);
|
||||
|
||||
return Pool::create()
|
||||
->concurrency($concurrency)
|
||||
->timeout(config('nntmux.multiprocessing_max_child_time'));
|
||||
}
|
||||
|
||||
protected function buildDnrCommand(string $args): string
|
||||
{
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
namespace Blacklight\libraries\Runners;
|
||||
|
||||
use App\Models\Settings;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Spatie\Async\Output\SerializableException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class BinariesRunner extends BaseRunner
|
||||
{
|
||||
@@ -37,26 +38,30 @@ class BinariesRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
|
||||
$this->headerStart('binaries', $count, $maxProcesses);
|
||||
|
||||
$taskNum = $count;
|
||||
foreach ($work as $group) {
|
||||
$pool->add(function () use ($group) {
|
||||
return $this->executeCommand(PHP_BINARY.' artisan update:binaries '.$group->name.' '.$group->max);
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($group, &$taskNum) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Task #'.$taskNum.' Updated group '.$group->name);
|
||||
$taskNum--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($work, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $group) {
|
||||
$command = PHP_BINARY.' artisan update:binaries '.$group->name.' '.$group->max;
|
||||
$tasks[$group->name] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $groupName => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Updated group '.$groupName);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Binaries batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function safeBinaries(): void
|
||||
@@ -132,25 +137,32 @@ class BinariesRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
$this->headerStart('safe_binaries', count($queues), $maxProcesses);
|
||||
|
||||
foreach ($queues as $queue) {
|
||||
preg_match('/alt\..+/i', $queue, $hit);
|
||||
$pool->add(function () use ($queue) {
|
||||
return $this->executeCommand($this->buildDnrCommand($queue));
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($hit) {
|
||||
if (! empty($hit)) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Updated group '.$hit[0]);
|
||||
}
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($queues, max(1, $maxProcesses), true);
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $queue) {
|
||||
preg_match('/alt\..+/i', $queue, $hit);
|
||||
$command = $this->buildDnrCommand($queue);
|
||||
$tasks[$idx] = fn () => ['output' => $this->executeCommand($command), 'group' => $hit[0] ?? ''];
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $result) {
|
||||
if (! empty($result['group'])) {
|
||||
echo $result['output'];
|
||||
$this->colorCli->primary('Updated group '.$result['group']);
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Safe binaries batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
namespace Blacklight\libraries\Runners;
|
||||
|
||||
use App\Models\Settings;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Spatie\Async\Output\SerializableException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class PostProcessRunner extends BaseRunner
|
||||
{
|
||||
@@ -30,29 +31,33 @@ class PostProcessRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
$count = count($releases);
|
||||
$this->headerStart('postprocess: '.$desc, $count, $maxProcesses);
|
||||
|
||||
foreach ($releases as $release) {
|
||||
$char = isset($release->id) ? substr((string) $release->id, 0, 1) : '';
|
||||
$pool->add(function () use ($char, $type) {
|
||||
// Use postprocess:guid command which accepts the GUID character
|
||||
return $this->executeCommand(PHP_BINARY.' artisan postprocess:guid '.$type.' '.$char);
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use (&$count, $desc) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Finished task #'.$count.' for '.$desc);
|
||||
$count--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
})->timeout(function () use ($desc, &$count) {
|
||||
$this->colorCli->notice('Task #'.$count.' ('.$desc.'): Timeout occurred.');
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($releases, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $release) {
|
||||
$char = isset($release->id) ? substr((string) $release->id, 0, 1) : '';
|
||||
// Use postprocess:guid command which accepts the GUID character
|
||||
$command = PHP_BINARY.' artisan postprocess:guid '.$type.' '.$char;
|
||||
$tasks[$idx] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $taskIdx => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Finished task for '.$desc);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Postprocess batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function processAdditional(): void
|
||||
@@ -215,30 +220,34 @@ class PostProcessRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
$count = count($releases);
|
||||
$this->headerStart('postprocess: '.$desc, $count, $maxProcesses);
|
||||
|
||||
foreach ($releases as $release) {
|
||||
$char = isset($release->id) ? substr((string) $release->id, 0, 1) : '';
|
||||
$renamed = isset($release->renamed) ? $release->renamed : '';
|
||||
$pool->add(function () use ($char, $renamed) {
|
||||
// Use the pipelined TV command for each GUID bucket
|
||||
return $this->executeCommand(PHP_BINARY.' artisan postprocess:tv-pipeline '.$char.($renamed ? ' '.$renamed : '').' --mode=pipeline');
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use (&$count, $desc) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Finished task #'.$count.' for '.$desc);
|
||||
$count--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
})->timeout(function () use ($desc, &$count) {
|
||||
$this->colorCli->notice('Task #'.$count.' ('.$desc.'): Timeout occurred.');
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($releases, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $release) {
|
||||
$char = isset($release->id) ? substr((string) $release->id, 0, 1) : '';
|
||||
$renamed = isset($release->renamed) ? $release->renamed : '';
|
||||
// Use the pipelined TV command for each GUID bucket
|
||||
$command = PHP_BINARY.' artisan postprocess:tv-pipeline '.$char.($renamed ? ' '.$renamed : '').' --mode=pipeline';
|
||||
$tasks[$idx] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $taskIdx => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Finished task for '.$desc);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('TV pipeline batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,9 +4,9 @@ namespace Blacklight\libraries\Runners;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Models\UsenetGroup;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Spatie\Async\Output\SerializableException;
|
||||
|
||||
class ReleasesRunner extends BaseRunner
|
||||
{
|
||||
@@ -47,26 +47,30 @@ class ReleasesRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
|
||||
$this->headerStart('releases', $count, $maxProcesses);
|
||||
|
||||
$taskNum = $count;
|
||||
foreach ($uGroups as $group) {
|
||||
$pool->add(function () use ($group) {
|
||||
return $this->executeCommand($this->buildDnrCommand('releases '.$group['id']));
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use (&$taskNum) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Task #'.$taskNum.' Finished performing release processing');
|
||||
$taskNum--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($uGroups, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $group) {
|
||||
$command = $this->buildDnrCommand('releases '.$group['id']);
|
||||
$tasks[$group['id']] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $groupId => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Finished performing release processing for group ID: '.$groupId);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Release processing batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function updatePerGroup(): void
|
||||
@@ -92,24 +96,31 @@ class ReleasesRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxProcesses);
|
||||
$this->headerStart('update_per_group', $count, $maxProcesses);
|
||||
|
||||
foreach ($groups as $group) {
|
||||
$pool->add(function () use ($group) {
|
||||
return $this->executeCommand($this->buildDnrCommand('update_per_group '.$group->id));
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use ($group) {
|
||||
echo $output;
|
||||
$name = UsenetGroup::getNameByID($group->id);
|
||||
$this->colorCli->primary('Finished updating binaries, processing releases and additional postprocessing for group:'.$name);
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
echo $serializableException->asThrowable()->getMessage();
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($groups, max(1, $maxProcesses));
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $group) {
|
||||
$command = $this->buildDnrCommand('update_per_group '.$group->id);
|
||||
$tasks[$group->id] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $groupId => $output) {
|
||||
echo $output;
|
||||
$name = UsenetGroup::getNameByID($groupId);
|
||||
$this->colorCli->primary('Finished updating binaries, processing releases and additional postprocessing for group: '.$name);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Update per group batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function fixRelNames(string $mode, int $maxPerRun, int $maxThreads): void
|
||||
@@ -157,26 +168,30 @@ class ReleasesRunner extends BaseRunner
|
||||
return;
|
||||
}
|
||||
|
||||
$pool = $this->createPool($maxThreads);
|
||||
|
||||
$this->headerStart('fixRelNames_'.$mode, $count, $maxThreads);
|
||||
|
||||
$taskNum = $count;
|
||||
foreach ($queues as $queue) {
|
||||
$pool->add(function () use ($queue) {
|
||||
// Updated to use new script location (modernized)
|
||||
return $this->executeCommand(PHP_BINARY.' app/Services/Tmux/Scripts/groupfixrelnames.php "'.$queue.'" true');
|
||||
}, self::ASYNC_BUFFER_SIZE)->then(function ($output) use (&$taskNum) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Task #'.$taskNum.' Finished fixing releases names');
|
||||
$taskNum--;
|
||||
})->catch(function (\Throwable $exception) {
|
||||
echo $exception->getMessage();
|
||||
})->catch(static function (SerializableException $serializableException) {
|
||||
// swallow
|
||||
});
|
||||
}
|
||||
// Process in batches using Laravel's native Concurrency facade
|
||||
$batches = array_chunk($queues, max(1, $maxThreads), true);
|
||||
|
||||
$pool->wait();
|
||||
foreach ($batches as $batchIndex => $batch) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $queue) {
|
||||
// Updated to use new script location (modernized)
|
||||
$command = PHP_BINARY.' app/Services/Tmux/Scripts/groupfixrelnames.php "'.$queue.'" true';
|
||||
$tasks[$idx] = fn () => $this->executeCommand($command);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
foreach ($results as $taskIdx => $output) {
|
||||
echo $output;
|
||||
$this->colorCli->primary('Task #'.$taskIdx.' Finished fixing releases names');
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Fix rel names batch failed: '.$e->getMessage());
|
||||
$this->colorCli->error('Batch '.($batchIndex + 1).' failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,8 +20,8 @@ use Blacklight\ReleaseImage;
|
||||
use Blacklight\Releases;
|
||||
use Illuminate\Pipeline\Pipeline;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Concurrency;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Spatie\Async\Pool;
|
||||
|
||||
/**
|
||||
* Pipeline-based adult movie processing service using Laravel Pipeline.
|
||||
@@ -29,7 +29,7 @@ use Spatie\Async\Pool;
|
||||
* This service uses Laravel's Pipeline to orchestrate multiple adult movie data providers
|
||||
* to process releases and match them against movie metadata from various sources.
|
||||
*
|
||||
* It also supports parallel processing of multiple releases using spatie/async.
|
||||
* It also supports parallel processing of multiple releases using Laravel's native Concurrency facade.
|
||||
*/
|
||||
class AdultProcessingPipeline
|
||||
{
|
||||
@@ -183,7 +183,7 @@ class AdultProcessingPipeline
|
||||
/**
|
||||
* Process all XXX releases where xxxinfo_id is 0.
|
||||
*
|
||||
* Uses spatie/async for parallel processing when possible.
|
||||
* Uses Laravel's native Concurrency facade for parallel processing when possible.
|
||||
*/
|
||||
public function processXXXReleases(): void
|
||||
{
|
||||
@@ -236,7 +236,7 @@ class AdultProcessingPipeline
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a batch of releases using spatie/async for parallel execution.
|
||||
* Process a batch of releases using Laravel's native Concurrency for parallel execution.
|
||||
*
|
||||
* Note: Due to serialization limitations with DOMDocument and HtmlDomParser,
|
||||
* we process releases sequentially within the batch but can process multiple
|
||||
@@ -278,29 +278,28 @@ class AdultProcessingPipeline
|
||||
return;
|
||||
}
|
||||
|
||||
// For async processing, we need to avoid serializing $this
|
||||
// Instead, we create independent tasks with only serializable data
|
||||
$pool = Pool::create()
|
||||
->concurrency(count($batch))
|
||||
->timeout(120);
|
||||
|
||||
// For async processing using Laravel's native Concurrency facade
|
||||
// We create independent tasks with only serializable data
|
||||
$cookie = $this->cookie;
|
||||
$echoOutput = $this->echoOutput;
|
||||
$imgSavePath = $this->imgSavePath;
|
||||
|
||||
foreach ($batch as $release) {
|
||||
$tasks = [];
|
||||
foreach ($batch as $idx => $release) {
|
||||
// Extract only the serializable data needed
|
||||
$releaseData = [
|
||||
'id' => $release['id'],
|
||||
'searchname' => $release['searchname'],
|
||||
];
|
||||
|
||||
$pool->add(function () use ($releaseData, $cookie, $echoOutput, $imgSavePath) {
|
||||
// Create a fresh pipeline instance in the child process
|
||||
// This avoids serialization issues with DOMDocument
|
||||
return self::processReleaseInChildProcess($releaseData, $cookie, $echoOutput, $imgSavePath);
|
||||
})->then(function ($result) {
|
||||
// Update stats based on result
|
||||
$tasks[$idx] = fn () => self::processReleaseInChildProcess($releaseData, $cookie, $echoOutput, $imgSavePath);
|
||||
}
|
||||
|
||||
try {
|
||||
$results = Concurrency::run($tasks);
|
||||
|
||||
// Update stats based on results
|
||||
foreach ($results as $result) {
|
||||
if (is_array($result)) {
|
||||
if ($result['matched']) {
|
||||
$this->stats['matched']++;
|
||||
@@ -311,20 +310,20 @@ class AdultProcessingPipeline
|
||||
}
|
||||
}
|
||||
$this->stats['processed']++;
|
||||
})->catch(function (\Throwable $e) use ($releaseData) {
|
||||
Log::error('Async processing failed for release ' . ($releaseData['id'] ?? 'unknown') . ': ' . $e->getMessage());
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Async batch processing failed: ' . $e->getMessage());
|
||||
|
||||
// Mark all releases in batch as processed with error to avoid infinite loop
|
||||
foreach ($batch as $release) {
|
||||
$this->stats['failed']++;
|
||||
|
||||
// Mark release as processed with error to avoid infinite loop
|
||||
try {
|
||||
Release::query()->where('id', $releaseData['id'])->update(['xxxinfo_id' => -2]);
|
||||
Release::query()->where('id', $release['id'])->update(['xxxinfo_id' => -2]);
|
||||
} catch (\Throwable $updateError) {
|
||||
Log::error('Failed to mark release ' . $releaseData['id'] . ' as processed: ' . $updateError->getMessage());
|
||||
Log::error('Failed to mark release ' . $release['id'] . ' as processed: ' . $updateError->getMessage());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
$pool->wait();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -613,7 +612,7 @@ class AdultProcessingPipeline
|
||||
*/
|
||||
protected function canUseAsync(): bool
|
||||
{
|
||||
return class_exists('Spatie\Async\Pool') &&
|
||||
return class_exists('Illuminate\Support\Facades\Concurrency') &&
|
||||
function_exists('pcntl_fork') &&
|
||||
!defined('HHVM_VERSION');
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ use voku\helper\HtmlDomParser;
|
||||
* Each pipe is responsible for processing releases through a specific adult site provider.
|
||||
*
|
||||
* Note: This class intentionally uses lazy loading for HtmlDomParser to avoid
|
||||
* serialization issues with DOMDocument when using spatie/async.
|
||||
* serialization issues with DOMDocument when using Laravel's Concurrency facade.
|
||||
*/
|
||||
abstract class AbstractAdultProviderPipe
|
||||
{
|
||||
|
||||
@@ -88,7 +88,6 @@
|
||||
"propaganistas/laravel-disposable-email": "^2.4",
|
||||
"riari/laravel-forum": "^7.0",
|
||||
"sentry/sentry-laravel": "^4.13",
|
||||
"spatie/async": "^1.7",
|
||||
"spatie/laravel-directory-cleanup": "^1.10",
|
||||
"spatie/laravel-fractal": "^6.3",
|
||||
"spatie/laravel-ignition": "^2.9",
|
||||
|
||||
Generated
+1
-68
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "7eea2735024f68804cd270d1aed9638b",
|
||||
"content-hash": "578bb49b5102526e3d269681e6d2d504",
|
||||
"packages": [
|
||||
{
|
||||
"name": "aharen/omdbapi",
|
||||
@@ -8353,73 +8353,6 @@
|
||||
],
|
||||
"time": "2025-12-02T10:37:40+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/async",
|
||||
"version": "1.8.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/spatie/async.git",
|
||||
"reference": "f9e9d4908ac1b7887a5725322399ae5935bea077"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/spatie/async/zipball/f9e9d4908ac1b7887a5725322399ae5935bea077",
|
||||
"reference": "f9e9d4908ac1b7887a5725322399ae5935bea077",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"laravel/serializable-closure": "^1.3.7|^2.0.0",
|
||||
"php": "^8.3|^8.4|^8.5",
|
||||
"symfony/process": "^7.2|^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"larapack/dd": "^1.1",
|
||||
"pestphp/pest": "^3.0|^4.0",
|
||||
"symfony/stopwatch": "^7.2|^8.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-pcntl": "Required to use async processes",
|
||||
"ext-posix": "Required to use async processes"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"files": [
|
||||
"src/helpers.php"
|
||||
],
|
||||
"psr-4": {
|
||||
"Spatie\\Async\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Brent Roose",
|
||||
"email": "brent@spatie.be",
|
||||
"homepage": "https://spatie.be",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Asynchronous and parallel PHP with the PCNTL extension",
|
||||
"homepage": "https://github.com/spatie/async",
|
||||
"keywords": [
|
||||
"async",
|
||||
"spatie"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/spatie/async/issues",
|
||||
"source": "https://github.com/spatie/async/tree/1.8.1"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/spatie",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2025-11-28T10:22:46+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/backtrace",
|
||||
"version": "1.8.1",
|
||||
|
||||
Reference in New Issue
Block a user