mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-28 17:01:16 +00:00
Refactor additional postprocessing
This commit is contained in:
@@ -79,7 +79,6 @@ NNTP_SOCKET_TIMEOUT_A=120
|
||||
|
||||
NN_MULTIPROCESSING_MAX_CHILD_TIME=1800
|
||||
NN_CONCURRENCY_TIMEOUT=
|
||||
|
||||
ADMIN_USER=
|
||||
ADMIN_PASS=
|
||||
ADMIN_EMAIL=
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingDiagnostics;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class AdditionalProcessingDiagnose extends Command
|
||||
{
|
||||
protected $signature = 'nntmux:additional-diagnose
|
||||
{--json : Output machine-readable JSON}';
|
||||
|
||||
protected $description = 'Inspect additional post-processing backlog, claims, capacity, indexes, and temp-path health';
|
||||
|
||||
public function __construct(private readonly AdditionalProcessingDiagnostics $diagnostics)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$report = $this->diagnostics->inspect();
|
||||
|
||||
if ($this->option('json')) {
|
||||
$this->line((string) json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR));
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->table(['Backlog', 'Count'], [
|
||||
['Total', (string) $report['backlog']['total']],
|
||||
['Available', (string) $report['backlog']['available']],
|
||||
['Active claims', (string) $report['backlog']['active_claims']],
|
||||
['Stale claims', (string) $report['backlog']['stale_claims']],
|
||||
]);
|
||||
$this->table(['Capacity', 'Value'], [
|
||||
['Pipeline', (string) $report['settings']['pipeline']],
|
||||
['Threads', (string) $report['capacity']['threads']],
|
||||
['Releases per batch', (string) $report['capacity']['releases_per_batch']],
|
||||
['Maximum in flight', (string) $report['capacity']['max_in_flight']],
|
||||
['Claim TTL', $report['claims']['ttl_seconds'].' seconds'],
|
||||
['Child timeout', $report['settings']['multiprocessing_child_timeout'].' seconds'],
|
||||
]);
|
||||
$this->table(['Preflight', 'Value'], [
|
||||
['Database driver', (string) $report['database_driver']],
|
||||
['Covering claim index', (string) ($report['indexes']['covering_index'] ?? 'missing')],
|
||||
['Temp path', (string) $report['temp_path']['path']],
|
||||
['Temp path writable', $report['temp_path']['writable'] ? 'yes' : 'no'],
|
||||
]);
|
||||
|
||||
if ($report['warnings'] === []) {
|
||||
$this->info('No additional-processing diagnostic warnings were found.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
foreach ($report['warnings'] as $warning) {
|
||||
$this->warn('['.$warning['code'].'] '.$warning['message']);
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
|
||||
use App\Services\NfoService;
|
||||
use App\Services\NNTP\NNTPService;
|
||||
use App\Services\PostProcessService;
|
||||
@@ -24,7 +25,8 @@ class PostProcessGuid extends Command
|
||||
{guid : First character of release guid (a-f, 0-9)}
|
||||
{renamed? : For movie/tv: process renamed only (optional)}
|
||||
{--worker : Drain multiple additional-processing batches}
|
||||
{--max-batches=4 : Maximum claim batches for worker mode}';
|
||||
{--max-batches=4 : Maximum claim batches for worker mode}
|
||||
{--profile : Emit one machine-readable performance record per additional batch}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
@@ -93,13 +95,16 @@ class PostProcessGuid extends Command
|
||||
|
||||
try {
|
||||
for ($batch = 0; $batch < $maxBatches && microtime(true) < $deadline; $batch++) {
|
||||
$stats = $this->additionalProcessor->start('', $guid, $workerToken, $excludedReleaseIds);
|
||||
if ($stats['claimed'] === 0) {
|
||||
$result = $this->additionalProcessor->start('', $guid, $workerToken, $excludedReleaseIds);
|
||||
if ((bool) $this->option('profile')) {
|
||||
$this->writeAdditionalProfile($guid, $batch + 1, $result);
|
||||
}
|
||||
if ($result->claimedCount() === 0) {
|
||||
break;
|
||||
}
|
||||
$excludedReleaseIds = array_values(array_unique([
|
||||
...$excludedReleaseIds,
|
||||
...$stats['claimed_ids'],
|
||||
...$result->claimedIds,
|
||||
]));
|
||||
}
|
||||
} finally {
|
||||
@@ -107,6 +112,39 @@ class PostProcessGuid extends Command
|
||||
}
|
||||
}
|
||||
|
||||
private function writeAdditionalProfile(string $guid, int $batch, AdditionalBatchResult $result): void
|
||||
{
|
||||
$downloadMetrics = $result->downloadMetrics();
|
||||
$persistenceMetrics = $result->persistenceMetrics();
|
||||
|
||||
$this->line(json_encode([
|
||||
'event' => 'additional-postprocessing-profile',
|
||||
'pipeline' => 'v2',
|
||||
'guid_char' => $guid,
|
||||
'batch' => $batch,
|
||||
'claimed' => $result->claimedCount(),
|
||||
'attempted' => $result->attemptedCount(),
|
||||
'successful' => $result->successfulCount(),
|
||||
'failed' => $result->unsuccessfulCount(),
|
||||
'outcomes' => $result->outcomeCounts(),
|
||||
'artifacts_created' => $result->artifactsCreatedCount(),
|
||||
'artifact_yield_percent' => round($result->artifactYieldPercent(), 2),
|
||||
'release_files_added' => $result->releaseFilesAdded(),
|
||||
'nntp_requests' => $downloadMetrics->networkRequests,
|
||||
'nntp_bytes' => $downloadMetrics->bytesDownloaded,
|
||||
'database_statements' => $persistenceMetrics->databaseStatements,
|
||||
'database_milliseconds' => round($persistenceMetrics->databaseMilliseconds, 3),
|
||||
'search_sync_requests' => $persistenceMetrics->searchSyncRequests,
|
||||
'search_sync_executions' => $persistenceMetrics->searchSyncExecutions,
|
||||
'duplicate_message_ids' => $result->duplicateMessageIdCount(),
|
||||
'elapsed_seconds' => round($result->elapsedSeconds, 6),
|
||||
'releases_per_hour' => round($result->releasesPerSecond() * 3600, 2),
|
||||
'average_release_seconds' => round($result->averageReleaseSeconds(), 6),
|
||||
'stage_seconds' => $result->stageDurationTotals(),
|
||||
'peak_memory_bytes' => $result->peakMemoryBytes,
|
||||
], JSON_THROW_ON_ERROR));
|
||||
}
|
||||
|
||||
/**
|
||||
* Process NFO files for releases.
|
||||
*/
|
||||
|
||||
@@ -7,6 +7,7 @@ namespace App\Console\Commands;
|
||||
use App\Facades\Search;
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class ProcessAdditionalGuid extends Command
|
||||
@@ -15,7 +16,8 @@ class ProcessAdditionalGuid extends Command
|
||||
{guid? : Release GUID (optional if using --id)}
|
||||
{--id= : Numeric release ID to run additional postprocessing on (alternative to GUID)}
|
||||
{--reset : Reset key additional postprocessing flags before running}
|
||||
{--no-progress : Suppress progress / status output (forces quiet mode)}';
|
||||
{--no-progress : Suppress progress / status output (forces quiet mode)}
|
||||
{--profile : Display stage, network, and persistence metrics}';
|
||||
|
||||
protected $description = 'Run additional postprocessing for a specific release by GUID or ID regardless of prior postprocessing status';
|
||||
|
||||
@@ -87,19 +89,89 @@ class ProcessAdditionalGuid extends Command
|
||||
$processor = app(AdditionalProcessingOrchestrator::class);
|
||||
|
||||
try {
|
||||
$ok = $processor->processSingleGuid($guid);
|
||||
$result = $processor->processSingleGuid($guid);
|
||||
} finally {
|
||||
$processor->finish();
|
||||
}
|
||||
|
||||
if (! $ok) {
|
||||
$this->error('Processing failed or nothing processed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.');
|
||||
$identifier = $idOpt ? 'ID '.$idOpt : 'GUID '.$guid;
|
||||
|
||||
$this->displayResult($result, $identifier);
|
||||
|
||||
if ($this->option('profile')) {
|
||||
$this->displayProfile($result);
|
||||
}
|
||||
|
||||
if (! $result->isSuccessful()) {
|
||||
return 3;
|
||||
}
|
||||
|
||||
$this->info('Additional postprocessing completed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.');
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private function displayResult(ReleaseProcessingResult $result, string $identifier): void
|
||||
{
|
||||
$rows = [
|
||||
['Release', $identifier],
|
||||
['Pipeline', 'v2'],
|
||||
['Outcome', $result->outcome->value],
|
||||
['Artifacts created', $result->artifactsCreated ? 'yes' : 'no'],
|
||||
['Release files added', (string) $result->releaseFilesAdded],
|
||||
['Duration', number_format($result->elapsedSeconds, 3).' seconds'],
|
||||
];
|
||||
|
||||
if ($result->reason !== '') {
|
||||
$rows[] = ['Reason', $result->reason];
|
||||
}
|
||||
|
||||
$this->table(['Result', 'Value'], $rows);
|
||||
|
||||
$message = 'Additional postprocessing ended with outcome '.$result->outcome->value.' for '.$identifier
|
||||
.' after '.number_format($result->elapsedSeconds, 3).' seconds.';
|
||||
|
||||
if ($result->isSuccessful()) {
|
||||
$this->info($message);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->error($message);
|
||||
}
|
||||
|
||||
private function displayProfile(ReleaseProcessingResult $result): void
|
||||
{
|
||||
$stageRows = [];
|
||||
foreach ($result->stageDurations as $stage => $duration) {
|
||||
$stageRows[] = [$stage, number_format($duration, 3)];
|
||||
}
|
||||
|
||||
if ($stageRows !== []) {
|
||||
$this->newLine();
|
||||
$this->line('Stage profile');
|
||||
$this->table(['Stage', 'Seconds'], $stageRows);
|
||||
}
|
||||
|
||||
if ($result->downloadMetrics !== null) {
|
||||
$this->newLine();
|
||||
$this->line('Download profile');
|
||||
$this->table(['Metric', 'Value'], [
|
||||
['Logical requests', (string) $result->downloadMetrics->logicalRequests],
|
||||
['Network requests', (string) $result->downloadMetrics->networkRequests],
|
||||
['Cache hits', (string) $result->downloadMetrics->cacheHits],
|
||||
['Bytes downloaded', (string) $result->downloadMetrics->bytesDownloaded],
|
||||
['Bytes reused', (string) $result->downloadMetrics->bytesReused],
|
||||
]);
|
||||
}
|
||||
|
||||
if ($result->persistenceMetrics !== null) {
|
||||
$this->newLine();
|
||||
$this->line('Persistence profile');
|
||||
$this->table(['Metric', 'Value'], [
|
||||
['Database statements', (string) $result->persistenceMetrics->databaseStatements],
|
||||
['Database milliseconds', number_format($result->persistenceMetrics->databaseMilliseconds, 3)],
|
||||
['Search sync requests', (string) $result->persistenceMetrics->searchSyncRequests],
|
||||
['Search sync executions', (string) $result->persistenceMetrics->searchSyncExecutions],
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ declare(strict_types=1);
|
||||
namespace App\Providers;
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
|
||||
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
|
||||
use App\Services\AdditionalProcessing\ArchiveExtractionService;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\ConsoleOutputService;
|
||||
@@ -13,15 +14,20 @@ use App\Services\AdditionalProcessing\NzbContentParser;
|
||||
use App\Services\AdditionalProcessing\ReleaseFileManager;
|
||||
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
|
||||
use App\Services\AdditionalProcessing\ReleaseProcessor;
|
||||
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\UsenetDownloadService;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use App\Services\NameFixing\NameFixingService;
|
||||
use App\Services\NameFixing\ReleaseUpdateService;
|
||||
use App\Services\NfoService;
|
||||
use App\Services\Nzb\NzbParserService;
|
||||
use App\Services\Nzb\NzbService;
|
||||
use App\Services\ReleaseExtraService;
|
||||
use App\Services\ReleaseImageService;
|
||||
use App\Services\TempWorkspaceService;
|
||||
use Illuminate\Database\Events\QueryExecuted;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\ServiceProvider;
|
||||
|
||||
/**
|
||||
@@ -40,6 +46,18 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
return new ProcessingConfiguration;
|
||||
});
|
||||
|
||||
$this->app->singleton(PersistenceMetricsCollector::class);
|
||||
$this->app->singleton(ReleaseSearchSyncCoordinator::class, function ($app) {
|
||||
return new ReleaseSearchSyncCoordinator(
|
||||
$app->make(PersistenceMetricsCollector::class),
|
||||
);
|
||||
});
|
||||
$this->app->bind(ReleaseUpdateService::class, function ($app) {
|
||||
return new ReleaseUpdateService(
|
||||
searchSyncCoordinator: $app->make(ReleaseSearchSyncCoordinator::class),
|
||||
);
|
||||
});
|
||||
|
||||
// Release extra service for video/audio/subtitle data
|
||||
$this->app->singleton(ReleaseExtraService::class, function () {
|
||||
return new ReleaseExtraService;
|
||||
@@ -64,6 +82,10 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
);
|
||||
});
|
||||
|
||||
$this->app->singleton(AdditionalWorkPlanner::class, function ($app) {
|
||||
return new AdditionalWorkPlanner($app->make(ProcessingConfiguration::class));
|
||||
});
|
||||
|
||||
// Archive extraction service
|
||||
$this->app->singleton(ArchiveExtractionService::class, function ($app) {
|
||||
return new ArchiveExtractionService(
|
||||
@@ -85,7 +107,9 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
$app->make(ReleaseImageService::class),
|
||||
new NfoService,
|
||||
$app->make(NzbService::class),
|
||||
new NameFixingService
|
||||
new NameFixingService($app->make(ReleaseUpdateService::class)),
|
||||
$app->make(ReleaseUpdateService::class),
|
||||
searchSyncCoordinator: $app->make(ReleaseSearchSyncCoordinator::class),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -95,18 +119,18 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
$app->make(ProcessingConfiguration::class),
|
||||
$app->make(ReleaseImageService::class),
|
||||
$app->make(ReleaseExtraService::class),
|
||||
new CategorizationService
|
||||
new CategorizationService,
|
||||
$app->make(ReleaseSearchSyncCoordinator::class),
|
||||
);
|
||||
});
|
||||
|
||||
$this->app->singleton(ReleaseFilesArchiveFallback::class, function ($app) {
|
||||
return new ReleaseFilesArchiveFallback(
|
||||
$app->make(ProcessingConfiguration::class),
|
||||
$app->make(ArchiveExtractionService::class),
|
||||
$app->make(MediaExtractionService::class),
|
||||
$app->make(UsenetDownloadService::class),
|
||||
$app->make(ReleaseFileManager::class),
|
||||
$app->make(ConsoleOutputService::class)
|
||||
$app->make(ConsoleOutputService::class),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -114,13 +138,16 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
return new ReleaseProcessor(
|
||||
$app->make(ProcessingConfiguration::class),
|
||||
$app->make(NzbContentParser::class),
|
||||
$app->make(AdditionalWorkPlanner::class),
|
||||
$app->make(ArchiveExtractionService::class),
|
||||
$app->make(MediaExtractionService::class),
|
||||
$app->make(UsenetDownloadService::class),
|
||||
$app->make(ReleaseFileManager::class),
|
||||
$app->make(ReleaseFilesArchiveFallback::class),
|
||||
$app->make(TempWorkspaceService::class),
|
||||
$app->make(ConsoleOutputService::class)
|
||||
$app->make(ConsoleOutputService::class),
|
||||
$app->make(ReleaseSearchSyncCoordinator::class),
|
||||
$app->make(PersistenceMetricsCollector::class),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -143,8 +170,12 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
|
||||
/**
|
||||
* Bootstrap services.
|
||||
*/
|
||||
public function boot(): void
|
||||
public function boot(PersistenceMetricsCollector $persistenceMetrics): void
|
||||
{
|
||||
DB::listen(static function (QueryExecuted $query) use ($persistenceMetrics): void {
|
||||
$persistenceMetrics->recordDatabaseStatement($query->time);
|
||||
});
|
||||
|
||||
$this->app->terminating(function (): void {
|
||||
if ($this->app->bound(AdditionalProcessingOrchestrator::class)) {
|
||||
$this->app->make(AdditionalProcessingOrchestrator::class)->finish();
|
||||
|
||||
@@ -390,14 +390,14 @@ final class AdditionalCandidateQuery
|
||||
});
|
||||
}
|
||||
|
||||
private static function claimTtlSeconds(): int
|
||||
public static function claimTtlSeconds(): int
|
||||
{
|
||||
$timeout = (int) (Settings::settingValue('releaseprocessingtimeout') ?: 120);
|
||||
|
||||
return max(300, $timeout * 2);
|
||||
}
|
||||
|
||||
private static function claimStaleBefore(): Carbon
|
||||
public static function claimStaleBefore(): Carbon
|
||||
{
|
||||
return now()->subSeconds(self::claimTtlSeconds());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Models\Settings;
|
||||
use App\Services\Runners\PostProcessRunner;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
final class AdditionalProcessingDiagnostics
|
||||
{
|
||||
private const int OVERSIZED_IN_FLIGHT_THRESHOLD = 500;
|
||||
|
||||
private const int OVERSIZED_THREAD_THRESHOLD = 32;
|
||||
|
||||
/**
|
||||
* @var list<string>
|
||||
*/
|
||||
private const array REQUIRED_CLAIM_INDEX_COLUMNS = [
|
||||
'passwordstatus',
|
||||
'haspreview',
|
||||
'nzbstatus',
|
||||
'leftguid',
|
||||
'additional_pp_claimed_at',
|
||||
'postdate',
|
||||
];
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public function inspect(): array
|
||||
{
|
||||
$warnings = [];
|
||||
$releaseTimeout = $this->setting('releaseprocessingtimeout', 120);
|
||||
$threads = max(1, $this->setting('postthreads', 1));
|
||||
$batchSize = max(1, $this->setting('maxaddprocessed', 25));
|
||||
$childTimeout = (int) (config('nntmux.concurrency_timeout')
|
||||
?? config('nntmux.multiprocessing_max_child_time', 1800));
|
||||
$claimTtl = AdditionalCandidateQuery::claimTtlSeconds();
|
||||
$backlog = $this->backlog($warnings);
|
||||
$indexes = $this->indexes($warnings);
|
||||
$tempPath = $this->tempPath($warnings);
|
||||
$maximumClaimLifetime = min(
|
||||
max(1, $childTimeout),
|
||||
max(1, $releaseTimeout) * $batchSize,
|
||||
);
|
||||
|
||||
if ($backlog['stale_claims'] > 0) {
|
||||
$warnings[] = $this->warning(
|
||||
'stale-claims',
|
||||
$backlog['stale_claims'].' additional-processing claim(s) are stale and eligible for recovery.',
|
||||
);
|
||||
}
|
||||
|
||||
if ($claimTtl <= $maximumClaimLifetime) {
|
||||
$warnings[] = $this->warning(
|
||||
'claim-ttl',
|
||||
'Claim TTL ('.$claimTtl.'s) does not exceed the estimated claimed-batch lifetime ('
|
||||
.$maximumClaimLifetime.'s); another worker may reclaim active work.',
|
||||
);
|
||||
}
|
||||
|
||||
$maxInFlight = $threads * $batchSize;
|
||||
if ($threads > self::OVERSIZED_THREAD_THRESHOLD || $maxInFlight > self::OVERSIZED_IN_FLIGHT_THRESHOLD) {
|
||||
$warnings[] = $this->warning(
|
||||
'oversized-capacity',
|
||||
'postthreads='.$threads.' and maxaddprocessed='.$batchSize.' allow up to '.$maxInFlight
|
||||
.' claimed releases at once; reduce threads or batch size and benchmark end-to-end throughput.',
|
||||
);
|
||||
}
|
||||
|
||||
return [
|
||||
'generated_at' => now()->toIso8601String(),
|
||||
'database_driver' => DB::getDriverName(),
|
||||
'backlog' => $backlog,
|
||||
'settings' => [
|
||||
'pipeline' => 'v2',
|
||||
'postthreads' => $threads,
|
||||
'maxaddprocessed' => $batchSize,
|
||||
'releaseprocessingtimeout' => $releaseTimeout,
|
||||
'maxpptimeoutcount' => $this->setting('maxpptimeoutcount', 3),
|
||||
'worker_max_batches' => PostProcessRunner::ADDITIONAL_WORKER_MAX_BATCHES,
|
||||
'multiprocessing_child_timeout' => $childTimeout,
|
||||
],
|
||||
'claims' => [
|
||||
'ttl_seconds' => $claimTtl,
|
||||
'stale_before' => AdditionalCandidateQuery::claimStaleBefore()->toIso8601String(),
|
||||
'estimated_claimed_batch_lifetime_seconds' => $maximumClaimLifetime,
|
||||
],
|
||||
'capacity' => [
|
||||
'threads' => $threads,
|
||||
'releases_per_batch' => $batchSize,
|
||||
'max_in_flight' => $maxInFlight,
|
||||
'max_per_worker_lifetime' => $batchSize * PostProcessRunner::ADDITIONAL_WORKER_MAX_BATCHES,
|
||||
],
|
||||
'indexes' => $indexes,
|
||||
'temp_path' => $tempPath,
|
||||
'warnings' => $warnings,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{code: string, message: string}> $warnings
|
||||
* @return array{total: int, available: int, active_claims: int, stale_claims: int, buckets: list<array{bucket: string, total: int, available: int}>}
|
||||
*/
|
||||
private function backlog(array &$warnings): array
|
||||
{
|
||||
$empty = [
|
||||
'total' => 0,
|
||||
'available' => 0,
|
||||
'active_claims' => 0,
|
||||
'stale_claims' => 0,
|
||||
'buckets' => [],
|
||||
];
|
||||
|
||||
if (! Schema::hasTable('releases') || ! Schema::hasTable('categories')) {
|
||||
$warnings[] = $this->warning(
|
||||
'missing-processing-tables',
|
||||
'The releases or categories table is missing; backlog health cannot be inspected.',
|
||||
);
|
||||
|
||||
return $empty;
|
||||
}
|
||||
|
||||
$counts = AdditionalCandidateQuery::backlogCounts();
|
||||
$activeClaims = 0;
|
||||
$staleClaims = 0;
|
||||
|
||||
if (AdditionalCandidateQuery::supportsClaims()) {
|
||||
$claimQuery = AdditionalCandidateQuery::baseBuilder(includeClaimed: true)
|
||||
->whereNotNull('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN);
|
||||
|
||||
$activeClaims = (clone $claimQuery)
|
||||
->where('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN, '>=', AdditionalCandidateQuery::claimStaleBefore())
|
||||
->count('r.id');
|
||||
$staleClaims = (clone $claimQuery)
|
||||
->where('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN, '<', AdditionalCandidateQuery::claimStaleBefore())
|
||||
->count('r.id');
|
||||
}
|
||||
|
||||
return [
|
||||
...$counts,
|
||||
'active_claims' => $activeClaims,
|
||||
'stale_claims' => $staleClaims,
|
||||
'buckets' => AdditionalCandidateQuery::bucketBacklog(),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{code: string, message: string}> $warnings
|
||||
* @return array{required_columns: list<string>, present: list<string>, covering_index: string|null, missing: bool}
|
||||
*/
|
||||
private function indexes(array &$warnings): array
|
||||
{
|
||||
$present = [];
|
||||
$coveringIndex = null;
|
||||
|
||||
if (Schema::hasTable('releases')) {
|
||||
foreach (Schema::getIndexes('releases') as $index) {
|
||||
$name = (string) ($index['name'] ?? '');
|
||||
$columns = array_map(
|
||||
static fn (mixed $column): string => strtolower((string) $column),
|
||||
is_array($index['columns'] ?? null) ? $index['columns'] : [],
|
||||
);
|
||||
|
||||
if ($name !== '') {
|
||||
$present[] = $name;
|
||||
}
|
||||
if ($columns === self::REQUIRED_CLAIM_INDEX_COLUMNS) {
|
||||
$coveringIndex = $name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort($present);
|
||||
$missing = $coveringIndex === null;
|
||||
if ($missing) {
|
||||
$warnings[] = $this->warning(
|
||||
'missing-index',
|
||||
'The releases claim-queue index is missing or has the wrong column order; run pending migrations before processing.',
|
||||
);
|
||||
}
|
||||
|
||||
return [
|
||||
'required_columns' => self::REQUIRED_CLAIM_INDEX_COLUMNS,
|
||||
'present' => $present,
|
||||
'covering_index' => $coveringIndex,
|
||||
'missing' => $missing,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array{code: string, message: string}> $warnings
|
||||
* @return array{path: string, exists: bool, writable: bool}
|
||||
*/
|
||||
private function tempPath(array &$warnings): array
|
||||
{
|
||||
$path = rtrim((string) config('nntmux.tmp_unrar_path'), '/\\');
|
||||
$exists = $path !== '' && is_dir($path);
|
||||
$writable = $exists ? is_writable($path) : $this->nearestExistingDirectoryIsWritable($path);
|
||||
|
||||
if (! $writable) {
|
||||
$warnings[] = $this->warning(
|
||||
'unwritable-temp-path',
|
||||
'Additional-processing temp path '.$path.' is not writable and cannot be created by the current user.',
|
||||
);
|
||||
}
|
||||
|
||||
return [
|
||||
'path' => $path,
|
||||
'exists' => $exists,
|
||||
'writable' => $writable,
|
||||
];
|
||||
}
|
||||
|
||||
private function nearestExistingDirectoryIsWritable(string $path): bool
|
||||
{
|
||||
if ($path === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
$candidate = $path;
|
||||
while (! is_dir($candidate)) {
|
||||
$parent = dirname($candidate);
|
||||
if ($parent === $candidate) {
|
||||
return false;
|
||||
}
|
||||
$candidate = $parent;
|
||||
}
|
||||
|
||||
return is_writable($candidate);
|
||||
}
|
||||
|
||||
private function setting(string $name, int $default): int
|
||||
{
|
||||
$value = Settings::settingValue($name);
|
||||
|
||||
return $value === null || $value === '' ? $default : (int) $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{code: string, message: string}
|
||||
*/
|
||||
private function warning(string $code, string $message): array
|
||||
{
|
||||
return ['code' => $code, 'message' => $message];
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,9 @@ namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\TempWorkspaceService;
|
||||
use Exception;
|
||||
@@ -31,6 +34,8 @@ class AdditionalProcessingOrchestrator
|
||||
|
||||
private string $claimToken = '';
|
||||
|
||||
private string $lastSetupError = '';
|
||||
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
private readonly ReleaseProcessor $processor,
|
||||
@@ -42,7 +47,6 @@ class AdditionalProcessingOrchestrator
|
||||
* Start the additional processing.
|
||||
*
|
||||
* @param list<int> $excludedReleaseIds
|
||||
* @return array{claimed: int, processed: int, failed: int, claimed_ids: list<int>}
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@@ -51,10 +55,10 @@ class AdditionalProcessingOrchestrator
|
||||
string $guidChar = '',
|
||||
string $workerToken = '',
|
||||
array $excludedReleaseIds = []
|
||||
): array {
|
||||
): AdditionalBatchResult {
|
||||
$this->finish();
|
||||
if (! $this->setupTempPath($guidChar, $groupID, $workerToken)) {
|
||||
return ['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []];
|
||||
return AdditionalBatchResult::setupFailed($this->lastSetupError);
|
||||
}
|
||||
|
||||
$this->fetchReleases($groupID, $guidChar, $excludedReleaseIds);
|
||||
@@ -65,40 +69,59 @@ class AdditionalProcessingOrchestrator
|
||||
return $this->processReleases($guidChar);
|
||||
}
|
||||
|
||||
return ['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []];
|
||||
return AdditionalBatchResult::empty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single release by GUID.
|
||||
*/
|
||||
public function processSingleGuid(string $guid): bool
|
||||
public function processSingleGuid(string $guid): ReleaseProcessingResult
|
||||
{
|
||||
$releaseId = 0;
|
||||
|
||||
try {
|
||||
$this->finish();
|
||||
$release = Release::where('guid', $guid)->first();
|
||||
if ($release === null) {
|
||||
$this->output->warning('Release not found for GUID: '.$guid);
|
||||
|
||||
return false;
|
||||
return new ReleaseProcessingResult(0, $guid, ProcessingOutcome::NotFound, reason: 'Release not found.');
|
||||
}
|
||||
|
||||
$releaseId = (int) $release->id;
|
||||
|
||||
$this->releases = collect([$release]);
|
||||
$this->totalReleases = 1;
|
||||
$guidChar = $release->leftguid ?? substr($release->guid, 0, 1);
|
||||
$groupID = '';
|
||||
if (! $this->setupTempPath($guidChar, $groupID)) {
|
||||
return false;
|
||||
return new ReleaseProcessingResult(
|
||||
$releaseId,
|
||||
$guid,
|
||||
ProcessingOutcome::TemporaryWorkspaceUnavailable,
|
||||
reason: $this->lastSetupError,
|
||||
);
|
||||
}
|
||||
|
||||
$this->processReleases($guidChar);
|
||||
$result = $this->processReleases($guidChar)->firstResult();
|
||||
|
||||
return true;
|
||||
return $result ?? new ReleaseProcessingResult(
|
||||
$releaseId,
|
||||
$guid,
|
||||
ProcessingOutcome::Failed,
|
||||
reason: 'No processing result was produced.',
|
||||
);
|
||||
} catch (\Throwable $e) {
|
||||
if ($this->config->debugMode) {
|
||||
Log::error('processSingleGuid failed: '.$e->getMessage());
|
||||
}
|
||||
|
||||
return false;
|
||||
return new ReleaseProcessingResult(
|
||||
$releaseId,
|
||||
$guid,
|
||||
ProcessingOutcome::Failed,
|
||||
reason: $e->getMessage(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,6 +130,8 @@ class AdditionalProcessingOrchestrator
|
||||
*/
|
||||
private function setupTempPath(string $guidChar, string $groupID, string $workerToken = ''): bool
|
||||
{
|
||||
$this->lastSetupError = '';
|
||||
|
||||
try {
|
||||
$this->mainTmpPath = $this->tempWorkspace->ensureMainTempPath(
|
||||
$this->config->tmpUnrarPath,
|
||||
@@ -116,6 +141,7 @@ class AdditionalProcessingOrchestrator
|
||||
);
|
||||
$this->tempWorkspace->clearDirectory($this->mainTmpPath, true);
|
||||
} catch (\Throwable $e) {
|
||||
$this->lastSetupError = $e->getMessage();
|
||||
$this->output->warning('Additional post-processing skipped: '.$e->getMessage());
|
||||
Log::error('Additional post-processing temp path is unavailable', [
|
||||
'tmp_unrar_path' => $this->config->tmpUnrarPath,
|
||||
@@ -183,27 +209,31 @@ class AdditionalProcessingOrchestrator
|
||||
* every subsequent cycle, and the "needs additional pp" backlog would
|
||||
* grow indefinitely without any visible failure.
|
||||
*
|
||||
* @return array{claimed: int, processed: int, failed: int, claimed_ids: list<int>}
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
private function processReleases(string $guidChar = ''): array
|
||||
private function processReleases(string $guidChar = ''): AdditionalBatchResult
|
||||
{
|
||||
$startedAt = microtime(true);
|
||||
$startedAt = hrtime(true);
|
||||
$claimedIds = $this->releases
|
||||
->pluck('id')
|
||||
->map(static fn (mixed $id): int => (int) $id)
|
||||
->values()
|
||||
->all();
|
||||
$processed = 0;
|
||||
$failed = 0;
|
||||
$results = [];
|
||||
|
||||
foreach ($this->releases as $release) {
|
||||
$releaseStartedAt = hrtime(true);
|
||||
|
||||
try {
|
||||
$this->processor->process(new ReleaseProcessingContext($release), $this->mainTmpPath);
|
||||
$processed++;
|
||||
$results[] = $this->processor->process(new ReleaseProcessingContext($release), $this->mainTmpPath);
|
||||
} catch (\Throwable $e) {
|
||||
$failed++;
|
||||
$results[] = new ReleaseProcessingResult(
|
||||
releaseId: (int) ($release->id ?? 0),
|
||||
guid: (string) ($release->guid ?? ''),
|
||||
outcome: ProcessingOutcome::Failed,
|
||||
reason: $e->getMessage(),
|
||||
elapsedSeconds: $this->elapsedSecondsSince($releaseStartedAt),
|
||||
);
|
||||
Log::error('Additional postprocessing failed for release '.($release->id ?? '?').': '.$e->getMessage(), [
|
||||
'release_id' => $release->id ?? null,
|
||||
'guid' => $release->guid ?? null,
|
||||
@@ -220,23 +250,63 @@ class AdditionalProcessingOrchestrator
|
||||
}
|
||||
}
|
||||
|
||||
$batchResult = new AdditionalBatchResult(
|
||||
claimedIds: $claimedIds,
|
||||
results: $results,
|
||||
elapsedSeconds: $this->elapsedSecondsSince($startedAt),
|
||||
peakMemoryBytes: memory_get_peak_usage(true),
|
||||
);
|
||||
$downloadMetrics = $batchResult->downloadMetrics();
|
||||
$persistenceMetrics = $batchResult->persistenceMetrics();
|
||||
|
||||
Log::info('Additional postprocessing run finished', [
|
||||
'pipeline' => 'v2',
|
||||
'guid_char' => $guidChar,
|
||||
'picked' => $this->totalReleases,
|
||||
'processed' => $processed,
|
||||
'failed' => $failed,
|
||||
'elapsed_seconds' => round(microtime(true) - $startedAt, 3),
|
||||
'peak_memory_bytes' => memory_get_peak_usage(true),
|
||||
'processed' => $batchResult->successfulCount(),
|
||||
'failed' => $batchResult->unsuccessfulCount(),
|
||||
'outcomes' => $batchResult->outcomeCounts(),
|
||||
'artifacts_created' => $batchResult->artifactsCreatedCount(),
|
||||
'artifact_yield_percent' => round($batchResult->artifactYieldPercent(), 2),
|
||||
'release_files_added' => $batchResult->releaseFilesAdded(),
|
||||
'download_requests' => $downloadMetrics->logicalRequests,
|
||||
'nntp_requests' => $downloadMetrics->networkRequests,
|
||||
'download_cache_hits' => $downloadMetrics->cacheHits,
|
||||
'nntp_bytes' => $downloadMetrics->bytesDownloaded,
|
||||
'reused_bytes' => $downloadMetrics->bytesReused,
|
||||
'database_statements' => $persistenceMetrics->databaseStatements,
|
||||
'database_milliseconds' => round($persistenceMetrics->databaseMilliseconds, 3),
|
||||
'search_sync_requests' => $persistenceMetrics->searchSyncRequests,
|
||||
'search_sync_executions' => $persistenceMetrics->searchSyncExecutions,
|
||||
'duplicate_message_ids' => $batchResult->duplicateMessageIdCount(),
|
||||
'unsupported_reasons' => $batchResult->unsupportedReasonCounts(),
|
||||
'elapsed_seconds' => round($batchResult->elapsedSeconds, 6),
|
||||
'releases_per_second' => round($batchResult->releasesPerSecond(), 4),
|
||||
'average_release_seconds' => round($batchResult->averageReleaseSeconds(), 6),
|
||||
'stage_seconds' => $this->roundedStageDurations($batchResult->stageDurationTotals()),
|
||||
'peak_memory_bytes' => $batchResult->peakMemoryBytes,
|
||||
]);
|
||||
|
||||
$this->output->endOutput();
|
||||
|
||||
return [
|
||||
'claimed' => $this->totalReleases,
|
||||
'processed' => $processed,
|
||||
'failed' => $failed,
|
||||
'claimed_ids' => $claimedIds,
|
||||
];
|
||||
return $batchResult;
|
||||
}
|
||||
|
||||
private function elapsedSecondsSince(int $startedAtNanoseconds): float
|
||||
{
|
||||
return (hrtime(true) - $startedAtNanoseconds) / 1_000_000_000;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, float> $durations
|
||||
* @return array<string, float>
|
||||
*/
|
||||
private function roundedStageDurations(array $durations): array
|
||||
{
|
||||
return array_map(
|
||||
static fn (float $duration): float => round($duration, 6),
|
||||
$durations,
|
||||
);
|
||||
}
|
||||
|
||||
public function finish(): void
|
||||
@@ -249,5 +319,6 @@ class AdditionalProcessingOrchestrator
|
||||
$this->releases = collect();
|
||||
$this->totalReleases = 0;
|
||||
$this->claimToken = '';
|
||||
$this->lastSetupError = '';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalWorkPlan;
|
||||
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
final readonly class AdditionalWorkPlanner
|
||||
{
|
||||
private const string ARCHIVE_PATTERN = '/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)("|\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i';
|
||||
|
||||
public function __construct(private ProcessingConfiguration $config) {}
|
||||
|
||||
/**
|
||||
* @param array<int|string, mixed> $nzbContents
|
||||
*/
|
||||
public function plan(array $nzbContents, string $groupName): AdditionalWorkPlan
|
||||
{
|
||||
$sampleMessageIds = [];
|
||||
$jpgMessageIds = [];
|
||||
$mediaInfoMessageId = '';
|
||||
$audioInfoMessageId = '';
|
||||
$audioInfoExtension = '';
|
||||
$archiveCandidates = [];
|
||||
$bookFileCount = 0;
|
||||
$duplicateMessageIdCount = 0;
|
||||
$seenMessageIds = [];
|
||||
|
||||
foreach (array_values($nzbContents) as $sourceIndex => $file) {
|
||||
if (! is_array($file)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$title = (string) ($file['title'] ?? '');
|
||||
$segments = is_array($file['segments'] ?? null) ? $file['segments'] : [];
|
||||
|
||||
if (preg_match($this->config->ignoreBookRegex, $title) === 1) {
|
||||
$bookFileCount++;
|
||||
}
|
||||
|
||||
if (preg_match(self::ARCHIVE_PATTERN, $title) === 1) {
|
||||
$archiveMessageIds = $this->extractSegments(
|
||||
$segments,
|
||||
$this->config->maximumRarSegments,
|
||||
$seenMessageIds,
|
||||
$duplicateMessageIdCount,
|
||||
);
|
||||
if ($archiveMessageIds !== []) {
|
||||
$archiveCandidates[] = new ArchiveCandidate(
|
||||
title: $title,
|
||||
messageIds: $archiveMessageIds,
|
||||
likelyFirstVolume: $this->isLikelyFirstVolume($title),
|
||||
sourceIndex: $sourceIndex,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->isSupportFile($title)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->config->processThumbnails && $sampleMessageIds === [] && $segments !== []
|
||||
&& stripos($title, 'sample') !== false
|
||||
&& preg_match('/\.(?:jpe?g|png|webp)$/i', $title) !== 1
|
||||
) {
|
||||
$sampleMessageIds = $this->extractSegments(
|
||||
$segments,
|
||||
$this->config->segmentsToDownload,
|
||||
$seenMessageIds,
|
||||
$duplicateMessageIdCount,
|
||||
);
|
||||
}
|
||||
|
||||
if ($this->config->processJPGSample && $jpgMessageIds === [] && $segments !== []
|
||||
&& preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName) !== 1
|
||||
&& preg_match('/\.(?:jpe?g|png|webp)[. ")\]]/i', $title) === 1
|
||||
) {
|
||||
$jpgMessageIds = $this->extractSegments(
|
||||
$segments,
|
||||
$this->config->segmentsToDownload,
|
||||
$seenMessageIds,
|
||||
$duplicateMessageIdCount,
|
||||
);
|
||||
}
|
||||
|
||||
if ($this->config->processMediaInfo && $mediaInfoMessageId === '' && isset($segments[0])
|
||||
&& stripos($title, 'sample') !== false
|
||||
&& preg_match('/'.$this->config->videoFileRegex.'[. ")\]]/i', $title) === 1
|
||||
) {
|
||||
$mediaInfoMessageId = (string) $segments[0];
|
||||
$this->recordMessageId($mediaInfoMessageId, $seenMessageIds, $duplicateMessageIdCount);
|
||||
}
|
||||
|
||||
if ($this->config->processAudioInfo && $audioInfoMessageId === '' && isset($segments[0])
|
||||
&& preg_match('/'.$this->config->audioFileRegex.'[. ")\]]/i', $title, $type) === 1
|
||||
) {
|
||||
$audioInfoExtension = (string) ($type[1] ?? '');
|
||||
$audioInfoMessageId = (string) $segments[0];
|
||||
$this->recordMessageId($audioInfoMessageId, $seenMessageIds, $duplicateMessageIdCount);
|
||||
}
|
||||
} catch (\ErrorException $e) {
|
||||
Log::debug($e->getTraceAsString());
|
||||
}
|
||||
}
|
||||
|
||||
$bookFlood = $bookFileCount > 80 && ($bookFileCount * 2) >= count($nzbContents);
|
||||
$unsupportedReasons = [];
|
||||
if ($bookFlood) {
|
||||
$unsupportedReasons[] = 'book-flood';
|
||||
}
|
||||
if ($archiveCandidates === []
|
||||
&& $sampleMessageIds === []
|
||||
&& $jpgMessageIds === []
|
||||
&& $mediaInfoMessageId === ''
|
||||
&& $audioInfoMessageId === ''
|
||||
) {
|
||||
$unsupportedReasons[] = 'no-supported-candidates';
|
||||
}
|
||||
|
||||
return new AdditionalWorkPlan(
|
||||
sampleMessageIds: $sampleMessageIds,
|
||||
jpgMessageIds: $jpgMessageIds,
|
||||
mediaInfoMessageId: $mediaInfoMessageId,
|
||||
audioInfoMessageId: $audioInfoMessageId,
|
||||
audioInfoExtension: $audioInfoExtension,
|
||||
archiveCandidates: $archiveCandidates,
|
||||
bookFileCount: $bookFileCount,
|
||||
bookFlood: $bookFlood,
|
||||
duplicateMessageIdCount: $duplicateMessageIdCount,
|
||||
unsupportedReasons: $unsupportedReasons,
|
||||
);
|
||||
}
|
||||
|
||||
private function isSupportFile(string $title): bool
|
||||
{
|
||||
return preg_match(
|
||||
'/(?:'.$this->config->supportFileRegex.'|nfo\b|inf\b|ofn\b)($|[ ")]|-])(?!.{20,})/i',
|
||||
$title,
|
||||
) === 1;
|
||||
}
|
||||
|
||||
private function isLikelyFirstVolume(string $title): bool
|
||||
{
|
||||
if (preg_match('/\.part0*(\d+)/i', $title, $part) === 1) {
|
||||
return (int) $part[1] === 1;
|
||||
}
|
||||
|
||||
if (preg_match('/"[a-f0-9]{32}\.[1-9]\d{1,2}".*\((\d+)\/\d{2,}\)$/i', $title, $position) === 1) {
|
||||
return (int) $position[1] === 1;
|
||||
}
|
||||
|
||||
return preg_match('/\.(rar|zip)($|[ ")]|-])/i', $title) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int|string, mixed> $segments
|
||||
* @param array<string, true> $seenMessageIds
|
||||
* @return list<string>
|
||||
*/
|
||||
private function extractSegments(
|
||||
array $segments,
|
||||
int $limit,
|
||||
array &$seenMessageIds,
|
||||
int &$duplicateMessageIdCount,
|
||||
): array {
|
||||
$messageIds = [];
|
||||
$requestMessageIds = [];
|
||||
|
||||
foreach (array_slice($segments, 0, max($limit, 0)) as $segment) {
|
||||
$messageId = (string) $segment;
|
||||
if ($messageId === '' || isset($requestMessageIds[$messageId])) {
|
||||
if ($messageId !== '') {
|
||||
$duplicateMessageIdCount++;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$requestMessageIds[$messageId] = true;
|
||||
$messageIds[] = $messageId;
|
||||
$this->recordMessageId($messageId, $seenMessageIds, $duplicateMessageIdCount);
|
||||
}
|
||||
|
||||
return $messageIds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, true> $seenMessageIds
|
||||
*/
|
||||
private function recordMessageId(string $messageId, array &$seenMessageIds, int &$duplicateMessageIdCount): void
|
||||
{
|
||||
if (isset($seenMessageIds[$messageId])) {
|
||||
$duplicateMessageIdCount++;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$seenMessageIds[$messageId] = true;
|
||||
}
|
||||
}
|
||||
@@ -167,8 +167,8 @@ class ArchiveExtractionService
|
||||
/**
|
||||
* Sort files to prioritize NFO files for processing.
|
||||
*
|
||||
* @param array<string, mixed> $files Array of file info arrays.
|
||||
* @return array<string, mixed> Sorted array with NFO files first.
|
||||
* @param list<array<string, mixed>> $files Array of file info arrays.
|
||||
* @return list<array<string, mixed>> Sorted array with NFO files first.
|
||||
*/
|
||||
public function sortFilesWithNfoPriority(array $files): array
|
||||
{
|
||||
@@ -325,12 +325,31 @@ class ArchiveExtractionService
|
||||
*/
|
||||
public function extractSpecificFile(string $compressedData, string $filename, string $tmpPath): ?string
|
||||
{
|
||||
// Try using ArchiveInfo's built-in extraction
|
||||
return $this->extractSpecificFiles($compressedData, [$filename], $tmpPath)[$filename] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect one archive and extract several exact filenames from it.
|
||||
*
|
||||
* @param list<string> $filenames
|
||||
* @return array<string, string>
|
||||
*/
|
||||
public function extractSpecificFiles(string $compressedData, array $filenames, string $tmpPath): array
|
||||
{
|
||||
$filenames = array_values(array_unique(array_filter(
|
||||
$filenames,
|
||||
static fn (string $filename): bool => $filename !== '',
|
||||
)));
|
||||
$extractedFiles = [];
|
||||
$remaining = $filenames;
|
||||
|
||||
if ($this->archiveInfo->setData($compressedData, true)) {
|
||||
foreach ($filenames as $filename) {
|
||||
try {
|
||||
$extracted = $this->archiveInfo->getFileData($filename);
|
||||
if ($extracted !== false && ! empty($extracted)) {
|
||||
return $extracted;
|
||||
if ($extracted !== false && $extracted !== '') {
|
||||
$extractedFiles[$filename] = $extracted;
|
||||
$remaining = array_values(array_diff($remaining, [$filename]));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
if ($this->config->debugMode) {
|
||||
@@ -338,10 +357,23 @@ class ArchiveExtractionService
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use external tools to extract to temp directory
|
||||
foreach ($remaining as $filename) {
|
||||
$extracted = $this->extractSpecificFileWithExternalTools($compressedData, $filename, $tmpPath);
|
||||
if ($extracted !== null) {
|
||||
$extractedFiles[$filename] = $extracted;
|
||||
}
|
||||
}
|
||||
|
||||
// Try using unrar for RAR files
|
||||
return $extractedFiles;
|
||||
}
|
||||
|
||||
private function extractSpecificFileWithExternalTools(
|
||||
string $compressedData,
|
||||
string $filename,
|
||||
string $tmpPath,
|
||||
): ?string {
|
||||
if ($this->config->unrarPath) {
|
||||
$extracted = $this->extractFileViaExternalTool(
|
||||
$compressedData,
|
||||
@@ -356,7 +388,6 @@ class ArchiveExtractionService
|
||||
}
|
||||
}
|
||||
|
||||
// Try using unzip for ZIP files
|
||||
if ($this->config->unzipPath) {
|
||||
$extracted = $this->extractFileViaExternalTool(
|
||||
$compressedData,
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
|
||||
final readonly class AdditionalBatchResult
|
||||
{
|
||||
/**
|
||||
* @param list<int> $claimedIds
|
||||
* @param list<ReleaseProcessingResult> $results
|
||||
*/
|
||||
public function __construct(
|
||||
public array $claimedIds = [],
|
||||
public array $results = [],
|
||||
public string $setupFailure = '',
|
||||
public float $elapsedSeconds = 0.0,
|
||||
public int $peakMemoryBytes = 0,
|
||||
) {}
|
||||
|
||||
public static function empty(): self
|
||||
{
|
||||
return new self;
|
||||
}
|
||||
|
||||
public static function setupFailed(string $reason): self
|
||||
{
|
||||
return new self(setupFailure: $reason);
|
||||
}
|
||||
|
||||
public function claimedCount(): int
|
||||
{
|
||||
return count($this->claimedIds);
|
||||
}
|
||||
|
||||
public function attemptedCount(): int
|
||||
{
|
||||
return count($this->results);
|
||||
}
|
||||
|
||||
public function successfulCount(): int
|
||||
{
|
||||
return count(array_filter(
|
||||
$this->results,
|
||||
static fn (ReleaseProcessingResult $result): bool => $result->isSuccessful(),
|
||||
));
|
||||
}
|
||||
|
||||
public function unsuccessfulCount(): int
|
||||
{
|
||||
return $this->attemptedCount() - $this->successfulCount();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, int>
|
||||
*/
|
||||
public function outcomeCounts(): array
|
||||
{
|
||||
$counts = [];
|
||||
|
||||
foreach ($this->results as $result) {
|
||||
$outcome = $result->outcome->value;
|
||||
$counts[$outcome] = ($counts[$outcome] ?? 0) + 1;
|
||||
}
|
||||
|
||||
return $counts;
|
||||
}
|
||||
|
||||
public function firstResult(): ?ReleaseProcessingResult
|
||||
{
|
||||
return $this->results[0] ?? null;
|
||||
}
|
||||
|
||||
public function hasOutcome(ProcessingOutcome $outcome): bool
|
||||
{
|
||||
foreach ($this->results as $result) {
|
||||
if ($result->outcome === $outcome) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public function withPerformance(float $elapsedSeconds, int $peakMemoryBytes): self
|
||||
{
|
||||
return new self(
|
||||
claimedIds: $this->claimedIds,
|
||||
results: $this->results,
|
||||
setupFailure: $this->setupFailure,
|
||||
elapsedSeconds: $elapsedSeconds,
|
||||
peakMemoryBytes: $peakMemoryBytes,
|
||||
);
|
||||
}
|
||||
|
||||
public function releasesPerSecond(): float
|
||||
{
|
||||
if ($this->elapsedSeconds <= 0.0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
return $this->attemptedCount() / $this->elapsedSeconds;
|
||||
}
|
||||
|
||||
public function averageReleaseSeconds(): float
|
||||
{
|
||||
if ($this->attemptedCount() === 0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
return array_sum(array_map(
|
||||
static fn (ReleaseProcessingResult $result): float => $result->elapsedSeconds,
|
||||
$this->results,
|
||||
)) / $this->attemptedCount();
|
||||
}
|
||||
|
||||
public function artifactsCreatedCount(): int
|
||||
{
|
||||
return count(array_filter(
|
||||
$this->results,
|
||||
static fn (ReleaseProcessingResult $result): bool => $result->artifactsCreated,
|
||||
));
|
||||
}
|
||||
|
||||
public function artifactYieldPercent(): float
|
||||
{
|
||||
if ($this->successfulCount() === 0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
$successfulArtifacts = count(array_filter(
|
||||
$this->results,
|
||||
static fn (ReleaseProcessingResult $result): bool => $result->isSuccessful() && $result->artifactsCreated,
|
||||
));
|
||||
|
||||
return ($successfulArtifacts / $this->successfulCount()) * 100;
|
||||
}
|
||||
|
||||
public function releaseFilesAdded(): int
|
||||
{
|
||||
return array_sum(array_map(
|
||||
static fn (ReleaseProcessingResult $result): int => $result->releaseFilesAdded,
|
||||
$this->results,
|
||||
));
|
||||
}
|
||||
|
||||
public function downloadMetrics(): DownloadMetrics
|
||||
{
|
||||
$logicalRequests = 0;
|
||||
$networkRequests = 0;
|
||||
$cacheHits = 0;
|
||||
$bytesDownloaded = 0;
|
||||
$bytesReused = 0;
|
||||
|
||||
foreach ($this->results as $result) {
|
||||
if ($result->downloadMetrics === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$logicalRequests += $result->downloadMetrics->logicalRequests;
|
||||
$networkRequests += $result->downloadMetrics->networkRequests;
|
||||
$cacheHits += $result->downloadMetrics->cacheHits;
|
||||
$bytesDownloaded += $result->downloadMetrics->bytesDownloaded;
|
||||
$bytesReused += $result->downloadMetrics->bytesReused;
|
||||
}
|
||||
|
||||
return new DownloadMetrics(
|
||||
logicalRequests: $logicalRequests,
|
||||
networkRequests: $networkRequests,
|
||||
cacheHits: $cacheHits,
|
||||
bytesDownloaded: $bytesDownloaded,
|
||||
bytesReused: $bytesReused,
|
||||
);
|
||||
}
|
||||
|
||||
public function persistenceMetrics(): PersistenceMetrics
|
||||
{
|
||||
$databaseStatements = 0;
|
||||
$databaseMilliseconds = 0.0;
|
||||
$searchSyncRequests = 0;
|
||||
$searchSyncExecutions = 0;
|
||||
|
||||
foreach ($this->results as $result) {
|
||||
if ($result->persistenceMetrics === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$databaseStatements += $result->persistenceMetrics->databaseStatements;
|
||||
$databaseMilliseconds += $result->persistenceMetrics->databaseMilliseconds;
|
||||
$searchSyncRequests += $result->persistenceMetrics->searchSyncRequests;
|
||||
$searchSyncExecutions += $result->persistenceMetrics->searchSyncExecutions;
|
||||
}
|
||||
|
||||
return new PersistenceMetrics(
|
||||
databaseStatements: $databaseStatements,
|
||||
databaseMilliseconds: $databaseMilliseconds,
|
||||
searchSyncRequests: $searchSyncRequests,
|
||||
searchSyncExecutions: $searchSyncExecutions,
|
||||
);
|
||||
}
|
||||
|
||||
public function duplicateMessageIdCount(): int
|
||||
{
|
||||
return array_sum(array_map(
|
||||
static fn (ReleaseProcessingResult $result): int => $result->duplicateMessageIdCount,
|
||||
$this->results,
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, int>
|
||||
*/
|
||||
public function unsupportedReasonCounts(): array
|
||||
{
|
||||
$counts = [];
|
||||
|
||||
foreach ($this->results as $result) {
|
||||
foreach ($result->unsupportedReasons as $reason) {
|
||||
$counts[$reason] = ($counts[$reason] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return $counts;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, float>
|
||||
*/
|
||||
public function stageDurationTotals(): array
|
||||
{
|
||||
$totals = [];
|
||||
|
||||
foreach ($this->results as $result) {
|
||||
foreach ($result->stageDurations as $stage => $duration) {
|
||||
$totals[$stage] = ($totals[$stage] ?? 0.0) + $duration;
|
||||
}
|
||||
}
|
||||
|
||||
return $totals;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
final readonly class AdditionalWorkPlan
|
||||
{
|
||||
/**
|
||||
* @param list<string> $sampleMessageIds
|
||||
* @param list<string> $jpgMessageIds
|
||||
* @param list<ArchiveCandidate> $archiveCandidates
|
||||
* @param list<string> $unsupportedReasons
|
||||
*/
|
||||
public function __construct(
|
||||
public array $sampleMessageIds = [],
|
||||
public array $jpgMessageIds = [],
|
||||
public string $mediaInfoMessageId = '',
|
||||
public string $audioInfoMessageId = '',
|
||||
public string $audioInfoExtension = '',
|
||||
public array $archiveCandidates = [],
|
||||
public int $bookFileCount = 0,
|
||||
public bool $bookFlood = false,
|
||||
public int $duplicateMessageIdCount = 0,
|
||||
public array $unsupportedReasons = [],
|
||||
) {}
|
||||
|
||||
public function hasCompressedFile(): bool
|
||||
{
|
||||
return $this->archiveCandidates !== [];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<ArchiveCandidate>
|
||||
*/
|
||||
public function orderedArchiveCandidates(bool $reverse = false): array
|
||||
{
|
||||
return $reverse ? array_reverse($this->archiveCandidates) : $this->archiveCandidates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a stable priority view without changing the legacy processing order.
|
||||
*
|
||||
* @return list<ArchiveCandidate>
|
||||
*/
|
||||
public function prioritizedArchiveCandidates(): array
|
||||
{
|
||||
$firstVolumes = [];
|
||||
$remaining = [];
|
||||
|
||||
foreach ($this->archiveCandidates as $candidate) {
|
||||
if ($candidate->likelyFirstVolume) {
|
||||
$firstVolumes[] = $candidate;
|
||||
} else {
|
||||
$remaining[] = $candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return [...$firstVolumes, ...$remaining];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
final readonly class ArchiveCandidate
|
||||
{
|
||||
/**
|
||||
* @param list<string> $messageIds
|
||||
*/
|
||||
public function __construct(
|
||||
public string $title,
|
||||
public array $messageIds,
|
||||
public bool $likelyFirstVolume,
|
||||
public int $sourceIndex,
|
||||
) {}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
final readonly class DownloadMetrics
|
||||
{
|
||||
public function __construct(
|
||||
public int $logicalRequests = 0,
|
||||
public int $networkRequests = 0,
|
||||
public int $cacheHits = 0,
|
||||
public int $bytesDownloaded = 0,
|
||||
public int $bytesReused = 0,
|
||||
) {}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
final readonly class DownloadedArchive
|
||||
{
|
||||
/**
|
||||
* @param array<int, array<string, mixed>> $files
|
||||
*/
|
||||
public function __construct(
|
||||
public string $title,
|
||||
public string $data,
|
||||
public array $files,
|
||||
) {}
|
||||
|
||||
public function byteSize(): int
|
||||
{
|
||||
return strlen($this->data);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
final readonly class PersistenceMetrics
|
||||
{
|
||||
public function __construct(
|
||||
public int $databaseStatements = 0,
|
||||
public float $databaseMilliseconds = 0.0,
|
||||
public int $searchSyncRequests = 0,
|
||||
public int $searchSyncExecutions = 0,
|
||||
) {}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\DTO;
|
||||
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
|
||||
final readonly class ReleaseProcessingResult
|
||||
{
|
||||
/**
|
||||
* @param array<string, float> $stageDurations
|
||||
* @param list<string> $unsupportedReasons
|
||||
*/
|
||||
public function __construct(
|
||||
public int $releaseId,
|
||||
public string $guid,
|
||||
public ProcessingOutcome $outcome,
|
||||
public bool $artifactsCreated = false,
|
||||
public int $releaseFilesAdded = 0,
|
||||
public string $reason = '',
|
||||
public float $elapsedSeconds = 0.0,
|
||||
public array $stageDurations = [],
|
||||
public ?DownloadMetrics $downloadMetrics = null,
|
||||
public ?PersistenceMetrics $persistenceMetrics = null,
|
||||
public int $duplicateMessageIdCount = 0,
|
||||
public array $unsupportedReasons = [],
|
||||
) {}
|
||||
|
||||
public function isSuccessful(): bool
|
||||
{
|
||||
return $this->outcome->isSuccessful();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, float> $stageDurations
|
||||
*/
|
||||
public function withPerformance(
|
||||
float $elapsedSeconds,
|
||||
array $stageDurations,
|
||||
?DownloadMetrics $downloadMetrics = null,
|
||||
?PersistenceMetrics $persistenceMetrics = null,
|
||||
): self {
|
||||
return new self(
|
||||
releaseId: $this->releaseId,
|
||||
guid: $this->guid,
|
||||
outcome: $this->outcome,
|
||||
artifactsCreated: $this->artifactsCreated,
|
||||
releaseFilesAdded: $this->releaseFilesAdded,
|
||||
reason: $this->reason,
|
||||
elapsedSeconds: $elapsedSeconds,
|
||||
stageDurations: $stageDurations,
|
||||
downloadMetrics: $downloadMetrics ?? $this->downloadMetrics,
|
||||
persistenceMetrics: $persistenceMetrics ?? $this->persistenceMetrics,
|
||||
duplicateMessageIdCount: $this->duplicateMessageIdCount,
|
||||
unsupportedReasons: $this->unsupportedReasons,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\Enums;
|
||||
|
||||
enum ProcessingOutcome: string
|
||||
{
|
||||
case Completed = 'completed';
|
||||
case NoUsefulArtifacts = 'no-useful-artifacts';
|
||||
case Passworded = 'passworded';
|
||||
case GroupUnavailable = 'group-unavailable';
|
||||
case TemporaryWorkspaceUnavailable = 'temporary-workspace-unavailable';
|
||||
case TimedOut = 'timed-out';
|
||||
case DeletedAfterTimeout = 'deleted-after-timeout';
|
||||
case DeletedBrokenNzb = 'deleted-broken-nzb';
|
||||
case NotFound = 'not-found';
|
||||
case Failed = 'failed';
|
||||
|
||||
public function isSuccessful(): bool
|
||||
{
|
||||
return match ($this) {
|
||||
self::Completed, self::NoUsefulArtifacts, self::Passworded => true,
|
||||
default => false,
|
||||
};
|
||||
}
|
||||
|
||||
public function isDeleted(): bool
|
||||
{
|
||||
return match ($this) {
|
||||
self::DeletedAfterTimeout, self::DeletedBrokenNzb => true,
|
||||
default => false,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\Enums;
|
||||
|
||||
enum ProcessingStage: string
|
||||
{
|
||||
case WorkspacePreparation = 'workspace-preparation';
|
||||
case NzbParsing = 'nzb-parsing';
|
||||
case ReleaseInitialization = 'release-initialization';
|
||||
case MessageIdSelection = 'message-id-selection';
|
||||
case DirectDownloads = 'direct-downloads';
|
||||
case ArchiveDownloads = 'archive-downloads';
|
||||
case ExtractedFiles = 'extracted-files';
|
||||
case ArchiveFallbacks = 'archive-fallbacks';
|
||||
case TimeoutHandling = 'timeout-handling';
|
||||
case Finalization = 'finalization';
|
||||
case WorkspaceCleanup = 'workspace-cleanup';
|
||||
}
|
||||
@@ -5,10 +5,10 @@ declare(strict_types=1);
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Enums\ImageAssetProfile;
|
||||
use App\Facades\Search;
|
||||
use App\Models\Category;
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use App\Services\NameFixing\ReleaseUpdateService;
|
||||
@@ -38,12 +38,20 @@ class MediaExtractionService
|
||||
|
||||
private ?MediaInfo $mediaInfo = null;
|
||||
|
||||
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
|
||||
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
private readonly ReleaseImageService $releaseImage,
|
||||
private readonly ReleaseExtraService $releaseExtra,
|
||||
private readonly CategorizationService $categorize
|
||||
) {}
|
||||
private readonly CategorizationService $categorize,
|
||||
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
|
||||
) {
|
||||
$this->searchSyncCoordinator = $searchSyncCoordinator
|
||||
?? new ReleaseSearchSyncCoordinator(
|
||||
new PersistenceMetricsCollector,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get video time code for sample extraction.
|
||||
@@ -329,7 +337,7 @@ class MediaExtractionService
|
||||
'proc_pp' => 1,
|
||||
]);
|
||||
|
||||
Search::updateRelease($context->release->id);
|
||||
$this->searchSyncCoordinator->request((int) $context->release->id);
|
||||
|
||||
if ($this->config->echoCLI) {
|
||||
$releaseInfo = (object) [
|
||||
|
||||
@@ -147,11 +147,11 @@ class NzbContentParser
|
||||
/**
|
||||
* Process NZB contents to extract message IDs for different file types.
|
||||
*
|
||||
* @param array<string, mixed> $nzbContents
|
||||
* @return array<string, mixed>
|
||||
* @param list<array<string, mixed>> $nzbContents
|
||||
* @return array{
|
||||
* hasCompressedFile: bool,
|
||||
* sampleMessageIDs: array,
|
||||
* jpgMessageIDs: array,
|
||||
* sampleMessageIDs: list<string>,
|
||||
* jpgMessageIDs: list<string>,
|
||||
* mediaInfoMessageID: string,
|
||||
* audioInfoMessageID: string,
|
||||
* audioInfoExtension: string,
|
||||
@@ -194,7 +194,7 @@ class NzbContentParser
|
||||
// Look for a video sample (not an image)
|
||||
if ($config->processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments)
|
||||
&& stripos($title, 'sample') !== false
|
||||
&& ! preg_match('/\.jpe?g$/i', $title)
|
||||
&& ! preg_match('/\.(?:jpe?g|png|webp)$/i', $title)
|
||||
) {
|
||||
$result['sampleMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
|
||||
}
|
||||
@@ -202,7 +202,7 @@ class NzbContentParser
|
||||
// Look for a JPG picture (not a CD cover)
|
||||
if ($config->processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments)
|
||||
&& ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName)
|
||||
&& preg_match('/\.jpe?g[. ")\]]/i', $title)
|
||||
&& preg_match('/\.(?:jpe?g|png|webp)[. ")\]]/i', $title)
|
||||
) {
|
||||
$result['jpgMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
|
||||
}
|
||||
@@ -238,18 +238,14 @@ class NzbContentParser
|
||||
/**
|
||||
* Extract segment message IDs up to a limit.
|
||||
*
|
||||
* @param array<string, mixed> $segments
|
||||
* @return array<string, mixed>
|
||||
* @param array<int|string, mixed> $segments
|
||||
* @return list<string>
|
||||
*/
|
||||
private function extractSegments(array $segments, int $limit): array
|
||||
{
|
||||
$ids = [];
|
||||
$segCount = count($segments) - 1;
|
||||
for ($i = 0; $i < $limit; $i++) {
|
||||
if ($i > $segCount) {
|
||||
break;
|
||||
}
|
||||
$ids[] = (string) $segments[$i]; // @phpstan-ignore offsetAccess.notFound
|
||||
foreach (array_slice(array_values($segments), 0, max(0, $limit)) as $segment) {
|
||||
$ids[] = (string) $segment;
|
||||
}
|
||||
|
||||
return $ids;
|
||||
|
||||
@@ -12,6 +12,7 @@ use App\Models\Predb;
|
||||
use App\Models\Release;
|
||||
use App\Models\ReleaseFile;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\NameFixing\FileNameCleaner;
|
||||
use App\Services\NameFixing\NameFixingService;
|
||||
@@ -24,6 +25,7 @@ use App\Services\Releases\ReleaseBrowseService;
|
||||
use dariusiii\rarinfo\Par2Info;
|
||||
use Illuminate\Contracts\Filesystem\FileNotFoundException;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\File;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
@@ -37,6 +39,8 @@ class ReleaseFileManager
|
||||
|
||||
private readonly FileNameCleaner $fileNameCleaner;
|
||||
|
||||
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
|
||||
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
private readonly ReleaseImageService $releaseImage,
|
||||
@@ -44,9 +48,15 @@ class ReleaseFileManager
|
||||
private readonly NzbService $nzb,
|
||||
private readonly NameFixingService $nameFixingService,
|
||||
?ReleaseUpdateService $releaseUpdateService = null,
|
||||
?FileNameCleaner $fileNameCleaner = null
|
||||
?FileNameCleaner $fileNameCleaner = null,
|
||||
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
|
||||
) {
|
||||
$this->releaseUpdateService = $releaseUpdateService ?? new ReleaseUpdateService;
|
||||
$this->searchSyncCoordinator = $searchSyncCoordinator
|
||||
?? new ReleaseSearchSyncCoordinator(
|
||||
new PersistenceMetricsCollector,
|
||||
);
|
||||
$this->releaseUpdateService = $releaseUpdateService
|
||||
?? new ReleaseUpdateService(searchSyncCoordinator: $this->searchSyncCoordinator);
|
||||
$this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner;
|
||||
}
|
||||
|
||||
@@ -156,7 +166,7 @@ class ReleaseFileManager
|
||||
*/
|
||||
public function updateSearchIndex(int $releaseId): void
|
||||
{
|
||||
Search::updateRelease($releaseId);
|
||||
$this->searchSyncCoordinator->request($releaseId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -213,8 +223,6 @@ class ReleaseFileManager
|
||||
*/
|
||||
public function finalizeRelease(ReleaseProcessingContext $context, bool $processPasswords): void
|
||||
{
|
||||
$this->flushQueuedReleaseFiles($context);
|
||||
|
||||
$updateRows = ['haspreview' => 0];
|
||||
|
||||
// Check for existing samples
|
||||
@@ -230,9 +238,6 @@ class ReleaseFileManager
|
||||
$updateRows['jpgstatus'] = 1;
|
||||
}
|
||||
|
||||
// Get file count
|
||||
$releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0;
|
||||
|
||||
$passwordStatus = $context->passwordStatus;
|
||||
|
||||
// Set to no password if processing is off
|
||||
@@ -242,7 +247,27 @@ class ReleaseFileManager
|
||||
|
||||
$updateRows = array_merge($updateRows, AdditionalCandidateQuery::claimResetValues());
|
||||
|
||||
// Update based on conditions
|
||||
$pendingReleaseFiles = array_values($context->pendingReleaseFiles);
|
||||
$pendingParHashes = array_values($context->pendingParHashes);
|
||||
|
||||
$insertedReleaseFiles = DB::transaction(function () use (
|
||||
$context,
|
||||
$pendingReleaseFiles,
|
||||
$pendingParHashes,
|
||||
$updateRows,
|
||||
$processPasswords,
|
||||
$passwordStatus,
|
||||
): int {
|
||||
$inserted = $pendingReleaseFiles === []
|
||||
? 0
|
||||
: ReleaseFile::query()->insertOrIgnore($pendingReleaseFiles);
|
||||
|
||||
if ($pendingParHashes !== []) {
|
||||
ParHash::query()->insertOrIgnore($pendingParHashes);
|
||||
}
|
||||
|
||||
$releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0;
|
||||
|
||||
if (! $context->releaseHasPassword && $context->nzbHasCompressedFile && $releaseFilesCount === 0) {
|
||||
Release::query()->where('id', $context->release->id)->update($updateRows);
|
||||
} else {
|
||||
@@ -251,7 +276,14 @@ class ReleaseFileManager
|
||||
Release::query()->where('id', $context->release->id)->update($updateRows);
|
||||
}
|
||||
|
||||
Search::updateRelease((int) $context->release->id);
|
||||
return $inserted;
|
||||
}, 3);
|
||||
|
||||
$context->pendingReleaseFiles = [];
|
||||
$context->pendingParHashes = [];
|
||||
$context->releaseFilesChanged = $context->releaseFilesChanged || $insertedReleaseFiles > 0;
|
||||
|
||||
$this->searchSyncCoordinator->request((int) $context->release->id);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -283,7 +315,7 @@ class ReleaseFileManager
|
||||
'passwordstatus' => ReleaseBrowseService::PASSWD_NONE,
|
||||
], AdditionalCandidateQuery::claimResetValues()));
|
||||
|
||||
Search::updateRelease((int) $release->id);
|
||||
$this->searchSyncCoordinator->request((int) $release->id);
|
||||
|
||||
Log::warning('Release '.$release->id.' skipped after post-processing timeout ('.$newCount.'/'.$maxTimeoutCount.')');
|
||||
|
||||
@@ -302,6 +334,7 @@ class ReleaseFileManager
|
||||
|
||||
$id = (int) $release->id;
|
||||
$guid = $release->guid ?? '';
|
||||
$this->searchSyncCoordinator->discard($id);
|
||||
|
||||
// Delete NZB file
|
||||
try {
|
||||
@@ -352,7 +385,7 @@ class ReleaseFileManager
|
||||
Release::where('id', $id)->delete();
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
} catch (\Throwable) { // @phpstan-ignore catch.neverThrown
|
||||
} catch (\Throwable) {
|
||||
// Last resort: swallow any exception
|
||||
}
|
||||
}
|
||||
@@ -693,20 +726,6 @@ class ReleaseFileManager
|
||||
return true;
|
||||
}
|
||||
|
||||
private function flushQueuedReleaseFiles(ReleaseProcessingContext $context): void
|
||||
{
|
||||
if ($context->pendingReleaseFiles !== []) {
|
||||
$inserted = ReleaseFile::query()->insertOrIgnore(array_values($context->pendingReleaseFiles));
|
||||
$context->releaseFilesChanged = $context->releaseFilesChanged || $inserted > 0;
|
||||
$context->pendingReleaseFiles = [];
|
||||
}
|
||||
|
||||
if ($context->pendingParHashes !== []) {
|
||||
ParHash::query()->insertOrIgnore(array_values($context->pendingParHashes));
|
||||
$context->pendingParHashes = [];
|
||||
}
|
||||
}
|
||||
|
||||
private function loadExistingReleaseFileNames(ReleaseProcessingContext $context): void
|
||||
{
|
||||
if ($context->existingReleaseFileNames !== null) {
|
||||
|
||||
@@ -5,7 +5,7 @@ declare(strict_types=1);
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Models\ReleaseFile;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
|
||||
use App\Services\AdditionalProcessing\Enums\DownloadKind;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
|
||||
@@ -17,7 +17,6 @@ use Illuminate\Support\Facades\File;
|
||||
class ReleaseFilesArchiveFallback
|
||||
{
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
private readonly ArchiveExtractionService $archiveService,
|
||||
private readonly MediaExtractionService $mediaService,
|
||||
private readonly UsenetDownloadService $downloadService,
|
||||
@@ -26,7 +25,7 @@ class ReleaseFilesArchiveFallback
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Try JPG/PNG files listed inside an already-downloaded archive summary.
|
||||
* Try JPG/PNG/WebP files listed inside an already-downloaded archive summary.
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $files
|
||||
*/
|
||||
@@ -37,7 +36,7 @@ class ReleaseFilesArchiveFallback
|
||||
): void {
|
||||
$imageFiles = array_values(array_filter(
|
||||
$files,
|
||||
static fn (array $file): bool => preg_match('/\.(jpe?g|png)$/i', (string) ($file['name'] ?? '')) === 1
|
||||
static fn (array $file): bool => preg_match('/\.(jpe?g|png|webp)$/i', (string) ($file['name'] ?? '')) === 1
|
||||
));
|
||||
|
||||
if ($imageFiles === []) {
|
||||
@@ -69,7 +68,7 @@ class ReleaseFilesArchiveFallback
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-download archives and extract stored JPG/PNG candidates.
|
||||
* Re-download archives and extract stored JPG/PNG/WebP candidates.
|
||||
*/
|
||||
public function processJpgFromReleaseFiles(ReleaseProcessingContext $context): void
|
||||
{
|
||||
@@ -78,7 +77,8 @@ class ReleaseFilesArchiveFallback
|
||||
->where(function ($query) {
|
||||
$query->where('name', 'like', '%.jpg')
|
||||
->orWhere('name', 'like', '%.jpeg')
|
||||
->orWhere('name', 'like', '%.png');
|
||||
->orWhere('name', 'like', '%.png')
|
||||
->orWhere('name', 'like', '%.webp');
|
||||
})
|
||||
->orderByDesc('size')
|
||||
->limit(3)
|
||||
@@ -144,6 +144,45 @@ class ReleaseFilesArchiveFallback
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract NFO candidates from archives already downloaded for this release.
|
||||
*/
|
||||
public function processNfoFromDownloadedArchives(ReleaseProcessingContext $context): void
|
||||
{
|
||||
if ($context->downloadedArchives === [] || ! $context->releaseHasNoNFO) {
|
||||
return;
|
||||
}
|
||||
|
||||
$nntp = $this->downloadService->getNNTP();
|
||||
|
||||
foreach ($context->downloadedArchives as $archive) {
|
||||
$nfoFiles = array_values(array_filter(
|
||||
$this->archiveService->sortFilesWithNfoPriority($archive->files),
|
||||
fn (array $file): bool => $this->archiveService->isNfoFile((string) ($file['name'] ?? '')),
|
||||
));
|
||||
|
||||
$this->processExtractedCandidatesFromData(
|
||||
$archive->data,
|
||||
array_slice($nfoFiles, 0, 3),
|
||||
$context,
|
||||
'downloaded_nfo_',
|
||||
function (string $tempPath, ReleaseProcessingContext $context) use ($nntp): bool {
|
||||
if (! $this->releaseManager->processNfoFile($tempPath, $context, $nntp)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->output->echoNfoFound();
|
||||
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
if (! $context->releaseHasNoNFO) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param EloquentCollection<int, ReleaseFile> $candidates
|
||||
* @param callable(ReleaseProcessingContext): bool $shouldStop
|
||||
@@ -156,34 +195,22 @@ class ReleaseFilesArchiveFallback
|
||||
callable $shouldStop,
|
||||
callable $handler
|
||||
): void {
|
||||
if ($candidates->isEmpty() || $context->nzbContents === []) {
|
||||
$archiveCandidates = $this->archiveCandidates($context);
|
||||
if ($candidates->isEmpty() || $archiveCandidates === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($context->nzbContents as $nzbFile) {
|
||||
foreach ($archiveCandidates as $archiveCandidate) {
|
||||
if ($shouldStop($context)) {
|
||||
break;
|
||||
}
|
||||
|
||||
$title = (string) ($nzbFile['title'] ?? '');
|
||||
if (preg_match(
|
||||
'/(\.(part0*1|rar|zip))(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
|
||||
$title
|
||||
) !== 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$messageIDs = $this->extractInitialSegments($nzbFile['segments'] ?? []);
|
||||
if ($messageIDs === []) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$result = $this->downloadService->download(
|
||||
DownloadKind::Compressed,
|
||||
$messageIDs,
|
||||
$archiveCandidate->messageIds,
|
||||
$context->releaseGroupName,
|
||||
$context->release->id,
|
||||
$title
|
||||
$archiveCandidate->title,
|
||||
);
|
||||
|
||||
if ($result['groupUnavailable']) {
|
||||
@@ -197,13 +224,20 @@ class ReleaseFilesArchiveFallback
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($candidates as $candidate) {
|
||||
$fileData = $this->archiveService->extractSpecificFile(
|
||||
$filenames = $candidates
|
||||
->pluck('name')
|
||||
->filter(static fn (mixed $filename): bool => is_string($filename) && $filename !== '')
|
||||
->values()
|
||||
->all();
|
||||
$extractedFiles = $this->extractCandidateFiles(
|
||||
$result['data'],
|
||||
$candidate->name,
|
||||
$context->tmpPath
|
||||
$filenames,
|
||||
$context->tmpPath,
|
||||
);
|
||||
|
||||
foreach ($candidates as $candidate) {
|
||||
$fileData = $extractedFiles[$candidate->name] ?? null;
|
||||
|
||||
if ($fileData === null || $fileData === '') {
|
||||
continue;
|
||||
}
|
||||
@@ -226,13 +260,23 @@ class ReleaseFilesArchiveFallback
|
||||
string $tempPrefix,
|
||||
callable $handler
|
||||
): void {
|
||||
$filenames = array_values(array_filter(array_map(
|
||||
static fn (array $candidate): string => (string) ($candidate['name'] ?? ''),
|
||||
$candidates,
|
||||
)));
|
||||
$extractedFiles = $this->extractCandidateFiles(
|
||||
$compressedData,
|
||||
$filenames,
|
||||
$context->tmpPath,
|
||||
);
|
||||
|
||||
foreach ($candidates as $candidate) {
|
||||
$filename = (string) ($candidate['name'] ?? '');
|
||||
if ($filename === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$fileData = $this->archiveService->extractSpecificFile($compressedData, $filename, $context->tmpPath);
|
||||
$fileData = $extractedFiles[$filename] ?? null;
|
||||
if ($fileData === null || $fileData === '') {
|
||||
continue;
|
||||
}
|
||||
@@ -243,6 +287,30 @@ class ReleaseFilesArchiveFallback
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<ArchiveCandidate>
|
||||
*/
|
||||
private function archiveCandidates(ReleaseProcessingContext $context): array
|
||||
{
|
||||
if ($context->workPlan === null) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return array_values(array_filter(
|
||||
$context->workPlan->archiveCandidates,
|
||||
static fn (ArchiveCandidate $candidate): bool => $candidate->likelyFirstVolume,
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<string> $filenames
|
||||
* @return array<string, string>
|
||||
*/
|
||||
private function extractCandidateFiles(string $archiveData, array $filenames, string $tmpPath): array
|
||||
{
|
||||
return $this->archiveService->extractSpecificFiles($archiveData, $filenames, $tmpPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param callable(string, ReleaseProcessingContext): bool $handler
|
||||
*/
|
||||
@@ -264,20 +332,4 @@ class ReleaseFilesArchiveFallback
|
||||
File::delete($tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, mixed> $segments
|
||||
* @return list<string>
|
||||
*/
|
||||
private function extractInitialSegments(array $segments): array
|
||||
{
|
||||
$messageIDs = [];
|
||||
$segmentCount = min(count($segments), $this->config->maximumRarSegments);
|
||||
|
||||
for ($index = 0; $index < $segmentCount; $index++) {
|
||||
$messageIDs[] = (string) $segments[$index];
|
||||
}
|
||||
|
||||
return $messageIDs;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,12 @@ namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use App\Services\AdditionalProcessing\Enums\DownloadKind;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\State\ProcessingMetrics;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\Releases\ReleaseBrowseService;
|
||||
use App\Services\TempWorkspaceService;
|
||||
@@ -25,99 +30,203 @@ class ReleaseProcessor
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
private readonly NzbContentParser $nzbParser,
|
||||
private readonly AdditionalWorkPlanner $workPlanner,
|
||||
private readonly ArchiveExtractionService $archiveService,
|
||||
private readonly MediaExtractionService $mediaService,
|
||||
private readonly UsenetDownloadService $downloadService,
|
||||
private readonly ReleaseFileManager $releaseManager,
|
||||
private readonly ReleaseFilesArchiveFallback $archiveFallback,
|
||||
private readonly TempWorkspaceService $tempWorkspace,
|
||||
private readonly ConsoleOutputService $output
|
||||
private readonly ConsoleOutputService $output,
|
||||
private readonly ?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
|
||||
private readonly ?PersistenceMetricsCollector $persistenceMetricsCollector = null,
|
||||
) {}
|
||||
|
||||
public function process(ReleaseProcessingContext $context, string $mainTmpPath): void
|
||||
public function process(ReleaseProcessingContext $context, string $mainTmpPath): ReleaseProcessingResult
|
||||
{
|
||||
$metrics = new ProcessingMetrics;
|
||||
$releaseId = (int) $context->release->id;
|
||||
$persistenceMetricsCollector = $this->persistenceMetricsCollector ?? new PersistenceMetricsCollector;
|
||||
$searchSyncCoordinator = $this->searchSyncCoordinator
|
||||
?? new ReleaseSearchSyncCoordinator(
|
||||
$persistenceMetricsCollector,
|
||||
);
|
||||
|
||||
$persistenceMetricsCollector->beginReleaseScope($releaseId);
|
||||
$searchSyncCoordinator->beginReleaseScope($releaseId);
|
||||
$this->downloadService->beginReleaseScope();
|
||||
|
||||
try {
|
||||
$result = $this->processRelease($context, $mainTmpPath, $metrics);
|
||||
} finally {
|
||||
$context->releaseDownloadedArchives();
|
||||
try {
|
||||
$searchSyncCoordinator->finishReleaseScope();
|
||||
} finally {
|
||||
$persistenceMetrics = $persistenceMetricsCollector->finishReleaseScope();
|
||||
$downloadMetrics = $this->downloadService->finishReleaseScope();
|
||||
}
|
||||
}
|
||||
|
||||
return $result->withPerformance(
|
||||
$metrics->elapsedSeconds(),
|
||||
$metrics->stageDurations(),
|
||||
$downloadMetrics,
|
||||
$persistenceMetrics,
|
||||
);
|
||||
}
|
||||
|
||||
private function processRelease(
|
||||
ReleaseProcessingContext $context,
|
||||
string $mainTmpPath,
|
||||
ProcessingMetrics $metrics,
|
||||
): ReleaseProcessingResult {
|
||||
$release = $context->release;
|
||||
|
||||
$this->output->echoReleaseStart($release->id, $release->size);
|
||||
$this->output->setProcessTitle((int) $release->id);
|
||||
|
||||
try {
|
||||
$context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid);
|
||||
$context->tmpPath = $metrics->measure(
|
||||
ProcessingStage::WorkspacePreparation,
|
||||
fn (): string => $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid),
|
||||
);
|
||||
} catch (\Throwable $e) {
|
||||
$this->output->warning('Unable to prepare release temp directory: '.$e->getMessage());
|
||||
|
||||
return;
|
||||
return $this->result(
|
||||
$context,
|
||||
ProcessingOutcome::TemporaryWorkspaceUnavailable,
|
||||
reason: $e->getMessage(),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
$nzbResult = $this->nzbParser->parseNzb($release->guid);
|
||||
$releaseNameChanged = false;
|
||||
$releaseNeededNfo = false;
|
||||
$nzbResult = $metrics->measure(
|
||||
ProcessingStage::NzbParsing,
|
||||
fn (): array => $this->nzbParser->parseNzb($release->guid),
|
||||
);
|
||||
if ($nzbResult['error'] !== null) {
|
||||
$this->output->warning($nzbResult['error']);
|
||||
$this->releaseManager->deleteRelease($release);
|
||||
|
||||
return;
|
||||
return $this->result(
|
||||
$context,
|
||||
ProcessingOutcome::DeletedBrokenNzb,
|
||||
reason: $nzbResult['error'],
|
||||
);
|
||||
}
|
||||
|
||||
$context->nzbContents = array_values($nzbResult['contents']);
|
||||
if ($this->checkReleaseTimeout($context)) {
|
||||
return;
|
||||
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
|
||||
return $timeoutResult;
|
||||
}
|
||||
|
||||
$releaseNameChanged = $metrics->measure(
|
||||
ProcessingStage::ReleaseInitialization,
|
||||
function () use ($context): bool {
|
||||
$this->initializeContext($context);
|
||||
$this->releaseManager->processReleaseNameFromNzbContents($context->nzbContents, $context);
|
||||
$messageIds = $this->nzbParser->extractMessageIDs($context->nzbContents, $context->releaseGroupName, $this->config);
|
||||
|
||||
$context->nzbHasCompressedFile = $messageIds['hasCompressedFile'];
|
||||
$context->sampleMessageIDs = $messageIds['sampleMessageIDs'];
|
||||
$context->jpgMessageIDs = $messageIds['jpgMessageIDs'];
|
||||
$context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID'];
|
||||
$context->audioInfoMessageIDs = $messageIds['audioInfoMessageID'];
|
||||
$context->audioInfoExtension = $messageIds['audioInfoExtension'];
|
||||
|
||||
$bookFlood = $messageIds['bookFileCount'] > 80
|
||||
&& ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents);
|
||||
return $this->releaseManager->processReleaseNameFromNzbContents($context->nzbContents, $context);
|
||||
},
|
||||
);
|
||||
$releaseNeededNfo = $context->releaseHasNoNFO;
|
||||
$bookFlood = $metrics->measure(
|
||||
ProcessingStage::MessageIdSelection,
|
||||
fn (): bool => $this->prepareMessageIds($context),
|
||||
);
|
||||
|
||||
if ($this->shouldProcessDownloads()) {
|
||||
$this->processMessageIdDownloads($context);
|
||||
if ($this->checkReleaseTimeout($context)) {
|
||||
return;
|
||||
$metrics->measure(
|
||||
ProcessingStage::DirectDownloads,
|
||||
fn () => $this->processMessageIdDownloads($context),
|
||||
);
|
||||
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
|
||||
return $timeoutResult;
|
||||
}
|
||||
|
||||
if (! $bookFlood && $context->nzbHasCompressedFile) {
|
||||
$triedCompressedMids = [];
|
||||
$metrics->measure(
|
||||
ProcessingStage::ArchiveDownloads,
|
||||
function () use ($context, &$triedCompressedMids): void {
|
||||
$this->processNzbCompressedFiles($context, false, $triedCompressedMids);
|
||||
if ($this->checkReleaseTimeout($context)) {
|
||||
return;
|
||||
},
|
||||
);
|
||||
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
|
||||
return $timeoutResult;
|
||||
}
|
||||
|
||||
if ($this->config->fetchLastFiles) {
|
||||
$metrics->measure(
|
||||
ProcessingStage::ArchiveDownloads,
|
||||
function () use ($context, &$triedCompressedMids): void {
|
||||
$this->processNzbCompressedFiles($context, true, $triedCompressedMids);
|
||||
if ($this->checkReleaseTimeout($context)) {
|
||||
return;
|
||||
},
|
||||
);
|
||||
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
|
||||
return $timeoutResult;
|
||||
}
|
||||
}
|
||||
|
||||
if (! $context->releaseHasPassword) {
|
||||
$this->processExtractedFiles($context);
|
||||
if ($this->checkReleaseTimeout($context)) {
|
||||
return;
|
||||
$metrics->measure(
|
||||
ProcessingStage::ExtractedFiles,
|
||||
fn () => $this->processExtractedFiles($context),
|
||||
);
|
||||
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
|
||||
return $timeoutResult;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$metrics->measure(ProcessingStage::ArchiveFallbacks, function () use ($context): void {
|
||||
if (! $context->foundJPGSample && $this->config->processJPGSample) {
|
||||
$this->archiveFallback->processJpgFromReleaseFiles($context);
|
||||
}
|
||||
|
||||
if ($context->releaseHasNoNFO) {
|
||||
$this->archiveFallback->processNfoFromReleaseFiles($context);
|
||||
}
|
||||
$this->archiveFallback->processNfoFromDownloadedArchives($context);
|
||||
}
|
||||
|
||||
$this->releaseManager->finalizeRelease($context, $this->config->processPasswords);
|
||||
if ($context->releaseHasNoNFO) {
|
||||
$this->archiveFallback->processNfoFromReleaseFiles($context);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
$metrics->measure(
|
||||
ProcessingStage::Finalization,
|
||||
fn () => $this->releaseManager->finalizeRelease($context, $this->config->processPasswords),
|
||||
);
|
||||
|
||||
$artifactsCreated = $releaseNameChanged || $this->createdArtifacts($context, $releaseNeededNfo);
|
||||
$outcome = match (true) {
|
||||
$context->releaseHasPassword => ProcessingOutcome::Passworded,
|
||||
$context->groupUnavailable => ProcessingOutcome::GroupUnavailable,
|
||||
! $artifactsCreated => ProcessingOutcome::NoUsefulArtifacts,
|
||||
default => ProcessingOutcome::Completed,
|
||||
};
|
||||
|
||||
return $this->result(
|
||||
$context,
|
||||
$outcome,
|
||||
$artifactsCreated,
|
||||
reason: match ($outcome) {
|
||||
ProcessingOutcome::Passworded => 'Password protection was detected.',
|
||||
ProcessingOutcome::GroupUnavailable => 'The release group was unavailable during processing.',
|
||||
ProcessingOutcome::NoUsefulArtifacts => 'Processing completed without creating useful artifacts.',
|
||||
default => '',
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
if ($context->tmpPath !== '') {
|
||||
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
|
||||
$metrics->measure(
|
||||
ProcessingStage::WorkspaceCleanup,
|
||||
fn () => $this->tempWorkspace->clearDirectory($context->tmpPath, false),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,6 +241,23 @@ class ReleaseProcessor
|
||||
|| $this->config->processJPGSample;
|
||||
}
|
||||
|
||||
private function prepareMessageIds(ReleaseProcessingContext $context): bool
|
||||
{
|
||||
$workPlan = $this->workPlanner->plan(
|
||||
$context->nzbContents,
|
||||
$context->releaseGroupName,
|
||||
);
|
||||
$context->workPlan = $workPlan;
|
||||
$context->nzbHasCompressedFile = $workPlan->hasCompressedFile();
|
||||
$context->sampleMessageIDs = $workPlan->sampleMessageIds;
|
||||
$context->jpgMessageIDs = $workPlan->jpgMessageIds;
|
||||
$context->mediaInfoMessageIDs = $workPlan->mediaInfoMessageId;
|
||||
$context->audioInfoMessageIDs = $workPlan->audioInfoMessageId;
|
||||
$context->audioInfoExtension = $workPlan->audioInfoExtension;
|
||||
|
||||
return $workPlan->bookFlood;
|
||||
}
|
||||
|
||||
private function initializeContext(ReleaseProcessingContext $context): void
|
||||
{
|
||||
$context->initializeFromConfig(
|
||||
@@ -159,15 +285,20 @@ class ReleaseProcessor
|
||||
$context->resetCounters();
|
||||
}
|
||||
|
||||
private function checkReleaseTimeout(ReleaseProcessingContext $context): bool
|
||||
{
|
||||
private function processingTimeoutResult(
|
||||
ReleaseProcessingContext $context,
|
||||
ProcessingMetrics $metrics,
|
||||
): ?ReleaseProcessingResult {
|
||||
if (! $context->isTimedOut($this->config->releaseProcessingTimeout)) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
$deleted = $this->releaseManager->handleReleaseTimeout(
|
||||
$deleted = $metrics->measure(
|
||||
ProcessingStage::TimeoutHandling,
|
||||
fn (): bool => $this->releaseManager->handleReleaseTimeout(
|
||||
$context->release,
|
||||
$this->config->maxPpTimeoutCount
|
||||
$this->config->maxPpTimeoutCount,
|
||||
),
|
||||
);
|
||||
|
||||
if ($deleted) {
|
||||
@@ -183,7 +314,44 @@ class ReleaseProcessor
|
||||
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
|
||||
}
|
||||
|
||||
return true;
|
||||
return $this->result(
|
||||
$context,
|
||||
$deleted ? ProcessingOutcome::DeletedAfterTimeout : ProcessingOutcome::TimedOut,
|
||||
reason: $deleted
|
||||
? 'The release was deleted after reaching the post-processing timeout limit.'
|
||||
: 'The release exceeded the post-processing timeout.',
|
||||
);
|
||||
}
|
||||
|
||||
private function createdArtifacts(ReleaseProcessingContext $context, bool $releaseNeededNfo): bool
|
||||
{
|
||||
return $context->releaseFilesChanged
|
||||
|| $context->foundPAR2Info
|
||||
|| ($releaseNeededNfo && ! $context->releaseHasNoNFO)
|
||||
|| ($this->config->processVideo && $context->foundVideo)
|
||||
|| ($this->config->processThumbnails && $context->foundSample)
|
||||
|| ($this->config->processJPGSample && $context->foundJPGSample)
|
||||
|| ($this->config->processMediaInfo && $context->foundMediaInfo)
|
||||
|| ($this->config->processAudioInfo && $context->foundAudioInfo)
|
||||
|| ($this->config->processAudioSample && $context->foundAudioSample);
|
||||
}
|
||||
|
||||
private function result(
|
||||
ReleaseProcessingContext $context,
|
||||
ProcessingOutcome $outcome,
|
||||
bool $artifactsCreated = false,
|
||||
string $reason = '',
|
||||
): ReleaseProcessingResult {
|
||||
return new ReleaseProcessingResult(
|
||||
releaseId: (int) $context->release->id,
|
||||
guid: (string) $context->release->guid,
|
||||
outcome: $outcome,
|
||||
artifactsCreated: $artifactsCreated,
|
||||
releaseFilesAdded: $context->addedFileInfo,
|
||||
reason: $reason,
|
||||
duplicateMessageIdCount: $context->duplicateMessageIdCount(),
|
||||
unsupportedReasons: $context->unsupportedReasons(),
|
||||
);
|
||||
}
|
||||
|
||||
private function processMessageIdDownloads(ReleaseProcessingContext $context): void
|
||||
@@ -318,61 +486,39 @@ class ReleaseProcessor
|
||||
return;
|
||||
}
|
||||
|
||||
$nzbContents = $context->nzbContents;
|
||||
if ($reverse) {
|
||||
krsort($nzbContents);
|
||||
} else {
|
||||
$archiveCandidates = match (true) {
|
||||
$context->workPlan === null => [],
|
||||
$reverse => $context->workPlan->orderedArchiveCandidates(true),
|
||||
default => $context->workPlan->prioritizedArchiveCandidates(),
|
||||
};
|
||||
if (! $reverse) {
|
||||
$triedCompressedMids = [];
|
||||
}
|
||||
|
||||
$failed = $downloaded = 0;
|
||||
|
||||
foreach ($nzbContents as $nzbFile) {
|
||||
foreach ($archiveCandidates as $archiveCandidate) {
|
||||
if ($downloaded >= $this->config->maximumRarSegments
|
||||
|| $failed >= $this->config->maximumRarPasswordChecks
|
||||
|| $context->releaseHasPassword
|
||||
|| $context->groupUnavailable
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
$title = (string) ($nzbFile['title'] ?? '');
|
||||
if (preg_match(
|
||||
'/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i',
|
||||
$title
|
||||
) !== 1) {
|
||||
if ($reverse && array_intersect($archiveCandidate->messageIds, $triedCompressedMids) !== []) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$segments = $nzbFile['segments'] ?? [];
|
||||
$messageIDs = [];
|
||||
$segmentCount = count($segments) - 1;
|
||||
|
||||
foreach (range(0, $this->config->maximumRarSegments - 1) as $index) {
|
||||
if ($index > $segmentCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
$segment = (string) $segments[$index];
|
||||
if (! $reverse) {
|
||||
$triedCompressedMids[] = $segment;
|
||||
} elseif (in_array($segment, $triedCompressedMids, false)) {
|
||||
continue 2;
|
||||
}
|
||||
|
||||
$messageIDs[] = $segment;
|
||||
}
|
||||
|
||||
if ($messageIDs === []) {
|
||||
continue;
|
||||
$triedCompressedMids = [...$triedCompressedMids, ...$archiveCandidate->messageIds];
|
||||
}
|
||||
|
||||
$result = $this->downloadService->download(
|
||||
DownloadKind::Compressed,
|
||||
$messageIDs,
|
||||
$archiveCandidate->messageIds,
|
||||
$context->releaseGroupName,
|
||||
$context->release->id,
|
||||
$title
|
||||
$archiveCandidate->title,
|
||||
);
|
||||
|
||||
if ($result['groupUnavailable']) {
|
||||
@@ -385,8 +531,13 @@ class ReleaseProcessor
|
||||
$this->output->echoCompressedDownload();
|
||||
$downloaded++;
|
||||
|
||||
$processed = $this->processCompressedData($result['data'], $context, $reverse);
|
||||
if ($processed || $context->releaseHasPassword) {
|
||||
$processed = $this->processCompressedData(
|
||||
$result['data'],
|
||||
$context,
|
||||
$reverse,
|
||||
$archiveCandidate->title,
|
||||
);
|
||||
if ($processed) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
@@ -399,7 +550,8 @@ class ReleaseProcessor
|
||||
private function processCompressedData(
|
||||
string $compressedData,
|
||||
ReleaseProcessingContext $context,
|
||||
bool $reverse
|
||||
bool $reverse,
|
||||
string $archiveTitle = '',
|
||||
): bool {
|
||||
$result = $this->archiveService->processCompressedData(
|
||||
$compressedData,
|
||||
@@ -411,7 +563,7 @@ class ReleaseProcessor
|
||||
$context->releaseHasPassword = true;
|
||||
$context->passwordStatus = $result['passwordStatus'];
|
||||
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isset($result['standaloneVideoType'])) {
|
||||
@@ -457,6 +609,13 @@ class ReleaseProcessor
|
||||
}
|
||||
}
|
||||
|
||||
if ($context->releaseHasNoNFO
|
||||
&& is_array($result['files'])
|
||||
&& $this->containsNfoCandidate($result['files'])
|
||||
) {
|
||||
$context->rememberDownloadedArchive($archiveTitle, $compressedData, $result['files']);
|
||||
}
|
||||
|
||||
if (! $context->foundJPGSample && $this->config->processJPGSample) {
|
||||
$this->archiveFallback->processJpgFromArchiveFileList($compressedData, $result['files'], $context);
|
||||
}
|
||||
@@ -464,6 +623,20 @@ class ReleaseProcessor
|
||||
return $context->totalFileInfo > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, array<string, mixed>> $files
|
||||
*/
|
||||
private function containsNfoCandidate(array $files): bool
|
||||
{
|
||||
foreach ($files as $file) {
|
||||
if ($this->archiveService->isNfoFile((string) ($file['name'] ?? ''))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function processExtractedFiles(ReleaseProcessingContext $context): void
|
||||
{
|
||||
$nestedLevels = 0;
|
||||
@@ -490,7 +663,7 @@ class ReleaseProcessor
|
||||
|
||||
$rarData = @File::get($filePath);
|
||||
if (! empty($rarData)) {
|
||||
$this->processCompressedData($rarData, $context, false);
|
||||
$this->processCompressedData($rarData, $context, false, basename($filePath));
|
||||
$foundCompressed = true;
|
||||
}
|
||||
File::delete($filePath);
|
||||
@@ -565,7 +738,7 @@ class ReleaseProcessor
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png)$/i', $filePath) === 1) {
|
||||
if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png|webp)$/i', $filePath) === 1) {
|
||||
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
|
||||
$context->markFound('jpgSample');
|
||||
$this->output->echoJpgSaved();
|
||||
@@ -588,7 +761,7 @@ class ReleaseProcessor
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG/i', $output) === 1) {
|
||||
if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG|^WebP/i', $output) === 1) {
|
||||
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
|
||||
$context->markFound('jpgSample');
|
||||
$this->output->echoJpgSaved();
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Facades\Search;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use Closure;
|
||||
|
||||
final class ReleaseSearchSyncCoordinator
|
||||
{
|
||||
private ?int $releaseId = null;
|
||||
|
||||
private bool $pending = false;
|
||||
|
||||
/** @var Closure(int): void */
|
||||
private readonly Closure $synchronize;
|
||||
|
||||
/**
|
||||
* @param (Closure(int): void)|null $synchronize
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly PersistenceMetricsCollector $metrics,
|
||||
?Closure $synchronize = null,
|
||||
private readonly bool $coalesce = true,
|
||||
) {
|
||||
$this->synchronize = $synchronize ?? static function (int $releaseId): void {
|
||||
Search::updateRelease($releaseId);
|
||||
};
|
||||
}
|
||||
|
||||
public function beginReleaseScope(int $releaseId): void
|
||||
{
|
||||
if (! $this->coalesce) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->releaseId = $releaseId;
|
||||
$this->pending = false;
|
||||
}
|
||||
|
||||
public function request(int $releaseId): void
|
||||
{
|
||||
$this->metrics->recordSearchSyncRequest($releaseId);
|
||||
|
||||
if ($this->coalesce && $this->releaseId === $releaseId) {
|
||||
$this->pending = true;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->execute($releaseId);
|
||||
}
|
||||
|
||||
public function discard(int $releaseId): void
|
||||
{
|
||||
if ($this->coalesce && $this->releaseId === $releaseId) {
|
||||
$this->pending = false;
|
||||
}
|
||||
}
|
||||
|
||||
public function finishReleaseScope(): void
|
||||
{
|
||||
if (! $this->coalesce) {
|
||||
return;
|
||||
}
|
||||
|
||||
$releaseId = $this->releaseId;
|
||||
$pending = $this->pending;
|
||||
$this->releaseId = null;
|
||||
$this->pending = false;
|
||||
|
||||
if ($releaseId !== null && $pending) {
|
||||
$this->execute($releaseId);
|
||||
}
|
||||
}
|
||||
|
||||
private function execute(int $releaseId): void
|
||||
{
|
||||
$this->metrics->recordSearchSyncExecution($releaseId);
|
||||
($this->synchronize)($releaseId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\State;
|
||||
|
||||
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
|
||||
|
||||
final class PersistenceMetricsCollector
|
||||
{
|
||||
private ?int $releaseId = null;
|
||||
|
||||
private int $databaseStatements = 0;
|
||||
|
||||
private float $databaseMilliseconds = 0.0;
|
||||
|
||||
private int $searchSyncRequests = 0;
|
||||
|
||||
private int $searchSyncExecutions = 0;
|
||||
|
||||
public function beginReleaseScope(int $releaseId): void
|
||||
{
|
||||
$this->releaseId = $releaseId;
|
||||
$this->databaseStatements = 0;
|
||||
$this->databaseMilliseconds = 0.0;
|
||||
$this->searchSyncRequests = 0;
|
||||
$this->searchSyncExecutions = 0;
|
||||
}
|
||||
|
||||
public function recordDatabaseStatement(float $milliseconds): void
|
||||
{
|
||||
if ($this->releaseId === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->databaseStatements++;
|
||||
$this->databaseMilliseconds += $milliseconds;
|
||||
}
|
||||
|
||||
public function recordSearchSyncRequest(int $releaseId): void
|
||||
{
|
||||
if ($this->releaseId === $releaseId) {
|
||||
$this->searchSyncRequests++;
|
||||
}
|
||||
}
|
||||
|
||||
public function recordSearchSyncExecution(int $releaseId): void
|
||||
{
|
||||
if ($this->releaseId === $releaseId) {
|
||||
$this->searchSyncExecutions++;
|
||||
}
|
||||
}
|
||||
|
||||
public function finishReleaseScope(): PersistenceMetrics
|
||||
{
|
||||
$metrics = new PersistenceMetrics(
|
||||
databaseStatements: $this->databaseStatements,
|
||||
databaseMilliseconds: $this->databaseMilliseconds,
|
||||
searchSyncRequests: $this->searchSyncRequests,
|
||||
searchSyncExecutions: $this->searchSyncExecutions,
|
||||
);
|
||||
|
||||
$this->releaseId = null;
|
||||
$this->databaseStatements = 0;
|
||||
$this->databaseMilliseconds = 0.0;
|
||||
$this->searchSyncRequests = 0;
|
||||
$this->searchSyncExecutions = 0;
|
||||
|
||||
return $metrics;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\AdditionalProcessing\State;
|
||||
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
|
||||
use Closure;
|
||||
|
||||
final class ProcessingMetrics
|
||||
{
|
||||
private readonly int $startedAtNanoseconds;
|
||||
|
||||
/**
|
||||
* @var array<string, int>
|
||||
*/
|
||||
private array $stageNanoseconds = [];
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->startedAtNanoseconds = hrtime(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @template TValue
|
||||
*
|
||||
* @param Closure(): TValue $operation
|
||||
* @return TValue
|
||||
*/
|
||||
public function measure(ProcessingStage $stage, Closure $operation): mixed
|
||||
{
|
||||
$startedAt = hrtime(true);
|
||||
|
||||
try {
|
||||
return $operation();
|
||||
} finally {
|
||||
$elapsed = hrtime(true) - $startedAt;
|
||||
$this->stageNanoseconds[$stage->value] = ($this->stageNanoseconds[$stage->value] ?? 0) + $elapsed;
|
||||
}
|
||||
}
|
||||
|
||||
public function elapsedSeconds(): float
|
||||
{
|
||||
return $this->nanosecondsToSeconds(hrtime(true) - $this->startedAtNanoseconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, float>
|
||||
*/
|
||||
public function stageDurations(): array
|
||||
{
|
||||
return array_map($this->nanosecondsToSeconds(...), $this->stageNanoseconds);
|
||||
}
|
||||
|
||||
private function nanosecondsToSeconds(int $nanoseconds): float
|
||||
{
|
||||
return $nanoseconds / 1_000_000_000;
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ declare(strict_types=1);
|
||||
namespace App\Services\AdditionalProcessing\State;
|
||||
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalWorkPlan;
|
||||
use App\Services\AdditionalProcessing\DTO\DownloadedArchive;
|
||||
|
||||
/**
|
||||
* Mutable context object that holds the processing state for a single release.
|
||||
@@ -17,6 +19,10 @@ use App\Models\Release;
|
||||
*/
|
||||
class ReleaseProcessingContext
|
||||
{
|
||||
private const int MAX_RETAINED_ARCHIVES = 2;
|
||||
|
||||
private const int MAX_RETAINED_ARCHIVE_BYTES = 16_777_216;
|
||||
|
||||
// The release being processed
|
||||
public Release $release;
|
||||
|
||||
@@ -56,6 +62,15 @@ class ReleaseProcessingContext
|
||||
*/
|
||||
public array $nzbContents = [];
|
||||
|
||||
public ?AdditionalWorkPlan $workPlan = null;
|
||||
|
||||
/**
|
||||
* @var list<DownloadedArchive>
|
||||
*/
|
||||
public array $downloadedArchives = [];
|
||||
|
||||
private int $retainedArchiveBytes = 0;
|
||||
|
||||
// Message IDs for downloading
|
||||
/**
|
||||
* @var list<string>
|
||||
@@ -177,10 +192,53 @@ class ReleaseProcessingContext
|
||||
$this->releaseHasPassword = false;
|
||||
$this->nzbHasCompressedFile = false;
|
||||
$this->groupUnavailable = false;
|
||||
$this->workPlan = null;
|
||||
$this->releaseDownloadedArchives();
|
||||
$this->resetMessageIDs();
|
||||
$this->resetCounters();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retain only small, immediately reusable archive payloads for this release.
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $files
|
||||
*/
|
||||
public function rememberDownloadedArchive(string $title, string $data, array $files): bool
|
||||
{
|
||||
$byteSize = strlen($data);
|
||||
if ($byteSize === 0
|
||||
|| count($this->downloadedArchives) >= self::MAX_RETAINED_ARCHIVES
|
||||
|| ($this->retainedArchiveBytes + $byteSize) > self::MAX_RETAINED_ARCHIVE_BYTES
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$archive = new DownloadedArchive($title, $data, $files);
|
||||
$this->downloadedArchives[] = $archive;
|
||||
$this->retainedArchiveBytes += $archive->byteSize();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function releaseDownloadedArchives(): void
|
||||
{
|
||||
$this->downloadedArchives = [];
|
||||
$this->retainedArchiveBytes = 0;
|
||||
}
|
||||
|
||||
public function duplicateMessageIdCount(): int
|
||||
{
|
||||
return $this->workPlan === null ? 0 : $this->workPlan->duplicateMessageIdCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<string>
|
||||
*/
|
||||
public function unsupportedReasons(): array
|
||||
{
|
||||
return $this->workPlan === null ? [] : $this->workPlan->unsupportedReasons;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a processing artifact as found.
|
||||
*/
|
||||
|
||||
@@ -5,6 +5,7 @@ declare(strict_types=1);
|
||||
namespace App\Services\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
|
||||
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
|
||||
use App\Services\AdditionalProcessing\Enums\DownloadKind;
|
||||
use App\Services\NNTP\NNTPService;
|
||||
use Exception;
|
||||
@@ -16,8 +17,33 @@ use Illuminate\Support\Facades\Log;
|
||||
*/
|
||||
class UsenetDownloadService
|
||||
{
|
||||
private const int MAX_CACHE_ENTRIES = 4;
|
||||
|
||||
private const int MAX_CACHE_ENTRY_BYTES = 8_388_608;
|
||||
|
||||
private const int MAX_CACHE_BYTES = 16_777_216;
|
||||
|
||||
private NNTPService $nntp;
|
||||
|
||||
private bool $releaseScopeActive = false;
|
||||
|
||||
/**
|
||||
* @var array<string, array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}>
|
||||
*/
|
||||
private array $releaseCache = [];
|
||||
|
||||
private int $cachedBytes = 0;
|
||||
|
||||
private int $logicalRequests = 0;
|
||||
|
||||
private int $networkRequests = 0;
|
||||
|
||||
private int $cacheHits = 0;
|
||||
|
||||
private int $bytesDownloaded = 0;
|
||||
|
||||
private int $bytesReused = 0;
|
||||
|
||||
public function __construct(
|
||||
private readonly ProcessingConfiguration $config,
|
||||
?NNTPService $nntp = null
|
||||
@@ -25,10 +51,31 @@ class UsenetDownloadService
|
||||
$this->nntp = $nntp ?? new NNTPService;
|
||||
}
|
||||
|
||||
public function beginReleaseScope(): void
|
||||
{
|
||||
$this->clearReleaseScope();
|
||||
$this->releaseScopeActive = true;
|
||||
}
|
||||
|
||||
public function finishReleaseScope(): DownloadMetrics
|
||||
{
|
||||
$metrics = new DownloadMetrics(
|
||||
logicalRequests: $this->logicalRequests,
|
||||
networkRequests: $this->networkRequests,
|
||||
cacheHits: $this->cacheHits,
|
||||
bytesDownloaded: $this->bytesDownloaded,
|
||||
bytesReused: $this->bytesReused,
|
||||
);
|
||||
|
||||
$this->clearReleaseScope();
|
||||
|
||||
return $metrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download binary content from usenet using message IDs.
|
||||
*
|
||||
* @param array<string, mixed>|string $messageIDs Single or array of message IDs
|
||||
* @param array<int|string, mixed>|string $messageIDs Single or array of message IDs
|
||||
* @param string $groupName Group name for logging
|
||||
* @param int|null $releaseId Release ID for logging
|
||||
* @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}
|
||||
@@ -61,7 +108,8 @@ class UsenetDownloadService
|
||||
if ($this->config->debugMode) {
|
||||
Log::debug('Attempting NNTP fetch', [
|
||||
'release_id' => $releaseId,
|
||||
'message_ids' => $messageIDs,
|
||||
'message_id_count' => count($messageIDs),
|
||||
'request_fingerprint' => $this->downloadFingerprint($messageIDs, $groupName),
|
||||
'group' => $groupName,
|
||||
]);
|
||||
}
|
||||
@@ -89,7 +137,8 @@ class UsenetDownloadService
|
||||
if ($this->config->debugMode) {
|
||||
Log::debug('NNTP fetch failed', [
|
||||
'release_id' => $releaseId,
|
||||
'message_ids' => $messageIDs,
|
||||
'message_id_count' => count($messageIDs),
|
||||
'request_fingerprint' => $this->downloadFingerprint($messageIDs, $groupName),
|
||||
'group' => $groupName,
|
||||
'error_object' => is_object($binary) ? get_class($binary) : null,
|
||||
'error_message' => $errorMessage,
|
||||
@@ -112,7 +161,7 @@ class UsenetDownloadService
|
||||
/**
|
||||
* Download content for a specific processing step.
|
||||
*
|
||||
* @param array<string, mixed>|string $messageIDs
|
||||
* @param array<int|string, mixed>|string $messageIDs
|
||||
* @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}
|
||||
*/
|
||||
public function download(
|
||||
@@ -122,24 +171,51 @@ class UsenetDownloadService
|
||||
?int $releaseId = null,
|
||||
?string $fileTitle = null
|
||||
): array {
|
||||
$messageIdList = is_array($messageIDs) ? array_values($messageIDs) : [$messageIDs];
|
||||
$cacheKey = $this->downloadFingerprint($messageIdList, $groupName);
|
||||
|
||||
if ($this->releaseScopeActive) {
|
||||
$this->logicalRequests++;
|
||||
|
||||
if (isset($this->releaseCache[$cacheKey])) {
|
||||
$cached = $this->releaseCache[$cacheKey];
|
||||
$this->cacheHits++;
|
||||
$this->bytesReused += is_string($cached['data']) ? strlen($cached['data']) : 0;
|
||||
|
||||
return $cached;
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->config->debugMode) {
|
||||
Log::debug('Attempting download', [
|
||||
'kind' => $kind->value,
|
||||
'release_id' => $releaseId,
|
||||
'file_title' => $fileTitle,
|
||||
'message_ids' => $messageIDs,
|
||||
'message_id_count' => count($messageIdList),
|
||||
'request_fingerprint' => $cacheKey,
|
||||
'group' => $groupName,
|
||||
]);
|
||||
}
|
||||
|
||||
if ($this->releaseScopeActive && $messageIdList !== [] && $messageIdList !== ['']) {
|
||||
$this->networkRequests++;
|
||||
}
|
||||
|
||||
$result = $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId);
|
||||
|
||||
if ($this->releaseScopeActive && $result['success'] && is_string($result['data'])) {
|
||||
$byteSize = strlen($result['data']);
|
||||
$this->bytesDownloaded += $byteSize;
|
||||
$this->rememberSuccessfulDownload($cacheKey, $result, $byteSize);
|
||||
}
|
||||
|
||||
if (! $result['success'] && $this->config->debugMode) {
|
||||
Log::debug('Download failed', [
|
||||
'kind' => $kind->value,
|
||||
'release_id' => $releaseId,
|
||||
'file_title' => $fileTitle,
|
||||
'message_ids' => $messageIDs,
|
||||
'message_id_count' => count($messageIdList),
|
||||
'request_fingerprint' => $cacheKey,
|
||||
'group' => $groupName,
|
||||
'error' => $result['error'],
|
||||
]);
|
||||
@@ -148,6 +224,46 @@ class UsenetDownloadService
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<mixed> $messageIds
|
||||
*/
|
||||
private function downloadFingerprint(array $messageIds, string $groupName): string
|
||||
{
|
||||
return hash('sha256', serialize([
|
||||
'message_ids' => array_map(static fn (mixed $messageId): string => (string) $messageId, $messageIds),
|
||||
'group' => $groupName,
|
||||
'alternate' => $this->config->alternateNNTP,
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{success: bool, data: string|null, groupUnavailable: bool, error: string|null} $result
|
||||
*/
|
||||
private function rememberSuccessfulDownload(string $cacheKey, array $result, int $byteSize): void
|
||||
{
|
||||
if ($byteSize > self::MAX_CACHE_ENTRY_BYTES
|
||||
|| count($this->releaseCache) >= self::MAX_CACHE_ENTRIES
|
||||
|| ($this->cachedBytes + $byteSize) > self::MAX_CACHE_BYTES
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->releaseCache[$cacheKey] = $result;
|
||||
$this->cachedBytes += $byteSize;
|
||||
}
|
||||
|
||||
private function clearReleaseScope(): void
|
||||
{
|
||||
$this->releaseScopeActive = false;
|
||||
$this->releaseCache = [];
|
||||
$this->cachedBytes = 0;
|
||||
$this->logicalRequests = 0;
|
||||
$this->networkRequests = 0;
|
||||
$this->cacheHits = 0;
|
||||
$this->bytesDownloaded = 0;
|
||||
$this->bytesReused = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the NNTP client instance.
|
||||
*/
|
||||
|
||||
@@ -5,10 +5,11 @@ declare(strict_types=1);
|
||||
namespace App\Services\NameFixing;
|
||||
|
||||
use App\Events\ReleaseNameFixed;
|
||||
use App\Facades\Search;
|
||||
use App\Models\Category;
|
||||
use App\Models\Release;
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use App\Services\ReleaseCleaningService;
|
||||
use Illuminate\Support\Arr;
|
||||
@@ -82,6 +83,8 @@ class ReleaseUpdateService
|
||||
|
||||
protected bool $echoOutput;
|
||||
|
||||
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
|
||||
|
||||
/**
|
||||
* The release ID we are trying to rename.
|
||||
*/
|
||||
@@ -109,10 +112,13 @@ class ReleaseUpdateService
|
||||
|
||||
public function __construct(
|
||||
?CategorizationService $category = null,
|
||||
?FileNameCleaner $fileNameCleaner = null
|
||||
?FileNameCleaner $fileNameCleaner = null,
|
||||
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
|
||||
) {
|
||||
$this->category = $category ?? new CategorizationService;
|
||||
$this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner;
|
||||
$this->searchSyncCoordinator = $searchSyncCoordinator
|
||||
?? new ReleaseSearchSyncCoordinator(new PersistenceMetricsCollector);
|
||||
$this->echoOutput = config('nntmux.echocli');
|
||||
}
|
||||
|
||||
@@ -331,8 +337,7 @@ class ReleaseUpdateService
|
||||
));
|
||||
});
|
||||
|
||||
// Update search index
|
||||
Search::updateRelease($release->releases_id);
|
||||
$this->searchSyncCoordinator->request((int) $release->releases_id);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -16,7 +16,7 @@ use Illuminate\Support\Facades\Log;
|
||||
|
||||
class PostProcessRunner extends BaseRunner
|
||||
{
|
||||
private const int ADDITIONAL_WORKER_MAX_BATCHES = 4;
|
||||
public const int ADDITIONAL_WORKER_MAX_BATCHES = 4;
|
||||
|
||||
private function guidBucketExpression(string $column = 'leftguid'): string
|
||||
{
|
||||
@@ -257,13 +257,13 @@ class PostProcessRunner extends BaseRunner
|
||||
$excludedReleaseIds = [];
|
||||
try {
|
||||
for ($batch = 0; $batch < self::ADDITIONAL_WORKER_MAX_BATCHES; $batch++) {
|
||||
$stats = $orchestrator->start('', $char, $workerToken, $excludedReleaseIds);
|
||||
if ($stats['claimed'] === 0) {
|
||||
$result = $orchestrator->start('', $char, $workerToken, $excludedReleaseIds);
|
||||
if ($result->claimedCount() === 0) {
|
||||
break;
|
||||
}
|
||||
$excludedReleaseIds = array_values(array_unique([
|
||||
...$excludedReleaseIds,
|
||||
...$stats['claimed_ids'],
|
||||
...$result->claimedIds,
|
||||
]));
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -480,24 +480,12 @@ parameters:
|
||||
count: 1
|
||||
path: app/Rules/RecaptchaRule.php
|
||||
|
||||
-
|
||||
message: '#^Method App\\Services\\AdditionalProcessing\\ArchiveExtractionService\:\:sortFilesWithNfoPriority\(\) should return array\<string, mixed\> but returns list\<mixed\>\.$#'
|
||||
identifier: return.type
|
||||
count: 1
|
||||
path: app/Services/AdditionalProcessing/ArchiveExtractionService.php
|
||||
|
||||
-
|
||||
message: '#^PHPDoc tag @return with type list\<mixed\> is incompatible with native type void\.$#'
|
||||
identifier: return.phpDocType
|
||||
count: 1
|
||||
path: app/Services/AdditionalProcessing/ArchiveExtractionService.php
|
||||
|
||||
-
|
||||
message: '#^Method App\\Services\\AdditionalProcessing\\NzbContentParser\:\:extractSegments\(\) should return array\<string, mixed\> but returns list\<string\>\.$#'
|
||||
identifier: return.type
|
||||
count: 1
|
||||
path: app/Services/AdditionalProcessing/NzbContentParser.php
|
||||
|
||||
-
|
||||
message: '#^Using nullsafe property access "\?\-\>id" on left side of \?\? is unnecessary\. Use \-\> instead\.$#'
|
||||
identifier: nullsafe.neverNull
|
||||
@@ -510,30 +498,6 @@ parameters:
|
||||
count: 2
|
||||
path: app/Services/AdditionalProcessing/ReleaseFileManager.php
|
||||
|
||||
-
|
||||
message: '#^Parameter \#2 \$messageIDs of method App\\Services\\AdditionalProcessing\\UsenetDownloadService\:\:download\(\) expects array\<string, mixed\>\|string, list\<string\> given\.$#'
|
||||
identifier: argument.type
|
||||
count: 1
|
||||
path: app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php
|
||||
|
||||
-
|
||||
message: '#^Parameter \#1 \$nzbContents of method App\\Services\\AdditionalProcessing\\NzbContentParser\:\:extractMessageIDs\(\) expects array\<string, mixed\>, list\<array\<string, mixed\>\> given\.$#'
|
||||
identifier: argument.type
|
||||
count: 1
|
||||
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
|
||||
|
||||
-
|
||||
message: '#^Parameter \#2 \$messageIDs of method App\\Services\\AdditionalProcessing\\UsenetDownloadService\:\:download\(\) expects array\<string, mixed\>\|string, list\<string\> given\.$#'
|
||||
identifier: argument.type
|
||||
count: 3
|
||||
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
|
||||
|
||||
-
|
||||
message: '#^Right side of \|\| is always false\.$#'
|
||||
identifier: booleanOr.rightAlwaysFalse
|
||||
count: 2
|
||||
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
|
||||
|
||||
-
|
||||
message: '#^Strict comparison using \=\=\= between non\-empty\-string and null will always evaluate to false\.$#'
|
||||
identifier: identical.alwaysFalse
|
||||
|
||||
@@ -154,6 +154,25 @@ class AdditionalProcessingNzbSplitRenameTest extends TestCase
|
||||
$this->assertSame('MLB.2026.Cincinnati.Reds.vs.Tampa.Bay.Rays.20.04.720pEN60fps', $result);
|
||||
}
|
||||
|
||||
public function test_additional_nfo_detection_accepts_alternative_and_scene_names(): void
|
||||
{
|
||||
$manager = $this->makeReleaseFileManager();
|
||||
|
||||
foreach (['file_id.diz', 'readme.txt', 'info.txt', '00-scene.nfo', 'group-release.nfo'] as $filename) {
|
||||
$this->assertTrue($manager->isNfoFilename($filename), $filename.' should be treated as NFO data');
|
||||
}
|
||||
}
|
||||
|
||||
public function test_additional_nfo_normalization_converts_utf16_and_removes_utf8_bom(): void
|
||||
{
|
||||
$manager = $this->makeReleaseFileManager();
|
||||
$normalize = new \ReflectionMethod($manager, 'normalizeNfoEncoding');
|
||||
$utf16 = "\xFF\xFE".mb_convert_encoding('Scene NFO', 'UTF-16LE', 'UTF-8');
|
||||
|
||||
$this->assertSame('Scene NFO', $normalize->invoke($manager, $utf16));
|
||||
$this->assertSame('Scene NFO', $normalize->invoke($manager, "\xEF\xBB\xBFScene NFO"));
|
||||
}
|
||||
|
||||
private function makeReleaseFileManager(?ReleaseUpdateService $updateService = null): ReleaseFileManager
|
||||
{
|
||||
$updateService ??= $this->createMock(ReleaseUpdateService::class);
|
||||
|
||||
@@ -7,13 +7,17 @@ namespace Tests\Feature;
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
|
||||
use App\Services\AdditionalProcessing\ConsoleOutputService;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use App\Services\AdditionalProcessing\ReleaseProcessor;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\TempWorkspaceService;
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Mockery;
|
||||
use PDO;
|
||||
use Tests\TestCase;
|
||||
use Tests\Unit\AdditionalProcessing\CreatesProcessingConfiguration;
|
||||
@@ -87,6 +91,8 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
|
||||
|
||||
public function test_handled_release_exception_clears_claim(): void
|
||||
{
|
||||
Log::spy();
|
||||
|
||||
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
|
||||
DB::table('releases')->insert([
|
||||
'id' => 1,
|
||||
@@ -121,15 +127,49 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
|
||||
$output
|
||||
);
|
||||
|
||||
$orchestrator->start('', 'a');
|
||||
$result = $orchestrator->start('', 'a');
|
||||
|
||||
$this->assertSame(1, $processor->processCalls);
|
||||
$this->assertSame([1], $result->claimedIds);
|
||||
$this->assertSame(0, $result->successfulCount());
|
||||
$this->assertSame(1, $result->unsuccessfulCount());
|
||||
$this->assertTrue($result->hasOutcome(ProcessingOutcome::Failed));
|
||||
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
|
||||
$this->assertGreaterThan(0.0, $result->averageReleaseSeconds());
|
||||
$this->assertGreaterThan(0.0, $result->releasesPerSecond());
|
||||
$this->assertGreaterThan(0, $result->peakMemoryBytes);
|
||||
$this->assertSame(1, $tempWorkspace->ensureMainTempPathCalls);
|
||||
$this->assertSame(1, $tempWorkspace->clearDirectoryCalls);
|
||||
$this->assertSame(1, $output->echoDescriptionCalls);
|
||||
$this->assertSame(1, $output->endOutputCalls);
|
||||
$this->assertNull(Release::query()->where('id', 1)->value('additional_pp_claimed_at'));
|
||||
$this->assertNull(Release::query()->where('id', 1)->value('additional_pp_claim_token'));
|
||||
|
||||
Log::shouldHaveReceived('info')
|
||||
->once()
|
||||
->with(
|
||||
'Additional postprocessing run finished',
|
||||
Mockery::on(static fn (array $context): bool => $context['picked'] === 1
|
||||
&& $context['processed'] === 0
|
||||
&& $context['failed'] === 1
|
||||
&& $context['outcomes'] === ['failed' => 1]
|
||||
&& array_key_exists('artifacts_created', $context)
|
||||
&& array_key_exists('artifact_yield_percent', $context)
|
||||
&& array_key_exists('release_files_added', $context)
|
||||
&& array_key_exists('download_requests', $context)
|
||||
&& array_key_exists('nntp_requests', $context)
|
||||
&& array_key_exists('download_cache_hits', $context)
|
||||
&& array_key_exists('database_statements', $context)
|
||||
&& array_key_exists('database_milliseconds', $context)
|
||||
&& array_key_exists('search_sync_requests', $context)
|
||||
&& array_key_exists('search_sync_executions', $context)
|
||||
&& array_key_exists('duplicate_message_ids', $context)
|
||||
&& array_key_exists('unsupported_reasons', $context)
|
||||
&& array_key_exists('releases_per_second', $context)
|
||||
&& array_key_exists('average_release_seconds', $context)
|
||||
&& array_key_exists('stage_seconds', $context)
|
||||
&& array_key_exists('peak_memory_bytes', $context)),
|
||||
);
|
||||
}
|
||||
|
||||
public function test_temp_setup_failure_does_not_claim_releases(): void
|
||||
@@ -168,9 +208,11 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
|
||||
$output
|
||||
);
|
||||
|
||||
$orchestrator->start('', 'a');
|
||||
$result = $orchestrator->start('', 'a');
|
||||
|
||||
$this->assertSame(0, $processor->processCalls);
|
||||
$this->assertSame(0, $result->claimedCount());
|
||||
$this->assertStringContainsString('not writable', $result->setupFailure);
|
||||
$this->assertSame(1, $tempWorkspace->ensureMainTempPathCalls);
|
||||
$this->assertSame(0, $output->echoDescriptionCalls);
|
||||
$this->assertSame(0, $output->endOutputCalls);
|
||||
@@ -246,7 +288,7 @@ class FailingAdditionalReleaseProcessor extends ReleaseProcessor
|
||||
|
||||
public function __construct() {}
|
||||
|
||||
public function process(ReleaseProcessingContext $context, string $mainTmpPath): void
|
||||
public function process(ReleaseProcessingContext $context, string $mainTmpPath): ReleaseProcessingResult
|
||||
{
|
||||
$this->processCalls++;
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ namespace Tests\Feature;
|
||||
use App\Facades\Search;
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\ReleaseFileManager;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\CollectionCleanupService;
|
||||
use App\Services\NameFixing\NameFixingService;
|
||||
@@ -14,6 +15,7 @@ use App\Services\NfoService;
|
||||
use App\Services\Nzb\NzbService;
|
||||
use App\Services\ReleaseImageService;
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Database\QueryException;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\File;
|
||||
@@ -129,6 +131,40 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
|
||||
$this->assertNull(DB::table('releases')->where('id', 1)->value('additional_pp_claim_token'));
|
||||
}
|
||||
|
||||
public function test_finalization_flushes_rows_and_synchronizes_search_immediately(): void
|
||||
{
|
||||
DB::table('releases')->insert($this->releaseRow());
|
||||
|
||||
Search::shouldReceive('updateRelease')->once()->with(1);
|
||||
|
||||
$manager = $this->makeManagerWithImageService(new ReleaseImageService);
|
||||
$context = new ReleaseProcessingContext(Release::query()->findOrFail(1));
|
||||
$this->assertTrue($manager->addFileInfo([
|
||||
'name' => 'Example.Movie.2026.mkv',
|
||||
'size' => 2048,
|
||||
'date' => 1_788_600_000,
|
||||
], $context, '\\.(?:par2|sfv|nzb)'));
|
||||
|
||||
$manager->finalizeRelease($context, false);
|
||||
|
||||
$this->assertSame(1, DB::table('release_files')->count());
|
||||
$this->assertSame(1, DB::table('releases')->where('id', 1)->value('rarinnerfilecount'));
|
||||
$this->assertSame([], $context->pendingReleaseFiles);
|
||||
}
|
||||
|
||||
public function test_database_statements_are_measured_inside_an_active_release_scope(): void
|
||||
{
|
||||
DB::table('releases')->insert($this->releaseRow());
|
||||
$collector = app(PersistenceMetricsCollector::class);
|
||||
|
||||
$collector->beginReleaseScope(1);
|
||||
DB::table('releases')->where('id', 1)->value('guid');
|
||||
$metrics = $collector->finishReleaseScope();
|
||||
|
||||
$this->assertSame(1, $metrics->databaseStatements);
|
||||
$this->assertGreaterThanOrEqual(0.0, $metrics->databaseMilliseconds);
|
||||
}
|
||||
|
||||
public function test_queued_par_hashes_flush_with_release_files(): void
|
||||
{
|
||||
DB::table('releases')->insert($this->releaseRow());
|
||||
@@ -156,6 +192,35 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
|
||||
$this->assertSame(1, DB::table('par_hashes')->count());
|
||||
}
|
||||
|
||||
public function test_failed_finalization_rolls_back_buffered_rows_and_keeps_them_for_retry(): void
|
||||
{
|
||||
DB::table('releases')->insert($this->releaseRow());
|
||||
|
||||
Search::shouldReceive('updateRelease')->never();
|
||||
|
||||
$manager = $this->makeManager();
|
||||
$context = new ReleaseProcessingContext(Release::query()->findOrFail(1));
|
||||
$this->assertTrue($manager->addFileInfo([
|
||||
'name' => 'Example.Movie.2026.mkv',
|
||||
'size' => 1024,
|
||||
'date' => 1_788_600_000,
|
||||
], $context, '\\.(?:par2|sfv|nzb)'));
|
||||
|
||||
DB::unprepared("CREATE TRIGGER fail_release_finalize BEFORE UPDATE ON releases BEGIN SELECT RAISE(ABORT, 'forced finalization failure'); END");
|
||||
|
||||
try {
|
||||
$manager->finalizeRelease($context, false);
|
||||
$this->fail('Finalization should have failed.');
|
||||
} catch (QueryException $exception) {
|
||||
$this->assertStringContainsString('forced finalization failure', $exception->getMessage());
|
||||
} finally {
|
||||
DB::unprepared('DROP TRIGGER IF EXISTS fail_release_finalize');
|
||||
}
|
||||
|
||||
$this->assertSame(0, DB::table('release_files')->count());
|
||||
$this->assertArrayHasKey('Example.Movie.2026.mkv', $context->pendingReleaseFiles);
|
||||
}
|
||||
|
||||
public function test_float_release_file_size_is_normalized_before_queueing(): void
|
||||
{
|
||||
DB::table('releases')->insert($this->releaseRow());
|
||||
@@ -230,12 +295,16 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
|
||||
return $this->makeManagerWithImageService(new ReleaseImageService, $nameFixing);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $configOverrides
|
||||
*/
|
||||
private function makeManagerWithImageService(
|
||||
ReleaseImageService $imageService,
|
||||
?NameFixingService $nameFixing = null,
|
||||
array $configOverrides = [],
|
||||
): ReleaseFileManager {
|
||||
return new ReleaseFileManager(
|
||||
$this->makeConfig(),
|
||||
$this->makeConfig($configOverrides),
|
||||
$imageService,
|
||||
new NfoService,
|
||||
new TestNzbService,
|
||||
|
||||
@@ -5,6 +5,11 @@ declare(strict_types=1);
|
||||
namespace Tests\Feature;
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
|
||||
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
|
||||
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use App\Services\Runners\PostProcessRunner;
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
@@ -186,7 +191,7 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
|
||||
$mock->shouldReceive('start')
|
||||
->once()
|
||||
->with('', 'a', Mockery::type('string'), [])
|
||||
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [1]]);
|
||||
->andReturn($this->successfulBatch(1));
|
||||
$mock->shouldReceive('finish')->once();
|
||||
});
|
||||
|
||||
@@ -205,17 +210,17 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
|
||||
->once()
|
||||
->ordered()
|
||||
->with('', 'a', Mockery::type('string'), [])
|
||||
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [1]]);
|
||||
->andReturn($this->successfulBatch(1));
|
||||
$mock->shouldReceive('start')
|
||||
->once()
|
||||
->ordered()
|
||||
->with('', 'a', Mockery::type('string'), [1])
|
||||
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [2]]);
|
||||
->andReturn($this->successfulBatch(2));
|
||||
$mock->shouldReceive('start')
|
||||
->once()
|
||||
->ordered()
|
||||
->with('', 'a', Mockery::type('string'), [1, 2])
|
||||
->andReturn(['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []]);
|
||||
->andReturn(AdditionalBatchResult::empty());
|
||||
$mock->shouldReceive('finish')->once();
|
||||
});
|
||||
|
||||
@@ -229,6 +234,154 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
|
||||
$this->assertSame(0, $status);
|
||||
}
|
||||
|
||||
public function test_worker_guid_command_emits_machine_readable_batch_profiles(): void
|
||||
{
|
||||
$batchResult = $this->successfulBatch(1)->withPerformance(2.0, 1024);
|
||||
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock) use ($batchResult): void {
|
||||
$mock->shouldReceive('start')->once()->andReturn($batchResult);
|
||||
$mock->shouldReceive('finish')->once();
|
||||
});
|
||||
|
||||
$status = Artisan::call('postprocess:guid', [
|
||||
'type' => 'additional',
|
||||
'guid' => 'a',
|
||||
'--profile' => true,
|
||||
]);
|
||||
$profile = json_decode(trim(Artisan::output()), true, flags: JSON_THROW_ON_ERROR);
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertSame('additional-postprocessing-profile', $profile['event']);
|
||||
$this->assertSame('v2', $profile['pipeline']);
|
||||
$this->assertSame(1, $profile['attempted']);
|
||||
$this->assertSame(1800, $profile['releases_per_hour']);
|
||||
$this->assertSame(1024, $profile['peak_memory_bytes']);
|
||||
}
|
||||
|
||||
public function test_targeted_additional_command_reports_a_successful_typed_outcome(): void
|
||||
{
|
||||
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
|
||||
DB::table('releases')->insert($this->releaseRow(1, 'a'));
|
||||
|
||||
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock): void {
|
||||
$mock->shouldReceive('processSingleGuid')
|
||||
->once()
|
||||
->with('a-guid-1')
|
||||
->andReturn(new ReleaseProcessingResult(
|
||||
1,
|
||||
'a-guid-1',
|
||||
ProcessingOutcome::Completed,
|
||||
artifactsCreated: true,
|
||||
releaseFilesAdded: 2,
|
||||
elapsedSeconds: 1.2345,
|
||||
stageDurations: ['nzb-parsing' => 0.5],
|
||||
downloadMetrics: new DownloadMetrics(2, 1, 1, 100, 100),
|
||||
persistenceMetrics: new PersistenceMetrics(4, 3.5, 2, 1),
|
||||
));
|
||||
$mock->shouldReceive('finish')->once();
|
||||
});
|
||||
|
||||
$status = Artisan::call('releases:additional', ['--id' => 1, '--profile' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertStringContainsString('outcome completed', $output);
|
||||
$this->assertStringContainsString('Artifacts created', $output);
|
||||
$this->assertStringContainsString('Release files added', $output);
|
||||
$this->assertStringContainsString('Stage profile', $output);
|
||||
$this->assertStringContainsString('Download profile', $output);
|
||||
$this->assertStringContainsString('Persistence profile', $output);
|
||||
}
|
||||
|
||||
public function test_targeted_additional_command_fails_for_an_unsuccessful_typed_outcome(): void
|
||||
{
|
||||
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
|
||||
DB::table('releases')->insert($this->releaseRow(1, 'a'));
|
||||
|
||||
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock): void {
|
||||
$mock->shouldReceive('processSingleGuid')
|
||||
->once()
|
||||
->with('a-guid-1')
|
||||
->andReturn(new ReleaseProcessingResult(
|
||||
1,
|
||||
'a-guid-1',
|
||||
ProcessingOutcome::TimedOut,
|
||||
reason: 'The release exceeded the post-processing timeout.',
|
||||
elapsedSeconds: 2.3456,
|
||||
));
|
||||
$mock->shouldReceive('finish')->once();
|
||||
});
|
||||
|
||||
$status = Artisan::call('releases:additional', ['--id' => 1]);
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(3, $status);
|
||||
$this->assertStringContainsString('outcome timed-out', $output);
|
||||
$this->assertStringContainsString('Reason', $output);
|
||||
$this->assertStringContainsString('The release exceeded the post-processing timeout.', $output);
|
||||
}
|
||||
|
||||
public function test_additional_diagnostics_reports_stale_claims_and_preflight_warnings_as_json(): void
|
||||
{
|
||||
config(['nntmux.tmp_unrar_path' => '']);
|
||||
DB::table('settings')->upsert([
|
||||
['name' => 'postthreads', 'value' => '33'],
|
||||
['name' => 'maxaddprocessed', 'value' => '25'],
|
||||
['name' => 'maxpptimeoutcount', 'value' => '3'],
|
||||
], ['name'], ['value']);
|
||||
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
|
||||
DB::table('releases')->insert([
|
||||
...$this->releaseRow(1, 'a'),
|
||||
'additional_pp_claimed_at' => now()->subHour(),
|
||||
'additional_pp_claim_token' => 'stale-claim',
|
||||
]);
|
||||
|
||||
$status = Artisan::call('nntmux:additional-diagnose', ['--json' => true]);
|
||||
$report = json_decode(Artisan::output(), true, flags: JSON_THROW_ON_ERROR);
|
||||
$warningCodes = array_column($report['warnings'], 'code');
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertSame(1, $report['backlog']['total']);
|
||||
$this->assertSame(1, $report['backlog']['available']);
|
||||
$this->assertSame(1, $report['backlog']['stale_claims']);
|
||||
$this->assertSame(825, $report['capacity']['max_in_flight']);
|
||||
$this->assertSame('v2', $report['settings']['pipeline']);
|
||||
$this->assertContains('stale-claims', $warningCodes);
|
||||
$this->assertContains('claim-ttl', $warningCodes);
|
||||
$this->assertContains('missing-index', $warningCodes);
|
||||
$this->assertContains('unwritable-temp-path', $warningCodes);
|
||||
$this->assertContains('oversized-capacity', $warningCodes);
|
||||
}
|
||||
|
||||
public function test_additional_diagnostics_recognizes_the_covering_claim_index(): void
|
||||
{
|
||||
Schema::table('releases', function (Blueprint $table): void {
|
||||
$table->index([
|
||||
'passwordstatus',
|
||||
'haspreview',
|
||||
'nzbstatus',
|
||||
'leftguid',
|
||||
'additional_pp_claimed_at',
|
||||
'postdate',
|
||||
], 'ix_releases_add_pp_claim_queue');
|
||||
});
|
||||
|
||||
$status = Artisan::call('nntmux:additional-diagnose', ['--json' => true]);
|
||||
$report = json_decode(Artisan::output(), true, flags: JSON_THROW_ON_ERROR);
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertFalse($report['indexes']['missing']);
|
||||
$this->assertSame('ix_releases_add_pp_claim_queue', $report['indexes']['covering_index']);
|
||||
$this->assertNotContains('missing-index', array_column($report['warnings'], 'code'));
|
||||
}
|
||||
|
||||
private function successfulBatch(int $releaseId): AdditionalBatchResult
|
||||
{
|
||||
return new AdditionalBatchResult(
|
||||
[$releaseId],
|
||||
[new ReleaseProcessingResult($releaseId, 'a-guid-'.$releaseId, ProcessingOutcome::Completed)],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Integration;
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalCandidateQuery;
|
||||
use Dotenv\Dotenv;
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use PDO;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use RuntimeException;
|
||||
use Tests\TestCase;
|
||||
|
||||
final class AdditionalCandidateQueryMariaDbTest extends TestCase
|
||||
{
|
||||
private string $tablePrefix;
|
||||
|
||||
public function createApplication()
|
||||
{
|
||||
Dotenv::createMutable(dirname(__DIR__, 2))->safeLoad();
|
||||
|
||||
$this->tablePrefix = 'phase4_'.getmypid().'_'.bin2hex(random_bytes(4)).'_';
|
||||
$database = (string) ($_ENV['DB_DATABASE'] ?? 'nntmux');
|
||||
|
||||
$this->setEnvironmentValue('APP_ENV', 'testing');
|
||||
$this->setEnvironmentValue('DB_URL', null);
|
||||
$this->setEnvironmentValue('DB_CONNECTION', 'mariadb');
|
||||
$this->setEnvironmentValue('DB_DATABASE', $database);
|
||||
|
||||
$app = require __DIR__.'/../../bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
$app->make('config')->set('database.connections.mariadb.prefix', $this->tablePrefix);
|
||||
$app->make('db')->purge('mariadb');
|
||||
|
||||
return $app;
|
||||
}
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
|
||||
if (DB::getDriverName() !== 'mariadb') {
|
||||
$this->markTestSkipped('MariaDB integration test.');
|
||||
}
|
||||
|
||||
$this->createSchema();
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
if (isset($this->tablePrefix) && preg_match('/^phase4_\d+_[a-f0-9]{8}_$/', $this->tablePrefix) === 1) {
|
||||
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('releases').'`');
|
||||
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('categories').'`');
|
||||
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('settings').'`');
|
||||
}
|
||||
|
||||
DB::disconnect();
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function current_indexes_bound_examined_rows_for_realistic_claim_and_backlog_plans(): void
|
||||
{
|
||||
$releasesTable = $this->tableName('releases');
|
||||
$categoriesTable = $this->tableName('categories');
|
||||
|
||||
DB::statement(<<<SQL
|
||||
INSERT INTO `{$releasesTable}` (
|
||||
id, guid, leftguid, passwordstatus, haspreview, nzbstatus,
|
||||
categories_id, size, postdate, additional_pp_claimed_at
|
||||
)
|
||||
SELECT
|
||||
seq,
|
||||
CONCAT(SUBSTRING('0123456789abcdef', MOD(seq, 16) + 1, 1), '-guid-', seq),
|
||||
SUBSTRING('0123456789abcdef', MOD(seq, 16) + 1, 1),
|
||||
IF(MOD(seq, 97) = 0, -1, 0),
|
||||
IF(MOD(seq, 97) = 0, -1, 0),
|
||||
1,
|
||||
1,
|
||||
2097152,
|
||||
DATE_SUB(NOW(), INTERVAL seq SECOND),
|
||||
IF(MOD(seq, 679) = 0, NOW(), NULL)
|
||||
FROM seq_1_to_50000
|
||||
SQL);
|
||||
|
||||
DB::statement("ANALYZE TABLE `{$releasesTable}`");
|
||||
|
||||
$claimPlan = $this->analyze(<<<SQL
|
||||
SELECT r.id
|
||||
FROM `{$releasesTable}` r
|
||||
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
|
||||
WHERE r.passwordstatus = -1
|
||||
AND r.haspreview = -1
|
||||
AND r.nzbstatus = 1
|
||||
AND c.disablepreview = 0
|
||||
AND r.size > 1048576
|
||||
AND r.size < 107374182400
|
||||
AND r.leftguid = 'a'
|
||||
AND (r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND))
|
||||
ORDER BY r.postdate DESC, r.id ASC
|
||||
LIMIT 25
|
||||
FOR UPDATE
|
||||
SQL);
|
||||
$bucketPlan = $this->analyze(<<<SQL
|
||||
SELECT r.leftguid, COUNT(*) AS total_count,
|
||||
SUM(CASE WHEN r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND) THEN 1 ELSE 0 END) AS available_count
|
||||
FROM `{$releasesTable}` r
|
||||
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
|
||||
WHERE r.passwordstatus = -1
|
||||
AND r.haspreview = -1
|
||||
AND r.nzbstatus = 1
|
||||
AND c.disablepreview = 0
|
||||
AND r.size > 1048576
|
||||
AND r.size < 107374182400
|
||||
GROUP BY r.leftguid
|
||||
ORDER BY r.leftguid
|
||||
SQL);
|
||||
$backlogPlan = $this->analyze(<<<SQL
|
||||
SELECT COUNT(*) AS total_count,
|
||||
SUM(CASE WHEN r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND) THEN 1 ELSE 0 END) AS available_count
|
||||
FROM `{$releasesTable}` r
|
||||
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
|
||||
WHERE r.passwordstatus = -1
|
||||
AND r.haspreview = -1
|
||||
AND r.nzbstatus = 1
|
||||
AND c.disablepreview = 0
|
||||
AND r.size > 1048576
|
||||
AND r.size < 107374182400
|
||||
SQL);
|
||||
|
||||
$this->assertStringContainsString('ix_releases_add_pp_claim_queue', $claimPlan);
|
||||
$this->assertLessThan(5_000, $this->examinedRowsFor($claimPlan, 'r'));
|
||||
$this->assertLessThanOrEqual(2_000, $this->examinedRowsFor($bucketPlan, 'r'));
|
||||
$this->assertLessThanOrEqual(2_000, $this->examinedRowsFor($backlogPlan, 'r'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function claims_exclude_fresh_rows_recover_stale_rows_and_preserve_ordering(): void
|
||||
{
|
||||
DB::table('releases')->insert([
|
||||
$this->releaseRow(1, '2026-08-10 12:00:00', now()),
|
||||
$this->releaseRow(2, '2026-08-10 11:00:00'),
|
||||
$this->releaseRow(3, '2026-08-10 11:00:00'),
|
||||
$this->releaseRow(4, '2026-08-10 10:00:00', now()->subSeconds(301)),
|
||||
]);
|
||||
|
||||
$firstClaim = AdditionalCandidateQuery::claimBatch('a', 3, 'worker-one', columns: ['id']);
|
||||
$secondClaim = AdditionalCandidateQuery::claimBatch('a', 3, 'worker-two', columns: ['id']);
|
||||
|
||||
$this->assertSame([2, 3, 4], $firstClaim->pluck('id')->all());
|
||||
$this->assertSame([], $secondClaim->pluck('id')->all());
|
||||
$this->assertSame('worker-one', DB::table('releases')->where('id', 4)->value('additional_pp_claim_token'));
|
||||
$this->assertSame('claimed', DB::table('releases')->where('id', 1)->value('additional_pp_claim_token'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function concurrent_workers_do_not_claim_the_same_hot_bucket_rows(): void
|
||||
{
|
||||
if (! function_exists('pcntl_fork')) {
|
||||
$this->markTestSkipped('The pcntl extension is required for the MariaDB contention test.');
|
||||
}
|
||||
|
||||
DB::table('releases')->insert(array_map(
|
||||
fn (int $id): array => $this->releaseRow($id, now()->subSeconds($id)->format('Y-m-d H:i:s')),
|
||||
range(1, 20),
|
||||
));
|
||||
|
||||
DB::beginTransaction();
|
||||
$firstWorkerIds = AdditionalCandidateQuery::baseBuilder(guidChar: 'a')
|
||||
->select('r.id')
|
||||
->orderByDesc('r.postdate')
|
||||
->orderBy('r.id')
|
||||
->limit(5)
|
||||
->lockForUpdate()
|
||||
->pluck('r.id')
|
||||
->map(static fn (mixed $id): int => (int) $id)
|
||||
->all();
|
||||
|
||||
$sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP);
|
||||
if ($sockets === false) {
|
||||
DB::rollBack();
|
||||
throw new RuntimeException('Unable to create an IPC socket pair.');
|
||||
}
|
||||
|
||||
$pid = pcntl_fork();
|
||||
if ($pid === -1) {
|
||||
DB::rollBack();
|
||||
throw new RuntimeException('Unable to fork the competing claim worker.');
|
||||
}
|
||||
|
||||
if ($pid === 0) {
|
||||
fclose($sockets[0]);
|
||||
|
||||
try {
|
||||
$connection = $this->createMariaDbConnection();
|
||||
$connection->beginTransaction();
|
||||
$releasesTable = $this->tableName('releases');
|
||||
$categoriesTable = $this->tableName('categories');
|
||||
$claimedIds = array_map(
|
||||
static fn (mixed $id): int => (int) $id,
|
||||
$connection->query(<<<SQL
|
||||
SELECT r.id
|
||||
FROM `{$releasesTable}` r
|
||||
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
|
||||
WHERE r.passwordstatus = -1
|
||||
AND r.haspreview = -1
|
||||
AND r.nzbstatus = 1
|
||||
AND c.disablepreview = 0
|
||||
AND r.leftguid = 'a'
|
||||
AND (r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND))
|
||||
ORDER BY r.postdate DESC, r.id ASC
|
||||
LIMIT 5
|
||||
FOR UPDATE
|
||||
SQL)->fetchAll(PDO::FETCH_COLUMN),
|
||||
);
|
||||
$placeholders = implode(',', array_fill(0, count($claimedIds), '?'));
|
||||
$update = $connection->prepare("UPDATE `{$releasesTable}` SET additional_pp_claimed_at = ?, additional_pp_claim_token = ? WHERE id IN ({$placeholders})");
|
||||
$update->execute([now()->format('Y-m-d H:i:s'), 'worker-two', ...$claimedIds]);
|
||||
$connection->commit();
|
||||
fwrite($sockets[1], json_encode(['ids' => $claimedIds], JSON_THROW_ON_ERROR));
|
||||
fclose($sockets[1]);
|
||||
exit(0);
|
||||
} catch (\Throwable $exception) {
|
||||
fwrite($sockets[1], json_encode(['error' => $exception->getMessage()], JSON_THROW_ON_ERROR));
|
||||
fclose($sockets[1]);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fclose($sockets[1]);
|
||||
usleep(200_000);
|
||||
DB::table('releases')->whereIn('id', $firstWorkerIds)->update([
|
||||
'additional_pp_claimed_at' => now(),
|
||||
'additional_pp_claim_token' => 'worker-one',
|
||||
]);
|
||||
DB::commit();
|
||||
|
||||
$childPayload = stream_get_contents($sockets[0]);
|
||||
fclose($sockets[0]);
|
||||
pcntl_waitpid($pid, $status);
|
||||
DB::purge();
|
||||
DB::reconnect();
|
||||
|
||||
/** @var array{ids?: list<int>, error?: string} $childResult */
|
||||
$childResult = json_decode($childPayload, true, flags: JSON_THROW_ON_ERROR);
|
||||
|
||||
$this->assertArrayNotHasKey('error', $childResult, $childResult['error'] ?? 'Competing worker failed.');
|
||||
$this->assertSame([1, 2, 3, 4, 5], $firstWorkerIds);
|
||||
$this->assertSame([6, 7, 8, 9, 10], $childResult['ids'] ?? []);
|
||||
$this->assertSame([], array_intersect($firstWorkerIds, $childResult['ids'] ?? []));
|
||||
$this->assertTrue(pcntl_wifexited($status));
|
||||
$this->assertSame(0, pcntl_wexitstatus($status));
|
||||
}
|
||||
|
||||
private function createSchema(): void
|
||||
{
|
||||
$settingsTable = $this->tableName('settings');
|
||||
$categoriesTable = $this->tableName('categories');
|
||||
$releasesTable = $this->tableName('releases');
|
||||
|
||||
DB::statement("CREATE TABLE `{$settingsTable}` (`name` VARCHAR(255) PRIMARY KEY, `value` TEXT NULL) ENGINE=InnoDB");
|
||||
DB::statement("CREATE TABLE `{$categoriesTable}` (id INT UNSIGNED PRIMARY KEY, disablepreview TINYINT(1) NOT NULL DEFAULT 0) ENGINE=InnoDB");
|
||||
DB::statement(<<<SQL
|
||||
CREATE TABLE `{$releasesTable}` (
|
||||
id INT UNSIGNED PRIMARY KEY,
|
||||
guid VARCHAR(255) NOT NULL,
|
||||
leftguid CHAR(1) NOT NULL,
|
||||
passwordstatus INT NOT NULL,
|
||||
haspreview INT NOT NULL,
|
||||
nzbstatus INT NOT NULL,
|
||||
categories_id INT UNSIGNED NOT NULL,
|
||||
size BIGINT UNSIGNED NOT NULL,
|
||||
postdate DATETIME NULL,
|
||||
additional_pp_claimed_at TIMESTAMP NULL,
|
||||
additional_pp_claim_token VARCHAR(64) NULL,
|
||||
KEY ix_releases_haspreview_passwordstatus (haspreview, passwordstatus),
|
||||
KEY ix_releases_add_pp_claim_queue (passwordstatus, haspreview, nzbstatus, leftguid, additional_pp_claimed_at, postdate DESC)
|
||||
) ENGINE=InnoDB
|
||||
SQL);
|
||||
DB::table('settings')->insert([
|
||||
['name' => 'categorizeforeign', 'value' => '0'],
|
||||
['name' => 'catwebdl', 'value' => '0'],
|
||||
['name' => 'releaseprocessingtimeout', 'value' => '120'],
|
||||
]);
|
||||
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function releaseRow(int $id, string $postdate, ?\DateTimeInterface $claimedAt = null): array
|
||||
{
|
||||
return [
|
||||
'id' => $id,
|
||||
'guid' => 'a-guid-'.$id,
|
||||
'leftguid' => 'a',
|
||||
'passwordstatus' => -1,
|
||||
'haspreview' => -1,
|
||||
'nzbstatus' => 1,
|
||||
'categories_id' => 1,
|
||||
'size' => 2 * 1048576,
|
||||
'postdate' => $postdate,
|
||||
'additional_pp_claimed_at' => $claimedAt?->format('Y-m-d H:i:s'),
|
||||
'additional_pp_claim_token' => $claimedAt === null ? null : 'claimed',
|
||||
];
|
||||
}
|
||||
|
||||
private function analyze(string $sql): string
|
||||
{
|
||||
$row = DB::selectOne('ANALYZE FORMAT=JSON '.$sql);
|
||||
$values = (array) $row;
|
||||
|
||||
return (string) reset($values);
|
||||
}
|
||||
|
||||
private function examinedRowsFor(string $plan, string $tableName): float
|
||||
{
|
||||
/** @var array<string, mixed> $decoded */
|
||||
$decoded = json_decode($plan, true, flags: JSON_THROW_ON_ERROR);
|
||||
$examinedRows = $this->findExaminedRows($decoded, $tableName);
|
||||
|
||||
if ($examinedRows === null) {
|
||||
throw new RuntimeException("Unable to find runtime row count for table [{$tableName}].");
|
||||
}
|
||||
|
||||
return $examinedRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, mixed> $node
|
||||
*/
|
||||
private function findExaminedRows(array $node, string $tableName): ?float
|
||||
{
|
||||
if (($node['table_name'] ?? null) === $tableName && isset($node['r_rows'])) {
|
||||
return (float) $node['r_rows'];
|
||||
}
|
||||
|
||||
foreach ($node as $value) {
|
||||
if (! is_array($value)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rows = $this->findExaminedRows($value, $tableName);
|
||||
if ($rows !== null) {
|
||||
return $rows;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function tableName(string $table): string
|
||||
{
|
||||
return $this->tablePrefix.$table;
|
||||
}
|
||||
|
||||
private function createMariaDbConnection(): PDO
|
||||
{
|
||||
$host = (string) ($_ENV['DB_HOST'] ?? 'mariadb');
|
||||
$port = (string) ($_ENV['DB_PORT'] ?? '3306');
|
||||
$database = (string) ($_ENV['DB_DATABASE'] ?? 'nntmux');
|
||||
$username = (string) ($_ENV['DB_USERNAME'] ?? 'nntmux');
|
||||
$password = (string) ($_ENV['DB_PASSWORD'] ?? '');
|
||||
|
||||
return new PDO(
|
||||
"mysql:host={$host};port={$port};dbname={$database};charset=utf8mb4",
|
||||
$username,
|
||||
$password,
|
||||
[PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION],
|
||||
);
|
||||
}
|
||||
|
||||
private function setEnvironmentValue(string $key, ?string $value): void
|
||||
{
|
||||
if ($value === null) {
|
||||
putenv($key);
|
||||
unset($_ENV[$key], $_SERVER[$key]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
putenv($key.'='.$value);
|
||||
$_ENV[$key] = $value;
|
||||
$_SERVER[$key] = $value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
|
||||
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class AdditionalWorkPlannerTest extends TestCase
|
||||
{
|
||||
use CreatesProcessingConfiguration;
|
||||
|
||||
#[Test]
|
||||
public function it_builds_one_ordered_plan_for_direct_and_archive_candidates(): void
|
||||
{
|
||||
$planner = new AdditionalWorkPlanner($this->makeConfig([
|
||||
'processThumbnails' => true,
|
||||
'processJPGSample' => true,
|
||||
'processMediaInfo' => true,
|
||||
'processAudioInfo' => true,
|
||||
]));
|
||||
|
||||
$plan = $planner->plan([
|
||||
['title' => 'release.part02.rar', 'segments' => ['<part-2>', '<shared>']],
|
||||
['title' => '"sample.mkv" yEnc', 'segments' => ['<sample>', '<sample-2>']],
|
||||
['title' => '"cover.jpg" yEnc', 'segments' => ['<cover>']],
|
||||
['title' => '"track.FLAC" yEnc', 'segments' => ['<audio>']],
|
||||
['title' => 'release.part01.rar', 'segments' => ['<part-1>', '<shared>', '<part-1>']],
|
||||
], 'alt.binaries.test');
|
||||
|
||||
$this->assertSame(['<sample>', '<sample-2>'], $plan->sampleMessageIds);
|
||||
$this->assertSame(['<cover>'], $plan->jpgMessageIds);
|
||||
$this->assertSame('<sample>', $plan->mediaInfoMessageId);
|
||||
$this->assertSame('<audio>', $plan->audioInfoMessageId);
|
||||
$this->assertSame('FLAC', $plan->audioInfoExtension);
|
||||
$this->assertTrue($plan->hasCompressedFile());
|
||||
$this->assertSame(
|
||||
['release.part02.rar', 'release.part01.rar'],
|
||||
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->archiveCandidates),
|
||||
);
|
||||
$this->assertFalse($plan->archiveCandidates[0]->likelyFirstVolume);
|
||||
$this->assertTrue($plan->archiveCandidates[1]->likelyFirstVolume);
|
||||
$this->assertSame(
|
||||
['release.part01.rar', 'release.part02.rar'],
|
||||
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->prioritizedArchiveCandidates()),
|
||||
);
|
||||
$this->assertSame(
|
||||
['release.part01.rar', 'release.part02.rar'],
|
||||
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->orderedArchiveCandidates(true)),
|
||||
);
|
||||
$this->assertSame(3, $plan->duplicateMessageIdCount);
|
||||
$this->assertSame([], $plan->unsupportedReasons);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reports_book_floods_and_releases_without_supported_candidates(): void
|
||||
{
|
||||
$planner = new AdditionalWorkPlanner($this->makeConfig());
|
||||
$contents = array_fill(0, 81, [
|
||||
'title' => 'book.epub',
|
||||
'segments' => ['<book>'],
|
||||
]);
|
||||
|
||||
$plan = $planner->plan($contents, 'alt.binaries.books');
|
||||
|
||||
$this->assertSame(81, $plan->bookFileCount);
|
||||
$this->assertTrue($plan->bookFlood);
|
||||
$this->assertSame(['book-flood', 'no-supported-candidates'], $plan->unsupportedReasons);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_selects_jpeg_png_and_webp_image_candidates(): void
|
||||
{
|
||||
$planner = new AdditionalWorkPlanner($this->makeConfig(['processJPGSample' => true]));
|
||||
|
||||
foreach (['cover.jpg', 'cover.png', 'cover.webp'] as $index => $filename) {
|
||||
$messageId = '<image-'.$index.'>';
|
||||
$plan = $planner->plan([
|
||||
['title' => '"'.$filename.'" yEnc', 'segments' => [$messageId]],
|
||||
], 'alt.binaries.test');
|
||||
|
||||
$this->assertSame([$messageId], $plan->jpgMessageIds, $filename.' should be selected');
|
||||
}
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_keeps_a_usable_last_volume_when_the_first_volume_is_missing(): void
|
||||
{
|
||||
$planner = new AdditionalWorkPlanner($this->makeConfig());
|
||||
$plan = $planner->plan([
|
||||
['title' => 'release.part99.rar', 'segments' => ['<last-volume>']],
|
||||
], 'alt.binaries.test');
|
||||
|
||||
$this->assertFalse($plan->archiveCandidates[0]->likelyFirstVolume);
|
||||
$this->assertSame(['<last-volume>'], $plan->orderedArchiveCandidates(true)[0]->messageIds);
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,14 @@
|
||||
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\ArchiveExtractionService;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\Releases\ReleaseBrowseService;
|
||||
use dariusiii\rarinfo\ArchiveInfo;
|
||||
use dariusiii\rarinfo\Par2Info;
|
||||
use Mockery;
|
||||
use PHPUnit\Framework\Attributes\DataProvider;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
@@ -35,6 +39,26 @@ class ArchiveExtractionServiceTest extends TestCase
|
||||
$this->assertSame('IMAGE-DATA', $service->extractSpecificFile('ARCHIVE', 'cover.jpg', sys_get_temp_dir().'/'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_inspects_an_archive_once_when_extracting_multiple_candidates(): void
|
||||
{
|
||||
$archiveInfo = Mockery::mock(ArchiveInfo::class);
|
||||
$archiveInfo->shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true);
|
||||
$archiveInfo->shouldReceive('getFileData')->once()->with('release.nfo')->andReturn('NFO-DATA');
|
||||
$archiveInfo->shouldReceive('getFileData')->once()->with('cover.jpg')->andReturn('IMAGE-DATA');
|
||||
|
||||
$service = new ArchiveExtractionService(
|
||||
$this->makeConfig(),
|
||||
$archiveInfo,
|
||||
Mockery::mock(Par2Info::class)
|
||||
);
|
||||
|
||||
$this->assertSame([
|
||||
'release.nfo' => 'NFO-DATA',
|
||||
'cover.jpg' => 'IMAGE-DATA',
|
||||
], $service->extractSpecificFiles('ARCHIVE', ['release.nfo', 'cover.jpg'], sys_get_temp_dir().'/'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_detects_common_standalone_video_signatures(): void
|
||||
{
|
||||
@@ -51,4 +75,68 @@ class ArchiveExtractionServiceTest extends TestCase
|
||||
$this->assertSame('mp4', $service->detectStandaloneVideo($mp4));
|
||||
$this->assertNull($service->detectStandaloneVideo('tiny'));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
#[DataProvider('archiveProtectionFixtures')]
|
||||
public function it_classifies_passworded_and_unpassworded_rar_and_zip_data(
|
||||
int $archiveType,
|
||||
string $expectedMarker,
|
||||
bool $encrypted,
|
||||
): void {
|
||||
$archiveInfo = Mockery::mock(ArchiveInfo::class);
|
||||
$archiveInfo->error = '';
|
||||
$archiveInfo->shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true);
|
||||
$archiveInfo->shouldReceive('getSummary')->once()->with(true)->andReturn([
|
||||
'main_type' => $archiveType,
|
||||
'is_encrypted' => $encrypted ? 1 : 0,
|
||||
'archives' => [
|
||||
'nested.rar' => ['file_list' => [['name' => 'Nested.Release.2026.mkv']]],
|
||||
],
|
||||
]);
|
||||
|
||||
if ($encrypted) {
|
||||
$archiveInfo->shouldNotReceive('getArchiveFileList');
|
||||
} else {
|
||||
$archiveInfo->shouldReceive('getArchiveFileList')->once()->andReturn([
|
||||
['name' => 'nested.rar', 'size' => 100],
|
||||
]);
|
||||
}
|
||||
|
||||
$service = new ArchiveExtractionService(
|
||||
$this->makeConfig(),
|
||||
$archiveInfo,
|
||||
Mockery::mock(Par2Info::class)
|
||||
);
|
||||
$context = new ReleaseProcessingContext(new Release(['id' => 1, 'guid' => 'fixture-guid']));
|
||||
|
||||
$result = $service->processCompressedData('ARCHIVE', $context, sys_get_temp_dir().'/');
|
||||
|
||||
$this->assertSame($encrypted, $result['hasPassword']);
|
||||
$this->assertSame(
|
||||
$encrypted ? ReleaseBrowseService::PASSWD_RAR : ReleaseBrowseService::PASSWD_NONE,
|
||||
$result['passwordStatus'],
|
||||
);
|
||||
|
||||
if (! $encrypted) {
|
||||
$this->assertTrue($result['success']);
|
||||
$this->assertSame($expectedMarker, $result['archiveMarker']);
|
||||
$this->assertSame(
|
||||
'Nested.Release.2026.mkv',
|
||||
$result['dataSummary']['archives']['nested.rar']['file_list'][0]['name'],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, array{int, string, bool}>
|
||||
*/
|
||||
public static function archiveProtectionFixtures(): array
|
||||
{
|
||||
return [
|
||||
'unpassworded RAR' => [ArchiveInfo::TYPE_RAR, 'r', false],
|
||||
'passworded RAR' => [ArchiveInfo::TYPE_RAR, 'r', true],
|
||||
'unpassworded ZIP' => [ArchiveInfo::TYPE_ZIP, 'z', false],
|
||||
'passworded ZIP' => [ArchiveInfo::TYPE_ZIP, 'z', true],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\MediaExtractionService;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use App\Services\ReleaseExtraService;
|
||||
use App\Services\ReleaseImageService;
|
||||
use Illuminate\Filesystem\Filesystem;
|
||||
use Illuminate\Foundation\Application;
|
||||
use Illuminate\Support\Facades\Facade;
|
||||
use Mockery;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class MediaExtractionServiceTest extends TestCase
|
||||
{
|
||||
use CreatesProcessingConfiguration;
|
||||
|
||||
private string $tmpPath;
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
|
||||
$container = new Application(sys_get_temp_dir());
|
||||
$container->instance('files', new Filesystem);
|
||||
Facade::setFacadeApplication($container);
|
||||
|
||||
$this->tmpPath = sys_get_temp_dir().'/additional-media-'.uniqid('', true).'/';
|
||||
(new Filesystem)->makeDirectory($this->tmpPath, 0777, true, true);
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
(new Filesystem)->deleteDirectory($this->tmpPath);
|
||||
Mockery::close();
|
||||
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_recognizes_jpeg_png_and_webp_samples_by_signature(): void
|
||||
{
|
||||
$fixtures = [
|
||||
'sample.jpg' => "\xFF\xD8\xFF\xE0\x00\x10JFIF\x00",
|
||||
'sample.png' => "\x89PNG\r\n\x1A\n\x00\x00\x00\rIHDR",
|
||||
'sample.webp' => "RIFF\x1A\x00\x00\x00WEBPVP8 ",
|
||||
];
|
||||
$service = new MediaExtractionService(
|
||||
$this->makeConfig(),
|
||||
Mockery::mock(ReleaseImageService::class),
|
||||
Mockery::mock(ReleaseExtraService::class),
|
||||
Mockery::mock(CategorizationService::class),
|
||||
);
|
||||
|
||||
foreach ($fixtures as $filename => $contents) {
|
||||
$path = $this->tmpPath.$filename;
|
||||
file_put_contents($path, $contents);
|
||||
|
||||
$this->assertTrue($service->isValidImage($path), $filename.' should be recognized');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
|
||||
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
|
||||
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
|
||||
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class ProcessingResultTest extends TestCase
|
||||
{
|
||||
#[Test]
|
||||
public function it_aggregates_typed_release_outcomes(): void
|
||||
{
|
||||
$batch = new AdditionalBatchResult(
|
||||
claimedIds: [10, 11, 12],
|
||||
results: [
|
||||
new ReleaseProcessingResult(
|
||||
10,
|
||||
'a-guid-10',
|
||||
ProcessingOutcome::Completed,
|
||||
artifactsCreated: true,
|
||||
releaseFilesAdded: 3,
|
||||
elapsedSeconds: 0.5,
|
||||
stageDurations: ['nzb-parsing' => 0.2, 'finalization' => 0.1],
|
||||
downloadMetrics: new DownloadMetrics(2, 1, 1, 100, 100),
|
||||
persistenceMetrics: new PersistenceMetrics(8, 12.5, 2, 1),
|
||||
duplicateMessageIdCount: 1,
|
||||
),
|
||||
new ReleaseProcessingResult(
|
||||
11,
|
||||
'a-guid-11',
|
||||
ProcessingOutcome::Passworded,
|
||||
elapsedSeconds: 0.75,
|
||||
stageDurations: ['nzb-parsing' => 0.3],
|
||||
downloadMetrics: new DownloadMetrics(1, 1, 0, 50, 0),
|
||||
persistenceMetrics: new PersistenceMetrics(5, 7.5, 1, 1),
|
||||
unsupportedReasons: ['book-flood'],
|
||||
),
|
||||
new ReleaseProcessingResult(
|
||||
12,
|
||||
'a-guid-12',
|
||||
ProcessingOutcome::TimedOut,
|
||||
elapsedSeconds: 1.25,
|
||||
stageDurations: ['nzb-parsing' => 0.5, 'timeout-handling' => 0.05],
|
||||
),
|
||||
],
|
||||
elapsedSeconds: 2.0,
|
||||
peakMemoryBytes: 32_000_000,
|
||||
);
|
||||
|
||||
$this->assertSame(3, $batch->claimedCount());
|
||||
$this->assertSame(3, $batch->attemptedCount());
|
||||
$this->assertSame(2, $batch->successfulCount());
|
||||
$this->assertSame(1, $batch->unsuccessfulCount());
|
||||
$this->assertSame([
|
||||
'completed' => 1,
|
||||
'passworded' => 1,
|
||||
'timed-out' => 1,
|
||||
], $batch->outcomeCounts());
|
||||
$this->assertTrue($batch->hasOutcome(ProcessingOutcome::TimedOut));
|
||||
$this->assertSame(ProcessingOutcome::Completed, $batch->firstResult()?->outcome);
|
||||
$this->assertSame(1.5, $batch->releasesPerSecond());
|
||||
$this->assertEqualsWithDelta(0.833333, $batch->averageReleaseSeconds(), 0.000001);
|
||||
$this->assertSame(1, $batch->artifactsCreatedCount());
|
||||
$this->assertSame(50.0, $batch->artifactYieldPercent());
|
||||
$this->assertSame(3, $batch->releaseFilesAdded());
|
||||
$this->assertEquals(new DownloadMetrics(3, 2, 1, 150, 100), $batch->downloadMetrics());
|
||||
$this->assertEquals(new PersistenceMetrics(13, 20.0, 3, 2), $batch->persistenceMetrics());
|
||||
$this->assertSame(1, $batch->duplicateMessageIdCount());
|
||||
$this->assertSame(['book-flood' => 1], $batch->unsupportedReasonCounts());
|
||||
$this->assertSame([
|
||||
'nzb-parsing' => 1.0,
|
||||
'finalization' => 0.1,
|
||||
'timeout-handling' => 0.05,
|
||||
], $batch->stageDurationTotals());
|
||||
$this->assertSame(32_000_000, $batch->peakMemoryBytes);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_represents_a_worker_setup_failure_without_claiming_releases(): void
|
||||
{
|
||||
$batch = AdditionalBatchResult::setupFailed('Temporary path is unavailable.');
|
||||
|
||||
$this->assertSame(0, $batch->claimedCount());
|
||||
$this->assertSame(0, $batch->attemptedCount());
|
||||
$this->assertSame('Temporary path is unavailable.', $batch->setupFailure);
|
||||
$this->assertNull($batch->firstResult());
|
||||
$this->assertSame(0.0, $batch->releasesPerSecond());
|
||||
$this->assertSame(0.0, $batch->artifactYieldPercent());
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function completing_without_useful_artifacts_is_a_truthful_successful_outcome(): void
|
||||
{
|
||||
$result = new ReleaseProcessingResult(
|
||||
10,
|
||||
'a-guid-10',
|
||||
ProcessingOutcome::NoUsefulArtifacts,
|
||||
reason: 'Processing completed without creating useful artifacts.',
|
||||
);
|
||||
|
||||
$this->assertTrue($result->isSuccessful());
|
||||
$this->assertFalse($result->artifactsCreated);
|
||||
$this->assertSame('Processing completed without creating useful artifacts.', $result->reason);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_can_attach_performance_data_without_changing_the_processing_outcome(): void
|
||||
{
|
||||
$result = new ReleaseProcessingResult(
|
||||
10,
|
||||
'a-guid-10',
|
||||
ProcessingOutcome::Completed,
|
||||
artifactsCreated: true,
|
||||
releaseFilesAdded: 2,
|
||||
);
|
||||
|
||||
$downloadMetrics = new DownloadMetrics(2, 1, 1, 100, 100);
|
||||
$persistenceMetrics = new PersistenceMetrics(4, 3.5, 2, 1);
|
||||
$measured = $result->withPerformance(
|
||||
1.25,
|
||||
['nzb-parsing' => 0.4],
|
||||
$downloadMetrics,
|
||||
$persistenceMetrics,
|
||||
);
|
||||
|
||||
$this->assertSame($result->releaseId, $measured->releaseId);
|
||||
$this->assertSame($result->outcome, $measured->outcome);
|
||||
$this->assertTrue($measured->artifactsCreated);
|
||||
$this->assertSame(2, $measured->releaseFilesAdded);
|
||||
$this->assertSame(1.25, $measured->elapsedSeconds);
|
||||
$this->assertSame(['nzb-parsing' => 0.4], $measured->stageDurations);
|
||||
$this->assertSame($downloadMetrics, $measured->downloadMetrics);
|
||||
$this->assertSame($persistenceMetrics, $measured->persistenceMetrics);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Models\Release;
|
||||
use App\Models\ReleaseFile;
|
||||
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
|
||||
use App\Services\AdditionalProcessing\ArchiveExtractionService;
|
||||
use App\Services\AdditionalProcessing\ConsoleOutputService;
|
||||
use App\Services\AdditionalProcessing\MediaExtractionService;
|
||||
@@ -59,10 +60,13 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_extracts_jpg_candidates_from_an_archive_file_list(): void
|
||||
public function it_extracts_webp_candidates_from_an_archive_file_list(): void
|
||||
{
|
||||
$archive = Mockery::mock(ArchiveExtractionService::class);
|
||||
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('JPEG-DATA');
|
||||
$archive->shouldReceive('extractSpecificFiles')
|
||||
->once()
|
||||
->with('ARCHIVE', ['cover.webp'], Mockery::type('string'))
|
||||
->andReturn(['cover.webp' => 'WEBP-DATA']);
|
||||
|
||||
$media = Mockery::mock(MediaExtractionService::class);
|
||||
$media->shouldReceive('isValidImage')->once()->andReturn(true);
|
||||
@@ -72,7 +76,6 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
$output->shouldReceive('echoJpgSaved')->once()->andReturnNull();
|
||||
|
||||
$fallback = new ReleaseFilesArchiveFallback(
|
||||
$this->makeConfig(['processJPGSample' => true]),
|
||||
$archive,
|
||||
$media,
|
||||
Mockery::mock(UsenetDownloadService::class),
|
||||
@@ -81,7 +84,7 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.jpg', 'size' => 99]], $context);
|
||||
$fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.webp', 'size' => 99]], $context);
|
||||
|
||||
$this->assertTrue($context->foundJPGSample);
|
||||
}
|
||||
@@ -96,7 +99,10 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
]);
|
||||
|
||||
$archive = Mockery::mock(ArchiveExtractionService::class);
|
||||
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('NFO DATA');
|
||||
$archive->shouldReceive('extractSpecificFiles')
|
||||
->once()
|
||||
->with('ARCHIVE', ['release.nfo'], Mockery::type('string'))
|
||||
->andReturn(['release.nfo' => 'NFO DATA']);
|
||||
|
||||
$download = Mockery::mock(UsenetDownloadService::class);
|
||||
$download->shouldReceive('download')->once()->andReturn([
|
||||
@@ -118,8 +124,8 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
|
||||
|
||||
$config = $this->makeConfig();
|
||||
$fallback = new ReleaseFilesArchiveFallback(
|
||||
$this->makeConfig(),
|
||||
$archive,
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$download,
|
||||
@@ -131,12 +137,133 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
|
||||
$context->releaseGroupName = 'alt.binaries.test';
|
||||
$context->releaseHasNoNFO = true;
|
||||
$context->nzbContents = [['title' => 'archive.part01.rar', 'segments' => ['<1>', '<2>']]];
|
||||
$context->workPlan = (new AdditionalWorkPlanner($config))->plan(
|
||||
$context->nzbContents,
|
||||
$context->releaseGroupName,
|
||||
);
|
||||
|
||||
$fallback->processNfoFromReleaseFiles($context);
|
||||
|
||||
$this->assertFalse($context->releaseHasNoNFO);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_recovers_an_nfo_from_an_already_downloaded_archive_without_another_download(): void
|
||||
{
|
||||
$files = [['name' => 'release.nfo', 'size' => 200]];
|
||||
|
||||
$archive = Mockery::mock(ArchiveExtractionService::class);
|
||||
$archive->shouldReceive('sortFilesWithNfoPriority')->once()->with($files)->andReturn($files);
|
||||
$archive->shouldReceive('isNfoFile')->once()->with('release.nfo')->andReturn(true);
|
||||
$archive->shouldReceive('extractSpecificFiles')
|
||||
->once()
|
||||
->with('ARCHIVE', ['release.nfo'], Mockery::type('string'))
|
||||
->andReturn(['release.nfo' => 'NFO DATA']);
|
||||
|
||||
$download = Mockery::mock(UsenetDownloadService::class);
|
||||
$download->shouldNotReceive('download');
|
||||
$download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class));
|
||||
|
||||
$manager = Mockery::mock(ReleaseFileManager::class);
|
||||
$manager->shouldReceive('processNfoFile')->once()->andReturnUsing(
|
||||
function (string $path, ReleaseProcessingContext $context): bool {
|
||||
$this->assertSame('NFO DATA', file_get_contents($path));
|
||||
$context->releaseHasNoNFO = false;
|
||||
|
||||
return true;
|
||||
}
|
||||
);
|
||||
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
|
||||
|
||||
$fallback = new ReleaseFilesArchiveFallback(
|
||||
$archive,
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$download,
|
||||
$manager,
|
||||
$output
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->releaseHasNoNFO = true;
|
||||
$this->assertTrue($context->rememberDownloadedArchive('release.part01.rar', 'ARCHIVE', $files));
|
||||
|
||||
$fallback->processNfoFromDownloadedArchives($context);
|
||||
|
||||
$this->assertFalse($context->releaseHasNoNFO);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_extracts_archive_candidates_in_one_batch(): void
|
||||
{
|
||||
$files = [
|
||||
['name' => 'first.nfo', 'size' => 200],
|
||||
['name' => 'second.nfo', 'size' => 200],
|
||||
];
|
||||
|
||||
$archive = Mockery::mock(ArchiveExtractionService::class);
|
||||
$archive->shouldReceive('sortFilesWithNfoPriority')->once()->with($files)->andReturn($files);
|
||||
$archive->shouldReceive('isNfoFile')->twice()->andReturn(true);
|
||||
$archive->shouldNotReceive('extractSpecificFile');
|
||||
$archive->shouldReceive('extractSpecificFiles')
|
||||
->once()
|
||||
->with('ARCHIVE', ['first.nfo', 'second.nfo'], Mockery::type('string'))
|
||||
->andReturn([
|
||||
'first.nfo' => '',
|
||||
'second.nfo' => 'NFO DATA',
|
||||
]);
|
||||
|
||||
$download = Mockery::mock(UsenetDownloadService::class);
|
||||
$download->shouldNotReceive('download');
|
||||
$download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class));
|
||||
|
||||
$manager = Mockery::mock(ReleaseFileManager::class);
|
||||
$manager->shouldReceive('processNfoFile')->once()->andReturnUsing(
|
||||
function (string $path, ReleaseProcessingContext $context): bool {
|
||||
$this->assertSame('NFO DATA', file_get_contents($path));
|
||||
$context->releaseHasNoNFO = false;
|
||||
|
||||
return true;
|
||||
}
|
||||
);
|
||||
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
|
||||
|
||||
$fallback = new ReleaseFilesArchiveFallback(
|
||||
$archive,
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$download,
|
||||
$manager,
|
||||
$output,
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->releaseHasNoNFO = true;
|
||||
$this->assertTrue($context->rememberDownloadedArchive('release.part01.rar', 'ARCHIVE', $files));
|
||||
|
||||
$fallback->processNfoFromDownloadedArchives($context);
|
||||
|
||||
$this->assertFalse($context->releaseHasNoNFO);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_bounds_and_releases_retained_archive_payloads(): void
|
||||
{
|
||||
$context = $this->makeContext();
|
||||
$files = [['name' => 'release.nfo', 'size' => 10]];
|
||||
|
||||
$this->assertTrue($context->rememberDownloadedArchive('first.rar', 'FIRST', $files));
|
||||
$this->assertTrue($context->rememberDownloadedArchive('second.rar', 'SECOND', $files));
|
||||
$this->assertFalse($context->rememberDownloadedArchive('third.rar', 'THIRD', $files));
|
||||
$this->assertCount(2, $context->downloadedArchives);
|
||||
|
||||
$context->releaseDownloadedArchives();
|
||||
|
||||
$this->assertSame([], $context->downloadedArchives);
|
||||
}
|
||||
|
||||
private function makeContext(int $releaseId = 12): ReleaseProcessingContext
|
||||
{
|
||||
$context = new ReleaseProcessingContext(new Release([
|
||||
|
||||
@@ -3,15 +3,23 @@
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Models\Release;
|
||||
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
|
||||
use App\Services\AdditionalProcessing\ArchiveExtractionService;
|
||||
use App\Services\AdditionalProcessing\ConsoleOutputService;
|
||||
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
|
||||
use App\Services\AdditionalProcessing\Enums\DownloadKind;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
|
||||
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
|
||||
use App\Services\AdditionalProcessing\MediaExtractionService;
|
||||
use App\Services\AdditionalProcessing\NzbContentParser;
|
||||
use App\Services\AdditionalProcessing\ReleaseFileManager;
|
||||
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
|
||||
use App\Services\AdditionalProcessing\ReleaseProcessor;
|
||||
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
|
||||
use App\Services\AdditionalProcessing\UsenetDownloadService;
|
||||
use App\Services\Releases\ReleaseBrowseService;
|
||||
use App\Services\TempWorkspaceService;
|
||||
use Mockery;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
@@ -44,9 +52,16 @@ class ReleaseProcessorTest extends TestCase
|
||||
->shouldReceive('warning')->once()->with('broken nzb')->andReturnNull()->getMock()
|
||||
);
|
||||
|
||||
$processor->process($this->makeContext(), '/tmp/main/');
|
||||
$result = $processor->process($this->makeContext(), '/tmp/main/');
|
||||
|
||||
$this->assertTrue(true);
|
||||
$this->assertSame(ProcessingOutcome::DeletedBrokenNzb, $result->outcome);
|
||||
$this->assertTrue($result->outcome->isDeleted());
|
||||
$this->assertFalse($result->isSuccessful());
|
||||
$this->assertSame('broken nzb', $result->reason);
|
||||
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
|
||||
$this->assertArrayHasKey(ProcessingStage::WorkspacePreparation->value, $result->stageDurations);
|
||||
$this->assertArrayHasKey(ProcessingStage::NzbParsing->value, $result->stageDurations);
|
||||
$this->assertArrayHasKey(ProcessingStage::WorkspaceCleanup->value, $result->stageDurations);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
@@ -58,19 +73,22 @@ class ReleaseProcessorTest extends TestCase
|
||||
'error' => null,
|
||||
'contents' => [['title' => 'file.nzb', 'segments' => []]],
|
||||
]);
|
||||
$nzbParser->shouldReceive('extractMessageIDs')->once()->andReturn([
|
||||
'hasCompressedFile' => false,
|
||||
'sampleMessageIDs' => [],
|
||||
'jpgMessageIDs' => [],
|
||||
'mediaInfoMessageID' => '',
|
||||
'audioInfoMessageID' => '',
|
||||
'audioInfoExtension' => '',
|
||||
'bookFileCount' => 0,
|
||||
]);
|
||||
|
||||
$releaseManager = Mockery::mock(ReleaseFileManager::class);
|
||||
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
|
||||
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
|
||||
$persistenceMetrics = new PersistenceMetricsCollector;
|
||||
$synchronizedReleaseIds = [];
|
||||
$searchSyncCoordinator = new ReleaseSearchSyncCoordinator(
|
||||
$persistenceMetrics,
|
||||
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
|
||||
$synchronizedReleaseIds[] = $releaseId;
|
||||
},
|
||||
);
|
||||
$releaseManager->shouldReceive('finalizeRelease')
|
||||
->once()
|
||||
->andReturnUsing(static function () use ($searchSyncCoordinator): void {
|
||||
$searchSyncCoordinator->request(1);
|
||||
$searchSyncCoordinator->request(1);
|
||||
});
|
||||
|
||||
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
|
||||
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
|
||||
@@ -83,18 +101,80 @@ class ReleaseProcessorTest extends TestCase
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
Mockery::mock(UsenetDownloadService::class),
|
||||
$this->scopedDownloadService(),
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$tempWorkspace,
|
||||
$output
|
||||
$output,
|
||||
$searchSyncCoordinator,
|
||||
$persistenceMetrics,
|
||||
);
|
||||
|
||||
$processor->process($this->makeContext(), '/tmp/main/');
|
||||
$context = $this->makeContext();
|
||||
$context->release->nfostatus = 1;
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertTrue(true);
|
||||
$this->assertSame(ProcessingOutcome::NoUsefulArtifacts, $result->outcome);
|
||||
$this->assertTrue($result->isSuccessful());
|
||||
$this->assertFalse($result->artifactsCreated);
|
||||
$this->assertSame('Processing completed without creating useful artifacts.', $result->reason);
|
||||
$this->assertSame(0, $result->releaseFilesAdded);
|
||||
$this->assertSame([1], $synchronizedReleaseIds);
|
||||
$this->assertSame(2, $result->persistenceMetrics?->searchSyncRequests);
|
||||
$this->assertSame(1, $result->persistenceMetrics?->searchSyncExecutions);
|
||||
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
|
||||
$this->assertSame([
|
||||
ProcessingStage::WorkspacePreparation->value,
|
||||
ProcessingStage::NzbParsing->value,
|
||||
ProcessingStage::ReleaseInitialization->value,
|
||||
ProcessingStage::MessageIdSelection->value,
|
||||
ProcessingStage::Finalization->value,
|
||||
ProcessingStage::WorkspaceCleanup->value,
|
||||
], array_keys($result->stageDurations));
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_always_uses_the_v2_work_plan_for_message_id_selection(): void
|
||||
{
|
||||
$config = $this->makeConfig();
|
||||
$nzbContents = [['title' => 'file.nzb', 'segments' => ['<message>']]];
|
||||
$nzbParser = Mockery::mock(NzbContentParser::class);
|
||||
$nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([
|
||||
'error' => null,
|
||||
'contents' => $nzbContents,
|
||||
]);
|
||||
$nzbParser->shouldNotReceive('extractMessageIDs');
|
||||
|
||||
$releaseManager = Mockery::mock(ReleaseFileManager::class);
|
||||
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
|
||||
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
|
||||
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$this->scopedDownloadService(),
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$this->successfulTempWorkspace(),
|
||||
Mockery::mock(ConsoleOutputService::class)
|
||||
->shouldReceive('echoReleaseStart')->once()->andReturnNull()
|
||||
->shouldReceive('setProcessTitle')->once()->andReturnNull()
|
||||
->getMock(),
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->release->nfostatus = 1;
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertSame(ProcessingOutcome::NoUsefulArtifacts, $result->outcome);
|
||||
$this->assertNotNull($context->workPlan);
|
||||
$this->assertSame(0, $result->duplicateMessageIdCount);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
@@ -124,9 +204,10 @@ class ReleaseProcessorTest extends TestCase
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
Mockery::mock(UsenetDownloadService::class),
|
||||
$this->scopedDownloadService(),
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$tempWorkspace,
|
||||
@@ -136,9 +217,172 @@ class ReleaseProcessorTest extends TestCase
|
||||
$context = $this->makeContext();
|
||||
$context->startTime = hrtime(true) - 2_000_000_000;
|
||||
|
||||
$processor->process($context, '/tmp/main/');
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertTrue(true);
|
||||
$this->assertSame(ProcessingOutcome::TimedOut, $result->outcome);
|
||||
$this->assertFalse($result->isSuccessful());
|
||||
$this->assertStringContainsString('exceeded', $result->reason);
|
||||
$this->assertArrayHasKey(ProcessingStage::TimeoutHandling->value, $result->stageDurations);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reports_a_deleted_timeout_separately(): void
|
||||
{
|
||||
$config = $this->makeConfig(['releaseProcessingTimeout' => 1, 'maxPpTimeoutCount' => 1]);
|
||||
$nzbParser = Mockery::mock(NzbContentParser::class);
|
||||
$nzbParser->shouldReceive('parseNzb')->once()->andReturn(['error' => null, 'contents' => []]);
|
||||
|
||||
$releaseManager = Mockery::mock(ReleaseFileManager::class);
|
||||
$releaseManager->shouldReceive('handleReleaseTimeout')->once()->andReturn(true);
|
||||
$releaseManager->shouldNotReceive('finalizeRelease');
|
||||
|
||||
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
|
||||
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
|
||||
$tempWorkspace->shouldReceive('clearDirectory')->twice()->with('/tmp/ap-release/', false)->andReturnNull();
|
||||
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
|
||||
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
|
||||
$output->shouldReceive('echoReleaseTimeoutDeleted')->once()->andReturnNull();
|
||||
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$this->scopedDownloadService(),
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$tempWorkspace,
|
||||
$output
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->startTime = hrtime(true) - 2_000_000_000;
|
||||
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertSame(ProcessingOutcome::DeletedAfterTimeout, $result->outcome);
|
||||
$this->assertTrue($result->outcome->isDeleted());
|
||||
$this->assertFalse($result->isSuccessful());
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reports_a_temporary_workspace_failure(): void
|
||||
{
|
||||
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
|
||||
$tempWorkspace->shouldReceive('createReleaseTempFolder')
|
||||
->once()
|
||||
->andThrow(new \RuntimeException('workspace is not writable'));
|
||||
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
|
||||
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
|
||||
$output->shouldReceive('warning')->once()->with('Unable to prepare release temp directory: workspace is not writable')->andReturnNull();
|
||||
|
||||
$processor = $this->makeProcessor(tempWorkspace: $tempWorkspace, output: $output);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->release->nfostatus = 1;
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertSame(ProcessingOutcome::TemporaryWorkspaceUnavailable, $result->outcome);
|
||||
$this->assertFalse($result->isSuccessful());
|
||||
$this->assertSame('workspace is not writable', $result->reason);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reports_password_detection_as_a_successful_outcome(): void
|
||||
{
|
||||
$config = $this->makeConfig(['processPasswords' => true]);
|
||||
$nzbParser = $this->compressedNzbParser();
|
||||
|
||||
$archiveService = Mockery::mock(ArchiveExtractionService::class);
|
||||
$archiveService->shouldReceive('processCompressedData')->once()->andReturn([
|
||||
'success' => false,
|
||||
'files' => [],
|
||||
'hasPassword' => true,
|
||||
'passwordStatus' => ReleaseBrowseService::PASSWD_RAR,
|
||||
]);
|
||||
|
||||
$downloadService = Mockery::mock(UsenetDownloadService::class);
|
||||
$this->expectDownloadScope($downloadService);
|
||||
$downloadService->shouldReceive('download')
|
||||
->once()
|
||||
->with(DownloadKind::Compressed, ['<archive>'], '', 1, 'archive.part01.rar')
|
||||
->andReturn(['success' => true, 'data' => 'ARCHIVE', 'groupUnavailable' => false, 'error' => null]);
|
||||
|
||||
$releaseManager = Mockery::mock(ReleaseFileManager::class);
|
||||
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
|
||||
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
|
||||
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
$archiveService,
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$downloadService,
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$this->successfulTempWorkspace(),
|
||||
$this->passwordOutput()
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->release->nfostatus = 1;
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertSame(ProcessingOutcome::Passworded, $result->outcome);
|
||||
$this->assertTrue($result->isSuccessful());
|
||||
$this->assertFalse($result->artifactsCreated);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reports_group_unavailability_as_an_unsuccessful_outcome(): void
|
||||
{
|
||||
$config = $this->makeConfig(['processPasswords' => true]);
|
||||
$nzbParser = $this->compressedNzbParser();
|
||||
|
||||
$downloadService = Mockery::mock(UsenetDownloadService::class);
|
||||
$this->expectDownloadScope($downloadService);
|
||||
$downloadService->shouldReceive('download')->once()->andReturn([
|
||||
'success' => false,
|
||||
'data' => null,
|
||||
'groupUnavailable' => true,
|
||||
'error' => 'Group unavailable',
|
||||
]);
|
||||
|
||||
$releaseManager = Mockery::mock(ReleaseFileManager::class);
|
||||
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
|
||||
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
|
||||
|
||||
$output = Mockery::mock(ConsoleOutputService::class);
|
||||
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
|
||||
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
|
||||
$output->shouldReceive('echoGroupUnavailable')->once()->andReturnNull();
|
||||
|
||||
$processor = new ReleaseProcessor(
|
||||
$config,
|
||||
$nzbParser,
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
$downloadService,
|
||||
$releaseManager,
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$this->successfulTempWorkspace(),
|
||||
$output
|
||||
);
|
||||
|
||||
$context = $this->makeContext();
|
||||
$context->release->nfostatus = 1;
|
||||
$result = $processor->process($context, '/tmp/main/');
|
||||
|
||||
$this->assertSame(ProcessingOutcome::GroupUnavailable, $result->outcome);
|
||||
$this->assertFalse($result->isSuccessful());
|
||||
$this->assertStringContainsString('unavailable', $result->reason);
|
||||
}
|
||||
|
||||
private function makeProcessor(
|
||||
@@ -147,12 +391,15 @@ class ReleaseProcessorTest extends TestCase
|
||||
?TempWorkspaceService $tempWorkspace = null,
|
||||
?ConsoleOutputService $output = null
|
||||
): ReleaseProcessor {
|
||||
$config = $this->makeConfig();
|
||||
|
||||
return new ReleaseProcessor(
|
||||
$this->makeConfig(),
|
||||
$config,
|
||||
$nzbParser ?? Mockery::mock(NzbContentParser::class),
|
||||
new AdditionalWorkPlanner($config),
|
||||
Mockery::mock(ArchiveExtractionService::class),
|
||||
Mockery::mock(MediaExtractionService::class),
|
||||
Mockery::mock(UsenetDownloadService::class),
|
||||
$this->scopedDownloadService(),
|
||||
$releaseManager ?? Mockery::mock(ReleaseFileManager::class),
|
||||
Mockery::mock(ReleaseFilesArchiveFallback::class),
|
||||
$tempWorkspace ?? Mockery::mock(TempWorkspaceService::class),
|
||||
@@ -171,4 +418,46 @@ class ReleaseProcessorTest extends TestCase
|
||||
'pp_timeout_count' => 0,
|
||||
]));
|
||||
}
|
||||
|
||||
private function compressedNzbParser(): NzbContentParser
|
||||
{
|
||||
$parser = Mockery::mock(NzbContentParser::class);
|
||||
$parser->shouldReceive('parseNzb')->once()->andReturn([
|
||||
'error' => null,
|
||||
'contents' => [['title' => 'archive.part01.rar', 'segments' => ['<archive>']]],
|
||||
]);
|
||||
|
||||
return $parser;
|
||||
}
|
||||
|
||||
private function scopedDownloadService(): UsenetDownloadService
|
||||
{
|
||||
$downloadService = Mockery::mock(UsenetDownloadService::class);
|
||||
$this->expectDownloadScope($downloadService);
|
||||
|
||||
return $downloadService;
|
||||
}
|
||||
|
||||
private function expectDownloadScope(UsenetDownloadService $downloadService): void
|
||||
{
|
||||
$downloadService->shouldReceive('beginReleaseScope')->once()->andReturnNull();
|
||||
$downloadService->shouldReceive('finishReleaseScope')->once()->andReturn(new DownloadMetrics);
|
||||
}
|
||||
|
||||
private function successfulTempWorkspace(): TempWorkspaceService
|
||||
{
|
||||
return Mockery::mock(TempWorkspaceService::class)
|
||||
->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/')
|
||||
->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull()
|
||||
->getMock();
|
||||
}
|
||||
|
||||
private function passwordOutput(): ConsoleOutputService
|
||||
{
|
||||
return Mockery::mock(ConsoleOutputService::class)
|
||||
->shouldReceive('echoReleaseStart')->once()->andReturnNull()
|
||||
->shouldReceive('setProcessTitle')->once()->andReturnNull()
|
||||
->shouldReceive('echoCompressedDownload')->once()->andReturnNull()
|
||||
->getMock();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\AdditionalProcessing;
|
||||
|
||||
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
|
||||
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
|
||||
use PHPUnit\Framework\Attributes\Test;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class ReleaseSearchSyncCoordinatorTest extends TestCase
|
||||
{
|
||||
#[Test]
|
||||
public function it_coalesces_multiple_requests_for_one_active_release(): void
|
||||
{
|
||||
$synchronizedReleaseIds = [];
|
||||
$metrics = new PersistenceMetricsCollector;
|
||||
$coordinator = new ReleaseSearchSyncCoordinator(
|
||||
$metrics,
|
||||
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
|
||||
$synchronizedReleaseIds[] = $releaseId;
|
||||
},
|
||||
);
|
||||
|
||||
$metrics->beginReleaseScope(42);
|
||||
$coordinator->beginReleaseScope(42);
|
||||
$coordinator->request(42);
|
||||
$coordinator->request(42);
|
||||
$coordinator->request(42);
|
||||
|
||||
$this->assertSame([], $synchronizedReleaseIds);
|
||||
|
||||
$coordinator->finishReleaseScope();
|
||||
$persistenceMetrics = $metrics->finishReleaseScope();
|
||||
|
||||
$this->assertSame([42], $synchronizedReleaseIds);
|
||||
$this->assertSame(3, $persistenceMetrics->searchSyncRequests);
|
||||
$this->assertSame(1, $persistenceMetrics->searchSyncExecutions);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_discards_a_pending_sync_when_the_release_is_deleted(): void
|
||||
{
|
||||
$synchronizedReleaseIds = [];
|
||||
$metrics = new PersistenceMetricsCollector;
|
||||
$coordinator = new ReleaseSearchSyncCoordinator(
|
||||
$metrics,
|
||||
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
|
||||
$synchronizedReleaseIds[] = $releaseId;
|
||||
},
|
||||
);
|
||||
|
||||
$metrics->beginReleaseScope(42);
|
||||
$coordinator->beginReleaseScope(42);
|
||||
$coordinator->request(42);
|
||||
$coordinator->discard(42);
|
||||
$coordinator->finishReleaseScope();
|
||||
$persistenceMetrics = $metrics->finishReleaseScope();
|
||||
|
||||
$this->assertSame([], $synchronizedReleaseIds);
|
||||
$this->assertSame(1, $persistenceMetrics->searchSyncRequests);
|
||||
$this->assertSame(0, $persistenceMetrics->searchSyncExecutions);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_synchronizes_immediately_outside_an_active_release_scope(): void
|
||||
{
|
||||
$synchronizedReleaseIds = [];
|
||||
$coordinator = new ReleaseSearchSyncCoordinator(
|
||||
new PersistenceMetricsCollector,
|
||||
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
|
||||
$synchronizedReleaseIds[] = $releaseId;
|
||||
},
|
||||
);
|
||||
|
||||
$coordinator->request(7);
|
||||
|
||||
$this->assertSame([7], $synchronizedReleaseIds);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_executes_every_request_immediately_when_coalescing_is_disabled(): void
|
||||
{
|
||||
$synchronizedReleaseIds = [];
|
||||
$metrics = new PersistenceMetricsCollector;
|
||||
$coordinator = new ReleaseSearchSyncCoordinator(
|
||||
$metrics,
|
||||
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
|
||||
$synchronizedReleaseIds[] = $releaseId;
|
||||
},
|
||||
coalesce: false,
|
||||
);
|
||||
|
||||
$metrics->beginReleaseScope(42);
|
||||
$coordinator->beginReleaseScope(42);
|
||||
$coordinator->request(42);
|
||||
$coordinator->request(42);
|
||||
$coordinator->finishReleaseScope();
|
||||
$persistenceMetrics = $metrics->finishReleaseScope();
|
||||
|
||||
$this->assertSame([42, 42], $synchronizedReleaseIds);
|
||||
$this->assertSame(2, $persistenceMetrics->searchSyncRequests);
|
||||
$this->assertSame(2, $persistenceMetrics->searchSyncExecutions);
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,99 @@ class UsenetDownloadServiceTest extends TestCase
|
||||
$this->assertStringContainsString('Group unavailable', (string) $result['error']);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_reuses_exact_successful_requests_only_within_the_current_release(): void
|
||||
{
|
||||
$nntp = Mockery::mock(NNTPService::class);
|
||||
$nntp->shouldReceive('getMessagesByMessageID')
|
||||
->twice()
|
||||
->with(['<shared>'], false)
|
||||
->andReturn('SHARED-DATA');
|
||||
|
||||
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
|
||||
$service->beginReleaseScope();
|
||||
|
||||
$first = $service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 10);
|
||||
$second = $service->download(DownloadKind::Jpg, ['<shared>'], 'alt.binaries.test', 10);
|
||||
$firstReleaseMetrics = $service->finishReleaseScope();
|
||||
|
||||
$this->assertSame($first, $second);
|
||||
$this->assertSame(2, $firstReleaseMetrics->logicalRequests);
|
||||
$this->assertSame(1, $firstReleaseMetrics->networkRequests);
|
||||
$this->assertSame(1, $firstReleaseMetrics->cacheHits);
|
||||
$this->assertSame(strlen('SHARED-DATA'), $firstReleaseMetrics->bytesDownloaded);
|
||||
$this->assertSame(strlen('SHARED-DATA'), $firstReleaseMetrics->bytesReused);
|
||||
|
||||
$service->beginReleaseScope();
|
||||
$service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 11);
|
||||
$secondReleaseMetrics = $service->finishReleaseScope();
|
||||
|
||||
$this->assertSame(1, $secondReleaseMetrics->logicalRequests);
|
||||
$this->assertSame(1, $secondReleaseMetrics->networkRequests);
|
||||
$this->assertSame(0, $secondReleaseMetrics->cacheHits);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_does_not_cache_failed_downloads(): void
|
||||
{
|
||||
$nntp = Mockery::mock(NNTPService::class);
|
||||
$nntp->shouldReceive('getMessagesByMessageID')
|
||||
->twice()
|
||||
->with(['<missing>'], false)
|
||||
->andReturn('');
|
||||
|
||||
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
|
||||
$service->beginReleaseScope();
|
||||
|
||||
$service->download(DownloadKind::Compressed, ['<missing>']);
|
||||
$service->download(DownloadKind::Compressed, ['<missing>']);
|
||||
$metrics = $service->finishReleaseScope();
|
||||
|
||||
$this->assertSame(2, $metrics->networkRequests);
|
||||
$this->assertSame(0, $metrics->cacheHits);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_does_not_reuse_successful_downloads_outside_a_release_scope(): void
|
||||
{
|
||||
$nntp = Mockery::mock(NNTPService::class);
|
||||
$nntp->shouldReceive('getMessagesByMessageID')
|
||||
->twice()
|
||||
->with(['<shared>'], false)
|
||||
->andReturn('SHARED-DATA');
|
||||
|
||||
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
|
||||
|
||||
$first = $service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 10);
|
||||
$second = $service->download(DownloadKind::Jpg, ['<shared>'], 'alt.binaries.test', 10);
|
||||
|
||||
$this->assertSame('SHARED-DATA', $first['data']);
|
||||
$this->assertSame('SHARED-DATA', $second['data']);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_retries_an_exact_request_after_a_partial_provider_failure(): void
|
||||
{
|
||||
$nntp = Mockery::mock(NNTPService::class);
|
||||
$nntp->shouldReceive('getMessagesByMessageID')
|
||||
->twice()
|
||||
->with(['<retry>'], false)
|
||||
->andReturn('', 'RECOVERED-DATA');
|
||||
|
||||
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
|
||||
$service->beginReleaseScope();
|
||||
|
||||
$failed = $service->download(DownloadKind::Compressed, ['<retry>']);
|
||||
$recovered = $service->download(DownloadKind::Compressed, ['<retry>']);
|
||||
$metrics = $service->finishReleaseScope();
|
||||
|
||||
$this->assertFalse($failed['success']);
|
||||
$this->assertTrue($recovered['success']);
|
||||
$this->assertSame('RECOVERED-DATA', $recovered['data']);
|
||||
$this->assertSame(2, $metrics->networkRequests);
|
||||
$this->assertSame(0, $metrics->cacheHits);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function it_checks_minimum_download_sizes(): void
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user