Refactor additional postprocessing

This commit is contained in:
DariusIII
2026-08-11 10:43:22 +02:00
parent 140b7c8c04
commit 014077b514
45 changed files with 3875 additions and 338 deletions
-1
View File
@@ -79,7 +79,6 @@ NNTP_SOCKET_TIMEOUT_A=120
NN_MULTIPROCESSING_MAX_CHILD_TIME=1800
NN_CONCURRENCY_TIMEOUT=
ADMIN_USER=
ADMIN_PASS=
ADMIN_EMAIL=
@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Services\AdditionalProcessing\AdditionalProcessingDiagnostics;
use Illuminate\Console\Command;
class AdditionalProcessingDiagnose extends Command
{
protected $signature = 'nntmux:additional-diagnose
{--json : Output machine-readable JSON}';
protected $description = 'Inspect additional post-processing backlog, claims, capacity, indexes, and temp-path health';
public function __construct(private readonly AdditionalProcessingDiagnostics $diagnostics)
{
parent::__construct();
}
public function handle(): int
{
$report = $this->diagnostics->inspect();
if ($this->option('json')) {
$this->line((string) json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR));
return self::SUCCESS;
}
$this->table(['Backlog', 'Count'], [
['Total', (string) $report['backlog']['total']],
['Available', (string) $report['backlog']['available']],
['Active claims', (string) $report['backlog']['active_claims']],
['Stale claims', (string) $report['backlog']['stale_claims']],
]);
$this->table(['Capacity', 'Value'], [
['Pipeline', (string) $report['settings']['pipeline']],
['Threads', (string) $report['capacity']['threads']],
['Releases per batch', (string) $report['capacity']['releases_per_batch']],
['Maximum in flight', (string) $report['capacity']['max_in_flight']],
['Claim TTL', $report['claims']['ttl_seconds'].' seconds'],
['Child timeout', $report['settings']['multiprocessing_child_timeout'].' seconds'],
]);
$this->table(['Preflight', 'Value'], [
['Database driver', (string) $report['database_driver']],
['Covering claim index', (string) ($report['indexes']['covering_index'] ?? 'missing')],
['Temp path', (string) $report['temp_path']['path']],
['Temp path writable', $report['temp_path']['writable'] ? 'yes' : 'no'],
]);
if ($report['warnings'] === []) {
$this->info('No additional-processing diagnostic warnings were found.');
return self::SUCCESS;
}
$this->newLine();
foreach ($report['warnings'] as $warning) {
$this->warn('['.$warning['code'].'] '.$warning['message']);
}
return self::SUCCESS;
}
}
+42 -4
View File
@@ -6,6 +6,7 @@ namespace App\Console\Commands;
use App\Models\Settings;
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
use App\Services\NfoService;
use App\Services\NNTP\NNTPService;
use App\Services\PostProcessService;
@@ -24,7 +25,8 @@ class PostProcessGuid extends Command
{guid : First character of release guid (a-f, 0-9)}
{renamed? : For movie/tv: process renamed only (optional)}
{--worker : Drain multiple additional-processing batches}
{--max-batches=4 : Maximum claim batches for worker mode}';
{--max-batches=4 : Maximum claim batches for worker mode}
{--profile : Emit one machine-readable performance record per additional batch}';
/**
* The console command description.
@@ -93,13 +95,16 @@ class PostProcessGuid extends Command
try {
for ($batch = 0; $batch < $maxBatches && microtime(true) < $deadline; $batch++) {
$stats = $this->additionalProcessor->start('', $guid, $workerToken, $excludedReleaseIds);
if ($stats['claimed'] === 0) {
$result = $this->additionalProcessor->start('', $guid, $workerToken, $excludedReleaseIds);
if ((bool) $this->option('profile')) {
$this->writeAdditionalProfile($guid, $batch + 1, $result);
}
if ($result->claimedCount() === 0) {
break;
}
$excludedReleaseIds = array_values(array_unique([
...$excludedReleaseIds,
...$stats['claimed_ids'],
...$result->claimedIds,
]));
}
} finally {
@@ -107,6 +112,39 @@ class PostProcessGuid extends Command
}
}
private function writeAdditionalProfile(string $guid, int $batch, AdditionalBatchResult $result): void
{
$downloadMetrics = $result->downloadMetrics();
$persistenceMetrics = $result->persistenceMetrics();
$this->line(json_encode([
'event' => 'additional-postprocessing-profile',
'pipeline' => 'v2',
'guid_char' => $guid,
'batch' => $batch,
'claimed' => $result->claimedCount(),
'attempted' => $result->attemptedCount(),
'successful' => $result->successfulCount(),
'failed' => $result->unsuccessfulCount(),
'outcomes' => $result->outcomeCounts(),
'artifacts_created' => $result->artifactsCreatedCount(),
'artifact_yield_percent' => round($result->artifactYieldPercent(), 2),
'release_files_added' => $result->releaseFilesAdded(),
'nntp_requests' => $downloadMetrics->networkRequests,
'nntp_bytes' => $downloadMetrics->bytesDownloaded,
'database_statements' => $persistenceMetrics->databaseStatements,
'database_milliseconds' => round($persistenceMetrics->databaseMilliseconds, 3),
'search_sync_requests' => $persistenceMetrics->searchSyncRequests,
'search_sync_executions' => $persistenceMetrics->searchSyncExecutions,
'duplicate_message_ids' => $result->duplicateMessageIdCount(),
'elapsed_seconds' => round($result->elapsedSeconds, 6),
'releases_per_hour' => round($result->releasesPerSecond() * 3600, 2),
'average_release_seconds' => round($result->averageReleaseSeconds(), 6),
'stage_seconds' => $result->stageDurationTotals(),
'peak_memory_bytes' => $result->peakMemoryBytes,
], JSON_THROW_ON_ERROR));
}
/**
* Process NFO files for releases.
*/
+78 -6
View File
@@ -7,6 +7,7 @@ namespace App\Console\Commands;
use App\Facades\Search;
use App\Models\Release;
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use Illuminate\Console\Command;
class ProcessAdditionalGuid extends Command
@@ -15,7 +16,8 @@ class ProcessAdditionalGuid extends Command
{guid? : Release GUID (optional if using --id)}
{--id= : Numeric release ID to run additional postprocessing on (alternative to GUID)}
{--reset : Reset key additional postprocessing flags before running}
{--no-progress : Suppress progress / status output (forces quiet mode)}';
{--no-progress : Suppress progress / status output (forces quiet mode)}
{--profile : Display stage, network, and persistence metrics}';
protected $description = 'Run additional postprocessing for a specific release by GUID or ID regardless of prior postprocessing status';
@@ -87,19 +89,89 @@ class ProcessAdditionalGuid extends Command
$processor = app(AdditionalProcessingOrchestrator::class);
try {
$ok = $processor->processSingleGuid($guid);
$result = $processor->processSingleGuid($guid);
} finally {
$processor->finish();
}
if (! $ok) {
$this->error('Processing failed or nothing processed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.');
$identifier = $idOpt ? 'ID '.$idOpt : 'GUID '.$guid;
$this->displayResult($result, $identifier);
if ($this->option('profile')) {
$this->displayProfile($result);
}
if (! $result->isSuccessful()) {
return 3;
}
$this->info('Additional postprocessing completed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.');
return 0;
}
private function displayResult(ReleaseProcessingResult $result, string $identifier): void
{
$rows = [
['Release', $identifier],
['Pipeline', 'v2'],
['Outcome', $result->outcome->value],
['Artifacts created', $result->artifactsCreated ? 'yes' : 'no'],
['Release files added', (string) $result->releaseFilesAdded],
['Duration', number_format($result->elapsedSeconds, 3).' seconds'],
];
if ($result->reason !== '') {
$rows[] = ['Reason', $result->reason];
}
$this->table(['Result', 'Value'], $rows);
$message = 'Additional postprocessing ended with outcome '.$result->outcome->value.' for '.$identifier
.' after '.number_format($result->elapsedSeconds, 3).' seconds.';
if ($result->isSuccessful()) {
$this->info($message);
return;
}
$this->error($message);
}
private function displayProfile(ReleaseProcessingResult $result): void
{
$stageRows = [];
foreach ($result->stageDurations as $stage => $duration) {
$stageRows[] = [$stage, number_format($duration, 3)];
}
if ($stageRows !== []) {
$this->newLine();
$this->line('Stage profile');
$this->table(['Stage', 'Seconds'], $stageRows);
}
if ($result->downloadMetrics !== null) {
$this->newLine();
$this->line('Download profile');
$this->table(['Metric', 'Value'], [
['Logical requests', (string) $result->downloadMetrics->logicalRequests],
['Network requests', (string) $result->downloadMetrics->networkRequests],
['Cache hits', (string) $result->downloadMetrics->cacheHits],
['Bytes downloaded', (string) $result->downloadMetrics->bytesDownloaded],
['Bytes reused', (string) $result->downloadMetrics->bytesReused],
]);
}
if ($result->persistenceMetrics !== null) {
$this->newLine();
$this->line('Persistence profile');
$this->table(['Metric', 'Value'], [
['Database statements', (string) $result->persistenceMetrics->databaseStatements],
['Database milliseconds', number_format($result->persistenceMetrics->databaseMilliseconds, 3)],
['Search sync requests', (string) $result->persistenceMetrics->searchSyncRequests],
['Search sync executions', (string) $result->persistenceMetrics->searchSyncExecutions],
]);
}
}
}
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace App\Providers;
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\ConsoleOutputService;
@@ -13,15 +14,20 @@ use App\Services\AdditionalProcessing\NzbContentParser;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
use App\Services\AdditionalProcessing\ReleaseProcessor;
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\Categorization\CategorizationService;
use App\Services\NameFixing\NameFixingService;
use App\Services\NameFixing\ReleaseUpdateService;
use App\Services\NfoService;
use App\Services\Nzb\NzbParserService;
use App\Services\Nzb\NzbService;
use App\Services\ReleaseExtraService;
use App\Services\ReleaseImageService;
use App\Services\TempWorkspaceService;
use Illuminate\Database\Events\QueryExecuted;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\ServiceProvider;
/**
@@ -40,6 +46,18 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
return new ProcessingConfiguration;
});
$this->app->singleton(PersistenceMetricsCollector::class);
$this->app->singleton(ReleaseSearchSyncCoordinator::class, function ($app) {
return new ReleaseSearchSyncCoordinator(
$app->make(PersistenceMetricsCollector::class),
);
});
$this->app->bind(ReleaseUpdateService::class, function ($app) {
return new ReleaseUpdateService(
searchSyncCoordinator: $app->make(ReleaseSearchSyncCoordinator::class),
);
});
// Release extra service for video/audio/subtitle data
$this->app->singleton(ReleaseExtraService::class, function () {
return new ReleaseExtraService;
@@ -64,6 +82,10 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
);
});
$this->app->singleton(AdditionalWorkPlanner::class, function ($app) {
return new AdditionalWorkPlanner($app->make(ProcessingConfiguration::class));
});
// Archive extraction service
$this->app->singleton(ArchiveExtractionService::class, function ($app) {
return new ArchiveExtractionService(
@@ -85,7 +107,9 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
$app->make(ReleaseImageService::class),
new NfoService,
$app->make(NzbService::class),
new NameFixingService
new NameFixingService($app->make(ReleaseUpdateService::class)),
$app->make(ReleaseUpdateService::class),
searchSyncCoordinator: $app->make(ReleaseSearchSyncCoordinator::class),
);
});
@@ -95,18 +119,18 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
$app->make(ProcessingConfiguration::class),
$app->make(ReleaseImageService::class),
$app->make(ReleaseExtraService::class),
new CategorizationService
new CategorizationService,
$app->make(ReleaseSearchSyncCoordinator::class),
);
});
$this->app->singleton(ReleaseFilesArchiveFallback::class, function ($app) {
return new ReleaseFilesArchiveFallback(
$app->make(ProcessingConfiguration::class),
$app->make(ArchiveExtractionService::class),
$app->make(MediaExtractionService::class),
$app->make(UsenetDownloadService::class),
$app->make(ReleaseFileManager::class),
$app->make(ConsoleOutputService::class)
$app->make(ConsoleOutputService::class),
);
});
@@ -114,13 +138,16 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
return new ReleaseProcessor(
$app->make(ProcessingConfiguration::class),
$app->make(NzbContentParser::class),
$app->make(AdditionalWorkPlanner::class),
$app->make(ArchiveExtractionService::class),
$app->make(MediaExtractionService::class),
$app->make(UsenetDownloadService::class),
$app->make(ReleaseFileManager::class),
$app->make(ReleaseFilesArchiveFallback::class),
$app->make(TempWorkspaceService::class),
$app->make(ConsoleOutputService::class)
$app->make(ConsoleOutputService::class),
$app->make(ReleaseSearchSyncCoordinator::class),
$app->make(PersistenceMetricsCollector::class),
);
});
@@ -143,8 +170,12 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
/**
* Bootstrap services.
*/
public function boot(): void
public function boot(PersistenceMetricsCollector $persistenceMetrics): void
{
DB::listen(static function (QueryExecuted $query) use ($persistenceMetrics): void {
$persistenceMetrics->recordDatabaseStatement($query->time);
});
$this->app->terminating(function (): void {
if ($this->app->bound(AdditionalProcessingOrchestrator::class)) {
$this->app->make(AdditionalProcessingOrchestrator::class)->finish();
@@ -390,14 +390,14 @@ final class AdditionalCandidateQuery
});
}
private static function claimTtlSeconds(): int
public static function claimTtlSeconds(): int
{
$timeout = (int) (Settings::settingValue('releaseprocessingtimeout') ?: 120);
return max(300, $timeout * 2);
}
private static function claimStaleBefore(): Carbon
public static function claimStaleBefore(): Carbon
{
return now()->subSeconds(self::claimTtlSeconds());
}
@@ -0,0 +1,250 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Models\Settings;
use App\Services\Runners\PostProcessRunner;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
final class AdditionalProcessingDiagnostics
{
private const int OVERSIZED_IN_FLIGHT_THRESHOLD = 500;
private const int OVERSIZED_THREAD_THRESHOLD = 32;
/**
* @var list<string>
*/
private const array REQUIRED_CLAIM_INDEX_COLUMNS = [
'passwordstatus',
'haspreview',
'nzbstatus',
'leftguid',
'additional_pp_claimed_at',
'postdate',
];
/**
* @return array<string, mixed>
*/
public function inspect(): array
{
$warnings = [];
$releaseTimeout = $this->setting('releaseprocessingtimeout', 120);
$threads = max(1, $this->setting('postthreads', 1));
$batchSize = max(1, $this->setting('maxaddprocessed', 25));
$childTimeout = (int) (config('nntmux.concurrency_timeout')
?? config('nntmux.multiprocessing_max_child_time', 1800));
$claimTtl = AdditionalCandidateQuery::claimTtlSeconds();
$backlog = $this->backlog($warnings);
$indexes = $this->indexes($warnings);
$tempPath = $this->tempPath($warnings);
$maximumClaimLifetime = min(
max(1, $childTimeout),
max(1, $releaseTimeout) * $batchSize,
);
if ($backlog['stale_claims'] > 0) {
$warnings[] = $this->warning(
'stale-claims',
$backlog['stale_claims'].' additional-processing claim(s) are stale and eligible for recovery.',
);
}
if ($claimTtl <= $maximumClaimLifetime) {
$warnings[] = $this->warning(
'claim-ttl',
'Claim TTL ('.$claimTtl.'s) does not exceed the estimated claimed-batch lifetime ('
.$maximumClaimLifetime.'s); another worker may reclaim active work.',
);
}
$maxInFlight = $threads * $batchSize;
if ($threads > self::OVERSIZED_THREAD_THRESHOLD || $maxInFlight > self::OVERSIZED_IN_FLIGHT_THRESHOLD) {
$warnings[] = $this->warning(
'oversized-capacity',
'postthreads='.$threads.' and maxaddprocessed='.$batchSize.' allow up to '.$maxInFlight
.' claimed releases at once; reduce threads or batch size and benchmark end-to-end throughput.',
);
}
return [
'generated_at' => now()->toIso8601String(),
'database_driver' => DB::getDriverName(),
'backlog' => $backlog,
'settings' => [
'pipeline' => 'v2',
'postthreads' => $threads,
'maxaddprocessed' => $batchSize,
'releaseprocessingtimeout' => $releaseTimeout,
'maxpptimeoutcount' => $this->setting('maxpptimeoutcount', 3),
'worker_max_batches' => PostProcessRunner::ADDITIONAL_WORKER_MAX_BATCHES,
'multiprocessing_child_timeout' => $childTimeout,
],
'claims' => [
'ttl_seconds' => $claimTtl,
'stale_before' => AdditionalCandidateQuery::claimStaleBefore()->toIso8601String(),
'estimated_claimed_batch_lifetime_seconds' => $maximumClaimLifetime,
],
'capacity' => [
'threads' => $threads,
'releases_per_batch' => $batchSize,
'max_in_flight' => $maxInFlight,
'max_per_worker_lifetime' => $batchSize * PostProcessRunner::ADDITIONAL_WORKER_MAX_BATCHES,
],
'indexes' => $indexes,
'temp_path' => $tempPath,
'warnings' => $warnings,
];
}
/**
* @param list<array{code: string, message: string}> $warnings
* @return array{total: int, available: int, active_claims: int, stale_claims: int, buckets: list<array{bucket: string, total: int, available: int}>}
*/
private function backlog(array &$warnings): array
{
$empty = [
'total' => 0,
'available' => 0,
'active_claims' => 0,
'stale_claims' => 0,
'buckets' => [],
];
if (! Schema::hasTable('releases') || ! Schema::hasTable('categories')) {
$warnings[] = $this->warning(
'missing-processing-tables',
'The releases or categories table is missing; backlog health cannot be inspected.',
);
return $empty;
}
$counts = AdditionalCandidateQuery::backlogCounts();
$activeClaims = 0;
$staleClaims = 0;
if (AdditionalCandidateQuery::supportsClaims()) {
$claimQuery = AdditionalCandidateQuery::baseBuilder(includeClaimed: true)
->whereNotNull('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN);
$activeClaims = (clone $claimQuery)
->where('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN, '>=', AdditionalCandidateQuery::claimStaleBefore())
->count('r.id');
$staleClaims = (clone $claimQuery)
->where('r.'.AdditionalCandidateQuery::CLAIMED_AT_COLUMN, '<', AdditionalCandidateQuery::claimStaleBefore())
->count('r.id');
}
return [
...$counts,
'active_claims' => $activeClaims,
'stale_claims' => $staleClaims,
'buckets' => AdditionalCandidateQuery::bucketBacklog(),
];
}
/**
* @param list<array{code: string, message: string}> $warnings
* @return array{required_columns: list<string>, present: list<string>, covering_index: string|null, missing: bool}
*/
private function indexes(array &$warnings): array
{
$present = [];
$coveringIndex = null;
if (Schema::hasTable('releases')) {
foreach (Schema::getIndexes('releases') as $index) {
$name = (string) ($index['name'] ?? '');
$columns = array_map(
static fn (mixed $column): string => strtolower((string) $column),
is_array($index['columns'] ?? null) ? $index['columns'] : [],
);
if ($name !== '') {
$present[] = $name;
}
if ($columns === self::REQUIRED_CLAIM_INDEX_COLUMNS) {
$coveringIndex = $name;
}
}
}
sort($present);
$missing = $coveringIndex === null;
if ($missing) {
$warnings[] = $this->warning(
'missing-index',
'The releases claim-queue index is missing or has the wrong column order; run pending migrations before processing.',
);
}
return [
'required_columns' => self::REQUIRED_CLAIM_INDEX_COLUMNS,
'present' => $present,
'covering_index' => $coveringIndex,
'missing' => $missing,
];
}
/**
* @param list<array{code: string, message: string}> $warnings
* @return array{path: string, exists: bool, writable: bool}
*/
private function tempPath(array &$warnings): array
{
$path = rtrim((string) config('nntmux.tmp_unrar_path'), '/\\');
$exists = $path !== '' && is_dir($path);
$writable = $exists ? is_writable($path) : $this->nearestExistingDirectoryIsWritable($path);
if (! $writable) {
$warnings[] = $this->warning(
'unwritable-temp-path',
'Additional-processing temp path '.$path.' is not writable and cannot be created by the current user.',
);
}
return [
'path' => $path,
'exists' => $exists,
'writable' => $writable,
];
}
private function nearestExistingDirectoryIsWritable(string $path): bool
{
if ($path === '') {
return false;
}
$candidate = $path;
while (! is_dir($candidate)) {
$parent = dirname($candidate);
if ($parent === $candidate) {
return false;
}
$candidate = $parent;
}
return is_writable($candidate);
}
private function setting(string $name, int $default): int
{
$value = Settings::settingValue($name);
return $value === null || $value === '' ? $default : (int) $value;
}
/**
* @return array{code: string, message: string}
*/
private function warning(string $code, string $message): array
{
return ['code' => $code, 'message' => $message];
}
}
@@ -6,6 +6,9 @@ namespace App\Services\AdditionalProcessing;
use App\Models\Release;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\TempWorkspaceService;
use Exception;
@@ -31,6 +34,8 @@ class AdditionalProcessingOrchestrator
private string $claimToken = '';
private string $lastSetupError = '';
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly ReleaseProcessor $processor,
@@ -42,7 +47,6 @@ class AdditionalProcessingOrchestrator
* Start the additional processing.
*
* @param list<int> $excludedReleaseIds
* @return array{claimed: int, processed: int, failed: int, claimed_ids: list<int>}
*
* @throws Exception
*/
@@ -51,10 +55,10 @@ class AdditionalProcessingOrchestrator
string $guidChar = '',
string $workerToken = '',
array $excludedReleaseIds = []
): array {
): AdditionalBatchResult {
$this->finish();
if (! $this->setupTempPath($guidChar, $groupID, $workerToken)) {
return ['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []];
return AdditionalBatchResult::setupFailed($this->lastSetupError);
}
$this->fetchReleases($groupID, $guidChar, $excludedReleaseIds);
@@ -65,40 +69,59 @@ class AdditionalProcessingOrchestrator
return $this->processReleases($guidChar);
}
return ['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []];
return AdditionalBatchResult::empty();
}
/**
* Process a single release by GUID.
*/
public function processSingleGuid(string $guid): bool
public function processSingleGuid(string $guid): ReleaseProcessingResult
{
$releaseId = 0;
try {
$this->finish();
$release = Release::where('guid', $guid)->first();
if ($release === null) {
$this->output->warning('Release not found for GUID: '.$guid);
return false;
return new ReleaseProcessingResult(0, $guid, ProcessingOutcome::NotFound, reason: 'Release not found.');
}
$releaseId = (int) $release->id;
$this->releases = collect([$release]);
$this->totalReleases = 1;
$guidChar = $release->leftguid ?? substr($release->guid, 0, 1);
$groupID = '';
if (! $this->setupTempPath($guidChar, $groupID)) {
return false;
return new ReleaseProcessingResult(
$releaseId,
$guid,
ProcessingOutcome::TemporaryWorkspaceUnavailable,
reason: $this->lastSetupError,
);
}
$this->processReleases($guidChar);
$result = $this->processReleases($guidChar)->firstResult();
return true;
return $result ?? new ReleaseProcessingResult(
$releaseId,
$guid,
ProcessingOutcome::Failed,
reason: 'No processing result was produced.',
);
} catch (\Throwable $e) {
if ($this->config->debugMode) {
Log::error('processSingleGuid failed: '.$e->getMessage());
}
return false;
return new ReleaseProcessingResult(
$releaseId,
$guid,
ProcessingOutcome::Failed,
reason: $e->getMessage(),
);
}
}
@@ -107,6 +130,8 @@ class AdditionalProcessingOrchestrator
*/
private function setupTempPath(string $guidChar, string $groupID, string $workerToken = ''): bool
{
$this->lastSetupError = '';
try {
$this->mainTmpPath = $this->tempWorkspace->ensureMainTempPath(
$this->config->tmpUnrarPath,
@@ -116,6 +141,7 @@ class AdditionalProcessingOrchestrator
);
$this->tempWorkspace->clearDirectory($this->mainTmpPath, true);
} catch (\Throwable $e) {
$this->lastSetupError = $e->getMessage();
$this->output->warning('Additional post-processing skipped: '.$e->getMessage());
Log::error('Additional post-processing temp path is unavailable', [
'tmp_unrar_path' => $this->config->tmpUnrarPath,
@@ -183,27 +209,31 @@ class AdditionalProcessingOrchestrator
* every subsequent cycle, and the "needs additional pp" backlog would
* grow indefinitely without any visible failure.
*
* @return array{claimed: int, processed: int, failed: int, claimed_ids: list<int>}
*
* @throws Exception
*/
private function processReleases(string $guidChar = ''): array
private function processReleases(string $guidChar = ''): AdditionalBatchResult
{
$startedAt = microtime(true);
$startedAt = hrtime(true);
$claimedIds = $this->releases
->pluck('id')
->map(static fn (mixed $id): int => (int) $id)
->values()
->all();
$processed = 0;
$failed = 0;
$results = [];
foreach ($this->releases as $release) {
$releaseStartedAt = hrtime(true);
try {
$this->processor->process(new ReleaseProcessingContext($release), $this->mainTmpPath);
$processed++;
$results[] = $this->processor->process(new ReleaseProcessingContext($release), $this->mainTmpPath);
} catch (\Throwable $e) {
$failed++;
$results[] = new ReleaseProcessingResult(
releaseId: (int) ($release->id ?? 0),
guid: (string) ($release->guid ?? ''),
outcome: ProcessingOutcome::Failed,
reason: $e->getMessage(),
elapsedSeconds: $this->elapsedSecondsSince($releaseStartedAt),
);
Log::error('Additional postprocessing failed for release '.($release->id ?? '?').': '.$e->getMessage(), [
'release_id' => $release->id ?? null,
'guid' => $release->guid ?? null,
@@ -220,23 +250,63 @@ class AdditionalProcessingOrchestrator
}
}
$batchResult = new AdditionalBatchResult(
claimedIds: $claimedIds,
results: $results,
elapsedSeconds: $this->elapsedSecondsSince($startedAt),
peakMemoryBytes: memory_get_peak_usage(true),
);
$downloadMetrics = $batchResult->downloadMetrics();
$persistenceMetrics = $batchResult->persistenceMetrics();
Log::info('Additional postprocessing run finished', [
'pipeline' => 'v2',
'guid_char' => $guidChar,
'picked' => $this->totalReleases,
'processed' => $processed,
'failed' => $failed,
'elapsed_seconds' => round(microtime(true) - $startedAt, 3),
'peak_memory_bytes' => memory_get_peak_usage(true),
'processed' => $batchResult->successfulCount(),
'failed' => $batchResult->unsuccessfulCount(),
'outcomes' => $batchResult->outcomeCounts(),
'artifacts_created' => $batchResult->artifactsCreatedCount(),
'artifact_yield_percent' => round($batchResult->artifactYieldPercent(), 2),
'release_files_added' => $batchResult->releaseFilesAdded(),
'download_requests' => $downloadMetrics->logicalRequests,
'nntp_requests' => $downloadMetrics->networkRequests,
'download_cache_hits' => $downloadMetrics->cacheHits,
'nntp_bytes' => $downloadMetrics->bytesDownloaded,
'reused_bytes' => $downloadMetrics->bytesReused,
'database_statements' => $persistenceMetrics->databaseStatements,
'database_milliseconds' => round($persistenceMetrics->databaseMilliseconds, 3),
'search_sync_requests' => $persistenceMetrics->searchSyncRequests,
'search_sync_executions' => $persistenceMetrics->searchSyncExecutions,
'duplicate_message_ids' => $batchResult->duplicateMessageIdCount(),
'unsupported_reasons' => $batchResult->unsupportedReasonCounts(),
'elapsed_seconds' => round($batchResult->elapsedSeconds, 6),
'releases_per_second' => round($batchResult->releasesPerSecond(), 4),
'average_release_seconds' => round($batchResult->averageReleaseSeconds(), 6),
'stage_seconds' => $this->roundedStageDurations($batchResult->stageDurationTotals()),
'peak_memory_bytes' => $batchResult->peakMemoryBytes,
]);
$this->output->endOutput();
return [
'claimed' => $this->totalReleases,
'processed' => $processed,
'failed' => $failed,
'claimed_ids' => $claimedIds,
];
return $batchResult;
}
private function elapsedSecondsSince(int $startedAtNanoseconds): float
{
return (hrtime(true) - $startedAtNanoseconds) / 1_000_000_000;
}
/**
* @param array<string, float> $durations
* @return array<string, float>
*/
private function roundedStageDurations(array $durations): array
{
return array_map(
static fn (float $duration): float => round($duration, 6),
$durations,
);
}
public function finish(): void
@@ -249,5 +319,6 @@ class AdditionalProcessingOrchestrator
$this->releases = collect();
$this->totalReleases = 0;
$this->claimToken = '';
$this->lastSetupError = '';
}
}
@@ -0,0 +1,205 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\AdditionalWorkPlan;
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
use Illuminate\Support\Facades\Log;
final readonly class AdditionalWorkPlanner
{
private const string ARCHIVE_PATTERN = '/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)("|\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i';
public function __construct(private ProcessingConfiguration $config) {}
/**
* @param array<int|string, mixed> $nzbContents
*/
public function plan(array $nzbContents, string $groupName): AdditionalWorkPlan
{
$sampleMessageIds = [];
$jpgMessageIds = [];
$mediaInfoMessageId = '';
$audioInfoMessageId = '';
$audioInfoExtension = '';
$archiveCandidates = [];
$bookFileCount = 0;
$duplicateMessageIdCount = 0;
$seenMessageIds = [];
foreach (array_values($nzbContents) as $sourceIndex => $file) {
if (! is_array($file)) {
continue;
}
try {
$title = (string) ($file['title'] ?? '');
$segments = is_array($file['segments'] ?? null) ? $file['segments'] : [];
if (preg_match($this->config->ignoreBookRegex, $title) === 1) {
$bookFileCount++;
}
if (preg_match(self::ARCHIVE_PATTERN, $title) === 1) {
$archiveMessageIds = $this->extractSegments(
$segments,
$this->config->maximumRarSegments,
$seenMessageIds,
$duplicateMessageIdCount,
);
if ($archiveMessageIds !== []) {
$archiveCandidates[] = new ArchiveCandidate(
title: $title,
messageIds: $archiveMessageIds,
likelyFirstVolume: $this->isLikelyFirstVolume($title),
sourceIndex: $sourceIndex,
);
}
}
if ($this->isSupportFile($title)) {
continue;
}
if ($this->config->processThumbnails && $sampleMessageIds === [] && $segments !== []
&& stripos($title, 'sample') !== false
&& preg_match('/\.(?:jpe?g|png|webp)$/i', $title) !== 1
) {
$sampleMessageIds = $this->extractSegments(
$segments,
$this->config->segmentsToDownload,
$seenMessageIds,
$duplicateMessageIdCount,
);
}
if ($this->config->processJPGSample && $jpgMessageIds === [] && $segments !== []
&& preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName) !== 1
&& preg_match('/\.(?:jpe?g|png|webp)[. ")\]]/i', $title) === 1
) {
$jpgMessageIds = $this->extractSegments(
$segments,
$this->config->segmentsToDownload,
$seenMessageIds,
$duplicateMessageIdCount,
);
}
if ($this->config->processMediaInfo && $mediaInfoMessageId === '' && isset($segments[0])
&& stripos($title, 'sample') !== false
&& preg_match('/'.$this->config->videoFileRegex.'[. ")\]]/i', $title) === 1
) {
$mediaInfoMessageId = (string) $segments[0];
$this->recordMessageId($mediaInfoMessageId, $seenMessageIds, $duplicateMessageIdCount);
}
if ($this->config->processAudioInfo && $audioInfoMessageId === '' && isset($segments[0])
&& preg_match('/'.$this->config->audioFileRegex.'[. ")\]]/i', $title, $type) === 1
) {
$audioInfoExtension = (string) ($type[1] ?? '');
$audioInfoMessageId = (string) $segments[0];
$this->recordMessageId($audioInfoMessageId, $seenMessageIds, $duplicateMessageIdCount);
}
} catch (\ErrorException $e) {
Log::debug($e->getTraceAsString());
}
}
$bookFlood = $bookFileCount > 80 && ($bookFileCount * 2) >= count($nzbContents);
$unsupportedReasons = [];
if ($bookFlood) {
$unsupportedReasons[] = 'book-flood';
}
if ($archiveCandidates === []
&& $sampleMessageIds === []
&& $jpgMessageIds === []
&& $mediaInfoMessageId === ''
&& $audioInfoMessageId === ''
) {
$unsupportedReasons[] = 'no-supported-candidates';
}
return new AdditionalWorkPlan(
sampleMessageIds: $sampleMessageIds,
jpgMessageIds: $jpgMessageIds,
mediaInfoMessageId: $mediaInfoMessageId,
audioInfoMessageId: $audioInfoMessageId,
audioInfoExtension: $audioInfoExtension,
archiveCandidates: $archiveCandidates,
bookFileCount: $bookFileCount,
bookFlood: $bookFlood,
duplicateMessageIdCount: $duplicateMessageIdCount,
unsupportedReasons: $unsupportedReasons,
);
}
private function isSupportFile(string $title): bool
{
return preg_match(
'/(?:'.$this->config->supportFileRegex.'|nfo\b|inf\b|ofn\b)($|[ ")]|-])(?!.{20,})/i',
$title,
) === 1;
}
private function isLikelyFirstVolume(string $title): bool
{
if (preg_match('/\.part0*(\d+)/i', $title, $part) === 1) {
return (int) $part[1] === 1;
}
if (preg_match('/"[a-f0-9]{32}\.[1-9]\d{1,2}".*\((\d+)\/\d{2,}\)$/i', $title, $position) === 1) {
return (int) $position[1] === 1;
}
return preg_match('/\.(rar|zip)($|[ ")]|-])/i', $title) === 1;
}
/**
* @param array<int|string, mixed> $segments
* @param array<string, true> $seenMessageIds
* @return list<string>
*/
private function extractSegments(
array $segments,
int $limit,
array &$seenMessageIds,
int &$duplicateMessageIdCount,
): array {
$messageIds = [];
$requestMessageIds = [];
foreach (array_slice($segments, 0, max($limit, 0)) as $segment) {
$messageId = (string) $segment;
if ($messageId === '' || isset($requestMessageIds[$messageId])) {
if ($messageId !== '') {
$duplicateMessageIdCount++;
}
continue;
}
$requestMessageIds[$messageId] = true;
$messageIds[] = $messageId;
$this->recordMessageId($messageId, $seenMessageIds, $duplicateMessageIdCount);
}
return $messageIds;
}
/**
* @param array<string, true> $seenMessageIds
*/
private function recordMessageId(string $messageId, array &$seenMessageIds, int &$duplicateMessageIdCount): void
{
if (isset($seenMessageIds[$messageId])) {
$duplicateMessageIdCount++;
return;
}
$seenMessageIds[$messageId] = true;
}
}
@@ -167,8 +167,8 @@ class ArchiveExtractionService
/**
* Sort files to prioritize NFO files for processing.
*
* @param array<string, mixed> $files Array of file info arrays.
* @return array<string, mixed> Sorted array with NFO files first.
* @param list<array<string, mixed>> $files Array of file info arrays.
* @return list<array<string, mixed>> Sorted array with NFO files first.
*/
public function sortFilesWithNfoPriority(array $files): array
{
@@ -325,12 +325,31 @@ class ArchiveExtractionService
*/
public function extractSpecificFile(string $compressedData, string $filename, string $tmpPath): ?string
{
// Try using ArchiveInfo's built-in extraction
return $this->extractSpecificFiles($compressedData, [$filename], $tmpPath)[$filename] ?? null;
}
/**
* Inspect one archive and extract several exact filenames from it.
*
* @param list<string> $filenames
* @return array<string, string>
*/
public function extractSpecificFiles(string $compressedData, array $filenames, string $tmpPath): array
{
$filenames = array_values(array_unique(array_filter(
$filenames,
static fn (string $filename): bool => $filename !== '',
)));
$extractedFiles = [];
$remaining = $filenames;
if ($this->archiveInfo->setData($compressedData, true)) {
foreach ($filenames as $filename) {
try {
$extracted = $this->archiveInfo->getFileData($filename);
if ($extracted !== false && ! empty($extracted)) {
return $extracted;
if ($extracted !== false && $extracted !== '') {
$extractedFiles[$filename] = $extracted;
$remaining = array_values(array_diff($remaining, [$filename]));
}
} catch (\Throwable $e) {
if ($this->config->debugMode) {
@@ -338,10 +357,23 @@ class ArchiveExtractionService
}
}
}
}
// Fallback: use external tools to extract to temp directory
foreach ($remaining as $filename) {
$extracted = $this->extractSpecificFileWithExternalTools($compressedData, $filename, $tmpPath);
if ($extracted !== null) {
$extractedFiles[$filename] = $extracted;
}
}
// Try using unrar for RAR files
return $extractedFiles;
}
private function extractSpecificFileWithExternalTools(
string $compressedData,
string $filename,
string $tmpPath,
): ?string {
if ($this->config->unrarPath) {
$extracted = $this->extractFileViaExternalTool(
$compressedData,
@@ -356,7 +388,6 @@ class ArchiveExtractionService
}
}
// Try using unzip for ZIP files
if ($this->config->unzipPath) {
$extracted = $this->extractFileViaExternalTool(
$compressedData,
@@ -0,0 +1,243 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
final readonly class AdditionalBatchResult
{
/**
* @param list<int> $claimedIds
* @param list<ReleaseProcessingResult> $results
*/
public function __construct(
public array $claimedIds = [],
public array $results = [],
public string $setupFailure = '',
public float $elapsedSeconds = 0.0,
public int $peakMemoryBytes = 0,
) {}
public static function empty(): self
{
return new self;
}
public static function setupFailed(string $reason): self
{
return new self(setupFailure: $reason);
}
public function claimedCount(): int
{
return count($this->claimedIds);
}
public function attemptedCount(): int
{
return count($this->results);
}
public function successfulCount(): int
{
return count(array_filter(
$this->results,
static fn (ReleaseProcessingResult $result): bool => $result->isSuccessful(),
));
}
public function unsuccessfulCount(): int
{
return $this->attemptedCount() - $this->successfulCount();
}
/**
* @return array<string, int>
*/
public function outcomeCounts(): array
{
$counts = [];
foreach ($this->results as $result) {
$outcome = $result->outcome->value;
$counts[$outcome] = ($counts[$outcome] ?? 0) + 1;
}
return $counts;
}
public function firstResult(): ?ReleaseProcessingResult
{
return $this->results[0] ?? null;
}
public function hasOutcome(ProcessingOutcome $outcome): bool
{
foreach ($this->results as $result) {
if ($result->outcome === $outcome) {
return true;
}
}
return false;
}
public function withPerformance(float $elapsedSeconds, int $peakMemoryBytes): self
{
return new self(
claimedIds: $this->claimedIds,
results: $this->results,
setupFailure: $this->setupFailure,
elapsedSeconds: $elapsedSeconds,
peakMemoryBytes: $peakMemoryBytes,
);
}
public function releasesPerSecond(): float
{
if ($this->elapsedSeconds <= 0.0) {
return 0.0;
}
return $this->attemptedCount() / $this->elapsedSeconds;
}
public function averageReleaseSeconds(): float
{
if ($this->attemptedCount() === 0) {
return 0.0;
}
return array_sum(array_map(
static fn (ReleaseProcessingResult $result): float => $result->elapsedSeconds,
$this->results,
)) / $this->attemptedCount();
}
public function artifactsCreatedCount(): int
{
return count(array_filter(
$this->results,
static fn (ReleaseProcessingResult $result): bool => $result->artifactsCreated,
));
}
public function artifactYieldPercent(): float
{
if ($this->successfulCount() === 0) {
return 0.0;
}
$successfulArtifacts = count(array_filter(
$this->results,
static fn (ReleaseProcessingResult $result): bool => $result->isSuccessful() && $result->artifactsCreated,
));
return ($successfulArtifacts / $this->successfulCount()) * 100;
}
public function releaseFilesAdded(): int
{
return array_sum(array_map(
static fn (ReleaseProcessingResult $result): int => $result->releaseFilesAdded,
$this->results,
));
}
public function downloadMetrics(): DownloadMetrics
{
$logicalRequests = 0;
$networkRequests = 0;
$cacheHits = 0;
$bytesDownloaded = 0;
$bytesReused = 0;
foreach ($this->results as $result) {
if ($result->downloadMetrics === null) {
continue;
}
$logicalRequests += $result->downloadMetrics->logicalRequests;
$networkRequests += $result->downloadMetrics->networkRequests;
$cacheHits += $result->downloadMetrics->cacheHits;
$bytesDownloaded += $result->downloadMetrics->bytesDownloaded;
$bytesReused += $result->downloadMetrics->bytesReused;
}
return new DownloadMetrics(
logicalRequests: $logicalRequests,
networkRequests: $networkRequests,
cacheHits: $cacheHits,
bytesDownloaded: $bytesDownloaded,
bytesReused: $bytesReused,
);
}
public function persistenceMetrics(): PersistenceMetrics
{
$databaseStatements = 0;
$databaseMilliseconds = 0.0;
$searchSyncRequests = 0;
$searchSyncExecutions = 0;
foreach ($this->results as $result) {
if ($result->persistenceMetrics === null) {
continue;
}
$databaseStatements += $result->persistenceMetrics->databaseStatements;
$databaseMilliseconds += $result->persistenceMetrics->databaseMilliseconds;
$searchSyncRequests += $result->persistenceMetrics->searchSyncRequests;
$searchSyncExecutions += $result->persistenceMetrics->searchSyncExecutions;
}
return new PersistenceMetrics(
databaseStatements: $databaseStatements,
databaseMilliseconds: $databaseMilliseconds,
searchSyncRequests: $searchSyncRequests,
searchSyncExecutions: $searchSyncExecutions,
);
}
public function duplicateMessageIdCount(): int
{
return array_sum(array_map(
static fn (ReleaseProcessingResult $result): int => $result->duplicateMessageIdCount,
$this->results,
));
}
/**
* @return array<string, int>
*/
public function unsupportedReasonCounts(): array
{
$counts = [];
foreach ($this->results as $result) {
foreach ($result->unsupportedReasons as $reason) {
$counts[$reason] = ($counts[$reason] ?? 0) + 1;
}
}
return $counts;
}
/**
* @return array<string, float>
*/
public function stageDurationTotals(): array
{
$totals = [];
foreach ($this->results as $result) {
foreach ($result->stageDurations as $stage => $duration) {
$totals[$stage] = ($totals[$stage] ?? 0.0) + $duration;
}
}
return $totals;
}
}
@@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
final readonly class AdditionalWorkPlan
{
/**
* @param list<string> $sampleMessageIds
* @param list<string> $jpgMessageIds
* @param list<ArchiveCandidate> $archiveCandidates
* @param list<string> $unsupportedReasons
*/
public function __construct(
public array $sampleMessageIds = [],
public array $jpgMessageIds = [],
public string $mediaInfoMessageId = '',
public string $audioInfoMessageId = '',
public string $audioInfoExtension = '',
public array $archiveCandidates = [],
public int $bookFileCount = 0,
public bool $bookFlood = false,
public int $duplicateMessageIdCount = 0,
public array $unsupportedReasons = [],
) {}
public function hasCompressedFile(): bool
{
return $this->archiveCandidates !== [];
}
/**
* @return list<ArchiveCandidate>
*/
public function orderedArchiveCandidates(bool $reverse = false): array
{
return $reverse ? array_reverse($this->archiveCandidates) : $this->archiveCandidates;
}
/**
* Return a stable priority view without changing the legacy processing order.
*
* @return list<ArchiveCandidate>
*/
public function prioritizedArchiveCandidates(): array
{
$firstVolumes = [];
$remaining = [];
foreach ($this->archiveCandidates as $candidate) {
if ($candidate->likelyFirstVolume) {
$firstVolumes[] = $candidate;
} else {
$remaining[] = $candidate;
}
}
return [...$firstVolumes, ...$remaining];
}
}
@@ -0,0 +1,18 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
final readonly class ArchiveCandidate
{
/**
* @param list<string> $messageIds
*/
public function __construct(
public string $title,
public array $messageIds,
public bool $likelyFirstVolume,
public int $sourceIndex,
) {}
}
@@ -0,0 +1,16 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
final readonly class DownloadMetrics
{
public function __construct(
public int $logicalRequests = 0,
public int $networkRequests = 0,
public int $cacheHits = 0,
public int $bytesDownloaded = 0,
public int $bytesReused = 0,
) {}
}
@@ -0,0 +1,22 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
final readonly class DownloadedArchive
{
/**
* @param array<int, array<string, mixed>> $files
*/
public function __construct(
public string $title,
public string $data,
public array $files,
) {}
public function byteSize(): int
{
return strlen($this->data);
}
}
@@ -0,0 +1,15 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
final readonly class PersistenceMetrics
{
public function __construct(
public int $databaseStatements = 0,
public float $databaseMilliseconds = 0.0,
public int $searchSyncRequests = 0,
public int $searchSyncExecutions = 0,
) {}
}
@@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\DTO;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
final readonly class ReleaseProcessingResult
{
/**
* @param array<string, float> $stageDurations
* @param list<string> $unsupportedReasons
*/
public function __construct(
public int $releaseId,
public string $guid,
public ProcessingOutcome $outcome,
public bool $artifactsCreated = false,
public int $releaseFilesAdded = 0,
public string $reason = '',
public float $elapsedSeconds = 0.0,
public array $stageDurations = [],
public ?DownloadMetrics $downloadMetrics = null,
public ?PersistenceMetrics $persistenceMetrics = null,
public int $duplicateMessageIdCount = 0,
public array $unsupportedReasons = [],
) {}
public function isSuccessful(): bool
{
return $this->outcome->isSuccessful();
}
/**
* @param array<string, float> $stageDurations
*/
public function withPerformance(
float $elapsedSeconds,
array $stageDurations,
?DownloadMetrics $downloadMetrics = null,
?PersistenceMetrics $persistenceMetrics = null,
): self {
return new self(
releaseId: $this->releaseId,
guid: $this->guid,
outcome: $this->outcome,
artifactsCreated: $this->artifactsCreated,
releaseFilesAdded: $this->releaseFilesAdded,
reason: $this->reason,
elapsedSeconds: $elapsedSeconds,
stageDurations: $stageDurations,
downloadMetrics: $downloadMetrics ?? $this->downloadMetrics,
persistenceMetrics: $persistenceMetrics ?? $this->persistenceMetrics,
duplicateMessageIdCount: $this->duplicateMessageIdCount,
unsupportedReasons: $this->unsupportedReasons,
);
}
}
@@ -0,0 +1,35 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\Enums;
enum ProcessingOutcome: string
{
case Completed = 'completed';
case NoUsefulArtifacts = 'no-useful-artifacts';
case Passworded = 'passworded';
case GroupUnavailable = 'group-unavailable';
case TemporaryWorkspaceUnavailable = 'temporary-workspace-unavailable';
case TimedOut = 'timed-out';
case DeletedAfterTimeout = 'deleted-after-timeout';
case DeletedBrokenNzb = 'deleted-broken-nzb';
case NotFound = 'not-found';
case Failed = 'failed';
public function isSuccessful(): bool
{
return match ($this) {
self::Completed, self::NoUsefulArtifacts, self::Passworded => true,
default => false,
};
}
public function isDeleted(): bool
{
return match ($this) {
self::DeletedAfterTimeout, self::DeletedBrokenNzb => true,
default => false,
};
}
}
@@ -0,0 +1,20 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\Enums;
enum ProcessingStage: string
{
case WorkspacePreparation = 'workspace-preparation';
case NzbParsing = 'nzb-parsing';
case ReleaseInitialization = 'release-initialization';
case MessageIdSelection = 'message-id-selection';
case DirectDownloads = 'direct-downloads';
case ArchiveDownloads = 'archive-downloads';
case ExtractedFiles = 'extracted-files';
case ArchiveFallbacks = 'archive-fallbacks';
case TimeoutHandling = 'timeout-handling';
case Finalization = 'finalization';
case WorkspaceCleanup = 'workspace-cleanup';
}
@@ -5,10 +5,10 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Enums\ImageAssetProfile;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Release;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\Categorization\CategorizationService;
use App\Services\NameFixing\ReleaseUpdateService;
@@ -38,12 +38,20 @@ class MediaExtractionService
private ?MediaInfo $mediaInfo = null;
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly ReleaseImageService $releaseImage,
private readonly ReleaseExtraService $releaseExtra,
private readonly CategorizationService $categorize
) {}
private readonly CategorizationService $categorize,
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
) {
$this->searchSyncCoordinator = $searchSyncCoordinator
?? new ReleaseSearchSyncCoordinator(
new PersistenceMetricsCollector,
);
}
/**
* Get video time code for sample extraction.
@@ -329,7 +337,7 @@ class MediaExtractionService
'proc_pp' => 1,
]);
Search::updateRelease($context->release->id);
$this->searchSyncCoordinator->request((int) $context->release->id);
if ($this->config->echoCLI) {
$releaseInfo = (object) [
@@ -147,11 +147,11 @@ class NzbContentParser
/**
* Process NZB contents to extract message IDs for different file types.
*
* @param array<string, mixed> $nzbContents
* @return array<string, mixed>
* @param list<array<string, mixed>> $nzbContents
* @return array{
* hasCompressedFile: bool,
* sampleMessageIDs: array,
* jpgMessageIDs: array,
* sampleMessageIDs: list<string>,
* jpgMessageIDs: list<string>,
* mediaInfoMessageID: string,
* audioInfoMessageID: string,
* audioInfoExtension: string,
@@ -194,7 +194,7 @@ class NzbContentParser
// Look for a video sample (not an image)
if ($config->processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments)
&& stripos($title, 'sample') !== false
&& ! preg_match('/\.jpe?g$/i', $title)
&& ! preg_match('/\.(?:jpe?g|png|webp)$/i', $title)
) {
$result['sampleMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
}
@@ -202,7 +202,7 @@ class NzbContentParser
// Look for a JPG picture (not a CD cover)
if ($config->processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments)
&& ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName)
&& preg_match('/\.jpe?g[. ")\]]/i', $title)
&& preg_match('/\.(?:jpe?g|png|webp)[. ")\]]/i', $title)
) {
$result['jpgMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
}
@@ -238,18 +238,14 @@ class NzbContentParser
/**
* Extract segment message IDs up to a limit.
*
* @param array<string, mixed> $segments
* @return array<string, mixed>
* @param array<int|string, mixed> $segments
* @return list<string>
*/
private function extractSegments(array $segments, int $limit): array
{
$ids = [];
$segCount = count($segments) - 1;
for ($i = 0; $i < $limit; $i++) {
if ($i > $segCount) {
break;
}
$ids[] = (string) $segments[$i]; // @phpstan-ignore offsetAccess.notFound
foreach (array_slice(array_values($segments), 0, max(0, $limit)) as $segment) {
$ids[] = (string) $segment;
}
return $ids;
@@ -12,6 +12,7 @@ use App\Models\Predb;
use App\Models\Release;
use App\Models\ReleaseFile;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\NameFixing\FileNameCleaner;
use App\Services\NameFixing\NameFixingService;
@@ -24,6 +25,7 @@ use App\Services\Releases\ReleaseBrowseService;
use dariusiii\rarinfo\Par2Info;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Log;
@@ -37,6 +39,8 @@ class ReleaseFileManager
private readonly FileNameCleaner $fileNameCleaner;
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly ReleaseImageService $releaseImage,
@@ -44,9 +48,15 @@ class ReleaseFileManager
private readonly NzbService $nzb,
private readonly NameFixingService $nameFixingService,
?ReleaseUpdateService $releaseUpdateService = null,
?FileNameCleaner $fileNameCleaner = null
?FileNameCleaner $fileNameCleaner = null,
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
) {
$this->releaseUpdateService = $releaseUpdateService ?? new ReleaseUpdateService;
$this->searchSyncCoordinator = $searchSyncCoordinator
?? new ReleaseSearchSyncCoordinator(
new PersistenceMetricsCollector,
);
$this->releaseUpdateService = $releaseUpdateService
?? new ReleaseUpdateService(searchSyncCoordinator: $this->searchSyncCoordinator);
$this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner;
}
@@ -156,7 +166,7 @@ class ReleaseFileManager
*/
public function updateSearchIndex(int $releaseId): void
{
Search::updateRelease($releaseId);
$this->searchSyncCoordinator->request($releaseId);
}
/**
@@ -213,8 +223,6 @@ class ReleaseFileManager
*/
public function finalizeRelease(ReleaseProcessingContext $context, bool $processPasswords): void
{
$this->flushQueuedReleaseFiles($context);
$updateRows = ['haspreview' => 0];
// Check for existing samples
@@ -230,9 +238,6 @@ class ReleaseFileManager
$updateRows['jpgstatus'] = 1;
}
// Get file count
$releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0;
$passwordStatus = $context->passwordStatus;
// Set to no password if processing is off
@@ -242,7 +247,27 @@ class ReleaseFileManager
$updateRows = array_merge($updateRows, AdditionalCandidateQuery::claimResetValues());
// Update based on conditions
$pendingReleaseFiles = array_values($context->pendingReleaseFiles);
$pendingParHashes = array_values($context->pendingParHashes);
$insertedReleaseFiles = DB::transaction(function () use (
$context,
$pendingReleaseFiles,
$pendingParHashes,
$updateRows,
$processPasswords,
$passwordStatus,
): int {
$inserted = $pendingReleaseFiles === []
? 0
: ReleaseFile::query()->insertOrIgnore($pendingReleaseFiles);
if ($pendingParHashes !== []) {
ParHash::query()->insertOrIgnore($pendingParHashes);
}
$releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0;
if (! $context->releaseHasPassword && $context->nzbHasCompressedFile && $releaseFilesCount === 0) {
Release::query()->where('id', $context->release->id)->update($updateRows);
} else {
@@ -251,7 +276,14 @@ class ReleaseFileManager
Release::query()->where('id', $context->release->id)->update($updateRows);
}
Search::updateRelease((int) $context->release->id);
return $inserted;
}, 3);
$context->pendingReleaseFiles = [];
$context->pendingParHashes = [];
$context->releaseFilesChanged = $context->releaseFilesChanged || $insertedReleaseFiles > 0;
$this->searchSyncCoordinator->request((int) $context->release->id);
}
/**
@@ -283,7 +315,7 @@ class ReleaseFileManager
'passwordstatus' => ReleaseBrowseService::PASSWD_NONE,
], AdditionalCandidateQuery::claimResetValues()));
Search::updateRelease((int) $release->id);
$this->searchSyncCoordinator->request((int) $release->id);
Log::warning('Release '.$release->id.' skipped after post-processing timeout ('.$newCount.'/'.$maxTimeoutCount.')');
@@ -302,6 +334,7 @@ class ReleaseFileManager
$id = (int) $release->id;
$guid = $release->guid ?? '';
$this->searchSyncCoordinator->discard($id);
// Delete NZB file
try {
@@ -352,7 +385,7 @@ class ReleaseFileManager
Release::where('id', $id)->delete();
} catch (\Throwable) {
}
} catch (\Throwable) { // @phpstan-ignore catch.neverThrown
} catch (\Throwable) {
// Last resort: swallow any exception
}
}
@@ -693,20 +726,6 @@ class ReleaseFileManager
return true;
}
private function flushQueuedReleaseFiles(ReleaseProcessingContext $context): void
{
if ($context->pendingReleaseFiles !== []) {
$inserted = ReleaseFile::query()->insertOrIgnore(array_values($context->pendingReleaseFiles));
$context->releaseFilesChanged = $context->releaseFilesChanged || $inserted > 0;
$context->pendingReleaseFiles = [];
}
if ($context->pendingParHashes !== []) {
ParHash::query()->insertOrIgnore(array_values($context->pendingParHashes));
$context->pendingParHashes = [];
}
}
private function loadExistingReleaseFileNames(ReleaseProcessingContext $context): void
{
if ($context->existingReleaseFileNames !== null) {
@@ -5,7 +5,7 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Models\ReleaseFile;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
@@ -17,7 +17,6 @@ use Illuminate\Support\Facades\File;
class ReleaseFilesArchiveFallback
{
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly ArchiveExtractionService $archiveService,
private readonly MediaExtractionService $mediaService,
private readonly UsenetDownloadService $downloadService,
@@ -26,7 +25,7 @@ class ReleaseFilesArchiveFallback
) {}
/**
* Try JPG/PNG files listed inside an already-downloaded archive summary.
* Try JPG/PNG/WebP files listed inside an already-downloaded archive summary.
*
* @param array<int, array<string, mixed>> $files
*/
@@ -37,7 +36,7 @@ class ReleaseFilesArchiveFallback
): void {
$imageFiles = array_values(array_filter(
$files,
static fn (array $file): bool => preg_match('/\.(jpe?g|png)$/i', (string) ($file['name'] ?? '')) === 1
static fn (array $file): bool => preg_match('/\.(jpe?g|png|webp)$/i', (string) ($file['name'] ?? '')) === 1
));
if ($imageFiles === []) {
@@ -69,7 +68,7 @@ class ReleaseFilesArchiveFallback
}
/**
* Re-download archives and extract stored JPG/PNG candidates.
* Re-download archives and extract stored JPG/PNG/WebP candidates.
*/
public function processJpgFromReleaseFiles(ReleaseProcessingContext $context): void
{
@@ -78,7 +77,8 @@ class ReleaseFilesArchiveFallback
->where(function ($query) {
$query->where('name', 'like', '%.jpg')
->orWhere('name', 'like', '%.jpeg')
->orWhere('name', 'like', '%.png');
->orWhere('name', 'like', '%.png')
->orWhere('name', 'like', '%.webp');
})
->orderByDesc('size')
->limit(3)
@@ -144,6 +144,45 @@ class ReleaseFilesArchiveFallback
);
}
/**
* Extract NFO candidates from archives already downloaded for this release.
*/
public function processNfoFromDownloadedArchives(ReleaseProcessingContext $context): void
{
if ($context->downloadedArchives === [] || ! $context->releaseHasNoNFO) {
return;
}
$nntp = $this->downloadService->getNNTP();
foreach ($context->downloadedArchives as $archive) {
$nfoFiles = array_values(array_filter(
$this->archiveService->sortFilesWithNfoPriority($archive->files),
fn (array $file): bool => $this->archiveService->isNfoFile((string) ($file['name'] ?? '')),
));
$this->processExtractedCandidatesFromData(
$archive->data,
array_slice($nfoFiles, 0, 3),
$context,
'downloaded_nfo_',
function (string $tempPath, ReleaseProcessingContext $context) use ($nntp): bool {
if (! $this->releaseManager->processNfoFile($tempPath, $context, $nntp)) {
return false;
}
$this->output->echoNfoFound();
return true;
},
);
if (! $context->releaseHasNoNFO) {
return;
}
}
}
/**
* @param EloquentCollection<int, ReleaseFile> $candidates
* @param callable(ReleaseProcessingContext): bool $shouldStop
@@ -156,34 +195,22 @@ class ReleaseFilesArchiveFallback
callable $shouldStop,
callable $handler
): void {
if ($candidates->isEmpty() || $context->nzbContents === []) {
$archiveCandidates = $this->archiveCandidates($context);
if ($candidates->isEmpty() || $archiveCandidates === []) {
return;
}
foreach ($context->nzbContents as $nzbFile) {
foreach ($archiveCandidates as $archiveCandidate) {
if ($shouldStop($context)) {
break;
}
$title = (string) ($nzbFile['title'] ?? '');
if (preg_match(
'/(\.(part0*1|rar|zip))(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
$title
) !== 1) {
continue;
}
$messageIDs = $this->extractInitialSegments($nzbFile['segments'] ?? []);
if ($messageIDs === []) {
continue;
}
$result = $this->downloadService->download(
DownloadKind::Compressed,
$messageIDs,
$archiveCandidate->messageIds,
$context->releaseGroupName,
$context->release->id,
$title
$archiveCandidate->title,
);
if ($result['groupUnavailable']) {
@@ -197,13 +224,20 @@ class ReleaseFilesArchiveFallback
continue;
}
foreach ($candidates as $candidate) {
$fileData = $this->archiveService->extractSpecificFile(
$filenames = $candidates
->pluck('name')
->filter(static fn (mixed $filename): bool => is_string($filename) && $filename !== '')
->values()
->all();
$extractedFiles = $this->extractCandidateFiles(
$result['data'],
$candidate->name,
$context->tmpPath
$filenames,
$context->tmpPath,
);
foreach ($candidates as $candidate) {
$fileData = $extractedFiles[$candidate->name] ?? null;
if ($fileData === null || $fileData === '') {
continue;
}
@@ -226,13 +260,23 @@ class ReleaseFilesArchiveFallback
string $tempPrefix,
callable $handler
): void {
$filenames = array_values(array_filter(array_map(
static fn (array $candidate): string => (string) ($candidate['name'] ?? ''),
$candidates,
)));
$extractedFiles = $this->extractCandidateFiles(
$compressedData,
$filenames,
$context->tmpPath,
);
foreach ($candidates as $candidate) {
$filename = (string) ($candidate['name'] ?? '');
if ($filename === '') {
continue;
}
$fileData = $this->archiveService->extractSpecificFile($compressedData, $filename, $context->tmpPath);
$fileData = $extractedFiles[$filename] ?? null;
if ($fileData === null || $fileData === '') {
continue;
}
@@ -243,6 +287,30 @@ class ReleaseFilesArchiveFallback
}
}
/**
* @return list<ArchiveCandidate>
*/
private function archiveCandidates(ReleaseProcessingContext $context): array
{
if ($context->workPlan === null) {
return [];
}
return array_values(array_filter(
$context->workPlan->archiveCandidates,
static fn (ArchiveCandidate $candidate): bool => $candidate->likelyFirstVolume,
));
}
/**
* @param list<string> $filenames
* @return array<string, string>
*/
private function extractCandidateFiles(string $archiveData, array $filenames, string $tmpPath): array
{
return $this->archiveService->extractSpecificFiles($archiveData, $filenames, $tmpPath);
}
/**
* @param callable(string, ReleaseProcessingContext): bool $handler
*/
@@ -264,20 +332,4 @@ class ReleaseFilesArchiveFallback
File::delete($tempPath);
}
}
/**
* @param array<int, mixed> $segments
* @return list<string>
*/
private function extractInitialSegments(array $segments): array
{
$messageIDs = [];
$segmentCount = min(count($segments), $this->config->maximumRarSegments);
for ($index = 0; $index < $segmentCount; $index++) {
$messageIDs[] = (string) $segments[$index];
}
return $messageIDs;
}
}
@@ -6,7 +6,12 @@ namespace App\Services\AdditionalProcessing;
use App\Models\UsenetGroup;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\State\ProcessingMetrics;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\Releases\ReleaseBrowseService;
use App\Services\TempWorkspaceService;
@@ -25,99 +30,203 @@ class ReleaseProcessor
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly NzbContentParser $nzbParser,
private readonly AdditionalWorkPlanner $workPlanner,
private readonly ArchiveExtractionService $archiveService,
private readonly MediaExtractionService $mediaService,
private readonly UsenetDownloadService $downloadService,
private readonly ReleaseFileManager $releaseManager,
private readonly ReleaseFilesArchiveFallback $archiveFallback,
private readonly TempWorkspaceService $tempWorkspace,
private readonly ConsoleOutputService $output
private readonly ConsoleOutputService $output,
private readonly ?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
private readonly ?PersistenceMetricsCollector $persistenceMetricsCollector = null,
) {}
public function process(ReleaseProcessingContext $context, string $mainTmpPath): void
public function process(ReleaseProcessingContext $context, string $mainTmpPath): ReleaseProcessingResult
{
$metrics = new ProcessingMetrics;
$releaseId = (int) $context->release->id;
$persistenceMetricsCollector = $this->persistenceMetricsCollector ?? new PersistenceMetricsCollector;
$searchSyncCoordinator = $this->searchSyncCoordinator
?? new ReleaseSearchSyncCoordinator(
$persistenceMetricsCollector,
);
$persistenceMetricsCollector->beginReleaseScope($releaseId);
$searchSyncCoordinator->beginReleaseScope($releaseId);
$this->downloadService->beginReleaseScope();
try {
$result = $this->processRelease($context, $mainTmpPath, $metrics);
} finally {
$context->releaseDownloadedArchives();
try {
$searchSyncCoordinator->finishReleaseScope();
} finally {
$persistenceMetrics = $persistenceMetricsCollector->finishReleaseScope();
$downloadMetrics = $this->downloadService->finishReleaseScope();
}
}
return $result->withPerformance(
$metrics->elapsedSeconds(),
$metrics->stageDurations(),
$downloadMetrics,
$persistenceMetrics,
);
}
private function processRelease(
ReleaseProcessingContext $context,
string $mainTmpPath,
ProcessingMetrics $metrics,
): ReleaseProcessingResult {
$release = $context->release;
$this->output->echoReleaseStart($release->id, $release->size);
$this->output->setProcessTitle((int) $release->id);
try {
$context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid);
$context->tmpPath = $metrics->measure(
ProcessingStage::WorkspacePreparation,
fn (): string => $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid),
);
} catch (\Throwable $e) {
$this->output->warning('Unable to prepare release temp directory: '.$e->getMessage());
return;
return $this->result(
$context,
ProcessingOutcome::TemporaryWorkspaceUnavailable,
reason: $e->getMessage(),
);
}
try {
$nzbResult = $this->nzbParser->parseNzb($release->guid);
$releaseNameChanged = false;
$releaseNeededNfo = false;
$nzbResult = $metrics->measure(
ProcessingStage::NzbParsing,
fn (): array => $this->nzbParser->parseNzb($release->guid),
);
if ($nzbResult['error'] !== null) {
$this->output->warning($nzbResult['error']);
$this->releaseManager->deleteRelease($release);
return;
return $this->result(
$context,
ProcessingOutcome::DeletedBrokenNzb,
reason: $nzbResult['error'],
);
}
$context->nzbContents = array_values($nzbResult['contents']);
if ($this->checkReleaseTimeout($context)) {
return;
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
return $timeoutResult;
}
$releaseNameChanged = $metrics->measure(
ProcessingStage::ReleaseInitialization,
function () use ($context): bool {
$this->initializeContext($context);
$this->releaseManager->processReleaseNameFromNzbContents($context->nzbContents, $context);
$messageIds = $this->nzbParser->extractMessageIDs($context->nzbContents, $context->releaseGroupName, $this->config);
$context->nzbHasCompressedFile = $messageIds['hasCompressedFile'];
$context->sampleMessageIDs = $messageIds['sampleMessageIDs'];
$context->jpgMessageIDs = $messageIds['jpgMessageIDs'];
$context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID'];
$context->audioInfoMessageIDs = $messageIds['audioInfoMessageID'];
$context->audioInfoExtension = $messageIds['audioInfoExtension'];
$bookFlood = $messageIds['bookFileCount'] > 80
&& ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents);
return $this->releaseManager->processReleaseNameFromNzbContents($context->nzbContents, $context);
},
);
$releaseNeededNfo = $context->releaseHasNoNFO;
$bookFlood = $metrics->measure(
ProcessingStage::MessageIdSelection,
fn (): bool => $this->prepareMessageIds($context),
);
if ($this->shouldProcessDownloads()) {
$this->processMessageIdDownloads($context);
if ($this->checkReleaseTimeout($context)) {
return;
$metrics->measure(
ProcessingStage::DirectDownloads,
fn () => $this->processMessageIdDownloads($context),
);
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
return $timeoutResult;
}
if (! $bookFlood && $context->nzbHasCompressedFile) {
$triedCompressedMids = [];
$metrics->measure(
ProcessingStage::ArchiveDownloads,
function () use ($context, &$triedCompressedMids): void {
$this->processNzbCompressedFiles($context, false, $triedCompressedMids);
if ($this->checkReleaseTimeout($context)) {
return;
},
);
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
return $timeoutResult;
}
if ($this->config->fetchLastFiles) {
$metrics->measure(
ProcessingStage::ArchiveDownloads,
function () use ($context, &$triedCompressedMids): void {
$this->processNzbCompressedFiles($context, true, $triedCompressedMids);
if ($this->checkReleaseTimeout($context)) {
return;
},
);
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
return $timeoutResult;
}
}
if (! $context->releaseHasPassword) {
$this->processExtractedFiles($context);
if ($this->checkReleaseTimeout($context)) {
return;
$metrics->measure(
ProcessingStage::ExtractedFiles,
fn () => $this->processExtractedFiles($context),
);
if (($timeoutResult = $this->processingTimeoutResult($context, $metrics)) !== null) {
return $timeoutResult;
}
}
}
$metrics->measure(ProcessingStage::ArchiveFallbacks, function () use ($context): void {
if (! $context->foundJPGSample && $this->config->processJPGSample) {
$this->archiveFallback->processJpgFromReleaseFiles($context);
}
if ($context->releaseHasNoNFO) {
$this->archiveFallback->processNfoFromReleaseFiles($context);
}
$this->archiveFallback->processNfoFromDownloadedArchives($context);
}
$this->releaseManager->finalizeRelease($context, $this->config->processPasswords);
if ($context->releaseHasNoNFO) {
$this->archiveFallback->processNfoFromReleaseFiles($context);
}
});
}
$metrics->measure(
ProcessingStage::Finalization,
fn () => $this->releaseManager->finalizeRelease($context, $this->config->processPasswords),
);
$artifactsCreated = $releaseNameChanged || $this->createdArtifacts($context, $releaseNeededNfo);
$outcome = match (true) {
$context->releaseHasPassword => ProcessingOutcome::Passworded,
$context->groupUnavailable => ProcessingOutcome::GroupUnavailable,
! $artifactsCreated => ProcessingOutcome::NoUsefulArtifacts,
default => ProcessingOutcome::Completed,
};
return $this->result(
$context,
$outcome,
$artifactsCreated,
reason: match ($outcome) {
ProcessingOutcome::Passworded => 'Password protection was detected.',
ProcessingOutcome::GroupUnavailable => 'The release group was unavailable during processing.',
ProcessingOutcome::NoUsefulArtifacts => 'Processing completed without creating useful artifacts.',
default => '',
},
);
} finally {
if ($context->tmpPath !== '') {
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
$metrics->measure(
ProcessingStage::WorkspaceCleanup,
fn () => $this->tempWorkspace->clearDirectory($context->tmpPath, false),
);
}
}
}
@@ -132,6 +241,23 @@ class ReleaseProcessor
|| $this->config->processJPGSample;
}
private function prepareMessageIds(ReleaseProcessingContext $context): bool
{
$workPlan = $this->workPlanner->plan(
$context->nzbContents,
$context->releaseGroupName,
);
$context->workPlan = $workPlan;
$context->nzbHasCompressedFile = $workPlan->hasCompressedFile();
$context->sampleMessageIDs = $workPlan->sampleMessageIds;
$context->jpgMessageIDs = $workPlan->jpgMessageIds;
$context->mediaInfoMessageIDs = $workPlan->mediaInfoMessageId;
$context->audioInfoMessageIDs = $workPlan->audioInfoMessageId;
$context->audioInfoExtension = $workPlan->audioInfoExtension;
return $workPlan->bookFlood;
}
private function initializeContext(ReleaseProcessingContext $context): void
{
$context->initializeFromConfig(
@@ -159,15 +285,20 @@ class ReleaseProcessor
$context->resetCounters();
}
private function checkReleaseTimeout(ReleaseProcessingContext $context): bool
{
private function processingTimeoutResult(
ReleaseProcessingContext $context,
ProcessingMetrics $metrics,
): ?ReleaseProcessingResult {
if (! $context->isTimedOut($this->config->releaseProcessingTimeout)) {
return false;
return null;
}
$deleted = $this->releaseManager->handleReleaseTimeout(
$deleted = $metrics->measure(
ProcessingStage::TimeoutHandling,
fn (): bool => $this->releaseManager->handleReleaseTimeout(
$context->release,
$this->config->maxPpTimeoutCount
$this->config->maxPpTimeoutCount,
),
);
if ($deleted) {
@@ -183,7 +314,44 @@ class ReleaseProcessor
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
}
return true;
return $this->result(
$context,
$deleted ? ProcessingOutcome::DeletedAfterTimeout : ProcessingOutcome::TimedOut,
reason: $deleted
? 'The release was deleted after reaching the post-processing timeout limit.'
: 'The release exceeded the post-processing timeout.',
);
}
private function createdArtifacts(ReleaseProcessingContext $context, bool $releaseNeededNfo): bool
{
return $context->releaseFilesChanged
|| $context->foundPAR2Info
|| ($releaseNeededNfo && ! $context->releaseHasNoNFO)
|| ($this->config->processVideo && $context->foundVideo)
|| ($this->config->processThumbnails && $context->foundSample)
|| ($this->config->processJPGSample && $context->foundJPGSample)
|| ($this->config->processMediaInfo && $context->foundMediaInfo)
|| ($this->config->processAudioInfo && $context->foundAudioInfo)
|| ($this->config->processAudioSample && $context->foundAudioSample);
}
private function result(
ReleaseProcessingContext $context,
ProcessingOutcome $outcome,
bool $artifactsCreated = false,
string $reason = '',
): ReleaseProcessingResult {
return new ReleaseProcessingResult(
releaseId: (int) $context->release->id,
guid: (string) $context->release->guid,
outcome: $outcome,
artifactsCreated: $artifactsCreated,
releaseFilesAdded: $context->addedFileInfo,
reason: $reason,
duplicateMessageIdCount: $context->duplicateMessageIdCount(),
unsupportedReasons: $context->unsupportedReasons(),
);
}
private function processMessageIdDownloads(ReleaseProcessingContext $context): void
@@ -318,61 +486,39 @@ class ReleaseProcessor
return;
}
$nzbContents = $context->nzbContents;
if ($reverse) {
krsort($nzbContents);
} else {
$archiveCandidates = match (true) {
$context->workPlan === null => [],
$reverse => $context->workPlan->orderedArchiveCandidates(true),
default => $context->workPlan->prioritizedArchiveCandidates(),
};
if (! $reverse) {
$triedCompressedMids = [];
}
$failed = $downloaded = 0;
foreach ($nzbContents as $nzbFile) {
foreach ($archiveCandidates as $archiveCandidate) {
if ($downloaded >= $this->config->maximumRarSegments
|| $failed >= $this->config->maximumRarPasswordChecks
|| $context->releaseHasPassword
|| $context->groupUnavailable
) {
break;
}
$title = (string) ($nzbFile['title'] ?? '');
if (preg_match(
'/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i',
$title
) !== 1) {
if ($reverse && array_intersect($archiveCandidate->messageIds, $triedCompressedMids) !== []) {
continue;
}
$segments = $nzbFile['segments'] ?? [];
$messageIDs = [];
$segmentCount = count($segments) - 1;
foreach (range(0, $this->config->maximumRarSegments - 1) as $index) {
if ($index > $segmentCount) {
break;
}
$segment = (string) $segments[$index];
if (! $reverse) {
$triedCompressedMids[] = $segment;
} elseif (in_array($segment, $triedCompressedMids, false)) {
continue 2;
}
$messageIDs[] = $segment;
}
if ($messageIDs === []) {
continue;
$triedCompressedMids = [...$triedCompressedMids, ...$archiveCandidate->messageIds];
}
$result = $this->downloadService->download(
DownloadKind::Compressed,
$messageIDs,
$archiveCandidate->messageIds,
$context->releaseGroupName,
$context->release->id,
$title
$archiveCandidate->title,
);
if ($result['groupUnavailable']) {
@@ -385,8 +531,13 @@ class ReleaseProcessor
$this->output->echoCompressedDownload();
$downloaded++;
$processed = $this->processCompressedData($result['data'], $context, $reverse);
if ($processed || $context->releaseHasPassword) {
$processed = $this->processCompressedData(
$result['data'],
$context,
$reverse,
$archiveCandidate->title,
);
if ($processed) {
break;
}
} else {
@@ -399,7 +550,8 @@ class ReleaseProcessor
private function processCompressedData(
string $compressedData,
ReleaseProcessingContext $context,
bool $reverse
bool $reverse,
string $archiveTitle = '',
): bool {
$result = $this->archiveService->processCompressedData(
$compressedData,
@@ -411,7 +563,7 @@ class ReleaseProcessor
$context->releaseHasPassword = true;
$context->passwordStatus = $result['passwordStatus'];
return false;
return true;
}
if (isset($result['standaloneVideoType'])) {
@@ -457,6 +609,13 @@ class ReleaseProcessor
}
}
if ($context->releaseHasNoNFO
&& is_array($result['files'])
&& $this->containsNfoCandidate($result['files'])
) {
$context->rememberDownloadedArchive($archiveTitle, $compressedData, $result['files']);
}
if (! $context->foundJPGSample && $this->config->processJPGSample) {
$this->archiveFallback->processJpgFromArchiveFileList($compressedData, $result['files'], $context);
}
@@ -464,6 +623,20 @@ class ReleaseProcessor
return $context->totalFileInfo > 0;
}
/**
* @param array<int, array<string, mixed>> $files
*/
private function containsNfoCandidate(array $files): bool
{
foreach ($files as $file) {
if ($this->archiveService->isNfoFile((string) ($file['name'] ?? ''))) {
return true;
}
}
return false;
}
private function processExtractedFiles(ReleaseProcessingContext $context): void
{
$nestedLevels = 0;
@@ -490,7 +663,7 @@ class ReleaseProcessor
$rarData = @File::get($filePath);
if (! empty($rarData)) {
$this->processCompressedData($rarData, $context, false);
$this->processCompressedData($rarData, $context, false, basename($filePath));
$foundCompressed = true;
}
File::delete($filePath);
@@ -565,7 +738,7 @@ class ReleaseProcessor
continue;
}
if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png)$/i', $filePath) === 1) {
if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png|webp)$/i', $filePath) === 1) {
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
@@ -588,7 +761,7 @@ class ReleaseProcessor
continue;
}
if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG/i', $output) === 1) {
if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG|^WebP/i', $output) === 1) {
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Facades\Search;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use Closure;
final class ReleaseSearchSyncCoordinator
{
private ?int $releaseId = null;
private bool $pending = false;
/** @var Closure(int): void */
private readonly Closure $synchronize;
/**
* @param (Closure(int): void)|null $synchronize
*/
public function __construct(
private readonly PersistenceMetricsCollector $metrics,
?Closure $synchronize = null,
private readonly bool $coalesce = true,
) {
$this->synchronize = $synchronize ?? static function (int $releaseId): void {
Search::updateRelease($releaseId);
};
}
public function beginReleaseScope(int $releaseId): void
{
if (! $this->coalesce) {
return;
}
$this->releaseId = $releaseId;
$this->pending = false;
}
public function request(int $releaseId): void
{
$this->metrics->recordSearchSyncRequest($releaseId);
if ($this->coalesce && $this->releaseId === $releaseId) {
$this->pending = true;
return;
}
$this->execute($releaseId);
}
public function discard(int $releaseId): void
{
if ($this->coalesce && $this->releaseId === $releaseId) {
$this->pending = false;
}
}
public function finishReleaseScope(): void
{
if (! $this->coalesce) {
return;
}
$releaseId = $this->releaseId;
$pending = $this->pending;
$this->releaseId = null;
$this->pending = false;
if ($releaseId !== null && $pending) {
$this->execute($releaseId);
}
}
private function execute(int $releaseId): void
{
$this->metrics->recordSearchSyncExecution($releaseId);
($this->synchronize)($releaseId);
}
}
@@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\State;
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
final class PersistenceMetricsCollector
{
private ?int $releaseId = null;
private int $databaseStatements = 0;
private float $databaseMilliseconds = 0.0;
private int $searchSyncRequests = 0;
private int $searchSyncExecutions = 0;
public function beginReleaseScope(int $releaseId): void
{
$this->releaseId = $releaseId;
$this->databaseStatements = 0;
$this->databaseMilliseconds = 0.0;
$this->searchSyncRequests = 0;
$this->searchSyncExecutions = 0;
}
public function recordDatabaseStatement(float $milliseconds): void
{
if ($this->releaseId === null) {
return;
}
$this->databaseStatements++;
$this->databaseMilliseconds += $milliseconds;
}
public function recordSearchSyncRequest(int $releaseId): void
{
if ($this->releaseId === $releaseId) {
$this->searchSyncRequests++;
}
}
public function recordSearchSyncExecution(int $releaseId): void
{
if ($this->releaseId === $releaseId) {
$this->searchSyncExecutions++;
}
}
public function finishReleaseScope(): PersistenceMetrics
{
$metrics = new PersistenceMetrics(
databaseStatements: $this->databaseStatements,
databaseMilliseconds: $this->databaseMilliseconds,
searchSyncRequests: $this->searchSyncRequests,
searchSyncExecutions: $this->searchSyncExecutions,
);
$this->releaseId = null;
$this->databaseStatements = 0;
$this->databaseMilliseconds = 0.0;
$this->searchSyncRequests = 0;
$this->searchSyncExecutions = 0;
return $metrics;
}
}
@@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\State;
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
use Closure;
final class ProcessingMetrics
{
private readonly int $startedAtNanoseconds;
/**
* @var array<string, int>
*/
private array $stageNanoseconds = [];
public function __construct()
{
$this->startedAtNanoseconds = hrtime(true);
}
/**
* @template TValue
*
* @param Closure(): TValue $operation
* @return TValue
*/
public function measure(ProcessingStage $stage, Closure $operation): mixed
{
$startedAt = hrtime(true);
try {
return $operation();
} finally {
$elapsed = hrtime(true) - $startedAt;
$this->stageNanoseconds[$stage->value] = ($this->stageNanoseconds[$stage->value] ?? 0) + $elapsed;
}
}
public function elapsedSeconds(): float
{
return $this->nanosecondsToSeconds(hrtime(true) - $this->startedAtNanoseconds);
}
/**
* @return array<string, float>
*/
public function stageDurations(): array
{
return array_map($this->nanosecondsToSeconds(...), $this->stageNanoseconds);
}
private function nanosecondsToSeconds(int $nanoseconds): float
{
return $nanoseconds / 1_000_000_000;
}
}
@@ -5,6 +5,8 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing\State;
use App\Models\Release;
use App\Services\AdditionalProcessing\DTO\AdditionalWorkPlan;
use App\Services\AdditionalProcessing\DTO\DownloadedArchive;
/**
* Mutable context object that holds the processing state for a single release.
@@ -17,6 +19,10 @@ use App\Models\Release;
*/
class ReleaseProcessingContext
{
private const int MAX_RETAINED_ARCHIVES = 2;
private const int MAX_RETAINED_ARCHIVE_BYTES = 16_777_216;
// The release being processed
public Release $release;
@@ -56,6 +62,15 @@ class ReleaseProcessingContext
*/
public array $nzbContents = [];
public ?AdditionalWorkPlan $workPlan = null;
/**
* @var list<DownloadedArchive>
*/
public array $downloadedArchives = [];
private int $retainedArchiveBytes = 0;
// Message IDs for downloading
/**
* @var list<string>
@@ -177,10 +192,53 @@ class ReleaseProcessingContext
$this->releaseHasPassword = false;
$this->nzbHasCompressedFile = false;
$this->groupUnavailable = false;
$this->workPlan = null;
$this->releaseDownloadedArchives();
$this->resetMessageIDs();
$this->resetCounters();
}
/**
* Retain only small, immediately reusable archive payloads for this release.
*
* @param array<int, array<string, mixed>> $files
*/
public function rememberDownloadedArchive(string $title, string $data, array $files): bool
{
$byteSize = strlen($data);
if ($byteSize === 0
|| count($this->downloadedArchives) >= self::MAX_RETAINED_ARCHIVES
|| ($this->retainedArchiveBytes + $byteSize) > self::MAX_RETAINED_ARCHIVE_BYTES
) {
return false;
}
$archive = new DownloadedArchive($title, $data, $files);
$this->downloadedArchives[] = $archive;
$this->retainedArchiveBytes += $archive->byteSize();
return true;
}
public function releaseDownloadedArchives(): void
{
$this->downloadedArchives = [];
$this->retainedArchiveBytes = 0;
}
public function duplicateMessageIdCount(): int
{
return $this->workPlan === null ? 0 : $this->workPlan->duplicateMessageIdCount;
}
/**
* @return list<string>
*/
public function unsupportedReasons(): array
{
return $this->workPlan === null ? [] : $this->workPlan->unsupportedReasons;
}
/**
* Mark a processing artifact as found.
*/
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\NNTP\NNTPService;
use Exception;
@@ -16,8 +17,33 @@ use Illuminate\Support\Facades\Log;
*/
class UsenetDownloadService
{
private const int MAX_CACHE_ENTRIES = 4;
private const int MAX_CACHE_ENTRY_BYTES = 8_388_608;
private const int MAX_CACHE_BYTES = 16_777_216;
private NNTPService $nntp;
private bool $releaseScopeActive = false;
/**
* @var array<string, array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}>
*/
private array $releaseCache = [];
private int $cachedBytes = 0;
private int $logicalRequests = 0;
private int $networkRequests = 0;
private int $cacheHits = 0;
private int $bytesDownloaded = 0;
private int $bytesReused = 0;
public function __construct(
private readonly ProcessingConfiguration $config,
?NNTPService $nntp = null
@@ -25,10 +51,31 @@ class UsenetDownloadService
$this->nntp = $nntp ?? new NNTPService;
}
public function beginReleaseScope(): void
{
$this->clearReleaseScope();
$this->releaseScopeActive = true;
}
public function finishReleaseScope(): DownloadMetrics
{
$metrics = new DownloadMetrics(
logicalRequests: $this->logicalRequests,
networkRequests: $this->networkRequests,
cacheHits: $this->cacheHits,
bytesDownloaded: $this->bytesDownloaded,
bytesReused: $this->bytesReused,
);
$this->clearReleaseScope();
return $metrics;
}
/**
* Download binary content from usenet using message IDs.
*
* @param array<string, mixed>|string $messageIDs Single or array of message IDs
* @param array<int|string, mixed>|string $messageIDs Single or array of message IDs
* @param string $groupName Group name for logging
* @param int|null $releaseId Release ID for logging
* @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}
@@ -61,7 +108,8 @@ class UsenetDownloadService
if ($this->config->debugMode) {
Log::debug('Attempting NNTP fetch', [
'release_id' => $releaseId,
'message_ids' => $messageIDs,
'message_id_count' => count($messageIDs),
'request_fingerprint' => $this->downloadFingerprint($messageIDs, $groupName),
'group' => $groupName,
]);
}
@@ -89,7 +137,8 @@ class UsenetDownloadService
if ($this->config->debugMode) {
Log::debug('NNTP fetch failed', [
'release_id' => $releaseId,
'message_ids' => $messageIDs,
'message_id_count' => count($messageIDs),
'request_fingerprint' => $this->downloadFingerprint($messageIDs, $groupName),
'group' => $groupName,
'error_object' => is_object($binary) ? get_class($binary) : null,
'error_message' => $errorMessage,
@@ -112,7 +161,7 @@ class UsenetDownloadService
/**
* Download content for a specific processing step.
*
* @param array<string, mixed>|string $messageIDs
* @param array<int|string, mixed>|string $messageIDs
* @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}
*/
public function download(
@@ -122,24 +171,51 @@ class UsenetDownloadService
?int $releaseId = null,
?string $fileTitle = null
): array {
$messageIdList = is_array($messageIDs) ? array_values($messageIDs) : [$messageIDs];
$cacheKey = $this->downloadFingerprint($messageIdList, $groupName);
if ($this->releaseScopeActive) {
$this->logicalRequests++;
if (isset($this->releaseCache[$cacheKey])) {
$cached = $this->releaseCache[$cacheKey];
$this->cacheHits++;
$this->bytesReused += is_string($cached['data']) ? strlen($cached['data']) : 0;
return $cached;
}
}
if ($this->config->debugMode) {
Log::debug('Attempting download', [
'kind' => $kind->value,
'release_id' => $releaseId,
'file_title' => $fileTitle,
'message_ids' => $messageIDs,
'message_id_count' => count($messageIdList),
'request_fingerprint' => $cacheKey,
'group' => $groupName,
]);
}
if ($this->releaseScopeActive && $messageIdList !== [] && $messageIdList !== ['']) {
$this->networkRequests++;
}
$result = $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId);
if ($this->releaseScopeActive && $result['success'] && is_string($result['data'])) {
$byteSize = strlen($result['data']);
$this->bytesDownloaded += $byteSize;
$this->rememberSuccessfulDownload($cacheKey, $result, $byteSize);
}
if (! $result['success'] && $this->config->debugMode) {
Log::debug('Download failed', [
'kind' => $kind->value,
'release_id' => $releaseId,
'file_title' => $fileTitle,
'message_ids' => $messageIDs,
'message_id_count' => count($messageIdList),
'request_fingerprint' => $cacheKey,
'group' => $groupName,
'error' => $result['error'],
]);
@@ -148,6 +224,46 @@ class UsenetDownloadService
return $result;
}
/**
* @param list<mixed> $messageIds
*/
private function downloadFingerprint(array $messageIds, string $groupName): string
{
return hash('sha256', serialize([
'message_ids' => array_map(static fn (mixed $messageId): string => (string) $messageId, $messageIds),
'group' => $groupName,
'alternate' => $this->config->alternateNNTP,
]));
}
/**
* @param array{success: bool, data: string|null, groupUnavailable: bool, error: string|null} $result
*/
private function rememberSuccessfulDownload(string $cacheKey, array $result, int $byteSize): void
{
if ($byteSize > self::MAX_CACHE_ENTRY_BYTES
|| count($this->releaseCache) >= self::MAX_CACHE_ENTRIES
|| ($this->cachedBytes + $byteSize) > self::MAX_CACHE_BYTES
) {
return;
}
$this->releaseCache[$cacheKey] = $result;
$this->cachedBytes += $byteSize;
}
private function clearReleaseScope(): void
{
$this->releaseScopeActive = false;
$this->releaseCache = [];
$this->cachedBytes = 0;
$this->logicalRequests = 0;
$this->networkRequests = 0;
$this->cacheHits = 0;
$this->bytesDownloaded = 0;
$this->bytesReused = 0;
}
/**
* Get the NNTP client instance.
*/
@@ -5,10 +5,11 @@ declare(strict_types=1);
namespace App\Services\NameFixing;
use App\Events\ReleaseNameFixed;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Release;
use App\Models\UsenetGroup;
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseCleaningService;
use Illuminate\Support\Arr;
@@ -82,6 +83,8 @@ class ReleaseUpdateService
protected bool $echoOutput;
private readonly ReleaseSearchSyncCoordinator $searchSyncCoordinator;
/**
* The release ID we are trying to rename.
*/
@@ -109,10 +112,13 @@ class ReleaseUpdateService
public function __construct(
?CategorizationService $category = null,
?FileNameCleaner $fileNameCleaner = null
?FileNameCleaner $fileNameCleaner = null,
?ReleaseSearchSyncCoordinator $searchSyncCoordinator = null,
) {
$this->category = $category ?? new CategorizationService;
$this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner;
$this->searchSyncCoordinator = $searchSyncCoordinator
?? new ReleaseSearchSyncCoordinator(new PersistenceMetricsCollector);
$this->echoOutput = config('nntmux.echocli');
}
@@ -331,8 +337,7 @@ class ReleaseUpdateService
));
});
// Update search index
Search::updateRelease($release->releases_id);
$this->searchSyncCoordinator->request((int) $release->releases_id);
}
/**
+4 -4
View File
@@ -16,7 +16,7 @@ use Illuminate\Support\Facades\Log;
class PostProcessRunner extends BaseRunner
{
private const int ADDITIONAL_WORKER_MAX_BATCHES = 4;
public const int ADDITIONAL_WORKER_MAX_BATCHES = 4;
private function guidBucketExpression(string $column = 'leftguid'): string
{
@@ -257,13 +257,13 @@ class PostProcessRunner extends BaseRunner
$excludedReleaseIds = [];
try {
for ($batch = 0; $batch < self::ADDITIONAL_WORKER_MAX_BATCHES; $batch++) {
$stats = $orchestrator->start('', $char, $workerToken, $excludedReleaseIds);
if ($stats['claimed'] === 0) {
$result = $orchestrator->start('', $char, $workerToken, $excludedReleaseIds);
if ($result->claimedCount() === 0) {
break;
}
$excludedReleaseIds = array_values(array_unique([
...$excludedReleaseIds,
...$stats['claimed_ids'],
...$result->claimedIds,
]));
}
} finally {
-36
View File
@@ -480,24 +480,12 @@ parameters:
count: 1
path: app/Rules/RecaptchaRule.php
-
message: '#^Method App\\Services\\AdditionalProcessing\\ArchiveExtractionService\:\:sortFilesWithNfoPriority\(\) should return array\<string, mixed\> but returns list\<mixed\>\.$#'
identifier: return.type
count: 1
path: app/Services/AdditionalProcessing/ArchiveExtractionService.php
-
message: '#^PHPDoc tag @return with type list\<mixed\> is incompatible with native type void\.$#'
identifier: return.phpDocType
count: 1
path: app/Services/AdditionalProcessing/ArchiveExtractionService.php
-
message: '#^Method App\\Services\\AdditionalProcessing\\NzbContentParser\:\:extractSegments\(\) should return array\<string, mixed\> but returns list\<string\>\.$#'
identifier: return.type
count: 1
path: app/Services/AdditionalProcessing/NzbContentParser.php
-
message: '#^Using nullsafe property access "\?\-\>id" on left side of \?\? is unnecessary\. Use \-\> instead\.$#'
identifier: nullsafe.neverNull
@@ -510,30 +498,6 @@ parameters:
count: 2
path: app/Services/AdditionalProcessing/ReleaseFileManager.php
-
message: '#^Parameter \#2 \$messageIDs of method App\\Services\\AdditionalProcessing\\UsenetDownloadService\:\:download\(\) expects array\<string, mixed\>\|string, list\<string\> given\.$#'
identifier: argument.type
count: 1
path: app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php
-
message: '#^Parameter \#1 \$nzbContents of method App\\Services\\AdditionalProcessing\\NzbContentParser\:\:extractMessageIDs\(\) expects array\<string, mixed\>, list\<array\<string, mixed\>\> given\.$#'
identifier: argument.type
count: 1
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
-
message: '#^Parameter \#2 \$messageIDs of method App\\Services\\AdditionalProcessing\\UsenetDownloadService\:\:download\(\) expects array\<string, mixed\>\|string, list\<string\> given\.$#'
identifier: argument.type
count: 3
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
-
message: '#^Right side of \|\| is always false\.$#'
identifier: booleanOr.rightAlwaysFalse
count: 2
path: app/Services/AdditionalProcessing/ReleaseProcessor.php
-
message: '#^Strict comparison using \=\=\= between non\-empty\-string and null will always evaluate to false\.$#'
identifier: identical.alwaysFalse
@@ -154,6 +154,25 @@ class AdditionalProcessingNzbSplitRenameTest extends TestCase
$this->assertSame('MLB.2026.Cincinnati.Reds.vs.Tampa.Bay.Rays.20.04.720pEN60fps', $result);
}
public function test_additional_nfo_detection_accepts_alternative_and_scene_names(): void
{
$manager = $this->makeReleaseFileManager();
foreach (['file_id.diz', 'readme.txt', 'info.txt', '00-scene.nfo', 'group-release.nfo'] as $filename) {
$this->assertTrue($manager->isNfoFilename($filename), $filename.' should be treated as NFO data');
}
}
public function test_additional_nfo_normalization_converts_utf16_and_removes_utf8_bom(): void
{
$manager = $this->makeReleaseFileManager();
$normalize = new \ReflectionMethod($manager, 'normalizeNfoEncoding');
$utf16 = "\xFF\xFE".mb_convert_encoding('Scene NFO', 'UTF-16LE', 'UTF-8');
$this->assertSame('Scene NFO', $normalize->invoke($manager, $utf16));
$this->assertSame('Scene NFO', $normalize->invoke($manager, "\xEF\xBB\xBFScene NFO"));
}
private function makeReleaseFileManager(?ReleaseUpdateService $updateService = null): ReleaseFileManager
{
$updateService ??= $this->createMock(ReleaseUpdateService::class);
@@ -7,13 +7,17 @@ namespace Tests\Feature;
use App\Models\Release;
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use App\Services\AdditionalProcessing\ReleaseProcessor;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\TempWorkspaceService;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Schema;
use Mockery;
use PDO;
use Tests\TestCase;
use Tests\Unit\AdditionalProcessing\CreatesProcessingConfiguration;
@@ -87,6 +91,8 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
public function test_handled_release_exception_clears_claim(): void
{
Log::spy();
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
DB::table('releases')->insert([
'id' => 1,
@@ -121,15 +127,49 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
$output
);
$orchestrator->start('', 'a');
$result = $orchestrator->start('', 'a');
$this->assertSame(1, $processor->processCalls);
$this->assertSame([1], $result->claimedIds);
$this->assertSame(0, $result->successfulCount());
$this->assertSame(1, $result->unsuccessfulCount());
$this->assertTrue($result->hasOutcome(ProcessingOutcome::Failed));
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
$this->assertGreaterThan(0.0, $result->averageReleaseSeconds());
$this->assertGreaterThan(0.0, $result->releasesPerSecond());
$this->assertGreaterThan(0, $result->peakMemoryBytes);
$this->assertSame(1, $tempWorkspace->ensureMainTempPathCalls);
$this->assertSame(1, $tempWorkspace->clearDirectoryCalls);
$this->assertSame(1, $output->echoDescriptionCalls);
$this->assertSame(1, $output->endOutputCalls);
$this->assertNull(Release::query()->where('id', 1)->value('additional_pp_claimed_at'));
$this->assertNull(Release::query()->where('id', 1)->value('additional_pp_claim_token'));
Log::shouldHaveReceived('info')
->once()
->with(
'Additional postprocessing run finished',
Mockery::on(static fn (array $context): bool => $context['picked'] === 1
&& $context['processed'] === 0
&& $context['failed'] === 1
&& $context['outcomes'] === ['failed' => 1]
&& array_key_exists('artifacts_created', $context)
&& array_key_exists('artifact_yield_percent', $context)
&& array_key_exists('release_files_added', $context)
&& array_key_exists('download_requests', $context)
&& array_key_exists('nntp_requests', $context)
&& array_key_exists('download_cache_hits', $context)
&& array_key_exists('database_statements', $context)
&& array_key_exists('database_milliseconds', $context)
&& array_key_exists('search_sync_requests', $context)
&& array_key_exists('search_sync_executions', $context)
&& array_key_exists('duplicate_message_ids', $context)
&& array_key_exists('unsupported_reasons', $context)
&& array_key_exists('releases_per_second', $context)
&& array_key_exists('average_release_seconds', $context)
&& array_key_exists('stage_seconds', $context)
&& array_key_exists('peak_memory_bytes', $context)),
);
}
public function test_temp_setup_failure_does_not_claim_releases(): void
@@ -168,9 +208,11 @@ class AdditionalProcessingOrchestratorClaimTest extends TestCase
$output
);
$orchestrator->start('', 'a');
$result = $orchestrator->start('', 'a');
$this->assertSame(0, $processor->processCalls);
$this->assertSame(0, $result->claimedCount());
$this->assertStringContainsString('not writable', $result->setupFailure);
$this->assertSame(1, $tempWorkspace->ensureMainTempPathCalls);
$this->assertSame(0, $output->echoDescriptionCalls);
$this->assertSame(0, $output->endOutputCalls);
@@ -246,7 +288,7 @@ class FailingAdditionalReleaseProcessor extends ReleaseProcessor
public function __construct() {}
public function process(ReleaseProcessingContext $context, string $mainTmpPath): void
public function process(ReleaseProcessingContext $context, string $mainTmpPath): ReleaseProcessingResult
{
$this->processCalls++;
@@ -7,6 +7,7 @@ namespace Tests\Feature;
use App\Facades\Search;
use App\Models\Release;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\CollectionCleanupService;
use App\Services\NameFixing\NameFixingService;
@@ -14,6 +15,7 @@ use App\Services\NfoService;
use App\Services\Nzb\NzbService;
use App\Services\ReleaseImageService;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\QueryException;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\File;
@@ -129,6 +131,40 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
$this->assertNull(DB::table('releases')->where('id', 1)->value('additional_pp_claim_token'));
}
public function test_finalization_flushes_rows_and_synchronizes_search_immediately(): void
{
DB::table('releases')->insert($this->releaseRow());
Search::shouldReceive('updateRelease')->once()->with(1);
$manager = $this->makeManagerWithImageService(new ReleaseImageService);
$context = new ReleaseProcessingContext(Release::query()->findOrFail(1));
$this->assertTrue($manager->addFileInfo([
'name' => 'Example.Movie.2026.mkv',
'size' => 2048,
'date' => 1_788_600_000,
], $context, '\\.(?:par2|sfv|nzb)'));
$manager->finalizeRelease($context, false);
$this->assertSame(1, DB::table('release_files')->count());
$this->assertSame(1, DB::table('releases')->where('id', 1)->value('rarinnerfilecount'));
$this->assertSame([], $context->pendingReleaseFiles);
}
public function test_database_statements_are_measured_inside_an_active_release_scope(): void
{
DB::table('releases')->insert($this->releaseRow());
$collector = app(PersistenceMetricsCollector::class);
$collector->beginReleaseScope(1);
DB::table('releases')->where('id', 1)->value('guid');
$metrics = $collector->finishReleaseScope();
$this->assertSame(1, $metrics->databaseStatements);
$this->assertGreaterThanOrEqual(0.0, $metrics->databaseMilliseconds);
}
public function test_queued_par_hashes_flush_with_release_files(): void
{
DB::table('releases')->insert($this->releaseRow());
@@ -156,6 +192,35 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
$this->assertSame(1, DB::table('par_hashes')->count());
}
public function test_failed_finalization_rolls_back_buffered_rows_and_keeps_them_for_retry(): void
{
DB::table('releases')->insert($this->releaseRow());
Search::shouldReceive('updateRelease')->never();
$manager = $this->makeManager();
$context = new ReleaseProcessingContext(Release::query()->findOrFail(1));
$this->assertTrue($manager->addFileInfo([
'name' => 'Example.Movie.2026.mkv',
'size' => 1024,
'date' => 1_788_600_000,
], $context, '\\.(?:par2|sfv|nzb)'));
DB::unprepared("CREATE TRIGGER fail_release_finalize BEFORE UPDATE ON releases BEGIN SELECT RAISE(ABORT, 'forced finalization failure'); END");
try {
$manager->finalizeRelease($context, false);
$this->fail('Finalization should have failed.');
} catch (QueryException $exception) {
$this->assertStringContainsString('forced finalization failure', $exception->getMessage());
} finally {
DB::unprepared('DROP TRIGGER IF EXISTS fail_release_finalize');
}
$this->assertSame(0, DB::table('release_files')->count());
$this->assertArrayHasKey('Example.Movie.2026.mkv', $context->pendingReleaseFiles);
}
public function test_float_release_file_size_is_normalized_before_queueing(): void
{
DB::table('releases')->insert($this->releaseRow());
@@ -230,12 +295,16 @@ class AdditionalProcessingReleaseFileManagerTest extends TestCase
return $this->makeManagerWithImageService(new ReleaseImageService, $nameFixing);
}
/**
* @param array<string, mixed> $configOverrides
*/
private function makeManagerWithImageService(
ReleaseImageService $imageService,
?NameFixingService $nameFixing = null,
array $configOverrides = [],
): ReleaseFileManager {
return new ReleaseFileManager(
$this->makeConfig(),
$this->makeConfig($configOverrides),
$imageService,
new NfoService,
new TestNzbService,
@@ -5,6 +5,11 @@ declare(strict_types=1);
namespace Tests\Feature;
use App\Services\AdditionalProcessing\AdditionalProcessingOrchestrator;
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use App\Services\Runners\PostProcessRunner;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
@@ -186,7 +191,7 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
$mock->shouldReceive('start')
->once()
->with('', 'a', Mockery::type('string'), [])
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [1]]);
->andReturn($this->successfulBatch(1));
$mock->shouldReceive('finish')->once();
});
@@ -205,17 +210,17 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
->once()
->ordered()
->with('', 'a', Mockery::type('string'), [])
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [1]]);
->andReturn($this->successfulBatch(1));
$mock->shouldReceive('start')
->once()
->ordered()
->with('', 'a', Mockery::type('string'), [1])
->andReturn(['claimed' => 1, 'processed' => 1, 'failed' => 0, 'claimed_ids' => [2]]);
->andReturn($this->successfulBatch(2));
$mock->shouldReceive('start')
->once()
->ordered()
->with('', 'a', Mockery::type('string'), [1, 2])
->andReturn(['claimed' => 0, 'processed' => 0, 'failed' => 0, 'claimed_ids' => []]);
->andReturn(AdditionalBatchResult::empty());
$mock->shouldReceive('finish')->once();
});
@@ -229,6 +234,154 @@ class PostProcessRunnerAdditionalThreadsTest extends TestCase
$this->assertSame(0, $status);
}
public function test_worker_guid_command_emits_machine_readable_batch_profiles(): void
{
$batchResult = $this->successfulBatch(1)->withPerformance(2.0, 1024);
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock) use ($batchResult): void {
$mock->shouldReceive('start')->once()->andReturn($batchResult);
$mock->shouldReceive('finish')->once();
});
$status = Artisan::call('postprocess:guid', [
'type' => 'additional',
'guid' => 'a',
'--profile' => true,
]);
$profile = json_decode(trim(Artisan::output()), true, flags: JSON_THROW_ON_ERROR);
$this->assertSame(0, $status);
$this->assertSame('additional-postprocessing-profile', $profile['event']);
$this->assertSame('v2', $profile['pipeline']);
$this->assertSame(1, $profile['attempted']);
$this->assertSame(1800, $profile['releases_per_hour']);
$this->assertSame(1024, $profile['peak_memory_bytes']);
}
public function test_targeted_additional_command_reports_a_successful_typed_outcome(): void
{
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
DB::table('releases')->insert($this->releaseRow(1, 'a'));
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock): void {
$mock->shouldReceive('processSingleGuid')
->once()
->with('a-guid-1')
->andReturn(new ReleaseProcessingResult(
1,
'a-guid-1',
ProcessingOutcome::Completed,
artifactsCreated: true,
releaseFilesAdded: 2,
elapsedSeconds: 1.2345,
stageDurations: ['nzb-parsing' => 0.5],
downloadMetrics: new DownloadMetrics(2, 1, 1, 100, 100),
persistenceMetrics: new PersistenceMetrics(4, 3.5, 2, 1),
));
$mock->shouldReceive('finish')->once();
});
$status = Artisan::call('releases:additional', ['--id' => 1, '--profile' => true]);
$output = Artisan::output();
$this->assertSame(0, $status);
$this->assertStringContainsString('outcome completed', $output);
$this->assertStringContainsString('Artifacts created', $output);
$this->assertStringContainsString('Release files added', $output);
$this->assertStringContainsString('Stage profile', $output);
$this->assertStringContainsString('Download profile', $output);
$this->assertStringContainsString('Persistence profile', $output);
}
public function test_targeted_additional_command_fails_for_an_unsuccessful_typed_outcome(): void
{
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
DB::table('releases')->insert($this->releaseRow(1, 'a'));
$this->mock(AdditionalProcessingOrchestrator::class, function (MockInterface $mock): void {
$mock->shouldReceive('processSingleGuid')
->once()
->with('a-guid-1')
->andReturn(new ReleaseProcessingResult(
1,
'a-guid-1',
ProcessingOutcome::TimedOut,
reason: 'The release exceeded the post-processing timeout.',
elapsedSeconds: 2.3456,
));
$mock->shouldReceive('finish')->once();
});
$status = Artisan::call('releases:additional', ['--id' => 1]);
$output = Artisan::output();
$this->assertSame(3, $status);
$this->assertStringContainsString('outcome timed-out', $output);
$this->assertStringContainsString('Reason', $output);
$this->assertStringContainsString('The release exceeded the post-processing timeout.', $output);
}
public function test_additional_diagnostics_reports_stale_claims_and_preflight_warnings_as_json(): void
{
config(['nntmux.tmp_unrar_path' => '']);
DB::table('settings')->upsert([
['name' => 'postthreads', 'value' => '33'],
['name' => 'maxaddprocessed', 'value' => '25'],
['name' => 'maxpptimeoutcount', 'value' => '3'],
], ['name'], ['value']);
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
DB::table('releases')->insert([
...$this->releaseRow(1, 'a'),
'additional_pp_claimed_at' => now()->subHour(),
'additional_pp_claim_token' => 'stale-claim',
]);
$status = Artisan::call('nntmux:additional-diagnose', ['--json' => true]);
$report = json_decode(Artisan::output(), true, flags: JSON_THROW_ON_ERROR);
$warningCodes = array_column($report['warnings'], 'code');
$this->assertSame(0, $status);
$this->assertSame(1, $report['backlog']['total']);
$this->assertSame(1, $report['backlog']['available']);
$this->assertSame(1, $report['backlog']['stale_claims']);
$this->assertSame(825, $report['capacity']['max_in_flight']);
$this->assertSame('v2', $report['settings']['pipeline']);
$this->assertContains('stale-claims', $warningCodes);
$this->assertContains('claim-ttl', $warningCodes);
$this->assertContains('missing-index', $warningCodes);
$this->assertContains('unwritable-temp-path', $warningCodes);
$this->assertContains('oversized-capacity', $warningCodes);
}
public function test_additional_diagnostics_recognizes_the_covering_claim_index(): void
{
Schema::table('releases', function (Blueprint $table): void {
$table->index([
'passwordstatus',
'haspreview',
'nzbstatus',
'leftguid',
'additional_pp_claimed_at',
'postdate',
], 'ix_releases_add_pp_claim_queue');
});
$status = Artisan::call('nntmux:additional-diagnose', ['--json' => true]);
$report = json_decode(Artisan::output(), true, flags: JSON_THROW_ON_ERROR);
$this->assertSame(0, $status);
$this->assertFalse($report['indexes']['missing']);
$this->assertSame('ix_releases_add_pp_claim_queue', $report['indexes']['covering_index']);
$this->assertNotContains('missing-index', array_column($report['warnings'], 'code'));
}
private function successfulBatch(int $releaseId): AdditionalBatchResult
{
return new AdditionalBatchResult(
[$releaseId],
[new ReleaseProcessingResult($releaseId, 'a-guid-'.$releaseId, ProcessingOutcome::Completed)],
);
}
/**
* @return array<string, mixed>
*/
@@ -0,0 +1,386 @@
<?php
declare(strict_types=1);
namespace Tests\Integration;
use App\Services\AdditionalProcessing\AdditionalCandidateQuery;
use Dotenv\Dotenv;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Support\Facades\DB;
use PDO;
use PHPUnit\Framework\Attributes\Test;
use RuntimeException;
use Tests\TestCase;
final class AdditionalCandidateQueryMariaDbTest extends TestCase
{
private string $tablePrefix;
public function createApplication()
{
Dotenv::createMutable(dirname(__DIR__, 2))->safeLoad();
$this->tablePrefix = 'phase4_'.getmypid().'_'.bin2hex(random_bytes(4)).'_';
$database = (string) ($_ENV['DB_DATABASE'] ?? 'nntmux');
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_URL', null);
$this->setEnvironmentValue('DB_CONNECTION', 'mariadb');
$this->setEnvironmentValue('DB_DATABASE', $database);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
$app->make('config')->set('database.connections.mariadb.prefix', $this->tablePrefix);
$app->make('db')->purge('mariadb');
return $app;
}
protected function setUp(): void
{
parent::setUp();
if (DB::getDriverName() !== 'mariadb') {
$this->markTestSkipped('MariaDB integration test.');
}
$this->createSchema();
}
protected function tearDown(): void
{
if (isset($this->tablePrefix) && preg_match('/^phase4_\d+_[a-f0-9]{8}_$/', $this->tablePrefix) === 1) {
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('releases').'`');
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('categories').'`');
DB::statement('DROP TABLE IF EXISTS `'.$this->tableName('settings').'`');
}
DB::disconnect();
parent::tearDown();
}
#[Test]
public function current_indexes_bound_examined_rows_for_realistic_claim_and_backlog_plans(): void
{
$releasesTable = $this->tableName('releases');
$categoriesTable = $this->tableName('categories');
DB::statement(<<<SQL
INSERT INTO `{$releasesTable}` (
id, guid, leftguid, passwordstatus, haspreview, nzbstatus,
categories_id, size, postdate, additional_pp_claimed_at
)
SELECT
seq,
CONCAT(SUBSTRING('0123456789abcdef', MOD(seq, 16) + 1, 1), '-guid-', seq),
SUBSTRING('0123456789abcdef', MOD(seq, 16) + 1, 1),
IF(MOD(seq, 97) = 0, -1, 0),
IF(MOD(seq, 97) = 0, -1, 0),
1,
1,
2097152,
DATE_SUB(NOW(), INTERVAL seq SECOND),
IF(MOD(seq, 679) = 0, NOW(), NULL)
FROM seq_1_to_50000
SQL);
DB::statement("ANALYZE TABLE `{$releasesTable}`");
$claimPlan = $this->analyze(<<<SQL
SELECT r.id
FROM `{$releasesTable}` r
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
WHERE r.passwordstatus = -1
AND r.haspreview = -1
AND r.nzbstatus = 1
AND c.disablepreview = 0
AND r.size > 1048576
AND r.size < 107374182400
AND r.leftguid = 'a'
AND (r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND))
ORDER BY r.postdate DESC, r.id ASC
LIMIT 25
FOR UPDATE
SQL);
$bucketPlan = $this->analyze(<<<SQL
SELECT r.leftguid, COUNT(*) AS total_count,
SUM(CASE WHEN r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND) THEN 1 ELSE 0 END) AS available_count
FROM `{$releasesTable}` r
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
WHERE r.passwordstatus = -1
AND r.haspreview = -1
AND r.nzbstatus = 1
AND c.disablepreview = 0
AND r.size > 1048576
AND r.size < 107374182400
GROUP BY r.leftguid
ORDER BY r.leftguid
SQL);
$backlogPlan = $this->analyze(<<<SQL
SELECT COUNT(*) AS total_count,
SUM(CASE WHEN r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND) THEN 1 ELSE 0 END) AS available_count
FROM `{$releasesTable}` r
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
WHERE r.passwordstatus = -1
AND r.haspreview = -1
AND r.nzbstatus = 1
AND c.disablepreview = 0
AND r.size > 1048576
AND r.size < 107374182400
SQL);
$this->assertStringContainsString('ix_releases_add_pp_claim_queue', $claimPlan);
$this->assertLessThan(5_000, $this->examinedRowsFor($claimPlan, 'r'));
$this->assertLessThanOrEqual(2_000, $this->examinedRowsFor($bucketPlan, 'r'));
$this->assertLessThanOrEqual(2_000, $this->examinedRowsFor($backlogPlan, 'r'));
}
#[Test]
public function claims_exclude_fresh_rows_recover_stale_rows_and_preserve_ordering(): void
{
DB::table('releases')->insert([
$this->releaseRow(1, '2026-08-10 12:00:00', now()),
$this->releaseRow(2, '2026-08-10 11:00:00'),
$this->releaseRow(3, '2026-08-10 11:00:00'),
$this->releaseRow(4, '2026-08-10 10:00:00', now()->subSeconds(301)),
]);
$firstClaim = AdditionalCandidateQuery::claimBatch('a', 3, 'worker-one', columns: ['id']);
$secondClaim = AdditionalCandidateQuery::claimBatch('a', 3, 'worker-two', columns: ['id']);
$this->assertSame([2, 3, 4], $firstClaim->pluck('id')->all());
$this->assertSame([], $secondClaim->pluck('id')->all());
$this->assertSame('worker-one', DB::table('releases')->where('id', 4)->value('additional_pp_claim_token'));
$this->assertSame('claimed', DB::table('releases')->where('id', 1)->value('additional_pp_claim_token'));
}
#[Test]
public function concurrent_workers_do_not_claim_the_same_hot_bucket_rows(): void
{
if (! function_exists('pcntl_fork')) {
$this->markTestSkipped('The pcntl extension is required for the MariaDB contention test.');
}
DB::table('releases')->insert(array_map(
fn (int $id): array => $this->releaseRow($id, now()->subSeconds($id)->format('Y-m-d H:i:s')),
range(1, 20),
));
DB::beginTransaction();
$firstWorkerIds = AdditionalCandidateQuery::baseBuilder(guidChar: 'a')
->select('r.id')
->orderByDesc('r.postdate')
->orderBy('r.id')
->limit(5)
->lockForUpdate()
->pluck('r.id')
->map(static fn (mixed $id): int => (int) $id)
->all();
$sockets = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP);
if ($sockets === false) {
DB::rollBack();
throw new RuntimeException('Unable to create an IPC socket pair.');
}
$pid = pcntl_fork();
if ($pid === -1) {
DB::rollBack();
throw new RuntimeException('Unable to fork the competing claim worker.');
}
if ($pid === 0) {
fclose($sockets[0]);
try {
$connection = $this->createMariaDbConnection();
$connection->beginTransaction();
$releasesTable = $this->tableName('releases');
$categoriesTable = $this->tableName('categories');
$claimedIds = array_map(
static fn (mixed $id): int => (int) $id,
$connection->query(<<<SQL
SELECT r.id
FROM `{$releasesTable}` r
LEFT JOIN `{$categoriesTable}` c ON c.id = r.categories_id
WHERE r.passwordstatus = -1
AND r.haspreview = -1
AND r.nzbstatus = 1
AND c.disablepreview = 0
AND r.leftguid = 'a'
AND (r.additional_pp_claimed_at IS NULL OR r.additional_pp_claimed_at < DATE_SUB(NOW(), INTERVAL 300 SECOND))
ORDER BY r.postdate DESC, r.id ASC
LIMIT 5
FOR UPDATE
SQL)->fetchAll(PDO::FETCH_COLUMN),
);
$placeholders = implode(',', array_fill(0, count($claimedIds), '?'));
$update = $connection->prepare("UPDATE `{$releasesTable}` SET additional_pp_claimed_at = ?, additional_pp_claim_token = ? WHERE id IN ({$placeholders})");
$update->execute([now()->format('Y-m-d H:i:s'), 'worker-two', ...$claimedIds]);
$connection->commit();
fwrite($sockets[1], json_encode(['ids' => $claimedIds], JSON_THROW_ON_ERROR));
fclose($sockets[1]);
exit(0);
} catch (\Throwable $exception) {
fwrite($sockets[1], json_encode(['error' => $exception->getMessage()], JSON_THROW_ON_ERROR));
fclose($sockets[1]);
exit(1);
}
}
fclose($sockets[1]);
usleep(200_000);
DB::table('releases')->whereIn('id', $firstWorkerIds)->update([
'additional_pp_claimed_at' => now(),
'additional_pp_claim_token' => 'worker-one',
]);
DB::commit();
$childPayload = stream_get_contents($sockets[0]);
fclose($sockets[0]);
pcntl_waitpid($pid, $status);
DB::purge();
DB::reconnect();
/** @var array{ids?: list<int>, error?: string} $childResult */
$childResult = json_decode($childPayload, true, flags: JSON_THROW_ON_ERROR);
$this->assertArrayNotHasKey('error', $childResult, $childResult['error'] ?? 'Competing worker failed.');
$this->assertSame([1, 2, 3, 4, 5], $firstWorkerIds);
$this->assertSame([6, 7, 8, 9, 10], $childResult['ids'] ?? []);
$this->assertSame([], array_intersect($firstWorkerIds, $childResult['ids'] ?? []));
$this->assertTrue(pcntl_wifexited($status));
$this->assertSame(0, pcntl_wexitstatus($status));
}
private function createSchema(): void
{
$settingsTable = $this->tableName('settings');
$categoriesTable = $this->tableName('categories');
$releasesTable = $this->tableName('releases');
DB::statement("CREATE TABLE `{$settingsTable}` (`name` VARCHAR(255) PRIMARY KEY, `value` TEXT NULL) ENGINE=InnoDB");
DB::statement("CREATE TABLE `{$categoriesTable}` (id INT UNSIGNED PRIMARY KEY, disablepreview TINYINT(1) NOT NULL DEFAULT 0) ENGINE=InnoDB");
DB::statement(<<<SQL
CREATE TABLE `{$releasesTable}` (
id INT UNSIGNED PRIMARY KEY,
guid VARCHAR(255) NOT NULL,
leftguid CHAR(1) NOT NULL,
passwordstatus INT NOT NULL,
haspreview INT NOT NULL,
nzbstatus INT NOT NULL,
categories_id INT UNSIGNED NOT NULL,
size BIGINT UNSIGNED NOT NULL,
postdate DATETIME NULL,
additional_pp_claimed_at TIMESTAMP NULL,
additional_pp_claim_token VARCHAR(64) NULL,
KEY ix_releases_haspreview_passwordstatus (haspreview, passwordstatus),
KEY ix_releases_add_pp_claim_queue (passwordstatus, haspreview, nzbstatus, leftguid, additional_pp_claimed_at, postdate DESC)
) ENGINE=InnoDB
SQL);
DB::table('settings')->insert([
['name' => 'categorizeforeign', 'value' => '0'],
['name' => 'catwebdl', 'value' => '0'],
['name' => 'releaseprocessingtimeout', 'value' => '120'],
]);
DB::table('categories')->insert(['id' => 1, 'disablepreview' => 0]);
}
/** @return array<string, mixed> */
private function releaseRow(int $id, string $postdate, ?\DateTimeInterface $claimedAt = null): array
{
return [
'id' => $id,
'guid' => 'a-guid-'.$id,
'leftguid' => 'a',
'passwordstatus' => -1,
'haspreview' => -1,
'nzbstatus' => 1,
'categories_id' => 1,
'size' => 2 * 1048576,
'postdate' => $postdate,
'additional_pp_claimed_at' => $claimedAt?->format('Y-m-d H:i:s'),
'additional_pp_claim_token' => $claimedAt === null ? null : 'claimed',
];
}
private function analyze(string $sql): string
{
$row = DB::selectOne('ANALYZE FORMAT=JSON '.$sql);
$values = (array) $row;
return (string) reset($values);
}
private function examinedRowsFor(string $plan, string $tableName): float
{
/** @var array<string, mixed> $decoded */
$decoded = json_decode($plan, true, flags: JSON_THROW_ON_ERROR);
$examinedRows = $this->findExaminedRows($decoded, $tableName);
if ($examinedRows === null) {
throw new RuntimeException("Unable to find runtime row count for table [{$tableName}].");
}
return $examinedRows;
}
/**
* @param array<string, mixed> $node
*/
private function findExaminedRows(array $node, string $tableName): ?float
{
if (($node['table_name'] ?? null) === $tableName && isset($node['r_rows'])) {
return (float) $node['r_rows'];
}
foreach ($node as $value) {
if (! is_array($value)) {
continue;
}
$rows = $this->findExaminedRows($value, $tableName);
if ($rows !== null) {
return $rows;
}
}
return null;
}
private function tableName(string $table): string
{
return $this->tablePrefix.$table;
}
private function createMariaDbConnection(): PDO
{
$host = (string) ($_ENV['DB_HOST'] ?? 'mariadb');
$port = (string) ($_ENV['DB_PORT'] ?? '3306');
$database = (string) ($_ENV['DB_DATABASE'] ?? 'nntmux');
$username = (string) ($_ENV['DB_USERNAME'] ?? 'nntmux');
$password = (string) ($_ENV['DB_PASSWORD'] ?? '');
return new PDO(
"mysql:host={$host};port={$port};dbname={$database};charset=utf8mb4",
$username,
$password,
[PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION],
);
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
}
@@ -0,0 +1,100 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
use App\Services\AdditionalProcessing\DTO\ArchiveCandidate;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class AdditionalWorkPlannerTest extends TestCase
{
use CreatesProcessingConfiguration;
#[Test]
public function it_builds_one_ordered_plan_for_direct_and_archive_candidates(): void
{
$planner = new AdditionalWorkPlanner($this->makeConfig([
'processThumbnails' => true,
'processJPGSample' => true,
'processMediaInfo' => true,
'processAudioInfo' => true,
]));
$plan = $planner->plan([
['title' => 'release.part02.rar', 'segments' => ['<part-2>', '<shared>']],
['title' => '"sample.mkv" yEnc', 'segments' => ['<sample>', '<sample-2>']],
['title' => '"cover.jpg" yEnc', 'segments' => ['<cover>']],
['title' => '"track.FLAC" yEnc', 'segments' => ['<audio>']],
['title' => 'release.part01.rar', 'segments' => ['<part-1>', '<shared>', '<part-1>']],
], 'alt.binaries.test');
$this->assertSame(['<sample>', '<sample-2>'], $plan->sampleMessageIds);
$this->assertSame(['<cover>'], $plan->jpgMessageIds);
$this->assertSame('<sample>', $plan->mediaInfoMessageId);
$this->assertSame('<audio>', $plan->audioInfoMessageId);
$this->assertSame('FLAC', $plan->audioInfoExtension);
$this->assertTrue($plan->hasCompressedFile());
$this->assertSame(
['release.part02.rar', 'release.part01.rar'],
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->archiveCandidates),
);
$this->assertFalse($plan->archiveCandidates[0]->likelyFirstVolume);
$this->assertTrue($plan->archiveCandidates[1]->likelyFirstVolume);
$this->assertSame(
['release.part01.rar', 'release.part02.rar'],
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->prioritizedArchiveCandidates()),
);
$this->assertSame(
['release.part01.rar', 'release.part02.rar'],
array_map(static fn (ArchiveCandidate $candidate): string => $candidate->title, $plan->orderedArchiveCandidates(true)),
);
$this->assertSame(3, $plan->duplicateMessageIdCount);
$this->assertSame([], $plan->unsupportedReasons);
}
#[Test]
public function it_reports_book_floods_and_releases_without_supported_candidates(): void
{
$planner = new AdditionalWorkPlanner($this->makeConfig());
$contents = array_fill(0, 81, [
'title' => 'book.epub',
'segments' => ['<book>'],
]);
$plan = $planner->plan($contents, 'alt.binaries.books');
$this->assertSame(81, $plan->bookFileCount);
$this->assertTrue($plan->bookFlood);
$this->assertSame(['book-flood', 'no-supported-candidates'], $plan->unsupportedReasons);
}
#[Test]
public function it_selects_jpeg_png_and_webp_image_candidates(): void
{
$planner = new AdditionalWorkPlanner($this->makeConfig(['processJPGSample' => true]));
foreach (['cover.jpg', 'cover.png', 'cover.webp'] as $index => $filename) {
$messageId = '<image-'.$index.'>';
$plan = $planner->plan([
['title' => '"'.$filename.'" yEnc', 'segments' => [$messageId]],
], 'alt.binaries.test');
$this->assertSame([$messageId], $plan->jpgMessageIds, $filename.' should be selected');
}
}
#[Test]
public function it_keeps_a_usable_last_volume_when_the_first_volume_is_missing(): void
{
$planner = new AdditionalWorkPlanner($this->makeConfig());
$plan = $planner->plan([
['title' => 'release.part99.rar', 'segments' => ['<last-volume>']],
], 'alt.binaries.test');
$this->assertFalse($plan->archiveCandidates[0]->likelyFirstVolume);
$this->assertSame(['<last-volume>'], $plan->orderedArchiveCandidates(true)[0]->messageIds);
}
}
@@ -2,10 +2,14 @@
namespace Tests\Unit\AdditionalProcessing;
use App\Models\Release;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\Releases\ReleaseBrowseService;
use dariusiii\rarinfo\ArchiveInfo;
use dariusiii\rarinfo\Par2Info;
use Mockery;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
@@ -35,6 +39,26 @@ class ArchiveExtractionServiceTest extends TestCase
$this->assertSame('IMAGE-DATA', $service->extractSpecificFile('ARCHIVE', 'cover.jpg', sys_get_temp_dir().'/'));
}
#[Test]
public function it_inspects_an_archive_once_when_extracting_multiple_candidates(): void
{
$archiveInfo = Mockery::mock(ArchiveInfo::class);
$archiveInfo->shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true);
$archiveInfo->shouldReceive('getFileData')->once()->with('release.nfo')->andReturn('NFO-DATA');
$archiveInfo->shouldReceive('getFileData')->once()->with('cover.jpg')->andReturn('IMAGE-DATA');
$service = new ArchiveExtractionService(
$this->makeConfig(),
$archiveInfo,
Mockery::mock(Par2Info::class)
);
$this->assertSame([
'release.nfo' => 'NFO-DATA',
'cover.jpg' => 'IMAGE-DATA',
], $service->extractSpecificFiles('ARCHIVE', ['release.nfo', 'cover.jpg'], sys_get_temp_dir().'/'));
}
#[Test]
public function it_detects_common_standalone_video_signatures(): void
{
@@ -51,4 +75,68 @@ class ArchiveExtractionServiceTest extends TestCase
$this->assertSame('mp4', $service->detectStandaloneVideo($mp4));
$this->assertNull($service->detectStandaloneVideo('tiny'));
}
#[Test]
#[DataProvider('archiveProtectionFixtures')]
public function it_classifies_passworded_and_unpassworded_rar_and_zip_data(
int $archiveType,
string $expectedMarker,
bool $encrypted,
): void {
$archiveInfo = Mockery::mock(ArchiveInfo::class);
$archiveInfo->error = '';
$archiveInfo->shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true);
$archiveInfo->shouldReceive('getSummary')->once()->with(true)->andReturn([
'main_type' => $archiveType,
'is_encrypted' => $encrypted ? 1 : 0,
'archives' => [
'nested.rar' => ['file_list' => [['name' => 'Nested.Release.2026.mkv']]],
],
]);
if ($encrypted) {
$archiveInfo->shouldNotReceive('getArchiveFileList');
} else {
$archiveInfo->shouldReceive('getArchiveFileList')->once()->andReturn([
['name' => 'nested.rar', 'size' => 100],
]);
}
$service = new ArchiveExtractionService(
$this->makeConfig(),
$archiveInfo,
Mockery::mock(Par2Info::class)
);
$context = new ReleaseProcessingContext(new Release(['id' => 1, 'guid' => 'fixture-guid']));
$result = $service->processCompressedData('ARCHIVE', $context, sys_get_temp_dir().'/');
$this->assertSame($encrypted, $result['hasPassword']);
$this->assertSame(
$encrypted ? ReleaseBrowseService::PASSWD_RAR : ReleaseBrowseService::PASSWD_NONE,
$result['passwordStatus'],
);
if (! $encrypted) {
$this->assertTrue($result['success']);
$this->assertSame($expectedMarker, $result['archiveMarker']);
$this->assertSame(
'Nested.Release.2026.mkv',
$result['dataSummary']['archives']['nested.rar']['file_list'][0]['name'],
);
}
}
/**
* @return array<string, array{int, string, bool}>
*/
public static function archiveProtectionFixtures(): array
{
return [
'unpassworded RAR' => [ArchiveInfo::TYPE_RAR, 'r', false],
'passworded RAR' => [ArchiveInfo::TYPE_RAR, 'r', true],
'unpassworded ZIP' => [ArchiveInfo::TYPE_ZIP, 'z', false],
'passworded ZIP' => [ArchiveInfo::TYPE_ZIP, 'z', true],
];
}
}
@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\MediaExtractionService;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseExtraService;
use App\Services\ReleaseImageService;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Foundation\Application;
use Illuminate\Support\Facades\Facade;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class MediaExtractionServiceTest extends TestCase
{
use CreatesProcessingConfiguration;
private string $tmpPath;
protected function setUp(): void
{
parent::setUp();
$container = new Application(sys_get_temp_dir());
$container->instance('files', new Filesystem);
Facade::setFacadeApplication($container);
$this->tmpPath = sys_get_temp_dir().'/additional-media-'.uniqid('', true).'/';
(new Filesystem)->makeDirectory($this->tmpPath, 0777, true, true);
}
protected function tearDown(): void
{
(new Filesystem)->deleteDirectory($this->tmpPath);
Mockery::close();
parent::tearDown();
}
#[Test]
public function it_recognizes_jpeg_png_and_webp_samples_by_signature(): void
{
$fixtures = [
'sample.jpg' => "\xFF\xD8\xFF\xE0\x00\x10JFIF\x00",
'sample.png' => "\x89PNG\r\n\x1A\n\x00\x00\x00\rIHDR",
'sample.webp' => "RIFF\x1A\x00\x00\x00WEBPVP8 ",
];
$service = new MediaExtractionService(
$this->makeConfig(),
Mockery::mock(ReleaseImageService::class),
Mockery::mock(ReleaseExtraService::class),
Mockery::mock(CategorizationService::class),
);
foreach ($fixtures as $filename => $contents) {
$path = $this->tmpPath.$filename;
file_put_contents($path, $contents);
$this->assertTrue($service->isValidImage($path), $filename.' should be recognized');
}
}
}
@@ -0,0 +1,142 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\DTO\AdditionalBatchResult;
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
use App\Services\AdditionalProcessing\DTO\PersistenceMetrics;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingResult;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class ProcessingResultTest extends TestCase
{
#[Test]
public function it_aggregates_typed_release_outcomes(): void
{
$batch = new AdditionalBatchResult(
claimedIds: [10, 11, 12],
results: [
new ReleaseProcessingResult(
10,
'a-guid-10',
ProcessingOutcome::Completed,
artifactsCreated: true,
releaseFilesAdded: 3,
elapsedSeconds: 0.5,
stageDurations: ['nzb-parsing' => 0.2, 'finalization' => 0.1],
downloadMetrics: new DownloadMetrics(2, 1, 1, 100, 100),
persistenceMetrics: new PersistenceMetrics(8, 12.5, 2, 1),
duplicateMessageIdCount: 1,
),
new ReleaseProcessingResult(
11,
'a-guid-11',
ProcessingOutcome::Passworded,
elapsedSeconds: 0.75,
stageDurations: ['nzb-parsing' => 0.3],
downloadMetrics: new DownloadMetrics(1, 1, 0, 50, 0),
persistenceMetrics: new PersistenceMetrics(5, 7.5, 1, 1),
unsupportedReasons: ['book-flood'],
),
new ReleaseProcessingResult(
12,
'a-guid-12',
ProcessingOutcome::TimedOut,
elapsedSeconds: 1.25,
stageDurations: ['nzb-parsing' => 0.5, 'timeout-handling' => 0.05],
),
],
elapsedSeconds: 2.0,
peakMemoryBytes: 32_000_000,
);
$this->assertSame(3, $batch->claimedCount());
$this->assertSame(3, $batch->attemptedCount());
$this->assertSame(2, $batch->successfulCount());
$this->assertSame(1, $batch->unsuccessfulCount());
$this->assertSame([
'completed' => 1,
'passworded' => 1,
'timed-out' => 1,
], $batch->outcomeCounts());
$this->assertTrue($batch->hasOutcome(ProcessingOutcome::TimedOut));
$this->assertSame(ProcessingOutcome::Completed, $batch->firstResult()?->outcome);
$this->assertSame(1.5, $batch->releasesPerSecond());
$this->assertEqualsWithDelta(0.833333, $batch->averageReleaseSeconds(), 0.000001);
$this->assertSame(1, $batch->artifactsCreatedCount());
$this->assertSame(50.0, $batch->artifactYieldPercent());
$this->assertSame(3, $batch->releaseFilesAdded());
$this->assertEquals(new DownloadMetrics(3, 2, 1, 150, 100), $batch->downloadMetrics());
$this->assertEquals(new PersistenceMetrics(13, 20.0, 3, 2), $batch->persistenceMetrics());
$this->assertSame(1, $batch->duplicateMessageIdCount());
$this->assertSame(['book-flood' => 1], $batch->unsupportedReasonCounts());
$this->assertSame([
'nzb-parsing' => 1.0,
'finalization' => 0.1,
'timeout-handling' => 0.05,
], $batch->stageDurationTotals());
$this->assertSame(32_000_000, $batch->peakMemoryBytes);
}
#[Test]
public function it_represents_a_worker_setup_failure_without_claiming_releases(): void
{
$batch = AdditionalBatchResult::setupFailed('Temporary path is unavailable.');
$this->assertSame(0, $batch->claimedCount());
$this->assertSame(0, $batch->attemptedCount());
$this->assertSame('Temporary path is unavailable.', $batch->setupFailure);
$this->assertNull($batch->firstResult());
$this->assertSame(0.0, $batch->releasesPerSecond());
$this->assertSame(0.0, $batch->artifactYieldPercent());
}
#[Test]
public function completing_without_useful_artifacts_is_a_truthful_successful_outcome(): void
{
$result = new ReleaseProcessingResult(
10,
'a-guid-10',
ProcessingOutcome::NoUsefulArtifacts,
reason: 'Processing completed without creating useful artifacts.',
);
$this->assertTrue($result->isSuccessful());
$this->assertFalse($result->artifactsCreated);
$this->assertSame('Processing completed without creating useful artifacts.', $result->reason);
}
#[Test]
public function it_can_attach_performance_data_without_changing_the_processing_outcome(): void
{
$result = new ReleaseProcessingResult(
10,
'a-guid-10',
ProcessingOutcome::Completed,
artifactsCreated: true,
releaseFilesAdded: 2,
);
$downloadMetrics = new DownloadMetrics(2, 1, 1, 100, 100);
$persistenceMetrics = new PersistenceMetrics(4, 3.5, 2, 1);
$measured = $result->withPerformance(
1.25,
['nzb-parsing' => 0.4],
$downloadMetrics,
$persistenceMetrics,
);
$this->assertSame($result->releaseId, $measured->releaseId);
$this->assertSame($result->outcome, $measured->outcome);
$this->assertTrue($measured->artifactsCreated);
$this->assertSame(2, $measured->releaseFilesAdded);
$this->assertSame(1.25, $measured->elapsedSeconds);
$this->assertSame(['nzb-parsing' => 0.4], $measured->stageDurations);
$this->assertSame($downloadMetrics, $measured->downloadMetrics);
$this->assertSame($persistenceMetrics, $measured->persistenceMetrics);
}
}
@@ -4,6 +4,7 @@ namespace Tests\Unit\AdditionalProcessing;
use App\Models\Release;
use App\Models\ReleaseFile;
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\MediaExtractionService;
@@ -59,10 +60,13 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
}
#[Test]
public function it_extracts_jpg_candidates_from_an_archive_file_list(): void
public function it_extracts_webp_candidates_from_an_archive_file_list(): void
{
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('JPEG-DATA');
$archive->shouldReceive('extractSpecificFiles')
->once()
->with('ARCHIVE', ['cover.webp'], Mockery::type('string'))
->andReturn(['cover.webp' => 'WEBP-DATA']);
$media = Mockery::mock(MediaExtractionService::class);
$media->shouldReceive('isValidImage')->once()->andReturn(true);
@@ -72,7 +76,6 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
$output->shouldReceive('echoJpgSaved')->once()->andReturnNull();
$fallback = new ReleaseFilesArchiveFallback(
$this->makeConfig(['processJPGSample' => true]),
$archive,
$media,
Mockery::mock(UsenetDownloadService::class),
@@ -81,7 +84,7 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
);
$context = $this->makeContext();
$fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.jpg', 'size' => 99]], $context);
$fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.webp', 'size' => 99]], $context);
$this->assertTrue($context->foundJPGSample);
}
@@ -96,7 +99,10 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
]);
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('NFO DATA');
$archive->shouldReceive('extractSpecificFiles')
->once()
->with('ARCHIVE', ['release.nfo'], Mockery::type('string'))
->andReturn(['release.nfo' => 'NFO DATA']);
$download = Mockery::mock(UsenetDownloadService::class);
$download->shouldReceive('download')->once()->andReturn([
@@ -118,8 +124,8 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
$config = $this->makeConfig();
$fallback = new ReleaseFilesArchiveFallback(
$this->makeConfig(),
$archive,
Mockery::mock(MediaExtractionService::class),
$download,
@@ -131,12 +137,133 @@ class ReleaseFilesArchiveFallbackTest extends TestCase
$context->releaseGroupName = 'alt.binaries.test';
$context->releaseHasNoNFO = true;
$context->nzbContents = [['title' => 'archive.part01.rar', 'segments' => ['<1>', '<2>']]];
$context->workPlan = (new AdditionalWorkPlanner($config))->plan(
$context->nzbContents,
$context->releaseGroupName,
);
$fallback->processNfoFromReleaseFiles($context);
$this->assertFalse($context->releaseHasNoNFO);
}
#[Test]
public function it_recovers_an_nfo_from_an_already_downloaded_archive_without_another_download(): void
{
$files = [['name' => 'release.nfo', 'size' => 200]];
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('sortFilesWithNfoPriority')->once()->with($files)->andReturn($files);
$archive->shouldReceive('isNfoFile')->once()->with('release.nfo')->andReturn(true);
$archive->shouldReceive('extractSpecificFiles')
->once()
->with('ARCHIVE', ['release.nfo'], Mockery::type('string'))
->andReturn(['release.nfo' => 'NFO DATA']);
$download = Mockery::mock(UsenetDownloadService::class);
$download->shouldNotReceive('download');
$download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class));
$manager = Mockery::mock(ReleaseFileManager::class);
$manager->shouldReceive('processNfoFile')->once()->andReturnUsing(
function (string $path, ReleaseProcessingContext $context): bool {
$this->assertSame('NFO DATA', file_get_contents($path));
$context->releaseHasNoNFO = false;
return true;
}
);
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
$fallback = new ReleaseFilesArchiveFallback(
$archive,
Mockery::mock(MediaExtractionService::class),
$download,
$manager,
$output
);
$context = $this->makeContext();
$context->releaseHasNoNFO = true;
$this->assertTrue($context->rememberDownloadedArchive('release.part01.rar', 'ARCHIVE', $files));
$fallback->processNfoFromDownloadedArchives($context);
$this->assertFalse($context->releaseHasNoNFO);
}
#[Test]
public function it_extracts_archive_candidates_in_one_batch(): void
{
$files = [
['name' => 'first.nfo', 'size' => 200],
['name' => 'second.nfo', 'size' => 200],
];
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('sortFilesWithNfoPriority')->once()->with($files)->andReturn($files);
$archive->shouldReceive('isNfoFile')->twice()->andReturn(true);
$archive->shouldNotReceive('extractSpecificFile');
$archive->shouldReceive('extractSpecificFiles')
->once()
->with('ARCHIVE', ['first.nfo', 'second.nfo'], Mockery::type('string'))
->andReturn([
'first.nfo' => '',
'second.nfo' => 'NFO DATA',
]);
$download = Mockery::mock(UsenetDownloadService::class);
$download->shouldNotReceive('download');
$download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class));
$manager = Mockery::mock(ReleaseFileManager::class);
$manager->shouldReceive('processNfoFile')->once()->andReturnUsing(
function (string $path, ReleaseProcessingContext $context): bool {
$this->assertSame('NFO DATA', file_get_contents($path));
$context->releaseHasNoNFO = false;
return true;
}
);
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
$fallback = new ReleaseFilesArchiveFallback(
$archive,
Mockery::mock(MediaExtractionService::class),
$download,
$manager,
$output,
);
$context = $this->makeContext();
$context->releaseHasNoNFO = true;
$this->assertTrue($context->rememberDownloadedArchive('release.part01.rar', 'ARCHIVE', $files));
$fallback->processNfoFromDownloadedArchives($context);
$this->assertFalse($context->releaseHasNoNFO);
}
#[Test]
public function it_bounds_and_releases_retained_archive_payloads(): void
{
$context = $this->makeContext();
$files = [['name' => 'release.nfo', 'size' => 10]];
$this->assertTrue($context->rememberDownloadedArchive('first.rar', 'FIRST', $files));
$this->assertTrue($context->rememberDownloadedArchive('second.rar', 'SECOND', $files));
$this->assertFalse($context->rememberDownloadedArchive('third.rar', 'THIRD', $files));
$this->assertCount(2, $context->downloadedArchives);
$context->releaseDownloadedArchives();
$this->assertSame([], $context->downloadedArchives);
}
private function makeContext(int $releaseId = 12): ReleaseProcessingContext
{
$context = new ReleaseProcessingContext(new Release([
@@ -3,15 +3,23 @@
namespace Tests\Unit\AdditionalProcessing;
use App\Models\Release;
use App\Services\AdditionalProcessing\AdditionalWorkPlanner;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\DTO\DownloadMetrics;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\AdditionalProcessing\Enums\ProcessingOutcome;
use App\Services\AdditionalProcessing\Enums\ProcessingStage;
use App\Services\AdditionalProcessing\MediaExtractionService;
use App\Services\AdditionalProcessing\NzbContentParser;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
use App\Services\AdditionalProcessing\ReleaseProcessor;
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use App\Services\AdditionalProcessing\State\ReleaseProcessingContext;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\Releases\ReleaseBrowseService;
use App\Services\TempWorkspaceService;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
@@ -44,9 +52,16 @@ class ReleaseProcessorTest extends TestCase
->shouldReceive('warning')->once()->with('broken nzb')->andReturnNull()->getMock()
);
$processor->process($this->makeContext(), '/tmp/main/');
$result = $processor->process($this->makeContext(), '/tmp/main/');
$this->assertTrue(true);
$this->assertSame(ProcessingOutcome::DeletedBrokenNzb, $result->outcome);
$this->assertTrue($result->outcome->isDeleted());
$this->assertFalse($result->isSuccessful());
$this->assertSame('broken nzb', $result->reason);
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
$this->assertArrayHasKey(ProcessingStage::WorkspacePreparation->value, $result->stageDurations);
$this->assertArrayHasKey(ProcessingStage::NzbParsing->value, $result->stageDurations);
$this->assertArrayHasKey(ProcessingStage::WorkspaceCleanup->value, $result->stageDurations);
}
#[Test]
@@ -58,19 +73,22 @@ class ReleaseProcessorTest extends TestCase
'error' => null,
'contents' => [['title' => 'file.nzb', 'segments' => []]],
]);
$nzbParser->shouldReceive('extractMessageIDs')->once()->andReturn([
'hasCompressedFile' => false,
'sampleMessageIDs' => [],
'jpgMessageIDs' => [],
'mediaInfoMessageID' => '',
'audioInfoMessageID' => '',
'audioInfoExtension' => '',
'bookFileCount' => 0,
]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
$persistenceMetrics = new PersistenceMetricsCollector;
$synchronizedReleaseIds = [];
$searchSyncCoordinator = new ReleaseSearchSyncCoordinator(
$persistenceMetrics,
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
$synchronizedReleaseIds[] = $releaseId;
},
);
$releaseManager->shouldReceive('finalizeRelease')
->once()
->andReturnUsing(static function () use ($searchSyncCoordinator): void {
$searchSyncCoordinator->request(1);
$searchSyncCoordinator->request(1);
});
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
@@ -83,18 +101,80 @@ class ReleaseProcessorTest extends TestCase
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$this->scopedDownloadService(),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace,
$output
$output,
$searchSyncCoordinator,
$persistenceMetrics,
);
$processor->process($this->makeContext(), '/tmp/main/');
$context = $this->makeContext();
$context->release->nfostatus = 1;
$result = $processor->process($context, '/tmp/main/');
$this->assertTrue(true);
$this->assertSame(ProcessingOutcome::NoUsefulArtifacts, $result->outcome);
$this->assertTrue($result->isSuccessful());
$this->assertFalse($result->artifactsCreated);
$this->assertSame('Processing completed without creating useful artifacts.', $result->reason);
$this->assertSame(0, $result->releaseFilesAdded);
$this->assertSame([1], $synchronizedReleaseIds);
$this->assertSame(2, $result->persistenceMetrics?->searchSyncRequests);
$this->assertSame(1, $result->persistenceMetrics?->searchSyncExecutions);
$this->assertGreaterThan(0.0, $result->elapsedSeconds);
$this->assertSame([
ProcessingStage::WorkspacePreparation->value,
ProcessingStage::NzbParsing->value,
ProcessingStage::ReleaseInitialization->value,
ProcessingStage::MessageIdSelection->value,
ProcessingStage::Finalization->value,
ProcessingStage::WorkspaceCleanup->value,
], array_keys($result->stageDurations));
}
#[Test]
public function it_always_uses_the_v2_work_plan_for_message_id_selection(): void
{
$config = $this->makeConfig();
$nzbContents = [['title' => 'file.nzb', 'segments' => ['<message>']]];
$nzbParser = Mockery::mock(NzbContentParser::class);
$nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([
'error' => null,
'contents' => $nzbContents,
]);
$nzbParser->shouldNotReceive('extractMessageIDs');
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
$this->scopedDownloadService(),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$this->successfulTempWorkspace(),
Mockery::mock(ConsoleOutputService::class)
->shouldReceive('echoReleaseStart')->once()->andReturnNull()
->shouldReceive('setProcessTitle')->once()->andReturnNull()
->getMock(),
);
$context = $this->makeContext();
$context->release->nfostatus = 1;
$result = $processor->process($context, '/tmp/main/');
$this->assertSame(ProcessingOutcome::NoUsefulArtifacts, $result->outcome);
$this->assertNotNull($context->workPlan);
$this->assertSame(0, $result->duplicateMessageIdCount);
}
#[Test]
@@ -124,9 +204,10 @@ class ReleaseProcessorTest extends TestCase
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$this->scopedDownloadService(),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace,
@@ -136,9 +217,172 @@ class ReleaseProcessorTest extends TestCase
$context = $this->makeContext();
$context->startTime = hrtime(true) - 2_000_000_000;
$processor->process($context, '/tmp/main/');
$result = $processor->process($context, '/tmp/main/');
$this->assertTrue(true);
$this->assertSame(ProcessingOutcome::TimedOut, $result->outcome);
$this->assertFalse($result->isSuccessful());
$this->assertStringContainsString('exceeded', $result->reason);
$this->assertArrayHasKey(ProcessingStage::TimeoutHandling->value, $result->stageDurations);
}
#[Test]
public function it_reports_a_deleted_timeout_separately(): void
{
$config = $this->makeConfig(['releaseProcessingTimeout' => 1, 'maxPpTimeoutCount' => 1]);
$nzbParser = Mockery::mock(NzbContentParser::class);
$nzbParser->shouldReceive('parseNzb')->once()->andReturn(['error' => null, 'contents' => []]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('handleReleaseTimeout')->once()->andReturn(true);
$releaseManager->shouldNotReceive('finalizeRelease');
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
$tempWorkspace->shouldReceive('clearDirectory')->twice()->with('/tmp/ap-release/', false)->andReturnNull();
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
$output->shouldReceive('echoReleaseTimeoutDeleted')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
$this->scopedDownloadService(),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace,
$output
);
$context = $this->makeContext();
$context->startTime = hrtime(true) - 2_000_000_000;
$result = $processor->process($context, '/tmp/main/');
$this->assertSame(ProcessingOutcome::DeletedAfterTimeout, $result->outcome);
$this->assertTrue($result->outcome->isDeleted());
$this->assertFalse($result->isSuccessful());
}
#[Test]
public function it_reports_a_temporary_workspace_failure(): void
{
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
$tempWorkspace->shouldReceive('createReleaseTempFolder')
->once()
->andThrow(new \RuntimeException('workspace is not writable'));
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
$output->shouldReceive('warning')->once()->with('Unable to prepare release temp directory: workspace is not writable')->andReturnNull();
$processor = $this->makeProcessor(tempWorkspace: $tempWorkspace, output: $output);
$context = $this->makeContext();
$context->release->nfostatus = 1;
$result = $processor->process($context, '/tmp/main/');
$this->assertSame(ProcessingOutcome::TemporaryWorkspaceUnavailable, $result->outcome);
$this->assertFalse($result->isSuccessful());
$this->assertSame('workspace is not writable', $result->reason);
}
#[Test]
public function it_reports_password_detection_as_a_successful_outcome(): void
{
$config = $this->makeConfig(['processPasswords' => true]);
$nzbParser = $this->compressedNzbParser();
$archiveService = Mockery::mock(ArchiveExtractionService::class);
$archiveService->shouldReceive('processCompressedData')->once()->andReturn([
'success' => false,
'files' => [],
'hasPassword' => true,
'passwordStatus' => ReleaseBrowseService::PASSWD_RAR,
]);
$downloadService = Mockery::mock(UsenetDownloadService::class);
$this->expectDownloadScope($downloadService);
$downloadService->shouldReceive('download')
->once()
->with(DownloadKind::Compressed, ['<archive>'], '', 1, 'archive.part01.rar')
->andReturn(['success' => true, 'data' => 'ARCHIVE', 'groupUnavailable' => false, 'error' => null]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
$archiveService,
Mockery::mock(MediaExtractionService::class),
$downloadService,
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$this->successfulTempWorkspace(),
$this->passwordOutput()
);
$context = $this->makeContext();
$context->release->nfostatus = 1;
$result = $processor->process($context, '/tmp/main/');
$this->assertSame(ProcessingOutcome::Passworded, $result->outcome);
$this->assertTrue($result->isSuccessful());
$this->assertFalse($result->artifactsCreated);
}
#[Test]
public function it_reports_group_unavailability_as_an_unsuccessful_outcome(): void
{
$config = $this->makeConfig(['processPasswords' => true]);
$nzbParser = $this->compressedNzbParser();
$downloadService = Mockery::mock(UsenetDownloadService::class);
$this->expectDownloadScope($downloadService);
$downloadService->shouldReceive('download')->once()->andReturn([
'success' => false,
'data' => null,
'groupUnavailable' => true,
'error' => 'Group unavailable',
]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('processReleaseNameFromNzbContents')->once()->andReturnFalse();
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
$output->shouldReceive('echoGroupUnavailable')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
$downloadService,
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$this->successfulTempWorkspace(),
$output
);
$context = $this->makeContext();
$context->release->nfostatus = 1;
$result = $processor->process($context, '/tmp/main/');
$this->assertSame(ProcessingOutcome::GroupUnavailable, $result->outcome);
$this->assertFalse($result->isSuccessful());
$this->assertStringContainsString('unavailable', $result->reason);
}
private function makeProcessor(
@@ -147,12 +391,15 @@ class ReleaseProcessorTest extends TestCase
?TempWorkspaceService $tempWorkspace = null,
?ConsoleOutputService $output = null
): ReleaseProcessor {
$config = $this->makeConfig();
return new ReleaseProcessor(
$this->makeConfig(),
$config,
$nzbParser ?? Mockery::mock(NzbContentParser::class),
new AdditionalWorkPlanner($config),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$this->scopedDownloadService(),
$releaseManager ?? Mockery::mock(ReleaseFileManager::class),
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace ?? Mockery::mock(TempWorkspaceService::class),
@@ -171,4 +418,46 @@ class ReleaseProcessorTest extends TestCase
'pp_timeout_count' => 0,
]));
}
private function compressedNzbParser(): NzbContentParser
{
$parser = Mockery::mock(NzbContentParser::class);
$parser->shouldReceive('parseNzb')->once()->andReturn([
'error' => null,
'contents' => [['title' => 'archive.part01.rar', 'segments' => ['<archive>']]],
]);
return $parser;
}
private function scopedDownloadService(): UsenetDownloadService
{
$downloadService = Mockery::mock(UsenetDownloadService::class);
$this->expectDownloadScope($downloadService);
return $downloadService;
}
private function expectDownloadScope(UsenetDownloadService $downloadService): void
{
$downloadService->shouldReceive('beginReleaseScope')->once()->andReturnNull();
$downloadService->shouldReceive('finishReleaseScope')->once()->andReturn(new DownloadMetrics);
}
private function successfulTempWorkspace(): TempWorkspaceService
{
return Mockery::mock(TempWorkspaceService::class)
->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/')
->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull()
->getMock();
}
private function passwordOutput(): ConsoleOutputService
{
return Mockery::mock(ConsoleOutputService::class)
->shouldReceive('echoReleaseStart')->once()->andReturnNull()
->shouldReceive('setProcessTitle')->once()->andReturnNull()
->shouldReceive('echoCompressedDownload')->once()->andReturnNull()
->getMock();
}
}
@@ -0,0 +1,106 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\ReleaseSearchSyncCoordinator;
use App\Services\AdditionalProcessing\State\PersistenceMetricsCollector;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class ReleaseSearchSyncCoordinatorTest extends TestCase
{
#[Test]
public function it_coalesces_multiple_requests_for_one_active_release(): void
{
$synchronizedReleaseIds = [];
$metrics = new PersistenceMetricsCollector;
$coordinator = new ReleaseSearchSyncCoordinator(
$metrics,
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
$synchronizedReleaseIds[] = $releaseId;
},
);
$metrics->beginReleaseScope(42);
$coordinator->beginReleaseScope(42);
$coordinator->request(42);
$coordinator->request(42);
$coordinator->request(42);
$this->assertSame([], $synchronizedReleaseIds);
$coordinator->finishReleaseScope();
$persistenceMetrics = $metrics->finishReleaseScope();
$this->assertSame([42], $synchronizedReleaseIds);
$this->assertSame(3, $persistenceMetrics->searchSyncRequests);
$this->assertSame(1, $persistenceMetrics->searchSyncExecutions);
}
#[Test]
public function it_discards_a_pending_sync_when_the_release_is_deleted(): void
{
$synchronizedReleaseIds = [];
$metrics = new PersistenceMetricsCollector;
$coordinator = new ReleaseSearchSyncCoordinator(
$metrics,
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
$synchronizedReleaseIds[] = $releaseId;
},
);
$metrics->beginReleaseScope(42);
$coordinator->beginReleaseScope(42);
$coordinator->request(42);
$coordinator->discard(42);
$coordinator->finishReleaseScope();
$persistenceMetrics = $metrics->finishReleaseScope();
$this->assertSame([], $synchronizedReleaseIds);
$this->assertSame(1, $persistenceMetrics->searchSyncRequests);
$this->assertSame(0, $persistenceMetrics->searchSyncExecutions);
}
#[Test]
public function it_synchronizes_immediately_outside_an_active_release_scope(): void
{
$synchronizedReleaseIds = [];
$coordinator = new ReleaseSearchSyncCoordinator(
new PersistenceMetricsCollector,
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
$synchronizedReleaseIds[] = $releaseId;
},
);
$coordinator->request(7);
$this->assertSame([7], $synchronizedReleaseIds);
}
#[Test]
public function it_executes_every_request_immediately_when_coalescing_is_disabled(): void
{
$synchronizedReleaseIds = [];
$metrics = new PersistenceMetricsCollector;
$coordinator = new ReleaseSearchSyncCoordinator(
$metrics,
static function (int $releaseId) use (&$synchronizedReleaseIds): void {
$synchronizedReleaseIds[] = $releaseId;
},
coalesce: false,
);
$metrics->beginReleaseScope(42);
$coordinator->beginReleaseScope(42);
$coordinator->request(42);
$coordinator->request(42);
$coordinator->finishReleaseScope();
$persistenceMetrics = $metrics->finishReleaseScope();
$this->assertSame([42, 42], $synchronizedReleaseIds);
$this->assertSame(2, $persistenceMetrics->searchSyncRequests);
$this->assertSame(2, $persistenceMetrics->searchSyncExecutions);
}
}
@@ -62,6 +62,99 @@ class UsenetDownloadServiceTest extends TestCase
$this->assertStringContainsString('Group unavailable', (string) $result['error']);
}
#[Test]
public function it_reuses_exact_successful_requests_only_within_the_current_release(): void
{
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->twice()
->with(['<shared>'], false)
->andReturn('SHARED-DATA');
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$service->beginReleaseScope();
$first = $service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 10);
$second = $service->download(DownloadKind::Jpg, ['<shared>'], 'alt.binaries.test', 10);
$firstReleaseMetrics = $service->finishReleaseScope();
$this->assertSame($first, $second);
$this->assertSame(2, $firstReleaseMetrics->logicalRequests);
$this->assertSame(1, $firstReleaseMetrics->networkRequests);
$this->assertSame(1, $firstReleaseMetrics->cacheHits);
$this->assertSame(strlen('SHARED-DATA'), $firstReleaseMetrics->bytesDownloaded);
$this->assertSame(strlen('SHARED-DATA'), $firstReleaseMetrics->bytesReused);
$service->beginReleaseScope();
$service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 11);
$secondReleaseMetrics = $service->finishReleaseScope();
$this->assertSame(1, $secondReleaseMetrics->logicalRequests);
$this->assertSame(1, $secondReleaseMetrics->networkRequests);
$this->assertSame(0, $secondReleaseMetrics->cacheHits);
}
#[Test]
public function it_does_not_cache_failed_downloads(): void
{
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->twice()
->with(['<missing>'], false)
->andReturn('');
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$service->beginReleaseScope();
$service->download(DownloadKind::Compressed, ['<missing>']);
$service->download(DownloadKind::Compressed, ['<missing>']);
$metrics = $service->finishReleaseScope();
$this->assertSame(2, $metrics->networkRequests);
$this->assertSame(0, $metrics->cacheHits);
}
#[Test]
public function it_does_not_reuse_successful_downloads_outside_a_release_scope(): void
{
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->twice()
->with(['<shared>'], false)
->andReturn('SHARED-DATA');
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$first = $service->download(DownloadKind::Compressed, ['<shared>'], 'alt.binaries.test', 10);
$second = $service->download(DownloadKind::Jpg, ['<shared>'], 'alt.binaries.test', 10);
$this->assertSame('SHARED-DATA', $first['data']);
$this->assertSame('SHARED-DATA', $second['data']);
}
#[Test]
public function it_retries_an_exact_request_after_a_partial_provider_failure(): void
{
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->twice()
->with(['<retry>'], false)
->andReturn('', 'RECOVERED-DATA');
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$service->beginReleaseScope();
$failed = $service->download(DownloadKind::Compressed, ['<retry>']);
$recovered = $service->download(DownloadKind::Compressed, ['<retry>']);
$metrics = $service->finishReleaseScope();
$this->assertFalse($failed['success']);
$this->assertTrue($recovered['success']);
$this->assertSame('RECOVERED-DATA', $recovered['data']);
$this->assertSame(2, $metrics->networkRequests);
$this->assertSame(0, $metrics->cacheHits);
}
#[Test]
public function it_checks_minimum_download_sizes(): void
{