starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update additional postprocessing code

Browse Source
This commit is contained in:
DariusIII
2026-04-21 15:03:05 +02:00
parent 0891e1f00c
commit f2c06224ee
20 changed files with 1633 additions and 1125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/PostProcessGuid.php
+5 -1
View File
@@ -80,7 +80,11 @@ class PostProcessGuid extends Command
*/
private function processAdditional(string $guid): void
{
$this->additionalProcessor->start('', $guid);
try {
$this->additionalProcessor->start('', $guid);
} finally {
$this->additionalProcessor->finish();
}
}
/**
app/Console/Commands/ProcessAdditionalGuid.php
+6 -1
View File
@@ -85,7 +85,12 @@ class ProcessAdditionalGuid extends Command
}
$processor = app(AdditionalProcessingOrchestrator::class);
$ok = $processor->processSingleGuid($guid);
try {
$ok = $processor->processSingleGuid($guid);
} finally {
$processor->finish();
}
if (! $ok) {
$this->error('Processing failed or nothing processed for '.($idOpt ? 'ID '.$idOpt : 'GUID '.$guid).'.');
app/Console/Commands/UpdatePerGroup.php
+6 -1
View File
@@ -77,7 +77,12 @@ class UpdatePerGroup extends Command
// Post process the releases
$this->info("Post-processing additional for group: {$groupMySQL['name']}");
app(AdditionalProcessingOrchestrator::class)->start($groupId);
$orchestrator = app(AdditionalProcessingOrchestrator::class);
try {
$orchestrator->start($groupId);
} finally {
$orchestrator->finish();
}
$this->info("Processing NFO files for group: {$groupMySQL['name']}");
(new NfoService)->processNfoFiles(
app/Providers/AdditionalProcessingServiceProvider.php
+33 -6
View File
@@ -11,6 +11,8 @@ use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\MediaExtractionService;
use App\Services\AdditionalProcessing\NzbContentParser;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
use App\Services\AdditionalProcessing\ReleaseProcessor;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\Categorization\CategorizationService;
use App\Services\NameFixing\NameFixingService;
@@ -97,6 +99,31 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
);
});
$this->app->singleton(ReleaseFilesArchiveFallback::class, function ($app) {
return new ReleaseFilesArchiveFallback(
$app->make(ProcessingConfiguration::class),
$app->make(ArchiveExtractionService::class),
$app->make(MediaExtractionService::class),
$app->make(UsenetDownloadService::class),
$app->make(ReleaseFileManager::class),
$app->make(ConsoleOutputService::class)
);
});
$this->app->singleton(ReleaseProcessor::class, function ($app) {
return new ReleaseProcessor(
$app->make(ProcessingConfiguration::class),
$app->make(NzbContentParser::class),
$app->make(ArchiveExtractionService::class),
$app->make(MediaExtractionService::class),
$app->make(UsenetDownloadService::class),
$app->make(ReleaseFileManager::class),
$app->make(ReleaseFilesArchiveFallback::class),
$app->make(TempWorkspaceService::class),
$app->make(ConsoleOutputService::class)
);
});
// Temp workspace service (might already be registered elsewhere)
$this->app->singleton(TempWorkspaceService::class, function () {
return new TempWorkspaceService;
@@ -106,11 +133,7 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
$this->app->singleton(AdditionalProcessingOrchestrator::class, function ($app) {
return new AdditionalProcessingOrchestrator(
$app->make(ProcessingConfiguration::class),
$app->make(NzbContentParser::class),
$app->make(ArchiveExtractionService::class),
$app->make(MediaExtractionService::class),
$app->make(UsenetDownloadService::class),
$app->make(ReleaseFileManager::class),
$app->make(ReleaseProcessor::class),
$app->make(TempWorkspaceService::class),
$app->make(ConsoleOutputService::class)
);
@@ -122,6 +145,10 @@ class AdditionalProcessingServiceProvider extends ServiceProvider
*/
public function boot(): void
{
//
$this->app->terminating(function (): void {
if ($this->app->bound(AdditionalProcessingOrchestrator::class)) {
$this->app->make(AdditionalProcessingOrchestrator::class)->finish();
}
});
}
}
app/Services/AdditionalProcessing/AdditionalProcessingOrchestrator.php
+44 -953
View File
File diff suppressed because it is too large Load Diff
app/Services/AdditionalProcessing/ArchiveExtractionService.php
+31 -61
View File
@@ -23,10 +23,12 @@ class ArchiveExtractionService
private Par2Info $par2Info;
public function __construct(
private readonly ProcessingConfiguration $config
private readonly ProcessingConfiguration $config,
?ArchiveInfo $archiveInfo = null,
?Par2Info $par2Info = null
) {
$this->archiveInfo = new ArchiveInfo;
$this->par2Info = new Par2Info;
$this->archiveInfo = $archiveInfo ?? new ArchiveInfo;
$this->par2Info = $par2Info ?? new Par2Info;
// Configure external clients for ArchiveInfo
if ($this->config->unrarPath) {
@@ -341,7 +343,14 @@ class ArchiveExtractionService
// Try using unrar for RAR files
if ($this->config->unrarPath) {
$extracted = $this->extractFileViaUnrar($compressedData, $filename, $tmpPath);
$extracted = $this->extractFileViaExternalTool(
$compressedData,
$filename,
$tmpPath,
'rar',
fn (string $archiveFile, string $extractDir): string => $this->config->getKillString().
$this->config->unrarPath.'" e -y -c- -inul -p- "'.$archiveFile.'" "'.$filename.'" "'.$extractDir.'"'
);
if ($extracted !== null) {
return $extracted;
}
@@ -349,7 +358,14 @@ class ArchiveExtractionService
// Try using unzip for ZIP files
if ($this->config->unzipPath) {
$extracted = $this->extractFileViaUnzip($compressedData, $filename, $tmpPath);
$extracted = $this->extractFileViaExternalTool(
$compressedData,
$filename,
$tmpPath,
'zip',
fn (string $archiveFile, string $extractDir): string => $this->config->unzipPath.
' -j "'.$archiveFile.'" "'.$filename.'" -d "'.$extractDir.'"'
);
if ($extracted !== null) {
return $extracted;
}
@@ -361,20 +377,23 @@ class ArchiveExtractionService
/**
* Extract a specific file using unrar.
*/
private function extractFileViaUnrar(string $compressedData, string $filename, string $tmpPath): ?string
{
private function extractFileViaExternalTool(
string $compressedData,
string $filename,
string $tmpPath,
string $archiveExtension,
callable $commandBuilder
): ?string {
try {
$extractDir = $tmpPath.'extract_'.uniqid('', true).'/';
if (! File::isDirectory($extractDir)) {
File::makeDirectory($extractDir, 0777, true, true);
}
$archiveFile = $tmpPath.'archive_'.uniqid('', true).'.rar';
$archiveFile = $tmpPath.'archive_'.uniqid('', true).'.'.$archiveExtension;
File::put($archiveFile, $compressedData);
// Extract specific file using unrar
$killString = $this->config->getKillString();
runCmd($killString.$this->config->unrarPath.'" e -y -c- -inul -p- "'.$archiveFile.'" "'.$filename.'" "'.$extractDir.'"');
runCmd($commandBuilder($archiveFile, $extractDir));
File::delete($archiveFile);
@@ -401,56 +420,7 @@ class ArchiveExtractionService
File::deleteDirectory($extractDir);
} catch (\Throwable $e) {
if ($this->config->debugMode) {
Log::debug('Unrar extraction failed: '.$e->getMessage());
}
}
return null;
}
/**
* Extract a specific file using unzip.
*/
private function extractFileViaUnzip(string $compressedData, string $filename, string $tmpPath): ?string
{
try {
$extractDir = $tmpPath.'extract_'.uniqid('', true).'/';
if (! File::isDirectory($extractDir)) {
File::makeDirectory($extractDir, 0777, true, true);
}
$archiveFile = $tmpPath.'archive_'.uniqid('', true).'.zip';
File::put($archiveFile, $compressedData);
// Extract specific file using unzip
runCmd($this->config->unzipPath.' -j "'.$archiveFile.'" "'.$filename.'" -d "'.$extractDir.'"');
File::delete($archiveFile);
// Look for extracted file
$extractedPath = $extractDir.basename($filename);
if (File::isFile($extractedPath)) {
$content = File::get($extractedPath);
File::deleteDirectory($extractDir);
return $content;
}
// Try to find it with glob
$files = File::allFiles($extractDir);
foreach ($files as $file) {
if (strtolower($file->getFilename()) === strtolower(basename($filename))) {
$content = File::get($file->getPathname());
File::deleteDirectory($extractDir);
return $content;
}
}
File::deleteDirectory($extractDir);
} catch (\Throwable $e) {
if ($this->config->debugMode) {
Log::debug('Unzip extraction failed: '.$e->getMessage());
Log::debug('External extraction failed: '.$e->getMessage());
}
}
app/Services/AdditionalProcessing/ConsoleOutputService.php
+7
View File
@@ -56,6 +56,13 @@ class ConsoleOutputService
$this->echo(PHP_EOL.'['.$releaseId.']['.human_filesize($size, 1).']', 'primaryOver');
}
public function setProcessTitle(int $releaseId): void
{
if (function_exists('cli_set_process_title')) {
@cli_set_process_title(PHP_BINARY.' '.__DIR__.'/AdditionalProcessingOrchestrator.php ReleaseID: '.$releaseId);
}
}
public function echoCompressedDownload(): void
{
$this->echo('(cB)', 'primaryOver');
app/Services/AdditionalProcessing/DTO/ReleaseProcessingContext.php
+23 -6
View File
@@ -47,33 +47,33 @@ class ReleaseProcessingContext
public bool $nzbHasCompressedFile = false;
/**
* @var array<string, mixed>
* @var list<array<string, mixed>>
*/
public array $nzbContents = [];
// Message IDs for downloading
/**
* @var array<string, mixed>
* @var list<string>
*/
public array $sampleMessageIDs = [];
/**
* @var array<string, mixed>
* @var list<string>
*/
public array $jpgMessageIDs = [];
/**
* @var array<string, mixed>
* @var list<string>|string
*/
public string|array $mediaInfoMessageIDs = [];
/**
* @var array<string, mixed>
* @var list<string>|string
*/
public string|array $audioInfoMessageIDs = [];
/**
* @var array<string, mixed>
* @var list<string>
*/
public array $rarFileMessageIDs = [];
@@ -155,6 +155,23 @@ class ReleaseProcessingContext
$this->resetCounters();
}
/**
* Mark a processing artifact as found.
*/
public function markFound(string $type): void
{
match ($type) {
'video' => $this->foundVideo = true,
'mediaInfo' => $this->foundMediaInfo = true,
'audioInfo' => $this->foundAudioInfo = true,
'audioSample' => $this->foundAudioSample = true,
'jpgSample' => $this->foundJPGSample = true,
'sample' => $this->foundSample = true,
'par2Info' => $this->foundPAR2Info = true,
default => null,
};
}
/**
* Check if more media processing is needed.
*/
app/Services/AdditionalProcessing/Enums/DownloadKind.php
+14
View File
@@ -0,0 +1,14 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing\Enums;
enum DownloadKind: string
{
case Sample = 'sample';
case MediaInfo = 'media-info';
case Audio = 'audio';
case Jpg = 'jpg';
case Compressed = 'compressed';
}
app/Services/AdditionalProcessing/NzbContentParser.php
+12 -21
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\Nzb\NzbParserService;
use App\Services\Nzb\NzbService;
use Illuminate\Support\Facades\File;
@@ -160,15 +161,7 @@ class NzbContentParser
public function extractMessageIDs(
array $nzbContents,
string $groupName,
int $segmentsToDownload,
bool $processThumbnails,
bool $processJPGSample,
bool $processMediaInfo,
bool $processAudioInfo,
string $audioFileRegex,
string $videoFileRegex,
string $supportFileRegex,
string $ignoreBookRegex
ProcessingConfiguration $config
): array {
$result = [
'hasCompressedFile' => false,
@@ -186,7 +179,7 @@ class NzbContentParser
$segments = $file['segments'] ?? [];
// Skip support/nfo files
if (preg_match('/(?:'.$supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $title)) {
if (preg_match('/(?:'.$config->supportFileRegex.'|nfo\\b|inf\\b|ofn\\b)($|[ ")]|-])(?!.{20,})/i', $title)) {
continue;
}
@@ -199,39 +192,39 @@ class NzbContentParser
}
// Look for a video sample (not an image)
if ($processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments)
if ($config->processThumbnails && empty($result['sampleMessageIDs']) && ! empty($segments)
&& stripos($title, 'sample') !== false
&& ! preg_match('/\.jpe?g$/i', $title)
) {
$result['sampleMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload);
$result['sampleMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
}
// Look for a JPG picture (not a CD cover)
if ($processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments)
if ($config->processJPGSample && empty($result['jpgMessageIDs']) && ! empty($segments)
&& ! preg_match('/flac|lossless|mp3|music|inner-sanctum|sound/i', $groupName)
&& preg_match('/\.jpe?g[. ")\]]/i', $title)
) {
$result['jpgMessageIDs'] = $this->extractSegments($segments, $segmentsToDownload);
$result['jpgMessageIDs'] = $this->extractSegments($segments, $config->segmentsToDownload);
}
// Look for a video file for MediaInfo (sample video)
if ($processMediaInfo && empty($result['mediaInfoMessageID']) && ! empty($segments[0])
if ($config->processMediaInfo && empty($result['mediaInfoMessageID']) && ! empty($segments[0])
&& stripos($title, 'sample') !== false
&& preg_match('/'.$videoFileRegex.'[. ")\]]/i', $title)
&& preg_match('/'.$config->videoFileRegex.'[. ")\]]/i', $title)
) {
$result['mediaInfoMessageID'] = (string) $segments[0];
}
// Look for an audio file
if ($processAudioInfo && empty($result['audioInfoMessageID']) && ! empty($segments)
&& preg_match('/'.$audioFileRegex.'[. ")\]]/i', $title, $type)
if ($config->processAudioInfo && empty($result['audioInfoMessageID']) && ! empty($segments)
&& preg_match('/'.$config->audioFileRegex.'[. ")\]]/i', $title, $type)
) {
$result['audioInfoExtension'] = $type[1];
$result['audioInfoMessageID'] = (string) $segments[0];
}
// Count book files
if (preg_match($ignoreBookRegex, $title)) {
if (preg_match($config->ignoreBookRegex, $title)) {
$result['bookFileCount']++;
}
} catch (\ErrorException $e) {
@@ -264,8 +257,6 @@ class NzbContentParser
/**
* Get the NZB path for a GUID.
*
* @return list<string>
*/
public function getNzbPath(string $guid): string|false
{
app/Services/AdditionalProcessing/ReleaseFileManager.php
+5 -5
View File
@@ -175,7 +175,7 @@ class ReleaseFileManager
// Get file count
$releaseFilesCount = ReleaseFile::whereReleasesId($context->release->id)->count('releases_id') ?? 0;
$passwordStatus = max([$context->passwordStatus]);
$passwordStatus = $context->passwordStatus;
// Set to no password if processing is off
if (! $processPasswords) {
@@ -514,8 +514,8 @@ class ReleaseFileManager
if ($extractedName !== null) {
$preCheck = Predb::whereTitle($extractedName)->first();
$context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0;
$candidate = $preCheck->title ?? $extractedName;
$context->release->preid = $preCheck?->id ?? 0;
$candidate = $preCheck?->title ?? $extractedName;
$candidate = $this->normalizeCandidateTitle($candidate);
if ($this->isPlausibleReleaseTitle($candidate)) {
@@ -540,8 +540,8 @@ class ReleaseFileManager
if ($extractedName !== null) {
$preCheck = Predb::whereTitle($extractedName)->first();
$context->release->preid = $preCheck !== null ? $preCheck->value('id') : 0;
$candidate = $preCheck->title ?? $extractedName;
$context->release->preid = $preCheck?->id ?? 0;
$candidate = $preCheck?->title ?? $extractedName;
$candidate = $this->normalizeCandidateTitle($candidate);
if ($this->isPlausibleReleaseTitle($candidate)) {
app/Services/AdditionalProcessing/ReleaseFilesArchiveFallback.php
+283
View File
@@ -0,0 +1,283 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Models\ReleaseFile;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Support\Facades\File;
/**
* Shared fallback logic for extracting known files back out of downloaded archives.
*/
class ReleaseFilesArchiveFallback
{
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly ArchiveExtractionService $archiveService,
private readonly MediaExtractionService $mediaService,
private readonly UsenetDownloadService $downloadService,
private readonly ReleaseFileManager $releaseManager,
private readonly ConsoleOutputService $output
) {}
/**
* Try JPG/PNG files listed inside an already-downloaded archive summary.
*
* @param array<int, array<string, mixed>> $files
*/
public function processJpgFromArchiveFileList(
string $compressedData,
array $files,
ReleaseProcessingContext $context
): void {
$imageFiles = array_values(array_filter(
$files,
static fn (array $file): bool => preg_match('/\.(jpe?g|png)$/i', (string) ($file['name'] ?? '')) === 1
));
if ($imageFiles === []) {
return;
}
usort($imageFiles, static fn (array $a, array $b): int => ((int) ($b['size'] ?? 0)) <=> ((int) ($a['size'] ?? 0)));
$this->processExtractedCandidatesFromData(
$compressedData,
array_slice($imageFiles, 0, 3),
$context,
'extracted_',
function (string $tempPath, ReleaseProcessingContext $context): bool {
if (! $this->mediaService->isValidImage($tempPath)) {
return false;
}
if (! $this->mediaService->getJPGSample($tempPath, $context->release->guid)) {
return false;
}
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
return true;
}
);
}
/**
* Re-download archives and extract stored JPG/PNG candidates.
*/
public function processJpgFromReleaseFiles(ReleaseProcessingContext $context): void
{
$imageFiles = ReleaseFile::query()
->where('releases_id', $context->release->id)
->where(function ($query) {
$query->where('name', 'like', '%.jpg')
->orWhere('name', 'like', '%.jpeg')
->orWhere('name', 'like', '%.png');
})
->orderByDesc('size')
->limit(3)
->get();
if ($imageFiles->isEmpty()) {
return;
}
$this->processStoredReleaseFiles(
$context,
$imageFiles,
'release_file_',
static fn (ReleaseProcessingContext $context): bool => $context->foundJPGSample,
function (string $tempPath, ReleaseProcessingContext $context): bool {
if (! $this->mediaService->isValidImage($tempPath)) {
return false;
}
if (! $this->mediaService->getJPGSample($tempPath, $context->release->guid)) {
return false;
}
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
return true;
}
);
}
/**
* Re-download archives and extract stored NFO-like candidates.
*/
public function processNfoFromReleaseFiles(ReleaseProcessingContext $context): void
{
$nfoFiles = ReleaseFile::query()
->where('releases_id', $context->release->id)
->nfoFiles()
->limit(3)
->get();
if ($nfoFiles->isEmpty()) {
return;
}
$nntp = $this->downloadService->getNNTP();
$this->processStoredReleaseFiles(
$context,
$nfoFiles,
'release_file_',
static fn (ReleaseProcessingContext $context): bool => ! $context->releaseHasNoNFO,
function (string $tempPath, ReleaseProcessingContext $context) use ($nntp): bool {
if (! $this->releaseManager->processNfoFile($tempPath, $context, $nntp)) {
return false;
}
$this->output->echoNfoFound();
return true;
}
);
}
/**
* @param EloquentCollection<int, ReleaseFile> $candidates
* @param callable(ReleaseProcessingContext): bool $shouldStop
* @param callable(string, ReleaseProcessingContext): bool $handler
*/
private function processStoredReleaseFiles(
ReleaseProcessingContext $context,
EloquentCollection $candidates,
string $tempPrefix,
callable $shouldStop,
callable $handler
): void {
if ($candidates->isEmpty() || $context->nzbContents === []) {
return;
}
foreach ($context->nzbContents as $nzbFile) {
if ($shouldStop($context)) {
break;
}
$title = (string) ($nzbFile['title'] ?? '');
if (preg_match(
'/(\.(part0*1|rar|zip))(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
$title
) !== 1) {
continue;
}
$messageIDs = $this->extractInitialSegments($nzbFile['segments'] ?? []);
if ($messageIDs === []) {
continue;
}
$result = $this->downloadService->download(
DownloadKind::Compressed,
$messageIDs,
$context->releaseGroupName,
$context->release->id,
$title
);
if ($result['groupUnavailable']) {
$context->groupUnavailable = true;
$this->output->echoGroupUnavailable();
return;
}
if (! $result['success'] || empty($result['data'])) {
continue;
}
foreach ($candidates as $candidate) {
$fileData = $this->archiveService->extractSpecificFile(
$result['data'],
$candidate->name,
$context->tmpPath
);
if ($fileData === null || $fileData === '') {
continue;
}
if ($this->withExtractedTempFile($candidate->name, $fileData, $context, $tempPrefix, $handler)) {
return;
}
}
}
}
/**
* @param array<int, array<string, mixed>> $candidates
* @param callable(string, ReleaseProcessingContext): bool $handler
*/
private function processExtractedCandidatesFromData(
string $compressedData,
array $candidates,
ReleaseProcessingContext $context,
string $tempPrefix,
callable $handler
): void {
foreach ($candidates as $candidate) {
$filename = (string) ($candidate['name'] ?? '');
if ($filename === '') {
continue;
}
$fileData = $this->archiveService->extractSpecificFile($compressedData, $filename, $context->tmpPath);
if ($fileData === null || $fileData === '') {
continue;
}
if ($this->withExtractedTempFile($filename, $fileData, $context, $tempPrefix, $handler)) {
return;
}
}
}
/**
* @param callable(string, ReleaseProcessingContext): bool $handler
*/
private function withExtractedTempFile(
string $filename,
string $fileData,
ReleaseProcessingContext $context,
string $tempPrefix,
callable $handler
): bool {
$extension = strtolower(pathinfo($filename, PATHINFO_EXTENSION));
$suffix = $extension === '' ? '' : '.'.$extension;
$tempPath = $context->tmpPath.$tempPrefix.uniqid('', true).$suffix;
File::put($tempPath, $fileData);
try {
return $handler($tempPath, $context);
} finally {
File::delete($tempPath);
}
}
/**
* @param array<int, mixed> $segments
* @return list<string>
*/
private function extractInitialSegments(array $segments): array
{
$messageIDs = [];
$segmentCount = min(count($segments), $this->config->maximumRarSegments);
for ($index = 0; $index < $segmentCount; $index++) {
$messageIDs[] = (string) $segments[$index];
}
return $messageIDs;
}
}
app/Services/AdditionalProcessing/ReleaseProcessor.php
+623
View File
@@ -0,0 +1,623 @@
<?php
declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Models\UsenetGroup;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\Releases\ReleaseBrowseService;
use App\Services\TempWorkspaceService;
use Illuminate\Support\Facades\File;
/**
* Stateless release processor for additional post-processing.
*/
class ReleaseProcessor
{
public function __construct(
private readonly ProcessingConfiguration $config,
private readonly NzbContentParser $nzbParser,
private readonly ArchiveExtractionService $archiveService,
private readonly MediaExtractionService $mediaService,
private readonly UsenetDownloadService $downloadService,
private readonly ReleaseFileManager $releaseManager,
private readonly ReleaseFilesArchiveFallback $archiveFallback,
private readonly TempWorkspaceService $tempWorkspace,
private readonly ConsoleOutputService $output
) {}
public function process(ReleaseProcessingContext $context, string $mainTmpPath): void
{
$release = $context->release;
$this->output->echoReleaseStart($release->id, $release->size);
$this->output->setProcessTitle((int) $release->id);
try {
$context->tmpPath = $this->tempWorkspace->createReleaseTempFolder($mainTmpPath, $release->guid);
} catch (\Throwable $e) {
$this->output->warning('Unable to create directory: '.$e->getMessage());
return;
}
try {
$nzbResult = $this->nzbParser->parseNzb($release->guid);
if ($nzbResult['error'] !== null) {
$this->output->warning($nzbResult['error']);
$this->releaseManager->deleteRelease($release);
return;
}
$context->nzbContents = array_values($nzbResult['contents']);
if ($this->checkReleaseTimeout($context)) {
return;
}
$this->initializeContext($context);
$messageIds = $this->nzbParser->extractMessageIDs($context->nzbContents, $context->releaseGroupName, $this->config);
$context->nzbHasCompressedFile = $messageIds['hasCompressedFile'];
$context->sampleMessageIDs = $messageIds['sampleMessageIDs'];
$context->jpgMessageIDs = $messageIds['jpgMessageIDs'];
$context->mediaInfoMessageIDs = $messageIds['mediaInfoMessageID'];
$context->audioInfoMessageIDs = $messageIds['audioInfoMessageID'];
$context->audioInfoExtension = $messageIds['audioInfoExtension'];
$bookFlood = $messageIds['bookFileCount'] > 80
&& ($messageIds['bookFileCount'] * 2) >= count($context->nzbContents);
if ($this->shouldProcessDownloads()) {
$this->processMessageIdDownloads($context);
if ($this->checkReleaseTimeout($context)) {
return;
}
if (! $bookFlood && $context->nzbHasCompressedFile) {
$triedCompressedMids = [];
$this->processNzbCompressedFiles($context, false, $triedCompressedMids);
if ($this->checkReleaseTimeout($context)) {
return;
}
if ($this->config->fetchLastFiles) {
$this->processNzbCompressedFiles($context, true, $triedCompressedMids);
if ($this->checkReleaseTimeout($context)) {
return;
}
}
if (! $context->releaseHasPassword) {
$this->processExtractedFiles($context);
if ($this->checkReleaseTimeout($context)) {
return;
}
}
}
if (! $context->foundJPGSample && $this->config->processJPGSample) {
$this->archiveFallback->processJpgFromReleaseFiles($context);
}
if ($context->releaseHasNoNFO) {
$this->archiveFallback->processNfoFromReleaseFiles($context);
}
}
$this->releaseManager->finalizeRelease($context, $this->config->processPasswords);
} finally {
if ($context->tmpPath !== '') {
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
}
}
}
private function shouldProcessDownloads(): bool
{
return $this->config->processPasswords
|| $this->config->processThumbnails
|| $this->config->processMediaInfo
|| $this->config->processAudioInfo
|| $this->config->processVideo
|| $this->config->processJPGSample;
}
private function initializeContext(ReleaseProcessingContext $context): void
{
$context->initializeFromConfig(
$this->config->processVideo,
$this->config->processMediaInfo,
$this->config->processAudioInfo,
$this->config->processAudioSample,
$this->config->processJPGSample,
$this->config->processThumbnails
);
$context->passwordStatus = ReleaseBrowseService::PASSWD_NONE;
$context->releaseHasPassword = false;
try {
$context->releaseGroupName = UsenetGroup::getNameByID($context->release->groups_id);
} catch (\Throwable) {
$context->releaseGroupName = '';
}
$context->releaseHasNoNFO = (int) $context->release->nfostatus !== 1;
$context->resetMessageIDs();
$context->resetCounters();
}
private function checkReleaseTimeout(ReleaseProcessingContext $context): bool
{
if (! $context->isTimedOut($this->config->releaseProcessingTimeout)) {
return false;
}
$deleted = $this->releaseManager->handleReleaseTimeout(
$context->release,
$this->config->maxPpTimeoutCount
);
if ($deleted) {
$this->output->echoReleaseTimeoutDeleted(
$context->release->id,
(int) ($context->release->pp_timeout_count ?? 0) + 1
);
} else {
$this->output->echoReleaseTimeout($context->release->id, $context->getElapsedSeconds());
}
if ($context->tmpPath !== '') {
$this->tempWorkspace->clearDirectory($context->tmpPath, false);
}
return true;
}
private function processMessageIdDownloads(ReleaseProcessingContext $context): void
{
if ((! $context->foundSample || ! $context->foundVideo) && $context->sampleMessageIDs !== []) {
$result = $this->downloadService->download(
DownloadKind::Sample,
$context->sampleMessageIDs,
$context->releaseGroupName,
$context->release->id
);
if ($result['success'] && is_string($result['data']) && $this->downloadService->meetsMinimumSize($result['data'])) {
$this->output->echoSampleDownload();
$fileLocation = $context->tmpPath.'sample_'.random_int(0, 99999).'.avi';
File::put($fileLocation, $result['data']);
if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('sample');
$this->output->echoSampleCreated();
}
if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('video');
$this->output->echoVideoCreated();
}
} elseif (! $result['success']) {
$this->output->echoSampleFailure();
}
}
if ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo)
&& ! empty($context->mediaInfoMessageIDs)
) {
$result = $this->downloadService->download(
DownloadKind::MediaInfo,
$context->mediaInfoMessageIDs,
$context->releaseGroupName,
$context->release->id
);
if ($result['success'] && is_string($result['data']) && $this->downloadService->meetsMinimumSize($result['data'])) {
$this->output->echoMediaInfoDownload();
$fileLocation = $context->tmpPath.'media.avi';
File::put($fileLocation, $result['data']);
if (! $context->foundMediaInfo && $this->mediaService->getMediaInfo($fileLocation, $context->release->id)) {
$context->markFound('mediaInfo');
$this->output->echoMediaInfoAdded();
}
if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('sample');
$this->output->echoSampleCreated();
}
if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('video');
$this->output->echoVideoCreated();
}
} elseif (! $result['success']) {
$this->output->echoMediaInfoFailure();
}
}
if ((! $context->foundAudioInfo || ! $context->foundAudioSample)
&& ! empty($context->audioInfoMessageIDs)
) {
$result = $this->downloadService->download(
DownloadKind::Audio,
$context->audioInfoMessageIDs,
$context->releaseGroupName,
$context->release->id
);
if ($result['success'] && is_string($result['data'])) {
$this->output->echoAudioDownload();
$fileLocation = $context->tmpPath.'audio.'.$context->audioInfoExtension;
File::put($fileLocation, $result['data']);
$audioResult = $this->mediaService->getAudioInfo(
$fileLocation,
$context->audioInfoExtension,
$context,
$context->tmpPath
);
if ($audioResult['audioInfo']) {
$this->output->echoAudioInfoAdded();
}
if ($audioResult['audioSample']) {
$this->output->echoAudioSampleCreated();
}
} elseif (! $result['success']) {
$this->output->echoAudioFailure();
}
}
if (! $context->foundJPGSample && $context->jpgMessageIDs !== []) {
$result = $this->downloadService->download(
DownloadKind::Jpg,
$context->jpgMessageIDs,
$context->releaseGroupName,
$context->release->id
);
if ($result['success'] && is_string($result['data'])) {
$this->output->echoJpgDownload();
$fileLocation = $context->tmpPath.'samplepicture.jpg';
File::put($fileLocation, $result['data']);
if ($this->mediaService->isJpegData($fileLocation)
&& $this->mediaService->getJPGSample($fileLocation, $context->release->guid)
) {
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
}
File::delete($fileLocation);
} elseif (! $result['success']) {
$this->output->echoJpgFailure();
}
}
}
/**
* @param list<string> $triedCompressedMids
*/
private function processNzbCompressedFiles(
ReleaseProcessingContext $context,
bool $reverse,
array &$triedCompressedMids
): void {
if ($context->groupUnavailable) {
return;
}
$nzbContents = $context->nzbContents;
if ($reverse) {
krsort($nzbContents);
} else {
$triedCompressedMids = [];
}
$failed = $downloaded = 0;
foreach ($nzbContents as $nzbFile) {
if ($downloaded >= $this->config->maximumRarSegments
|| $failed >= $this->config->maximumRarPasswordChecks
|| $context->releaseHasPassword
|| $context->groupUnavailable
) {
break;
}
$title = (string) ($nzbFile['title'] ?? '');
if (preg_match(
'/(\.(part\d+|[rz]\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")]|-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$)/i',
$title
) !== 1) {
continue;
}
$segments = $nzbFile['segments'] ?? [];
$messageIDs = [];
$segmentCount = count($segments) - 1;
foreach (range(0, $this->config->maximumRarSegments - 1) as $index) {
if ($index > $segmentCount) {
break;
}
$segment = (string) $segments[$index];
if (! $reverse) {
$triedCompressedMids[] = $segment;
} elseif (in_array($segment, $triedCompressedMids, false)) {
continue 2;
}
$messageIDs[] = $segment;
}
if ($messageIDs === []) {
continue;
}
$result = $this->downloadService->download(
DownloadKind::Compressed,
$messageIDs,
$context->releaseGroupName,
$context->release->id,
$title
);
if ($result['groupUnavailable']) {
$context->groupUnavailable = true;
$this->output->echoGroupUnavailable();
break;
}
if ($result['success'] && is_string($result['data'])) {
$this->output->echoCompressedDownload();
$downloaded++;
$processed = $this->processCompressedData($result['data'], $context, $reverse);
if ($processed || $context->releaseHasPassword) {
break;
}
} else {
$failed++;
$this->output->echoCompressedFailure($failed);
}
}
}
private function processCompressedData(
string $compressedData,
ReleaseProcessingContext $context,
bool $reverse
): bool {
$result = $this->archiveService->processCompressedData(
$compressedData,
$context,
$context->tmpPath
);
if ($result['hasPassword']) {
$context->releaseHasPassword = true;
$context->passwordStatus = $result['passwordStatus'];
return false;
}
if (isset($result['standaloneVideoType'])) {
$this->output->echoInlineVideo();
$fileLocation = $context->tmpPath.'inline_video_'.uniqid('', true).'.'.$result['standaloneVideoType'];
File::put($fileLocation, $result['standaloneVideoData']);
if (! $context->foundMediaInfo && $this->mediaService->getMediaInfo($fileLocation, $context->release->id)) {
$context->markFound('mediaInfo');
$this->output->echoMediaInfoAdded();
}
if (! $context->foundSample && $this->mediaService->getSample($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('sample');
$this->output->echoSampleCreated();
}
if (! $context->foundVideo && $this->mediaService->getVideo($fileLocation, $context->tmpPath, $context->release->guid)) {
$context->markFound('video');
$this->output->echoVideoCreated();
}
return $context->foundMediaInfo || $context->foundSample || $context->foundVideo;
}
if (! $result['success']) {
return false;
}
if (! empty($result['archiveMarker'])) {
$this->output->echoArchiveMarker($result['archiveMarker']);
}
if ($reverse && ! empty($result['dataSummary'])) {
$this->releaseManager->processReleaseNameFromRar($result['dataSummary'], $context);
}
foreach ($result['files'] as $file) {
if ($context->releaseHasPassword) {
break;
}
if ($this->releaseManager->addFileInfo($file, $context, $this->config->supportFileRegex)) {
$this->output->echoFileInfoAdded();
}
}
if ($context->addedFileInfo > 0) {
$this->releaseManager->updateSearchIndex($context->release->id);
}
if (! $context->foundJPGSample && $this->config->processJPGSample) {
$this->archiveFallback->processJpgFromArchiveFileList($compressedData, $result['files'], $context);
}
return $context->totalFileInfo > 0;
}
private function processExtractedFiles(ReleaseProcessingContext $context): void
{
$nestedLevels = 0;
while ($nestedLevels < $this->config->maxNestedLevels) {
if ($context->compressedFilesChecked >= AdditionalProcessingOrchestrator::MAX_COMPRESSED_FILES_TO_CHECK) {
break;
}
$foundCompressed = false;
$pattern = '/.*\.([rz]\d{2,}|rar|zipx?|0{0,2}1)($|[^a-z0-9])/i';
try {
$files = $this->tempWorkspace->listFiles($context->tmpPath, $pattern);
} catch (\Throwable) {
break;
}
foreach ($files as $file) {
$filePath = is_array($file) ? $file[0] : $file->getPathname();
if (! File::isFile($filePath)) {
continue;
}
$rarData = @File::get($filePath);
if (! empty($rarData)) {
$this->processCompressedData($rarData, $context, false);
$foundCompressed = true;
}
File::delete($filePath);
}
if (! $foundCompressed) {
break;
}
$nestedLevels++;
}
try {
$files = $this->tempWorkspace->listFiles($context->tmpPath);
} catch (\Throwable) {
return;
}
foreach ($files as $file) {
$filePath = is_object($file) ? $file->getPathname() : $file;
if (preg_match('/[\/\\\\]\.{1,2}$/', $filePath) === 1 || ! File::isFile($filePath)) {
continue;
}
if (! $context->foundPAR2Info && preg_match('/\.par2$/i', $filePath) === 1) {
$this->releaseManager->processPar2File(
$filePath,
$context,
$this->archiveService->getPar2Info()
);
continue;
}
if ($context->releaseHasNoNFO) {
if (preg_match('/(\.(nfo|inf|ofn|diz)|info\.txt)$/i', $filePath) === 1) {
if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) {
$this->output->echoNfoFound();
}
continue;
}
if ($this->releaseManager->isNfoFilename($filePath)) {
if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) {
$this->output->echoNfoFound();
}
continue;
}
}
if ((! $context->foundAudioInfo || ! $context->foundAudioSample)
&& preg_match('/(.*)'.$this->config->audioFileRegex.'$/i', $filePath, $fileType) === 1
) {
$audioPath = $context->tmpPath.'audiofile.'.$fileType[2];
File::move($filePath, $audioPath);
$audioResult = $this->mediaService->getAudioInfo(
$audioPath,
$fileType[2],
$context,
$context->tmpPath
);
if ($audioResult['audioInfo']) {
$this->output->echoAudioInfoAdded();
}
if ($audioResult['audioSample']) {
$this->output->echoAudioSampleCreated();
}
File::delete($audioPath);
continue;
}
if (! $context->foundJPGSample && preg_match('/\.(jpe?g|png)$/i', $filePath) === 1) {
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
}
File::delete($filePath);
continue;
}
if ((! $context->foundSample || ! $context->foundVideo || ! $context->foundMediaInfo)
&& preg_match('/(.*)'.$this->config->videoFileRegex.'$/i', $filePath) === 1
) {
$this->mediaService->processVideoFile($filePath, $context, $context->tmpPath);
continue;
}
$output = fileInfo($filePath);
if ($output === '' || $output === null) {
continue;
}
if (! $context->foundJPGSample && preg_match('/^JPE?G|^PNG/i', $output) === 1) {
if ($this->mediaService->getJPGSample($filePath, $context->release->guid)) {
$context->markFound('jpgSample');
$this->output->echoJpgSaved();
}
File::delete($filePath);
} elseif ((! $context->foundMediaInfo || ! $context->foundSample || ! $context->foundVideo)
&& preg_match('/Matroska data|MPEG v4|MPEG sequence, v2|\WAVI\W/i', $output) === 1
) {
$this->mediaService->processVideoFile($filePath, $context, $context->tmpPath);
} elseif ((! $context->foundAudioSample || ! $context->foundAudioInfo)
&& preg_match('/^FLAC|layer III|Vorbis audio/i', $output, $audioType) === 1
) {
$extension = match ($audioType[0]) {
'FLAC' => 'FLAC',
'layer III' => 'MP3',
'Vorbis audio' => 'OGG',
default => 'audio',
};
$audioPath = $context->tmpPath.'audiofile.'.$extension;
File::move($filePath, $audioPath);
$audioResult = $this->mediaService->getAudioInfo($audioPath, $extension, $context, $context->tmpPath);
if ($audioResult['audioInfo']) {
$this->output->echoAudioInfoAdded();
}
if ($audioResult['audioSample']) {
$this->output->echoAudioSampleCreated();
}
File::delete($audioPath);
} elseif (! $context->foundPAR2Info && stripos($output, 'Parity') === 0) {
$this->releaseManager->processPar2File(
$filePath,
$context,
$this->archiveService->getPar2Info()
);
}
}
}
}
app/Services/AdditionalProcessing/UsenetDownloadService.php
+13 -69
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace App\Services\AdditionalProcessing;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\NNTP\NNTPService;
use Exception;
use Illuminate\Support\Facades\Log;
@@ -18,9 +19,10 @@ class UsenetDownloadService
private NNTPService $nntp;
public function __construct(
private readonly ProcessingConfiguration $config
private readonly ProcessingConfiguration $config,
?NNTPService $nntp = null
) {
$this->nntp = new NNTPService;
$this->nntp = $nntp ?? new NNTPService;
}
/**
@@ -108,80 +110,21 @@ class UsenetDownloadService
}
/**
* Download sample video content.
* Download content for a specific processing step.
*
* @param array<string, mixed> $messageIDs
* @return array<string, mixed>
*/
public function downloadSample(
array $messageIDs,
string $groupName = '',
?int $releaseId = null
): array {
return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId);
}
/**
* Download media info video content.
*
* @param array<string, mixed> $messageID
* @return array<string, mixed>
*/
public function downloadMediaInfo(
string|array $messageID,
string $groupName = '',
?int $releaseId = null
): array {
return $this->downloadByMessageIDs($messageID, $groupName, $releaseId);
}
/**
* Download audio content.
*
* @param array<string, mixed> $messageID
* @return array<string, mixed>
*/
public function downloadAudio(
string|array $messageID,
string $groupName = '',
?int $releaseId = null
): array {
return $this->downloadByMessageIDs($messageID, $groupName, $releaseId);
}
/**
* Download JPG content.
*
* @param array<string, mixed> $messageIDs
* @return array<string, mixed>
*/
public function downloadJPG(
array $messageIDs,
string $groupName = '',
?int $releaseId = null
): array {
return $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId);
}
/**
* Download compressed file content (RAR, ZIP, etc.).
*
* @param array<string, mixed> $messageIDs Message IDs to download
* @param string $groupName Group name for logging
* @param int|null $releaseId Release ID for logging
* @param string|null $fileTitle File title for logging
* @param array<string, mixed>|string $messageIDs
* @return array{success: bool, data: string|null, groupUnavailable: bool, error: string|null}
*
* @throws Exception
*/
public function downloadCompressedFile(
array $messageIDs,
public function download(
DownloadKind $kind,
array|string $messageIDs,
string $groupName = '',
?int $releaseId = null,
?string $fileTitle = null
): array {
if ($this->config->debugMode) {
Log::debug('Attempting compressed fetch', [
Log::debug('Attempting download', [
'kind' => $kind->value,
'release_id' => $releaseId,
'file_title' => $fileTitle,
'message_ids' => $messageIDs,
@@ -192,7 +135,8 @@ class UsenetDownloadService
$result = $this->downloadByMessageIDs($messageIDs, $groupName, $releaseId);
if (! $result['success'] && $this->config->debugMode) {
Log::debug('Compressed fetch failed', [
Log::debug('Download failed', [
'kind' => $kind->value,
'release_id' => $releaseId,
'file_title' => $fileTitle,
'message_ids' => $messageIDs,
app/Services/PostProcessService.php
+7 -1
View File
@@ -235,7 +235,13 @@ final class PostProcessService
*/
public function processAdditional(int|string $groupID = '', string $guidChar = ''): void
{
app(AdditionalProcessingOrchestrator::class)->start($groupID, $guidChar);
$orchestrator = app(AdditionalProcessingOrchestrator::class);
try {
$orchestrator->start($groupID, $guidChar);
} finally {
$orchestrator->finish();
}
}
/**
tests/Unit/AdditionalProcessing/ArchiveExtractionServiceTest.php
+54
View File
@@ -0,0 +1,54 @@
<?php
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use dariusiii\rarinfo\ArchiveInfo;
use dariusiii\rarinfo\Par2Info;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class ArchiveExtractionServiceTest extends TestCase
{
use CreatesProcessingConfiguration;
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
#[Test]
public function it_returns_file_data_directly_from_archive_info_when_available(): void
{
$archiveInfo = Mockery::mock(ArchiveInfo::class);
$archiveInfo->shouldReceive('setData')->once()->with('ARCHIVE', true)->andReturn(true);
$archiveInfo->shouldReceive('getFileData')->once()->with('cover.jpg')->andReturn('IMAGE-DATA');
$service = new ArchiveExtractionService(
$this->makeConfig(),
$archiveInfo,
Mockery::mock(Par2Info::class)
);
$this->assertSame('IMAGE-DATA', $service->extractSpecificFile('ARCHIVE', 'cover.jpg', sys_get_temp_dir().'/'));
}
#[Test]
public function it_detects_common_standalone_video_signatures(): void
{
$service = new ArchiveExtractionService(
$this->makeConfig(),
Mockery::mock(ArchiveInfo::class),
Mockery::mock(Par2Info::class)
);
$avi = 'RIFF'.str_repeat("\0", 4).'AVI '.str_repeat("\0", 16);
$mp4 = str_repeat("\0", 4).'ftypisom'.str_repeat("\0", 16);
$this->assertSame('avi', $service->detectStandaloneVideo($avi));
$this->assertSame('mp4', $service->detectStandaloneVideo($mp4));
$this->assertNull($service->detectStandaloneVideo('tiny'));
}
}
tests/Unit/AdditionalProcessing/CreatesProcessingConfiguration.php
+70
View File
@@ -0,0 +1,70 @@
<?php
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\Config\ProcessingConfiguration;
use ReflectionClass;
use ReflectionProperty;
trait CreatesProcessingConfiguration
{
/**
* @param array<string, mixed> $overrides
*/
private function makeConfig(array $overrides = []): ProcessingConfiguration
{
$reflection = new ReflectionClass(ProcessingConfiguration::class);
/** @var ProcessingConfiguration $config */
$config = $reflection->newInstanceWithoutConstructor();
$values = array_merge([
'echoCLI' => false,
'innerFileBlacklist' => false,
'maxNestedLevels' => 1,
'extractUsingRarInfo' => true,
'fetchLastFiles' => false,
'unrarPath' => false,
'unzipPath' => false,
'timeoutPath' => false,
'timeoutSeconds' => 0,
'queryLimit' => 25,
'segmentsToDownload' => 2,
'maximumRarSegments' => 3,
'maximumRarPasswordChecks' => 1,
'maxSizeGB' => 100,
'minSizeMB' => 0,
'alternateNNTP' => false,
'ffmpegDuration' => 5,
'addPAR2Files' => false,
'processVideo' => false,
'processThumbnails' => false,
'processAudioSample' => false,
'processJPGSample' => false,
'processMediaInfo' => false,
'processAudioInfo' => false,
'processPasswords' => false,
'audioSavePath' => sys_get_temp_dir().'/',
'tmpUnrarPath' => sys_get_temp_dir().'/',
'debugMode' => false,
'searchEnabled' => false,
'searchDriver' => 'manticore',
'renameMusicMediaInfo' => false,
'renamePar2' => false,
'ffmpegPath' => false,
'mediaInfoPath' => false,
'releaseProcessingTimeout' => 0,
'maxPpTimeoutCount' => 3,
'audioFileRegex' => '\\.(MP3|FLAC|OGG)',
'ignoreBookRegex' => '/\\b(epub|pdf)\\b/i',
'supportFileRegex' => '\\.(?:par2|sfv|nzb)',
'videoFileRegex' => '\\.(AVI|MKV|MP4)',
], $overrides);
foreach ($values as $property => $value) {
$prop = new ReflectionProperty(ProcessingConfiguration::class, $property);
$prop->setValue($config, $value);
}
return $config;
}
}
tests/Unit/AdditionalProcessing/ReleaseFilesArchiveFallbackTest.php
+151
View File
@@ -0,0 +1,151 @@
<?php
namespace Tests\Unit\AdditionalProcessing;
use App\Models\Release;
use App\Models\ReleaseFile;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext;
use App\Services\AdditionalProcessing\MediaExtractionService;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\NNTP\NNTPService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Events\Dispatcher;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Foundation\Application;
use Illuminate\Support\Facades\Facade;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class ReleaseFilesArchiveFallbackTest extends TestCase
{
use CreatesProcessingConfiguration;
private string $tmpPath;
protected function setUp(): void
{
parent::setUp();
$container = new Application(sys_get_temp_dir());
$container->instance('files', new Filesystem);
Facade::setFacadeApplication($container);
$capsule = new Capsule($container);
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$capsule->setEventDispatcher(new Dispatcher($container));
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema()->create('release_files', function ($table): void {
$table->integer('releases_id');
$table->string('name');
$table->integer('size')->default(0);
$table->timestamps();
});
$this->tmpPath = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR.'fallback_'.uniqid('', true).DIRECTORY_SEPARATOR;
(new Filesystem)->makeDirectory($this->tmpPath, 0777, true, true);
}
protected function tearDown(): void
{
(new Filesystem)->deleteDirectory($this->tmpPath);
Mockery::close();
parent::tearDown();
}
#[Test]
public function it_extracts_jpg_candidates_from_an_archive_file_list(): void
{
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('JPEG-DATA');
$media = Mockery::mock(MediaExtractionService::class);
$media->shouldReceive('isValidImage')->once()->andReturn(true);
$media->shouldReceive('getJPGSample')->once()->andReturn(true);
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoJpgSaved')->once()->andReturnNull();
$fallback = new ReleaseFilesArchiveFallback(
$this->makeConfig(['processJPGSample' => true]),
$archive,
$media,
Mockery::mock(UsenetDownloadService::class),
Mockery::mock(ReleaseFileManager::class),
$output
);
$context = $this->makeContext();
$fallback->processJpgFromArchiveFileList('ARCHIVE', [['name' => 'cover.jpg', 'size' => 99]], $context);
$this->assertTrue($context->foundJPGSample);
}
#[Test]
public function it_uses_stored_release_files_to_recover_an_nfo(): void
{
ReleaseFile::query()->create([
'releases_id' => 12,
'name' => 'release.nfo',
'size' => 200,
]);
$archive = Mockery::mock(ArchiveExtractionService::class);
$archive->shouldReceive('extractSpecificFile')->once()->andReturn('NFO DATA');
$download = Mockery::mock(UsenetDownloadService::class);
$download->shouldReceive('download')->once()->andReturn([
'success' => true,
'data' => 'ARCHIVE',
'groupUnavailable' => false,
'error' => null,
]);
$download->shouldReceive('getNNTP')->once()->andReturn(Mockery::mock(NNTPService::class));
$manager = Mockery::mock(ReleaseFileManager::class);
$manager->shouldReceive('processNfoFile')->once()->andReturnUsing(function (string $path, ReleaseProcessingContext $context): bool {
$this->assertFileExists($path);
$context->releaseHasNoNFO = false;
return true;
});
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoNfoFound')->once()->andReturnNull();
$fallback = new ReleaseFilesArchiveFallback(
$this->makeConfig(),
$archive,
Mockery::mock(MediaExtractionService::class),
$download,
$manager,
$output
);
$context = $this->makeContext(12);
$context->releaseGroupName = 'alt.binaries.test';
$context->releaseHasNoNFO = true;
$context->nzbContents = [['title' => 'archive.part01.rar', 'segments' => ['<1>', '<2>']]];
$fallback->processNfoFromReleaseFiles($context);
$this->assertFalse($context->releaseHasNoNFO);
}
private function makeContext(int $releaseId = 12): ReleaseProcessingContext
{
$context = new ReleaseProcessingContext(new Release([
'id' => $releaseId,
'guid' => 'guid-'.$releaseId,
'groups_id' => 22,
]));
$context->tmpPath = $this->tmpPath;
return $context;
}
}
tests/Unit/AdditionalProcessing/ReleaseProcessorTest.php
+173
View File
@@ -0,0 +1,173 @@
<?php
namespace Tests\Unit\AdditionalProcessing;
use App\Models\Release;
use App\Services\AdditionalProcessing\ArchiveExtractionService;
use App\Services\AdditionalProcessing\ConsoleOutputService;
use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext;
use App\Services\AdditionalProcessing\MediaExtractionService;
use App\Services\AdditionalProcessing\NzbContentParser;
use App\Services\AdditionalProcessing\ReleaseFileManager;
use App\Services\AdditionalProcessing\ReleaseFilesArchiveFallback;
use App\Services\AdditionalProcessing\ReleaseProcessor;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\TempWorkspaceService;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class ReleaseProcessorTest extends TestCase
{
use CreatesProcessingConfiguration;
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
#[Test]
public function it_deletes_release_when_nzb_parsing_fails(): void
{
$processor = $this->makeProcessor(
nzbParser: Mockery::mock(NzbContentParser::class)
->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn(['error' => 'broken nzb', 'contents' => []])->getMock(),
releaseManager: Mockery::mock(ReleaseFileManager::class)
->shouldReceive('deleteRelease')->once()->andReturnNull()->getMock(),
tempWorkspace: Mockery::mock(TempWorkspaceService::class)
->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/')
->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull()->getMock(),
output: Mockery::mock(ConsoleOutputService::class)
->shouldReceive('echoReleaseStart')->once()->andReturnNull()
->shouldReceive('setProcessTitle')->once()->andReturnNull()
->shouldReceive('warning')->once()->with('broken nzb')->andReturnNull()->getMock()
);
$processor->process($this->makeContext(), '/tmp/main/');
$this->assertTrue(true);
}
#[Test]
public function it_finalizes_a_release_after_basic_successful_processing(): void
{
$config = $this->makeConfig();
$nzbParser = Mockery::mock(NzbContentParser::class);
$nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([
'error' => null,
'contents' => [['title' => 'file.nzb', 'segments' => []]],
]);
$nzbParser->shouldReceive('extractMessageIDs')->once()->andReturn([
'hasCompressedFile' => false,
'sampleMessageIDs' => [],
'jpgMessageIDs' => [],
'mediaInfoMessageID' => '',
'audioInfoMessageID' => '',
'audioInfoExtension' => '',
'bookFileCount' => 0,
]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('finalizeRelease')->once()->andReturnNull();
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
$tempWorkspace->shouldReceive('clearDirectory')->once()->with('/tmp/ap-release/', false)->andReturnNull();
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace,
$output
);
$processor->process($this->makeContext(), '/tmp/main/');
$this->assertTrue(true);
}
#[Test]
public function it_marks_release_timeout_before_full_processing_continues(): void
{
$config = $this->makeConfig(['releaseProcessingTimeout' => 1, 'maxPpTimeoutCount' => 2]);
$nzbParser = Mockery::mock(NzbContentParser::class);
$nzbParser->shouldReceive('parseNzb')->once()->with('guid-1')->andReturn([
'error' => null,
'contents' => [],
]);
$releaseManager = Mockery::mock(ReleaseFileManager::class);
$releaseManager->shouldReceive('handleReleaseTimeout')->once()->andReturn(false);
$releaseManager->shouldNotReceive('finalizeRelease');
$tempWorkspace = Mockery::mock(TempWorkspaceService::class);
$tempWorkspace->shouldReceive('createReleaseTempFolder')->once()->andReturn('/tmp/ap-release/');
$tempWorkspace->shouldReceive('clearDirectory')->twice()->with('/tmp/ap-release/', false)->andReturnNull();
$output = Mockery::mock(ConsoleOutputService::class);
$output->shouldReceive('echoReleaseStart')->once()->andReturnNull();
$output->shouldReceive('setProcessTitle')->once()->andReturnNull();
$output->shouldReceive('echoReleaseTimeout')->once()->andReturnNull();
$processor = new ReleaseProcessor(
$config,
$nzbParser,
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$releaseManager,
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace,
$output
);
$context = $this->makeContext();
$context->startTime = hrtime(true) - 2_000_000_000;
$processor->process($context, '/tmp/main/');
$this->assertTrue(true);
}
private function makeProcessor(
?NzbContentParser $nzbParser = null,
?ReleaseFileManager $releaseManager = null,
?TempWorkspaceService $tempWorkspace = null,
?ConsoleOutputService $output = null
): ReleaseProcessor {
return new ReleaseProcessor(
$this->makeConfig(),
$nzbParser ?? Mockery::mock(NzbContentParser::class),
Mockery::mock(ArchiveExtractionService::class),
Mockery::mock(MediaExtractionService::class),
Mockery::mock(UsenetDownloadService::class),
$releaseManager ?? Mockery::mock(ReleaseFileManager::class),
Mockery::mock(ReleaseFilesArchiveFallback::class),
$tempWorkspace ?? Mockery::mock(TempWorkspaceService::class),
$output ?? Mockery::mock(ConsoleOutputService::class)
);
}
private function makeContext(): ReleaseProcessingContext
{
return new ReleaseProcessingContext(new Release([
'id' => 1,
'guid' => 'guid-1',
'size' => 1024,
'groups_id' => 10,
'nfostatus' => -1,
'pp_timeout_count' => 0,
]));
}
}
tests/Unit/AdditionalProcessing/UsenetDownloadServiceTest.php
+73
View File
@@ -0,0 +1,73 @@
<?php
namespace Tests\Unit\AdditionalProcessing;
use App\Services\AdditionalProcessing\Enums\DownloadKind;
use App\Services\AdditionalProcessing\UsenetDownloadService;
use App\Services\NNTP\NNTPService;
use Mockery;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
class UsenetDownloadServiceTest extends TestCase
{
use CreatesProcessingConfiguration;
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
#[Test]
public function it_downloads_binary_payloads_through_the_enum_driven_api(): void
{
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->once()
->with(['<abc>'], false)
->andReturn('BINARY-DATA');
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$result = $service->download(DownloadKind::Compressed, ['<abc>'], 'alt.binaries', 55, 'archive.rar');
$this->assertTrue($result['success']);
$this->assertSame('BINARY-DATA', $result['data']);
}
#[Test]
public function it_marks_group_unavailable_errors_explicitly(): void
{
$error = new class
{
public function getMessage(): string
{
return 'No such news group';
}
};
$nntp = Mockery::mock(NNTPService::class);
$nntp->shouldReceive('getMessagesByMessageID')
->once()
->with(['<missing>'], false)
->andReturn($error);
$service = new UsenetDownloadService($this->makeConfig(), $nntp);
$result = $service->download(DownloadKind::Sample, ['<missing>']);
$this->assertFalse($result['success']);
$this->assertTrue($result['groupUnavailable']);
$this->assertStringContainsString('Group unavailable', (string) $result['error']);
}
#[Test]
public function it_checks_minimum_download_sizes(): void
{
$service = new UsenetDownloadService($this->makeConfig(), Mockery::mock(NNTPService::class));
$this->assertFalse($service->meetsMinimumSize(str_repeat('a', 40)));
$this->assertTrue($service->meetsMinimumSize(str_repeat('a', 41)));
}
}