starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 00:01:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update namefixing

Browse Source
This commit is contained in:
DariusIII
2026-08-04 08:49:07 +02:00
parent 341b9f7e21
commit a10aae7f05
18 changed files with 1889 additions and 993 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/MatchPrefiles.php
+2 -5
View File
@@ -28,10 +28,8 @@ class MatchPrefiles extends Command
/**
* Execute the console command.
*
* @return int
*/
public function handle()
public function handle(NameFixingService $nameFixingService): int
{
$limit = $this->argument('limit');
@@ -53,8 +51,7 @@ class MatchPrefiles extends Command
}
try {
$nameFixingService = new NameFixingService;
$nameFixingService->getPreFileNames($argv); // @phpstan-ignore argument.type
$nameFixingService->getPreFileNames($argv);
return 0;
} catch (Exception $e) {
app/Console/Commands/ReleasesFixNamesGroup.php
+64 -252
View File
@@ -4,15 +4,14 @@ declare(strict_types=1);
namespace App\Console\Commands;
use App\Models\Category;
use App\Models\Predb;
use App\Models\Release;
use App\Services\NameFixing\NameFixingQueryService;
use App\Services\NameFixing\NameFixingService;
use App\Services\NfoService;
use App\Services\NNTP\NNTPService;
use App\Services\Nzb\NzbContentsService;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use RuntimeException;
class ReleasesFixNamesGroup extends Command
{
@@ -25,7 +24,8 @@ class ReleasesFixNamesGroup extends Command
{type : Type of fix (standard|predbft)}
{--guid-char= : GUID character to process (for standard type)}
{--limit=1000 : Maximum releases to process}
{--thread=1 : Thread number (for predbft type)}';
{--thread=1 : Worker number (for predbft type)}
{--workers=1 : Total workers (for predbft type)}';
/**
* The console command description.
@@ -36,6 +36,8 @@ class ReleasesFixNamesGroup extends Command
private NameFixingService $nameFixingService;
private NameFixingQueryService $queries;
private int $checked = 0;
/**
@@ -46,7 +48,8 @@ class ReleasesFixNamesGroup extends Command
$type = $this->argument('type');
$maxPerRun = (int) $this->option('limit');
$this->nameFixingService = new NameFixingService;
$this->nameFixingService = app(NameFixingService::class);
$this->queries = app(NameFixingQueryService::class);
switch ($type) {
case 'standard':
@@ -78,173 +81,52 @@ class ReleasesFixNamesGroup extends Command
$this->info("Processing releases with GUID starting with: {$guidChar}");
$this->info("Maximum per run: {$maxPerRun}");
// Allow for larger filename return sets
DB::statement('SET SESSION group_concat_max_len = 65536');
$nntp = null;
$nzbcontents = null;
$connectionAttempted = false;
$par2Processor = function (object $release) use (&$connectionAttempted, &$nntp, &$nzbcontents): bool {
if (! $connectionAttempted) {
$connectionAttempted = true;
$nntp = app(NNTPService::class);
$compressedHeaders = config('nntmux_nntp.compressed_headers');
$connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true
? $nntp->doConnect($compressedHeaders, true)
: $nntp->doConnect();
// Find releases to process
$releases = $this->fetchReleases($guidChar, $maxPerRun);
if ($connectResult !== true) {
$errorMessage = 'Unable to connect to usenet for PAR2 processing';
if (NNTPService::isError($connectResult)) {
$errorMessage .= ' Error: '.$connectResult->getMessage();
}
$this->warn($errorMessage);
} else {
$nzbcontents = app(NzbContentsService::class);
$nzbcontents->setNntp($nntp);
$nzbcontents->setNfo(app(NfoService::class));
$nzbcontents->setEchoOutput(false);
}
}
if ($releases->isEmpty()) {
return $nzbcontents !== null && $nzbcontents->checkPar2(
$release->guid,
$release->releases_id,
$release->groups_id,
1,
1
);
};
$stats = $this->nameFixingService->processStandardBatch($guidChar, $maxPerRun, true, $par2Processor);
$this->checked += $stats['checked'];
if ($stats['checked'] === 0) {
$this->info('No releases to process');
return Command::SUCCESS;
}
$this->info("Found {$releases->count()} releases to process");
$bar = $this->output->createProgressBar($releases->count());
$bar->start();
$nntp = null;
$nzbcontents = null;
foreach ($releases as $release) {
$this->checked++;
$this->nameFixingService->reset();
// Process UID
if ((int) $release->proc_uid === NameFixingService::PROC_UID_NONE &&
(! empty($release->uid) || ! empty($release->mediainfo))) {
if (! empty($release->uid)) {
$this->nameFixingService->checkName($release, true, 'UID, ', true, true);
}
if (! $this->nameFixingService->getUpdateService()->matched && ! empty($release->mediainfo)) {
$this->nameFixingService->checkName($release, true, 'Mediainfo, ', true, true);
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_uid', NameFixingService::PROC_UID_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process CRC32
if ((int) $release->proc_crc32 === NameFixingService::PROC_CRC_NONE && ! empty($release->crc)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'CRC32, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_crc32', NameFixingService::PROC_CRC_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process SRR
if ((int) $release->proc_srr === NameFixingService::PROC_SRR_NONE) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'SRR, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_srr', NameFixingService::PROC_SRR_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process PAR2 hash
if ((int) $release->proc_hash16k === NameFixingService::PROC_HASH16K_NONE && ! empty($release->hash)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'PAR2 hash, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_hash16k', NameFixingService::PROC_HASH16K_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process NFO
if ((int) $release->nfostatus === NfoService::NFO_FOUND &&
(int) $release->proc_nfo === NameFixingService::PROC_NFO_NONE &&
! empty($release->textstring) &&
! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'NFO, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_nfo', NameFixingService::PROC_NFO_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process filenames
if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixingService::PROC_FILES_NONE) {
$this->nameFixingService->reset();
$fileNames = explode('|', $release->filestring);
$releaseFile = $release;
foreach ($fileNames as $fileName) {
if (! $this->nameFixingService->getUpdateService()->matched) {
$releaseFile->textstring = $fileName;
$this->nameFixingService->checkName($releaseFile, true, 'Filenames, ', true, true);
}
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_files', NameFixingService::PROC_FILES_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process PAR2
if ((int) $release->proc_par2 === NameFixingService::PROC_PAR2_NONE) {
// Initialize NZB contents if needed
if (! isset($nzbcontents)) {
$nntp = new NNTPService;
$compressedHeaders = config('nntmux_nntp.compressed_headers');
$connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true
? $nntp->doConnect($compressedHeaders, true)
: $nntp->doConnect();
if ($connectResult !== true) {
$errorMessage = 'Unable to connect to usenet for PAR2 processing';
if (NNTPService::isError($connectResult)) {
$errorMessage .= ' Error: '.$connectResult->getMessage();
}
$this->warn($errorMessage);
} else {
$Nfo = new NfoService;
$nzbcontents = app(NzbContentsService::class);
$nzbcontents->setNntp($nntp);
$nzbcontents->setNfo($Nfo);
$nzbcontents->setEchoOutput(false);
}
}
if (isset($nzbcontents)) {
$nzbcontents->checkPar2($release->guid, $release->releases_id, $release->groups_id, 1, 1);
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_par2', NameFixingService::PROC_PAR2_DONE, $release->releases_id);
$bar->advance();
}
$bar->finish();
$this->newLine(2);
$this->info("✅ Processed {$this->checked} releases");
$this->info("✅ Fixed {$this->nameFixingService->getUpdateService()->fixed} release names");
$this->info("Processed {$stats['checked']} releases");
$this->info("Fixed {$stats['fixed']} release names");
return Command::SUCCESS;
}
@@ -255,51 +137,45 @@ class ReleasesFixNamesGroup extends Command
protected function processPredbFulltext(int $maxPerRun): int
{
$thread = (int) $this->option('thread');
$offset = $thread * $maxPerRun - $maxPerRun;
$workers = (int) $this->option('workers');
$this->info('Processing PreDB fulltext matching');
$this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}");
$this->info("Worker: {$thread}/{$workers}, Limit: {$maxPerRun}");
$pres = Predb::fromQuery(
sprintf(
'
SELECT p.id AS predb_id, p.title, p.source, p.searched
FROM predb p
WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]"
AND p.searched = 0
AND p.predate < (NOW() - INTERVAL 1 DAY)
ORDER BY p.predate ASC
LIMIT %s
OFFSET %s',
$maxPerRun,
$offset
)
);
$pres = $this->queries->predbBatch($thread, $workers, $maxPerRun);
if ($pres->isEmpty()) {
if ($pres === []) {
$this->info('No PreDB entries to process');
return Command::SUCCESS;
}
$this->info("Found {$pres->count()} PreDB entries to process");
$bar = $this->output->createProgressBar($pres->count());
$this->info('Found '.count($pres).' PreDB entries to process');
$bar = $this->output->createProgressBar(count($pres));
$bar->start();
foreach ($pres as $pre) {
$searched = 0;
$ftmatched = $this->matchPredbFT($pre);
try {
$ftmatched = $this->nameFixingService->matchPredbFulltext($pre);
} catch (RuntimeException $exception) {
$bar->finish();
$this->newLine(2);
$this->error($exception->getMessage());
return Command::FAILURE;
}
if ($ftmatched > 0) {
$searched = 1;
} elseif ($ftmatched < 0) {
$searched = -6;
} else {
$searched = $pre['searched'] - 1;
$searched = (int) $pre->searched - 1;
}
Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]);
DB::update('UPDATE predb SET searched = ? WHERE id = ?', [$searched, $pre->predb_id]);
$this->checked++;
$bar->advance();
@@ -312,68 +188,4 @@ class ReleasesFixNamesGroup extends Command
return Command::SUCCESS;
}
/**
* Match a PreDB title to releases using full-text search.
*/
protected function matchPredbFT(object $pre): int
{
// This is a simplified version - the full implementation would use Manticore/Elasticsearch
return 0;
}
/**
* Fetch releases for processing
*/
protected function fetchReleases(string $guidChar, int $maxPerRun): mixed
{
return Release::fromQuery(sprintf("
SELECT
r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo,
r.proc_uid, r.proc_files, r.proc_par2, r.nfostatus,
r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32,
IFNULL(rf.releases_id, 0) AS fileid,
IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring,
IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring,
IFNULL(ru.uniqueid, '') AS uid,
IFNULL(ph.hash, 0) AS hash,
IFNULL(rf.crc32, '') AS crc
FROM releases r
LEFT JOIN release_nfos rn ON rn.releases_id = r.id
LEFT JOIN release_files rf ON rf.releases_id = r.id
LEFT JOIN release_unique ru ON ru.releases_id = r.id
LEFT JOIN par_hashes ph ON ph.releases_id = r.id
WHERE r.leftguid = %s
AND r.isrenamed = %d
AND r.predb_id = 0
AND r.passwordstatus >= 0
AND r.nfostatus > %d
AND (
(r.nfostatus = %d AND r.proc_nfo = %d)
OR r.proc_files = %d
OR r.proc_uid = %d
OR r.proc_par2 = %d
OR r.proc_srr = %d
OR r.proc_hash16k = %d
OR r.proc_crc32 = %d
)
AND r.categories_id IN (%s)
GROUP BY r.id
ORDER BY r.id DESC
LIMIT %s",
escapeString($guidChar),
NameFixingService::IS_RENAMED_NONE,
NfoService::NFO_UNPROC,
NfoService::NFO_FOUND,
NameFixingService::PROC_NFO_NONE,
NameFixingService::PROC_FILES_NONE,
NameFixingService::PROC_UID_NONE,
NameFixingService::PROC_PAR2_NONE,
NameFixingService::PROC_SRR_NONE,
NameFixingService::PROC_HASH16K_NONE,
NameFixingService::PROC_CRC_NONE,
Category::getCategoryOthersGroup(),
$maxPerRun
));
}
}
app/Services/NameFixing/DonorMatchSelector.php
+48
View File
@@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
namespace App\Services\NameFixing;
final class DonorMatchSelector
{
/**
* @param list<object> $donors
*/
public function select(array $donors, int $targetSize, int $tolerancePercent): ?object
{
if ($targetSize <= 0 || $donors === []) {
return null;
}
$eligible = [];
foreach ($donors as $donor) {
$donorSize = (int) ($donor->relsize ?? $donor->size ?? 0);
if ($donorSize <= 0) {
continue;
}
$difference = abs($donorSize - $targetSize);
$differencePercent = ($difference / $donorSize) * 100;
if ($differencePercent <= $tolerancePercent) {
$eligible[] = [
'donor' => $donor,
'difference' => $difference,
'id' => (int) ($donor->releases_id ?? $donor->id ?? PHP_INT_MAX),
];
}
}
if ($eligible === []) {
return null;
}
usort($eligible, static function (array $left, array $right): int {
return [$left['difference'], $left['id']] <=> [$right['difference'], $right['id']];
});
return $eligible[0]['donor'];
}
}
app/Services/NameFixing/FilePrioritizer.php
+6 -6
View File
@@ -58,8 +58,8 @@ class FilePrioritizer
* 5. NFO files
* 6. Other files
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Sorted array of filenames
* @param list<string> $files Array of filenames
* @return list<string> Sorted array of filenames
*/
public function prioritizeForMatching(array $files): array
{
@@ -82,8 +82,8 @@ class FilePrioritizer
* Similar to prioritizeForMatching but with slightly different priorities
* optimized for PreDB filename lookups.
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Sorted array of filenames
* @param list<string> $files Array of filenames
* @return list<string> Sorted array of filenames
*/
public function prioritizeForPreDb(array $files): array
{
@@ -102,8 +102,8 @@ class FilePrioritizer
/**
* Categorize files into groups by type.
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Categorized files
* @param list<string> $files Array of filenames
* @return array{srr: list<string>, mainRar: list<string>, firstPart: list<string>, video: list<string>, nfo: list<string>, other: list<string>} Categorized files
*/
protected function categorizeFiles(array $files): array
{
app/Services/NameFixing/NameFixingQueryService.php
+481
View File
@@ -0,0 +1,481 @@
<?php
declare(strict_types=1);
namespace App\Services\NameFixing;
use App\Models\Category;
use Carbon\CarbonImmutable;
use Illuminate\Database\ConnectionInterface;
use Illuminate\Support\Facades\DB;
use InvalidArgumentException;
final class NameFixingQueryService
{
public const SOURCE_NFO = 'nfo';
public const SOURCE_FILES = 'files';
public const SOURCE_SRR = 'srr';
public const SOURCE_CRC = 'crc';
public const SOURCE_UID = 'uid';
public const SOURCE_HASH = 'hash';
public const SOURCE_PAR2 = 'par2';
public const SOURCE_XXX = 'xxx';
public const SOURCE_MEDIA_MOVIE = 'media_movie';
public const BATCH_SIZE = 1000;
private ConnectionInterface $database;
/**
* @var array<string, string>
*/
private const STATUS_COLUMNS = [
self::SOURCE_NFO => 'proc_nfo',
self::SOURCE_FILES => 'proc_files',
self::SOURCE_SRR => 'proc_srr',
self::SOURCE_CRC => 'proc_crc32',
self::SOURCE_UID => 'proc_uid',
self::SOURCE_HASH => 'proc_hash16k',
self::SOURCE_PAR2 => 'proc_par2',
self::SOURCE_XXX => 'proc_files',
self::SOURCE_MEDIA_MOVIE => 'proc_uid',
];
/**
* @var array<string, string>
*/
private const SOURCE_EXISTS = [
self::SOURCE_NFO => 'EXISTS (SELECT 1 FROM release_nfos source_nfo WHERE source_nfo.releases_id = r.id)',
self::SOURCE_FILES => 'EXISTS (SELECT 1 FROM release_files source_file WHERE source_file.releases_id = r.id)',
self::SOURCE_SRR => "EXISTS (SELECT 1 FROM release_files source_srr WHERE source_srr.releases_id = r.id AND (source_srr.name LIKE '%.srr' OR source_srr.name LIKE '%.srs'))",
self::SOURCE_CRC => "EXISTS (SELECT 1 FROM release_files source_crc WHERE source_crc.releases_id = r.id AND source_crc.crc32 IS NOT NULL AND source_crc.crc32 != '')",
self::SOURCE_UID => "EXISTS (SELECT 1 FROM media_infos source_media WHERE source_media.releases_id = r.id AND source_media.unique_id IS NOT NULL AND source_media.unique_id != '') OR EXISTS (SELECT 1 FROM release_unique source_unique WHERE source_unique.releases_id = r.id AND source_unique.uniqueid != '')",
self::SOURCE_HASH => "EXISTS (SELECT 1 FROM par_hashes source_hash WHERE source_hash.releases_id = r.id AND source_hash.hash != '')",
self::SOURCE_PAR2 => '1 = 1',
self::SOURCE_XXX => "EXISTS (SELECT 1 FROM release_files source_xxx WHERE source_xxx.releases_id = r.id AND source_xxx.name LIKE '%SDPORN%')",
self::SOURCE_MEDIA_MOVIE => "EXISTS (SELECT 1 FROM media_infos source_movie WHERE source_movie.releases_id = r.id AND source_movie.movie_name IS NOT NULL AND source_movie.movie_name != '')",
];
public function __construct(?ConnectionInterface $database = null)
{
$this->database = $database ?? DB::connection();
}
/**
* @return list<object>
*/
public function candidateBatch(
string $source,
int $time,
int $categories,
int $afterId = 0,
int $limit = self::BATCH_SIZE
): array {
[$where, $bindings] = $this->candidateWhere($source, $time, $categories);
$bindings[] = $afterId;
$bindings[] = max(1, $limit);
return $this->database->select(
"SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.groups_id,
r.categories_id, r.size AS relsize, r.guid, r.predb_id, r.nfostatus,
r.proc_nfo, r.proc_files, r.proc_par2, r.proc_uid, r.proc_srr,
r.proc_hash16k, r.proc_crc32
FROM releases r
WHERE {$where}
AND r.id > ?
ORDER BY r.id ASC
LIMIT ?",
$bindings
);
}
public function countCandidates(string $source, int $time, int $categories): int
{
[$where, $bindings] = $this->candidateWhere($source, $time, $categories);
$rows = $this->database->select(
"SELECT COUNT(*) AS aggregate FROM releases r WHERE {$where}",
$bindings
);
return (int) ($rows[0]->aggregate ?? 0);
}
/**
* @return list<object>
*/
public function standardCandidateBatch(string $leftGuid, int $limit): array
{
return $this->database->select(
'SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.guid,
r.groups_id, r.categories_id, r.size AS relsize, r.predb_id, r.nfostatus,
r.proc_nfo, r.proc_uid, r.proc_files, r.proc_par2, r.proc_hash16k,
r.proc_srr, r.proc_crc32
FROM releases r
WHERE r.leftguid = ?
AND r.isrenamed = 0
AND r.predb_id = 0
AND r.passwordstatus >= 0
AND r.nfostatus > -1
AND (
(r.nfostatus = 1 AND r.proc_nfo = 0)
OR r.proc_files = 0
OR r.proc_uid = 0
OR r.proc_par2 = 0
OR r.proc_srr = 0
OR r.proc_hash16k = 0
OR r.proc_crc32 = 0
)
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')
ORDER BY r.id DESC
LIMIT ?',
[$leftGuid, max(1, $limit)]
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function nfoRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT rn.releases_id, UNCOMPRESS(rn.nfo) AS textstring
FROM release_nfos rn
WHERE rn.releases_id IN (%s)
ORDER BY rn.releases_id',
$releaseIds
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function fileRows(array $releaseIds, string $source = self::SOURCE_FILES): array
{
$filter = match ($source) {
self::SOURCE_FILES => '',
self::SOURCE_SRR => " AND (rf.name LIKE '%.srr' OR rf.name LIKE '%.srs')",
self::SOURCE_CRC => " AND rf.crc32 IS NOT NULL AND rf.crc32 != ''",
self::SOURCE_XXX => " AND rf.name LIKE '%SDPORN%'",
default => throw new InvalidArgumentException("Unsupported release-file source [{$source}]."),
};
return $this->selectForReleaseIds(
'SELECT rf.releases_id, rf.name AS textstring, rf.name AS filename, rf.crc32
FROM release_files rf
WHERE rf.releases_id IN (%s)'.$filter.'
ORDER BY rf.releases_id, rf.name',
$releaseIds
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function mediaRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT mi.releases_id, mi.unique_id AS uid, mi.movie_name, mi.file_name
FROM media_infos mi
WHERE mi.releases_id IN (%s)
UNION ALL
SELECT ru.releases_id, ru.uniqueid AS uid, NULL AS movie_name, NULL AS file_name
FROM release_unique ru
WHERE ru.releases_id IN (%s)
ORDER BY releases_id',
array_merge($releaseIds, $releaseIds),
2
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function hashRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT ph.releases_id, ph.hash
FROM par_hashes ph
WHERE ph.releases_id IN (%s)
ORDER BY ph.releases_id, ph.hash',
$releaseIds
);
}
/**
* @param list<string> $uniqueIds
* @return array<string, list<object>>
*/
public function uidDonors(array $uniqueIds): array
{
if ($uniqueIds === []) {
return [];
}
$placeholders = $this->placeholders(count($uniqueIds));
$bindings = array_merge($uniqueIds, ['nonscene@Ef.net (EF)'], $uniqueIds);
$rows = $this->database->select(
"SELECT mi.unique_id AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM media_infos mi
INNER JOIN releases r ON r.id = mi.releases_id
WHERE mi.unique_id IN ({$placeholders})
AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = ?)
UNION ALL
SELECT ru.uniqueid AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM release_unique ru
INNER JOIN releases r ON r.id = ru.releases_id
WHERE ru.uniqueid IN ({$placeholders})
AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = 'nonscene@Ef.net (EF)')",
$bindings
);
return $this->groupByKey($rows, 'match_key');
}
/**
* @param list<string> $hashes
* @return array<string, list<object>>
*/
public function hashDonors(array $hashes): array
{
return $this->donors(
'SELECT ph.hash AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM par_hashes ph
INNER JOIN releases r ON r.id = ph.releases_id
WHERE ph.hash IN (%s)
AND (r.predb_id > 0 OR r.anidbid > 0)',
$hashes
);
}
/**
* @param list<string> $crcs
* @return array<string, list<object>>
*/
public function crcDonors(array $crcs): array
{
return $this->donors(
'SELECT rf.crc32 AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM release_files rf
INNER JOIN releases r ON r.id = rf.releases_id
WHERE rf.crc32 IN (%s)
AND r.predb_id > 0',
$crcs
);
}
/**
* @return list<object>
*/
public function predbBatch(int $worker, int $workers, int $limit): array
{
$workerCount = max(1, min(16, $workers));
$workerSlot = max(1, min($workerCount, $worker)) - 1;
return $this->database->select(
'SELECT p.id AS predb_id, p.title, p.source, p.searched
FROM predb p
WHERE LENGTH(p.title) >= 15
AND p.title NOT REGEXP \'["<> ]\'
AND p.searched = 0
AND p.predate < ?
AND MOD(p.id, ?) = ?
ORDER BY p.predate ASC, p.id ASC
LIMIT ?',
[
CarbonImmutable::now()->subDay()->toDateTimeString(),
$workerCount,
$workerSlot,
max(1, $limit),
]
);
}
/**
* @param list<int> $candidateIds
* @return list<object>
*/
public function confirmPredbCandidates(array $candidateIds, string $title): array
{
if ($candidateIds === []) {
return [];
}
$like = '%'.$this->escapeLike($title).'%';
return $this->database->select(
'SELECT r.id AS releases_id, r.name, r.fromname, r.searchname,
r.groups_id, r.categories_id
FROM releases r
WHERE r.id IN ('.$this->placeholders(count($candidateIds)).')
AND r.predb_id = 0
AND (r.name LIKE ? ESCAPE \'\\\\\' OR r.searchname LIKE ? ESCAPE \'\\\\\')
ORDER BY r.id ASC
LIMIT 21',
array_merge($candidateIds, [$like, $like])
);
}
public function countPrefileCandidates(): int
{
$rows = $this->database->select(
'SELECT COUNT(*) AS aggregate
FROM releases r
WHERE r.predb_id = 0
AND r.isrenamed = 0
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')
AND EXISTS (
SELECT 1 FROM release_files rf
WHERE rf.releases_id = r.id
AND rf.name IS NOT NULL
)'
);
return (int) ($rows[0]->aggregate ?? 0);
}
/**
* @return list<object>
*/
public function prefileCandidateBatch(int $cursor, int $limit, bool $descending): array
{
$operator = $descending ? '<' : '>';
$direction = $descending ? 'DESC' : 'ASC';
return $this->database->select(
'SELECT r.id AS releases_id, r.name, r.searchname, r.fromname,
r.groups_id, r.categories_id
FROM releases r
WHERE r.predb_id = 0
AND r.isrenamed = 0
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).")
AND r.id {$operator} ?
AND EXISTS (
SELECT 1 FROM release_files rf
WHERE rf.releases_id = r.id
AND rf.name IS NOT NULL
)
ORDER BY r.id {$direction}
LIMIT ?",
[$cursor, max(1, $limit)]
);
}
/**
* @param list<object> $rows
* @return array<int, list<object>>
*/
public function groupByReleaseId(array $rows): array
{
return $this->groupByKey($rows, 'releases_id');
}
/**
* @return array{0: string, 1: list<mixed>}
*/
private function candidateWhere(string $source, int $time, int $categories): array
{
if (! isset(self::SOURCE_EXISTS[$source], self::STATUS_COLUMNS[$source])) {
throw new InvalidArgumentException("Unsupported name-fixing source [{$source}].");
}
$where = ['r.predb_id = 0', '('.self::SOURCE_EXISTS[$source].')'];
$bindings = [];
if ($categories !== 3) {
$statusColumn = self::STATUS_COLUMNS[$source];
$where[] = '(r.isrenamed = 0 OR r.categories_id IN (?, ?))';
$bindings[] = Category::OTHER_MISC;
$bindings[] = Category::OTHER_HASHED;
$where[] = "r.{$statusColumn} = 0";
}
if ($time === 1) {
$where[] = 'r.adddate > ?';
$bindings[] = CarbonImmutable::now()->subHours(6)->toDateTimeString();
}
if ($categories === 1) {
$where[] = 'r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')';
}
return [implode(' AND ', $where), $bindings];
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
private function selectForReleaseIds(
string $sql,
array $releaseIds,
int $placeholderGroups = 1
): array {
if ($releaseIds === []) {
return [];
}
$placeholder = $this->placeholders((int) (count($releaseIds) / $placeholderGroups));
$sql = vsprintf($sql, array_fill(0, $placeholderGroups, $placeholder));
return $this->database->select($sql, $releaseIds);
}
/**
* @param list<string> $values
* @return array<string, list<object>>
*/
private function donors(string $sql, array $values): array
{
if ($values === []) {
return [];
}
$rows = $this->database->select(
sprintf($sql, $this->placeholders(count($values))),
$values
);
return $this->groupByKey($rows, 'match_key');
}
/**
* @param list<object> $rows
* @return array<int|string, list<object>>
*/
private function groupByKey(array $rows, string $key): array
{
$grouped = [];
foreach ($rows as $row) {
$grouped[$row->{$key}][] = $row;
}
return $grouped;
}
private function placeholders(int $count): string
{
return implode(',', array_fill(0, $count, '?'));
}
private function escapeLike(string $value): string
{
return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value);
}
}
app/Services/NameFixing/NameFixingService.php
+705 -715
View File
File diff suppressed because it is too large Load Diff
app/Services/NameFixing/ReleaseUpdateService.php
+63 -11
View File
@@ -7,13 +7,13 @@ namespace App\Services\NameFixing;
use App\Events\ReleaseNameFixed;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Predb;
use App\Models\Release;
use App\Models\UsenetGroup;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseCleaningService;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use InvalidArgumentException;
/**
* Service for updating releases with new names.
@@ -23,6 +23,20 @@ use Illuminate\Support\Facades\DB;
*/
class ReleaseUpdateService
{
/**
* @var list<string>
*/
private const SINGLE_UPDATE_COLUMNS = [
'predb_id',
'proc_nfo',
'proc_files',
'proc_par2',
'proc_uid',
'proc_hash16k',
'proc_srr',
'proc_crc32',
];
/**
* PreDB regex pattern for scene release names.
*/
@@ -347,10 +361,25 @@ class ReleaseUpdateService
*/
public function updateSingleColumn(string $column, int $status, int $id): void
{
if ($column !== '' && $id !== 0) {
Release::query()->where('id', $id)->update([$column => $status]);
Search::updateRelease($id);
if (! in_array($column, self::SINGLE_UPDATE_COLUMNS, true)) {
throw new InvalidArgumentException("Unsupported release status column [{$column}].");
}
if ($id !== 0) {
DB::update("UPDATE releases SET {$column} = ? WHERE id = ?", [$status, $id]);
}
}
public function attachPredbId(int $releaseId, int $predbId): void
{
if ($releaseId === 0 || $predbId === 0) {
return;
}
$this->updateSingleColumn('predb_id', $predbId, $releaseId);
$this->relid = $releaseId;
$this->matched = true;
$this->done = true;
}
/**
@@ -360,17 +389,40 @@ class ReleaseUpdateService
*/
public function checkPreDbMatch(object $release, string $textstring): ?array
{
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) {
foreach ($hits as $hit) {
foreach ($hit as $val) {
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
if ($title !== null) {
return ['title' => $title['title'], 'id' => $title['id']];
}
if (! preg_match_all(self::PREDB_REGEX, $textstring, $hits) || preg_match('/Source\s:/i', $textstring)) {
return null;
}
$candidates = [];
foreach ($hits as $hit) {
foreach ($hit as $value) {
$title = trim($value);
if ($title !== '') {
$candidates[$title] = $title;
}
}
}
if ($candidates === []) {
return null;
}
$candidateValues = array_values($candidates);
$rows = DB::select(
'SELECT id, title FROM predb WHERE title IN ('.implode(',', array_fill(0, count($candidateValues), '?')).')',
$candidateValues
);
$matches = [];
foreach ($rows as $row) {
$matches[(string) $row->title] = ['title' => (string) $row->title, 'id' => (int) $row->id];
}
foreach ($candidateValues as $candidate) {
if (isset($matches[$candidate])) {
return $matches[$candidate];
}
}
return null;
}
app/Services/Runners/ReleasesRunner.php
+2 -1
View File
@@ -144,10 +144,11 @@ class ReleasesRunner extends BaseRunner
$queues = [];
$idx = 0;
$workerCount = count($leftGuids);
foreach ($leftGuids as $leftGuid) {
if ($maxPerRun > 0) {
$idx++;
$queues[$idx] = sprintf('%s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx);
$queues[$idx] = sprintf('%s %s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx, $workerCount);
}
}
app/Services/Tmux/Scripts/groupfixrelnames.php
+3 -3
View File
@@ -15,8 +15,8 @@ if (! isset($argv[1])) {
exit(1);
}
// Parse arguments: "type guidChar maxPerRun thread"
[$type, $guidChar, $maxPerRun, $thread] = explode(' ', $argv[1]);
// Parse arguments: "type guidChar maxPerRun worker workers"
[$type, $guidChar, $maxPerRun, $thread, $workers] = array_pad(explode(' ', $argv[1]), 5, '1');
// Build artisan command based on type
$artisan = dirname(__DIR__, 4).'/artisan';
@@ -37,7 +37,7 @@ switch ($type) {
exit(1);
}
$command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread}";
$command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread} --workers={$workers}";
break;
default:
database/migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php
+84
View File
@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* @var array<string, array<string, list<string>>>
*/
private const INDEXES = [
'par_hashes' => [
'ix_par_hashes_hash_releases_id' => ['hash', 'releases_id'],
],
'release_files' => [
'ix_release_files_crc32_releases_id' => ['crc32', 'releases_id'],
],
'predb' => [
'ix_predb_searched_predate_id' => ['searched', 'predate', 'id'],
],
];
/**
* Run the migrations.
*/
public function up(): void
{
foreach (self::INDEXES as $table => $indexes) {
if (! Schema::hasTable($table)) {
continue;
}
foreach ($indexes as $name => $columns) {
if ($this->hasUsablePrefix($table, $columns)) {
continue;
}
Schema::table($table, static function (Blueprint $blueprint) use ($columns, $name): void {
$blueprint->index($columns, $name);
});
}
}
}
/**
* Reverse the migrations.
*/
public function down(): void
{
foreach (self::INDEXES as $table => $indexes) {
if (! Schema::hasTable($table)) {
continue;
}
foreach (array_keys($indexes) as $name) {
if (! Schema::hasIndex($table, $name)) {
continue;
}
Schema::table($table, static function (Blueprint $blueprint) use ($name): void {
$blueprint->dropIndex($name);
});
}
}
}
/**
* @param list<string> $requiredColumns
*/
private function hasUsablePrefix(string $table, array $requiredColumns): bool
{
foreach (Schema::getIndexes($table) as $index) {
$columns = array_values($index['columns'] ?? []);
if (array_slice($columns, 0, count($requiredColumns)) === $requiredColumns) {
return true;
}
}
return false;
}
};
database/schema/mariadb-schema.sql
+3
View File
@@ -580,6 +580,7 @@ CREATE TABLE `par_hashes` (
`releases_id` int(10) unsigned NOT NULL COMMENT 'FK to releases.id',
`hash` varchar(32) NOT NULL COMMENT 'hash_16k block of par2',
PRIMARY KEY (`releases_id`,`hash`),
KEY `ix_par_hashes_hash_releases_id` (`hash`,`releases_id`),
CONSTRAINT `FK_ph_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
@@ -705,6 +706,7 @@ CREATE TABLE `predb` (
KEY `ix_predb_predate` (`predate`),
KEY `ix_predb_source` (`source`),
KEY `ix_predb_searched` (`searched`),
KEY `ix_predb_searched_predate_id` (`searched`,`predate`,`id`),
FULLTEXT KEY `ft_predb_filename` (`filename`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
DROP TABLE IF EXISTS `predb_crcs`;
@@ -829,6 +831,7 @@ CREATE TABLE `release_files` (
`updated_at` timestamp NULL DEFAULT NULL,
`passworded` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`releases_id`,`name`),
KEY `ix_release_files_crc32_releases_id` (`crc32`,`releases_id`),
CONSTRAINT `FK_rf_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
DROP TABLE IF EXISTS `release_informs`;
database/schema/mysql-schema.sql
+3
View File
@@ -580,6 +580,7 @@ CREATE TABLE `par_hashes` (
`releases_id` int(10) unsigned NOT NULL COMMENT 'FK to releases.id',
`hash` varchar(32) NOT NULL COMMENT 'hash_16k block of par2',
PRIMARY KEY (`releases_id`,`hash`),
KEY `ix_par_hashes_hash_releases_id` (`hash`,`releases_id`),
CONSTRAINT `FK_ph_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
@@ -705,6 +706,7 @@ CREATE TABLE `predb` (
KEY `ix_predb_predate` (`predate`),
KEY `ix_predb_source` (`source`),
KEY `ix_predb_searched` (`searched`),
KEY `ix_predb_searched_predate_id` (`searched`,`predate`,`id`),
FULLTEXT KEY `ft_predb_filename` (`filename`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
DROP TABLE IF EXISTS `predb_crcs`;
@@ -829,6 +831,7 @@ CREATE TABLE `release_files` (
`updated_at` timestamp NULL DEFAULT NULL,
`passworded` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`releases_id`,`name`),
KEY `ix_release_files_crc32_releases_id` (`crc32`,`releases_id`),
CONSTRAINT `FK_rf_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
DROP TABLE IF EXISTS `release_informs`;
tests/Feature/FixReleaseNameQueryIndexesMigrationTest.php
+125
View File
@@ -0,0 +1,125 @@
<?php
declare(strict_types=1);
namespace Tests\Feature;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
use PDO;
use Tests\TestCase;
class FixReleaseNameQueryIndexesMigrationTest extends TestCase
{
/**
* @var array<string, string|false>
*/
private array $originalEnvironment = [];
private string $databasePath;
public function createApplication()
{
$this->databasePath = sys_get_temp_dir().'/nntmux-name-fixing-indexes-test.sqlite';
$this->originalEnvironment = [
'APP_ENV' => getenv('APP_ENV'),
'DB_CONNECTION' => getenv('DB_CONNECTION'),
'DB_DATABASE' => getenv('DB_DATABASE'),
];
if (file_exists($this->databasePath)) {
unlink($this->databasePath);
}
$pdo = new PDO('sqlite:'.$this->databasePath);
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
$pdo->exec("INSERT INTO settings (name, value) VALUES ('categorizeforeign', '0'), ('catwebdl', '1')");
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
return $app;
}
protected function setUp(): void
{
parent::setUp();
config([
'database.default' => 'sqlite',
'database.connections.sqlite.database' => $this->databasePath,
]);
$this->createSchema();
}
protected function tearDown(): void
{
if ($this->databasePath !== '' && file_exists($this->databasePath)) {
unlink($this->databasePath);
}
parent::tearDown();
foreach ($this->originalEnvironment as $key => $value) {
$this->setEnvironmentValue($key, $value === false ? null : $value);
}
}
public function test_it_adds_only_missing_reverse_lookup_and_queue_indexes_idempotently(): void
{
$migration = require database_path('migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php');
$migration->up();
$migration->up();
$this->assertTrue(Schema::hasIndex('par_hashes', 'ix_par_hashes_hash_releases_id'));
$this->assertTrue(Schema::hasIndex('release_files', 'ix_release_files_crc32_releases_id'));
$this->assertTrue(Schema::hasIndex('predb', 'ix_predb_searched_predate_id'));
$migration->down();
$this->assertFalse(Schema::hasIndex('par_hashes', 'ix_par_hashes_hash_releases_id'));
$this->assertFalse(Schema::hasIndex('release_files', 'ix_release_files_crc32_releases_id'));
$this->assertFalse(Schema::hasIndex('predb', 'ix_predb_searched_predate_id'));
}
private function createSchema(): void
{
Schema::create('par_hashes', function (Blueprint $table): void {
$table->unsignedInteger('releases_id');
$table->string('hash', 32);
$table->primary(['releases_id', 'hash']);
});
Schema::create('release_files', function (Blueprint $table): void {
$table->unsignedInteger('releases_id');
$table->string('name');
$table->string('crc32')->default('');
$table->primary(['releases_id', 'name']);
});
Schema::create('predb', function (Blueprint $table): void {
$table->increments('id');
$table->tinyInteger('searched')->default(0)->index();
$table->dateTime('predate')->nullable()->index();
});
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
}
tests/Feature/PredbFulltextNameFixingTest.php
+107
View File
@@ -0,0 +1,107 @@
<?php
declare(strict_types=1);
namespace Tests\Feature;
use App\Facades\Search;
use App\Services\NameFixing\NameFixingQueryService;
use App\Services\NameFixing\NameFixingService;
use App\Services\NameFixing\ReleaseUpdateService;
use Illuminate\Database\ConnectionInterface;
use RuntimeException;
use Tests\TestCase;
class PredbFulltextNameFixingTest extends TestCase
{
public function test_it_attaches_predb_id_when_the_confirmed_title_is_already_correct(): void
{
$title = 'Valid.Scene.Release.2026-GROUP';
Search::shouldReceive('isAvailable')->once()->andReturnTrue();
Search::shouldReceive('searchReleases')
->once()
->with(['name' => $title, 'searchname' => $title], 21)
->andReturn([42]);
$database = $this->createMock(ConnectionInterface::class);
$database->expects($this->once())
->method('select')
->willReturn([
(object) [
'releases_id' => 42,
'name' => $title,
'searchname' => $title,
'fromname' => 'poster',
'groups_id' => 1,
'categories_id' => 7010,
],
]);
$updates = $this->createMock(ReleaseUpdateService::class);
$updates->expects($this->once())->method('attachPredbId')->with(42, 7);
$service = new NameFixingService(
updateService: $updates,
queries: new NameFixingQueryService($database)
);
$matched = $service->matchPredbFulltext((object) [
'predb_id' => 7,
'title' => $title,
'source' => 'srrdb',
]);
$this->assertSame(1, $matched);
}
public function test_it_marks_sixteen_confirmed_candidates_as_a_flood(): void
{
$title = 'Valid.Scene.Release.2026-GROUP';
Search::shouldReceive('isAvailable')->once()->andReturnTrue();
Search::shouldReceive('searchReleases')->once()->andReturn(range(1, 21));
$database = $this->createStub(ConnectionInterface::class);
$database->method('select')->willReturn(array_map(
static fn (int $id): object => (object) [
'releases_id' => $id,
'name' => $title,
'searchname' => 'Old.Name-'.$id,
'fromname' => 'poster',
'groups_id' => 1,
'categories_id' => 7010,
],
range(1, 16)
));
$service = new NameFixingService(
updateService: $this->createStub(ReleaseUpdateService::class),
queries: new NameFixingQueryService($database)
);
$matched = $service->matchPredbFulltext((object) [
'predb_id' => 7,
'title' => $title,
'source' => 'srrdb',
]);
$this->assertSame(-1, $matched);
}
public function test_backend_unavailability_is_not_recorded_as_a_no_match(): void
{
Search::shouldReceive('isAvailable')->once()->andReturnFalse();
Search::shouldReceive('searchReleases')->never();
$database = $this->createMock(ConnectionInterface::class);
$database->expects($this->never())->method('select');
$service = new NameFixingService(
updateService: $this->createStub(ReleaseUpdateService::class),
queries: new NameFixingQueryService($database)
);
$this->expectException(RuntimeException::class);
$service->matchPredbFulltext((object) [
'predb_id' => 7,
'title' => 'Valid.Scene.Release.2026-GROUP',
'source' => 'srrdb',
]);
}
}
tests/Feature/ReleaseNameFixedRecategorizationTest.php
+23
View File
@@ -222,6 +222,29 @@ class ReleaseNameFixedRecategorizationTest extends TestCase
$this->assertSame(1, (int) $release->isrenamed);
}
public function test_internal_processing_status_updates_do_not_refresh_the_search_index(): void
{
Search::shouldReceive('updateRelease')->once();
$group = UsenetGroup::query()->create([
'name' => 'alt.binaries.test',
'active' => 1,
'backfill' => 0,
]);
$release = Release::factory()->create([
'groups_id' => $group->id,
'guid' => str_repeat('d', 40),
'leftguid' => 'd',
'proc_nfo' => 0,
]);
app(ReleaseUpdateService::class)->updateSingleColumn('proc_nfo', 1, $release->id);
app(ReleaseUpdateService::class)->attachPredbId($release->id, 123);
$this->assertSame(1, (int) $release->fresh()->proc_nfo);
$this->assertSame(123, (int) $release->fresh()->predb_id);
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
tests/Unit/Services/NameFixing/DonorMatchSelectorTest.php
+38
View File
@@ -0,0 +1,38 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Services\NameFixing;
use App\Services\NameFixing\DonorMatchSelector;
use PHPUnit\Framework\TestCase;
class DonorMatchSelectorTest extends TestCase
{
public function test_it_selects_the_closest_donor_with_a_stable_id_tie_breaker(): void
{
$selector = new DonorMatchSelector;
$selected = $selector->select([
(object) ['releases_id' => 20, 'relsize' => 950],
(object) ['releases_id' => 10, 'relsize' => 1050],
(object) ['releases_id' => 30, 'relsize' => 900],
], 1000, 10);
$this->assertNotNull($selected);
$this->assertSame(10, $selected->releases_id);
}
public function test_it_rejects_zero_sized_and_out_of_tolerance_donors(): void
{
$selector = new DonorMatchSelector;
$selected = $selector->select([
(object) ['releases_id' => 10, 'relsize' => 0],
(object) ['releases_id' => 20, 'relsize' => 500],
], 1000, 10);
$this->assertNull($selected);
$this->assertNull($selector->select([(object) ['releases_id' => 30, 'relsize' => 1000]], 0, 10));
}
}
tests/Unit/Services/NameFixing/NameFixingBatchTest.php
+72
View File
@@ -0,0 +1,72 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Services\NameFixing;
use App\Services\NameFixing\NameFixingQueryService;
use App\Services\NameFixing\NameFixingService;
use App\Services\NameFixing\ReleaseUpdateService;
use Illuminate\Database\ConnectionInterface;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
use ReflectionClass;
class NameFixingBatchTest extends TestCase
{
#[Test]
public function grouped_processing_fetches_sources_once_for_the_whole_batch(): void
{
$database = $this->createMock(ConnectionInterface::class);
$database->expects($this->exactly(5))
->method('select')
->willReturnCallback(function (string $sql): array {
$this->assertStringNotContainsString('GROUP_CONCAT', $sql);
$this->assertStringNotContainsString('GROUP BY r.id', $sql);
if (str_contains($sql, 'r.proc_nfo')) {
$this->assertStringContainsString('r.nfostatus > -1', $sql);
return [$this->processedRelease(10), $this->processedRelease(20)];
}
return [];
});
$serviceReflection = new ReflectionClass(NameFixingService::class);
$service = $serviceReflection->newInstanceWithoutConstructor();
$serviceReflection->getProperty('queries')
->setValue($service, new NameFixingQueryService($database));
$serviceReflection->getProperty('updateService')
->setValue(
$service,
(new ReflectionClass(ReleaseUpdateService::class))->newInstanceWithoutConstructor()
);
$this->assertSame(
['checked' => 2, 'fixed' => 0],
$service->processStandardBatch('a', 100, false)
);
}
private function processedRelease(int $id): object
{
return (object) [
'releases_id' => $id,
'guid' => "guid-{$id}",
'name' => "release-{$id}",
'searchname' => "release-{$id}",
'size' => 1000,
'groupName' => 'alt.binaries.test',
'categoryid' => 8010,
'nfostatus' => 0,
'proc_nfo' => NameFixingService::PROC_NFO_DONE,
'proc_files' => NameFixingService::PROC_FILES_DONE,
'proc_par2' => NameFixingService::PROC_PAR2_DONE,
'proc_uid' => NameFixingService::PROC_UID_DONE,
'proc_hash16k' => NameFixingService::PROC_HASH16K_DONE,
'proc_srr' => NameFixingService::PROC_SRR_DONE,
'proc_crc32' => NameFixingService::PROC_CRC_DONE,
];
}
}
tests/Unit/Services/NameFixing/NameFixingQueryServiceTest.php
+60
View File
@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Services\NameFixing;
use App\Services\NameFixing\NameFixingQueryService;
use Illuminate\Database\ConnectionInterface;
use PHPUnit\Framework\TestCase;
class NameFixingQueryServiceTest extends TestCase
{
public function test_it_preserves_every_source_row_when_grouping_by_release(): void
{
$database = $this->createStub(ConnectionInterface::class);
$service = new NameFixingQueryService($database);
$first = (object) ['releases_id' => 10, 'textstring' => 'first.rar'];
$second = (object) ['releases_id' => 10, 'textstring' => 'second.rar'];
$third = (object) ['releases_id' => 20, 'textstring' => 'third.rar'];
$grouped = $service->groupByReleaseId([$first, $second, $third]);
$this->assertSame([$first, $second], $grouped[10]);
$this->assertSame([$third], $grouped[20]);
}
public function test_file_candidates_use_exists_without_lossy_grouping(): void
{
$database = $this->createMock(ConnectionInterface::class);
$database->expects($this->once())
->method('select')
->with(
$this->callback(static fn (string $sql): bool => str_contains($sql, 'EXISTS (SELECT 1 FROM release_files')
&& ! str_contains($sql, 'GROUP BY')),
$this->callback(static fn (array $bindings): bool => $bindings[array_key_last($bindings)] === 100)
)
->willReturn([]);
$service = new NameFixingQueryService($database);
$service->candidateBatch(NameFixingQueryService::SOURCE_FILES, 2, 2, 0, 100);
}
public function test_predb_workers_use_disjoint_modulo_partitions_without_offsets(): void
{
$database = $this->createMock(ConnectionInterface::class);
$database->expects($this->once())
->method('select')
->with(
$this->callback(static fn (string $sql): bool => str_contains($sql, 'MOD(p.id, ?) = ?')
&& ! str_contains($sql, 'OFFSET')),
$this->callback(static fn (array $bindings): bool => $bindings[1] === 4
&& $bindings[2] === 2
&& $bindings[3] === 250)
)
->willReturn([]);
$service = new NameFixingQueryService($database);
$service->predbBatch(3, 4, 250);
}
}