From a10aae7f05799d550f9b677d541a244b9f136a67 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Tue, 4 Aug 2026 08:49:07 +0200 Subject: [PATCH] Update namefixing --- app/Console/Commands/MatchPrefiles.php | 7 +- .../Commands/ReleasesFixNamesGroup.php | 316 +--- .../NameFixing/DonorMatchSelector.php | 48 + app/Services/NameFixing/FilePrioritizer.php | 12 +- .../NameFixing/NameFixingQueryService.php | 481 ++++++ app/Services/NameFixing/NameFixingService.php | 1420 ++++++++--------- .../NameFixing/ReleaseUpdateService.php | 74 +- app/Services/Runners/ReleasesRunner.php | 3 +- .../Tmux/Scripts/groupfixrelnames.php | 6 +- ...439_add_fix_release_name_query_indexes.php | 84 + database/schema/mariadb-schema.sql | 3 + database/schema/mysql-schema.sql | 3 + ...ixReleaseNameQueryIndexesMigrationTest.php | 125 ++ tests/Feature/PredbFulltextNameFixingTest.php | 107 ++ .../ReleaseNameFixedRecategorizationTest.php | 23 + .../NameFixing/DonorMatchSelectorTest.php | 38 + .../NameFixing/NameFixingBatchTest.php | 72 + .../NameFixing/NameFixingQueryServiceTest.php | 60 + 18 files changed, 1889 insertions(+), 993 deletions(-) create mode 100644 app/Services/NameFixing/DonorMatchSelector.php create mode 100644 app/Services/NameFixing/NameFixingQueryService.php create mode 100644 database/migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php create mode 100644 tests/Feature/FixReleaseNameQueryIndexesMigrationTest.php create mode 100644 tests/Feature/PredbFulltextNameFixingTest.php create mode 100644 tests/Unit/Services/NameFixing/DonorMatchSelectorTest.php create mode 100644 tests/Unit/Services/NameFixing/NameFixingBatchTest.php create mode 100644 tests/Unit/Services/NameFixing/NameFixingQueryServiceTest.php diff --git a/app/Console/Commands/MatchPrefiles.php b/app/Console/Commands/MatchPrefiles.php index 507028862..44c11bd33 100644 --- a/app/Console/Commands/MatchPrefiles.php +++ b/app/Console/Commands/MatchPrefiles.php @@ -28,10 +28,8 @@ class MatchPrefiles extends Command /** * Execute the console command. - * - * @return int */ - public function handle() + public function handle(NameFixingService $nameFixingService): int { $limit = $this->argument('limit'); @@ -53,8 +51,7 @@ class MatchPrefiles extends Command } try { - $nameFixingService = new NameFixingService; - $nameFixingService->getPreFileNames($argv); // @phpstan-ignore argument.type + $nameFixingService->getPreFileNames($argv); return 0; } catch (Exception $e) { diff --git a/app/Console/Commands/ReleasesFixNamesGroup.php b/app/Console/Commands/ReleasesFixNamesGroup.php index 93c6027ea..b0cea685d 100644 --- a/app/Console/Commands/ReleasesFixNamesGroup.php +++ b/app/Console/Commands/ReleasesFixNamesGroup.php @@ -4,15 +4,14 @@ declare(strict_types=1); namespace App\Console\Commands; -use App\Models\Category; -use App\Models\Predb; -use App\Models\Release; +use App\Services\NameFixing\NameFixingQueryService; use App\Services\NameFixing\NameFixingService; use App\Services\NfoService; use App\Services\NNTP\NNTPService; use App\Services\Nzb\NzbContentsService; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; +use RuntimeException; class ReleasesFixNamesGroup extends Command { @@ -25,7 +24,8 @@ class ReleasesFixNamesGroup extends Command {type : Type of fix (standard|predbft)} {--guid-char= : GUID character to process (for standard type)} {--limit=1000 : Maximum releases to process} - {--thread=1 : Thread number (for predbft type)}'; + {--thread=1 : Worker number (for predbft type)} + {--workers=1 : Total workers (for predbft type)}'; /** * The console command description. @@ -36,6 +36,8 @@ class ReleasesFixNamesGroup extends Command private NameFixingService $nameFixingService; + private NameFixingQueryService $queries; + private int $checked = 0; /** @@ -46,7 +48,8 @@ class ReleasesFixNamesGroup extends Command $type = $this->argument('type'); $maxPerRun = (int) $this->option('limit'); - $this->nameFixingService = new NameFixingService; + $this->nameFixingService = app(NameFixingService::class); + $this->queries = app(NameFixingQueryService::class); switch ($type) { case 'standard': @@ -78,173 +81,52 @@ class ReleasesFixNamesGroup extends Command $this->info("Processing releases with GUID starting with: {$guidChar}"); $this->info("Maximum per run: {$maxPerRun}"); - // Allow for larger filename return sets - DB::statement('SET SESSION group_concat_max_len = 65536'); + $nntp = null; + $nzbcontents = null; + $connectionAttempted = false; + $par2Processor = function (object $release) use (&$connectionAttempted, &$nntp, &$nzbcontents): bool { + if (! $connectionAttempted) { + $connectionAttempted = true; + $nntp = app(NNTPService::class); + $compressedHeaders = config('nntmux_nntp.compressed_headers'); + $connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true + ? $nntp->doConnect($compressedHeaders, true) + : $nntp->doConnect(); - // Find releases to process - $releases = $this->fetchReleases($guidChar, $maxPerRun); + if ($connectResult !== true) { + $errorMessage = 'Unable to connect to usenet for PAR2 processing'; + if (NNTPService::isError($connectResult)) { + $errorMessage .= ' Error: '.$connectResult->getMessage(); + } + $this->warn($errorMessage); + } else { + $nzbcontents = app(NzbContentsService::class); + $nzbcontents->setNntp($nntp); + $nzbcontents->setNfo(app(NfoService::class)); + $nzbcontents->setEchoOutput(false); + } + } - if ($releases->isEmpty()) { + return $nzbcontents !== null && $nzbcontents->checkPar2( + $release->guid, + $release->releases_id, + $release->groups_id, + 1, + 1 + ); + }; + + $stats = $this->nameFixingService->processStandardBatch($guidChar, $maxPerRun, true, $par2Processor); + $this->checked += $stats['checked']; + + if ($stats['checked'] === 0) { $this->info('No releases to process'); return Command::SUCCESS; } - $this->info("Found {$releases->count()} releases to process"); - $bar = $this->output->createProgressBar($releases->count()); - $bar->start(); - - $nntp = null; - $nzbcontents = null; - - foreach ($releases as $release) { - $this->checked++; - $this->nameFixingService->reset(); - - // Process UID - if ((int) $release->proc_uid === NameFixingService::PROC_UID_NONE && - (! empty($release->uid) || ! empty($release->mediainfo))) { - - if (! empty($release->uid)) { - $this->nameFixingService->checkName($release, true, 'UID, ', true, true); - } - - if (! $this->nameFixingService->getUpdateService()->matched && ! empty($release->mediainfo)) { - $this->nameFixingService->checkName($release, true, 'Mediainfo, ', true, true); - } - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_uid', NameFixingService::PROC_UID_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process CRC32 - if ((int) $release->proc_crc32 === NameFixingService::PROC_CRC_NONE && ! empty($release->crc)) { - $this->nameFixingService->reset(); - $this->nameFixingService->checkName($release, true, 'CRC32, ', true, true); - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_crc32', NameFixingService::PROC_CRC_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process SRR - if ((int) $release->proc_srr === NameFixingService::PROC_SRR_NONE) { - $this->nameFixingService->reset(); - $this->nameFixingService->checkName($release, true, 'SRR, ', true, true); - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_srr', NameFixingService::PROC_SRR_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process PAR2 hash - if ((int) $release->proc_hash16k === NameFixingService::PROC_HASH16K_NONE && ! empty($release->hash)) { - $this->nameFixingService->reset(); - $this->nameFixingService->checkName($release, true, 'PAR2 hash, ', true, true); - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_hash16k', NameFixingService::PROC_HASH16K_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process NFO - if ((int) $release->nfostatus === NfoService::NFO_FOUND && - (int) $release->proc_nfo === NameFixingService::PROC_NFO_NONE && - ! empty($release->textstring) && - ! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) { - - $this->nameFixingService->reset(); - $this->nameFixingService->checkName($release, true, 'NFO, ', true, true); - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_nfo', NameFixingService::PROC_NFO_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process filenames - if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixingService::PROC_FILES_NONE) { - $this->nameFixingService->reset(); - $fileNames = explode('|', $release->filestring); - - $releaseFile = $release; - foreach ($fileNames as $fileName) { - if (! $this->nameFixingService->getUpdateService()->matched) { - $releaseFile->textstring = $fileName; - $this->nameFixingService->checkName($releaseFile, true, 'Filenames, ', true, true); - } - } - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_files', NameFixingService::PROC_FILES_DONE, $release->releases_id); - - if ($this->nameFixingService->getUpdateService()->matched) { - $bar->advance(); - - continue; - } - - // Process PAR2 - if ((int) $release->proc_par2 === NameFixingService::PROC_PAR2_NONE) { - // Initialize NZB contents if needed - if (! isset($nzbcontents)) { - $nntp = new NNTPService; - $compressedHeaders = config('nntmux_nntp.compressed_headers'); - - $connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true - ? $nntp->doConnect($compressedHeaders, true) - : $nntp->doConnect(); - - if ($connectResult !== true) { - $errorMessage = 'Unable to connect to usenet for PAR2 processing'; - if (NNTPService::isError($connectResult)) { - $errorMessage .= ' Error: '.$connectResult->getMessage(); - } - $this->warn($errorMessage); - } else { - $Nfo = new NfoService; - $nzbcontents = app(NzbContentsService::class); - $nzbcontents->setNntp($nntp); - $nzbcontents->setNfo($Nfo); - $nzbcontents->setEchoOutput(false); - } - } - - if (isset($nzbcontents)) { - $nzbcontents->checkPar2($release->guid, $release->releases_id, $release->groups_id, 1, 1); - } - } - - $this->nameFixingService->getUpdateService()->updateSingleColumn('proc_par2', NameFixingService::PROC_PAR2_DONE, $release->releases_id); - - $bar->advance(); - } - - $bar->finish(); - $this->newLine(2); - - $this->info("✅ Processed {$this->checked} releases"); - $this->info("✅ Fixed {$this->nameFixingService->getUpdateService()->fixed} release names"); + $this->info("Processed {$stats['checked']} releases"); + $this->info("Fixed {$stats['fixed']} release names"); return Command::SUCCESS; } @@ -255,51 +137,45 @@ class ReleasesFixNamesGroup extends Command protected function processPredbFulltext(int $maxPerRun): int { $thread = (int) $this->option('thread'); - $offset = $thread * $maxPerRun - $maxPerRun; + $workers = (int) $this->option('workers'); $this->info('Processing PreDB fulltext matching'); - $this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}"); + $this->info("Worker: {$thread}/{$workers}, Limit: {$maxPerRun}"); - $pres = Predb::fromQuery( - sprintf( - ' - SELECT p.id AS predb_id, p.title, p.source, p.searched - FROM predb p - WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]" - AND p.searched = 0 - AND p.predate < (NOW() - INTERVAL 1 DAY) - ORDER BY p.predate ASC - LIMIT %s - OFFSET %s', - $maxPerRun, - $offset - ) - ); + $pres = $this->queries->predbBatch($thread, $workers, $maxPerRun); - if ($pres->isEmpty()) { + if ($pres === []) { $this->info('No PreDB entries to process'); return Command::SUCCESS; } - $this->info("Found {$pres->count()} PreDB entries to process"); - $bar = $this->output->createProgressBar($pres->count()); + $this->info('Found '.count($pres).' PreDB entries to process'); + $bar = $this->output->createProgressBar(count($pres)); $bar->start(); foreach ($pres as $pre) { $searched = 0; - $ftmatched = $this->matchPredbFT($pre); + try { + $ftmatched = $this->nameFixingService->matchPredbFulltext($pre); + } catch (RuntimeException $exception) { + $bar->finish(); + $this->newLine(2); + $this->error($exception->getMessage()); + + return Command::FAILURE; + } if ($ftmatched > 0) { $searched = 1; } elseif ($ftmatched < 0) { $searched = -6; } else { - $searched = $pre['searched'] - 1; + $searched = (int) $pre->searched - 1; } - Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]); + DB::update('UPDATE predb SET searched = ? WHERE id = ?', [$searched, $pre->predb_id]); $this->checked++; $bar->advance(); @@ -312,68 +188,4 @@ class ReleasesFixNamesGroup extends Command return Command::SUCCESS; } - - /** - * Match a PreDB title to releases using full-text search. - */ - protected function matchPredbFT(object $pre): int - { - // This is a simplified version - the full implementation would use Manticore/Elasticsearch - return 0; - } - - /** - * Fetch releases for processing - */ - protected function fetchReleases(string $guidChar, int $maxPerRun): mixed - { - return Release::fromQuery(sprintf(" - SELECT - r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo, - r.proc_uid, r.proc_files, r.proc_par2, r.nfostatus, - r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32, - IFNULL(rf.releases_id, 0) AS fileid, - IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring, - IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring, - IFNULL(ru.uniqueid, '') AS uid, - IFNULL(ph.hash, 0) AS hash, - IFNULL(rf.crc32, '') AS crc - FROM releases r - LEFT JOIN release_nfos rn ON rn.releases_id = r.id - LEFT JOIN release_files rf ON rf.releases_id = r.id - LEFT JOIN release_unique ru ON ru.releases_id = r.id - LEFT JOIN par_hashes ph ON ph.releases_id = r.id - WHERE r.leftguid = %s - AND r.isrenamed = %d - AND r.predb_id = 0 - AND r.passwordstatus >= 0 - AND r.nfostatus > %d - AND ( - (r.nfostatus = %d AND r.proc_nfo = %d) - OR r.proc_files = %d - OR r.proc_uid = %d - OR r.proc_par2 = %d - OR r.proc_srr = %d - OR r.proc_hash16k = %d - OR r.proc_crc32 = %d - ) - AND r.categories_id IN (%s) - GROUP BY r.id - ORDER BY r.id DESC - LIMIT %s", - escapeString($guidChar), - NameFixingService::IS_RENAMED_NONE, - NfoService::NFO_UNPROC, - NfoService::NFO_FOUND, - NameFixingService::PROC_NFO_NONE, - NameFixingService::PROC_FILES_NONE, - NameFixingService::PROC_UID_NONE, - NameFixingService::PROC_PAR2_NONE, - NameFixingService::PROC_SRR_NONE, - NameFixingService::PROC_HASH16K_NONE, - NameFixingService::PROC_CRC_NONE, - Category::getCategoryOthersGroup(), - $maxPerRun - )); - } } diff --git a/app/Services/NameFixing/DonorMatchSelector.php b/app/Services/NameFixing/DonorMatchSelector.php new file mode 100644 index 000000000..2e45def63 --- /dev/null +++ b/app/Services/NameFixing/DonorMatchSelector.php @@ -0,0 +1,48 @@ + $donors + */ + public function select(array $donors, int $targetSize, int $tolerancePercent): ?object + { + if ($targetSize <= 0 || $donors === []) { + return null; + } + + $eligible = []; + + foreach ($donors as $donor) { + $donorSize = (int) ($donor->relsize ?? $donor->size ?? 0); + if ($donorSize <= 0) { + continue; + } + + $difference = abs($donorSize - $targetSize); + $differencePercent = ($difference / $donorSize) * 100; + + if ($differencePercent <= $tolerancePercent) { + $eligible[] = [ + 'donor' => $donor, + 'difference' => $difference, + 'id' => (int) ($donor->releases_id ?? $donor->id ?? PHP_INT_MAX), + ]; + } + } + + if ($eligible === []) { + return null; + } + + usort($eligible, static function (array $left, array $right): int { + return [$left['difference'], $left['id']] <=> [$right['difference'], $right['id']]; + }); + + return $eligible[0]['donor']; + } +} diff --git a/app/Services/NameFixing/FilePrioritizer.php b/app/Services/NameFixing/FilePrioritizer.php index 7a37d88be..251c18087 100644 --- a/app/Services/NameFixing/FilePrioritizer.php +++ b/app/Services/NameFixing/FilePrioritizer.php @@ -58,8 +58,8 @@ class FilePrioritizer * 5. NFO files * 6. Other files * - * @param array $files Array of filenames - * @return array Sorted array of filenames + * @param list $files Array of filenames + * @return list Sorted array of filenames */ public function prioritizeForMatching(array $files): array { @@ -82,8 +82,8 @@ class FilePrioritizer * Similar to prioritizeForMatching but with slightly different priorities * optimized for PreDB filename lookups. * - * @param array $files Array of filenames - * @return array Sorted array of filenames + * @param list $files Array of filenames + * @return list Sorted array of filenames */ public function prioritizeForPreDb(array $files): array { @@ -102,8 +102,8 @@ class FilePrioritizer /** * Categorize files into groups by type. * - * @param array $files Array of filenames - * @return array Categorized files + * @param list $files Array of filenames + * @return array{srr: list, mainRar: list, firstPart: list, video: list, nfo: list, other: list} Categorized files */ protected function categorizeFiles(array $files): array { diff --git a/app/Services/NameFixing/NameFixingQueryService.php b/app/Services/NameFixing/NameFixingQueryService.php new file mode 100644 index 000000000..218b13aeb --- /dev/null +++ b/app/Services/NameFixing/NameFixingQueryService.php @@ -0,0 +1,481 @@ + + */ + private const STATUS_COLUMNS = [ + self::SOURCE_NFO => 'proc_nfo', + self::SOURCE_FILES => 'proc_files', + self::SOURCE_SRR => 'proc_srr', + self::SOURCE_CRC => 'proc_crc32', + self::SOURCE_UID => 'proc_uid', + self::SOURCE_HASH => 'proc_hash16k', + self::SOURCE_PAR2 => 'proc_par2', + self::SOURCE_XXX => 'proc_files', + self::SOURCE_MEDIA_MOVIE => 'proc_uid', + ]; + + /** + * @var array + */ + private const SOURCE_EXISTS = [ + self::SOURCE_NFO => 'EXISTS (SELECT 1 FROM release_nfos source_nfo WHERE source_nfo.releases_id = r.id)', + self::SOURCE_FILES => 'EXISTS (SELECT 1 FROM release_files source_file WHERE source_file.releases_id = r.id)', + self::SOURCE_SRR => "EXISTS (SELECT 1 FROM release_files source_srr WHERE source_srr.releases_id = r.id AND (source_srr.name LIKE '%.srr' OR source_srr.name LIKE '%.srs'))", + self::SOURCE_CRC => "EXISTS (SELECT 1 FROM release_files source_crc WHERE source_crc.releases_id = r.id AND source_crc.crc32 IS NOT NULL AND source_crc.crc32 != '')", + self::SOURCE_UID => "EXISTS (SELECT 1 FROM media_infos source_media WHERE source_media.releases_id = r.id AND source_media.unique_id IS NOT NULL AND source_media.unique_id != '') OR EXISTS (SELECT 1 FROM release_unique source_unique WHERE source_unique.releases_id = r.id AND source_unique.uniqueid != '')", + self::SOURCE_HASH => "EXISTS (SELECT 1 FROM par_hashes source_hash WHERE source_hash.releases_id = r.id AND source_hash.hash != '')", + self::SOURCE_PAR2 => '1 = 1', + self::SOURCE_XXX => "EXISTS (SELECT 1 FROM release_files source_xxx WHERE source_xxx.releases_id = r.id AND source_xxx.name LIKE '%SDPORN%')", + self::SOURCE_MEDIA_MOVIE => "EXISTS (SELECT 1 FROM media_infos source_movie WHERE source_movie.releases_id = r.id AND source_movie.movie_name IS NOT NULL AND source_movie.movie_name != '')", + ]; + + public function __construct(?ConnectionInterface $database = null) + { + $this->database = $database ?? DB::connection(); + } + + /** + * @return list + */ + public function candidateBatch( + string $source, + int $time, + int $categories, + int $afterId = 0, + int $limit = self::BATCH_SIZE + ): array { + [$where, $bindings] = $this->candidateWhere($source, $time, $categories); + $bindings[] = $afterId; + $bindings[] = max(1, $limit); + + return $this->database->select( + "SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.groups_id, + r.categories_id, r.size AS relsize, r.guid, r.predb_id, r.nfostatus, + r.proc_nfo, r.proc_files, r.proc_par2, r.proc_uid, r.proc_srr, + r.proc_hash16k, r.proc_crc32 + FROM releases r + WHERE {$where} + AND r.id > ? + ORDER BY r.id ASC + LIMIT ?", + $bindings + ); + } + + public function countCandidates(string $source, int $time, int $categories): int + { + [$where, $bindings] = $this->candidateWhere($source, $time, $categories); + $rows = $this->database->select( + "SELECT COUNT(*) AS aggregate FROM releases r WHERE {$where}", + $bindings + ); + + return (int) ($rows[0]->aggregate ?? 0); + } + + /** + * @return list + */ + public function standardCandidateBatch(string $leftGuid, int $limit): array + { + return $this->database->select( + 'SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.guid, + r.groups_id, r.categories_id, r.size AS relsize, r.predb_id, r.nfostatus, + r.proc_nfo, r.proc_uid, r.proc_files, r.proc_par2, r.proc_hash16k, + r.proc_srr, r.proc_crc32 + FROM releases r + WHERE r.leftguid = ? + AND r.isrenamed = 0 + AND r.predb_id = 0 + AND r.passwordstatus >= 0 + AND r.nfostatus > -1 + AND ( + (r.nfostatus = 1 AND r.proc_nfo = 0) + OR r.proc_files = 0 + OR r.proc_uid = 0 + OR r.proc_par2 = 0 + OR r.proc_srr = 0 + OR r.proc_hash16k = 0 + OR r.proc_crc32 = 0 + ) + AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).') + ORDER BY r.id DESC + LIMIT ?', + [$leftGuid, max(1, $limit)] + ); + } + + /** + * @param list $releaseIds + * @return list + */ + public function nfoRows(array $releaseIds): array + { + return $this->selectForReleaseIds( + 'SELECT rn.releases_id, UNCOMPRESS(rn.nfo) AS textstring + FROM release_nfos rn + WHERE rn.releases_id IN (%s) + ORDER BY rn.releases_id', + $releaseIds + ); + } + + /** + * @param list $releaseIds + * @return list + */ + public function fileRows(array $releaseIds, string $source = self::SOURCE_FILES): array + { + $filter = match ($source) { + self::SOURCE_FILES => '', + self::SOURCE_SRR => " AND (rf.name LIKE '%.srr' OR rf.name LIKE '%.srs')", + self::SOURCE_CRC => " AND rf.crc32 IS NOT NULL AND rf.crc32 != ''", + self::SOURCE_XXX => " AND rf.name LIKE '%SDPORN%'", + default => throw new InvalidArgumentException("Unsupported release-file source [{$source}]."), + }; + + return $this->selectForReleaseIds( + 'SELECT rf.releases_id, rf.name AS textstring, rf.name AS filename, rf.crc32 + FROM release_files rf + WHERE rf.releases_id IN (%s)'.$filter.' + ORDER BY rf.releases_id, rf.name', + $releaseIds + ); + } + + /** + * @param list $releaseIds + * @return list + */ + public function mediaRows(array $releaseIds): array + { + return $this->selectForReleaseIds( + 'SELECT mi.releases_id, mi.unique_id AS uid, mi.movie_name, mi.file_name + FROM media_infos mi + WHERE mi.releases_id IN (%s) + UNION ALL + SELECT ru.releases_id, ru.uniqueid AS uid, NULL AS movie_name, NULL AS file_name + FROM release_unique ru + WHERE ru.releases_id IN (%s) + ORDER BY releases_id', + array_merge($releaseIds, $releaseIds), + 2 + ); + } + + /** + * @param list $releaseIds + * @return list + */ + public function hashRows(array $releaseIds): array + { + return $this->selectForReleaseIds( + 'SELECT ph.releases_id, ph.hash + FROM par_hashes ph + WHERE ph.releases_id IN (%s) + ORDER BY ph.releases_id, ph.hash', + $releaseIds + ); + } + + /** + * @param list $uniqueIds + * @return array> + */ + public function uidDonors(array $uniqueIds): array + { + if ($uniqueIds === []) { + return []; + } + + $placeholders = $this->placeholders(count($uniqueIds)); + $bindings = array_merge($uniqueIds, ['nonscene@Ef.net (EF)'], $uniqueIds); + $rows = $this->database->select( + "SELECT mi.unique_id AS match_key, r.id AS releases_id, r.size AS relsize, + r.searchname, r.fromname, r.predb_id + FROM media_infos mi + INNER JOIN releases r ON r.id = mi.releases_id + WHERE mi.unique_id IN ({$placeholders}) + AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = ?) + UNION ALL + SELECT ru.uniqueid AS match_key, r.id AS releases_id, r.size AS relsize, + r.searchname, r.fromname, r.predb_id + FROM release_unique ru + INNER JOIN releases r ON r.id = ru.releases_id + WHERE ru.uniqueid IN ({$placeholders}) + AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = 'nonscene@Ef.net (EF)')", + $bindings + ); + + return $this->groupByKey($rows, 'match_key'); + } + + /** + * @param list $hashes + * @return array> + */ + public function hashDonors(array $hashes): array + { + return $this->donors( + 'SELECT ph.hash AS match_key, r.id AS releases_id, r.size AS relsize, + r.searchname, r.fromname, r.predb_id + FROM par_hashes ph + INNER JOIN releases r ON r.id = ph.releases_id + WHERE ph.hash IN (%s) + AND (r.predb_id > 0 OR r.anidbid > 0)', + $hashes + ); + } + + /** + * @param list $crcs + * @return array> + */ + public function crcDonors(array $crcs): array + { + return $this->donors( + 'SELECT rf.crc32 AS match_key, r.id AS releases_id, r.size AS relsize, + r.searchname, r.fromname, r.predb_id + FROM release_files rf + INNER JOIN releases r ON r.id = rf.releases_id + WHERE rf.crc32 IN (%s) + AND r.predb_id > 0', + $crcs + ); + } + + /** + * @return list + */ + public function predbBatch(int $worker, int $workers, int $limit): array + { + $workerCount = max(1, min(16, $workers)); + $workerSlot = max(1, min($workerCount, $worker)) - 1; + + return $this->database->select( + 'SELECT p.id AS predb_id, p.title, p.source, p.searched + FROM predb p + WHERE LENGTH(p.title) >= 15 + AND p.title NOT REGEXP \'["<> ]\' + AND p.searched = 0 + AND p.predate < ? + AND MOD(p.id, ?) = ? + ORDER BY p.predate ASC, p.id ASC + LIMIT ?', + [ + CarbonImmutable::now()->subDay()->toDateTimeString(), + $workerCount, + $workerSlot, + max(1, $limit), + ] + ); + } + + /** + * @param list $candidateIds + * @return list + */ + public function confirmPredbCandidates(array $candidateIds, string $title): array + { + if ($candidateIds === []) { + return []; + } + + $like = '%'.$this->escapeLike($title).'%'; + + return $this->database->select( + 'SELECT r.id AS releases_id, r.name, r.fromname, r.searchname, + r.groups_id, r.categories_id + FROM releases r + WHERE r.id IN ('.$this->placeholders(count($candidateIds)).') + AND r.predb_id = 0 + AND (r.name LIKE ? ESCAPE \'\\\\\' OR r.searchname LIKE ? ESCAPE \'\\\\\') + ORDER BY r.id ASC + LIMIT 21', + array_merge($candidateIds, [$like, $like]) + ); + } + + public function countPrefileCandidates(): int + { + $rows = $this->database->select( + 'SELECT COUNT(*) AS aggregate + FROM releases r + WHERE r.predb_id = 0 + AND r.isrenamed = 0 + AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).') + AND EXISTS ( + SELECT 1 FROM release_files rf + WHERE rf.releases_id = r.id + AND rf.name IS NOT NULL + )' + ); + + return (int) ($rows[0]->aggregate ?? 0); + } + + /** + * @return list + */ + public function prefileCandidateBatch(int $cursor, int $limit, bool $descending): array + { + $operator = $descending ? '<' : '>'; + $direction = $descending ? 'DESC' : 'ASC'; + + return $this->database->select( + 'SELECT r.id AS releases_id, r.name, r.searchname, r.fromname, + r.groups_id, r.categories_id + FROM releases r + WHERE r.predb_id = 0 + AND r.isrenamed = 0 + AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).") + AND r.id {$operator} ? + AND EXISTS ( + SELECT 1 FROM release_files rf + WHERE rf.releases_id = r.id + AND rf.name IS NOT NULL + ) + ORDER BY r.id {$direction} + LIMIT ?", + [$cursor, max(1, $limit)] + ); + } + + /** + * @param list $rows + * @return array> + */ + public function groupByReleaseId(array $rows): array + { + return $this->groupByKey($rows, 'releases_id'); + } + + /** + * @return array{0: string, 1: list} + */ + private function candidateWhere(string $source, int $time, int $categories): array + { + if (! isset(self::SOURCE_EXISTS[$source], self::STATUS_COLUMNS[$source])) { + throw new InvalidArgumentException("Unsupported name-fixing source [{$source}]."); + } + + $where = ['r.predb_id = 0', '('.self::SOURCE_EXISTS[$source].')']; + $bindings = []; + + if ($categories !== 3) { + $statusColumn = self::STATUS_COLUMNS[$source]; + $where[] = '(r.isrenamed = 0 OR r.categories_id IN (?, ?))'; + $bindings[] = Category::OTHER_MISC; + $bindings[] = Category::OTHER_HASHED; + $where[] = "r.{$statusColumn} = 0"; + } + + if ($time === 1) { + $where[] = 'r.adddate > ?'; + $bindings[] = CarbonImmutable::now()->subHours(6)->toDateTimeString(); + } + + if ($categories === 1) { + $where[] = 'r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')'; + } + + return [implode(' AND ', $where), $bindings]; + } + + /** + * @param list $releaseIds + * @return list + */ + private function selectForReleaseIds( + string $sql, + array $releaseIds, + int $placeholderGroups = 1 + ): array { + if ($releaseIds === []) { + return []; + } + + $placeholder = $this->placeholders((int) (count($releaseIds) / $placeholderGroups)); + $sql = vsprintf($sql, array_fill(0, $placeholderGroups, $placeholder)); + + return $this->database->select($sql, $releaseIds); + } + + /** + * @param list $values + * @return array> + */ + private function donors(string $sql, array $values): array + { + if ($values === []) { + return []; + } + + $rows = $this->database->select( + sprintf($sql, $this->placeholders(count($values))), + $values + ); + + return $this->groupByKey($rows, 'match_key'); + } + + /** + * @param list $rows + * @return array> + */ + private function groupByKey(array $rows, string $key): array + { + $grouped = []; + + foreach ($rows as $row) { + $grouped[$row->{$key}][] = $row; + } + + return $grouped; + } + + private function placeholders(int $count): string + { + return implode(',', array_fill(0, $count, '?')); + } + + private function escapeLike(string $value): string + { + return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value); + } +} diff --git a/app/Services/NameFixing/NameFixingService.php b/app/Services/NameFixing/NameFixingService.php index 6c9b035df..00e3cc9ea 100644 --- a/app/Services/NameFixing/NameFixingService.php +++ b/app/Services/NameFixing/NameFixingService.php @@ -5,12 +5,11 @@ declare(strict_types=1); namespace App\Services\NameFixing; use App\Facades\Search; -use App\Models\Category; -use App\Models\Release; use App\Services\NameFixing\Extractors\FileNameExtractor; use App\Services\NameFixing\Extractors\NfoNameExtractor; use App\Services\NNTP\NNTPService; use App\Services\Nzb\NzbContentsService; +use RuntimeException; /** * Main service for name fixing operations. @@ -68,18 +67,17 @@ class NameFixingService protected PredbMatchSelector $predbMatchSelector; + protected NameFixingQueryService $queries; + + protected DonorMatchSelector $donorMatchSelector; + + /** + * @var array|null> + */ + protected array $predbMatchCache = []; + protected bool $echoOutput; - protected string $othercats; - - protected string $timeother; - - protected string $timeall; - - protected string $fullother; - - protected string $fullall; - protected int $_totalReleases = 0; public function __construct( @@ -89,7 +87,9 @@ class NameFixingService ?FileNameExtractor $fileExtractor = null, ?FileNameCleaner $fileNameCleaner = null, ?FilePrioritizer $filePrioritizer = null, - ?PredbMatchSelector $predbMatchSelector = null + ?PredbMatchSelector $predbMatchSelector = null, + ?NameFixingQueryService $queries = null, + ?DonorMatchSelector $donorMatchSelector = null ) { $this->updateService = $updateService ?? new ReleaseUpdateService; $this->checkerService = $checkerService ?? new NameCheckerService; @@ -98,13 +98,9 @@ class NameFixingService $this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner; $this->filePrioritizer = $filePrioritizer ?? new FilePrioritizer; $this->predbMatchSelector = $predbMatchSelector ?? new PredbMatchSelector($this->fileNameCleaner); + $this->queries = $queries ?? new NameFixingQueryService; + $this->donorMatchSelector = $donorMatchSelector ?? new DonorMatchSelector; $this->echoOutput = config('nntmux.echocli'); - - $this->othercats = implode(',', Category::OTHERS_GROUP); - $this->timeother = sprintf(' AND rel.adddate > (NOW() - INTERVAL 6 HOUR) AND rel.categories_id IN (%s) GROUP BY rel.id ORDER BY postdate DESC', $this->othercats); - $this->timeall = ' AND rel.adddate > (NOW() - INTERVAL 6 HOUR) GROUP BY rel.id ORDER BY postdate DESC'; - $this->fullother = sprintf(' AND rel.categories_id IN (%s) GROUP BY rel.id', $this->othercats); - $this->fullall = ''; } /** @@ -114,68 +110,38 @@ class NameFixingService { $this->echoStartMessage($time, '.nfo files'); $type = 'NFO, '; + $preIdOnly = $cats === 3; - $preId = false; - if ($cats === 3) { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.fromname - FROM releases rel - INNER JOIN release_nfos nfo ON (nfo.releases_id = rel.id) - WHERE rel.predb_id = 0' - ); - $cats = 2; - $preId = true; - } else { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.fromname - FROM releases rel - INNER JOIN release_nfos nfo ON (nfo.releases_id = rel.id) - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND rel.proc_nfo = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_NFO_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_NFO, $time, $cats, ' releases to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); - - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' releases to process.'); - - foreach ($releases as $rel) { - /** @var Release $rel */ - $releaseRow = Release::fromQuery( - sprintf( - 'SELECT nfo.releases_id AS nfoid, rel.groups_id, rel.fromname, rel.categories_id, rel.name, rel.searchname, - UNCOMPRESS(nfo) AS textstring, rel.id AS releases_id - FROM releases rel - INNER JOIN release_nfos nfo ON (nfo.releases_id = rel.id) - WHERE rel.id = %d LIMIT 1', - $rel->releases_id - ) - ); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_NFO, $time, $cats) as $releases) { + $nfos = $this->queries->groupByReleaseId($this->queries->nfoRows($this->releaseIds($releases))); + foreach ($releases as $release) { $this->updateService->incrementChecked(); - - // Ignore encrypted NFOs - if (preg_match('/^=newz\[NZB\]=\w+/', $releaseRow[0]->textstring)) { - $this->updateService->updateSingleColumn('proc_nfo', self::PROC_NFO_DONE, $rel->releases_id); + $this->updateService->reset(); + $nfo = $nfos[(int) $release->releases_id][0] ?? null; + if ($nfo === null) { + $this->markProcessed($echo, $nameStatus, 'proc_nfo', (int) $release->releases_id); continue; } - $this->updateService->reset(); + $release->textstring = (string) ($nfo->textstring ?? ''); + if (preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) { + $this->markProcessed($echo, $nameStatus, 'proc_nfo', (int) $release->releases_id); - // Try NFO extraction - $nfoResult = $this->nfoExtractor->extractFromNfo($releaseRow[0]->textstring); + continue; + } + + $nfoResult = $this->nfoExtractor->extractFromNfo($release->textstring); if ($nfoResult !== null) { $this->updateService->updateRelease( - $releaseRow[0], + $release, $nfoResult->newName, 'nfoCheck: '.$nfoResult->method, $echo, @@ -185,17 +151,19 @@ class NameFixingService ); } - // If NFO extraction didn't work, try pattern checkers if (! $this->updateService->matched) { - $this->checkWithPatternMatchers($releaseRow[0], $echo, $type, $nameStatus, $show, $preId); + $this->checkWithPatternMatchers($release, $echo, $type, $nameStatus, $show, $preIdOnly); + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_nfo', (int) $release->releases_id); } $this->echoRenamed($show); } - $this->echoFoundCount($echo, ' NFO\'s'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' NFO\'s'); } /** @@ -206,101 +174,21 @@ class NameFixingService $this->echoStartMessage($time, 'file names'); $type = 'Filenames, '; - $preId = false; - if ($cats === 3) { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE predb_id = 0' - ); - $cats = 2; - $preId = true; - } else { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND proc_files = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_FILES_NONE - ); - } - - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); - - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' file names to process.'); - - // Group files by release - $releaseFiles = []; - foreach ($releases as $release) { - /** @var Release $release */ - $releaseId = $release->releases_id; - if (! isset($releaseFiles[$releaseId])) { - $releaseFiles[$releaseId] = [ - 'release' => $release, - 'files' => [], - ]; - } - $releaseFiles[$releaseId]['files'][] = $release->textstring; - } - - foreach ($releaseFiles as $releaseId => $data) { - $this->updateService->reset(); - $this->updateService->incrementChecked(); - - // Prioritize files for matching - $prioritizedFiles = $this->filePrioritizer->prioritizeForMatching($data['files']); // @phpstan-ignore argument.type - - foreach ($prioritizedFiles as $filename) { - /** @var Release $release */ - $release = clone $data['release']; - $release->textstring = $filename; - - // Try file name extraction - $fileResult = $this->fileExtractor->extractFromFile($filename); - if ($fileResult !== null) { - $this->updateService->updateRelease( - $release, - $fileResult->newName, - 'fileCheck: '.$fileResult->method, - $echo, - $type, - $nameStatus, - $show - ); - } - - // If not matched, try PreDB search - if (! $this->updateService->matched) { - $this->preDbFileCheck($release, $echo, $type, $nameStatus, $show); - } - - if (! $this->updateService->matched) { - $this->preDbTitleCheck($release, $echo, $type, $nameStatus, $show); - } - - if ($this->updateService->matched) { - break; - } - } - - $this->echoRenamed($show); - } - - $this->echoFoundCount($echo, ' files'); - } else { + if (! $this->startBatch(NameFixingQueryService::SOURCE_FILES, $time, $cats, ' file names to process.')) { cli()->info('Nothing to fix.'); + + return; } + + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_FILES, $time, $cats) as $releases) { + $files = $this->queries->groupByReleaseId($this->queries->fileRows($this->releaseIds($releases))); + + foreach ($releases as $release) { + $this->processFileCandidates($release, $files[(int) $release->releases_id] ?? [], $echo, $nameStatus, $show); + } + } + + $this->echoFoundCount($echo, ' files'); } /** @@ -311,56 +199,38 @@ class NameFixingService $this->echoStartMessage($time, 'SRR file names'); $type = 'SRR, '; - if ($cats === 3) { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON (rf.releases_id = rel.id) - WHERE predb_id = 0 - AND (rf.name LIKE %s OR rf.name LIKE %s)', - escapeString('%.srr'), - escapeString('%.srs') - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON (rf.releases_id = rel.id) - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND (rf.name LIKE %s OR rf.name LIKE %s) - AND rel.proc_srr = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - escapeString('%.srr'), - escapeString('%.srs'), - self::PROC_SRR_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_SRR, $time, $cats, ' srr file extensions to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); - - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' srr file extensions to process.'); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_SRR, $time, $cats) as $releases) { + $files = $this->queries->groupByReleaseId( + $this->queries->fileRows($this->releaseIds($releases), NameFixingQueryService::SOURCE_SRR) + ); foreach ($releases as $release) { $this->updateService->reset(); $this->updateService->incrementChecked(); - $this->srrNameCheck($release, $echo, $type, $nameStatus, $show); + foreach ($files[(int) $release->releases_id] ?? [] as $file) { + $candidate = clone $release; + $candidate->textstring = (string) $file->textstring; + if ($this->srrNameCheck($candidate, $echo, $type, $nameStatus, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_srr', (int) $release->releases_id); + } + $this->echoRenamed($show); } - - $this->echoFoundCount($echo, ' files'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' files'); } /** @@ -371,87 +241,47 @@ class NameFixingService $this->echoStartMessage($time, 'CRC32'); $type = 'CRC32, '; - $preId = false; - if ($cats === 3) { - $query = sprintf( - 'SELECT rf.crc32 AS textstring, rf.name AS filename, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE predb_id = 0 - AND rf.crc32 != \'\' - AND rf.crc32 IS NOT NULL' - ); - $cats = 2; - $preId = true; - } else { - $query = sprintf( - 'SELECT rf.crc32 AS textstring, rf.name AS filename, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND rel.proc_crc32 = %d - AND rf.crc32 != \'\' - AND rf.crc32 IS NOT NULL', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_CRC_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_CRC, $time, $cats, ' CRC32\'s to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_CRC, $time, $cats) as $releases) { + $files = $this->queries->groupByReleaseId( + $this->queries->fileRows($this->releaseIds($releases), NameFixingQueryService::SOURCE_CRC) + ); + $crcs = $this->distinctValues($files, 'crc32'); + $donors = $this->queries->crcDonors($crcs); - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' CRC32\'s to process.'); - - // Group by release - $releasesCrc = []; foreach ($releases as $release) { - /** @var Release $release */ - $releaseId = $release->releases_id; - if (! isset($releasesCrc[$releaseId])) { - $releasesCrc[$releaseId] = [ - 'release' => $release, - 'crcs' => [], - ]; - } - if (! empty($release->textstring)) { - $priority = $this->filePrioritizer->getCrcPriority($release->filename ?? ''); - $releasesCrc[$releaseId]['crcs'][$priority][] = $release->textstring; - } - } - - foreach ($releasesCrc as $releaseId => $data) { $this->updateService->reset(); $this->updateService->incrementChecked(); + $prioritized = []; - ksort($data['crcs']); - foreach ($data['crcs'] as $crcs) { - foreach ($crcs as $crc) { - /** @var Release $release */ - $release = clone $data['release']; - $release->textstring = $crc; + foreach ($files[(int) $release->releases_id] ?? [] as $file) { + $priority = $this->filePrioritizer->getCrcPriority((string) $file->filename); + $prioritized[$priority][(string) $file->crc32] = (string) $file->crc32; + } - $this->crcCheck($release, $echo, $type, $nameStatus, $show, $preId); - - if ($this->updateService->matched) { + ksort($prioritized); + foreach ($prioritized as $values) { + foreach ($values as $crc) { + if ($this->applyDonorMatch($release, $donors[$crc] ?? [], 5, 'crcCheck: CRC32', $type, $echo, $nameStatus, $show)) { break 2; } } } + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_crc32', (int) $release->releases_id); + } + $this->echoRenamed($show); } - - $this->echoFoundCount($echo, ' crc32\'s'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' crc32\'s'); } /** @@ -462,53 +292,37 @@ class NameFixingService $type = 'UID, '; $this->echoStartMessage($time, 'mediainfo Unique_IDs'); - if ($cats === 3) { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.fromname, rel.categories_id, - rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, - ru.unique_id AS uid - FROM releases rel - LEFT JOIN media_infos ru ON ru.releases_id = rel.id - WHERE ru.releases_id IS NOT NULL - AND rel.predb_id = 0' - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.fromname, rel.categories_id, - rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, - ru.unique_id AS uid - FROM releases rel - LEFT JOIN media_infos ru ON ru.releases_id = rel.id - WHERE ru.releases_id IS NOT NULL - AND (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND rel.proc_uid = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_UID_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_UID, $time, $cats, ' unique ids to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_UID, $time, $cats) as $releases) { + $media = $this->queries->groupByReleaseId($this->queries->mediaRows($this->releaseIds($releases))); + $uids = $this->distinctValues($media, 'uid'); + $donors = $this->queries->uidDonors($uids); - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' unique ids to process.'); - - foreach ($releases as $rel) { + foreach ($releases as $release) { $this->updateService->reset(); $this->updateService->incrementChecked(); - $this->uidCheck($rel, $echo, $type, $nameStatus, $show); + + foreach ($media[(int) $release->releases_id] ?? [] as $row) { + $uid = (string) ($row->uid ?? ''); + if ($uid !== '' && $this->applyDonorMatch($release, $donors[$uid] ?? [], 10, 'uidCheck: Unique_ID', $type, $echo, $nameStatus, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_uid', (int) $release->releases_id); + } + $this->echoRenamed($show); } - - $this->echoFoundCount($echo, ' UID\'s'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' UID\'s'); } /** @@ -519,53 +333,37 @@ class NameFixingService $type = 'PAR2 hash, '; $this->echoStartMessage($time, 'PAR2 hash_16K'); - if ($cats === 3) { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.fromname, rel.categories_id, - rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, - IFNULL(ph.hash, \'\') AS hash - FROM releases rel - LEFT JOIN par_hashes ph ON ph.releases_id = rel.id - WHERE ph.hash != \'\' - AND rel.predb_id = 0' - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.fromname, rel.categories_id, - rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, - IFNULL(ph.hash, \'\') AS hash - FROM releases rel - LEFT JOIN par_hashes ph ON ph.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND ph.hash != \'\' - AND rel.proc_hash16k = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_HASH16K_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_HASH, $time, $cats, ' hash_16K to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases->count(); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_HASH, $time, $cats) as $releases) { + $hashes = $this->queries->groupByReleaseId($this->queries->hashRows($this->releaseIds($releases))); + $hashValues = $this->distinctValues($hashes, 'hash'); + $donors = $this->queries->hashDonors($hashValues); - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' hash_16K to process.'); - - foreach ($releases as $rel) { + foreach ($releases as $release) { $this->updateService->reset(); $this->updateService->incrementChecked(); - $this->hashCheck($rel, $echo, $type, $nameStatus, $show); + + foreach ($hashes[(int) $release->releases_id] ?? [] as $row) { + $hash = (string) ($row->hash ?? ''); + if ($hash !== '' && $this->applyDonorMatch($release, $donors[$hash] ?? [], 5, 'hashCheck: PAR2 hash_16K', $type, $echo, $nameStatus, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_hash16k', (int) $release->releases_id); + } + $this->echoRenamed($show); } - - $this->echoFoundCount($echo, ' hashes'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' hashes'); } /** @@ -638,147 +436,10 @@ class NameFixingService $show ); - return true; + return $this->updateService->matched; } } - $this->updateService->updateSingleColumn('proc_srr', self::PROC_SRR_DONE, $release->releases_id); - - return false; - } - - /** - * Check CRC32 for matches. - */ - protected function crcCheck(object $release, bool $echo, string $type, bool $nameStatus, bool $show, bool $preId): bool - { - if ($release->textstring === '') { - $this->updateService->updateSingleColumn('proc_crc32', self::PROC_CRC_DONE, $release->releases_id); - - return false; - } - - $result = Release::fromQuery( - sprintf( - 'SELECT rf.crc32, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, rel.size as relsize, rel.predb_id as predb_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - LEFT JOIN release_files rf ON rf.releases_id = rel.id - WHERE rel.predb_id > 0 - AND rf.crc32 = %s', - escapeString($release->textstring) - ) - ); - - foreach ($result as $res) { - /** @var Release $res */ - $floor = round(($res->relsize - $release->relsize) / $res->relsize * 100, 1); - if ($floor >= -5 && $floor <= 5) { - $this->updateService->updateRelease( - $release, - $res->searchname, - 'crcCheck: CRC32', - $echo, - $type, - $nameStatus, - $show, - $res->predb_id - ); - - return true; - } - } - - $this->updateService->updateSingleColumn('proc_crc32', self::PROC_CRC_DONE, $release->releases_id); - - return false; - } - - /** - * Check UID for matches. - */ - protected function uidCheck(object $release, bool $echo, string $type, bool $nameStatus, bool $show): bool - { - if (empty($release->uid)) { - $this->updateService->updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release->releases_id); - - return false; - } - - $result = Release::fromQuery(sprintf( - 'SELECT r.id AS releases_id, r.size AS relsize, r.name AS textstring, r.searchname, r.fromname, r.predb_id - FROM releases r - LEFT JOIN media_infos ru ON ru.releases_id = r.id - WHERE ru.releases_id IS NOT NULL - AND ru.unique_id = %s - AND ru.releases_id != %d - AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = %s)', - escapeString($release->uid), - $release->releases_id, - escapeString('nonscene@Ef.net (EF)') - )); - - foreach ($result as $res) { - /** @var Release $res */ - $floor = round(($res->relsize - $release->relsize) / $res->relsize * 100, 1); - if ($floor >= -10 && $floor <= 10) { - $this->updateService->updateRelease( - $release, - $res->searchname, - 'uidCheck: Unique_ID', - $echo, - $type, - $nameStatus, - $show, - $res->predb_id - ); - - return true; - } - } - - $this->updateService->updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release->releases_id); - - return false; - } - - /** - * Check PAR2 hash for matches. - */ - protected function hashCheck(object $release, bool $echo, string $type, bool $nameStatus, bool $show): bool - { - $result = Release::fromQuery(sprintf( - 'SELECT r.id AS releases_id, r.size AS relsize, r.name AS textstring, r.searchname, r.fromname, r.predb_id - FROM releases r - LEFT JOIN par_hashes ph ON ph.releases_id = r.id - WHERE ph.hash = %s - AND ph.releases_id != %d - AND (r.predb_id > 0 OR r.anidbid > 0)', - escapeString($release->hash), - $release->releases_id - )); - - foreach ($result as $res) { - /** @var Release $res */ - $floor = round(($res->relsize - $release->relsize) / $res->relsize * 100, 1); - if ($floor >= -5 && $floor <= 5) { - $this->updateService->updateRelease( - $release, - $res->searchname, - 'hashCheck: PAR2 hash_16K', - $echo, - $type, - $nameStatus, - $show, - $res->predb_id - ); - - return true; - } - } - - $this->updateService->updateSingleColumn('proc_hash16k', self::PROC_HASH16K_DONE, $release->releases_id); - return false; } @@ -795,6 +456,15 @@ class NameFixingService $bestMatch = $this->findBestPredbMatch($fileName); if ($bestMatch !== null) { + if (strcasecmp((string) ($bestMatch['title'] ?? ''), (string) $release->searchname) === 0) { + $this->updateService->attachPredbId( + (int) $release->releases_id, + (int) ($bestMatch['id'] ?? 0) + ); + + return $this->updateService->matched; + } + $this->updateService->updateRelease( $release, $bestMatch['title'] ?? '', @@ -806,37 +476,7 @@ class NameFixingService $bestMatch['id'] ?? null ); - return true; - } - - return false; - } - - /** - * Check PreDB for title matches. - */ - protected function preDbTitleCheck(object $release, bool $echo, string $type, bool $nameStatus, bool $show): bool - { - $fileName = $this->fileNameCleaner->cleanForMatching($release->textstring); - - if (empty($fileName)) { - return false; - } - - $bestMatch = $this->findBestPredbMatch($fileName); - if ($bestMatch !== null) { - $this->updateService->updateRelease( - $release, - $bestMatch['title'] ?? '', - 'PreDb: Title match', - $echo, - $type, - $nameStatus, - $show, - $bestMatch['id'] ?? null - ); - - return true; + return $this->updateService->matched; } return false; @@ -847,35 +487,186 @@ class NameFixingService */ protected function findBestPredbMatch(string $fileName): ?array { + if (array_key_exists($fileName, $this->predbMatchCache)) { + return $this->predbMatchCache[$fileName]; + } + + if (count($this->predbMatchCache) >= 5000) { + $this->predbMatchCache = []; + } + $results = Search::searchPredb($fileName); - return $this->predbMatchSelector->selectBestMatch($fileName, $results); + return $this->predbMatchCache[$fileName] = $this->predbMatchSelector->selectBestMatch($fileName, $results); } /** - * Get releases based on time and category parameters. - * - * @return Collection + * @return \Generator> */ - protected function getReleases(int $time, int $cats, string $query, int $limit = 0): \Illuminate\Database\Eloquent\Collection|bool // @phpstan-ignore class.notFound, missingType.generics, return.phpDocType + protected function candidateBatches(string $source, int $time, int $categories): \Generator { - $releases = false; - $queryLimit = ($limit === 0) ? '' : ' LIMIT '.$limit; + $afterId = 0; - if ($time === 1 && $cats === 1) { - $releases = Release::fromQuery($query.$this->timeother.$queryLimit); - } - if ($time === 1 && $cats === 2) { - $releases = Release::fromQuery($query.$this->timeall.$queryLimit); - } - if ($time === 2 && $cats === 1) { - $releases = Release::fromQuery($query.$this->fullother.$queryLimit); - } - if ($time === 2 && $cats === 2) { - $releases = Release::fromQuery($query.$this->fullall.$queryLimit); + do { + $batch = $this->queries->candidateBatch($source, $time, $categories, $afterId); + if ($batch === []) { + break; + } + + yield $batch; + $afterId = (int) $batch[array_key_last($batch)]->releases_id; + } while (count($batch) === NameFixingQueryService::BATCH_SIZE); + } + + protected function startBatch(string $source, int $time, int $categories, string $message): bool + { + $total = $this->queries->countCandidates($source, $time, $categories); + $this->_totalReleases = $total; + + if ($total === 0) { + return false; } - return $releases; + cli()->info(number_format($total).$message); + + return true; + } + + /** + * @param list $releases + * @return list + */ + protected function releaseIds(array $releases): array + { + return array_values(array_map( + static fn (object $release): int => (int) $release->releases_id, + $releases + )); + } + + /** + * @param array> $groupedRows + * @return list + */ + protected function distinctValues(array $groupedRows, string $column): array + { + $values = []; + + foreach ($groupedRows as $rows) { + foreach ($rows as $row) { + $value = (string) ($row->{$column} ?? ''); + if ($value !== '') { + $values[$value] = $value; + } + } + } + + return array_values($values); + } + + /** + * @param list $donors + */ + protected function applyDonorMatch( + object $release, + array $donors, + int $tolerancePercent, + string $method, + string $type, + bool $echo, + bool $nameStatus, + bool $show + ): bool { + $donor = $this->donorMatchSelector->select($donors, (int) $release->relsize, $tolerancePercent); + if ($donor === null) { + return false; + } + + if (strcasecmp((string) $donor->searchname, (string) $release->searchname) === 0) { + $this->updateService->attachPredbId( + (int) $release->releases_id, + (int) $donor->predb_id + ); + + return $this->updateService->matched; + } + + $this->updateService->updateRelease( + $release, + (string) $donor->searchname, + $method, + $echo, + $type, + $nameStatus, + $show, + (int) $donor->predb_id + ); + + return $this->updateService->matched; + } + + /** + * @param list $files + */ + protected function processFileCandidates( + object $release, + array $files, + bool $echo, + bool $nameStatus, + bool $show, + bool $incrementChecked = true, + bool $showProgress = true + ): void { + $this->updateService->reset(); + if ($incrementChecked) { + $this->updateService->incrementChecked(); + } + $filenames = array_values(array_unique(array_map( + static fn (object $file): string => (string) $file->textstring, + $files + ))); + $prioritized = $this->filePrioritizer->prioritizeForMatching($filenames); + + foreach ($prioritized as $filename) { + $candidate = clone $release; + $candidate->textstring = $filename; + $fileResult = $this->fileExtractor->extractFromFile($filename); + + if ($fileResult !== null) { + $this->updateService->updateRelease( + $candidate, + $fileResult->newName, + 'fileCheck: '.$fileResult->method, + $echo, + 'Filenames, ', + $nameStatus, + $show + ); + } + + if (! $this->updateService->matched) { + $this->preDbFileCheck($candidate, $echo, 'Filenames, ', $nameStatus, $show); + } + + if ($this->updateService->matched) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_files', (int) $release->releases_id); + } + + if ($showProgress) { + $this->echoRenamed($show); + } + } + + protected function markProcessed(bool $echo, bool $nameStatus, string $column, int $releaseId): void + { + if ($echo && $nameStatus) { + $this->updateService->updateSingleColumn($column, 1, $releaseId); + } } /** @@ -952,6 +743,252 @@ class NameFixingService } } + /** + * Process one GUID-partitioned release batch using all standard name sources. + * + * @param null|callable(object): bool $par2Processor + * @return array{checked: int, fixed: int} + */ + public function processStandardBatch( + string $leftGuid, + int $limit, + bool $show, + ?callable $par2Processor = null + ): array { + $releases = $this->queries->standardCandidateBatch($leftGuid, $limit); + if ($releases === []) { + return ['checked' => 0, 'fixed' => 0]; + } + + $releaseIds = $this->releaseIds($releases); + $nfos = $this->queries->groupByReleaseId($this->queries->nfoRows($releaseIds)); + $files = $this->queries->groupByReleaseId($this->queries->fileRows($releaseIds)); + $media = $this->queries->groupByReleaseId($this->queries->mediaRows($releaseIds)); + $hashes = $this->queries->groupByReleaseId($this->queries->hashRows($releaseIds)); + $uidDonors = $this->queries->uidDonors($this->distinctValues($media, 'uid')); + $crcDonors = $this->queries->crcDonors($this->distinctValues($files, 'crc32')); + $hashDonors = $this->queries->hashDonors($this->distinctValues($hashes, 'hash')); + $fixedBefore = $this->updateService->fixed; + + foreach ($releases as $release) { + $this->updateService->incrementChecked(); + $this->updateService->reset(); + $releaseId = (int) $release->releases_id; + $releaseMedia = $media[$releaseId] ?? []; + $releaseFiles = $files[$releaseId] ?? []; + + if ((int) $release->proc_uid === self::PROC_UID_NONE) { + $this->updateService->reset(); + foreach ($releaseMedia as $row) { + $uid = (string) ($row->uid ?? ''); + if ($uid !== '' && $this->applyDonorMatch($release, $uidDonors[$uid] ?? [], 10, 'uidCheck: Unique_ID', 'UID, ', true, true, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + foreach ($releaseMedia as $row) { + if (empty($row->movie_name)) { + continue; + } + + $candidate = clone $release; + $candidate->movie_name = $row->movie_name; + $candidate->file_name = $row->file_name; + if ($this->mediaMovieNameCheck($candidate, true, 'Mediainfo, ', true, $show)) { + break; + } + } + } + + if (! $this->updateService->matched) { + $this->updateService->updateSingleColumn('proc_uid', self::PROC_UID_DONE, $releaseId); + } + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_crc32 === self::PROC_CRC_NONE) { + $this->updateService->reset(); + $prioritizedCrcs = []; + foreach ($releaseFiles as $file) { + $crc = (string) ($file->crc32 ?? ''); + if ($crc === '') { + continue; + } + + $priority = $this->filePrioritizer->getCrcPriority((string) $file->filename); + $prioritizedCrcs[$priority][$crc] = $crc; + } + ksort($prioritizedCrcs); + + foreach ($prioritizedCrcs as $crcs) { + foreach ($crcs as $crc) { + if ($this->applyDonorMatch($release, $crcDonors[$crc] ?? [], 5, 'crcCheck: CRC32', 'CRC32, ', true, true, $show)) { + break 2; + } + } + } + + if (! $this->updateService->matched) { + $this->updateService->updateSingleColumn('proc_crc32', self::PROC_CRC_DONE, $releaseId); + } + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_srr === self::PROC_SRR_NONE) { + $this->updateService->reset(); + foreach ($releaseFiles as $file) { + $candidate = clone $release; + $candidate->textstring = (string) $file->textstring; + if ($this->srrNameCheck($candidate, true, 'SRR, ', true, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->updateService->updateSingleColumn('proc_srr', self::PROC_SRR_DONE, $releaseId); + } + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_hash16k === self::PROC_HASH16K_NONE) { + $this->updateService->reset(); + foreach ($hashes[$releaseId] ?? [] as $row) { + $hash = (string) ($row->hash ?? ''); + if ($hash !== '' && $this->applyDonorMatch($release, $hashDonors[$hash] ?? [], 5, 'hashCheck: PAR2 hash_16K', 'PAR2 hash, ', true, true, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->updateService->updateSingleColumn('proc_hash16k', self::PROC_HASH16K_DONE, $releaseId); + } + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_nfo === self::PROC_NFO_NONE) { + $this->updateService->reset(); + $nfo = $nfos[$releaseId][0] ?? null; + if ($nfo !== null) { + $release->textstring = (string) ($nfo->textstring ?? ''); + if (! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) { + $nfoResult = $this->nfoExtractor->extractFromNfo($release->textstring); + if ($nfoResult !== null) { + $this->updateService->updateRelease( + $release, + $nfoResult->newName, + 'nfoCheck: '.$nfoResult->method, + true, + 'NFO, ', + true, + $show + ); + } + + if (! $this->updateService->matched) { + $this->checkWithPatternMatchers($release, true, 'NFO, ', true, $show, false); + } + } + } + + if (! $this->updateService->matched) { + $this->updateService->updateSingleColumn('proc_nfo', self::PROC_NFO_DONE, $releaseId); + } + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_files === self::PROC_FILES_NONE) { + $this->processFileCandidates($release, $releaseFiles, true, true, $show, false, false); + } + + if ($this->updateService->matched) { + continue; + } + + if ((int) $release->proc_par2 === self::PROC_PAR2_NONE) { + $this->updateService->reset(); + $matched = $par2Processor !== null && $par2Processor($release); + if ($matched) { + $this->updateService->fixed++; + } else { + $this->updateService->updateSingleColumn('proc_par2', self::PROC_PAR2_DONE, $releaseId); + } + } + } + + return [ + 'checked' => count($releases), + 'fixed' => $this->updateService->fixed - $fixedBefore, + ]; + } + + /** + * Match one PreDB title against indexed release candidates. + */ + public function matchPredbFulltext(object $pre, bool $show = false): int + { + $title = (string) ($pre->title ?? ''); + if (strlen($title) < 15 || ! preg_match('/([\w()]+[\s._-]){2,}[\w()]+-\w+/u', $title)) { + return 0; + } + + if (! Search::isAvailable()) { + throw new RuntimeException('The configured search backend is unavailable for PreDB matching.'); + } + + $searchResults = Search::searchReleases(['name' => $title, 'searchname' => $title], 21); + $candidateIds = []; + foreach ($searchResults as $key => $value) { + $candidateId = is_numeric($value) ? (int) $value : (is_numeric($key) ? (int) $key : 0); + if ($candidateId > 0) { + $candidateIds[$candidateId] = $candidateId; + } + } + + $matches = $this->queries->confirmPredbCandidates(array_values($candidateIds), $title); + if (count($matches) >= 16) { + return -1; + } + + $matching = 0; + foreach ($matches as $release) { + if (strcasecmp($title, (string) $release->searchname) === 0) { + $this->updateService->attachPredbId((int) $release->releases_id, (int) $pre->predb_id); + } else { + $this->updateService->reset(); + $this->updateService->updateRelease( + $release, + $title, + 'Title Match source: '.(string) ($pre->source ?? ''), + true, + 'PreDB FT Exact, ', + true, + $show, + (int) $pre->predb_id + ); + } + + $matching++; + } + + return $matching; + } + /** * Get the update service. */ @@ -975,48 +1012,33 @@ class NameFixingService { $this->echoStartMessage($time, 'par2 files'); - if ($cats === 3) { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.guid, rel.groups_id, rel.fromname - FROM releases rel - WHERE rel.predb_id = 0' - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.guid, rel.groups_id, rel.fromname - FROM releases rel - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND rel.proc_par2 = %d', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - self::PROC_PAR2_NONE - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_PAR2, $time, $cats, ' releases to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases ? $releases->count() : 0; - - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' releases to process.'); - $nzbContentsService = app(NzbContentsService::class); - + $nzbContentsService = app(NzbContentsService::class); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_PAR2, $time, $cats) as $releases) { foreach ($releases as $release) { - /** @var Release $release */ - if ($nzbContentsService->checkPar2($release->guid, $release->releases_id, $release->groups_id, (int) $nameStatus, (int) $show)) { + if ($nzbContentsService->checkPar2( + $release->guid, + $release->releases_id, + $release->groups_id, + (int) ($echo && $nameStatus), + (int) $show + )) { $this->updateService->fixed++; + } else { + $this->markProcessed($echo, $nameStatus, 'proc_par2', (int) $release->releases_id); } $this->updateService->incrementChecked(); $this->echoRenamed($show); } - $this->echoFoundCount($echo, ' files'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' files'); } /** @@ -1027,48 +1049,38 @@ class NameFixingService $this->echoStartMessage($time, 'file names'); $type = 'Filenames, '; - if ($cats === 3) { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE predb_id = 0' - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - INNER JOIN release_files rf ON rf.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rel.predb_id = 0 - AND rf.name LIKE %s', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - escapeString('%SDPORN%') - ); + if (! $this->startBatch(NameFixingQueryService::SOURCE_XXX, $time, $cats, ' xxx file names to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases ? $releases->count() : 0; - - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' xxx file names to process.'); + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_XXX, $time, $cats) as $releases) { + $files = $this->queries->groupByReleaseId( + $this->queries->fileRows($this->releaseIds($releases), NameFixingQueryService::SOURCE_XXX) + ); foreach ($releases as $release) { $this->updateService->reset(); - $this->xxxNameCheck($release, $echo, $type, $nameStatus, $show); $this->updateService->incrementChecked(); + + foreach ($files[(int) $release->releases_id] ?? [] as $file) { + $candidate = clone $release; + $candidate->textstring = (string) $file->textstring; + if ($this->xxxNameCheck($candidate, $echo, $type, $nameStatus, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_files', (int) $release->releases_id); + } + $this->echoRenamed($show); } - $this->echoFoundCount($echo, ' files'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' files'); } /** @@ -1079,45 +1091,41 @@ class NameFixingService $type = 'Mediainfo, '; $this->echoStartMessage($time, 'Mediainfo movie_name'); - if ($cats === 3) { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, rel.fromname, rel.groups_id, rel.categories_id, rel.id AS releases_id, rf.movie_name as movie_name - FROM releases rel - INNER JOIN media_infos rf ON rf.releases_id = rel.id - WHERE rel.predb_id = 0' - ); - $cats = 2; - } else { - $query = sprintf( - 'SELECT rel.id AS releases_id, rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, rel.fromname, rel.groups_id, rel.categories_id, rel.id AS releases_id, rf.movie_name as movie_name, rf.file_name as file_name - FROM releases rel - INNER JOIN media_infos rf ON rf.releases_id = rel.id - WHERE rel.isrenamed = %d - AND rel.predb_id = 0', - self::IS_RENAMED_NONE - ); - if ($cats === 2) { - $query .= PHP_EOL.'AND rel.categories_id IN ('.Category::OTHER_MISC.','.Category::OTHER_HASHED.')'; - } + if (! $this->startBatch(NameFixingQueryService::SOURCE_MEDIA_MOVIE, $time, $cats, ' mediainfo movie names to process.')) { + cli()->info('Nothing to fix.'); + + return; } - $releases = $this->getReleases($time, $cats, $query); - $total = $releases ? $releases->count() : 0; + foreach ($this->candidateBatches(NameFixingQueryService::SOURCE_MEDIA_MOVIE, $time, $cats) as $releases) { + $media = $this->queries->groupByReleaseId($this->queries->mediaRows($this->releaseIds($releases))); - if ($total > 0) { - $this->_totalReleases = $total; - cli()->info(number_format($total).' mediainfo movie names to process.'); - - foreach ($releases as $rel) { + foreach ($releases as $release) { $this->updateService->incrementChecked(); $this->updateService->reset(); - $this->mediaMovieNameCheck($rel, $echo, $type, $nameStatus, $show); + + foreach ($media[(int) $release->releases_id] ?? [] as $row) { + if (empty($row->movie_name)) { + continue; + } + + $candidate = clone $release; + $candidate->movie_name = $row->movie_name; + $candidate->file_name = $row->file_name; + if ($this->mediaMovieNameCheck($candidate, $echo, $type, $nameStatus, $show)) { + break; + } + } + + if (! $this->updateService->matched) { + $this->markProcessed($echo, $nameStatus, 'proc_uid', (int) $release->releases_id); + } + $this->echoRenamed($show); } - $this->echoFoundCount($echo, ' MediaInfo\'s'); - } else { - cli()->info('Nothing to fix.'); } + + $this->echoFoundCount($echo, ' MediaInfo\'s'); } /** @@ -1128,11 +1136,9 @@ class NameFixingService if (preg_match('/^.+?SDPORN/i', $release->textstring, $hit)) { $this->updateService->updateRelease($release, $hit[0], 'fileCheck: XXX SDPORN', $echo, $type, $nameStatus, $show); - return true; + return $this->updateService->matched; } - $this->updateService->updateSingleColumn('proc_files', self::PROC_FILES_DONE, $release->releases_id); - return false; } @@ -1156,11 +1162,9 @@ class NameFixingService if ($newName !== '') { $this->updateService->updateRelease($release, $newName, 'MediaInfo: Movie Name', $echo, $type, $nameStatus, $show, $release->predb_id ?? 0); - return true; + return $this->updateService->matched; } - $this->updateService->updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release->releases_id); - return false; } @@ -1174,9 +1178,15 @@ class NameFixingService // Check PreDB first $preDbMatch = $this->updateService->checkPreDbMatch($release, $release->textstring); if ($preDbMatch !== null) { - $this->updateService->updateRelease($release, $preDbMatch['title'], 'preDB: Match', $echo, $type, $nameStatus, $show, $preDbMatch['id']); + if (strcasecmp((string) $preDbMatch['title'], (string) $release->searchname) === 0) { + if ($echo) { + $this->updateService->attachPredbId((int) $release->releases_id, (int) $preDbMatch['id']); + } + } else { + $this->updateService->updateRelease($release, $preDbMatch['title'], 'preDB: Match', $echo, $type, $nameStatus, $show, $preDbMatch['id']); + } - return true; + return $this->updateService->matched; } if ($preId) { @@ -1211,17 +1221,6 @@ class NameFixingService if (! $this->updateService->matched) { $this->preDbFileCheck($release, $echo, $type, $nameStatus, $show); } - // Try PreDB title check - if (! $this->updateService->matched) { - $this->preDbTitleCheck($release, $echo, $type, $nameStatus, $show); - } - // Try file name extraction - if (! $this->updateService->matched) { - $result = $this->fileExtractor->extractFromFile($release->textstring); - if ($result !== null) { - $this->updateService->updateRelease($release, $result->newName, 'fileCheck: '.$result->method, $echo, $type, $nameStatus, $show); - } - } break; default: @@ -1233,7 +1232,7 @@ class NameFixingService } // Update processing flags if not matched - if ($nameStatus === true && ! $this->updateService->matched) { + if ($echo && $nameStatus && ! $this->updateService->matched) { $this->updateProcessingFlags($type, $release->releases_id); } @@ -1281,7 +1280,7 @@ class NameFixingService $matching = 0; $files = explode('||', $release->filename ?? ''); - $prioritizedFiles = $this->filePrioritizer->prioritizeForPreDb($files); // @phpstan-ignore argument.type + $prioritizedFiles = $this->filePrioritizer->prioritizeForPreDb($files); foreach ($prioritizedFiles as $fileName) { $cleanedFileName = $this->fileNameCleaner->cleanForMatching($fileName); @@ -1290,21 +1289,17 @@ class NameFixingService continue; } - $results = Search::searchPredb($cleanedFileName); + $bestMatch = $this->findBestPredbMatch($cleanedFileName); - if (! empty($results)) { - $bestMatch = $this->predbMatchSelector->selectBestMatch($cleanedFileName, $results); - - if ($bestMatch !== null) { - if ($bestMatch['title'] !== $release->searchname) { - $this->updateService->updateRelease($release, $bestMatch['title'], 'file matched source: '.($bestMatch['source'] ?? ''), $echo, 'PreDB file match, ', $nameStatus, $show); - } else { - $this->updateService->updateSingleColumn('predb_id', $bestMatch['id'] ?? 0, $release->releases_id); - } - $matching++; - - return $matching; + if ($bestMatch !== null) { + if (strcasecmp((string) $bestMatch['title'], (string) $release->searchname) !== 0) { + $this->updateService->updateRelease($release, $bestMatch['title'], 'file matched source: '.($bestMatch['source'] ?? ''), $echo, 'PreDB file match, ', $nameStatus, $show); + } elseif ($echo) { + $this->updateService->attachPredbId((int) $release->releases_id, (int) ($bestMatch['id'] ?? 0)); } + $matching++; + + return $matching; } } @@ -1333,66 +1328,61 @@ class NameFixingService /** * Retrieves releases and their file names to attempt PreDB matches. * - * @param array $args + * @param list $args * * @throws \Exception */ public function getPreFileNames(array $args = []): void { $show = isset($args[2]) && $args[2] === 'show'; - - if (isset($args[1]) && is_numeric($args[1])) { - $limit = 'LIMIT '.$args[1]; - $orderBy = 'ORDER BY r.id DESC'; - } else { - $orderBy = 'ORDER BY r.id ASC'; - $limit = 'LIMIT 1000000'; - } + $limited = isset($args[1]) && is_numeric($args[1]); + $requestedLimit = $limited ? max(1, (int) $args[1]) : PHP_INT_MAX; cli()->info(PHP_EOL.'Match PreFiles '.($args[1] ?? 'all').' Started at '.now()); cli()->info('Matching predb filename to cleaned release_files.name.'); $counter = $counted = 0; $timeStart = now(); + $available = $this->queries->countPrefileCandidates(); + $total = min($available, $requestedLimit); + if ($total === 0) { + cli()->info('Nothing to do.'); - $query = Release::fromQuery( - sprintf( - "SELECT r.id AS releases_id, r.name, r.searchname, - r.fromname, r.groups_id, r.categories_id, - GROUP_CONCAT(rf.name ORDER BY LENGTH(rf.name) DESC SEPARATOR '||') AS filename - FROM releases r - INNER JOIN release_files rf ON r.id = rf.releases_id - WHERE rf.name IS NOT NULL - AND r.predb_id = 0 - AND r.categories_id IN (%s) - AND r.isrenamed = 0 - GROUP BY r.id - %s %s", - implode(',', Category::OTHERS_GROUP), - $orderBy, - $limit - ) - ); - - if ($query->isNotEmpty()) { - $total = $query->count(); - - if ($total > 0) { - cli()->info(PHP_EOL.number_format($total).' releases to process.'); - - foreach ($query as $row) { - $success = $this->matchPreDbFiles($row, true, true, $show); - if ($success === 1) { - $counted++; - } - if ($show === false) { - cli()->info('Renamed Releases: ['.number_format($counted).'] '.cli()->percentString(++$counter, $total)); - } - } - cli()->info(PHP_EOL.'Renamed '.number_format($counted).' releases in '.now()->diffInSeconds($timeStart, true).' seconds.'); - } else { - cli()->info('Nothing to do.'); - } + return; } + + cli()->info(PHP_EOL.number_format($total).' releases to process.'); + $cursor = $limited ? PHP_INT_MAX : 0; + + while ($counter < $total) { + $batchLimit = min(NameFixingQueryService::BATCH_SIZE, $total - $counter); + $batch = $this->queries->prefileCandidateBatch($cursor, $batchLimit, $limited); + if ($batch === []) { + break; + } + + $files = $this->queries->groupByReleaseId($this->queries->fileRows($this->releaseIds($batch))); + foreach ($batch as $row) { + $releaseFiles = $files[(int) $row->releases_id] ?? []; + usort($releaseFiles, static fn (object $left, object $right): int => strlen((string) $right->textstring) <=> strlen((string) $left->textstring)); + $row->filename = implode('||', array_map( + static fn (object $file): string => (string) $file->textstring, + $releaseFiles + )); + + if ($this->matchPreDbFiles($row, true, true, $show) === 1) { + $counted++; + } + $counter++; + + if (! $show) { + cli()->info('Renamed Releases: ['.number_format($counted).'] '.cli()->percentString($counter, $total)); + } + } + + $cursor = (int) $batch[array_key_last($batch)]->releases_id; + } + + cli()->info(PHP_EOL.'Renamed '.number_format($counted).' releases in '.now()->diffInSeconds($timeStart, true).' seconds.'); } } diff --git a/app/Services/NameFixing/ReleaseUpdateService.php b/app/Services/NameFixing/ReleaseUpdateService.php index fc75cf858..9c94d51be 100644 --- a/app/Services/NameFixing/ReleaseUpdateService.php +++ b/app/Services/NameFixing/ReleaseUpdateService.php @@ -7,13 +7,13 @@ namespace App\Services\NameFixing; use App\Events\ReleaseNameFixed; use App\Facades\Search; use App\Models\Category; -use App\Models\Predb; use App\Models\Release; use App\Models\UsenetGroup; use App\Services\Categorization\CategorizationService; use App\Services\ReleaseCleaningService; use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; +use InvalidArgumentException; /** * Service for updating releases with new names. @@ -23,6 +23,20 @@ use Illuminate\Support\Facades\DB; */ class ReleaseUpdateService { + /** + * @var list + */ + private const SINGLE_UPDATE_COLUMNS = [ + 'predb_id', + 'proc_nfo', + 'proc_files', + 'proc_par2', + 'proc_uid', + 'proc_hash16k', + 'proc_srr', + 'proc_crc32', + ]; + /** * PreDB regex pattern for scene release names. */ @@ -347,10 +361,25 @@ class ReleaseUpdateService */ public function updateSingleColumn(string $column, int $status, int $id): void { - if ($column !== '' && $id !== 0) { - Release::query()->where('id', $id)->update([$column => $status]); - Search::updateRelease($id); + if (! in_array($column, self::SINGLE_UPDATE_COLUMNS, true)) { + throw new InvalidArgumentException("Unsupported release status column [{$column}]."); } + + if ($id !== 0) { + DB::update("UPDATE releases SET {$column} = ? WHERE id = ?", [$status, $id]); + } + } + + public function attachPredbId(int $releaseId, int $predbId): void + { + if ($releaseId === 0 || $predbId === 0) { + return; + } + + $this->updateSingleColumn('predb_id', $predbId, $releaseId); + $this->relid = $releaseId; + $this->matched = true; + $this->done = true; } /** @@ -360,17 +389,40 @@ class ReleaseUpdateService */ public function checkPreDbMatch(object $release, string $textstring): ?array { - if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) { - foreach ($hits as $hit) { - foreach ($hit as $val) { - $title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first(); - if ($title !== null) { - return ['title' => $title['title'], 'id' => $title['id']]; - } + if (! preg_match_all(self::PREDB_REGEX, $textstring, $hits) || preg_match('/Source\s:/i', $textstring)) { + return null; + } + + $candidates = []; + foreach ($hits as $hit) { + foreach ($hit as $value) { + $title = trim($value); + if ($title !== '') { + $candidates[$title] = $title; } } } + if ($candidates === []) { + return null; + } + + $candidateValues = array_values($candidates); + $rows = DB::select( + 'SELECT id, title FROM predb WHERE title IN ('.implode(',', array_fill(0, count($candidateValues), '?')).')', + $candidateValues + ); + $matches = []; + foreach ($rows as $row) { + $matches[(string) $row->title] = ['title' => (string) $row->title, 'id' => (int) $row->id]; + } + + foreach ($candidateValues as $candidate) { + if (isset($matches[$candidate])) { + return $matches[$candidate]; + } + } + return null; } diff --git a/app/Services/Runners/ReleasesRunner.php b/app/Services/Runners/ReleasesRunner.php index 8395d4c65..faf7a2624 100644 --- a/app/Services/Runners/ReleasesRunner.php +++ b/app/Services/Runners/ReleasesRunner.php @@ -144,10 +144,11 @@ class ReleasesRunner extends BaseRunner $queues = []; $idx = 0; + $workerCount = count($leftGuids); foreach ($leftGuids as $leftGuid) { if ($maxPerRun > 0) { $idx++; - $queues[$idx] = sprintf('%s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx); + $queues[$idx] = sprintf('%s %s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx, $workerCount); } } diff --git a/app/Services/Tmux/Scripts/groupfixrelnames.php b/app/Services/Tmux/Scripts/groupfixrelnames.php index e182c55c9..4e2037d74 100644 --- a/app/Services/Tmux/Scripts/groupfixrelnames.php +++ b/app/Services/Tmux/Scripts/groupfixrelnames.php @@ -15,8 +15,8 @@ if (! isset($argv[1])) { exit(1); } -// Parse arguments: "type guidChar maxPerRun thread" -[$type, $guidChar, $maxPerRun, $thread] = explode(' ', $argv[1]); +// Parse arguments: "type guidChar maxPerRun worker workers" +[$type, $guidChar, $maxPerRun, $thread, $workers] = array_pad(explode(' ', $argv[1]), 5, '1'); // Build artisan command based on type $artisan = dirname(__DIR__, 4).'/artisan'; @@ -37,7 +37,7 @@ switch ($type) { exit(1); } - $command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread}"; + $command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread} --workers={$workers}"; break; default: diff --git a/database/migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php b/database/migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php new file mode 100644 index 000000000..43ef95e2a --- /dev/null +++ b/database/migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php @@ -0,0 +1,84 @@ +>> + */ + private const INDEXES = [ + 'par_hashes' => [ + 'ix_par_hashes_hash_releases_id' => ['hash', 'releases_id'], + ], + 'release_files' => [ + 'ix_release_files_crc32_releases_id' => ['crc32', 'releases_id'], + ], + 'predb' => [ + 'ix_predb_searched_predate_id' => ['searched', 'predate', 'id'], + ], + ]; + + /** + * Run the migrations. + */ + public function up(): void + { + foreach (self::INDEXES as $table => $indexes) { + if (! Schema::hasTable($table)) { + continue; + } + + foreach ($indexes as $name => $columns) { + if ($this->hasUsablePrefix($table, $columns)) { + continue; + } + + Schema::table($table, static function (Blueprint $blueprint) use ($columns, $name): void { + $blueprint->index($columns, $name); + }); + } + } + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + foreach (self::INDEXES as $table => $indexes) { + if (! Schema::hasTable($table)) { + continue; + } + + foreach (array_keys($indexes) as $name) { + if (! Schema::hasIndex($table, $name)) { + continue; + } + + Schema::table($table, static function (Blueprint $blueprint) use ($name): void { + $blueprint->dropIndex($name); + }); + } + } + } + + /** + * @param list $requiredColumns + */ + private function hasUsablePrefix(string $table, array $requiredColumns): bool + { + foreach (Schema::getIndexes($table) as $index) { + $columns = array_values($index['columns'] ?? []); + if (array_slice($columns, 0, count($requiredColumns)) === $requiredColumns) { + return true; + } + } + + return false; + } +}; diff --git a/database/schema/mariadb-schema.sql b/database/schema/mariadb-schema.sql index a05371d0e..c32dcd4dc 100644 --- a/database/schema/mariadb-schema.sql +++ b/database/schema/mariadb-schema.sql @@ -580,6 +580,7 @@ CREATE TABLE `par_hashes` ( `releases_id` int(10) unsigned NOT NULL COMMENT 'FK to releases.id', `hash` varchar(32) NOT NULL COMMENT 'hash_16k block of par2', PRIMARY KEY (`releases_id`,`hash`), + KEY `ix_par_hashes_hash_releases_id` (`hash`,`releases_id`), CONSTRAINT `FK_ph_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; @@ -705,6 +706,7 @@ CREATE TABLE `predb` ( KEY `ix_predb_predate` (`predate`), KEY `ix_predb_source` (`source`), KEY `ix_predb_searched` (`searched`), + KEY `ix_predb_searched_predate_id` (`searched`,`predate`,`id`), FULLTEXT KEY `ft_predb_filename` (`filename`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; DROP TABLE IF EXISTS `predb_crcs`; @@ -829,6 +831,7 @@ CREATE TABLE `release_files` ( `updated_at` timestamp NULL DEFAULT NULL, `passworded` tinyint(1) NOT NULL DEFAULT 0, PRIMARY KEY (`releases_id`,`name`), + KEY `ix_release_files_crc32_releases_id` (`crc32`,`releases_id`), CONSTRAINT `FK_rf_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; DROP TABLE IF EXISTS `release_informs`; diff --git a/database/schema/mysql-schema.sql b/database/schema/mysql-schema.sql index a05371d0e..c32dcd4dc 100644 --- a/database/schema/mysql-schema.sql +++ b/database/schema/mysql-schema.sql @@ -580,6 +580,7 @@ CREATE TABLE `par_hashes` ( `releases_id` int(10) unsigned NOT NULL COMMENT 'FK to releases.id', `hash` varchar(32) NOT NULL COMMENT 'hash_16k block of par2', PRIMARY KEY (`releases_id`,`hash`), + KEY `ix_par_hashes_hash_releases_id` (`hash`,`releases_id`), CONSTRAINT `FK_ph_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; @@ -705,6 +706,7 @@ CREATE TABLE `predb` ( KEY `ix_predb_predate` (`predate`), KEY `ix_predb_source` (`source`), KEY `ix_predb_searched` (`searched`), + KEY `ix_predb_searched_predate_id` (`searched`,`predate`,`id`), FULLTEXT KEY `ft_predb_filename` (`filename`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; DROP TABLE IF EXISTS `predb_crcs`; @@ -829,6 +831,7 @@ CREATE TABLE `release_files` ( `updated_at` timestamp NULL DEFAULT NULL, `passworded` tinyint(1) NOT NULL DEFAULT 0, PRIMARY KEY (`releases_id`,`name`), + KEY `ix_release_files_crc32_releases_id` (`crc32`,`releases_id`), CONSTRAINT `FK_rf_releases` FOREIGN KEY (`releases_id`) REFERENCES `releases` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; DROP TABLE IF EXISTS `release_informs`; diff --git a/tests/Feature/FixReleaseNameQueryIndexesMigrationTest.php b/tests/Feature/FixReleaseNameQueryIndexesMigrationTest.php new file mode 100644 index 000000000..58ff70ecf --- /dev/null +++ b/tests/Feature/FixReleaseNameQueryIndexesMigrationTest.php @@ -0,0 +1,125 @@ + + */ + private array $originalEnvironment = []; + + private string $databasePath; + + public function createApplication() + { + $this->databasePath = sys_get_temp_dir().'/nntmux-name-fixing-indexes-test.sqlite'; + $this->originalEnvironment = [ + 'APP_ENV' => getenv('APP_ENV'), + 'DB_CONNECTION' => getenv('DB_CONNECTION'), + 'DB_DATABASE' => getenv('DB_DATABASE'), + ]; + + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings (name, value) VALUES ('categorizeforeign', '0'), ('catwebdl', '1')"); + + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $this->databasePath, + ]); + + $this->createSchema(); + } + + protected function tearDown(): void + { + if ($this->databasePath !== '' && file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + parent::tearDown(); + + foreach ($this->originalEnvironment as $key => $value) { + $this->setEnvironmentValue($key, $value === false ? null : $value); + } + } + + public function test_it_adds_only_missing_reverse_lookup_and_queue_indexes_idempotently(): void + { + $migration = require database_path('migrations/2026_08_04_082439_add_fix_release_name_query_indexes.php'); + + $migration->up(); + $migration->up(); + + $this->assertTrue(Schema::hasIndex('par_hashes', 'ix_par_hashes_hash_releases_id')); + $this->assertTrue(Schema::hasIndex('release_files', 'ix_release_files_crc32_releases_id')); + $this->assertTrue(Schema::hasIndex('predb', 'ix_predb_searched_predate_id')); + + $migration->down(); + + $this->assertFalse(Schema::hasIndex('par_hashes', 'ix_par_hashes_hash_releases_id')); + $this->assertFalse(Schema::hasIndex('release_files', 'ix_release_files_crc32_releases_id')); + $this->assertFalse(Schema::hasIndex('predb', 'ix_predb_searched_predate_id')); + } + + private function createSchema(): void + { + Schema::create('par_hashes', function (Blueprint $table): void { + $table->unsignedInteger('releases_id'); + $table->string('hash', 32); + $table->primary(['releases_id', 'hash']); + }); + Schema::create('release_files', function (Blueprint $table): void { + $table->unsignedInteger('releases_id'); + $table->string('name'); + $table->string('crc32')->default(''); + $table->primary(['releases_id', 'name']); + }); + Schema::create('predb', function (Blueprint $table): void { + $table->increments('id'); + $table->tinyInteger('searched')->default(0)->index(); + $table->dateTime('predate')->nullable()->index(); + }); + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv($key.'='.$value); + $_ENV[$key] = $value; + $_SERVER[$key] = $value; + } +} diff --git a/tests/Feature/PredbFulltextNameFixingTest.php b/tests/Feature/PredbFulltextNameFixingTest.php new file mode 100644 index 000000000..4827babf5 --- /dev/null +++ b/tests/Feature/PredbFulltextNameFixingTest.php @@ -0,0 +1,107 @@ +once()->andReturnTrue(); + Search::shouldReceive('searchReleases') + ->once() + ->with(['name' => $title, 'searchname' => $title], 21) + ->andReturn([42]); + + $database = $this->createMock(ConnectionInterface::class); + $database->expects($this->once()) + ->method('select') + ->willReturn([ + (object) [ + 'releases_id' => 42, + 'name' => $title, + 'searchname' => $title, + 'fromname' => 'poster', + 'groups_id' => 1, + 'categories_id' => 7010, + ], + ]); + + $updates = $this->createMock(ReleaseUpdateService::class); + $updates->expects($this->once())->method('attachPredbId')->with(42, 7); + $service = new NameFixingService( + updateService: $updates, + queries: new NameFixingQueryService($database) + ); + + $matched = $service->matchPredbFulltext((object) [ + 'predb_id' => 7, + 'title' => $title, + 'source' => 'srrdb', + ]); + + $this->assertSame(1, $matched); + } + + public function test_it_marks_sixteen_confirmed_candidates_as_a_flood(): void + { + $title = 'Valid.Scene.Release.2026-GROUP'; + Search::shouldReceive('isAvailable')->once()->andReturnTrue(); + Search::shouldReceive('searchReleases')->once()->andReturn(range(1, 21)); + + $database = $this->createStub(ConnectionInterface::class); + $database->method('select')->willReturn(array_map( + static fn (int $id): object => (object) [ + 'releases_id' => $id, + 'name' => $title, + 'searchname' => 'Old.Name-'.$id, + 'fromname' => 'poster', + 'groups_id' => 1, + 'categories_id' => 7010, + ], + range(1, 16) + )); + + $service = new NameFixingService( + updateService: $this->createStub(ReleaseUpdateService::class), + queries: new NameFixingQueryService($database) + ); + + $matched = $service->matchPredbFulltext((object) [ + 'predb_id' => 7, + 'title' => $title, + 'source' => 'srrdb', + ]); + + $this->assertSame(-1, $matched); + } + + public function test_backend_unavailability_is_not_recorded_as_a_no_match(): void + { + Search::shouldReceive('isAvailable')->once()->andReturnFalse(); + Search::shouldReceive('searchReleases')->never(); + $database = $this->createMock(ConnectionInterface::class); + $database->expects($this->never())->method('select'); + $service = new NameFixingService( + updateService: $this->createStub(ReleaseUpdateService::class), + queries: new NameFixingQueryService($database) + ); + + $this->expectException(RuntimeException::class); + $service->matchPredbFulltext((object) [ + 'predb_id' => 7, + 'title' => 'Valid.Scene.Release.2026-GROUP', + 'source' => 'srrdb', + ]); + } +} diff --git a/tests/Feature/ReleaseNameFixedRecategorizationTest.php b/tests/Feature/ReleaseNameFixedRecategorizationTest.php index 5df4044b1..2086f5f53 100644 --- a/tests/Feature/ReleaseNameFixedRecategorizationTest.php +++ b/tests/Feature/ReleaseNameFixedRecategorizationTest.php @@ -222,6 +222,29 @@ class ReleaseNameFixedRecategorizationTest extends TestCase $this->assertSame(1, (int) $release->isrenamed); } + public function test_internal_processing_status_updates_do_not_refresh_the_search_index(): void + { + Search::shouldReceive('updateRelease')->once(); + + $group = UsenetGroup::query()->create([ + 'name' => 'alt.binaries.test', + 'active' => 1, + 'backfill' => 0, + ]); + $release = Release::factory()->create([ + 'groups_id' => $group->id, + 'guid' => str_repeat('d', 40), + 'leftguid' => 'd', + 'proc_nfo' => 0, + ]); + + app(ReleaseUpdateService::class)->updateSingleColumn('proc_nfo', 1, $release->id); + app(ReleaseUpdateService::class)->attachPredbId($release->id, 123); + + $this->assertSame(1, (int) $release->fresh()->proc_nfo); + $this->assertSame(123, (int) $release->fresh()->predb_id); + } + private function setEnvironmentValue(string $key, ?string $value): void { if ($value === null) { diff --git a/tests/Unit/Services/NameFixing/DonorMatchSelectorTest.php b/tests/Unit/Services/NameFixing/DonorMatchSelectorTest.php new file mode 100644 index 000000000..1b3078f8b --- /dev/null +++ b/tests/Unit/Services/NameFixing/DonorMatchSelectorTest.php @@ -0,0 +1,38 @@ +select([ + (object) ['releases_id' => 20, 'relsize' => 950], + (object) ['releases_id' => 10, 'relsize' => 1050], + (object) ['releases_id' => 30, 'relsize' => 900], + ], 1000, 10); + + $this->assertNotNull($selected); + $this->assertSame(10, $selected->releases_id); + } + + public function test_it_rejects_zero_sized_and_out_of_tolerance_donors(): void + { + $selector = new DonorMatchSelector; + + $selected = $selector->select([ + (object) ['releases_id' => 10, 'relsize' => 0], + (object) ['releases_id' => 20, 'relsize' => 500], + ], 1000, 10); + + $this->assertNull($selected); + $this->assertNull($selector->select([(object) ['releases_id' => 30, 'relsize' => 1000]], 0, 10)); + } +} diff --git a/tests/Unit/Services/NameFixing/NameFixingBatchTest.php b/tests/Unit/Services/NameFixing/NameFixingBatchTest.php new file mode 100644 index 000000000..1e59de0c6 --- /dev/null +++ b/tests/Unit/Services/NameFixing/NameFixingBatchTest.php @@ -0,0 +1,72 @@ +createMock(ConnectionInterface::class); + $database->expects($this->exactly(5)) + ->method('select') + ->willReturnCallback(function (string $sql): array { + $this->assertStringNotContainsString('GROUP_CONCAT', $sql); + $this->assertStringNotContainsString('GROUP BY r.id', $sql); + + if (str_contains($sql, 'r.proc_nfo')) { + $this->assertStringContainsString('r.nfostatus > -1', $sql); + + return [$this->processedRelease(10), $this->processedRelease(20)]; + } + + return []; + }); + + $serviceReflection = new ReflectionClass(NameFixingService::class); + $service = $serviceReflection->newInstanceWithoutConstructor(); + $serviceReflection->getProperty('queries') + ->setValue($service, new NameFixingQueryService($database)); + $serviceReflection->getProperty('updateService') + ->setValue( + $service, + (new ReflectionClass(ReleaseUpdateService::class))->newInstanceWithoutConstructor() + ); + + $this->assertSame( + ['checked' => 2, 'fixed' => 0], + $service->processStandardBatch('a', 100, false) + ); + } + + private function processedRelease(int $id): object + { + return (object) [ + 'releases_id' => $id, + 'guid' => "guid-{$id}", + 'name' => "release-{$id}", + 'searchname' => "release-{$id}", + 'size' => 1000, + 'groupName' => 'alt.binaries.test', + 'categoryid' => 8010, + 'nfostatus' => 0, + 'proc_nfo' => NameFixingService::PROC_NFO_DONE, + 'proc_files' => NameFixingService::PROC_FILES_DONE, + 'proc_par2' => NameFixingService::PROC_PAR2_DONE, + 'proc_uid' => NameFixingService::PROC_UID_DONE, + 'proc_hash16k' => NameFixingService::PROC_HASH16K_DONE, + 'proc_srr' => NameFixingService::PROC_SRR_DONE, + 'proc_crc32' => NameFixingService::PROC_CRC_DONE, + ]; + } +} diff --git a/tests/Unit/Services/NameFixing/NameFixingQueryServiceTest.php b/tests/Unit/Services/NameFixing/NameFixingQueryServiceTest.php new file mode 100644 index 000000000..213e33181 --- /dev/null +++ b/tests/Unit/Services/NameFixing/NameFixingQueryServiceTest.php @@ -0,0 +1,60 @@ +createStub(ConnectionInterface::class); + $service = new NameFixingQueryService($database); + $first = (object) ['releases_id' => 10, 'textstring' => 'first.rar']; + $second = (object) ['releases_id' => 10, 'textstring' => 'second.rar']; + $third = (object) ['releases_id' => 20, 'textstring' => 'third.rar']; + + $grouped = $service->groupByReleaseId([$first, $second, $third]); + + $this->assertSame([$first, $second], $grouped[10]); + $this->assertSame([$third], $grouped[20]); + } + + public function test_file_candidates_use_exists_without_lossy_grouping(): void + { + $database = $this->createMock(ConnectionInterface::class); + $database->expects($this->once()) + ->method('select') + ->with( + $this->callback(static fn (string $sql): bool => str_contains($sql, 'EXISTS (SELECT 1 FROM release_files') + && ! str_contains($sql, 'GROUP BY')), + $this->callback(static fn (array $bindings): bool => $bindings[array_key_last($bindings)] === 100) + ) + ->willReturn([]); + + $service = new NameFixingQueryService($database); + $service->candidateBatch(NameFixingQueryService::SOURCE_FILES, 2, 2, 0, 100); + } + + public function test_predb_workers_use_disjoint_modulo_partitions_without_offsets(): void + { + $database = $this->createMock(ConnectionInterface::class); + $database->expects($this->once()) + ->method('select') + ->with( + $this->callback(static fn (string $sql): bool => str_contains($sql, 'MOD(p.id, ?) = ?') + && ! str_contains($sql, 'OFFSET')), + $this->callback(static fn (array $bindings): bool => $bindings[1] === 4 + && $bindings[2] === 2 + && $bindings[3] === 250) + ) + ->willReturn([]); + + $service = new NameFixingQueryService($database); + $service->predbBatch(3, 4, 250); + } +}