starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update namefixing

Browse Source
This commit is contained in:
DariusIII
2026-08-04 08:49:07 +02:00
parent 341b9f7e21
commit a10aae7f05
18 changed files with 1889 additions and 993 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/MatchPrefiles.php
+2 -5
View File
@@ -28,10 +28,8 @@ class MatchPrefiles extends Command
/**
* Execute the console command.
*
* @return int
*/
public function handle()
public function handle(NameFixingService $nameFixingService): int
{
$limit = $this->argument('limit');
@@ -53,8 +51,7 @@ class MatchPrefiles extends Command
}
try {
$nameFixingService = new NameFixingService;
$nameFixingService->getPreFileNames($argv); // @phpstan-ignore argument.type
$nameFixingService->getPreFileNames($argv);
return 0;
} catch (Exception $e) {
app/Console/Commands/ReleasesFixNamesGroup.php
+64 -252
View File
@@ -4,15 +4,14 @@ declare(strict_types=1);
namespace App\Console\Commands;
use App\Models\Category;
use App\Models\Predb;
use App\Models\Release;
use App\Services\NameFixing\NameFixingQueryService;
use App\Services\NameFixing\NameFixingService;
use App\Services\NfoService;
use App\Services\NNTP\NNTPService;
use App\Services\Nzb\NzbContentsService;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use RuntimeException;
class ReleasesFixNamesGroup extends Command
{
@@ -25,7 +24,8 @@ class ReleasesFixNamesGroup extends Command
{type : Type of fix (standard|predbft)}
{--guid-char= : GUID character to process (for standard type)}
{--limit=1000 : Maximum releases to process}
{--thread=1 : Thread number (for predbft type)}';
{--thread=1 : Worker number (for predbft type)}
{--workers=1 : Total workers (for predbft type)}';
/**
* The console command description.
@@ -36,6 +36,8 @@ class ReleasesFixNamesGroup extends Command
private NameFixingService $nameFixingService;
private NameFixingQueryService $queries;
private int $checked = 0;
/**
@@ -46,7 +48,8 @@ class ReleasesFixNamesGroup extends Command
$type = $this->argument('type');
$maxPerRun = (int) $this->option('limit');
$this->nameFixingService = new NameFixingService;
$this->nameFixingService = app(NameFixingService::class);
$this->queries = app(NameFixingQueryService::class);
switch ($type) {
case 'standard':
@@ -78,173 +81,52 @@ class ReleasesFixNamesGroup extends Command
$this->info("Processing releases with GUID starting with: {$guidChar}");
$this->info("Maximum per run: {$maxPerRun}");
// Allow for larger filename return sets
DB::statement('SET SESSION group_concat_max_len = 65536');
$nntp = null;
$nzbcontents = null;
$connectionAttempted = false;
$par2Processor = function (object $release) use (&$connectionAttempted, &$nntp, &$nzbcontents): bool {
if (! $connectionAttempted) {
$connectionAttempted = true;
$nntp = app(NNTPService::class);
$compressedHeaders = config('nntmux_nntp.compressed_headers');
$connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true
? $nntp->doConnect($compressedHeaders, true)
: $nntp->doConnect();
// Find releases to process
$releases = $this->fetchReleases($guidChar, $maxPerRun);
if ($connectResult !== true) {
$errorMessage = 'Unable to connect to usenet for PAR2 processing';
if (NNTPService::isError($connectResult)) {
$errorMessage .= ' Error: '.$connectResult->getMessage();
}
$this->warn($errorMessage);
} else {
$nzbcontents = app(NzbContentsService::class);
$nzbcontents->setNntp($nntp);
$nzbcontents->setNfo(app(NfoService::class));
$nzbcontents->setEchoOutput(false);
}
}
if ($releases->isEmpty()) {
return $nzbcontents !== null && $nzbcontents->checkPar2(
$release->guid,
$release->releases_id,
$release->groups_id,
1,
1
);
};
$stats = $this->nameFixingService->processStandardBatch($guidChar, $maxPerRun, true, $par2Processor);
$this->checked += $stats['checked'];
if ($stats['checked'] === 0) {
$this->info('No releases to process');
return Command::SUCCESS;
}
$this->info("Found {$releases->count()} releases to process");
$bar = $this->output->createProgressBar($releases->count());
$bar->start();
$nntp = null;
$nzbcontents = null;
foreach ($releases as $release) {
$this->checked++;
$this->nameFixingService->reset();
// Process UID
if ((int) $release->proc_uid === NameFixingService::PROC_UID_NONE &&
(! empty($release->uid) || ! empty($release->mediainfo))) {
if (! empty($release->uid)) {
$this->nameFixingService->checkName($release, true, 'UID, ', true, true);
}
if (! $this->nameFixingService->getUpdateService()->matched && ! empty($release->mediainfo)) {
$this->nameFixingService->checkName($release, true, 'Mediainfo, ', true, true);
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_uid', NameFixingService::PROC_UID_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process CRC32
if ((int) $release->proc_crc32 === NameFixingService::PROC_CRC_NONE && ! empty($release->crc)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'CRC32, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_crc32', NameFixingService::PROC_CRC_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process SRR
if ((int) $release->proc_srr === NameFixingService::PROC_SRR_NONE) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'SRR, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_srr', NameFixingService::PROC_SRR_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process PAR2 hash
if ((int) $release->proc_hash16k === NameFixingService::PROC_HASH16K_NONE && ! empty($release->hash)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'PAR2 hash, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_hash16k', NameFixingService::PROC_HASH16K_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process NFO
if ((int) $release->nfostatus === NfoService::NFO_FOUND &&
(int) $release->proc_nfo === NameFixingService::PROC_NFO_NONE &&
! empty($release->textstring) &&
! preg_match('/^=newz\[NZB\]=\w+/', $release->textstring)) {
$this->nameFixingService->reset();
$this->nameFixingService->checkName($release, true, 'NFO, ', true, true);
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_nfo', NameFixingService::PROC_NFO_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process filenames
if ((int) $release->fileid > 0 && (int) $release->proc_files === NameFixingService::PROC_FILES_NONE) {
$this->nameFixingService->reset();
$fileNames = explode('|', $release->filestring);
$releaseFile = $release;
foreach ($fileNames as $fileName) {
if (! $this->nameFixingService->getUpdateService()->matched) {
$releaseFile->textstring = $fileName;
$this->nameFixingService->checkName($releaseFile, true, 'Filenames, ', true, true);
}
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_files', NameFixingService::PROC_FILES_DONE, $release->releases_id);
if ($this->nameFixingService->getUpdateService()->matched) {
$bar->advance();
continue;
}
// Process PAR2
if ((int) $release->proc_par2 === NameFixingService::PROC_PAR2_NONE) {
// Initialize NZB contents if needed
if (! isset($nzbcontents)) {
$nntp = new NNTPService;
$compressedHeaders = config('nntmux_nntp.compressed_headers');
$connectResult = config('nntmux_nntp.use_alternate_nntp_server') === true
? $nntp->doConnect($compressedHeaders, true)
: $nntp->doConnect();
if ($connectResult !== true) {
$errorMessage = 'Unable to connect to usenet for PAR2 processing';
if (NNTPService::isError($connectResult)) {
$errorMessage .= ' Error: '.$connectResult->getMessage();
}
$this->warn($errorMessage);
} else {
$Nfo = new NfoService;
$nzbcontents = app(NzbContentsService::class);
$nzbcontents->setNntp($nntp);
$nzbcontents->setNfo($Nfo);
$nzbcontents->setEchoOutput(false);
}
}
if (isset($nzbcontents)) {
$nzbcontents->checkPar2($release->guid, $release->releases_id, $release->groups_id, 1, 1);
}
}
$this->nameFixingService->getUpdateService()->updateSingleColumn('proc_par2', NameFixingService::PROC_PAR2_DONE, $release->releases_id);
$bar->advance();
}
$bar->finish();
$this->newLine(2);
$this->info("✅ Processed {$this->checked} releases");
$this->info("✅ Fixed {$this->nameFixingService->getUpdateService()->fixed} release names");
$this->info("Processed {$stats['checked']} releases");
$this->info("Fixed {$stats['fixed']} release names");
return Command::SUCCESS;
}
@@ -255,51 +137,45 @@ class ReleasesFixNamesGroup extends Command
protected function processPredbFulltext(int $maxPerRun): int
{
$thread = (int) $this->option('thread');
$offset = $thread * $maxPerRun - $maxPerRun;
$workers = (int) $this->option('workers');
$this->info('Processing PreDB fulltext matching');
$this->info("Thread: {$thread}, Limit: {$maxPerRun}, Offset: {$offset}");
$this->info("Worker: {$thread}/{$workers}, Limit: {$maxPerRun}");
$pres = Predb::fromQuery(
sprintf(
'
SELECT p.id AS predb_id, p.title, p.source, p.searched
FROM predb p
WHERE LENGTH(p.title) >= 15 AND p.title NOT REGEXP "[\"\<\> ]"
AND p.searched = 0
AND p.predate < (NOW() - INTERVAL 1 DAY)
ORDER BY p.predate ASC
LIMIT %s
OFFSET %s',
$maxPerRun,
$offset
)
);
$pres = $this->queries->predbBatch($thread, $workers, $maxPerRun);
if ($pres->isEmpty()) {
if ($pres === []) {
$this->info('No PreDB entries to process');
return Command::SUCCESS;
}
$this->info("Found {$pres->count()} PreDB entries to process");
$bar = $this->output->createProgressBar($pres->count());
$this->info('Found '.count($pres).' PreDB entries to process');
$bar = $this->output->createProgressBar(count($pres));
$bar->start();
foreach ($pres as $pre) {
$searched = 0;
$ftmatched = $this->matchPredbFT($pre);
try {
$ftmatched = $this->nameFixingService->matchPredbFulltext($pre);
} catch (RuntimeException $exception) {
$bar->finish();
$this->newLine(2);
$this->error($exception->getMessage());
return Command::FAILURE;
}
if ($ftmatched > 0) {
$searched = 1;
} elseif ($ftmatched < 0) {
$searched = -6;
} else {
$searched = $pre['searched'] - 1;
$searched = (int) $pre->searched - 1;
}
Predb::query()->where('id', $pre['predb_id'])->update(['searched' => $searched]);
DB::update('UPDATE predb SET searched = ? WHERE id = ?', [$searched, $pre->predb_id]);
$this->checked++;
$bar->advance();
@@ -312,68 +188,4 @@ class ReleasesFixNamesGroup extends Command
return Command::SUCCESS;
}
/**
* Match a PreDB title to releases using full-text search.
*/
protected function matchPredbFT(object $pre): int
{
// This is a simplified version - the full implementation would use Manticore/Elasticsearch
return 0;
}
/**
* Fetch releases for processing
*/
protected function fetchReleases(string $guidChar, int $maxPerRun): mixed
{
return Release::fromQuery(sprintf("
SELECT
r.id AS releases_id, r.fromname, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo,
r.proc_uid, r.proc_files, r.proc_par2, r.nfostatus,
r.size AS relsize, r.predb_id, r.proc_hash16k, r.proc_srr, r.proc_crc32,
IFNULL(rf.releases_id, 0) AS fileid,
IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring,
IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring,
IFNULL(ru.uniqueid, '') AS uid,
IFNULL(ph.hash, 0) AS hash,
IFNULL(rf.crc32, '') AS crc
FROM releases r
LEFT JOIN release_nfos rn ON rn.releases_id = r.id
LEFT JOIN release_files rf ON rf.releases_id = r.id
LEFT JOIN release_unique ru ON ru.releases_id = r.id
LEFT JOIN par_hashes ph ON ph.releases_id = r.id
WHERE r.leftguid = %s
AND r.isrenamed = %d
AND r.predb_id = 0
AND r.passwordstatus >= 0
AND r.nfostatus > %d
AND (
(r.nfostatus = %d AND r.proc_nfo = %d)
OR r.proc_files = %d
OR r.proc_uid = %d
OR r.proc_par2 = %d
OR r.proc_srr = %d
OR r.proc_hash16k = %d
OR r.proc_crc32 = %d
)
AND r.categories_id IN (%s)
GROUP BY r.id
ORDER BY r.id DESC
LIMIT %s",
escapeString($guidChar),
NameFixingService::IS_RENAMED_NONE,
NfoService::NFO_UNPROC,
NfoService::NFO_FOUND,
NameFixingService::PROC_NFO_NONE,
NameFixingService::PROC_FILES_NONE,
NameFixingService::PROC_UID_NONE,
NameFixingService::PROC_PAR2_NONE,
NameFixingService::PROC_SRR_NONE,
NameFixingService::PROC_HASH16K_NONE,
NameFixingService::PROC_CRC_NONE,
Category::getCategoryOthersGroup(),
$maxPerRun
));
}
}
app/Services/NameFixing/DonorMatchSelector.php
+48
View File
@@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
namespace App\Services\NameFixing;
final class DonorMatchSelector
{
/**
* @param list<object> $donors
*/
public function select(array $donors, int $targetSize, int $tolerancePercent): ?object
{
if ($targetSize <= 0 || $donors === []) {
return null;
}
$eligible = [];
foreach ($donors as $donor) {
$donorSize = (int) ($donor->relsize ?? $donor->size ?? 0);
if ($donorSize <= 0) {
continue;
}
$difference = abs($donorSize - $targetSize);
$differencePercent = ($difference / $donorSize) * 100;
if ($differencePercent <= $tolerancePercent) {
$eligible[] = [
'donor' => $donor,
'difference' => $difference,
'id' => (int) ($donor->releases_id ?? $donor->id ?? PHP_INT_MAX),
];
}
}
if ($eligible === []) {
return null;
}
usort($eligible, static function (array $left, array $right): int {
return [$left['difference'], $left['id']] <=> [$right['difference'], $right['id']];
});
return $eligible[0]['donor'];
}
}
app/Services/NameFixing/FilePrioritizer.php
+6 -6
View File
@@ -58,8 +58,8 @@ class FilePrioritizer
* 5. NFO files
* 6. Other files
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Sorted array of filenames
* @param list<string> $files Array of filenames
* @return list<string> Sorted array of filenames
*/
public function prioritizeForMatching(array $files): array
{
@@ -82,8 +82,8 @@ class FilePrioritizer
* Similar to prioritizeForMatching but with slightly different priorities
* optimized for PreDB filename lookups.
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Sorted array of filenames
* @param list<string> $files Array of filenames
* @return list<string> Sorted array of filenames
*/
public function prioritizeForPreDb(array $files): array
{
@@ -102,8 +102,8 @@ class FilePrioritizer
/**
* Categorize files into groups by type.
*
* @param array<string, mixed> $files Array of filenames
* @return array<string, mixed> Categorized files
* @param list<string> $files Array of filenames
* @return array{srr: list<string>, mainRar: list<string>, firstPart: list<string>, video: list<string>, nfo: list<string>, other: list<string>} Categorized files
*/
protected function categorizeFiles(array $files): array
{
app/Services/NameFixing/NameFixingQueryService.php
+481
View File
@@ -0,0 +1,481 @@
<?php
declare(strict_types=1);
namespace App\Services\NameFixing;
use App\Models\Category;
use Carbon\CarbonImmutable;
use Illuminate\Database\ConnectionInterface;
use Illuminate\Support\Facades\DB;
use InvalidArgumentException;
final class NameFixingQueryService
{
public const SOURCE_NFO = 'nfo';
public const SOURCE_FILES = 'files';
public const SOURCE_SRR = 'srr';
public const SOURCE_CRC = 'crc';
public const SOURCE_UID = 'uid';
public const SOURCE_HASH = 'hash';
public const SOURCE_PAR2 = 'par2';
public const SOURCE_XXX = 'xxx';
public const SOURCE_MEDIA_MOVIE = 'media_movie';
public const BATCH_SIZE = 1000;
private ConnectionInterface $database;
/**
* @var array<string, string>
*/
private const STATUS_COLUMNS = [
self::SOURCE_NFO => 'proc_nfo',
self::SOURCE_FILES => 'proc_files',
self::SOURCE_SRR => 'proc_srr',
self::SOURCE_CRC => 'proc_crc32',
self::SOURCE_UID => 'proc_uid',
self::SOURCE_HASH => 'proc_hash16k',
self::SOURCE_PAR2 => 'proc_par2',
self::SOURCE_XXX => 'proc_files',
self::SOURCE_MEDIA_MOVIE => 'proc_uid',
];
/**
* @var array<string, string>
*/
private const SOURCE_EXISTS = [
self::SOURCE_NFO => 'EXISTS (SELECT 1 FROM release_nfos source_nfo WHERE source_nfo.releases_id = r.id)',
self::SOURCE_FILES => 'EXISTS (SELECT 1 FROM release_files source_file WHERE source_file.releases_id = r.id)',
self::SOURCE_SRR => "EXISTS (SELECT 1 FROM release_files source_srr WHERE source_srr.releases_id = r.id AND (source_srr.name LIKE '%.srr' OR source_srr.name LIKE '%.srs'))",
self::SOURCE_CRC => "EXISTS (SELECT 1 FROM release_files source_crc WHERE source_crc.releases_id = r.id AND source_crc.crc32 IS NOT NULL AND source_crc.crc32 != '')",
self::SOURCE_UID => "EXISTS (SELECT 1 FROM media_infos source_media WHERE source_media.releases_id = r.id AND source_media.unique_id IS NOT NULL AND source_media.unique_id != '') OR EXISTS (SELECT 1 FROM release_unique source_unique WHERE source_unique.releases_id = r.id AND source_unique.uniqueid != '')",
self::SOURCE_HASH => "EXISTS (SELECT 1 FROM par_hashes source_hash WHERE source_hash.releases_id = r.id AND source_hash.hash != '')",
self::SOURCE_PAR2 => '1 = 1',
self::SOURCE_XXX => "EXISTS (SELECT 1 FROM release_files source_xxx WHERE source_xxx.releases_id = r.id AND source_xxx.name LIKE '%SDPORN%')",
self::SOURCE_MEDIA_MOVIE => "EXISTS (SELECT 1 FROM media_infos source_movie WHERE source_movie.releases_id = r.id AND source_movie.movie_name IS NOT NULL AND source_movie.movie_name != '')",
];
public function __construct(?ConnectionInterface $database = null)
{
$this->database = $database ?? DB::connection();
}
/**
* @return list<object>
*/
public function candidateBatch(
string $source,
int $time,
int $categories,
int $afterId = 0,
int $limit = self::BATCH_SIZE
): array {
[$where, $bindings] = $this->candidateWhere($source, $time, $categories);
$bindings[] = $afterId;
$bindings[] = max(1, $limit);
return $this->database->select(
"SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.groups_id,
r.categories_id, r.size AS relsize, r.guid, r.predb_id, r.nfostatus,
r.proc_nfo, r.proc_files, r.proc_par2, r.proc_uid, r.proc_srr,
r.proc_hash16k, r.proc_crc32
FROM releases r
WHERE {$where}
AND r.id > ?
ORDER BY r.id ASC
LIMIT ?",
$bindings
);
}
public function countCandidates(string $source, int $time, int $categories): int
{
[$where, $bindings] = $this->candidateWhere($source, $time, $categories);
$rows = $this->database->select(
"SELECT COUNT(*) AS aggregate FROM releases r WHERE {$where}",
$bindings
);
return (int) ($rows[0]->aggregate ?? 0);
}
/**
* @return list<object>
*/
public function standardCandidateBatch(string $leftGuid, int $limit): array
{
return $this->database->select(
'SELECT r.id AS releases_id, r.id, r.name, r.searchname, r.fromname, r.guid,
r.groups_id, r.categories_id, r.size AS relsize, r.predb_id, r.nfostatus,
r.proc_nfo, r.proc_uid, r.proc_files, r.proc_par2, r.proc_hash16k,
r.proc_srr, r.proc_crc32
FROM releases r
WHERE r.leftguid = ?
AND r.isrenamed = 0
AND r.predb_id = 0
AND r.passwordstatus >= 0
AND r.nfostatus > -1
AND (
(r.nfostatus = 1 AND r.proc_nfo = 0)
OR r.proc_files = 0
OR r.proc_uid = 0
OR r.proc_par2 = 0
OR r.proc_srr = 0
OR r.proc_hash16k = 0
OR r.proc_crc32 = 0
)
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')
ORDER BY r.id DESC
LIMIT ?',
[$leftGuid, max(1, $limit)]
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function nfoRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT rn.releases_id, UNCOMPRESS(rn.nfo) AS textstring
FROM release_nfos rn
WHERE rn.releases_id IN (%s)
ORDER BY rn.releases_id',
$releaseIds
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function fileRows(array $releaseIds, string $source = self::SOURCE_FILES): array
{
$filter = match ($source) {
self::SOURCE_FILES => '',
self::SOURCE_SRR => " AND (rf.name LIKE '%.srr' OR rf.name LIKE '%.srs')",
self::SOURCE_CRC => " AND rf.crc32 IS NOT NULL AND rf.crc32 != ''",
self::SOURCE_XXX => " AND rf.name LIKE '%SDPORN%'",
default => throw new InvalidArgumentException("Unsupported release-file source [{$source}]."),
};
return $this->selectForReleaseIds(
'SELECT rf.releases_id, rf.name AS textstring, rf.name AS filename, rf.crc32
FROM release_files rf
WHERE rf.releases_id IN (%s)'.$filter.'
ORDER BY rf.releases_id, rf.name',
$releaseIds
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function mediaRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT mi.releases_id, mi.unique_id AS uid, mi.movie_name, mi.file_name
FROM media_infos mi
WHERE mi.releases_id IN (%s)
UNION ALL
SELECT ru.releases_id, ru.uniqueid AS uid, NULL AS movie_name, NULL AS file_name
FROM release_unique ru
WHERE ru.releases_id IN (%s)
ORDER BY releases_id',
array_merge($releaseIds, $releaseIds),
2
);
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
public function hashRows(array $releaseIds): array
{
return $this->selectForReleaseIds(
'SELECT ph.releases_id, ph.hash
FROM par_hashes ph
WHERE ph.releases_id IN (%s)
ORDER BY ph.releases_id, ph.hash',
$releaseIds
);
}
/**
* @param list<string> $uniqueIds
* @return array<string, list<object>>
*/
public function uidDonors(array $uniqueIds): array
{
if ($uniqueIds === []) {
return [];
}
$placeholders = $this->placeholders(count($uniqueIds));
$bindings = array_merge($uniqueIds, ['nonscene@Ef.net (EF)'], $uniqueIds);
$rows = $this->database->select(
"SELECT mi.unique_id AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM media_infos mi
INNER JOIN releases r ON r.id = mi.releases_id
WHERE mi.unique_id IN ({$placeholders})
AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = ?)
UNION ALL
SELECT ru.uniqueid AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM release_unique ru
INNER JOIN releases r ON r.id = ru.releases_id
WHERE ru.uniqueid IN ({$placeholders})
AND (r.predb_id > 0 OR r.anidbid > 0 OR r.fromname = 'nonscene@Ef.net (EF)')",
$bindings
);
return $this->groupByKey($rows, 'match_key');
}
/**
* @param list<string> $hashes
* @return array<string, list<object>>
*/
public function hashDonors(array $hashes): array
{
return $this->donors(
'SELECT ph.hash AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM par_hashes ph
INNER JOIN releases r ON r.id = ph.releases_id
WHERE ph.hash IN (%s)
AND (r.predb_id > 0 OR r.anidbid > 0)',
$hashes
);
}
/**
* @param list<string> $crcs
* @return array<string, list<object>>
*/
public function crcDonors(array $crcs): array
{
return $this->donors(
'SELECT rf.crc32 AS match_key, r.id AS releases_id, r.size AS relsize,
r.searchname, r.fromname, r.predb_id
FROM release_files rf
INNER JOIN releases r ON r.id = rf.releases_id
WHERE rf.crc32 IN (%s)
AND r.predb_id > 0',
$crcs
);
}
/**
* @return list<object>
*/
public function predbBatch(int $worker, int $workers, int $limit): array
{
$workerCount = max(1, min(16, $workers));
$workerSlot = max(1, min($workerCount, $worker)) - 1;
return $this->database->select(
'SELECT p.id AS predb_id, p.title, p.source, p.searched
FROM predb p
WHERE LENGTH(p.title) >= 15
AND p.title NOT REGEXP \'["<> ]\'
AND p.searched = 0
AND p.predate < ?
AND MOD(p.id, ?) = ?
ORDER BY p.predate ASC, p.id ASC
LIMIT ?',
[
CarbonImmutable::now()->subDay()->toDateTimeString(),
$workerCount,
$workerSlot,
max(1, $limit),
]
);
}
/**
* @param list<int> $candidateIds
* @return list<object>
*/
public function confirmPredbCandidates(array $candidateIds, string $title): array
{
if ($candidateIds === []) {
return [];
}
$like = '%'.$this->escapeLike($title).'%';
return $this->database->select(
'SELECT r.id AS releases_id, r.name, r.fromname, r.searchname,
r.groups_id, r.categories_id
FROM releases r
WHERE r.id IN ('.$this->placeholders(count($candidateIds)).')
AND r.predb_id = 0
AND (r.name LIKE ? ESCAPE \'\\\\\' OR r.searchname LIKE ? ESCAPE \'\\\\\')
ORDER BY r.id ASC
LIMIT 21',
array_merge($candidateIds, [$like, $like])
);
}
public function countPrefileCandidates(): int
{
$rows = $this->database->select(
'SELECT COUNT(*) AS aggregate
FROM releases r
WHERE r.predb_id = 0
AND r.isrenamed = 0
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')
AND EXISTS (
SELECT 1 FROM release_files rf
WHERE rf.releases_id = r.id
AND rf.name IS NOT NULL
)'
);
return (int) ($rows[0]->aggregate ?? 0);
}
/**
* @return list<object>
*/
public function prefileCandidateBatch(int $cursor, int $limit, bool $descending): array
{
$operator = $descending ? '<' : '>';
$direction = $descending ? 'DESC' : 'ASC';
return $this->database->select(
'SELECT r.id AS releases_id, r.name, r.searchname, r.fromname,
r.groups_id, r.categories_id
FROM releases r
WHERE r.predb_id = 0
AND r.isrenamed = 0
AND r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).")
AND r.id {$operator} ?
AND EXISTS (
SELECT 1 FROM release_files rf
WHERE rf.releases_id = r.id
AND rf.name IS NOT NULL
)
ORDER BY r.id {$direction}
LIMIT ?",
[$cursor, max(1, $limit)]
);
}
/**
* @param list<object> $rows
* @return array<int, list<object>>
*/
public function groupByReleaseId(array $rows): array
{
return $this->groupByKey($rows, 'releases_id');
}
/**
* @return array{0: string, 1: list<mixed>}
*/
private function candidateWhere(string $source, int $time, int $categories): array
{
if (! isset(self::SOURCE_EXISTS[$source], self::STATUS_COLUMNS[$source])) {
throw new InvalidArgumentException("Unsupported name-fixing source [{$source}].");
}
$where = ['r.predb_id = 0', '('.self::SOURCE_EXISTS[$source].')'];
$bindings = [];
if ($categories !== 3) {
$statusColumn = self::STATUS_COLUMNS[$source];
$where[] = '(r.isrenamed = 0 OR r.categories_id IN (?, ?))';
$bindings[] = Category::OTHER_MISC;
$bindings[] = Category::OTHER_HASHED;
$where[] = "r.{$statusColumn} = 0";
}
if ($time === 1) {
$where[] = 'r.adddate > ?';
$bindings[] = CarbonImmutable::now()->subHours(6)->toDateTimeString();
}
if ($categories === 1) {
$where[] = 'r.categories_id IN ('.implode(',', Category::OTHERS_GROUP).')';
}
return [implode(' AND ', $where), $bindings];
}
/**
* @param list<int> $releaseIds
* @return list<object>
*/
private function selectForReleaseIds(
string $sql,
array $releaseIds,
int $placeholderGroups = 1
): array {
if ($releaseIds === []) {
return [];
}
$placeholder = $this->placeholders((int) (count($releaseIds) / $placeholderGroups));
$sql = vsprintf($sql, array_fill(0, $placeholderGroups, $placeholder));
return $this->database->select($sql, $releaseIds);
}
/**
* @param list<string> $values
* @return array<string, list<object>>
*/
private function donors(string $sql, array $values): array
{
if ($values === []) {
return [];
}
$rows = $this->database->select(
sprintf($sql, $this->placeholders(count($values))),
$values
);
return $this->groupByKey($rows, 'match_key');
}
/**
* @param list<object> $rows
* @return array<int|string, list<object>>
*/
private function groupByKey(array $rows, string $key): array
{
$grouped = [];
foreach ($rows as $row) {
$grouped[$row->{$key}][] = $row;
}
return $grouped;
}
private function placeholders(int $count): string
{
return implode(',', array_fill(0, $count, '?'));
}
private function escapeLike(string $value): string
{
return str_replace(['\\', '%', '_'], ['\\\\', '\\%', '\\_'], $value);
}
}
app/Services/NameFixing/NameFixingService.php
+705 -715
View File
File diff suppressed because it is too large Load Diff
app/Services/NameFixing/ReleaseUpdateService.php
+63 -11
View File
@@ -7,13 +7,13 @@ namespace App\Services\NameFixing;
use App\Events\ReleaseNameFixed;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Predb;
use App\Models\Release;
use App\Models\UsenetGroup;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseCleaningService;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use InvalidArgumentException;
/**
* Service for updating releases with new names.
@@ -23,6 +23,20 @@ use Illuminate\Support\Facades\DB;
*/
class ReleaseUpdateService
{
/**
* @var list<string>
*/
private const SINGLE_UPDATE_COLUMNS = [
'predb_id',
'proc_nfo',
'proc_files',
'proc_par2',
'proc_uid',
'proc_hash16k',
'proc_srr',
'proc_crc32',
];
/**
* PreDB regex pattern for scene release names.
*/
@@ -347,10 +361,25 @@ class ReleaseUpdateService
*/
public function updateSingleColumn(string $column, int $status, int $id): void
{
if ($column !== '' && $id !== 0) {
Release::query()->where('id', $id)->update([$column => $status]);
Search::updateRelease($id);
if (! in_array($column, self::SINGLE_UPDATE_COLUMNS, true)) {
throw new InvalidArgumentException("Unsupported release status column [{$column}].");
}
if ($id !== 0) {
DB::update("UPDATE releases SET {$column} = ? WHERE id = ?", [$status, $id]);
}
}
public function attachPredbId(int $releaseId, int $predbId): void
{
if ($releaseId === 0 || $predbId === 0) {
return;
}
$this->updateSingleColumn('predb_id', $predbId, $releaseId);
$this->relid = $releaseId;
$this->matched = true;
$this->done = true;
}
/**
@@ -360,17 +389,40 @@ class ReleaseUpdateService
*/
public function checkPreDbMatch(object $release, string $textstring): ?array
{
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) {
foreach ($hits as $hit) {
foreach ($hit as $val) {
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
if ($title !== null) {
return ['title' => $title['title'], 'id' => $title['id']];
}
if (! preg_match_all(self::PREDB_REGEX, $textstring, $hits) || preg_match('/Source\s:/i', $textstring)) {
return null;
}
$candidates = [];
foreach ($hits as $hit) {
foreach ($hit as $value) {
$title = trim($value);
if ($title !== '') {
$candidates[$title] = $title;
}
}
}
if ($candidates === []) {
return null;
}
$candidateValues = array_values($candidates);
$rows = DB::select(
'SELECT id, title FROM predb WHERE title IN ('.implode(',', array_fill(0, count($candidateValues), '?')).')',
$candidateValues
);
$matches = [];
foreach ($rows as $row) {
$matches[(string) $row->title] = ['title' => (string) $row->title, 'id' => (int) $row->id];
}
foreach ($candidateValues as $candidate) {
if (isset($matches[$candidate])) {
return $matches[$candidate];
}
}
return null;
}
app/Services/Runners/ReleasesRunner.php
+2 -1
View File
@@ -144,10 +144,11 @@ class ReleasesRunner extends BaseRunner
$queues = [];
$idx = 0;
$workerCount = count($leftGuids);
foreach ($leftGuids as $leftGuid) {
if ($maxPerRun > 0) {
$idx++;
$queues[$idx] = sprintf('%s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx);
$queues[$idx] = sprintf('%s %s %s %s %s', $mode, $leftGuid, $maxPerRun, $idx, $workerCount);
}
}
app/Services/Tmux/Scripts/groupfixrelnames.php
+3 -3
View File
@@ -15,8 +15,8 @@ if (! isset($argv[1])) {
exit(1);
}
// Parse arguments: "type guidChar maxPerRun thread"
[$type, $guidChar, $maxPerRun, $thread] = explode(' ', $argv[1]);
// Parse arguments: "type guidChar maxPerRun worker workers"
[$type, $guidChar, $maxPerRun, $thread, $workers] = array_pad(explode(' ', $argv[1]), 5, '1');
// Build artisan command based on type
$artisan = dirname(__DIR__, 4).'/artisan';
@@ -37,7 +37,7 @@ switch ($type) {
exit(1);
}
$command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread}";
$command = "php {$artisan} releases:fix-names-group predbft --limit={$maxPerRun} --thread={$thread} --workers={$workers}";
break;
default: