From bd7f540c231ea52454238b0eeec89f672fdcb049 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Tue, 21 Apr 2026 17:35:52 +0200 Subject: [PATCH] Update categorization --- app/Events/ReleaseNameFixed.php | 23 ++ .../RecategorizeReleaseAfterNameFix.php | 60 +++++ app/Providers/AppServiceProvider.php | 3 + .../Categorization/CategorizationPipeline.php | 29 ++- .../Categorization/CategorizationResult.php | 24 ++ .../Categorizers/GroupNameCategorizer.php | 14 +- .../Categorizers/MiscCategorizer.php | 99 +++++++- .../Pipes/CategorizationPassable.php | 8 + .../Categorization/Pipes/MiscPipe.php | 1 + .../Pipes/MiscSafetyNetPipe.php | 66 ++++++ .../NameFixing/ReleaseUpdateService.php | 111 ++++----- app/Traits/DetectsHashedNames.php | 59 ++++- config/nntmux.php | 3 + .../ReleaseNameFixedRecategorizationTest.php | 211 ++++++++++++++++++ .../Unit/HashedReleaseCategorizationTest.php | 37 ++- 15 files changed, 685 insertions(+), 63 deletions(-) create mode 100644 app/Events/ReleaseNameFixed.php create mode 100644 app/Listeners/RecategorizeReleaseAfterNameFix.php create mode 100644 app/Services/Categorization/Pipes/MiscSafetyNetPipe.php create mode 100644 tests/Feature/ReleaseNameFixedRecategorizationTest.php diff --git a/app/Events/ReleaseNameFixed.php b/app/Events/ReleaseNameFixed.php new file mode 100644 index 000000000..682cf276a --- /dev/null +++ b/app/Events/ReleaseNameFixed.php @@ -0,0 +1,23 @@ +find($event->releaseId, [ + 'id', + 'groups_id', + 'fromname', + 'categories_id', + 'iscategorized', + 'searchname', + ]); + + if ($release === null) { + return; + } + + $result = $this->categorization->determineCategory( + $release->groups_id, + $event->newName, + (string) ($release->fromname ?? $event->poster) + ); + + $newCategoryId = (int) ($result['categories_id'] ?? $release->categories_id); + + if ((int) $release->categories_id === $newCategoryId && (int) $release->iscategorized === 1) { + return; + } + + Release::query() + ->where('id', $release->id) + ->update([ + 'categories_id' => $newCategoryId, + 'iscategorized' => 1, + ]); + + if (config('nntmux.categorization.log', false)) { + Log::info('categorization.rename_recategorized', [ + 'release_id' => $release->id, + 'old_name' => $event->oldName, + 'new_name' => $event->newName, + 'old_category_id' => $event->oldCategoryId, + 'new_category_id' => $newCategoryId, + ]); + } + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 98937cd73..37c071d7b 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -4,6 +4,8 @@ declare(strict_types=1); namespace App\Providers; +use App\Events\ReleaseNameFixed; +use App\Listeners\RecategorizeReleaseAfterNameFix; use App\Models\AnidbInfo; use App\Models\AnidbTitle; use App\Models\BookInfo; @@ -59,6 +61,7 @@ class AppServiceProvider extends ServiceProvider return $user->hasRole('Admin'); }); Event::listen(Login::class, LoginViaRemember::class); + Event::listen(ReleaseNameFixed::class, RecategorizeReleaseAfterNameFix::class); // Register observers RolePromotion::observe(RolePromotionObserver::class); diff --git a/app/Services/Categorization/CategorizationPipeline.php b/app/Services/Categorization/CategorizationPipeline.php index 7493fe8f3..01ccb7281 100644 --- a/app/Services/Categorization/CategorizationPipeline.php +++ b/app/Services/Categorization/CategorizationPipeline.php @@ -10,6 +10,7 @@ use App\Services\Categorization\Pipes\AbstractCategorizationPipe; use App\Services\Categorization\Pipes\CategorizationPassable; use Illuminate\Pipeline\Pipeline; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; /** * Pipeline-based categorization service using Laravel Pipeline. @@ -88,9 +89,34 @@ class CategorizationPipeline ->through($this->pipes->values()->all()) ->thenReturn(); + $this->logCategorization($result); + return $result->toArray(); } + protected function logCategorization(CategorizationPassable $result): void + { + if (! config('nntmux.categorization.log', false)) { + return; + } + + $payload = [ + 'release_name' => $result->context->releaseName, + 'group_name' => $result->context->groupName, + 'category_id' => $result->bestResult->categoryId, + 'matched_by' => $result->bestResult->matchedBy, + 'confidence' => $result->bestResult->confidence, + 'locked_to_misc' => $result->lockedToMisc, + 'misc_analysis' => $result->miscAnalysis, + ]; + + if ($result->lockedToMisc || $result->bestResult->matchedBy === 'group_only_low_signal') { + Log::info('categorization.decision', $payload); + } + + Log::debug('categorization.trace', $payload + ['all_results' => $result->allResults]); + } + /** * Get all registered categorizers (pipes). * @@ -107,6 +133,7 @@ class CategorizationPipeline public static function createDefault(): self { return new self([ + new Pipes\MiscPipe, new Pipes\GroupNamePipe, new Pipes\XxxPipe, new Pipes\TvPipe, @@ -115,7 +142,7 @@ class CategorizationPipeline new Pipes\MusicPipe, new Pipes\PcPipe, new Pipes\ConsolePipe, - new Pipes\MiscPipe, + new Pipes\MiscSafetyNetPipe, ]); } } diff --git a/app/Services/Categorization/CategorizationResult.php b/app/Services/Categorization/CategorizationResult.php index 4b59bd41e..79a259c6a 100644 --- a/app/Services/Categorization/CategorizationResult.php +++ b/app/Services/Categorization/CategorizationResult.php @@ -51,6 +51,11 @@ class CategorizationResult */ public function shouldOverride(CategorizationResult $other): bool { + if ($this->categoryId !== Category::OTHER_MISC && $this->categoryId !== Category::OTHER_HASHED && + $other->isProtectedMiscResult() && $this->confidence < 0.6) { + return false; + } + // Higher confidence always wins if ($this->confidence > $other->confidence) { return true; @@ -64,6 +69,25 @@ class CategorizationResult return false; } + /** + * Determine whether this result should resist weak downstream overrides. + */ + public function isProtectedMiscResult(): bool + { + if ($this->categoryId === Category::OTHER_HASHED) { + return true; + } + + if ($this->categoryId !== Category::OTHER_MISC) { + return false; + } + + return str_starts_with($this->matchedBy, 'hash_') + || str_starts_with($this->matchedBy, 'obfuscated_') + || str_starts_with($this->matchedBy, 'gibberish_') + || $this->matchedBy === 'group_only_low_signal'; + } + /** * Create a failed/empty result. */ diff --git a/app/Services/Categorization/Categorizers/GroupNameCategorizer.php b/app/Services/Categorization/Categorizers/GroupNameCategorizer.php index ff7e791d8..c3d041941 100644 --- a/app/Services/Categorization/Categorizers/GroupNameCategorizer.php +++ b/app/Services/Categorization/Categorizers/GroupNameCategorizer.php @@ -24,25 +24,25 @@ class GroupNameCategorizer extends AbstractCategorizer return $this->noMatch(); } if (preg_match('/alt\.binaries\..*?(tv|hdtv|tvseries)/i', $groupName)) { - return $this->matched(Category::TV_OTHER, 0.6, 'group_tv'); + return $this->matched(Category::TV_OTHER, 0.6, 'group_name_tv'); } if (preg_match('/alt\.binaries\..*?(movies?|dvd|bluray|x264)/i', $groupName)) { - return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_movie'); + return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_name_movie'); } if (preg_match('/alt\.binaries\..*?(erotica|pictures\.erotica|xxx)/i', $groupName)) { - return $this->matched(Category::XXX_OTHER, 0.7, 'group_xxx'); + return $this->matched(Category::XXX_OTHER, 0.7, 'group_name_xxx'); } if (preg_match('/alt\.binaries\..*?(sounds?|mp3|music|lossless)/i', $groupName)) { - return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_music'); + return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_name_music'); } if (preg_match('/alt\.binaries\..*?(games?|console|psx|nintendo)/i', $groupName)) { - return $this->matched(Category::GAME_OTHER, 0.6, 'group_game'); + return $this->matched(Category::GAME_OTHER, 0.6, 'group_name_game'); } if (preg_match('/alt\.binaries\..*?(warez|0day|apps?|software)/i', $groupName)) { - return $this->matched(Category::PC_0DAY, 0.6, 'group_pc'); + return $this->matched(Category::PC_0DAY, 0.6, 'group_name_pc'); } if (preg_match('/alt\.binaries\..*?(e-?book|ebook|comics?)/i', $groupName)) { - return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_book'); + return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_name_book'); } return $this->noMatch(); diff --git a/app/Services/Categorization/Categorizers/MiscCategorizer.php b/app/Services/Categorization/Categorizers/MiscCategorizer.php index 8df19cda4..2a0a0484a 100644 --- a/app/Services/Categorization/Categorizers/MiscCategorizer.php +++ b/app/Services/Categorization/Categorizers/MiscCategorizer.php @@ -34,11 +34,21 @@ class MiscCategorizer extends AbstractCategorizer return $result; } + $analysis = $this->inspectSignals($name); + if ($this->isZeroVowelLongToken($analysis['coreName'])) { + return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels'); + } + // Check for obfuscated/encoded patterns if ($result = $this->checkObfuscated($name)) { return $result; } + // Check low-signal names that only contain random-looking tokens + if ($result = $this->checkLowSignal($name)) { + return $result; + } + // Check for gibberish patterns (character-analysis heuristics) if ($result = $this->checkGibberish($name)) { return $result; @@ -57,6 +67,50 @@ class MiscCategorizer extends AbstractCategorizer return $this->noMatch(); } + /** + * Inspect a release name for media signal markers used by the safety-net pipe. + * + * @return array{coreName: string, coreLength: int, signalScore: int, markers: list, lowSignal: bool} + */ + public function inspectSignals(ReleaseContext|string $context): array + { + $name = $context instanceof ReleaseContext ? $context->releaseName : $context; + $cleaned = $this->stripExtensionsForAnalysis($name); + $coreName = $this->getCoreNameWithoutSeparators($cleaned); + + $patterns = [ + 'season_episode' => '/\bS\d{1,3}[._ -]?E\d{1,4}\b/i', + 'season_pack' => '/\bS\d{1,3}\b/i', + 'resolution' => '/\b(480p|576p|720p|1080[pi]?|2160p|4k|uhd)\b/i', + 'codec' => '/\b(x264|x265|h\.?264|h\.?265|hevc|xvid|av1)\b/i', + 'source' => '/\b(bluray|bdrip|brrip|hdtv|web[._ -]?dl|web[._ -]?rip|dvdrip|remux)\b/i', + 'audio' => '/\b(aac|ac3|ddp|dts|flac|mp3)\b/i', + 'scene_tag' => '/\b(proper|repack|internal|limited|complete|dubbed|subbed|readnfo)\b/i', + 'year' => '/\b(19|20)\d{2}\b/', + 'release_group' => '/-[A-Za-z0-9][A-Za-z0-9._-]{1,20}$/', + 'known_extension' => '/\.(mkv|avi|mp4|mp3|flac|iso|epub|pdf|exe|nzb|rar|7z)$/i', + ]; + + $markers = []; + foreach ($patterns as $marker => $pattern) { + if (preg_match($pattern, $name)) { + $markers[] = $marker; + } + } + + $signalScore = count($markers); + $isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1; + $lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12; + + return [ + 'coreName' => $coreName, + 'coreLength' => strlen($coreName), + 'signalScore' => $signalScore, + 'markers' => $markers, + 'lowSignal' => $lowSignal, + ]; + } + protected function checkHash(string $name): ?CategorizationResult { // MD5 hash (32 hex characters) @@ -79,6 +133,10 @@ class MiscCategorizer extends AbstractCategorizer return $this->matched(Category::OTHER_HASHED, 0.95, 'hash_generic'); } + if ($this->isBase64LikeToken($name)) { + return $this->matched(Category::OTHER_HASHED, 0.9, 'hash_base64_like'); + } + // Strip extensions and separators for core-name checks $cleaned = $this->stripExtensionsForAnalysis($name); $coreName = $this->getCoreNameWithoutSeparators($cleaned); @@ -115,7 +173,18 @@ class MiscCategorizer extends AbstractCategorizer // Only punctuation and numbers with no clear structure if ($this->isObfuscatedPunctuation($name)) { - return $this->matched(Category::OTHER_MISC, 0.5, 'obfuscated_pattern'); + $analysis = $this->inspectSignals($name); + $hashLike = $this->isBase64LikeToken($name) + || $this->isBoundedGenericHash($name) + || $this->isZeroVowelLongToken($analysis['coreName'], 12) + || $analysis['lowSignal']; + + return $this->matched( + $hashLike ? Category::OTHER_HASHED : Category::OTHER_MISC, + $hashLike ? 0.75 : 0.5, + 'obfuscated_pattern', + ['signal_score' => $analysis['signalScore'], 'markers' => $analysis['markers']] + ); } return null; @@ -146,6 +215,34 @@ class MiscCategorizer extends AbstractCategorizer return $this->matched(Category::OTHER_HASHED, 0.7, 'gibberish_random_digits'); } + if ($this->isZeroVowelLongToken($coreName)) { + return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels'); + } + + return null; + } + + protected function checkLowSignal(string $name): ?CategorizationResult + { + $analysis = $this->inspectSignals($name); + + if ($this->isZeroVowelLongToken($analysis['coreName'])) { + return null; + } + + if ($analysis['lowSignal'] && $analysis['coreLength'] >= 20) { + return $this->matched( + Category::OTHER_HASHED, + 0.8, + 'gibberish_no_signal', + [ + 'signal_score' => $analysis['signalScore'], + 'markers' => $analysis['markers'], + 'core_length' => $analysis['coreLength'], + ] + ); + } + return null; } diff --git a/app/Services/Categorization/Pipes/CategorizationPassable.php b/app/Services/Categorization/Pipes/CategorizationPassable.php index 17b69b180..73a86a4ec 100644 --- a/app/Services/Categorization/Pipes/CategorizationPassable.php +++ b/app/Services/Categorization/Pipes/CategorizationPassable.php @@ -29,6 +29,13 @@ class CategorizationPassable */ public array $allResults = []; + /** + * Misc signal analysis captured by MiscPipe for downstream safety-net decisions. + * + * @var array + */ + public array $miscAnalysis = []; + public function __construct(ReleaseContext $context, bool $debug = false) { $this->context = $context; @@ -90,6 +97,7 @@ class CategorizationPassable 'locked_to_misc' => $this->lockedToMisc, 'release_name' => $this->context->releaseName, 'group_name' => $this->context->groupName, + 'misc_analysis' => $this->miscAnalysis, 'all_results' => $this->allResults, 'categorizer_details' => $this->bestResult->debug, ]; diff --git a/app/Services/Categorization/Pipes/MiscPipe.php b/app/Services/Categorization/Pipes/MiscPipe.php index 7cd9b442d..220d7e538 100644 --- a/app/Services/Categorization/Pipes/MiscPipe.php +++ b/app/Services/Categorization/Pipes/MiscPipe.php @@ -50,6 +50,7 @@ class MiscPipe extends AbstractCategorizationPipe public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable { // MiscPipe always runs regardless of shouldStopProcessing — it IS the first pipe. + $passable->miscAnalysis = $this->categorizer->inspectSignals($passable->context); $result = $this->categorize($passable->context); // Record the result for debug diff --git a/app/Services/Categorization/Pipes/MiscSafetyNetPipe.php b/app/Services/Categorization/Pipes/MiscSafetyNetPipe.php new file mode 100644 index 000000000..5dbc0b3f8 --- /dev/null +++ b/app/Services/Categorization/Pipes/MiscSafetyNetPipe.php @@ -0,0 +1,66 @@ +shouldDowngradeToMisc($passable)) { + $result = new CategorizationResult( + Category::OTHER_MISC, + 0.65, + 'group_only_low_signal', + [ + 'group_match' => $passable->bestResult->matchedBy, + 'misc_analysis' => $passable->miscAnalysis, + ] + ); + + if ($passable->debug) { + $passable->allResults[$this->getName()] = [ + 'category_id' => $result->categoryId, + 'confidence' => $result->confidence, + 'matched_by' => $result->matchedBy, + ]; + } + + $passable->bestResult = $result; + $passable->lockToMisc(); + } + + return $next($passable); + } + + protected function categorize(ReleaseContext $context): CategorizationResult + { + return $this->noMatch(); + } + + private function shouldDowngradeToMisc(CategorizationPassable $passable): bool + { + return ! $passable->lockedToMisc + && ($passable->miscAnalysis['lowSignal'] ?? false) === true + && str_starts_with($passable->bestResult->matchedBy, 'group_name_'); + } +} diff --git a/app/Services/NameFixing/ReleaseUpdateService.php b/app/Services/NameFixing/ReleaseUpdateService.php index e7043db83..00a70f34b 100644 --- a/app/Services/NameFixing/ReleaseUpdateService.php +++ b/app/Services/NameFixing/ReleaseUpdateService.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace App\Services\NameFixing; +use App\Events\ReleaseNameFixed; use App\Facades\Search; use App\Models\Category; use App\Models\Predb; @@ -12,6 +13,7 @@ use App\Models\UsenetGroup; use App\Services\Categorization\CategorizationService; use App\Services\ReleaseCleaningService; use Illuminate\Support\Arr; +use Illuminate\Support\Facades\DB; /** * Service for updating releases with new names. @@ -153,12 +155,6 @@ class ReleaseUpdateService $this->matched = true; $this->relid = (int) $release->releases_id; - $determinedCategory = $this->category->determineCategory( - $release->groups_id, - $newName, - ! empty($release->fromname) ? $release->fromname : '' - ); - if ($type === 'PAR2, ') { $newName = ucwords($newName); if (preg_match('/(.+?)\.[a-z0-9]{2,3}(PAR2)?$/i', $name, $hit)) { @@ -172,14 +168,21 @@ class ReleaseUpdateService $newName = explode('\\', $newName); $newName = preg_replace(['/^[=_.:\s-]+/', '/[=_.:\s-]+$/'], '', $newName[0]); - if ($this->echoOutput && $show) { - $this->echoReleaseInfo($release, $newName, $determinedCategory, $type, $method); - } - $newTitle = substr($newName, 0, 299); + $determinedCategory = null; + if ($this->echoOutput && $show) { + $determinedCategory = $this->category->determineCategory( + $release->groups_id, + $newTitle, + ! empty($release->fromname) ? $release->fromname : '' + ); + + $this->echoReleaseInfo($release, $newTitle, $determinedCategory, $type, $method); + } + if ($echo === true) { - $this->performDatabaseUpdate($release, $newTitle, $determinedCategory, $type, $nameStatus, $preId); + $this->performDatabaseUpdate($release, $newTitle, $type, $nameStatus, $preId); } } } @@ -253,59 +256,65 @@ class ReleaseUpdateService /** * Perform the actual database update. - * - * @param array $determinedCategory */ protected function performDatabaseUpdate( object $release, string $newTitle, - array $determinedCategory, string $type, bool $nameStatus, int $preId ): void { - if ($nameStatus === true) { - $status = $this->getStatusColumnsForType($type); + DB::transaction(function () use ($release, $newTitle, $type, $nameStatus, $preId): void { + if ($nameStatus === true) { + $status = $this->getStatusColumnsForType($type); - $updateColumns = [ - 'videos_id' => 0, - 'tv_episodes_id' => 0, - 'imdbid' => null, - 'musicinfo_id' => '', - 'consoleinfo_id' => '', - 'bookinfo_id' => '', - 'anidbid' => '', - 'predb_id' => $preId, - 'searchname' => $newTitle, - 'categories_id' => $determinedCategory['categories_id'], - ]; - - if (! empty($status)) { - foreach ($status as $key => $stat) { - $updateColumns = Arr::add($updateColumns, $key, $stat); - } - } - - Release::query() - ->where('id', $release->releases_id) - ->update($updateColumns); - } else { - Release::query() - ->where('id', $release->releases_id) - ->update([ + $updateColumns = [ 'videos_id' => 0, 'tv_episodes_id' => 0, 'imdbid' => null, - 'musicinfo_id' => null, - 'consoleinfo_id' => null, - 'bookinfo_id' => null, - 'anidbid' => null, + 'musicinfo_id' => '', + 'consoleinfo_id' => '', + 'bookinfo_id' => '', + 'anidbid' => '', 'predb_id' => $preId, 'searchname' => $newTitle, - 'categories_id' => $determinedCategory['categories_id'], - 'iscategorized' => 1, - ]); - } + ]; + + if (! empty($status)) { + foreach ($status as $key => $stat) { + $updateColumns = Arr::add($updateColumns, $key, $stat); + } + } + + Release::query() + ->where('id', $release->releases_id) + ->update($updateColumns); + } else { + Release::query() + ->where('id', $release->releases_id) + ->update([ + 'videos_id' => 0, + 'tv_episodes_id' => 0, + 'imdbid' => null, + 'musicinfo_id' => null, + 'consoleinfo_id' => null, + 'bookinfo_id' => null, + 'anidbid' => null, + 'predb_id' => $preId, + 'searchname' => $newTitle, + 'iscategorized' => 1, + ]); + } + + event(new ReleaseNameFixed( + (int) $release->releases_id, + (string) $release->searchname, + $newTitle, + (int) $release->categories_id, + $release->groups_id, + (string) ($release->fromname ?? '') + )); + }); // Update search index Search::updateRelease($release->releases_id); @@ -350,7 +359,7 @@ class ReleaseUpdateService */ public function checkPreDbMatch(object $release, string $textstring): ?array { - if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s\:/i', $textstring)) { + if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) { foreach ($hits as $hit) { foreach ($hit as $val) { $title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first(); diff --git a/app/Traits/DetectsHashedNames.php b/app/Traits/DetectsHashedNames.php index f06f38a3a..6431a7b75 100644 --- a/app/Traits/DetectsHashedNames.php +++ b/app/Traits/DetectsHashedNames.php @@ -29,11 +29,13 @@ trait DetectsHashedNames || $this->isBoundedSha1Hash($name) || $this->isBoundedSha256Hash($name) || $this->isBoundedGenericHash($name) + || $this->isBase64LikeToken($name) || $this->isObfuscatedUppercaseString($name) || $this->isObfuscatedMixedAlphanumeric($name) || $this->isObfuscatedUsenetFilename($name) || $this->isRandomByCharacterAnalysis($coreName, $name) || $this->hasInsufficientWordStructure($coreName) + || $this->isZeroVowelLongToken($coreName) || $this->isRandomDigitPattern($coreName); } @@ -70,7 +72,40 @@ trait DetectsHashedNames */ protected function isBoundedGenericHash(string $name): bool { - return (bool) preg_match('/(?:^|["\'\s\[\]\/\-])([a-f0-9]{32,128})(?:["\'\s\[\]\/.\-]|$)/i', $name); + if (! preg_match('/(?:^|["\'\s\[\]\/\-])([a-z0-9]{24,128})(?:["\'\s\[\]\/.\-]|$)/i', $name, $matches)) { + return false; + } + + $token = $matches[1]; + + if (preg_match('/\b(19|20)\d{2}\b/', $token)) { + return false; + } + + return preg_match('/\d/', $token) === 1 + && $this->looksLikeRandomString($token); + } + + /** + * Detect base64/base64url-like tokens that commonly appear in obfuscated subjects. + */ + protected function isBase64LikeToken(string $name): bool + { + if (! preg_match('/(?:^|["\'\s\[\]\/\-])([A-Za-z0-9+\/_=-]{24,})(?:["\'\s\[\]\/.\-]|$)/', $name, $matches)) { + return false; + } + + $token = rtrim($matches[1], '='); + + if (strlen($token) < 24 || preg_match('/\b(19|20)\d{2}\b/', $token)) { + return false; + } + + if (! preg_match('/[A-Z]/', $token) || ! preg_match('/[a-z]/', $token) || ! preg_match('/\d/', $token)) { + return false; + } + + return $this->looksLikeRandomString($token); } // --------------------------------------------------------------- @@ -207,6 +242,28 @@ trait DetectsHashedNames ); } + /** + * Detect long alphanumeric tokens with effectively no vowel structure. + */ + protected function isZeroVowelLongToken(string $coreName, int $minLength = 20): bool + { + if (strlen($coreName) < $minLength || ! preg_match('/^[a-zA-Z0-9]+$/', $coreName)) { + return false; + } + + if (preg_match('/\b(19|20)\d{2}\b/', $coreName)) { + return false; + } + + $letterOnly = preg_replace('/\d+/', '', $coreName); + + if ($letterOnly === '' || strlen($letterOnly) < 10) { + return false; + } + + return preg_match('/[aeiou]/i', $letterOnly) !== 1; + } + /** * Check if a string looks like a random/obfuscated string rather than a real title. * diff --git a/config/nntmux.php b/config/nntmux.php index e7c4e0b82..18b187edb 100644 --- a/config/nntmux.php +++ b/config/nntmux.php @@ -28,4 +28,7 @@ return [ 'nzb_upload_folder' => env('NZB_UPLOAD_FOLDER'), 'redis_fast_degrade' => (bool) env('REDIS_FAST_DEGRADE', true), 'redis_tcp_check_seconds' => (float) env('REDIS_TCP_CHECK_SECONDS', 0.2), + 'categorization' => [ + 'log' => (bool) env('NNTMUX_CATEGORIZATION_LOG', false), + ], ]; diff --git a/tests/Feature/ReleaseNameFixedRecategorizationTest.php b/tests/Feature/ReleaseNameFixedRecategorizationTest.php new file mode 100644 index 000000000..d7560ffa7 --- /dev/null +++ b/tests/Feature/ReleaseNameFixedRecategorizationTest.php @@ -0,0 +1,211 @@ + + */ + private array $originalEnvironment = []; + + public function createApplication() + { + $this->databasePath = sys_get_temp_dir().'/nntmux-release-name-fixed-test.sqlite'; + + $this->originalEnvironment = [ + 'APP_ENV' => getenv('APP_ENV'), + 'DB_CONNECTION' => getenv('DB_CONNECTION'), + 'DB_DATABASE' => getenv('DB_DATABASE'), + ]; + + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings (name, value) VALUES ('categorizeforeign', '0'), ('catwebdl', '1')"); + + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + DB::table('settings')->upsert([ + ['name' => 'categorizeforeign', 'value' => '0'], + ['name' => 'catwebdl', 'value' => '1'], + ], ['name'], ['value']); + + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $this->databasePath, + ]); + + DB::purge(); + DB::reconnect(); + + $this->createSchema(); + } + + protected function tearDown(): void + { + if ($this->databasePath !== '' && file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + parent::tearDown(); + + foreach ($this->originalEnvironment as $key => $value) { + $this->setEnvironmentValue($key, $value === false ? null : $value); + } + } + + public function test_renaming_hashed_release_recategorizes_it_synchronously(): void + { + Search::shouldReceive('updateRelease')->twice(); + + $group = UsenetGroup::query()->create([ + 'name' => 'alt.binaries.hdtv', + 'active' => 1, + 'backfill' => 0, + ]); + + $release = Release::factory()->create([ + 'name' => 'd41d8cd98f00b204e9800998ecf8427e', + 'searchname' => 'd41d8cd98f00b204e9800998ecf8427e', + 'fromname' => 'poster@example.com', + 'groups_id' => $group->id, + 'categories_id' => Category::OTHER_HASHED, + 'iscategorized' => 1, + 'isrenamed' => 0, + 'guid' => str_repeat('a', 40), + 'leftguid' => 'a', + 'nzb_guid' => 'test', + 'size' => 1, + 'postdate' => now(), + 'adddate' => now(), + ]); + + $service = app(ReleaseUpdateService::class); + $service->updateRelease( + $release->fresh(), + 'Show.Name.S03E05.720p.HDTV.x264-GROUP', + 'nfoCheck: Title Match', + true, + 'NFO, ', + true, + false, + ); + + $release->refresh(); + + $this->assertSame('Show.Name.S03E05.720p.HDTV.x264-GROUP', $release->searchname); + $this->assertSame(Category::TV_HD, $release->categories_id); + $this->assertSame(1, (int) $release->iscategorized); + $this->assertSame(1, (int) $release->isrenamed); + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv($key.'='.$value); + $_ENV[$key] = $value; + $_SERVER[$key] = $value; + } + + private function createSchema(): void + { + if (! Schema::hasTable('settings')) { + Schema::create('settings', function (Blueprint $table): void { + $table->string('name')->primary(); + $table->text('value')->nullable(); + }); + } + + if (! Schema::hasTable('usenet_groups')) { + Schema::create('usenet_groups', function (Blueprint $table): void { + $table->increments('id'); + $table->string('name')->unique(); + $table->integer('backfill_target')->default(1); + $table->unsignedBigInteger('first_record')->default(0); + $table->dateTime('first_record_postdate')->nullable(); + $table->unsignedBigInteger('last_record')->default(0); + $table->dateTime('last_record_postdate')->nullable(); + $table->dateTime('last_updated')->nullable(); + $table->integer('minfilestoformrelease')->nullable(); + $table->bigInteger('minsizetoformrelease')->nullable(); + $table->boolean('active')->default(false); + $table->boolean('backfill')->default(false); + $table->string('description')->nullable(); + }); + } + + if (! Schema::hasTable('releases')) { + Schema::create('releases', function (Blueprint $table): void { + $table->increments('id'); + $table->string('name')->default(''); + $table->string('searchname')->default(''); + $table->unsignedInteger('groups_id')->default(0); + $table->unsignedBigInteger('size')->default(0); + $table->dateTime('postdate')->nullable(); + $table->dateTime('adddate')->nullable(); + $table->string('guid', 40); + $table->char('leftguid', 1); + $table->string('fromname')->nullable(); + $table->integer('categories_id')->default(Category::OTHER_MISC); + $table->unsignedInteger('videos_id')->default(0); + $table->integer('tv_episodes_id')->default(0); + $table->string('imdbid')->nullable(); + $table->integer('musicinfo_id')->nullable(); + $table->integer('consoleinfo_id')->nullable(); + $table->integer('bookinfo_id')->nullable(); + $table->integer('anidbid')->nullable(); + $table->unsignedInteger('predb_id')->default(0); + $table->tinyInteger('iscategorized')->default(0); + $table->tinyInteger('isrenamed')->default(0); + $table->tinyInteger('proc_nfo')->default(0); + $table->tinyInteger('proc_files')->default(0); + $table->tinyInteger('proc_par2')->default(0); + $table->tinyInteger('proc_uid')->default(0); + $table->tinyInteger('proc_hash16k')->default(0); + $table->tinyInteger('proc_srr')->default(0); + $table->tinyInteger('proc_crc32')->default(0); + $table->tinyInteger('passwordstatus')->default(0); + $table->tinyInteger('nzbstatus')->default(0); + $table->binary('nzb_guid')->nullable(); + }); + } + } +} diff --git a/tests/Unit/HashedReleaseCategorizationTest.php b/tests/Unit/HashedReleaseCategorizationTest.php index 8a4e60bc8..0f187ef2d 100644 --- a/tests/Unit/HashedReleaseCategorizationTest.php +++ b/tests/Unit/HashedReleaseCategorizationTest.php @@ -7,10 +7,12 @@ namespace Tests\Unit; use App\Models\Category; use App\Services\Categorization\CategorizationResult; use App\Services\Categorization\Categorizers\MiscCategorizer; +use App\Services\Categorization\Pipes\BookPipe; use App\Services\Categorization\Pipes\CategorizationPassable; use App\Services\Categorization\Pipes\ConsolePipe; use App\Services\Categorization\Pipes\GroupNamePipe; use App\Services\Categorization\Pipes\MiscPipe; +use App\Services\Categorization\Pipes\MiscSafetyNetPipe; use App\Services\Categorization\Pipes\MoviePipe; use App\Services\Categorization\Pipes\MusicPipe; use App\Services\Categorization\Pipes\PcPipe; @@ -25,7 +27,7 @@ class HashedReleaseCategorizationTest extends TestCase /** * The ordered list of pipes that matches CategorizationPipeline::createDefault(). * Sorted by priority: MiscPipe(1), GroupNamePipe(5), XxxPipe(10), TvPipe(20), - * MoviePipe(25), BookPipe, MusicPipe, PcPipe, ConsolePipe. + * MoviePipe(25), BookPipe, MusicPipe, PcPipe, ConsolePipe, MiscSafetyNetPipe. * * @return list */ @@ -37,9 +39,11 @@ class HashedReleaseCategorizationTest extends TestCase new XxxPipe, new TvPipe, new MoviePipe, + new BookPipe, new MusicPipe, new PcPipe, new ConsolePipe, + new MiscSafetyNetPipe, ]; } @@ -87,6 +91,7 @@ class HashedReleaseCategorizationTest extends TestCase 'All uppercase 20 chars' => ['ABCDEFGH1234567890XY', 'obfuscated_uppercase'], 'Mixed alphanumeric random' => ['AA7Jl2toE8Q53yNZmQ5R6G', 'obfuscated_mixed_alphanumeric'], 'Usenet obfuscated filename' => ['[01/10] - "xK9mR2pL4qW7nT3vB.part01.rar"', 'obfuscated_usenet_filename'], + 'Base64-like token' => ['VGhpc0lzTm90QVNob3dOYW1lMTIzNDU2Nzg5MA==', 'hash_base64_like'], ]; } @@ -106,10 +111,12 @@ class HashedReleaseCategorizationTest extends TestCase // Lowercase with dots and digits — bypasses all obfuscated checks. // After stripping, coreName ≥20 with maxConsecutiveLetters < 5 but low transition rate. // Grouped letter/digit runs keep transition rate ≤ 0.35. - 'No word structure long' => ['xyz.1234.wvut.5678.srqp.9012', 'gibberish_no_word_structure'], + 'No word structure long' => ['xyz.1234.wvut.5678.srqp.9012.rar', 'gibberish_no_word_structure'], // Lowercase with dots and digits — bypasses all obfuscated checks. // After stripping, coreName matches digit-heavy pattern (1-3 letters + 6+ digits). 'Digit-heavy pattern' => ['xz.123456789012', 'gibberish_random_digits'], + 'Zero vowel core' => ['xkcdqwrtypsdfghjklmnbvcxz', 'gibberish_zero_vowels'], + 'No signal long token' => ['A1b2.C3d4.E5f6.G7h8.I9j0', 'gibberish_no_signal'], ]; } @@ -144,6 +151,22 @@ class HashedReleaseCategorizationTest extends TestCase 'Gibberish in games group' => ['aB3cD4eF5gH6iJ7kL8m', 'alt.binaries.games'], 'Hex in warez group' => ['aabbccdd0011223344ff', 'alt.binaries.warez'], 'Random digits in ebook group' => ['ab12345678901234', 'alt.binaries.e-book'], + 'Base64-like token in TV group' => ['VGhpc0lzTm90QVNob3dOYW1lMTIzNDU2Nzg5MA==', 'alt.binaries.hdtv'], + 'Zero vowel token in movie group' => ['xkcdqwrtypsdfghjklmnbvcxz', 'alt.binaries.movies'], + ]; + } + + /** + * Low-signal names that are not strong enough for an early lock but should + * still be kept out of content categories when only the group matched. + * + * @return array + */ + public static function lowSignalGroupOnlyProvider(): array + { + return [ + 'Short low-signal token in TV group' => ['ab12.cd34.ef56', 'alt.binaries.hdtv'], + 'Short low-signal token in movies group' => ['zx90.cv78.bn56', 'alt.binaries.movies'], ]; } @@ -237,6 +260,16 @@ class HashedReleaseCategorizationTest extends TestCase ); } + #[DataProvider('lowSignalGroupOnlyProvider')] + public function test_group_only_low_signal_releases_fall_back_to_other_misc(string $name, string $groupName): void + { + $passable = $this->runPipeline($name, $groupName); + + $this->assertTrue($passable->lockedToMisc, "Expected lockedToMisc for low-signal release: $name"); + $this->assertSame(Category::OTHER_MISC, $passable->bestResult->categoryId); + $this->assertSame('group_only_low_signal', $passable->bestResult->matchedBy); + } + // ------------------------------------------------------------------ // Tests: Legitimate releases still categorize normally // ------------------------------------------------------------------