Update categorization

This commit is contained in:
DariusIII
2026-04-21 17:35:52 +02:00
parent f2c06224ee
commit bd7f540c23
15 changed files with 685 additions and 63 deletions
+23
View File
@@ -0,0 +1,23 @@
<?php
declare(strict_types=1);
namespace App\Events;
use Illuminate\Foundation\Events\Dispatchable;
use Illuminate\Queue\SerializesModels;
class ReleaseNameFixed
{
use Dispatchable;
use SerializesModels;
public function __construct(
public readonly int $releaseId,
public readonly string $oldName,
public readonly string $newName,
public readonly int $oldCategoryId,
public readonly int|string $groupId,
public readonly string $poster = '',
) {}
}
@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
namespace App\Listeners;
use App\Events\ReleaseNameFixed;
use App\Models\Release;
use App\Services\Categorization\CategorizationService;
use Illuminate\Support\Facades\Log;
class RecategorizeReleaseAfterNameFix
{
public function __construct(private readonly CategorizationService $categorization) {}
public function handle(ReleaseNameFixed $event): void
{
$release = Release::query()->find($event->releaseId, [
'id',
'groups_id',
'fromname',
'categories_id',
'iscategorized',
'searchname',
]);
if ($release === null) {
return;
}
$result = $this->categorization->determineCategory(
$release->groups_id,
$event->newName,
(string) ($release->fromname ?? $event->poster)
);
$newCategoryId = (int) ($result['categories_id'] ?? $release->categories_id);
if ((int) $release->categories_id === $newCategoryId && (int) $release->iscategorized === 1) {
return;
}
Release::query()
->where('id', $release->id)
->update([
'categories_id' => $newCategoryId,
'iscategorized' => 1,
]);
if (config('nntmux.categorization.log', false)) {
Log::info('categorization.rename_recategorized', [
'release_id' => $release->id,
'old_name' => $event->oldName,
'new_name' => $event->newName,
'old_category_id' => $event->oldCategoryId,
'new_category_id' => $newCategoryId,
]);
}
}
}
+3
View File
@@ -4,6 +4,8 @@ declare(strict_types=1);
namespace App\Providers;
use App\Events\ReleaseNameFixed;
use App\Listeners\RecategorizeReleaseAfterNameFix;
use App\Models\AnidbInfo;
use App\Models\AnidbTitle;
use App\Models\BookInfo;
@@ -59,6 +61,7 @@ class AppServiceProvider extends ServiceProvider
return $user->hasRole('Admin');
});
Event::listen(Login::class, LoginViaRemember::class);
Event::listen(ReleaseNameFixed::class, RecategorizeReleaseAfterNameFix::class);
// Register observers
RolePromotion::observe(RolePromotionObserver::class);
@@ -10,6 +10,7 @@ use App\Services\Categorization\Pipes\AbstractCategorizationPipe;
use App\Services\Categorization\Pipes\CategorizationPassable;
use Illuminate\Pipeline\Pipeline;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
/**
* Pipeline-based categorization service using Laravel Pipeline.
@@ -88,9 +89,34 @@ class CategorizationPipeline
->through($this->pipes->values()->all())
->thenReturn();
$this->logCategorization($result);
return $result->toArray();
}
protected function logCategorization(CategorizationPassable $result): void
{
if (! config('nntmux.categorization.log', false)) {
return;
}
$payload = [
'release_name' => $result->context->releaseName,
'group_name' => $result->context->groupName,
'category_id' => $result->bestResult->categoryId,
'matched_by' => $result->bestResult->matchedBy,
'confidence' => $result->bestResult->confidence,
'locked_to_misc' => $result->lockedToMisc,
'misc_analysis' => $result->miscAnalysis,
];
if ($result->lockedToMisc || $result->bestResult->matchedBy === 'group_only_low_signal') {
Log::info('categorization.decision', $payload);
}
Log::debug('categorization.trace', $payload + ['all_results' => $result->allResults]);
}
/**
* Get all registered categorizers (pipes).
*
@@ -107,6 +133,7 @@ class CategorizationPipeline
public static function createDefault(): self
{
return new self([
new Pipes\MiscPipe,
new Pipes\GroupNamePipe,
new Pipes\XxxPipe,
new Pipes\TvPipe,
@@ -115,7 +142,7 @@ class CategorizationPipeline
new Pipes\MusicPipe,
new Pipes\PcPipe,
new Pipes\ConsolePipe,
new Pipes\MiscPipe,
new Pipes\MiscSafetyNetPipe,
]);
}
}
@@ -51,6 +51,11 @@ class CategorizationResult
*/
public function shouldOverride(CategorizationResult $other): bool
{
if ($this->categoryId !== Category::OTHER_MISC && $this->categoryId !== Category::OTHER_HASHED &&
$other->isProtectedMiscResult() && $this->confidence < 0.6) {
return false;
}
// Higher confidence always wins
if ($this->confidence > $other->confidence) {
return true;
@@ -64,6 +69,25 @@ class CategorizationResult
return false;
}
/**
* Determine whether this result should resist weak downstream overrides.
*/
public function isProtectedMiscResult(): bool
{
if ($this->categoryId === Category::OTHER_HASHED) {
return true;
}
if ($this->categoryId !== Category::OTHER_MISC) {
return false;
}
return str_starts_with($this->matchedBy, 'hash_')
|| str_starts_with($this->matchedBy, 'obfuscated_')
|| str_starts_with($this->matchedBy, 'gibberish_')
|| $this->matchedBy === 'group_only_low_signal';
}
/**
* Create a failed/empty result.
*/
@@ -24,25 +24,25 @@ class GroupNameCategorizer extends AbstractCategorizer
return $this->noMatch();
}
if (preg_match('/alt\.binaries\..*?(tv|hdtv|tvseries)/i', $groupName)) {
return $this->matched(Category::TV_OTHER, 0.6, 'group_tv');
return $this->matched(Category::TV_OTHER, 0.6, 'group_name_tv');
}
if (preg_match('/alt\.binaries\..*?(movies?|dvd|bluray|x264)/i', $groupName)) {
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_movie');
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_name_movie');
}
if (preg_match('/alt\.binaries\..*?(erotica|pictures\.erotica|xxx)/i', $groupName)) {
return $this->matched(Category::XXX_OTHER, 0.7, 'group_xxx');
return $this->matched(Category::XXX_OTHER, 0.7, 'group_name_xxx');
}
if (preg_match('/alt\.binaries\..*?(sounds?|mp3|music|lossless)/i', $groupName)) {
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_music');
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_name_music');
}
if (preg_match('/alt\.binaries\..*?(games?|console|psx|nintendo)/i', $groupName)) {
return $this->matched(Category::GAME_OTHER, 0.6, 'group_game');
return $this->matched(Category::GAME_OTHER, 0.6, 'group_name_game');
}
if (preg_match('/alt\.binaries\..*?(warez|0day|apps?|software)/i', $groupName)) {
return $this->matched(Category::PC_0DAY, 0.6, 'group_pc');
return $this->matched(Category::PC_0DAY, 0.6, 'group_name_pc');
}
if (preg_match('/alt\.binaries\..*?(e-?book|ebook|comics?)/i', $groupName)) {
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_book');
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_name_book');
}
return $this->noMatch();
@@ -34,11 +34,21 @@ class MiscCategorizer extends AbstractCategorizer
return $result;
}
$analysis = $this->inspectSignals($name);
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
}
// Check for obfuscated/encoded patterns
if ($result = $this->checkObfuscated($name)) {
return $result;
}
// Check low-signal names that only contain random-looking tokens
if ($result = $this->checkLowSignal($name)) {
return $result;
}
// Check for gibberish patterns (character-analysis heuristics)
if ($result = $this->checkGibberish($name)) {
return $result;
@@ -57,6 +67,50 @@ class MiscCategorizer extends AbstractCategorizer
return $this->noMatch();
}
/**
* Inspect a release name for media signal markers used by the safety-net pipe.
*
* @return array{coreName: string, coreLength: int, signalScore: int, markers: list<string>, lowSignal: bool}
*/
public function inspectSignals(ReleaseContext|string $context): array
{
$name = $context instanceof ReleaseContext ? $context->releaseName : $context;
$cleaned = $this->stripExtensionsForAnalysis($name);
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
$patterns = [
'season_episode' => '/\bS\d{1,3}[._ -]?E\d{1,4}\b/i',
'season_pack' => '/\bS\d{1,3}\b/i',
'resolution' => '/\b(480p|576p|720p|1080[pi]?|2160p|4k|uhd)\b/i',
'codec' => '/\b(x264|x265|h\.?264|h\.?265|hevc|xvid|av1)\b/i',
'source' => '/\b(bluray|bdrip|brrip|hdtv|web[._ -]?dl|web[._ -]?rip|dvdrip|remux)\b/i',
'audio' => '/\b(aac|ac3|ddp|dts|flac|mp3)\b/i',
'scene_tag' => '/\b(proper|repack|internal|limited|complete|dubbed|subbed|readnfo)\b/i',
'year' => '/\b(19|20)\d{2}\b/',
'release_group' => '/-[A-Za-z0-9][A-Za-z0-9._-]{1,20}$/',
'known_extension' => '/\.(mkv|avi|mp4|mp3|flac|iso|epub|pdf|exe|nzb|rar|7z)$/i',
];
$markers = [];
foreach ($patterns as $marker => $pattern) {
if (preg_match($pattern, $name)) {
$markers[] = $marker;
}
}
$signalScore = count($markers);
$isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
$lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12;
return [
'coreName' => $coreName,
'coreLength' => strlen($coreName),
'signalScore' => $signalScore,
'markers' => $markers,
'lowSignal' => $lowSignal,
];
}
protected function checkHash(string $name): ?CategorizationResult
{
// MD5 hash (32 hex characters)
@@ -79,6 +133,10 @@ class MiscCategorizer extends AbstractCategorizer
return $this->matched(Category::OTHER_HASHED, 0.95, 'hash_generic');
}
if ($this->isBase64LikeToken($name)) {
return $this->matched(Category::OTHER_HASHED, 0.9, 'hash_base64_like');
}
// Strip extensions and separators for core-name checks
$cleaned = $this->stripExtensionsForAnalysis($name);
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
@@ -115,7 +173,18 @@ class MiscCategorizer extends AbstractCategorizer
// Only punctuation and numbers with no clear structure
if ($this->isObfuscatedPunctuation($name)) {
return $this->matched(Category::OTHER_MISC, 0.5, 'obfuscated_pattern');
$analysis = $this->inspectSignals($name);
$hashLike = $this->isBase64LikeToken($name)
|| $this->isBoundedGenericHash($name)
|| $this->isZeroVowelLongToken($analysis['coreName'], 12)
|| $analysis['lowSignal'];
return $this->matched(
$hashLike ? Category::OTHER_HASHED : Category::OTHER_MISC,
$hashLike ? 0.75 : 0.5,
'obfuscated_pattern',
['signal_score' => $analysis['signalScore'], 'markers' => $analysis['markers']]
);
}
return null;
@@ -146,6 +215,34 @@ class MiscCategorizer extends AbstractCategorizer
return $this->matched(Category::OTHER_HASHED, 0.7, 'gibberish_random_digits');
}
if ($this->isZeroVowelLongToken($coreName)) {
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
}
return null;
}
protected function checkLowSignal(string $name): ?CategorizationResult
{
$analysis = $this->inspectSignals($name);
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return null;
}
if ($analysis['lowSignal'] && $analysis['coreLength'] >= 20) {
return $this->matched(
Category::OTHER_HASHED,
0.8,
'gibberish_no_signal',
[
'signal_score' => $analysis['signalScore'],
'markers' => $analysis['markers'],
'core_length' => $analysis['coreLength'],
]
);
}
return null;
}
@@ -29,6 +29,13 @@ class CategorizationPassable
*/
public array $allResults = [];
/**
* Misc signal analysis captured by MiscPipe for downstream safety-net decisions.
*
* @var array<string, mixed>
*/
public array $miscAnalysis = [];
public function __construct(ReleaseContext $context, bool $debug = false)
{
$this->context = $context;
@@ -90,6 +97,7 @@ class CategorizationPassable
'locked_to_misc' => $this->lockedToMisc,
'release_name' => $this->context->releaseName,
'group_name' => $this->context->groupName,
'misc_analysis' => $this->miscAnalysis,
'all_results' => $this->allResults,
'categorizer_details' => $this->bestResult->debug,
];
@@ -50,6 +50,7 @@ class MiscPipe extends AbstractCategorizationPipe
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
{
// MiscPipe always runs regardless of shouldStopProcessing — it IS the first pipe.
$passable->miscAnalysis = $this->categorizer->inspectSignals($passable->context);
$result = $this->categorize($passable->context);
// Record the result for debug
@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace App\Services\Categorization\Pipes;
use App\Models\Category;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\ReleaseContext;
use Closure;
/**
* Final guardrail for low-signal names.
*
* If a release has no meaningful media markers and only matched because of the
* group name, prefer OTHER_MISC over trusting the group alone.
*/
class MiscSafetyNetPipe extends AbstractCategorizationPipe
{
protected int $priority = 100;
public function getName(): string
{
return 'MiscSafetyNet';
}
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
{
if ($this->shouldDowngradeToMisc($passable)) {
$result = new CategorizationResult(
Category::OTHER_MISC,
0.65,
'group_only_low_signal',
[
'group_match' => $passable->bestResult->matchedBy,
'misc_analysis' => $passable->miscAnalysis,
]
);
if ($passable->debug) {
$passable->allResults[$this->getName()] = [
'category_id' => $result->categoryId,
'confidence' => $result->confidence,
'matched_by' => $result->matchedBy,
];
}
$passable->bestResult = $result;
$passable->lockToMisc();
}
return $next($passable);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->noMatch();
}
private function shouldDowngradeToMisc(CategorizationPassable $passable): bool
{
return ! $passable->lockedToMisc
&& ($passable->miscAnalysis['lowSignal'] ?? false) === true
&& str_starts_with($passable->bestResult->matchedBy, 'group_name_');
}
}
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Services\NameFixing;
use App\Events\ReleaseNameFixed;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Predb;
@@ -12,6 +13,7 @@ use App\Models\UsenetGroup;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseCleaningService;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
/**
* Service for updating releases with new names.
@@ -153,12 +155,6 @@ class ReleaseUpdateService
$this->matched = true;
$this->relid = (int) $release->releases_id;
$determinedCategory = $this->category->determineCategory(
$release->groups_id,
$newName,
! empty($release->fromname) ? $release->fromname : ''
);
if ($type === 'PAR2, ') {
$newName = ucwords($newName);
if (preg_match('/(.+?)\.[a-z0-9]{2,3}(PAR2)?$/i', $name, $hit)) {
@@ -172,14 +168,21 @@ class ReleaseUpdateService
$newName = explode('\\', $newName);
$newName = preg_replace(['/^[=_.:\s-]+/', '/[=_.:\s-]+$/'], '', $newName[0]);
if ($this->echoOutput && $show) {
$this->echoReleaseInfo($release, $newName, $determinedCategory, $type, $method);
}
$newTitle = substr($newName, 0, 299);
$determinedCategory = null;
if ($this->echoOutput && $show) {
$determinedCategory = $this->category->determineCategory(
$release->groups_id,
$newTitle,
! empty($release->fromname) ? $release->fromname : ''
);
$this->echoReleaseInfo($release, $newTitle, $determinedCategory, $type, $method);
}
if ($echo === true) {
$this->performDatabaseUpdate($release, $newTitle, $determinedCategory, $type, $nameStatus, $preId);
$this->performDatabaseUpdate($release, $newTitle, $type, $nameStatus, $preId);
}
}
}
@@ -253,59 +256,65 @@ class ReleaseUpdateService
/**
* Perform the actual database update.
*
* @param array<string, mixed> $determinedCategory
*/
protected function performDatabaseUpdate(
object $release,
string $newTitle,
array $determinedCategory,
string $type,
bool $nameStatus,
int $preId
): void {
if ($nameStatus === true) {
$status = $this->getStatusColumnsForType($type);
DB::transaction(function () use ($release, $newTitle, $type, $nameStatus, $preId): void {
if ($nameStatus === true) {
$status = $this->getStatusColumnsForType($type);
$updateColumns = [
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => '',
'consoleinfo_id' => '',
'bookinfo_id' => '',
'anidbid' => '',
'predb_id' => $preId,
'searchname' => $newTitle,
'categories_id' => $determinedCategory['categories_id'],
];
if (! empty($status)) {
foreach ($status as $key => $stat) {
$updateColumns = Arr::add($updateColumns, $key, $stat);
}
}
Release::query()
->where('id', $release->releases_id)
->update($updateColumns);
} else {
Release::query()
->where('id', $release->releases_id)
->update([
$updateColumns = [
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => null,
'consoleinfo_id' => null,
'bookinfo_id' => null,
'anidbid' => null,
'musicinfo_id' => '',
'consoleinfo_id' => '',
'bookinfo_id' => '',
'anidbid' => '',
'predb_id' => $preId,
'searchname' => $newTitle,
'categories_id' => $determinedCategory['categories_id'],
'iscategorized' => 1,
]);
}
];
if (! empty($status)) {
foreach ($status as $key => $stat) {
$updateColumns = Arr::add($updateColumns, $key, $stat);
}
}
Release::query()
->where('id', $release->releases_id)
->update($updateColumns);
} else {
Release::query()
->where('id', $release->releases_id)
->update([
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => null,
'consoleinfo_id' => null,
'bookinfo_id' => null,
'anidbid' => null,
'predb_id' => $preId,
'searchname' => $newTitle,
'iscategorized' => 1,
]);
}
event(new ReleaseNameFixed(
(int) $release->releases_id,
(string) $release->searchname,
$newTitle,
(int) $release->categories_id,
$release->groups_id,
(string) ($release->fromname ?? '')
));
});
// Update search index
Search::updateRelease($release->releases_id);
@@ -350,7 +359,7 @@ class ReleaseUpdateService
*/
public function checkPreDbMatch(object $release, string $textstring): ?array
{
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s\:/i', $textstring)) {
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) {
foreach ($hits as $hit) {
foreach ($hit as $val) {
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
+58 -1
View File
@@ -29,11 +29,13 @@ trait DetectsHashedNames
|| $this->isBoundedSha1Hash($name)
|| $this->isBoundedSha256Hash($name)
|| $this->isBoundedGenericHash($name)
|| $this->isBase64LikeToken($name)
|| $this->isObfuscatedUppercaseString($name)
|| $this->isObfuscatedMixedAlphanumeric($name)
|| $this->isObfuscatedUsenetFilename($name)
|| $this->isRandomByCharacterAnalysis($coreName, $name)
|| $this->hasInsufficientWordStructure($coreName)
|| $this->isZeroVowelLongToken($coreName)
|| $this->isRandomDigitPattern($coreName);
}
@@ -70,7 +72,40 @@ trait DetectsHashedNames
*/
protected function isBoundedGenericHash(string $name): bool
{
return (bool) preg_match('/(?:^|["\'\s\[\]\/\-])([a-f0-9]{32,128})(?:["\'\s\[\]\/.\-]|$)/i', $name);
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([a-z0-9]{24,128})(?:["\'\s\[\]\/.\-]|$)/i', $name, $matches)) {
return false;
}
$token = $matches[1];
if (preg_match('/\b(19|20)\d{2}\b/', $token)) {
return false;
}
return preg_match('/\d/', $token) === 1
&& $this->looksLikeRandomString($token);
}
/**
* Detect base64/base64url-like tokens that commonly appear in obfuscated subjects.
*/
protected function isBase64LikeToken(string $name): bool
{
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([A-Za-z0-9+\/_=-]{24,})(?:["\'\s\[\]\/.\-]|$)/', $name, $matches)) {
return false;
}
$token = rtrim($matches[1], '=');
if (strlen($token) < 24 || preg_match('/\b(19|20)\d{2}\b/', $token)) {
return false;
}
if (! preg_match('/[A-Z]/', $token) || ! preg_match('/[a-z]/', $token) || ! preg_match('/\d/', $token)) {
return false;
}
return $this->looksLikeRandomString($token);
}
// ---------------------------------------------------------------
@@ -207,6 +242,28 @@ trait DetectsHashedNames
);
}
/**
* Detect long alphanumeric tokens with effectively no vowel structure.
*/
protected function isZeroVowelLongToken(string $coreName, int $minLength = 20): bool
{
if (strlen($coreName) < $minLength || ! preg_match('/^[a-zA-Z0-9]+$/', $coreName)) {
return false;
}
if (preg_match('/\b(19|20)\d{2}\b/', $coreName)) {
return false;
}
$letterOnly = preg_replace('/\d+/', '', $coreName);
if ($letterOnly === '' || strlen($letterOnly) < 10) {
return false;
}
return preg_match('/[aeiou]/i', $letterOnly) !== 1;
}
/**
* Check if a string looks like a random/obfuscated string rather than a real title.
*