starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 22:01:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update categorization

Browse Source
This commit is contained in:
DariusIII
2026-04-21 17:35:52 +02:00
parent f2c06224ee
commit bd7f540c23
15 changed files with 685 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Events/ReleaseNameFixed.php
+23
View File
@@ -0,0 +1,23 @@
<?php
declare(strict_types=1);
namespace App\Events;
use Illuminate\Foundation\Events\Dispatchable;
use Illuminate\Queue\SerializesModels;
class ReleaseNameFixed
{
use Dispatchable;
use SerializesModels;
public function __construct(
public readonly int $releaseId,
public readonly string $oldName,
public readonly string $newName,
public readonly int $oldCategoryId,
public readonly int|string $groupId,
public readonly string $poster = '',
) {}
}
app/Listeners/RecategorizeReleaseAfterNameFix.php
+60
View File
@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
namespace App\Listeners;
use App\Events\ReleaseNameFixed;
use App\Models\Release;
use App\Services\Categorization\CategorizationService;
use Illuminate\Support\Facades\Log;
class RecategorizeReleaseAfterNameFix
{
public function __construct(private readonly CategorizationService $categorization) {}
public function handle(ReleaseNameFixed $event): void
{
$release = Release::query()->find($event->releaseId, [
'id',
'groups_id',
'fromname',
'categories_id',
'iscategorized',
'searchname',
]);
if ($release === null) {
return;
}
$result = $this->categorization->determineCategory(
$release->groups_id,
$event->newName,
(string) ($release->fromname ?? $event->poster)
);
$newCategoryId = (int) ($result['categories_id'] ?? $release->categories_id);
if ((int) $release->categories_id === $newCategoryId && (int) $release->iscategorized === 1) {
return;
}
Release::query()
->where('id', $release->id)
->update([
'categories_id' => $newCategoryId,
'iscategorized' => 1,
]);
if (config('nntmux.categorization.log', false)) {
Log::info('categorization.rename_recategorized', [
'release_id' => $release->id,
'old_name' => $event->oldName,
'new_name' => $event->newName,
'old_category_id' => $event->oldCategoryId,
'new_category_id' => $newCategoryId,
]);
}
}
}
app/Providers/AppServiceProvider.php
+3
View File
@@ -4,6 +4,8 @@ declare(strict_types=1);
namespace App\Providers;
use App\Events\ReleaseNameFixed;
use App\Listeners\RecategorizeReleaseAfterNameFix;
use App\Models\AnidbInfo;
use App\Models\AnidbTitle;
use App\Models\BookInfo;
@@ -59,6 +61,7 @@ class AppServiceProvider extends ServiceProvider
return $user->hasRole('Admin');
});
Event::listen(Login::class, LoginViaRemember::class);
Event::listen(ReleaseNameFixed::class, RecategorizeReleaseAfterNameFix::class);
// Register observers
RolePromotion::observe(RolePromotionObserver::class);
app/Services/Categorization/CategorizationPipeline.php
+28 -1
View File
@@ -10,6 +10,7 @@ use App\Services\Categorization\Pipes\AbstractCategorizationPipe;
use App\Services\Categorization\Pipes\CategorizationPassable;
use Illuminate\Pipeline\Pipeline;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
/**
* Pipeline-based categorization service using Laravel Pipeline.
@@ -88,9 +89,34 @@ class CategorizationPipeline
->through($this->pipes->values()->all())
->thenReturn();
$this->logCategorization($result);
return $result->toArray();
}
protected function logCategorization(CategorizationPassable $result): void
{
if (! config('nntmux.categorization.log', false)) {
return;
}
$payload = [
'release_name' => $result->context->releaseName,
'group_name' => $result->context->groupName,
'category_id' => $result->bestResult->categoryId,
'matched_by' => $result->bestResult->matchedBy,
'confidence' => $result->bestResult->confidence,
'locked_to_misc' => $result->lockedToMisc,
'misc_analysis' => $result->miscAnalysis,
];
if ($result->lockedToMisc || $result->bestResult->matchedBy === 'group_only_low_signal') {
Log::info('categorization.decision', $payload);
}
Log::debug('categorization.trace', $payload + ['all_results' => $result->allResults]);
}
/**
* Get all registered categorizers (pipes).
*
@@ -107,6 +133,7 @@ class CategorizationPipeline
public static function createDefault(): self
{
return new self([
new Pipes\MiscPipe,
new Pipes\GroupNamePipe,
new Pipes\XxxPipe,
new Pipes\TvPipe,
@@ -115,7 +142,7 @@ class CategorizationPipeline
new Pipes\MusicPipe,
new Pipes\PcPipe,
new Pipes\ConsolePipe,
new Pipes\MiscPipe,
new Pipes\MiscSafetyNetPipe,
]);
}
}
app/Services/Categorization/CategorizationResult.php
+24
View File
@@ -51,6 +51,11 @@ class CategorizationResult
*/
public function shouldOverride(CategorizationResult $other): bool
{
if ($this->categoryId !== Category::OTHER_MISC && $this->categoryId !== Category::OTHER_HASHED &&
$other->isProtectedMiscResult() && $this->confidence < 0.6) {
return false;
}
// Higher confidence always wins
if ($this->confidence > $other->confidence) {
return true;
@@ -64,6 +69,25 @@ class CategorizationResult
return false;
}
/**
* Determine whether this result should resist weak downstream overrides.
*/
public function isProtectedMiscResult(): bool
{
if ($this->categoryId === Category::OTHER_HASHED) {
return true;
}
if ($this->categoryId !== Category::OTHER_MISC) {
return false;
}
return str_starts_with($this->matchedBy, 'hash_')
|| str_starts_with($this->matchedBy, 'obfuscated_')
|| str_starts_with($this->matchedBy, 'gibberish_')
|| $this->matchedBy === 'group_only_low_signal';
}
/**
* Create a failed/empty result.
*/
app/Services/Categorization/Categorizers/GroupNameCategorizer.php
+7 -7
View File
@@ -24,25 +24,25 @@ class GroupNameCategorizer extends AbstractCategorizer
return $this->noMatch();
}
if (preg_match('/alt\.binaries\..*?(tv|hdtv|tvseries)/i', $groupName)) {
return $this->matched(Category::TV_OTHER, 0.6, 'group_tv');
return $this->matched(Category::TV_OTHER, 0.6, 'group_name_tv');
}
if (preg_match('/alt\.binaries\..*?(movies?|dvd|bluray|x264)/i', $groupName)) {
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_movie');
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_name_movie');
}
if (preg_match('/alt\.binaries\..*?(erotica|pictures\.erotica|xxx)/i', $groupName)) {
return $this->matched(Category::XXX_OTHER, 0.7, 'group_xxx');
return $this->matched(Category::XXX_OTHER, 0.7, 'group_name_xxx');
}
if (preg_match('/alt\.binaries\..*?(sounds?|mp3|music|lossless)/i', $groupName)) {
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_music');
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_name_music');
}
if (preg_match('/alt\.binaries\..*?(games?|console|psx|nintendo)/i', $groupName)) {
return $this->matched(Category::GAME_OTHER, 0.6, 'group_game');
return $this->matched(Category::GAME_OTHER, 0.6, 'group_name_game');
}
if (preg_match('/alt\.binaries\..*?(warez|0day|apps?|software)/i', $groupName)) {
return $this->matched(Category::PC_0DAY, 0.6, 'group_pc');
return $this->matched(Category::PC_0DAY, 0.6, 'group_name_pc');
}
if (preg_match('/alt\.binaries\..*?(e-?book|ebook|comics?)/i', $groupName)) {
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_book');
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_name_book');
}
return $this->noMatch();
app/Services/Categorization/Categorizers/MiscCategorizer.php
+98 -1
View File
@@ -34,11 +34,21 @@ class MiscCategorizer extends AbstractCategorizer
return $result;
}
$analysis = $this->inspectSignals($name);
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
}
// Check for obfuscated/encoded patterns
if ($result = $this->checkObfuscated($name)) {
return $result;
}
// Check low-signal names that only contain random-looking tokens
if ($result = $this->checkLowSignal($name)) {
return $result;
}
// Check for gibberish patterns (character-analysis heuristics)
if ($result = $this->checkGibberish($name)) {
return $result;
@@ -57,6 +67,50 @@ class MiscCategorizer extends AbstractCategorizer
return $this->noMatch();
}
/**
* Inspect a release name for media signal markers used by the safety-net pipe.
*
* @return array{coreName: string, coreLength: int, signalScore: int, markers: list<string>, lowSignal: bool}
*/
public function inspectSignals(ReleaseContext|string $context): array
{
$name = $context instanceof ReleaseContext ? $context->releaseName : $context;
$cleaned = $this->stripExtensionsForAnalysis($name);
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
$patterns = [
'season_episode' => '/\bS\d{1,3}[._ -]?E\d{1,4}\b/i',
'season_pack' => '/\bS\d{1,3}\b/i',
'resolution' => '/\b(480p|576p|720p|1080[pi]?|2160p|4k|uhd)\b/i',
'codec' => '/\b(x264|x265|h\.?264|h\.?265|hevc|xvid|av1)\b/i',
'source' => '/\b(bluray|bdrip|brrip|hdtv|web[._ -]?dl|web[._ -]?rip|dvdrip|remux)\b/i',
'audio' => '/\b(aac|ac3|ddp|dts|flac|mp3)\b/i',
'scene_tag' => '/\b(proper|repack|internal|limited|complete|dubbed|subbed|readnfo)\b/i',
'year' => '/\b(19|20)\d{2}\b/',
'release_group' => '/-[A-Za-z0-9][A-Za-z0-9._-]{1,20}$/',
'known_extension' => '/\.(mkv|avi|mp4|mp3|flac|iso|epub|pdf|exe|nzb|rar|7z)$/i',
];
$markers = [];
foreach ($patterns as $marker => $pattern) {
if (preg_match($pattern, $name)) {
$markers[] = $marker;
}
}
$signalScore = count($markers);
$isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
$lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12;
return [
'coreName' => $coreName,
'coreLength' => strlen($coreName),
'signalScore' => $signalScore,
'markers' => $markers,
'lowSignal' => $lowSignal,
];
}
protected function checkHash(string $name): ?CategorizationResult
{
// MD5 hash (32 hex characters)
@@ -79,6 +133,10 @@ class MiscCategorizer extends AbstractCategorizer
return $this->matched(Category::OTHER_HASHED, 0.95, 'hash_generic');
}
if ($this->isBase64LikeToken($name)) {
return $this->matched(Category::OTHER_HASHED, 0.9, 'hash_base64_like');
}
// Strip extensions and separators for core-name checks
$cleaned = $this->stripExtensionsForAnalysis($name);
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
@@ -115,7 +173,18 @@ class MiscCategorizer extends AbstractCategorizer
// Only punctuation and numbers with no clear structure
if ($this->isObfuscatedPunctuation($name)) {
return $this->matched(Category::OTHER_MISC, 0.5, 'obfuscated_pattern');
$analysis = $this->inspectSignals($name);
$hashLike = $this->isBase64LikeToken($name)
|| $this->isBoundedGenericHash($name)
|| $this->isZeroVowelLongToken($analysis['coreName'], 12)
|| $analysis['lowSignal'];
return $this->matched(
$hashLike ? Category::OTHER_HASHED : Category::OTHER_MISC,
$hashLike ? 0.75 : 0.5,
'obfuscated_pattern',
['signal_score' => $analysis['signalScore'], 'markers' => $analysis['markers']]
);
}
return null;
@@ -146,6 +215,34 @@ class MiscCategorizer extends AbstractCategorizer
return $this->matched(Category::OTHER_HASHED, 0.7, 'gibberish_random_digits');
}
if ($this->isZeroVowelLongToken($coreName)) {
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
}
return null;
}
protected function checkLowSignal(string $name): ?CategorizationResult
{
$analysis = $this->inspectSignals($name);
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return null;
}
if ($analysis['lowSignal'] && $analysis['coreLength'] >= 20) {
return $this->matched(
Category::OTHER_HASHED,
0.8,
'gibberish_no_signal',
[
'signal_score' => $analysis['signalScore'],
'markers' => $analysis['markers'],
'core_length' => $analysis['coreLength'],
]
);
}
return null;
}
app/Services/Categorization/Pipes/CategorizationPassable.php
+8
View File
@@ -29,6 +29,13 @@ class CategorizationPassable
*/
public array $allResults = [];
/**
* Misc signal analysis captured by MiscPipe for downstream safety-net decisions.
*
* @var array<string, mixed>
*/
public array $miscAnalysis = [];
public function __construct(ReleaseContext $context, bool $debug = false)
{
$this->context = $context;
@@ -90,6 +97,7 @@ class CategorizationPassable
'locked_to_misc' => $this->lockedToMisc,
'release_name' => $this->context->releaseName,
'group_name' => $this->context->groupName,
'misc_analysis' => $this->miscAnalysis,
'all_results' => $this->allResults,
'categorizer_details' => $this->bestResult->debug,
];
app/Services/Categorization/Pipes/MiscPipe.php
+1
View File
@@ -50,6 +50,7 @@ class MiscPipe extends AbstractCategorizationPipe
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
{
// MiscPipe always runs regardless of shouldStopProcessing — it IS the first pipe.
$passable->miscAnalysis = $this->categorizer->inspectSignals($passable->context);
$result = $this->categorize($passable->context);
// Record the result for debug
app/Services/Categorization/Pipes/MiscSafetyNetPipe.php
+66
View File
@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace App\Services\Categorization\Pipes;
use App\Models\Category;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\ReleaseContext;
use Closure;
/**
* Final guardrail for low-signal names.
*
* If a release has no meaningful media markers and only matched because of the
* group name, prefer OTHER_MISC over trusting the group alone.
*/
class MiscSafetyNetPipe extends AbstractCategorizationPipe
{
protected int $priority = 100;
public function getName(): string
{
return 'MiscSafetyNet';
}
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
{
if ($this->shouldDowngradeToMisc($passable)) {
$result = new CategorizationResult(
Category::OTHER_MISC,
0.65,
'group_only_low_signal',
[
'group_match' => $passable->bestResult->matchedBy,
'misc_analysis' => $passable->miscAnalysis,
]
);
if ($passable->debug) {
$passable->allResults[$this->getName()] = [
'category_id' => $result->categoryId,
'confidence' => $result->confidence,
'matched_by' => $result->matchedBy,
];
}
$passable->bestResult = $result;
$passable->lockToMisc();
}
return $next($passable);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->noMatch();
}
private function shouldDowngradeToMisc(CategorizationPassable $passable): bool
{
return ! $passable->lockedToMisc
&& ($passable->miscAnalysis['lowSignal'] ?? false) === true
&& str_starts_with($passable->bestResult->matchedBy, 'group_name_');
}
}
app/Services/NameFixing/ReleaseUpdateService.php
+60 -51
View File
@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Services\NameFixing;
use App\Events\ReleaseNameFixed;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Predb;
@@ -12,6 +13,7 @@ use App\Models\UsenetGroup;
use App\Services\Categorization\CategorizationService;
use App\Services\ReleaseCleaningService;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
/**
* Service for updating releases with new names.
@@ -153,12 +155,6 @@ class ReleaseUpdateService
$this->matched = true;
$this->relid = (int) $release->releases_id;
$determinedCategory = $this->category->determineCategory(
$release->groups_id,
$newName,
! empty($release->fromname) ? $release->fromname : ''
);
if ($type === 'PAR2, ') {
$newName = ucwords($newName);
if (preg_match('/(.+?)\.[a-z0-9]{2,3}(PAR2)?$/i', $name, $hit)) {
@@ -172,14 +168,21 @@ class ReleaseUpdateService
$newName = explode('\\', $newName);
$newName = preg_replace(['/^[=_.:\s-]+/', '/[=_.:\s-]+$/'], '', $newName[0]);
if ($this->echoOutput && $show) {
$this->echoReleaseInfo($release, $newName, $determinedCategory, $type, $method);
}
$newTitle = substr($newName, 0, 299);
$determinedCategory = null;
if ($this->echoOutput && $show) {
$determinedCategory = $this->category->determineCategory(
$release->groups_id,
$newTitle,
! empty($release->fromname) ? $release->fromname : ''
);
$this->echoReleaseInfo($release, $newTitle, $determinedCategory, $type, $method);
}
if ($echo === true) {
$this->performDatabaseUpdate($release, $newTitle, $determinedCategory, $type, $nameStatus, $preId);
$this->performDatabaseUpdate($release, $newTitle, $type, $nameStatus, $preId);
}
}
}
@@ -253,59 +256,65 @@ class ReleaseUpdateService
/**
* Perform the actual database update.
*
* @param array<string, mixed> $determinedCategory
*/
protected function performDatabaseUpdate(
object $release,
string $newTitle,
array $determinedCategory,
string $type,
bool $nameStatus,
int $preId
): void {
if ($nameStatus === true) {
$status = $this->getStatusColumnsForType($type);
DB::transaction(function () use ($release, $newTitle, $type, $nameStatus, $preId): void {
if ($nameStatus === true) {
$status = $this->getStatusColumnsForType($type);
$updateColumns = [
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => '',
'consoleinfo_id' => '',
'bookinfo_id' => '',
'anidbid' => '',
'predb_id' => $preId,
'searchname' => $newTitle,
'categories_id' => $determinedCategory['categories_id'],
];
if (! empty($status)) {
foreach ($status as $key => $stat) {
$updateColumns = Arr::add($updateColumns, $key, $stat);
}
}
Release::query()
->where('id', $release->releases_id)
->update($updateColumns);
} else {
Release::query()
->where('id', $release->releases_id)
->update([
$updateColumns = [
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => null,
'consoleinfo_id' => null,
'bookinfo_id' => null,
'anidbid' => null,
'musicinfo_id' => '',
'consoleinfo_id' => '',
'bookinfo_id' => '',
'anidbid' => '',
'predb_id' => $preId,
'searchname' => $newTitle,
'categories_id' => $determinedCategory['categories_id'],
'iscategorized' => 1,
]);
}
];
if (! empty($status)) {
foreach ($status as $key => $stat) {
$updateColumns = Arr::add($updateColumns, $key, $stat);
}
}
Release::query()
->where('id', $release->releases_id)
->update($updateColumns);
} else {
Release::query()
->where('id', $release->releases_id)
->update([
'videos_id' => 0,
'tv_episodes_id' => 0,
'imdbid' => null,
'musicinfo_id' => null,
'consoleinfo_id' => null,
'bookinfo_id' => null,
'anidbid' => null,
'predb_id' => $preId,
'searchname' => $newTitle,
'iscategorized' => 1,
]);
}
event(new ReleaseNameFixed(
(int) $release->releases_id,
(string) $release->searchname,
$newTitle,
(int) $release->categories_id,
$release->groups_id,
(string) ($release->fromname ?? '')
));
});
// Update search index
Search::updateRelease($release->releases_id);
@@ -350,7 +359,7 @@ class ReleaseUpdateService
*/
public function checkPreDbMatch(object $release, string $textstring): ?array
{
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s\:/i', $textstring)) {
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) {
foreach ($hits as $hit) {
foreach ($hit as $val) {
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
app/Traits/DetectsHashedNames.php
+58 -1
View File
@@ -29,11 +29,13 @@ trait DetectsHashedNames
|| $this->isBoundedSha1Hash($name)
|| $this->isBoundedSha256Hash($name)
|| $this->isBoundedGenericHash($name)
|| $this->isBase64LikeToken($name)
|| $this->isObfuscatedUppercaseString($name)
|| $this->isObfuscatedMixedAlphanumeric($name)
|| $this->isObfuscatedUsenetFilename($name)
|| $this->isRandomByCharacterAnalysis($coreName, $name)
|| $this->hasInsufficientWordStructure($coreName)
|| $this->isZeroVowelLongToken($coreName)
|| $this->isRandomDigitPattern($coreName);
}
@@ -70,7 +72,40 @@ trait DetectsHashedNames
*/
protected function isBoundedGenericHash(string $name): bool
{
return (bool) preg_match('/(?:^|["\'\s\[\]\/\-])([a-f0-9]{32,128})(?:["\'\s\[\]\/.\-]|$)/i', $name);
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([a-z0-9]{24,128})(?:["\'\s\[\]\/.\-]|$)/i', $name, $matches)) {
return false;
}
$token = $matches[1];
if (preg_match('/\b(19|20)\d{2}\b/', $token)) {
return false;
}
return preg_match('/\d/', $token) === 1
&& $this->looksLikeRandomString($token);
}
/**
* Detect base64/base64url-like tokens that commonly appear in obfuscated subjects.
*/
protected function isBase64LikeToken(string $name): bool
{
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([A-Za-z0-9+\/_=-]{24,})(?:["\'\s\[\]\/.\-]|$)/', $name, $matches)) {
return false;
}
$token = rtrim($matches[1], '=');
if (strlen($token) < 24 || preg_match('/\b(19|20)\d{2}\b/', $token)) {
return false;
}
if (! preg_match('/[A-Z]/', $token) || ! preg_match('/[a-z]/', $token) || ! preg_match('/\d/', $token)) {
return false;
}
return $this->looksLikeRandomString($token);
}
// ---------------------------------------------------------------
@@ -207,6 +242,28 @@ trait DetectsHashedNames
);
}
/**
* Detect long alphanumeric tokens with effectively no vowel structure.
*/
protected function isZeroVowelLongToken(string $coreName, int $minLength = 20): bool
{
if (strlen($coreName) < $minLength || ! preg_match('/^[a-zA-Z0-9]+$/', $coreName)) {
return false;
}
if (preg_match('/\b(19|20)\d{2}\b/', $coreName)) {
return false;
}
$letterOnly = preg_replace('/\d+/', '', $coreName);
if ($letterOnly === '' || strlen($letterOnly) < 10) {
return false;
}
return preg_match('/[aeiou]/i', $letterOnly) !== 1;
}
/**
* Check if a string looks like a random/obfuscated string rather than a real title.
*
config/nntmux.php
+3
View File
@@ -28,4 +28,7 @@ return [
'nzb_upload_folder' => env('NZB_UPLOAD_FOLDER'),
'redis_fast_degrade' => (bool) env('REDIS_FAST_DEGRADE', true),
'redis_tcp_check_seconds' => (float) env('REDIS_TCP_CHECK_SECONDS', 0.2),
'categorization' => [
'log' => (bool) env('NNTMUX_CATEGORIZATION_LOG', false),
],
];
tests/Feature/ReleaseNameFixedRecategorizationTest.php
+211
View File
@@ -0,0 +1,211 @@
<?php
declare(strict_types=1);
namespace Tests\Feature;
use App\Facades\Search;
use App\Models\Category;
use App\Models\Release;
use App\Models\UsenetGroup;
use App\Services\NameFixing\ReleaseUpdateService;
use Illuminate\Contracts\Console\Kernel;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use PDO;
use Tests\TestCase;
class ReleaseNameFixedRecategorizationTest extends TestCase
{
private string $databasePath;
/**
* @var array<string, string|false>
*/
private array $originalEnvironment = [];
public function createApplication()
{
$this->databasePath = sys_get_temp_dir().'/nntmux-release-name-fixed-test.sqlite';
$this->originalEnvironment = [
'APP_ENV' => getenv('APP_ENV'),
'DB_CONNECTION' => getenv('DB_CONNECTION'),
'DB_DATABASE' => getenv('DB_DATABASE'),
];
if (file_exists($this->databasePath)) {
unlink($this->databasePath);
}
$pdo = new PDO('sqlite:'.$this->databasePath);
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
$pdo->exec("INSERT INTO settings (name, value) VALUES ('categorizeforeign', '0'), ('catwebdl', '1')");
$this->setEnvironmentValue('APP_ENV', 'testing');
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
$app = require __DIR__.'/../../bootstrap/app.php';
$app->make(Kernel::class)->bootstrap();
return $app;
}
protected function setUp(): void
{
parent::setUp();
DB::table('settings')->upsert([
['name' => 'categorizeforeign', 'value' => '0'],
['name' => 'catwebdl', 'value' => '1'],
], ['name'], ['value']);
config([
'database.default' => 'sqlite',
'database.connections.sqlite.database' => $this->databasePath,
]);
DB::purge();
DB::reconnect();
$this->createSchema();
}
protected function tearDown(): void
{
if ($this->databasePath !== '' && file_exists($this->databasePath)) {
unlink($this->databasePath);
}
parent::tearDown();
foreach ($this->originalEnvironment as $key => $value) {
$this->setEnvironmentValue($key, $value === false ? null : $value);
}
}
public function test_renaming_hashed_release_recategorizes_it_synchronously(): void
{
Search::shouldReceive('updateRelease')->twice();
$group = UsenetGroup::query()->create([
'name' => 'alt.binaries.hdtv',
'active' => 1,
'backfill' => 0,
]);
$release = Release::factory()->create([
'name' => 'd41d8cd98f00b204e9800998ecf8427e',
'searchname' => 'd41d8cd98f00b204e9800998ecf8427e',
'fromname' => 'poster@example.com',
'groups_id' => $group->id,
'categories_id' => Category::OTHER_HASHED,
'iscategorized' => 1,
'isrenamed' => 0,
'guid' => str_repeat('a', 40),
'leftguid' => 'a',
'nzb_guid' => 'test',
'size' => 1,
'postdate' => now(),
'adddate' => now(),
]);
$service = app(ReleaseUpdateService::class);
$service->updateRelease(
$release->fresh(),
'Show.Name.S03E05.720p.HDTV.x264-GROUP',
'nfoCheck: Title Match',
true,
'NFO, ',
true,
false,
);
$release->refresh();
$this->assertSame('Show.Name.S03E05.720p.HDTV.x264-GROUP', $release->searchname);
$this->assertSame(Category::TV_HD, $release->categories_id);
$this->assertSame(1, (int) $release->iscategorized);
$this->assertSame(1, (int) $release->isrenamed);
}
private function setEnvironmentValue(string $key, ?string $value): void
{
if ($value === null) {
putenv($key);
unset($_ENV[$key], $_SERVER[$key]);
return;
}
putenv($key.'='.$value);
$_ENV[$key] = $value;
$_SERVER[$key] = $value;
}
private function createSchema(): void
{
if (! Schema::hasTable('settings')) {
Schema::create('settings', function (Blueprint $table): void {
$table->string('name')->primary();
$table->text('value')->nullable();
});
}
if (! Schema::hasTable('usenet_groups')) {
Schema::create('usenet_groups', function (Blueprint $table): void {
$table->increments('id');
$table->string('name')->unique();
$table->integer('backfill_target')->default(1);
$table->unsignedBigInteger('first_record')->default(0);
$table->dateTime('first_record_postdate')->nullable();
$table->unsignedBigInteger('last_record')->default(0);
$table->dateTime('last_record_postdate')->nullable();
$table->dateTime('last_updated')->nullable();
$table->integer('minfilestoformrelease')->nullable();
$table->bigInteger('minsizetoformrelease')->nullable();
$table->boolean('active')->default(false);
$table->boolean('backfill')->default(false);
$table->string('description')->nullable();
});
}
if (! Schema::hasTable('releases')) {
Schema::create('releases', function (Blueprint $table): void {
$table->increments('id');
$table->string('name')->default('');
$table->string('searchname')->default('');
$table->unsignedInteger('groups_id')->default(0);
$table->unsignedBigInteger('size')->default(0);
$table->dateTime('postdate')->nullable();
$table->dateTime('adddate')->nullable();
$table->string('guid', 40);
$table->char('leftguid', 1);
$table->string('fromname')->nullable();
$table->integer('categories_id')->default(Category::OTHER_MISC);
$table->unsignedInteger('videos_id')->default(0);
$table->integer('tv_episodes_id')->default(0);
$table->string('imdbid')->nullable();
$table->integer('musicinfo_id')->nullable();
$table->integer('consoleinfo_id')->nullable();
$table->integer('bookinfo_id')->nullable();
$table->integer('anidbid')->nullable();
$table->unsignedInteger('predb_id')->default(0);
$table->tinyInteger('iscategorized')->default(0);
$table->tinyInteger('isrenamed')->default(0);
$table->tinyInteger('proc_nfo')->default(0);
$table->tinyInteger('proc_files')->default(0);
$table->tinyInteger('proc_par2')->default(0);
$table->tinyInteger('proc_uid')->default(0);
$table->tinyInteger('proc_hash16k')->default(0);
$table->tinyInteger('proc_srr')->default(0);
$table->tinyInteger('proc_crc32')->default(0);
$table->tinyInteger('passwordstatus')->default(0);
$table->tinyInteger('nzbstatus')->default(0);
$table->binary('nzb_guid')->nullable();
});
}
}
}
tests/Unit/HashedReleaseCategorizationTest.php
+35 -2
View File
@@ -7,10 +7,12 @@ namespace Tests\Unit;
use App\Models\Category;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\MiscCategorizer;
use App\Services\Categorization\Pipes\BookPipe;
use App\Services\Categorization\Pipes\CategorizationPassable;
use App\Services\Categorization\Pipes\ConsolePipe;
use App\Services\Categorization\Pipes\GroupNamePipe;
use App\Services\Categorization\Pipes\MiscPipe;
use App\Services\Categorization\Pipes\MiscSafetyNetPipe;
use App\Services\Categorization\Pipes\MoviePipe;
use App\Services\Categorization\Pipes\MusicPipe;
use App\Services\Categorization\Pipes\PcPipe;
@@ -25,7 +27,7 @@ class HashedReleaseCategorizationTest extends TestCase
/**
* The ordered list of pipes that matches CategorizationPipeline::createDefault().
* Sorted by priority: MiscPipe(1), GroupNamePipe(5), XxxPipe(10), TvPipe(20),
* MoviePipe(25), BookPipe, MusicPipe, PcPipe, ConsolePipe.
* MoviePipe(25), BookPipe, MusicPipe, PcPipe, ConsolePipe, MiscSafetyNetPipe.
*
* @return list<object>
*/
@@ -37,9 +39,11 @@ class HashedReleaseCategorizationTest extends TestCase
new XxxPipe,
new TvPipe,
new MoviePipe,
new BookPipe,
new MusicPipe,
new PcPipe,
new ConsolePipe,
new MiscSafetyNetPipe,
];
}
@@ -87,6 +91,7 @@ class HashedReleaseCategorizationTest extends TestCase
'All uppercase 20 chars' => ['ABCDEFGH1234567890XY', 'obfuscated_uppercase'],
'Mixed alphanumeric random' => ['AA7Jl2toE8Q53yNZmQ5R6G', 'obfuscated_mixed_alphanumeric'],
'Usenet obfuscated filename' => ['[01/10] - "xK9mR2pL4qW7nT3vB.part01.rar"', 'obfuscated_usenet_filename'],
'Base64-like token' => ['VGhpc0lzTm90QVNob3dOYW1lMTIzNDU2Nzg5MA==', 'hash_base64_like'],
];
}
@@ -106,10 +111,12 @@ class HashedReleaseCategorizationTest extends TestCase
// Lowercase with dots and digits — bypasses all obfuscated checks.
// After stripping, coreName ≥20 with maxConsecutiveLetters < 5 but low transition rate.
// Grouped letter/digit runs keep transition rate ≤ 0.35.
'No word structure long' => ['xyz.1234.wvut.5678.srqp.9012', 'gibberish_no_word_structure'],
'No word structure long' => ['xyz.1234.wvut.5678.srqp.9012.rar', 'gibberish_no_word_structure'],
// Lowercase with dots and digits — bypasses all obfuscated checks.
// After stripping, coreName matches digit-heavy pattern (1-3 letters + 6+ digits).
'Digit-heavy pattern' => ['xz.123456789012', 'gibberish_random_digits'],
'Zero vowel core' => ['xkcdqwrtypsdfghjklmnbvcxz', 'gibberish_zero_vowels'],
'No signal long token' => ['A1b2.C3d4.E5f6.G7h8.I9j0', 'gibberish_no_signal'],
];
}
@@ -144,6 +151,22 @@ class HashedReleaseCategorizationTest extends TestCase
'Gibberish in games group' => ['aB3cD4eF5gH6iJ7kL8m', 'alt.binaries.games'],
'Hex in warez group' => ['aabbccdd0011223344ff', 'alt.binaries.warez'],
'Random digits in ebook group' => ['ab12345678901234', 'alt.binaries.e-book'],
'Base64-like token in TV group' => ['VGhpc0lzTm90QVNob3dOYW1lMTIzNDU2Nzg5MA==', 'alt.binaries.hdtv'],
'Zero vowel token in movie group' => ['xkcdqwrtypsdfghjklmnbvcxz', 'alt.binaries.movies'],
];
}
/**
* Low-signal names that are not strong enough for an early lock but should
* still be kept out of content categories when only the group matched.
*
* @return array<string, array{0: string, 1: string}>
*/
public static function lowSignalGroupOnlyProvider(): array
{
return [
'Short low-signal token in TV group' => ['ab12.cd34.ef56', 'alt.binaries.hdtv'],
'Short low-signal token in movies group' => ['zx90.cv78.bn56', 'alt.binaries.movies'],
];
}
@@ -237,6 +260,16 @@ class HashedReleaseCategorizationTest extends TestCase
);
}
#[DataProvider('lowSignalGroupOnlyProvider')]
public function test_group_only_low_signal_releases_fall_back_to_other_misc(string $name, string $groupName): void
{
$passable = $this->runPipeline($name, $groupName);
$this->assertTrue($passable->lockedToMisc, "Expected lockedToMisc for low-signal release: $name");
$this->assertSame(Category::OTHER_MISC, $passable->bestResult->categoryId);
$this->assertSame('group_only_low_signal', $passable->bestResult->matchedBy);
}
// ------------------------------------------------------------------
// Tests: Legitimate releases still categorize normally
// ------------------------------------------------------------------