mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Update categorization
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Events;
|
||||
|
||||
use Illuminate\Foundation\Events\Dispatchable;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
|
||||
class ReleaseNameFixed
|
||||
{
|
||||
use Dispatchable;
|
||||
use SerializesModels;
|
||||
|
||||
public function __construct(
|
||||
public readonly int $releaseId,
|
||||
public readonly string $oldName,
|
||||
public readonly string $newName,
|
||||
public readonly int $oldCategoryId,
|
||||
public readonly int|string $groupId,
|
||||
public readonly string $poster = '',
|
||||
) {}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Listeners;
|
||||
|
||||
use App\Events\ReleaseNameFixed;
|
||||
use App\Models\Release;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class RecategorizeReleaseAfterNameFix
|
||||
{
|
||||
public function __construct(private readonly CategorizationService $categorization) {}
|
||||
|
||||
public function handle(ReleaseNameFixed $event): void
|
||||
{
|
||||
$release = Release::query()->find($event->releaseId, [
|
||||
'id',
|
||||
'groups_id',
|
||||
'fromname',
|
||||
'categories_id',
|
||||
'iscategorized',
|
||||
'searchname',
|
||||
]);
|
||||
|
||||
if ($release === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$result = $this->categorization->determineCategory(
|
||||
$release->groups_id,
|
||||
$event->newName,
|
||||
(string) ($release->fromname ?? $event->poster)
|
||||
);
|
||||
|
||||
$newCategoryId = (int) ($result['categories_id'] ?? $release->categories_id);
|
||||
|
||||
if ((int) $release->categories_id === $newCategoryId && (int) $release->iscategorized === 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
Release::query()
|
||||
->where('id', $release->id)
|
||||
->update([
|
||||
'categories_id' => $newCategoryId,
|
||||
'iscategorized' => 1,
|
||||
]);
|
||||
|
||||
if (config('nntmux.categorization.log', false)) {
|
||||
Log::info('categorization.rename_recategorized', [
|
||||
'release_id' => $release->id,
|
||||
'old_name' => $event->oldName,
|
||||
'new_name' => $event->newName,
|
||||
'old_category_id' => $event->oldCategoryId,
|
||||
'new_category_id' => $newCategoryId,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ declare(strict_types=1);
|
||||
|
||||
namespace App\Providers;
|
||||
|
||||
use App\Events\ReleaseNameFixed;
|
||||
use App\Listeners\RecategorizeReleaseAfterNameFix;
|
||||
use App\Models\AnidbInfo;
|
||||
use App\Models\AnidbTitle;
|
||||
use App\Models\BookInfo;
|
||||
@@ -59,6 +61,7 @@ class AppServiceProvider extends ServiceProvider
|
||||
return $user->hasRole('Admin');
|
||||
});
|
||||
Event::listen(Login::class, LoginViaRemember::class);
|
||||
Event::listen(ReleaseNameFixed::class, RecategorizeReleaseAfterNameFix::class);
|
||||
|
||||
// Register observers
|
||||
RolePromotion::observe(RolePromotionObserver::class);
|
||||
|
||||
@@ -10,6 +10,7 @@ use App\Services\Categorization\Pipes\AbstractCategorizationPipe;
|
||||
use App\Services\Categorization\Pipes\CategorizationPassable;
|
||||
use Illuminate\Pipeline\Pipeline;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Pipeline-based categorization service using Laravel Pipeline.
|
||||
@@ -88,9 +89,34 @@ class CategorizationPipeline
|
||||
->through($this->pipes->values()->all())
|
||||
->thenReturn();
|
||||
|
||||
$this->logCategorization($result);
|
||||
|
||||
return $result->toArray();
|
||||
}
|
||||
|
||||
protected function logCategorization(CategorizationPassable $result): void
|
||||
{
|
||||
if (! config('nntmux.categorization.log', false)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$payload = [
|
||||
'release_name' => $result->context->releaseName,
|
||||
'group_name' => $result->context->groupName,
|
||||
'category_id' => $result->bestResult->categoryId,
|
||||
'matched_by' => $result->bestResult->matchedBy,
|
||||
'confidence' => $result->bestResult->confidence,
|
||||
'locked_to_misc' => $result->lockedToMisc,
|
||||
'misc_analysis' => $result->miscAnalysis,
|
||||
];
|
||||
|
||||
if ($result->lockedToMisc || $result->bestResult->matchedBy === 'group_only_low_signal') {
|
||||
Log::info('categorization.decision', $payload);
|
||||
}
|
||||
|
||||
Log::debug('categorization.trace', $payload + ['all_results' => $result->allResults]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered categorizers (pipes).
|
||||
*
|
||||
@@ -107,6 +133,7 @@ class CategorizationPipeline
|
||||
public static function createDefault(): self
|
||||
{
|
||||
return new self([
|
||||
new Pipes\MiscPipe,
|
||||
new Pipes\GroupNamePipe,
|
||||
new Pipes\XxxPipe,
|
||||
new Pipes\TvPipe,
|
||||
@@ -115,7 +142,7 @@ class CategorizationPipeline
|
||||
new Pipes\MusicPipe,
|
||||
new Pipes\PcPipe,
|
||||
new Pipes\ConsolePipe,
|
||||
new Pipes\MiscPipe,
|
||||
new Pipes\MiscSafetyNetPipe,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,11 @@ class CategorizationResult
|
||||
*/
|
||||
public function shouldOverride(CategorizationResult $other): bool
|
||||
{
|
||||
if ($this->categoryId !== Category::OTHER_MISC && $this->categoryId !== Category::OTHER_HASHED &&
|
||||
$other->isProtectedMiscResult() && $this->confidence < 0.6) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Higher confidence always wins
|
||||
if ($this->confidence > $other->confidence) {
|
||||
return true;
|
||||
@@ -64,6 +69,25 @@ class CategorizationResult
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether this result should resist weak downstream overrides.
|
||||
*/
|
||||
public function isProtectedMiscResult(): bool
|
||||
{
|
||||
if ($this->categoryId === Category::OTHER_HASHED) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if ($this->categoryId !== Category::OTHER_MISC) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return str_starts_with($this->matchedBy, 'hash_')
|
||||
|| str_starts_with($this->matchedBy, 'obfuscated_')
|
||||
|| str_starts_with($this->matchedBy, 'gibberish_')
|
||||
|| $this->matchedBy === 'group_only_low_signal';
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a failed/empty result.
|
||||
*/
|
||||
|
||||
@@ -24,25 +24,25 @@ class GroupNameCategorizer extends AbstractCategorizer
|
||||
return $this->noMatch();
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(tv|hdtv|tvseries)/i', $groupName)) {
|
||||
return $this->matched(Category::TV_OTHER, 0.6, 'group_tv');
|
||||
return $this->matched(Category::TV_OTHER, 0.6, 'group_name_tv');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(movies?|dvd|bluray|x264)/i', $groupName)) {
|
||||
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_movie');
|
||||
return $this->matched(Category::MOVIE_OTHER, 0.6, 'group_name_movie');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(erotica|pictures\.erotica|xxx)/i', $groupName)) {
|
||||
return $this->matched(Category::XXX_OTHER, 0.7, 'group_xxx');
|
||||
return $this->matched(Category::XXX_OTHER, 0.7, 'group_name_xxx');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(sounds?|mp3|music|lossless)/i', $groupName)) {
|
||||
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_music');
|
||||
return $this->matched(Category::MUSIC_OTHER, 0.6, 'group_name_music');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(games?|console|psx|nintendo)/i', $groupName)) {
|
||||
return $this->matched(Category::GAME_OTHER, 0.6, 'group_game');
|
||||
return $this->matched(Category::GAME_OTHER, 0.6, 'group_name_game');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(warez|0day|apps?|software)/i', $groupName)) {
|
||||
return $this->matched(Category::PC_0DAY, 0.6, 'group_pc');
|
||||
return $this->matched(Category::PC_0DAY, 0.6, 'group_name_pc');
|
||||
}
|
||||
if (preg_match('/alt\.binaries\..*?(e-?book|ebook|comics?)/i', $groupName)) {
|
||||
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_book');
|
||||
return $this->matched(Category::BOOKS_EBOOK, 0.5, 'group_name_book');
|
||||
}
|
||||
|
||||
return $this->noMatch();
|
||||
|
||||
@@ -34,11 +34,21 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
return $result;
|
||||
}
|
||||
|
||||
$analysis = $this->inspectSignals($name);
|
||||
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
|
||||
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
|
||||
}
|
||||
|
||||
// Check for obfuscated/encoded patterns
|
||||
if ($result = $this->checkObfuscated($name)) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
// Check low-signal names that only contain random-looking tokens
|
||||
if ($result = $this->checkLowSignal($name)) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
// Check for gibberish patterns (character-analysis heuristics)
|
||||
if ($result = $this->checkGibberish($name)) {
|
||||
return $result;
|
||||
@@ -57,6 +67,50 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
return $this->noMatch();
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect a release name for media signal markers used by the safety-net pipe.
|
||||
*
|
||||
* @return array{coreName: string, coreLength: int, signalScore: int, markers: list<string>, lowSignal: bool}
|
||||
*/
|
||||
public function inspectSignals(ReleaseContext|string $context): array
|
||||
{
|
||||
$name = $context instanceof ReleaseContext ? $context->releaseName : $context;
|
||||
$cleaned = $this->stripExtensionsForAnalysis($name);
|
||||
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
|
||||
|
||||
$patterns = [
|
||||
'season_episode' => '/\bS\d{1,3}[._ -]?E\d{1,4}\b/i',
|
||||
'season_pack' => '/\bS\d{1,3}\b/i',
|
||||
'resolution' => '/\b(480p|576p|720p|1080[pi]?|2160p|4k|uhd)\b/i',
|
||||
'codec' => '/\b(x264|x265|h\.?264|h\.?265|hevc|xvid|av1)\b/i',
|
||||
'source' => '/\b(bluray|bdrip|brrip|hdtv|web[._ -]?dl|web[._ -]?rip|dvdrip|remux)\b/i',
|
||||
'audio' => '/\b(aac|ac3|ddp|dts|flac|mp3)\b/i',
|
||||
'scene_tag' => '/\b(proper|repack|internal|limited|complete|dubbed|subbed|readnfo)\b/i',
|
||||
'year' => '/\b(19|20)\d{2}\b/',
|
||||
'release_group' => '/-[A-Za-z0-9][A-Za-z0-9._-]{1,20}$/',
|
||||
'known_extension' => '/\.(mkv|avi|mp4|mp3|flac|iso|epub|pdf|exe|nzb|rar|7z)$/i',
|
||||
];
|
||||
|
||||
$markers = [];
|
||||
foreach ($patterns as $marker => $pattern) {
|
||||
if (preg_match($pattern, $name)) {
|
||||
$markers[] = $marker;
|
||||
}
|
||||
}
|
||||
|
||||
$signalScore = count($markers);
|
||||
$isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
|
||||
$lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12;
|
||||
|
||||
return [
|
||||
'coreName' => $coreName,
|
||||
'coreLength' => strlen($coreName),
|
||||
'signalScore' => $signalScore,
|
||||
'markers' => $markers,
|
||||
'lowSignal' => $lowSignal,
|
||||
];
|
||||
}
|
||||
|
||||
protected function checkHash(string $name): ?CategorizationResult
|
||||
{
|
||||
// MD5 hash (32 hex characters)
|
||||
@@ -79,6 +133,10 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
return $this->matched(Category::OTHER_HASHED, 0.95, 'hash_generic');
|
||||
}
|
||||
|
||||
if ($this->isBase64LikeToken($name)) {
|
||||
return $this->matched(Category::OTHER_HASHED, 0.9, 'hash_base64_like');
|
||||
}
|
||||
|
||||
// Strip extensions and separators for core-name checks
|
||||
$cleaned = $this->stripExtensionsForAnalysis($name);
|
||||
$coreName = $this->getCoreNameWithoutSeparators($cleaned);
|
||||
@@ -115,7 +173,18 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
|
||||
// Only punctuation and numbers with no clear structure
|
||||
if ($this->isObfuscatedPunctuation($name)) {
|
||||
return $this->matched(Category::OTHER_MISC, 0.5, 'obfuscated_pattern');
|
||||
$analysis = $this->inspectSignals($name);
|
||||
$hashLike = $this->isBase64LikeToken($name)
|
||||
|| $this->isBoundedGenericHash($name)
|
||||
|| $this->isZeroVowelLongToken($analysis['coreName'], 12)
|
||||
|| $analysis['lowSignal'];
|
||||
|
||||
return $this->matched(
|
||||
$hashLike ? Category::OTHER_HASHED : Category::OTHER_MISC,
|
||||
$hashLike ? 0.75 : 0.5,
|
||||
'obfuscated_pattern',
|
||||
['signal_score' => $analysis['signalScore'], 'markers' => $analysis['markers']]
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -146,6 +215,34 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
return $this->matched(Category::OTHER_HASHED, 0.7, 'gibberish_random_digits');
|
||||
}
|
||||
|
||||
if ($this->isZeroVowelLongToken($coreName)) {
|
||||
return $this->matched(Category::OTHER_HASHED, 0.78, 'gibberish_zero_vowels');
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
protected function checkLowSignal(string $name): ?CategorizationResult
|
||||
{
|
||||
$analysis = $this->inspectSignals($name);
|
||||
|
||||
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($analysis['lowSignal'] && $analysis['coreLength'] >= 20) {
|
||||
return $this->matched(
|
||||
Category::OTHER_HASHED,
|
||||
0.8,
|
||||
'gibberish_no_signal',
|
||||
[
|
||||
'signal_score' => $analysis['signalScore'],
|
||||
'markers' => $analysis['markers'],
|
||||
'core_length' => $analysis['coreLength'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,13 @@ class CategorizationPassable
|
||||
*/
|
||||
public array $allResults = [];
|
||||
|
||||
/**
|
||||
* Misc signal analysis captured by MiscPipe for downstream safety-net decisions.
|
||||
*
|
||||
* @var array<string, mixed>
|
||||
*/
|
||||
public array $miscAnalysis = [];
|
||||
|
||||
public function __construct(ReleaseContext $context, bool $debug = false)
|
||||
{
|
||||
$this->context = $context;
|
||||
@@ -90,6 +97,7 @@ class CategorizationPassable
|
||||
'locked_to_misc' => $this->lockedToMisc,
|
||||
'release_name' => $this->context->releaseName,
|
||||
'group_name' => $this->context->groupName,
|
||||
'misc_analysis' => $this->miscAnalysis,
|
||||
'all_results' => $this->allResults,
|
||||
'categorizer_details' => $this->bestResult->debug,
|
||||
];
|
||||
|
||||
@@ -50,6 +50,7 @@ class MiscPipe extends AbstractCategorizationPipe
|
||||
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
|
||||
{
|
||||
// MiscPipe always runs regardless of shouldStopProcessing — it IS the first pipe.
|
||||
$passable->miscAnalysis = $this->categorizer->inspectSignals($passable->context);
|
||||
$result = $this->categorize($passable->context);
|
||||
|
||||
// Record the result for debug
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Categorization\Pipes;
|
||||
|
||||
use App\Models\Category;
|
||||
use App\Services\Categorization\CategorizationResult;
|
||||
use App\Services\Categorization\ReleaseContext;
|
||||
use Closure;
|
||||
|
||||
/**
|
||||
* Final guardrail for low-signal names.
|
||||
*
|
||||
* If a release has no meaningful media markers and only matched because of the
|
||||
* group name, prefer OTHER_MISC over trusting the group alone.
|
||||
*/
|
||||
class MiscSafetyNetPipe extends AbstractCategorizationPipe
|
||||
{
|
||||
protected int $priority = 100;
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return 'MiscSafetyNet';
|
||||
}
|
||||
|
||||
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
|
||||
{
|
||||
if ($this->shouldDowngradeToMisc($passable)) {
|
||||
$result = new CategorizationResult(
|
||||
Category::OTHER_MISC,
|
||||
0.65,
|
||||
'group_only_low_signal',
|
||||
[
|
||||
'group_match' => $passable->bestResult->matchedBy,
|
||||
'misc_analysis' => $passable->miscAnalysis,
|
||||
]
|
||||
);
|
||||
|
||||
if ($passable->debug) {
|
||||
$passable->allResults[$this->getName()] = [
|
||||
'category_id' => $result->categoryId,
|
||||
'confidence' => $result->confidence,
|
||||
'matched_by' => $result->matchedBy,
|
||||
];
|
||||
}
|
||||
|
||||
$passable->bestResult = $result;
|
||||
$passable->lockToMisc();
|
||||
}
|
||||
|
||||
return $next($passable);
|
||||
}
|
||||
|
||||
protected function categorize(ReleaseContext $context): CategorizationResult
|
||||
{
|
||||
return $this->noMatch();
|
||||
}
|
||||
|
||||
private function shouldDowngradeToMisc(CategorizationPassable $passable): bool
|
||||
{
|
||||
return ! $passable->lockedToMisc
|
||||
&& ($passable->miscAnalysis['lowSignal'] ?? false) === true
|
||||
&& str_starts_with($passable->bestResult->matchedBy, 'group_name_');
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ declare(strict_types=1);
|
||||
|
||||
namespace App\Services\NameFixing;
|
||||
|
||||
use App\Events\ReleaseNameFixed;
|
||||
use App\Facades\Search;
|
||||
use App\Models\Category;
|
||||
use App\Models\Predb;
|
||||
@@ -12,6 +13,7 @@ use App\Models\UsenetGroup;
|
||||
use App\Services\Categorization\CategorizationService;
|
||||
use App\Services\ReleaseCleaningService;
|
||||
use Illuminate\Support\Arr;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
/**
|
||||
* Service for updating releases with new names.
|
||||
@@ -153,12 +155,6 @@ class ReleaseUpdateService
|
||||
$this->matched = true;
|
||||
$this->relid = (int) $release->releases_id;
|
||||
|
||||
$determinedCategory = $this->category->determineCategory(
|
||||
$release->groups_id,
|
||||
$newName,
|
||||
! empty($release->fromname) ? $release->fromname : ''
|
||||
);
|
||||
|
||||
if ($type === 'PAR2, ') {
|
||||
$newName = ucwords($newName);
|
||||
if (preg_match('/(.+?)\.[a-z0-9]{2,3}(PAR2)?$/i', $name, $hit)) {
|
||||
@@ -172,14 +168,21 @@ class ReleaseUpdateService
|
||||
$newName = explode('\\', $newName);
|
||||
$newName = preg_replace(['/^[=_.:\s-]+/', '/[=_.:\s-]+$/'], '', $newName[0]);
|
||||
|
||||
if ($this->echoOutput && $show) {
|
||||
$this->echoReleaseInfo($release, $newName, $determinedCategory, $type, $method);
|
||||
}
|
||||
|
||||
$newTitle = substr($newName, 0, 299);
|
||||
|
||||
$determinedCategory = null;
|
||||
if ($this->echoOutput && $show) {
|
||||
$determinedCategory = $this->category->determineCategory(
|
||||
$release->groups_id,
|
||||
$newTitle,
|
||||
! empty($release->fromname) ? $release->fromname : ''
|
||||
);
|
||||
|
||||
$this->echoReleaseInfo($release, $newTitle, $determinedCategory, $type, $method);
|
||||
}
|
||||
|
||||
if ($echo === true) {
|
||||
$this->performDatabaseUpdate($release, $newTitle, $determinedCategory, $type, $nameStatus, $preId);
|
||||
$this->performDatabaseUpdate($release, $newTitle, $type, $nameStatus, $preId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -253,59 +256,65 @@ class ReleaseUpdateService
|
||||
|
||||
/**
|
||||
* Perform the actual database update.
|
||||
*
|
||||
* @param array<string, mixed> $determinedCategory
|
||||
*/
|
||||
protected function performDatabaseUpdate(
|
||||
object $release,
|
||||
string $newTitle,
|
||||
array $determinedCategory,
|
||||
string $type,
|
||||
bool $nameStatus,
|
||||
int $preId
|
||||
): void {
|
||||
if ($nameStatus === true) {
|
||||
$status = $this->getStatusColumnsForType($type);
|
||||
DB::transaction(function () use ($release, $newTitle, $type, $nameStatus, $preId): void {
|
||||
if ($nameStatus === true) {
|
||||
$status = $this->getStatusColumnsForType($type);
|
||||
|
||||
$updateColumns = [
|
||||
'videos_id' => 0,
|
||||
'tv_episodes_id' => 0,
|
||||
'imdbid' => null,
|
||||
'musicinfo_id' => '',
|
||||
'consoleinfo_id' => '',
|
||||
'bookinfo_id' => '',
|
||||
'anidbid' => '',
|
||||
'predb_id' => $preId,
|
||||
'searchname' => $newTitle,
|
||||
'categories_id' => $determinedCategory['categories_id'],
|
||||
];
|
||||
|
||||
if (! empty($status)) {
|
||||
foreach ($status as $key => $stat) {
|
||||
$updateColumns = Arr::add($updateColumns, $key, $stat);
|
||||
}
|
||||
}
|
||||
|
||||
Release::query()
|
||||
->where('id', $release->releases_id)
|
||||
->update($updateColumns);
|
||||
} else {
|
||||
Release::query()
|
||||
->where('id', $release->releases_id)
|
||||
->update([
|
||||
$updateColumns = [
|
||||
'videos_id' => 0,
|
||||
'tv_episodes_id' => 0,
|
||||
'imdbid' => null,
|
||||
'musicinfo_id' => null,
|
||||
'consoleinfo_id' => null,
|
||||
'bookinfo_id' => null,
|
||||
'anidbid' => null,
|
||||
'musicinfo_id' => '',
|
||||
'consoleinfo_id' => '',
|
||||
'bookinfo_id' => '',
|
||||
'anidbid' => '',
|
||||
'predb_id' => $preId,
|
||||
'searchname' => $newTitle,
|
||||
'categories_id' => $determinedCategory['categories_id'],
|
||||
'iscategorized' => 1,
|
||||
]);
|
||||
}
|
||||
];
|
||||
|
||||
if (! empty($status)) {
|
||||
foreach ($status as $key => $stat) {
|
||||
$updateColumns = Arr::add($updateColumns, $key, $stat);
|
||||
}
|
||||
}
|
||||
|
||||
Release::query()
|
||||
->where('id', $release->releases_id)
|
||||
->update($updateColumns);
|
||||
} else {
|
||||
Release::query()
|
||||
->where('id', $release->releases_id)
|
||||
->update([
|
||||
'videos_id' => 0,
|
||||
'tv_episodes_id' => 0,
|
||||
'imdbid' => null,
|
||||
'musicinfo_id' => null,
|
||||
'consoleinfo_id' => null,
|
||||
'bookinfo_id' => null,
|
||||
'anidbid' => null,
|
||||
'predb_id' => $preId,
|
||||
'searchname' => $newTitle,
|
||||
'iscategorized' => 1,
|
||||
]);
|
||||
}
|
||||
|
||||
event(new ReleaseNameFixed(
|
||||
(int) $release->releases_id,
|
||||
(string) $release->searchname,
|
||||
$newTitle,
|
||||
(int) $release->categories_id,
|
||||
$release->groups_id,
|
||||
(string) ($release->fromname ?? '')
|
||||
));
|
||||
});
|
||||
|
||||
// Update search index
|
||||
Search::updateRelease($release->releases_id);
|
||||
@@ -350,7 +359,7 @@ class ReleaseUpdateService
|
||||
*/
|
||||
public function checkPreDbMatch(object $release, string $textstring): ?array
|
||||
{
|
||||
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s\:/i', $textstring)) {
|
||||
if (preg_match_all(self::PREDB_REGEX, $textstring, $hits) && ! preg_match('/Source\s:/i', $textstring)) {
|
||||
foreach ($hits as $hit) {
|
||||
foreach ($hit as $val) {
|
||||
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
|
||||
|
||||
@@ -29,11 +29,13 @@ trait DetectsHashedNames
|
||||
|| $this->isBoundedSha1Hash($name)
|
||||
|| $this->isBoundedSha256Hash($name)
|
||||
|| $this->isBoundedGenericHash($name)
|
||||
|| $this->isBase64LikeToken($name)
|
||||
|| $this->isObfuscatedUppercaseString($name)
|
||||
|| $this->isObfuscatedMixedAlphanumeric($name)
|
||||
|| $this->isObfuscatedUsenetFilename($name)
|
||||
|| $this->isRandomByCharacterAnalysis($coreName, $name)
|
||||
|| $this->hasInsufficientWordStructure($coreName)
|
||||
|| $this->isZeroVowelLongToken($coreName)
|
||||
|| $this->isRandomDigitPattern($coreName);
|
||||
}
|
||||
|
||||
@@ -70,7 +72,40 @@ trait DetectsHashedNames
|
||||
*/
|
||||
protected function isBoundedGenericHash(string $name): bool
|
||||
{
|
||||
return (bool) preg_match('/(?:^|["\'\s\[\]\/\-])([a-f0-9]{32,128})(?:["\'\s\[\]\/.\-]|$)/i', $name);
|
||||
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([a-z0-9]{24,128})(?:["\'\s\[\]\/.\-]|$)/i', $name, $matches)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$token = $matches[1];
|
||||
|
||||
if (preg_match('/\b(19|20)\d{2}\b/', $token)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return preg_match('/\d/', $token) === 1
|
||||
&& $this->looksLikeRandomString($token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect base64/base64url-like tokens that commonly appear in obfuscated subjects.
|
||||
*/
|
||||
protected function isBase64LikeToken(string $name): bool
|
||||
{
|
||||
if (! preg_match('/(?:^|["\'\s\[\]\/\-])([A-Za-z0-9+\/_=-]{24,})(?:["\'\s\[\]\/.\-]|$)/', $name, $matches)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$token = rtrim($matches[1], '=');
|
||||
|
||||
if (strlen($token) < 24 || preg_match('/\b(19|20)\d{2}\b/', $token)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (! preg_match('/[A-Z]/', $token) || ! preg_match('/[a-z]/', $token) || ! preg_match('/\d/', $token)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $this->looksLikeRandomString($token);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
@@ -207,6 +242,28 @@ trait DetectsHashedNames
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect long alphanumeric tokens with effectively no vowel structure.
|
||||
*/
|
||||
protected function isZeroVowelLongToken(string $coreName, int $minLength = 20): bool
|
||||
{
|
||||
if (strlen($coreName) < $minLength || ! preg_match('/^[a-zA-Z0-9]+$/', $coreName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (preg_match('/\b(19|20)\d{2}\b/', $coreName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$letterOnly = preg_replace('/\d+/', '', $coreName);
|
||||
|
||||
if ($letterOnly === '' || strlen($letterOnly) < 10) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return preg_match('/[aeiou]/i', $letterOnly) !== 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a string looks like a random/obfuscated string rather than a real title.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user