From bbc54feaf277c9077702c60e862d5b6155ae47a4 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Tue, 21 Apr 2026 22:13:34 +0200 Subject: [PATCH] Update other categorization --- .../Categorizers/MiscCategorizer.php | 26 +++++++++++++- .../Categorizers/PcCategorizer.php | 4 +++ app/Traits/DetectsHashedNames.php | 19 +++++++++- tests/Unit/CategorizePcGameTest.php | 30 ++++++++++++++++ .../Unit/HashedReleaseCategorizationTest.php | 36 +++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/app/Services/Categorization/Categorizers/MiscCategorizer.php b/app/Services/Categorization/Categorizers/MiscCategorizer.php index 2a0a0484a..db63ca852 100644 --- a/app/Services/Categorization/Categorizers/MiscCategorizer.php +++ b/app/Services/Categorization/Categorizers/MiscCategorizer.php @@ -100,7 +100,10 @@ class MiscCategorizer extends AbstractCategorizer $signalScore = count($markers); $isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1; - $lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12; + $lowSignal = $signalScore === 0 + && $isCoreToken + && strlen($coreName) >= 12 + && ! $this->hasStrongWordStructure($name, $coreName); return [ 'coreName' => $coreName, @@ -226,6 +229,10 @@ class MiscCategorizer extends AbstractCategorizer { $analysis = $this->inspectSignals($name); + if ($this->hasStrongWordStructure($name, $analysis['coreName'])) { + return null; + } + if ($this->isZeroVowelLongToken($analysis['coreName'])) { return null; } @@ -246,6 +253,23 @@ class MiscCategorizer extends AbstractCategorizer return null; } + protected function hasStrongWordStructure(string $name, string $coreName): bool + { + return $this->getMaxConsecutiveLetters($coreName) >= 5 + && $this->hasNormalVowelRatio($coreName) + && $this->countAlphabeticWordTokens($name) >= 2; + } + + protected function countAlphabeticWordTokens(string $name): int + { + $tokens = preg_split('/[.\s_-]+/', $this->stripExtensionsForAnalysis($name)) ?: []; + + return count(array_filter( + $tokens, + static fn (string $token): bool => preg_match('/[a-z]{3,}/i', $token) === 1 + )); + } + protected function checkArchive(string $name): ?CategorizationResult { if (preg_match('/\.(zip|rar|7z|tar|gz|bz2|xz|tgz|tbz2|cab|iso|img|dmg|pkg|archive)$/i', $name)) { diff --git a/app/Services/Categorization/Categorizers/PcCategorizer.php b/app/Services/Categorization/Categorizers/PcCategorizer.php index d31b32e02..9a825be62 100644 --- a/app/Services/Categorization/Categorizers/PcCategorizer.php +++ b/app/Services/Categorization/Categorizers/PcCategorizer.php @@ -148,6 +148,10 @@ class PcCategorizer extends AbstractCategorizer protected function check0day(string $name): ?CategorizationResult { + if (preg_match('/\.(msix|msixbundle|appx|appxbundle|msi)$/i', $name)) { + return $this->matched(Category::PC_0DAY, 0.9, '0day_msix_installer'); + } + // Explicit 0day indicators if (preg_match('/[._ -]exe$|[._ -](utorrent|Virtualbox)[._ -]|\b0DAY\b|incl.+crack| DRM$|>DRMmatched(Category::PC_0DAY, 0.9, '0day_explicit'); diff --git a/app/Traits/DetectsHashedNames.php b/app/Traits/DetectsHashedNames.php index 6431a7b75..6e667bc21 100644 --- a/app/Traits/DetectsHashedNames.php +++ b/app/Traits/DetectsHashedNames.php @@ -264,6 +264,23 @@ trait DetectsHashedNames return preg_match('/[aeiou]/i', $letterOnly) !== 1; } + /** + * Check whether the letter-only portion of a token has a normal vowel ratio. + */ + protected function hasNormalVowelRatio(string $str, float $min = 0.2): bool + { + $letters = preg_replace('/[^a-z]/i', '', $str); + + if ($letters === '' || strlen($letters) < 5) { + return false; + } + + preg_match_all('/[aeiou]/i', $letters, $matches); + $vowelCount = count($matches[0]); + + return ($vowelCount / strlen($letters)) >= $min; + } + /** * Check if a string looks like a random/obfuscated string rather than a real title. * @@ -315,7 +332,7 @@ trait DetectsHashedNames protected function stripExtensionsForAnalysis(string $name): string { return preg_replace( - '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001)$/i', + '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001|msix|msixbundle|appx|appxbundle|apk|xap|ipa|deb|rpm|pkg|dmg|exe|msi)$/i', '', trim($name) ); diff --git a/tests/Unit/CategorizePcGameTest.php b/tests/Unit/CategorizePcGameTest.php index e64011231..21f13755b 100644 --- a/tests/Unit/CategorizePcGameTest.php +++ b/tests/Unit/CategorizePcGameTest.php @@ -7,10 +7,23 @@ namespace Tests\Unit; use App\Models\Category; use App\Services\Categorization\Categorizers\PcCategorizer; use App\Services\Categorization\ReleaseContext; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; class CategorizePcGameTest extends TestCase { + /** + * @return array + */ + public static function installerPackageProvider(): array + { + return [ + 'msix bundle' => ['Adobe Express Photos.Msixbundle'], + 'appx bundle' => ['Microsoft.To.Do.appxbundle'], + 'msi package' => ['Utility Installer.msi'], + ]; + } + private PcCategorizer $categorizer; protected function setUp(): void @@ -96,4 +109,21 @@ class CategorizePcGameTest extends TestCase } } } + + #[DataProvider('installerPackageProvider')] + public function test_installer_packages_are_classified_as_pc_0day(string $name): void + { + $context = new ReleaseContext( + releaseName: $name, + groupId: 0, + groupName: '', + poster: '' + ); + + $result = $this->categorizer->categorize($context); + + $this->assertTrue($result->isSuccessful(), "Expected PC 0day match for installer package: $name"); + $this->assertSame(Category::PC_0DAY, $result->categoryId, "Expected PC_0DAY for installer package: $name"); + $this->assertSame('0day_msix_installer', $result->matchedBy); + } } diff --git a/tests/Unit/HashedReleaseCategorizationTest.php b/tests/Unit/HashedReleaseCategorizationTest.php index 0f187ef2d..13aba30f5 100644 --- a/tests/Unit/HashedReleaseCategorizationTest.php +++ b/tests/Unit/HashedReleaseCategorizationTest.php @@ -132,6 +132,21 @@ class HashedReleaseCategorizationTest extends TestCase 'TV episode' => ['Show.Name.S03E05.720p.HDTV.x264-GROUP', 'alt.binaries.hdtv'], 'Music album' => ['Artist.Name-Album.Title-2024-FLAC-GROUP', 'alt.binaries.sounds.mp3'], 'Game release' => ['Starfield-RUNE', 'alt.binaries.games'], + 'Readable software package' => ['Microsoft Office Suite Installer', 'alt.binaries.warez'], + 'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx'], + ]; + } + + /** + * Readable software-like names that must not be classified as hashed by the misc categorizer. + * + * @return array + */ + public static function readableSoftwareNamesProvider(): array + { + return [ + 'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle'], + 'Office installer words' => ['Microsoft Office Suite Installer'], ]; } @@ -202,6 +217,18 @@ class HashedReleaseCategorizationTest extends TestCase $this->assertSame(Category::OTHER_HASHED, $result->categoryId, "Expected OTHER_HASHED for: $name"); } + #[DataProvider('readableSoftwareNamesProvider')] + public function test_misc_categorizer_does_not_hash_readable_software_names(string $name): void + { + $categorizer = new MiscCategorizer; + $context = new ReleaseContext(releaseName: $name, groupId: 0); + $result = $categorizer->categorize($context); + + $this->assertFalse($result->isSuccessful(), "Readable software name '$name' should not be matched by misc hash heuristics"); + $this->assertSame(Category::OTHER_MISC, $result->categoryId); + $this->assertSame('no_match', $result->matchedBy); + } + // ------------------------------------------------------------------ // Tests: MiscPipe lock mechanism // ------------------------------------------------------------------ @@ -289,6 +316,15 @@ class HashedReleaseCategorizationTest extends TestCase ); } + public function test_adobe_msix_bundle_reaches_pc_0day_in_full_pipeline(): void + { + $passable = $this->runPipeline('Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx'); + + $this->assertFalse($passable->lockedToMisc); + $this->assertSame(Category::PC_0DAY, $passable->bestResult->categoryId); + $this->assertSame('0day_msix_installer', $passable->bestResult->matchedBy); + } + // ------------------------------------------------------------------ // Tests: shouldStopProcessing() respects the lock // ------------------------------------------------------------------