Update other categorization

This commit is contained in:
DariusIII
2026-04-21 22:13:34 +02:00
parent bd7f540c23
commit bbc54feaf2
5 changed files with 113 additions and 2 deletions
@@ -100,7 +100,10 @@ class MiscCategorizer extends AbstractCategorizer
$signalScore = count($markers); $signalScore = count($markers);
$isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1; $isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
$lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12; $lowSignal = $signalScore === 0
&& $isCoreToken
&& strlen($coreName) >= 12
&& ! $this->hasStrongWordStructure($name, $coreName);
return [ return [
'coreName' => $coreName, 'coreName' => $coreName,
@@ -226,6 +229,10 @@ class MiscCategorizer extends AbstractCategorizer
{ {
$analysis = $this->inspectSignals($name); $analysis = $this->inspectSignals($name);
if ($this->hasStrongWordStructure($name, $analysis['coreName'])) {
return null;
}
if ($this->isZeroVowelLongToken($analysis['coreName'])) { if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return null; return null;
} }
@@ -246,6 +253,23 @@ class MiscCategorizer extends AbstractCategorizer
return null; return null;
} }
protected function hasStrongWordStructure(string $name, string $coreName): bool
{
return $this->getMaxConsecutiveLetters($coreName) >= 5
&& $this->hasNormalVowelRatio($coreName)
&& $this->countAlphabeticWordTokens($name) >= 2;
}
protected function countAlphabeticWordTokens(string $name): int
{
$tokens = preg_split('/[.\s_-]+/', $this->stripExtensionsForAnalysis($name)) ?: [];
return count(array_filter(
$tokens,
static fn (string $token): bool => preg_match('/[a-z]{3,}/i', $token) === 1
));
}
protected function checkArchive(string $name): ?CategorizationResult protected function checkArchive(string $name): ?CategorizationResult
{ {
if (preg_match('/\.(zip|rar|7z|tar|gz|bz2|xz|tgz|tbz2|cab|iso|img|dmg|pkg|archive)$/i', $name)) { if (preg_match('/\.(zip|rar|7z|tar|gz|bz2|xz|tgz|tbz2|cab|iso|img|dmg|pkg|archive)$/i', $name)) {
@@ -148,6 +148,10 @@ class PcCategorizer extends AbstractCategorizer
protected function check0day(string $name): ?CategorizationResult protected function check0day(string $name): ?CategorizationResult
{ {
if (preg_match('/\.(msix|msixbundle|appx|appxbundle|msi)$/i', $name)) {
return $this->matched(Category::PC_0DAY, 0.9, '0day_msix_installer');
}
// Explicit 0day indicators // Explicit 0day indicators
if (preg_match('/[._ -]exe$|[._ -](utorrent|Virtualbox)[._ -]|\b0DAY\b|incl.+crack| DRM$|>DRM</i', $name)) { if (preg_match('/[._ -]exe$|[._ -](utorrent|Virtualbox)[._ -]|\b0DAY\b|incl.+crack| DRM$|>DRM</i', $name)) {
return $this->matched(Category::PC_0DAY, 0.9, '0day_explicit'); return $this->matched(Category::PC_0DAY, 0.9, '0day_explicit');
+18 -1
View File
@@ -264,6 +264,23 @@ trait DetectsHashedNames
return preg_match('/[aeiou]/i', $letterOnly) !== 1; return preg_match('/[aeiou]/i', $letterOnly) !== 1;
} }
/**
* Check whether the letter-only portion of a token has a normal vowel ratio.
*/
protected function hasNormalVowelRatio(string $str, float $min = 0.2): bool
{
$letters = preg_replace('/[^a-z]/i', '', $str);
if ($letters === '' || strlen($letters) < 5) {
return false;
}
preg_match_all('/[aeiou]/i', $letters, $matches);
$vowelCount = count($matches[0]);
return ($vowelCount / strlen($letters)) >= $min;
}
/** /**
* Check if a string looks like a random/obfuscated string rather than a real title. * Check if a string looks like a random/obfuscated string rather than a real title.
* *
@@ -315,7 +332,7 @@ trait DetectsHashedNames
protected function stripExtensionsForAnalysis(string $name): string protected function stripExtensionsForAnalysis(string $name): string
{ {
return preg_replace( return preg_replace(
'/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001)$/i', '/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001|msix|msixbundle|appx|appxbundle|apk|xap|ipa|deb|rpm|pkg|dmg|exe|msi)$/i',
'', '',
trim($name) trim($name)
); );
+30
View File
@@ -7,10 +7,23 @@ namespace Tests\Unit;
use App\Models\Category; use App\Models\Category;
use App\Services\Categorization\Categorizers\PcCategorizer; use App\Services\Categorization\Categorizers\PcCategorizer;
use App\Services\Categorization\ReleaseContext; use App\Services\Categorization\ReleaseContext;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
class CategorizePcGameTest extends TestCase class CategorizePcGameTest extends TestCase
{ {
/**
* @return array<string, array{0: string}>
*/
public static function installerPackageProvider(): array
{
return [
'msix bundle' => ['Adobe Express Photos.Msixbundle'],
'appx bundle' => ['Microsoft.To.Do.appxbundle'],
'msi package' => ['Utility Installer.msi'],
];
}
private PcCategorizer $categorizer; private PcCategorizer $categorizer;
protected function setUp(): void protected function setUp(): void
@@ -96,4 +109,21 @@ class CategorizePcGameTest extends TestCase
} }
} }
} }
#[DataProvider('installerPackageProvider')]
public function test_installer_packages_are_classified_as_pc_0day(string $name): void
{
$context = new ReleaseContext(
releaseName: $name,
groupId: 0,
groupName: '',
poster: ''
);
$result = $this->categorizer->categorize($context);
$this->assertTrue($result->isSuccessful(), "Expected PC 0day match for installer package: $name");
$this->assertSame(Category::PC_0DAY, $result->categoryId, "Expected PC_0DAY for installer package: $name");
$this->assertSame('0day_msix_installer', $result->matchedBy);
}
} }
@@ -132,6 +132,21 @@ class HashedReleaseCategorizationTest extends TestCase
'TV episode' => ['Show.Name.S03E05.720p.HDTV.x264-GROUP', 'alt.binaries.hdtv'], 'TV episode' => ['Show.Name.S03E05.720p.HDTV.x264-GROUP', 'alt.binaries.hdtv'],
'Music album' => ['Artist.Name-Album.Title-2024-FLAC-GROUP', 'alt.binaries.sounds.mp3'], 'Music album' => ['Artist.Name-Album.Title-2024-FLAC-GROUP', 'alt.binaries.sounds.mp3'],
'Game release' => ['Starfield-RUNE', 'alt.binaries.games'], 'Game release' => ['Starfield-RUNE', 'alt.binaries.games'],
'Readable software package' => ['Microsoft Office Suite Installer', 'alt.binaries.warez'],
'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx'],
];
}
/**
* Readable software-like names that must not be classified as hashed by the misc categorizer.
*
* @return array<string, array{0: string}>
*/
public static function readableSoftwareNamesProvider(): array
{
return [
'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle'],
'Office installer words' => ['Microsoft Office Suite Installer'],
]; ];
} }
@@ -202,6 +217,18 @@ class HashedReleaseCategorizationTest extends TestCase
$this->assertSame(Category::OTHER_HASHED, $result->categoryId, "Expected OTHER_HASHED for: $name"); $this->assertSame(Category::OTHER_HASHED, $result->categoryId, "Expected OTHER_HASHED for: $name");
} }
#[DataProvider('readableSoftwareNamesProvider')]
public function test_misc_categorizer_does_not_hash_readable_software_names(string $name): void
{
$categorizer = new MiscCategorizer;
$context = new ReleaseContext(releaseName: $name, groupId: 0);
$result = $categorizer->categorize($context);
$this->assertFalse($result->isSuccessful(), "Readable software name '$name' should not be matched by misc hash heuristics");
$this->assertSame(Category::OTHER_MISC, $result->categoryId);
$this->assertSame('no_match', $result->matchedBy);
}
// ------------------------------------------------------------------ // ------------------------------------------------------------------
// Tests: MiscPipe lock mechanism // Tests: MiscPipe lock mechanism
// ------------------------------------------------------------------ // ------------------------------------------------------------------
@@ -289,6 +316,15 @@ class HashedReleaseCategorizationTest extends TestCase
); );
} }
public function test_adobe_msix_bundle_reaches_pc_0day_in_full_pipeline(): void
{
$passable = $this->runPipeline('Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx');
$this->assertFalse($passable->lockedToMisc);
$this->assertSame(Category::PC_0DAY, $passable->bestResult->categoryId);
$this->assertSame('0day_msix_installer', $passable->bestResult->matchedBy);
}
// ------------------------------------------------------------------ // ------------------------------------------------------------------
// Tests: shouldStopProcessing() respects the lock // Tests: shouldStopProcessing() respects the lock
// ------------------------------------------------------------------ // ------------------------------------------------------------------