Update other categorization

This commit is contained in:
DariusIII
2026-04-21 22:13:34 +02:00
parent bd7f540c23
commit bbc54feaf2
5 changed files with 113 additions and 2 deletions
@@ -100,7 +100,10 @@ class MiscCategorizer extends AbstractCategorizer
$signalScore = count($markers);
$isCoreToken = preg_match('/^[A-Za-z0-9+\/_=-]+$/', $coreName) === 1;
$lowSignal = $signalScore === 0 && $isCoreToken && strlen($coreName) >= 12;
$lowSignal = $signalScore === 0
&& $isCoreToken
&& strlen($coreName) >= 12
&& ! $this->hasStrongWordStructure($name, $coreName);
return [
'coreName' => $coreName,
@@ -226,6 +229,10 @@ class MiscCategorizer extends AbstractCategorizer
{
$analysis = $this->inspectSignals($name);
if ($this->hasStrongWordStructure($name, $analysis['coreName'])) {
return null;
}
if ($this->isZeroVowelLongToken($analysis['coreName'])) {
return null;
}
@@ -246,6 +253,23 @@ class MiscCategorizer extends AbstractCategorizer
return null;
}
protected function hasStrongWordStructure(string $name, string $coreName): bool
{
return $this->getMaxConsecutiveLetters($coreName) >= 5
&& $this->hasNormalVowelRatio($coreName)
&& $this->countAlphabeticWordTokens($name) >= 2;
}
protected function countAlphabeticWordTokens(string $name): int
{
$tokens = preg_split('/[.\s_-]+/', $this->stripExtensionsForAnalysis($name)) ?: [];
return count(array_filter(
$tokens,
static fn (string $token): bool => preg_match('/[a-z]{3,}/i', $token) === 1
));
}
protected function checkArchive(string $name): ?CategorizationResult
{
if (preg_match('/\.(zip|rar|7z|tar|gz|bz2|xz|tgz|tbz2|cab|iso|img|dmg|pkg|archive)$/i', $name)) {
@@ -148,6 +148,10 @@ class PcCategorizer extends AbstractCategorizer
protected function check0day(string $name): ?CategorizationResult
{
if (preg_match('/\.(msix|msixbundle|appx|appxbundle|msi)$/i', $name)) {
return $this->matched(Category::PC_0DAY, 0.9, '0day_msix_installer');
}
// Explicit 0day indicators
if (preg_match('/[._ -]exe$|[._ -](utorrent|Virtualbox)[._ -]|\b0DAY\b|incl.+crack| DRM$|>DRM</i', $name)) {
return $this->matched(Category::PC_0DAY, 0.9, '0day_explicit');
+18 -1
View File
@@ -264,6 +264,23 @@ trait DetectsHashedNames
return preg_match('/[aeiou]/i', $letterOnly) !== 1;
}
/**
* Check whether the letter-only portion of a token has a normal vowel ratio.
*/
protected function hasNormalVowelRatio(string $str, float $min = 0.2): bool
{
$letters = preg_replace('/[^a-z]/i', '', $str);
if ($letters === '' || strlen($letters) < 5) {
return false;
}
preg_match_all('/[aeiou]/i', $letters, $matches);
$vowelCount = count($matches[0]);
return ($vowelCount / strlen($letters)) >= $min;
}
/**
* Check if a string looks like a random/obfuscated string rather than a real title.
*
@@ -315,7 +332,7 @@ trait DetectsHashedNames
protected function stripExtensionsForAnalysis(string $name): string
{
return preg_replace(
'/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001)$/i',
'/\.(mkv|avi|mp4|m4v|mpg|mpeg|wmv|flv|mov|ts|vob|iso|divx|par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip|7z|gz|tar|001|msix|msixbundle|appx|appxbundle|apk|xap|ipa|deb|rpm|pkg|dmg|exe|msi)$/i',
'',
trim($name)
);
+30
View File
@@ -7,10 +7,23 @@ namespace Tests\Unit;
use App\Models\Category;
use App\Services\Categorization\Categorizers\PcCategorizer;
use App\Services\Categorization\ReleaseContext;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\TestCase;
class CategorizePcGameTest extends TestCase
{
/**
* @return array<string, array{0: string}>
*/
public static function installerPackageProvider(): array
{
return [
'msix bundle' => ['Adobe Express Photos.Msixbundle'],
'appx bundle' => ['Microsoft.To.Do.appxbundle'],
'msi package' => ['Utility Installer.msi'],
];
}
private PcCategorizer $categorizer;
protected function setUp(): void
@@ -96,4 +109,21 @@ class CategorizePcGameTest extends TestCase
}
}
}
#[DataProvider('installerPackageProvider')]
public function test_installer_packages_are_classified_as_pc_0day(string $name): void
{
$context = new ReleaseContext(
releaseName: $name,
groupId: 0,
groupName: '',
poster: ''
);
$result = $this->categorizer->categorize($context);
$this->assertTrue($result->isSuccessful(), "Expected PC 0day match for installer package: $name");
$this->assertSame(Category::PC_0DAY, $result->categoryId, "Expected PC_0DAY for installer package: $name");
$this->assertSame('0day_msix_installer', $result->matchedBy);
}
}
@@ -132,6 +132,21 @@ class HashedReleaseCategorizationTest extends TestCase
'TV episode' => ['Show.Name.S03E05.720p.HDTV.x264-GROUP', 'alt.binaries.hdtv'],
'Music album' => ['Artist.Name-Album.Title-2024-FLAC-GROUP', 'alt.binaries.sounds.mp3'],
'Game release' => ['Starfield-RUNE', 'alt.binaries.games'],
'Readable software package' => ['Microsoft Office Suite Installer', 'alt.binaries.warez'],
'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx'],
];
}
/**
* Readable software-like names that must not be classified as hashed by the misc categorizer.
*
* @return array<string, array{0: string}>
*/
public static function readableSoftwareNamesProvider(): array
{
return [
'Adobe msix bundle' => ['Adobe Express Photos.Msixbundle'],
'Office installer words' => ['Microsoft Office Suite Installer'],
];
}
@@ -202,6 +217,18 @@ class HashedReleaseCategorizationTest extends TestCase
$this->assertSame(Category::OTHER_HASHED, $result->categoryId, "Expected OTHER_HASHED for: $name");
}
#[DataProvider('readableSoftwareNamesProvider')]
public function test_misc_categorizer_does_not_hash_readable_software_names(string $name): void
{
$categorizer = new MiscCategorizer;
$context = new ReleaseContext(releaseName: $name, groupId: 0);
$result = $categorizer->categorize($context);
$this->assertFalse($result->isSuccessful(), "Readable software name '$name' should not be matched by misc hash heuristics");
$this->assertSame(Category::OTHER_MISC, $result->categoryId);
$this->assertSame('no_match', $result->matchedBy);
}
// ------------------------------------------------------------------
// Tests: MiscPipe lock mechanism
// ------------------------------------------------------------------
@@ -289,6 +316,15 @@ class HashedReleaseCategorizationTest extends TestCase
);
}
public function test_adobe_msix_bundle_reaches_pc_0day_in_full_pipeline(): void
{
$passable = $this->runPipeline('Adobe Express Photos.Msixbundle', 'alt.binaries.erotica.divx');
$this->assertFalse($passable->lockedToMisc);
$this->assertSame(Category::PC_0DAY, $passable->bestResult->categoryId);
$this->assertSame('0day_msix_installer', $passable->bestResult->matchedBy);
}
// ------------------------------------------------------------------
// Tests: shouldStopProcessing() respects the lock
// ------------------------------------------------------------------