Use Laravel Pipeline for categorization

This commit is contained in:
DariusIII
2025-12-10 00:20:19 +01:00
parent 179943db8a
commit 36e5bc2973
13 changed files with 518 additions and 78 deletions
@@ -2,54 +2,56 @@
namespace App\Services\Categorization;
use App\Models\Category;
use App\Models\Settings;
use App\Models\UsenetGroup;
use App\Services\Categorization\Contracts\CategorizerInterface;
use App\Services\Categorization\Pipes\AbstractCategorizationPipe;
use App\Services\Categorization\Pipes\CategorizationPassable;
use Illuminate\Pipeline\Pipeline;
use Illuminate\Support\Collection;
/**
* Pipeline-based categorization service.
* Pipeline-based categorization service using Laravel Pipeline.
*
* This service orchestrates multiple categorizers to determine the best
* category for a release. Each categorizer is responsible for a specific
* category domain and returns a result with a confidence score.
* This service uses Laravel's Pipeline to orchestrate multiple categorizers
* to determine the best category for a release. Each categorizer (pipe) is
* responsible for a specific category domain and returns a result with a
* confidence score.
*/
class CategorizationPipeline
{
/**
* @var Collection<CategorizerInterface>
* @var Collection<AbstractCategorizationPipe>
*/
protected Collection $categorizers;
protected Collection $pipes;
protected bool $categorizeForeign;
protected bool $catWebDL;
/**
* @param iterable<CategorizerInterface> $categorizers
* @param iterable<AbstractCategorizationPipe> $pipes
*/
public function __construct(iterable $categorizers = [])
public function __construct(iterable $pipes = [])
{
$this->categorizers = collect($categorizers)
->sortBy(fn (CategorizerInterface $c) => $c->getPriority());
$this->pipes = collect($pipes)
->sortBy(fn (AbstractCategorizationPipe $p) => $p->getPriority());
$this->categorizeForeign = (bool) Settings::settingValue('categorizeforeign');
$this->catWebDL = (bool) Settings::settingValue('catwebdl');
}
/**
* Register a categorizer in the pipeline.
* Register a categorizer pipe in the pipeline.
*/
public function addCategorizer(CategorizerInterface $categorizer): self
public function addCategorizer(AbstractCategorizationPipe $pipe): self
{
$this->categorizers->push($categorizer);
$this->categorizers = $this->categorizers->sortBy(fn (CategorizerInterface $c) => $c->getPriority());
$this->pipes->push($pipe);
$this->pipes = $this->pipes->sortBy(fn (AbstractCategorizationPipe $p) => $p->getPriority());
return $this;
}
/**
* Determine the category for a release.
* Determine the category for a release using Laravel Pipeline.
*
* @param int|string $groupId The usenet group ID
* @param string $releaseName The name of the release
@@ -74,62 +76,25 @@ class CategorizationPipeline
catWebDL: $this->catWebDL,
);
$bestResult = CategorizationResult::noMatch();
$allResults = [];
$passable = new CategorizationPassable($context, $debug);
foreach ($this->categorizers as $categorizer) {
// Skip if categorizer determines it shouldn't process this release
if ($categorizer->shouldSkip($context)) {
continue;
}
/** @var CategorizationPassable $result */
$result = app(Pipeline::class)
->send($passable)
->through($this->pipes->values()->all())
->thenReturn();
$result = $categorizer->categorize($context);
if ($debug) {
$allResults[$categorizer->getName()] = [
'category_id' => $result->categoryId,
'confidence' => $result->confidence,
'matched_by' => $result->matchedBy,
];
}
// If this result is better than our current best, use it
if ($result->isSuccessful() && $result->shouldOverride($bestResult)) {
$bestResult = $result;
// If we have a very high confidence match, we can stop early
if ($result->confidence >= 0.95) {
break;
}
}
}
// Build the return array
$returnValue = ['categories_id' => $bestResult->categoryId];
if ($debug) {
$returnValue['debug'] = [
'final_category' => $bestResult->categoryId,
'final_confidence' => $bestResult->confidence,
'matched_by' => $bestResult->matchedBy,
'release_name' => $releaseName,
'group_name' => $groupName,
'all_results' => $allResults,
'categorizer_details' => $bestResult->debug,
];
}
return $returnValue;
return $result->toArray();
}
/**
* Get all registered categorizers.
* Get all registered categorizers (pipes).
*
* @return Collection<CategorizerInterface>
* @return Collection<AbstractCategorizationPipe>
*/
public function getCategorizers(): Collection
{
return $this->categorizers;
return $this->pipes;
}
/**
@@ -138,15 +103,15 @@ class CategorizationPipeline
public static function createDefault(): self
{
return new self([
new Categorizers\GroupNameCategorizer(),
new Categorizers\XxxCategorizer(),
new Categorizers\TvCategorizer(),
new Categorizers\MovieCategorizer(),
new Categorizers\BookCategorizer(),
new Categorizers\MusicCategorizer(),
new Categorizers\PcCategorizer(),
new Categorizers\ConsoleCategorizer(),
new Categorizers\MiscCategorizer(),
new Pipes\GroupNamePipe(),
new Pipes\XxxPipe(),
new Pipes\TvPipe(),
new Pipes\MoviePipe(),
new Pipes\BookPipe(),
new Pipes\MusicPipe(),
new Pipes\PcPipe(),
new Pipes\ConsolePipe(),
new Pipes\MiscPipe(),
]);
}
}
@@ -2,10 +2,10 @@
namespace App\Services\Categorization;
use App\Models\Category;
use App\Services\Categorization\Pipes\AbstractCategorizationPipe;
/**
* Categorization service using the new pipeline-based system.
* Categorization service using Laravel Pipeline.
*
* This class is a drop-in replacement for the legacy Blacklight\Categorize
* with additional features like confidence scoring and debug information.
@@ -70,11 +70,11 @@ class CategorizationService
}
/**
* Add a custom categorizer to the pipeline.
* Add a custom categorizer pipe to the pipeline.
*/
public function addCategorizer(Contracts\CategorizerInterface $categorizer): self
public function addCategorizer(AbstractCategorizationPipe $pipe): self
{
$this->pipeline->addCategorizer($categorizer);
$this->pipeline->addCategorizer($pipe);
return $this;
}
@@ -0,0 +1,84 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\ReleaseContext;
use Closure;
/**
* Base class for categorization pipe handlers.
*
* Each pipe is responsible for categorizing a specific type of release.
*/
abstract class AbstractCategorizationPipe
{
protected int $priority = 50;
/**
* Handle the categorization request.
*/
public function handle(CategorizationPassable $passable, Closure $next): CategorizationPassable
{
// If we already have a high-confidence match, skip processing
if ($passable->shouldStopProcessing()) {
return $next($passable);
}
// Skip if this categorizer shouldn't process this release
if ($this->shouldSkip($passable->context)) {
return $next($passable);
}
// Attempt categorization
$result = $this->categorize($passable->context);
// Update the best result
$passable->updateBestResult($result, $this->getName());
return $next($passable);
}
/**
* Get the priority of this categorizer (lower = higher priority).
*/
public function getPriority(): int
{
return $this->priority;
}
/**
* Get the name of this categorizer for debugging/logging.
*/
abstract public function getName(): string;
/**
* Attempt to categorize the given release.
*/
abstract protected function categorize(ReleaseContext $context): CategorizationResult;
/**
* Check if this categorizer should be skipped for the given context.
*/
protected function shouldSkip(ReleaseContext $context): bool
{
return false;
}
/**
* Create a successful categorization result.
*/
protected function matched(int $categoryId, float $confidence, string $matchedBy, array $debug = []): CategorizationResult
{
return new CategorizationResult($categoryId, $confidence, $matchedBy, $debug);
}
/**
* Create a no-match result.
*/
protected function noMatch(): CategorizationResult
{
return CategorizationResult::noMatch();
}
}
@@ -0,0 +1,36 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\BookCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for Book content categorization.
*/
class BookPipe extends AbstractCategorizationPipe
{
protected int $priority = 45;
private BookCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new BookCategorizer();
}
public function getName(): string
{
return 'Book';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,73 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\ReleaseContext;
/**
* Passable object that travels through the categorization pipeline.
*/
class CategorizationPassable
{
public ReleaseContext $context;
public CategorizationResult $bestResult;
public bool $debug;
public array $allResults = [];
public function __construct(ReleaseContext $context, bool $debug = false)
{
$this->context = $context;
$this->debug = $debug;
$this->bestResult = CategorizationResult::noMatch();
}
/**
* Check if we should stop processing (high confidence match found).
*/
public function shouldStopProcessing(): bool
{
return $this->bestResult->confidence >= 0.95;
}
/**
* Update the best result if the new result is better.
*/
public function updateBestResult(CategorizationResult $result, string $categorizerName): void
{
if ($this->debug) {
$this->allResults[$categorizerName] = [
'category_id' => $result->categoryId,
'confidence' => $result->confidence,
'matched_by' => $result->matchedBy,
];
}
if ($result->isSuccessful() && $result->shouldOverride($this->bestResult)) {
$this->bestResult = $result;
}
}
/**
* Build the final result array.
*/
public function toArray(): array
{
$returnValue = ['categories_id' => $this->bestResult->categoryId];
if ($this->debug) {
$returnValue['debug'] = [
'final_category' => $this->bestResult->categoryId,
'final_confidence' => $this->bestResult->confidence,
'matched_by' => $this->bestResult->matchedBy,
'release_name' => $this->context->releaseName,
'group_name' => $this->context->groupName,
'all_results' => $this->allResults,
'categorizer_details' => $this->bestResult->debug,
];
}
return $returnValue;
}
}
@@ -0,0 +1,37 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\ConsoleCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for Console game categorization.
*/
class ConsolePipe extends AbstractCategorizationPipe
{
protected int $priority = 35;
private ConsoleCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new ConsoleCategorizer();
}
public function getName(): string
{
return 'Console';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,32 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\GroupNameCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for group-based categorization.
*/
class GroupNamePipe extends AbstractCategorizationPipe
{
protected int $priority = 5;
private GroupNameCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new GroupNameCategorizer();
}
public function getName(): string
{
return 'GroupName';
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,33 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\MiscCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for miscellaneous content and hash detection.
* This runs last as a fallback.
*/
class MiscPipe extends AbstractCategorizationPipe
{
protected int $priority = 100;
private MiscCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new MiscCategorizer();
}
public function getName(): string
{
return 'Misc';
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,37 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\MovieCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for Movie content categorization.
*/
class MoviePipe extends AbstractCategorizationPipe
{
protected int $priority = 25;
private MovieCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new MovieCategorizer();
}
public function getName(): string
{
return 'Movie';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,37 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\MusicCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for Music content categorization.
*/
class MusicPipe extends AbstractCategorizationPipe
{
protected int $priority = 40;
private MusicCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new MusicCategorizer();
}
public function getName(): string
{
return 'Music';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,37 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\PcCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for PC content categorization (Games, Software, etc.).
*/
class PcPipe extends AbstractCategorizationPipe
{
protected int $priority = 30;
private PcCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new PcCategorizer();
}
public function getName(): string
{
return 'PC';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,37 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\TvCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for TV content categorization.
*/
class TvPipe extends AbstractCategorizationPipe
{
protected int $priority = 20;
private TvCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new TvCategorizer();
}
public function getName(): string
{
return 'TV';
}
protected function shouldSkip(ReleaseContext $context): bool
{
return $this->categorizer->shouldSkip($context);
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}
@@ -0,0 +1,32 @@
<?php
namespace App\Services\Categorization\Pipes;
use App\Services\Categorization\CategorizationResult;
use App\Services\Categorization\Categorizers\XxxCategorizer;
use App\Services\Categorization\ReleaseContext;
/**
* Pipe for XXX/Adult content categorization.
*/
class XxxPipe extends AbstractCategorizationPipe
{
protected int $priority = 10;
private XxxCategorizer $categorizer;
public function __construct()
{
$this->categorizer = new XxxCategorizer();
}
public function getName(): string
{
return 'XXX';
}
protected function categorize(ReleaseContext $context): CategorizationResult
{
return $this->categorizer->categorize($context);
}
}