diff --git a/Blacklight/IRCScraper.php b/Blacklight/IRCScraper.php index 7d9013bb8..710452627 100755 --- a/Blacklight/IRCScraper.php +++ b/Blacklight/IRCScraper.php @@ -4,6 +4,8 @@ namespace Blacklight; use App\Models\Predb; use App\Models\UsenetGroup; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; @@ -49,9 +51,9 @@ class IRCScraper extends IRCClient */ protected string|false $_titleIgnoreRegex = false; - protected ManticoreSearch $manticoreSearch; + protected ManticoreSearchService $manticoreSearch; - private ElasticSearchSiteSearch $elasticsearch; + private ElasticSearchService $elasticsearch; /** * Construct. @@ -106,8 +108,8 @@ class IRCScraper extends IRCClient $this->_titleIgnoreRegex = (string) config('irc_settings.scrape_irc_title_ignore'); } - $this->elasticsearch = new ElasticSearchSiteSearch; - $this->manticoreSearch = new ManticoreSearch; + $this->elasticsearch = app(ElasticSearchService::class); + $this->manticoreSearch = app(ManticoreSearchService::class); $this->_groupList = []; $this->_silent = $silent; diff --git a/Blacklight/ReleaseRemover.php b/Blacklight/ReleaseRemover.php index 9b42d52d6..5cad8cf56 100755 --- a/Blacklight/ReleaseRemover.php +++ b/Blacklight/ReleaseRemover.php @@ -576,10 +576,10 @@ class ReleaseRemover private function performSearch(string $regexMatch): array|string { if (config('nntmux.elasticsearch_enabled') === true) { - return (new ElasticSearchSiteSearch)->indexSearch($regexMatch, 100); + return app(\App\Services\Search\ElasticSearchService::class)->indexSearch($regexMatch, 100); } - $searchResult = (new ManticoreSearch)->searchIndexes('releases_rt', $regexMatch, ['name,searchname']); + $searchResult = app(\App\Services\Search\ManticoreSearchService::class)->searchIndexes('releases_rt', $regexMatch, ['name,searchname']); return ! empty($searchResult) ? Arr::wrap(Arr::get($searchResult, 'id')) : ''; } diff --git a/Blacklight/Releases.php b/Blacklight/Releases.php index e1e64f43a..d32e6c02a 100644 --- a/Blacklight/Releases.php +++ b/Blacklight/Releases.php @@ -6,6 +6,8 @@ use App\Models\Category; use App\Models\Release; use App\Models\Settings; use App\Models\UsenetGroup; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Elasticsearch; use Elasticsearch\Common\Exceptions\Missing404Exception; use Illuminate\Database\Eloquent\Collection; @@ -29,9 +31,9 @@ class Releases extends Release public int $passwordStatus; - private ManticoreSearch $manticoreSearch; + private ManticoreSearchService $manticoreSearch; - private ElasticSearchSiteSearch $elasticSearch; + private ElasticSearchService $elasticSearch; /** * @var array Class instances. @@ -41,8 +43,8 @@ class Releases extends Release public function __construct() { parent::__construct(); - $this->manticoreSearch = new ManticoreSearch; - $this->elasticSearch = new ElasticSearchSiteSearch; + $this->manticoreSearch = app(ManticoreSearchService::class); + $this->elasticSearch = app(ElasticSearchService::class); } /** diff --git a/app/Console/Commands/DeleteReleases.php b/app/Console/Commands/DeleteReleases.php index b577c7a12..d54f8d7cf 100644 --- a/app/Console/Commands/DeleteReleases.php +++ b/app/Console/Commands/DeleteReleases.php @@ -2,8 +2,8 @@ namespace App\Console\Commands; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; @@ -147,9 +147,8 @@ class DeleteReleases extends Command DB::delete('DELETE FROM releases WHERE id = ?', [$releaseId]); // Delete from search indexes - $identifiers = ['id' => $releaseId, 'g' => $release->guid]; - ManticoreSearch::deleteRelease($identifiers); - ElasticSearchSiteSearch::deleteRelease($releaseId); + app(ManticoreSearchService::class)->deleteRelease($releaseId); + app(ElasticSearchService::class)->deleteRelease($releaseId); $deleted++; diff --git a/app/Console/Commands/NntmuxOffsetPopulate.php b/app/Console/Commands/NntmuxOffsetPopulate.php index 49f331320..1b13f76ec 100644 --- a/app/Console/Commands/NntmuxOffsetPopulate.php +++ b/app/Console/Commands/NntmuxOffsetPopulate.php @@ -4,7 +4,7 @@ namespace App\Console\Commands; use App\Models\Predb; use App\Models\Release; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Exception; use Illuminate\Console\Command; use Illuminate\Support\Arr; @@ -276,7 +276,7 @@ class NntmuxOffsetPopulate extends Command private function clearIndex(string $engine, string $index): void { if ($engine === 'manticore') { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $indexName = $index === 'releases' ? 'releases_rt' : 'predb_rt'; $manticore->truncateRTIndex(Arr::wrap($indexName)); $this->info("Truncated ManticoreSearch index: {$indexName}"); diff --git a/app/Console/Commands/NntmuxOffsetWorker.php b/app/Console/Commands/NntmuxOffsetWorker.php index 16b82aa5c..2e7640ed1 100644 --- a/app/Console/Commands/NntmuxOffsetWorker.php +++ b/app/Console/Commands/NntmuxOffsetWorker.php @@ -4,7 +4,7 @@ namespace App\Console\Commands; use App\Models\Predb; use App\Models\Release; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Exception; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; @@ -92,7 +92,7 @@ class NntmuxOffsetWorker extends Command $startTime = microtime(true); if ($engine === 'manticore') { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $batchData = []; $query->chunk($batchSize, function ($items) use ($manticore, $indexName, $transformer, &$processed, &$errors, &$batchData, $batchSize, $workerId) { @@ -248,7 +248,7 @@ class NntmuxOffsetWorker extends Command /** * Process ManticoreSearch batch */ - private function processManticoreBatch(ManticoreSearch $manticore, string $indexName, array $data, int $workerId): void + private function processManticoreBatch(ManticoreSearchService $manticore, string $indexName, array $data, int $workerId): void { $retries = 3; $attempt = 0; diff --git a/app/Console/Commands/NntmuxPopulateSearchIndexes.php b/app/Console/Commands/NntmuxPopulateSearchIndexes.php index aa986e6eb..e1eeded62 100644 --- a/app/Console/Commands/NntmuxPopulateSearchIndexes.php +++ b/app/Console/Commands/NntmuxPopulateSearchIndexes.php @@ -4,7 +4,7 @@ namespace App\Console\Commands; use App\Models\Predb; use App\Models\Release; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Exception; use Illuminate\Console\Command; use Illuminate\Support\Arr; @@ -116,7 +116,7 @@ class NntmuxPopulateSearchIndexes extends Command $this->info('Optimizing ManticoreSearch indexes...'); try { - (new ManticoreSearch)->optimizeRTIndex(); + app(ManticoreSearchService::class)->optimizeRTIndex(); $this->info('Optimization completed successfully!'); return Command::SUCCESS; @@ -155,7 +155,7 @@ class NntmuxPopulateSearchIndexes extends Command private function manticoreReleases(): int { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $indexName = 'releases_rt'; $manticore->truncateRTIndex(Arr::wrap($indexName)); @@ -208,7 +208,7 @@ class NntmuxPopulateSearchIndexes extends Command */ private function manticorePredb(): int { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $indexName = 'predb_rt'; $manticore->truncateRTIndex([$indexName]); @@ -244,7 +244,7 @@ class NntmuxPopulateSearchIndexes extends Command */ private function processManticoreData(string $indexName, int $total, $query, callable $transformer): int { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $chunkSize = $this->getChunkSize(); $batchSize = $this->getBatchSize(); @@ -487,7 +487,7 @@ class NntmuxPopulateSearchIndexes extends Command /** * Process ManticoreSearch batch with retry logic */ - private function processBatch(ManticoreSearch $manticore, string $indexName, array $data): void + private function processBatch(ManticoreSearchService $manticore, string $indexName, array $data): void { $retries = 3; $attempt = 0; diff --git a/app/Console/Commands/NntmuxRemoveBadReleases.php b/app/Console/Commands/NntmuxRemoveBadReleases.php index d14e39b89..bef4310a3 100644 --- a/app/Console/Commands/NntmuxRemoveBadReleases.php +++ b/app/Console/Commands/NntmuxRemoveBadReleases.php @@ -4,7 +4,7 @@ namespace App\Console\Commands; use App\Models\Release; use App\Models\ReleaseFile; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Blacklight\NZB; use Blacklight\ReleaseImage; use Elasticsearch; @@ -64,12 +64,8 @@ class NntmuxRemoveBadReleases extends Command // we do nothing here just catch the error, we don't care if release is missing from ES, we are deleting it anyway } } else { - $identifiers = [ - 'i' => $badRelease->id, - 'g' => $badRelease->guid, - ]; - // Delete from sphinx. - (new ManticoreSearch)->deleteRelease($identifiers); + // Delete from Manticore + app(ManticoreSearchService::class)->deleteRelease($badRelease->id); } $badRelease->delete(); } diff --git a/app/Console/Commands/NntmuxResetDb.php b/app/Console/Commands/NntmuxResetDb.php index 4153bf1a1..383073072 100644 --- a/app/Console/Commands/NntmuxResetDb.php +++ b/app/Console/Commands/NntmuxResetDb.php @@ -3,7 +3,7 @@ namespace App\Console\Commands; use App\Models\UsenetGroup; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; @@ -171,7 +171,7 @@ class NntmuxResetDb extends Command $this->info('All done! ElasticSearch indexes are deleted and recreated.'); } else { - (new ManticoreSearch)->truncateRTIndex(['releases_rt', 'predb_rt']); + app(ManticoreSearchService::class)->truncateRTIndex(['releases_rt', 'predb_rt']); } $this->info('Deleting nzbfiles subfolders.'); diff --git a/app/Http/Controllers/SearchController.php b/app/Http/Controllers/SearchController.php index 6d3eb25d3..525fa7aba 100644 --- a/app/Http/Controllers/SearchController.php +++ b/app/Http/Controllers/SearchController.php @@ -4,18 +4,18 @@ namespace App\Http\Controllers; use App\Models\Category; use App\Models\UsenetGroup; -use Blacklight\ManticoreSearch; +use App\Services\Search\Contracts\SearchServiceInterface; use Blacklight\Releases; use Illuminate\Http\Request; class SearchController extends BasePageController { - private ManticoreSearch $manticore; + private SearchServiceInterface $searchService; - public function __construct() + public function __construct(SearchServiceInterface $searchService) { parent::__construct(); - $this->manticore = new ManticoreSearch; + $this->searchService = $searchService; } /** @@ -145,9 +145,9 @@ class SearchController extends BasePageController // Get spell correction suggestions if we have a search query but few/no results $spellSuggestion = null; $searchQuery = $search ?: ($searchVars['searchadvr'] ?? ''); - if (! empty($searchQuery) && $this->manticore->isSuggestEnabled()) { - // Get suggestions from ManticoreSearch - $suggestions = $this->manticore->suggest($searchQuery); + if (! empty($searchQuery) && $this->searchService->isSuggestEnabled()) { + // Get suggestions from search service + $suggestions = $this->searchService->suggest($searchQuery); if (! empty($suggestions)) { // Sort by doc count descending to get best suggestion usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']); @@ -214,10 +214,10 @@ class SearchController extends BasePageController 'meta_title' => 'Search Nzbs', 'meta_keywords' => 'search,nzb,description,details', 'meta_description' => 'Search for Nzbs', - // ManticoreSearch enhanced features + // Search enhanced features 'spellSuggestion' => $spellSuggestion, - 'autocompleteEnabled' => $this->manticore->isAutocompleteEnabled(), - 'suggestEnabled' => $this->manticore->isSuggestEnabled(), + 'autocompleteEnabled' => $this->searchService->isAutocompleteEnabled(), + 'suggestEnabled' => $this->searchService->isSuggestEnabled(), ]); return view('search.index', $this->viewData); diff --git a/app/Http/Controllers/SearchSuggestController.php b/app/Http/Controllers/SearchSuggestController.php index d933663cc..cf1e6c51a 100644 --- a/app/Http/Controllers/SearchSuggestController.php +++ b/app/Http/Controllers/SearchSuggestController.php @@ -2,24 +2,18 @@ namespace App\Http\Controllers; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\Contracts\SearchServiceInterface; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\RateLimiter; class SearchSuggestController extends Controller { - private ManticoreSearch|ElasticSearchSiteSearch $searchEngine; + private SearchServiceInterface $searchEngine; - public function __construct() + public function __construct(SearchServiceInterface $searchEngine) { - // Use Elasticsearch if enabled, otherwise fall back to ManticoreSearch - if (config('nntmux.elasticsearch_enabled', false)) { - $this->searchEngine = new ElasticSearchSiteSearch; - } else { - $this->searchEngine = new ManticoreSearch; - } + $this->searchEngine = $searchEngine; } /** diff --git a/app/Models/Predb.php b/app/Models/Predb.php index d554af281..5a69048e4 100644 --- a/app/Models/Predb.php +++ b/app/Models/Predb.php @@ -2,10 +2,10 @@ namespace App\Models; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ColorCLI; use Blacklight\ConsoleTools; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Support\Arr; @@ -201,9 +201,9 @@ class Predb extends Model ->orderByDesc('predb.predate'); if (! empty($search)) { if (config('nntmux.elasticsearch_enabled') === true) { - $ids = (new ElasticSearchSiteSearch)->predbIndexSearch($search); + $ids = app(ElasticSearchService::class)->predbIndexSearch($search); } else { - $manticore = new ManticoreSearch; + $manticore = app(ManticoreSearchService::class); $ids = Arr::get($manticore->searchIndexes('predb_rt', $search, ['title']), 'id'); } $sql->whereIn('predb.id', $ids); diff --git a/app/Models/Release.php b/app/Models/Release.php index f6a6eed16..b05ed176d 100644 --- a/app/Models/Release.php +++ b/app/Models/Release.php @@ -2,8 +2,8 @@ namespace App\Models; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Illuminate\Contracts\Pagination\LengthAwarePaginator; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; @@ -136,9 +136,9 @@ class Release extends Model ); if (config('nntmux.elasticsearch_enabled') === true) { - (new ElasticSearchSiteSearch)->insertRelease($parameters); + app(ElasticSearchService::class)->insertRelease($parameters); } else { - (new ManticoreSearch)->insertRelease($parameters); + app(ManticoreSearchService::class)->insertRelease($parameters); } return $parameters['id']; @@ -173,9 +173,9 @@ class Release extends Model ); if (config('nntmux.elasticsearch_enabled') === true) { - (new ElasticSearchSiteSearch)->updateRelease($id); + app(ElasticSearchService::class)->updateRelease($id); } else { - (new ManticoreSearch)->updateRelease($id); + app(ManticoreSearchService::class)->updateRelease($id); } } @@ -397,9 +397,9 @@ class Release extends Model if (! empty($similar)) { if (config('nntmux.elasticsearch_enabled') === true) { - $searchResult = (new ElasticSearchSiteSearch)->indexSearch($similar[1], 10); + $searchResult = app(ElasticSearchService::class)->indexSearch($similar[1], 10); } else { - $searchResult = (new ManticoreSearch)->searchIndexes('releases_rt', $similar[1]); + $searchResult = app(ManticoreSearchService::class)->searchIndexes('releases_rt', $similar[1]); if (! empty($searchResult)) { $searchResult = Arr::wrap(Arr::get($searchResult, 'id')); } diff --git a/app/Models/ReleaseFile.php b/app/Models/ReleaseFile.php index 60b29e040..0256006b3 100644 --- a/app/Models/ReleaseFile.php +++ b/app/Models/ReleaseFile.php @@ -2,8 +2,8 @@ namespace App\Models; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Log; @@ -126,9 +126,9 @@ class ReleaseFile extends Model ParHash::insertOrIgnore(['releases_id' => $id, 'hash' => $hash]); } if (config('nntmux.elasticsearch_enabled') === true) { - (new ElasticSearchSiteSearch)->updateRelease($id); + app(ElasticSearchService::class)->updateRelease($id); } else { - (new ManticoreSearch)->updateRelease($id); + app(ManticoreSearchService::class)->updateRelease($id); } } diff --git a/app/Providers/SearchServiceProvider.php b/app/Providers/SearchServiceProvider.php new file mode 100644 index 000000000..1c52d2b2d --- /dev/null +++ b/app/Providers/SearchServiceProvider.php @@ -0,0 +1,48 @@ +app->singleton(ManticoreSearchService::class, function ($app) { + return new ManticoreSearchService(); + }); + + // Register ElasticSearchService as singleton + $this->app->singleton(ElasticSearchService::class, function ($app) { + return new ElasticSearchService(); + }); + + // Bind the interface to the appropriate implementation based on config + $this->app->singleton(SearchServiceInterface::class, function ($app) { + if (config('nntmux.elasticsearch_enabled') === true) { + return $app->make(ElasticSearchService::class); + } + + return $app->make(ManticoreSearchService::class); + }); + + // Register alias for ManticoreSearch (for backward compatibility) + $this->app->alias(ManticoreSearchService::class, 'manticore'); + } + + /** + * Bootstrap any application services. + */ + public function boot(): void + { + // + } +} + diff --git a/app/Services/AdditionalProcessing/MediaExtractionService.php b/app/Services/AdditionalProcessing/MediaExtractionService.php index 02801e75f..34102c0f9 100644 --- a/app/Services/AdditionalProcessing/MediaExtractionService.php +++ b/app/Services/AdditionalProcessing/MediaExtractionService.php @@ -7,9 +7,8 @@ use App\Models\Release; use App\Services\AdditionalProcessing\Config\ProcessingConfiguration; use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext; use App\Services\Categorization\CategorizationService; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; -use Blacklight\NameFixer; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ReleaseExtra; use Blacklight\ReleaseImage; use FFMpeg\Coordinate\Dimension; @@ -32,8 +31,8 @@ class MediaExtractionService private ?FFMpeg $ffmpeg = null; private ?FFProbe $ffprobe = null; private ?MediaInfo $mediaInfo = null; - private ?ManticoreSearch $manticore = null; - private ?ElasticSearchSiteSearch $elasticsearch = null; + private ?ManticoreSearchService $manticore = null; + private ?ElasticSearchService $elasticsearch = null; public function __construct( private readonly ProcessingConfiguration $config, @@ -471,18 +470,18 @@ class MediaExtractionService return $this->mediaInfo; } - private function manticore(): ManticoreSearch + private function manticore(): ManticoreSearchService { if ($this->manticore === null) { - $this->manticore = new ManticoreSearch(); + $this->manticore = app(ManticoreSearchService::class); } return $this->manticore; } - private function elasticsearch(): ElasticSearchSiteSearch + private function elasticsearch(): ElasticSearchService { if ($this->elasticsearch === null) { - $this->elasticsearch = new ElasticSearchSiteSearch(); + $this->elasticsearch = app(ElasticSearchService::class); } return $this->elasticsearch; } diff --git a/app/Services/AdditionalProcessing/ReleaseFileManager.php b/app/Services/AdditionalProcessing/ReleaseFileManager.php index 211fed0a7..18f165618 100644 --- a/app/Services/AdditionalProcessing/ReleaseFileManager.php +++ b/app/Services/AdditionalProcessing/ReleaseFileManager.php @@ -6,13 +6,13 @@ use App\Models\MediaInfo as MediaInfoModel; use App\Models\Predb; use App\Models\Release; use App\Models\ReleaseFile; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\Releases; use App\Services\AdditionalProcessing\Config\ProcessingConfiguration; use App\Services\AdditionalProcessing\DTO\ReleaseProcessingContext; use App\Services\NameFixing\NameFixingService; use App\Services\NameFixing\ReleaseUpdateService; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; use Blacklight\Nfo; use Blacklight\NZB; use Blacklight\ReleaseExtra; @@ -28,8 +28,8 @@ use Illuminate\Support\Facades\Log; */ class ReleaseFileManager { - private ?ManticoreSearch $manticore = null; - private ?ElasticSearchSiteSearch $elasticsearch = null; + private ?ManticoreSearchService $manticore = null; + private ?ElasticSearchService $elasticsearch = null; public function __construct( private readonly ProcessingConfiguration $config, @@ -590,18 +590,18 @@ class ReleaseFileManager return $hasGroupSuffix || ($hasTV && $hasQuality) || ($hasYear && ($hasQuality || $hasTV)) || $hasXXX; } - private function manticore(): ManticoreSearch + private function manticore(): ManticoreSearchService { if ($this->manticore === null) { - $this->manticore = new ManticoreSearch(); + $this->manticore = app(ManticoreSearchService::class); } return $this->manticore; } - private function elasticsearch(): ElasticSearchSiteSearch + private function elasticsearch(): ElasticSearchService { if ($this->elasticsearch === null) { - $this->elasticsearch = new ElasticSearchSiteSearch(); + $this->elasticsearch = app(ElasticSearchService::class); } return $this->elasticsearch; } diff --git a/app/Services/MediaProcessingService.php b/app/Services/MediaProcessingService.php index d81637bc3..e262f7947 100644 --- a/app/Services/MediaProcessingService.php +++ b/app/Services/MediaProcessingService.php @@ -5,8 +5,8 @@ namespace App\Services; use App\Models\Category; use App\Models\Release; use App\Services\Categorization\CategorizationService; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ReleaseExtra; use Blacklight\ReleaseImage; use FFMpeg\Coordinate\Dimension; @@ -28,8 +28,8 @@ class MediaProcessingService private readonly MediaInfo $mediaInfo, private readonly ReleaseImage $releaseImage, private readonly ReleaseExtra $releaseExtra, - private readonly ManticoreSearch $manticore, - private readonly ElasticSearchSiteSearch $elasticsearch, + private readonly ManticoreSearchService $manticore, + private readonly ElasticSearchService $elasticsearch, private readonly CategorizationService $categorize, ) {} diff --git a/app/Services/NameFixing/NameFixingService.php b/app/Services/NameFixing/NameFixingService.php index 6ca63f421..60fa82600 100644 --- a/app/Services/NameFixing/NameFixingService.php +++ b/app/Services/NameFixing/NameFixingService.php @@ -10,10 +10,10 @@ use App\Services\NameFixing\Contracts\NameSourceFixerInterface; use App\Services\NameFixing\DTO\NameFixResult; use App\Services\NameFixing\Extractors\NfoNameExtractor; use App\Services\NameFixing\Extractors\FileNameExtractor; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ColorCLI; use Blacklight\ConsoleTools; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; use Illuminate\Support\Arr; /** @@ -50,8 +50,8 @@ class NameFixingService protected FileNameExtractor $fileExtractor; protected FileNameCleaner $fileNameCleaner; protected FilePrioritizer $filePrioritizer; - protected ManticoreSearch $manticore; - protected ElasticSearchSiteSearch $elasticsearch; + protected ManticoreSearchService $manticore; + protected ElasticSearchService $elasticsearch; protected ColorCLI $colorCLI; protected bool $echoOutput; @@ -70,8 +70,8 @@ class NameFixingService ?FileNameExtractor $fileExtractor = null, ?FileNameCleaner $fileNameCleaner = null, ?FilePrioritizer $filePrioritizer = null, - ?ManticoreSearch $manticore = null, - ?ElasticSearchSiteSearch $elasticsearch = null, + ?ManticoreSearchService $manticore = null, + ?ElasticSearchService $elasticsearch = null, ?ColorCLI $colorCLI = null ) { $this->updateService = $updateService ?? new ReleaseUpdateService(); @@ -80,8 +80,8 @@ class NameFixingService $this->fileExtractor = $fileExtractor ?? new FileNameExtractor(); $this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner(); $this->filePrioritizer = $filePrioritizer ?? new FilePrioritizer(); - $this->manticore = $manticore ?? new ManticoreSearch(); - $this->elasticsearch = $elasticsearch ?? new ElasticSearchSiteSearch(); + $this->manticore = $manticore ?? app(ManticoreSearchService::class); + $this->elasticsearch = $elasticsearch ?? app(ElasticSearchService::class); $this->colorCLI = $colorCLI ?? new ColorCLI(); $this->echoOutput = config('nntmux.echocli'); diff --git a/app/Services/NameFixing/ReleaseUpdateService.php b/app/Services/NameFixing/ReleaseUpdateService.php index d8797305a..85c8a108c 100644 --- a/app/Services/NameFixing/ReleaseUpdateService.php +++ b/app/Services/NameFixing/ReleaseUpdateService.php @@ -9,10 +9,10 @@ use App\Models\Predb; use App\Models\Release; use App\Models\UsenetGroup; use App\Services\Categorization\CategorizationService; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ColorCLI; use Blacklight\ConsoleTools; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; use Blacklight\ReleaseCleaning; use Illuminate\Support\Arr; @@ -50,8 +50,8 @@ class ReleaseUpdateService public const IS_RENAMED_DONE = 1; protected CategorizationService $category; - protected ManticoreSearch $manticore; - protected ElasticSearchSiteSearch $elasticsearch; + protected ManticoreSearchService $manticore; + protected ElasticSearchService $elasticsearch; protected FileNameCleaner $fileNameCleaner; protected ColorCLI $colorCLI; protected bool $echoOutput; @@ -83,14 +83,14 @@ class ReleaseUpdateService public function __construct( ?CategorizationService $category = null, - ?ManticoreSearch $manticore = null, - ?ElasticSearchSiteSearch $elasticsearch = null, + ?ManticoreSearchService $manticore = null, + ?ElasticSearchService $elasticsearch = null, ?FileNameCleaner $fileNameCleaner = null, ?ColorCLI $colorCLI = null ) { $this->category = $category ?? new CategorizationService(); - $this->manticore = $manticore ?? new ManticoreSearch(); - $this->elasticsearch = $elasticsearch ?? new ElasticSearchSiteSearch(); + $this->manticore = $manticore ?? app(ManticoreSearchService::class); + $this->elasticsearch = $elasticsearch ?? app(ElasticSearchService::class); $this->fileNameCleaner = $fileNameCleaner ?? new FileNameCleaner(); $this->colorCLI = $colorCLI ?? new ColorCLI(); $this->echoOutput = config('nntmux.echocli'); diff --git a/app/Services/Search/Contracts/SearchServiceInterface.php b/app/Services/Search/Contracts/SearchServiceInterface.php new file mode 100644 index 000000000..f0a890c8d --- /dev/null +++ b/app/Services/Search/Contracts/SearchServiceInterface.php @@ -0,0 +1,102 @@ + + */ + public function autocomplete(string $query, ?string $index = null): array; + + /** + * Get spell correction suggestions ("Did you mean?"). + * + * @param string $query The search query to check + * @param string|null $index Index to use for suggestions + * @return array + */ + public function suggest(string $query, ?string $index = null): array; +} + diff --git a/app/Services/Search/ElasticSearchService.php b/app/Services/Search/ElasticSearchService.php new file mode 100644 index 000000000..d940319ef --- /dev/null +++ b/app/Services/Search/ElasticSearchService.php @@ -0,0 +1,1429 @@ +buildHostsArray($config['hosts'] ?? []); + + if (empty($hosts)) { + throw new RuntimeException('No Elasticsearch hosts configured'); + } + + if (config('app.debug')) { + Log::debug('Elasticsearch client initializing', [ + 'hosts' => $hosts, + ]); + } + + $clientBuilder->setHosts($hosts); + $clientBuilder->setHandler(new CurlHandler); + $clientBuilder->setConnectionParams([ + 'timeout' => config('elasticsearch.connections.default.timeout', self::DEFAULT_TIMEOUT), + 'connect_timeout' => config('elasticsearch.connections.default.connect_timeout', self::DEFAULT_CONNECT_TIMEOUT), + ]); + + // Enable retries for better resilience + $clientBuilder->setRetries(2); + + self::$client = $clientBuilder->build(); + + if (config('app.debug')) { + Log::debug('Elasticsearch client initialized successfully'); + } + + } catch (\Throwable $e) { + Log::error('Failed to initialize Elasticsearch client: '.$e->getMessage(), [ + 'exception_class' => get_class($e), + ]); + throw new RuntimeException('Elasticsearch client initialization failed: '.$e->getMessage()); + } + } + + return self::$client; + } + + /** + * Build hosts array from configuration. + * + * @param array $configHosts Configuration hosts array + * @return array Formatted hosts array for Elasticsearch client + */ + private function buildHostsArray(array $configHosts): array + { + $hosts = []; + + foreach ($configHosts as $host) { + $hostConfig = [ + 'host' => $host['host'] ?? 'localhost', + 'port' => $host['port'] ?? 9200, + ]; + + if (! empty($host['scheme'])) { + $hostConfig['scheme'] = $host['scheme']; + } + + if (! empty($host['user']) && ! empty($host['pass'])) { + $hostConfig['user'] = $host['user']; + $hostConfig['pass'] = $host['pass']; + } + + $hosts[] = $hostConfig; + } + + return $hosts; + } + + /** + * Check if Elasticsearch is available. + * + * Caches the availability status to avoid frequent ping requests. + */ + private function isElasticsearchAvailable(): bool + { + $now = time(); + + // Return cached result if still valid + if (self::$availabilityCache !== null + && self::$availabilityCacheTime !== null + && ($now - self::$availabilityCacheTime) < self::AVAILABILITY_CHECK_CACHE_TTL) { + return self::$availabilityCache; + } + + try { + $client = $this->getClient(); + $result = $client->ping(); + + if (config('app.debug')) { + Log::debug('Elasticsearch ping result', ['available' => $result]); + } + + self::$availabilityCache = $result; + self::$availabilityCacheTime = $now; + + return $result; + } catch (\Throwable $e) { + Log::warning('Elasticsearch is not available: '.$e->getMessage(), [ + 'exception_class' => get_class($e), + ]); + + self::$availabilityCache = false; + self::$availabilityCacheTime = $now; + + return false; + } + } + + /** + * Reset the client connection (useful for testing or reconnection). + */ + public function resetConnection(): void + { + self::$client = null; + self::$availabilityCache = null; + self::$availabilityCacheTime = null; + } + + /** + * Check if autocomplete is enabled. + */ + public function isAutocompleteEnabled(): bool + { + return config('elasticsearch.autocomplete.enabled', true) && $this->isElasticsearchAvailable(); + } + + /** + * Check if suggest is enabled. + */ + public function isSuggestEnabled(): bool + { + return config('elasticsearch.suggest.enabled', true) && $this->isElasticsearchAvailable(); + } + + /** + * Get the releases index name. + */ + public function getReleasesIndex(): string + { + return self::INDEX_RELEASES; + } + + /** + * Get the predb index name. + */ + public function getPredbIndex(): string + { + return self::INDEX_PREDB; + } + + /** + * Get autocomplete suggestions for a search query. + * Searches the releases index and returns matching searchnames. + * + * @param string $query The partial search query + * @param string|null $index Index to search (defaults to releases index) + * @return array + */ + public function autocomplete(string $query, ?string $index = null): array + { + if (! $this->isAutocompleteEnabled()) { + return []; + } + + $query = trim($query); + $minLength = (int) config('elasticsearch.autocomplete.min_length', self::AUTOCOMPLETE_MIN_LENGTH); + if (strlen($query) < $minLength) { + return []; + } + + $index = $index ?? self::INDEX_RELEASES; + $cacheKey = 'es:autocomplete:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + $maxResults = (int) config('elasticsearch.autocomplete.max_results', self::AUTOCOMPLETE_MAX_RESULTS); + + try { + $client = $this->getClient(); + + // Use a prefix/match query on searchname field + $searchParams = [ + 'index' => $index, + 'body' => [ + 'query' => [ + 'bool' => [ + 'should' => [ + // Prefix match for autocomplete-like behavior + [ + 'match_phrase_prefix' => [ + 'searchname' => [ + 'query' => $query, + 'max_expansions' => 50, + ], + ], + ], + // Also include regular match for better results + [ + 'match' => [ + 'searchname' => [ + 'query' => $query, + 'fuzziness' => 'AUTO', + ], + ], + ], + ], + 'minimum_should_match' => 1, + ], + ], + 'size' => $maxResults * 3, + '_source' => ['searchname'], + 'sort' => [ + // Sort by date first to get latest results, then by score + ['add_date' => ['order' => 'desc', 'unmapped_type' => 'date', 'missing' => '_last']], + ['_score' => ['order' => 'desc']], + ], + ], + ]; + + $response = $client->search($searchParams); + + $seen = []; + if (isset($response['hits']['hits'])) { + foreach ($response['hits']['hits'] as $hit) { + $searchname = $hit['_source']['searchname'] ?? ''; + + if (empty($searchname)) { + continue; + } + + // Create a clean suggestion from the searchname + $suggestion = $this->extractSuggestion($searchname, $query); + + if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) { + $seen[strtolower($suggestion)] = true; + $suggestions[] = [ + 'suggest' => $suggestion, + 'distance' => 0, + 'docs' => 1, + ]; + } + + if (count($suggestions) >= $maxResults) { + break; + } + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::warning('ElasticSearch autocomplete error: '.$e->getMessage()); + } + } + + if (! empty($suggestions)) { + $cacheMinutes = (int) config('elasticsearch.autocomplete.cache_minutes', self::AUTOCOMPLETE_CACHE_MINUTES); + Cache::put($cacheKey, $suggestions, now()->addMinutes($cacheMinutes)); + } + + return $suggestions; + } + + /** + * Get spell correction suggestions ("Did you mean?"). + * + * @param string $query The search query to check + * @param string|null $index Index to use for suggestions + * @return array + */ + public function suggest(string $query, ?string $index = null): array + { + if (! $this->isSuggestEnabled()) { + return []; + } + + $query = trim($query); + if (empty($query)) { + return []; + } + + $index = $index ?? self::INDEX_RELEASES; + $cacheKey = 'es:suggest:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + + try { + $client = $this->getClient(); + + // Use Elasticsearch suggest API with phrase suggester + $searchParams = [ + 'index' => $index, + 'body' => [ + 'suggest' => [ + 'text' => $query, + 'searchname_suggest' => [ + 'phrase' => [ + 'field' => 'searchname', + 'size' => 5, + 'gram_size' => 3, + 'direct_generator' => [ + [ + 'field' => 'searchname', + 'suggest_mode' => 'popular', + ], + ], + 'highlight' => [ + 'pre_tag' => '', + 'post_tag' => '', + ], + ], + ], + ], + ], + ]; + + $response = $client->search($searchParams); + + if (isset($response['suggest']['searchname_suggest'][0]['options'])) { + foreach ($response['suggest']['searchname_suggest'][0]['options'] as $option) { + $suggestedText = $option['text'] ?? ''; + if (! empty($suggestedText) && strtolower($suggestedText) !== strtolower($query)) { + $suggestions[] = [ + 'suggest' => $suggestedText, + 'distance' => 1, + 'docs' => (int) ($option['freq'] ?? 1), + ]; + } + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::debug('ElasticSearch native suggest failed: '.$e->getMessage()); + } + } + + // Fallback: if native suggest didn't work, use fuzzy search + if (empty($suggestions)) { + $suggestions = $this->suggestFallback($query, $index); + } + + if (! empty($suggestions)) { + Cache::put($cacheKey, $suggestions, now()->addMinutes(self::CACHE_TTL_MINUTES)); + } + + return $suggestions; + } + + /** + * Fallback suggest using fuzzy search on searchnames. + * + * @param string $query The search query + * @param string $index Index to search + * @return array + */ + private function suggestFallback(string $query, string $index): array + { + try { + $client = $this->getClient(); + + // Use fuzzy match to find similar terms + $searchParams = [ + 'index' => $index, + 'body' => [ + 'query' => [ + 'match' => [ + 'searchname' => [ + 'query' => $query, + 'fuzziness' => 'AUTO', + ], + ], + ], + 'size' => 20, + '_source' => ['searchname'], + ], + ]; + + $response = $client->search($searchParams); + + // Extract common terms from results that differ from the query + $termCounts = []; + if (isset($response['hits']['hits'])) { + foreach ($response['hits']['hits'] as $hit) { + $searchname = $hit['_source']['searchname'] ?? ''; + $words = preg_split('/[\s.\-_]+/', strtolower($searchname)); + + foreach ($words as $word) { + if (strlen($word) >= 3 && $word !== strtolower($query)) { + $distance = levenshtein(strtolower($query), $word); + if ($distance > 0 && $distance <= 3) { + if (! isset($termCounts[$word])) { + $termCounts[$word] = ['count' => 0, 'distance' => $distance]; + } + $termCounts[$word]['count']++; + } + } + } + } + } + + // Sort by count + uasort($termCounts, fn ($a, $b) => $b['count'] - $a['count']); + + $suggestions = []; + foreach (array_slice($termCounts, 0, 5, true) as $term => $data) { + $suggestions[] = [ + 'suggest' => $term, + 'distance' => $data['distance'], + 'docs' => $data['count'], + ]; + } + + return $suggestions; + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::warning('ElasticSearch suggest fallback error: '.$e->getMessage()); + } + + return []; + } + } + + /** + * Extract a clean suggestion from a searchname. + * + * @param string $searchname The full searchname + * @param string $query The user's query + * @return string|null The extracted suggestion + */ + private function extractSuggestion(string $searchname, string $query): ?string + { + // Clean up the searchname - remove file extensions + $clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname); + + // Replace dots and underscores with spaces for readability + $clean = str_replace(['.', '_'], ' ', $clean); + + // Remove multiple spaces + $clean = preg_replace('/\s+/', ' ', $clean); + $clean = trim($clean); + + if (empty($clean)) { + return null; + } + + // If the clean name is reasonable length, use it + if (strlen($clean) <= 80) { + return $clean; + } + + // For very long names, try to extract the relevant part + $pos = stripos($clean, $query); + if ($pos !== false) { + $start = max(0, $pos - 10); + $extracted = substr($clean, $start, 80); + + if ($start > 0) { + $extracted = preg_replace('/^\S*\s/', '', $extracted); + } + $extracted = preg_replace('/\s\S*$/', '', $extracted); + + return trim($extracted); + } + + return substr($clean, 0, 80); + } + + /** + * Search releases index. + * + * @param array|string $phrases Search phrases + * @param int $limit Maximum number of results + * @return array Array of release IDs + */ + public function searchReleases(array|string $phrases, int $limit = 1000): array + { + $result = $this->indexSearch($phrases, $limit); + + return is_array($result) ? $result : $result->toArray(); + } + + /** + * Search the predb index. + * + * @param array|string $searchTerm Search term(s) + * @return array Array of predb records + */ + public function searchPredb(array|string $searchTerm): array + { + $result = $this->predbIndexSearch($searchTerm); + + return is_array($result) ? $result : $result->toArray(); + } + + /** + * Search releases index. + * + * @param array|string $phrases Search phrases + * @param int $limit Maximum number of results + * @return array|Collection Array of release IDs + */ + public function indexSearch(array|string $phrases, int $limit): array|Collection + { + if (empty($phrases) || ! $this->isElasticsearchAvailable()) { + if (config('app.debug')) { + Log::debug('ElasticSearch indexSearch: empty phrases or ES not available', [ + 'phrases_empty' => empty($phrases), + 'es_available' => $this->isElasticsearchAvailable(), + ]); + } + + return []; + } + + $keywords = $this->sanitizeSearchTerms($phrases); + + if (config('app.debug')) { + Log::debug('ElasticSearch indexSearch: sanitized keywords', [ + 'original' => is_array($phrases) ? implode(' ', $phrases) : $phrases, + 'sanitized' => $keywords, + ]); + } + + if (empty($keywords)) { + if (config('app.debug')) { + Log::debug('ElasticSearch indexSearch: keywords empty after sanitization'); + } + + return []; + } + + $cacheKey = $this->buildCacheKey('index_search', [$keywords, $limit]); + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + try { + $search = $this->buildSearchQuery( + index: self::INDEX_RELEASES, + keywords: $keywords, + fields: ['searchname^2', 'plainsearchname^1.5', 'fromname', 'filename', 'name^1.2'], + limit: $limit + ); + + $result = $this->executeSearch($search); + Cache::put($cacheKey, $result, now()->addMinutes(self::CACHE_TTL_MINUTES)); + + return $result; + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch indexSearch error: '.$e->getMessage(), [ + 'keywords' => $keywords, + 'limit' => $limit, + ]); + + return []; + } + } + + /** + * Search releases for API requests. + * + * @param array|string $searchName Search name(s) + * @param int $limit Maximum number of results + * @return array|Collection Array of release IDs + */ + public function indexSearchApi(array|string $searchName, int $limit): array|Collection + { + if (empty($searchName) || ! $this->isElasticsearchAvailable()) { + return []; + } + + $keywords = $this->sanitizeSearchTerms($searchName); + if (empty($keywords)) { + return []; + } + + $cacheKey = $this->buildCacheKey('api_search', [$keywords, $limit]); + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + try { + $search = $this->buildSearchQuery( + index: self::INDEX_RELEASES, + keywords: $keywords, + fields: ['searchname^2', 'plainsearchname^1.5', 'fromname', 'filename', 'name^1.2'], + limit: $limit + ); + + $result = $this->executeSearch($search); + Cache::put($cacheKey, $result, now()->addMinutes(self::CACHE_TTL_MINUTES)); + + return $result; + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch indexSearchApi error: '.$e->getMessage(), [ + 'keywords' => $keywords, + 'limit' => $limit, + ]); + + return []; + } + } + + /** + * Search releases for TV/Movie/Audio (TMA) matching. + * + * @param array|string $name Name(s) to search + * @param int $limit Maximum number of results + * @return array|Collection Array of release IDs + */ + public function indexSearchTMA(array|string $name, int $limit): array|Collection + { + if (empty($name) || ! $this->isElasticsearchAvailable()) { + return []; + } + + $keywords = $this->sanitizeSearchTerms($name); + if (empty($keywords)) { + return []; + } + + $cacheKey = $this->buildCacheKey('tma_search', [$keywords, $limit]); + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + try { + $search = $this->buildSearchQuery( + index: self::INDEX_RELEASES, + keywords: $keywords, + fields: ['searchname^2', 'plainsearchname^1.5'], + limit: $limit, + options: [ + 'boost' => 1.2, + ] + ); + + $result = $this->executeSearch($search); + Cache::put($cacheKey, $result, now()->addMinutes(self::CACHE_TTL_MINUTES)); + + return $result; + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch indexSearchTMA error: '.$e->getMessage(), [ + 'keywords' => $keywords, + 'limit' => $limit, + ]); + + return []; + } + } + + /** + * Search predb index. + * + * @param array|string $searchTerm Search term(s) + * @return array|Collection Array of predb records + */ + public function predbIndexSearch(array|string $searchTerm): array|Collection + { + if (empty($searchTerm) || ! $this->isElasticsearchAvailable()) { + return []; + } + + $keywords = $this->sanitizeSearchTerms($searchTerm); + if (empty($keywords)) { + return []; + } + + $cacheKey = $this->buildCacheKey('predb_search', [$keywords]); + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + try { + $search = $this->buildSearchQuery( + index: self::INDEX_PREDB, + keywords: $keywords, + fields: ['title^2', 'filename'], + limit: 1000, + options: [ + 'fuzziness' => 'AUTO', + ], + includeDateSort: false + ); + + $result = $this->executeSearch($search, fullResults: true); + Cache::put($cacheKey, $result, now()->addMinutes(self::CACHE_TTL_MINUTES)); + + return $result; + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch predbIndexSearch error: '.$e->getMessage(), [ + 'keywords' => $keywords, + ]); + + return []; + } + } + + /** + * Insert a release into the index. + * + * @param array $parameters Release data with 'id', 'name', 'searchname', etc. + */ + public function insertRelease(array $parameters): void + { + if (empty($parameters['id']) || ! $this->isElasticsearchAvailable()) { + if (empty($parameters['id'])) { + Log::warning('ElasticSearch: Cannot insert release without ID'); + } + + return; + } + + try { + $client = $this->getClient(); + $client->index($this->buildReleaseDocument($parameters)); + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch insertRelease error: '.$e->getMessage(), [ + 'release_id' => $parameters['id'], + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch insertRelease unexpected error: '.$e->getMessage(), [ + 'release_id' => $parameters['id'], + ]); + } + } + + /** + * Bulk insert multiple releases into the index. + * + * @param array $releases Array of release data arrays + * @return array Results with 'success' and 'errors' counts + */ + public function bulkInsertReleases(array $releases): array + { + if (empty($releases) || ! $this->isElasticsearchAvailable()) { + return ['success' => 0, 'errors' => 0]; + } + + $params = ['body' => []]; + $validReleases = 0; + + foreach ($releases as $release) { + if (empty($release['id'])) { + continue; + } + + $params['body'][] = [ + 'index' => [ + '_index' => self::INDEX_RELEASES, + '_id' => $release['id'], + ], + ]; + + $document = $this->buildReleaseDocument($release); + $params['body'][] = $document['body']; + $validReleases++; + + // Send batch when reaching 500 documents + if ($validReleases % 500 === 0) { + $this->executeBulk($params); + $params = ['body' => []]; + } + } + + // Send remaining documents + if (! empty($params['body'])) { + $this->executeBulk($params); + } + + return ['success' => $validReleases, 'errors' => 0]; + } + + /** + * Update a release in the index. + * + * @param int|string $releaseID Release ID + */ + public function updateRelease(int|string $releaseID): void + { + if (empty($releaseID)) { + Log::warning('ElasticSearch: Cannot update release without ID'); + + return; + } + + if (! $this->isElasticsearchAvailable()) { + return; + } + + try { + $release = Release::query() + ->where('releases.id', $releaseID) + ->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id') + ->select([ + 'releases.id', + 'releases.name', + 'releases.searchname', + 'releases.fromname', + 'releases.categories_id', + DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename'), + ]) + ->groupBy('releases.id') + ->first(); + + if ($release === null) { + Log::warning('ElasticSearch: Release not found for update', ['id' => $releaseID]); + + return; + } + + $searchNameDotless = $this->createPlainSearchName($release->searchname); + $data = [ + 'body' => [ + 'doc' => [ + 'id' => $release->id, + 'name' => $release->name, + 'searchname' => $release->searchname, + 'plainsearchname' => $searchNameDotless, + 'fromname' => $release->fromname, + 'categories_id' => $release->categories_id, + 'filename' => $release->filename, + ], + 'doc_as_upsert' => true, + ], + 'index' => self::INDEX_RELEASES, + 'id' => $release->id, + ]; + + $client = $this->getClient(); + $client->update($data); + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch updateRelease error: '.$e->getMessage(), [ + 'release_id' => $releaseID, + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch updateRelease unexpected error: '.$e->getMessage(), [ + 'release_id' => $releaseID, + ]); + } + } + + + /** + * Insert a predb record into the index. + * + * @param array $parameters Predb data with 'id', 'title', 'source', 'filename' + */ + public function insertPredb(array $parameters): void + { + if (empty($parameters['id'])) { + Log::warning('ElasticSearch: Cannot insert predb without ID'); + + return; + } + + if (! $this->isElasticsearchAvailable()) { + return; + } + + try { + $data = [ + 'body' => [ + 'id' => $parameters['id'], + 'title' => $parameters['title'] ?? '', + 'source' => $parameters['source'] ?? '', + 'filename' => $parameters['filename'] ?? '', + ], + 'index' => self::INDEX_PREDB, + 'id' => $parameters['id'], + ]; + + $client = $this->getClient(); + $client->index($data); + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch insertPreDb error: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch insertPreDb unexpected error: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } + } + + /** + * Update a predb record in the index. + * + * @param array $parameters Predb data with 'id', 'title', 'source', 'filename' + */ + public function updatePreDb(array $parameters): void + { + if (empty($parameters['id'])) { + Log::warning('ElasticSearch: Cannot update predb without ID'); + + return; + } + + if (! $this->isElasticsearchAvailable()) { + return; + } + + try { + $data = [ + 'body' => [ + 'doc' => [ + 'id' => $parameters['id'], + 'title' => $parameters['title'] ?? '', + 'filename' => $parameters['filename'] ?? '', + 'source' => $parameters['source'] ?? '', + ], + 'doc_as_upsert' => true, + ], + 'index' => self::INDEX_PREDB, + 'id' => $parameters['id'], + ]; + + $client = $this->getClient(); + $client->update($data); + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch updatePreDb error: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch updatePreDb unexpected error: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } + } + + /** + * Delete a release from the index. + * + * @param int $id Release ID + */ + public function deleteRelease(int $id): void + { + if (empty($id) || ! $this->isElasticsearchAvailable()) { + if (empty($id)) { + Log::warning('ElasticSearch: Cannot delete release without ID'); + } + + return; + } + + try { + $client = $this->getClient(); + $client->delete([ + 'index' => self::INDEX_RELEASES, + 'id' => $id, + ]); + + } catch (Missing404Exception $e) { + // Document already deleted, not an error + if (config('app.debug')) { + Log::debug('ElasticSearch deleteRelease: document not found', ['release_id' => $id]); + } + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch deleteRelease error: '.$e->getMessage(), [ + 'release_id' => $id, + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch deleteRelease unexpected error: '.$e->getMessage(), [ + 'release_id' => $id, + ]); + } + } + + /** + * Delete a predb record from the index. + * + * @param int $id Predb ID + */ + public function deletePreDb(int $id): void + { + if (empty($id) || ! $this->isElasticsearchAvailable()) { + if (empty($id)) { + Log::warning('ElasticSearch: Cannot delete predb without ID'); + } + + return; + } + + try { + $client = $this->getClient(); + $client->delete([ + 'index' => self::INDEX_PREDB, + 'id' => $id, + ]); + + } catch (Missing404Exception $e) { + if (config('app.debug')) { + Log::debug('ElasticSearch deletePreDb: document not found', ['predb_id' => $id]); + } + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch deletePreDb error: '.$e->getMessage(), [ + 'predb_id' => $id, + ]); + } catch (\Throwable $e) { + Log::error('ElasticSearch deletePreDb unexpected error: '.$e->getMessage(), [ + 'predb_id' => $id, + ]); + } + } + + /** + * Check if an index exists. + * + * @param string $index Index name + */ + public function indexExists(string $index): bool + { + if (! $this->isElasticsearchAvailable()) { + return false; + } + + try { + $client = $this->getClient(); + + return $client->indices()->exists(['index' => $index]); + } catch (\Throwable $e) { + Log::error('ElasticSearch indexExists error: '.$e->getMessage(), ['index' => $index]); + + return false; + } + } + + /** + * Get cluster health information. + * + * @return array Health information or empty array on failure + */ + public function getClusterHealth(): array + { + if (! $this->isElasticsearchAvailable()) { + return []; + } + + try { + $client = $this->getClient(); + + return $client->cluster()->health(); + } catch (\Throwable $e) { + Log::error('ElasticSearch getClusterHealth error: '.$e->getMessage()); + + return []; + } + } + + /** + * Get index statistics. + * + * @param string $index Index name + * @return array Statistics or empty array on failure + */ + public function getIndexStats(string $index): array + { + if (! $this->isElasticsearchAvailable()) { + return []; + } + + try { + $client = $this->getClient(); + + return $client->indices()->stats(['index' => $index]); + } catch (\Throwable $e) { + Log::error('ElasticSearch getIndexStats error: '.$e->getMessage(), ['index' => $index]); + + return []; + } + } + + /** + * Sanitize search terms for Elasticsearch. + * + * Uses the global sanitize() helper function which properly escapes + * Elasticsearch query string special characters. + * + * @param array|string $terms Search terms + * @return string Sanitized search string + */ + private function sanitizeSearchTerms(array|string $terms): string + { + if (is_array($terms)) { + $terms = implode(' ', array_filter($terms)); + } + + $terms = trim($terms); + if (empty($terms)) { + return ''; + } + + // Use the original sanitize() helper function that properly handles + // Elasticsearch query string escaping + if (function_exists('sanitize')) { + return sanitize($terms); + } + + // Fallback if sanitize function doesn't exist + // Replace dots with spaces for release name searches + $terms = str_replace('.', ' ', $terms); + + // Remove multiple consecutive spaces + $terms = preg_replace('/\s+/', ' ', trim($terms)); + + return $terms; + } + + /** + * Build a cache key for search results. + * + * @param string $prefix Cache key prefix + * @param array $params Parameters to include in key + */ + private function buildCacheKey(string $prefix, array $params): string + { + return 'es_'.$prefix.'_'.md5(serialize($params)); + } + + /** + * Build a search query array. + * + * @param string $index Index name + * @param string $keywords Sanitized keywords + * @param array $fields Fields to search with boosts + * @param int $limit Maximum results + * @param array $options Additional query_string options + * @param bool $includeDateSort Include date sorting + */ + private function buildSearchQuery( + string $index, + string $keywords, + array $fields, + int $limit, + array $options = [], + bool $includeDateSort = true + ): array { + $queryString = array_merge([ + 'query' => $keywords, + 'fields' => $fields, + 'analyze_wildcard' => true, + 'default_operator' => 'and', + ], $options); + + $sort = [['_score' => ['order' => 'desc']]]; + + if ($includeDateSort) { + $sort[] = ['add_date' => ['order' => 'desc', 'unmapped_type' => 'date', 'missing' => '_last']]; + $sort[] = ['post_date' => ['order' => 'desc', 'unmapped_type' => 'date', 'missing' => '_last']]; + } + + return [ + 'scroll' => self::SCROLL_TIMEOUT, + 'index' => $index, + 'body' => [ + 'query' => [ + 'query_string' => $queryString, + ], + 'size' => min($limit, self::MAX_RESULTS), + 'sort' => $sort, + '_source' => ['id'], + 'track_total_hits' => true, + ], + ]; + } + + /** + * Build a release document for indexing. + * + * @param array $parameters Release parameters + */ + private function buildReleaseDocument(array $parameters): array + { + $searchNameDotless = $this->createPlainSearchName($parameters['searchname'] ?? ''); + + return [ + 'body' => [ + 'id' => $parameters['id'], + 'name' => $parameters['name'] ?? '', + 'searchname' => $parameters['searchname'] ?? '', + 'plainsearchname' => $searchNameDotless, + 'fromname' => $parameters['fromname'] ?? '', + 'categories_id' => $parameters['categories_id'] ?? 0, + 'filename' => $parameters['filename'] ?? '', + 'add_date' => now()->format('Y-m-d H:i:s'), + 'post_date' => $parameters['postdate'] ?? now()->format('Y-m-d H:i:s'), + ], + 'index' => self::INDEX_RELEASES, + 'id' => $parameters['id'], + ]; + } + + /** + * Create a plain search name by removing dots and dashes. + * + * @param string $searchName Original search name + */ + private function createPlainSearchName(string $searchName): string + { + return str_replace(['.', '-'], ' ', $searchName); + } + + /** + * Execute a search with scroll support. + * + * @param array $search Search query + * @param bool $fullResults Return full source documents instead of just IDs + */ + protected function executeSearch(array $search, bool $fullResults = false): array + { + if (empty($search) || ! $this->isElasticsearchAvailable()) { + return []; + } + + $scrollId = null; + + try { + $client = $this->getClient(); + + // Log the search query for debugging + if (config('app.debug')) { + Log::debug('ElasticSearch executing search', [ + 'index' => $search['index'] ?? 'unknown', + 'query' => $search['body']['query'] ?? [], + ]); + } + + $results = $client->search($search); + + // Log the number of hits + if (config('app.debug')) { + $totalHits = $results['hits']['total']['value'] ?? $results['hits']['total'] ?? 0; + Log::debug('ElasticSearch search results', [ + 'total_hits' => $totalHits, + 'returned_hits' => count($results['hits']['hits'] ?? []), + ]); + } + + $searchResult = []; + + while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) { + foreach ($results['hits']['hits'] as $result) { + if ($fullResults) { + $searchResult[] = $result['_source']; + } else { + $searchResult[] = $result['_source']['id'] ?? $result['_id']; + } + } + + // Handle scrolling for large result sets + if (! isset($results['_scroll_id'])) { + break; + } + + $scrollId = $results['_scroll_id']; + $results = $client->scroll([ + 'scroll_id' => $scrollId, + 'scroll' => self::SCROLL_TIMEOUT, + ]); + } + + return $searchResult; + + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch search error: '.$e->getMessage()); + + return []; + } catch (\Throwable $e) { + Log::error('ElasticSearch search unexpected error: '.$e->getMessage()); + + return []; + } finally { + // Clean up scroll context + $this->clearScrollContext($scrollId); + } + } + + /** + * Clear a scroll context to free server resources. + * + * @param string|null $scrollId Scroll ID to clear + */ + private function clearScrollContext(?string $scrollId): void + { + if ($scrollId === null) { + return; + } + + try { + $client = $this->getClient(); + $client->clearScroll(['scroll_id' => $scrollId]); + } catch (\Throwable $e) { + // Ignore errors when clearing scroll - it's just cleanup + if (config('app.debug')) { + Log::debug('Failed to clear scroll context: '.$e->getMessage()); + } + } + } + + /** + * Execute a bulk operation. + * + * @param array $params Bulk operation parameters + */ + private function executeBulk(array $params): void + { + if (empty($params['body'])) { + return; + } + + try { + $client = $this->getClient(); + $response = $client->bulk($params); + + if (! empty($response['errors'])) { + foreach ($response['items'] as $item) { + $operation = $item['index'] ?? $item['update'] ?? $item['delete'] ?? []; + if (isset($operation['error'])) { + Log::error('ElasticSearch bulk operation error', [ + 'id' => $operation['_id'] ?? 'unknown', + 'error' => $operation['error'], + ]); + } + } + } + } catch (ElasticsearchException $e) { + Log::error('ElasticSearch bulk error: '.$e->getMessage()); + } catch (\Throwable $e) { + Log::error('ElasticSearch bulk unexpected error: '.$e->getMessage()); + } + } +} + diff --git a/app/Services/Search/ManticoreSearchService.php b/app/Services/Search/ManticoreSearchService.php new file mode 100644 index 000000000..e599f9a14 --- /dev/null +++ b/app/Services/Search/ManticoreSearchService.php @@ -0,0 +1,831 @@ +config = config('manticoresearch'); + $this->connection = ['host' => $this->config['host'], 'port' => $this->config['port']]; + $this->manticoreSearch = new Client($this->connection); + $this->search = new Search($this->manticoreSearch); + $this->cli = new ColorCLI; + } + + /** + * Check if autocomplete is enabled. + */ + public function isAutocompleteEnabled(): bool + { + return ($this->config['autocomplete']['enabled'] ?? true) === true; + } + + /** + * Check if suggest is enabled. + */ + public function isSuggestEnabled(): bool + { + return ($this->config['suggest']['enabled'] ?? true) === true; + } + + /** + * Get the releases index name. + */ + public function getReleasesIndex(): string + { + return $this->config['indexes']['releases'] ?? 'releases_rt'; + } + + /** + * Get the predb index name. + */ + public function getPredbIndex(): string + { + return $this->config['indexes']['predb'] ?? 'predb_rt'; + } + + /** + * Insert release into ManticoreSearch releases_rt realtime index + */ + public function insertRelease(array $parameters): void + { + if (empty($parameters['id'])) { + Log::warning('ManticoreSearch: Cannot insert release without ID'); + + return; + } + + try { + $document = [ + 'name' => $parameters['name'] ?? '', + 'searchname' => $parameters['searchname'] ?? '', + 'fromname' => $parameters['fromname'] ?? '', + 'categories_id' => (string) ($parameters['categories_id'] ?? ''), + 'filename' => $parameters['filename'] ?? '', + ]; + + $this->manticoreSearch->table($this->config['indexes']['releases']) + ->replaceDocument($document, $parameters['id']); + + } catch (ResponseException $e) { + Log::error('ManticoreSearch insertRelease ResponseException: '.$e->getMessage(), [ + 'release_id' => $parameters['id'], + 'index' => $this->config['indexes']['releases'], + ]); + } catch (RuntimeException $e) { + Log::error('ManticoreSearch insertRelease RuntimeException: '.$e->getMessage(), [ + 'release_id' => $parameters['id'], + ]); + } catch (\Throwable $e) { + Log::error('ManticoreSearch insertRelease unexpected error: '.$e->getMessage(), [ + 'release_id' => $parameters['id'], + 'trace' => $e->getTraceAsString(), + ]); + } + } + + /** + * Insert release into Manticore RT table. + */ + public function insertPredb(array $parameters): void + { + if (empty($parameters['id'])) { + Log::warning('ManticoreSearch: Cannot insert predb without ID'); + + return; + } + + try { + $document = [ + 'title' => $parameters['title'] ?? '', + 'filename' => $parameters['filename'] ?? '', + 'source' => $parameters['source'] ?? '', + ]; + + $this->manticoreSearch->table($this->config['indexes']['predb']) + ->replaceDocument($document, $parameters['id']); + + } catch (ResponseException $e) { + Log::error('ManticoreSearch insertPredb ResponseException: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } catch (RuntimeException $e) { + Log::error('ManticoreSearch insertPredb RuntimeException: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } catch (\Throwable $e) { + Log::error('ManticoreSearch insertPredb unexpected error: '.$e->getMessage(), [ + 'predb_id' => $parameters['id'], + ]); + } + } + + /** + * Delete release from Manticore RT tables. + * + * @param int $id Release ID + */ + public function deleteRelease(int $id): void + { + if (empty($id)) { + Log::warning('ManticoreSearch: Cannot delete release without ID'); + + return; + } + + try { + $this->manticoreSearch->table($this->config['indexes']['releases']) + ->deleteDocument($id); + } catch (ResponseException $e) { + Log::error('ManticoreSearch deleteRelease error: '.$e->getMessage(), [ + 'id' => $id, + ]); + } + } + + /** + * Delete release from Manticore RT tables by GUID. + * + * @param array $identifiers ['g' => Release GUID(mandatory), 'id' => ReleaseID(optional, pass false)] + */ + public function deleteReleaseByGuid(array $identifiers): void + { + if (empty($identifiers['g'])) { + Log::warning('ManticoreSearch: Cannot delete release without GUID'); + + return; + } + + try { + if ($identifiers['i'] === false || empty($identifiers['i'])) { + $release = Release::query()->where('guid', $identifiers['g'])->first(['id']); + $identifiers['i'] = $release?->id; + } + + if (! empty($identifiers['i'])) { + $this->manticoreSearch->table($this->config['indexes']['releases']) + ->deleteDocument($identifiers['i']); + } else { + Log::warning('ManticoreSearch: Could not find release ID for deletion', [ + 'guid' => $identifiers['g'], + ]); + } + } catch (ResponseException $e) { + Log::error('ManticoreSearch deleteRelease error: '.$e->getMessage(), [ + 'guid' => $identifiers['g'], + 'id' => $identifiers['i'] ?? null, + ]); + } + } + + /** + * Escapes characters that are treated as special operators by the query language parser. + */ + public static function escapeString(string $string): string + { + if ($string === '*' || empty($string)) { + return ''; + } + + $from = ['\\', '(', ')', '@', '~', '"', '&', '/', '$', '=', "'", '--', '[', ']', '!', '-']; + $to = ['\\\\', '\(', '\)', '\@', '\~', '\"', '\&', '\/', '\$', '\=', "\'", '\--', '\[', '\]', '\!', '\-']; + + $string = str_replace($from, $to, $string); + + // Clean up trailing special characters + $string = rtrim($string, '-!'); + + return trim($string); + } + + public function updateRelease(int|string $releaseID): void + { + if (empty($releaseID)) { + Log::warning('ManticoreSearch: Cannot update release without ID'); + + return; + } + + try { + $release = Release::query() + ->where('releases.id', $releaseID) + ->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id') + ->select([ + 'releases.id', + 'releases.name', + 'releases.searchname', + 'releases.fromname', + 'releases.categories_id', + DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename'), + ]) + ->groupBy('releases.id') + ->first(); + + if ($release !== null) { + $this->insertRelease($release->toArray()); + } else { + Log::warning('ManticoreSearch: Release not found for update', ['id' => $releaseID]); + } + } catch (\Throwable $e) { + Log::error('ManticoreSearch updateRelease error: '.$e->getMessage(), [ + 'release_id' => $releaseID, + ]); + } + } + + /** + * Update Manticore Predb index for given predb_id. + */ + public function updatePreDb(array $parameters): void + { + if (empty($parameters)) { + Log::warning('ManticoreSearch: Cannot update predb with empty parameters'); + + return; + } + + $this->insertPredb($parameters); + } + + public function truncateRTIndex(array $indexes = []): bool + { + if (empty($indexes)) { + $this->cli->error('You need to provide index name to truncate'); + + return false; + } + + $success = true; + foreach ($indexes as $index) { + if (! \in_array($index, $this->config['indexes'], true)) { + $this->cli->error('Unsupported index: '.$index); + $success = false; + + continue; + } + + try { + $this->manticoreSearch->table($index)->truncate(); + $this->cli->info('Truncating index '.$index.' finished.'); + } catch (ResponseException $e) { + if ($e->getMessage() === 'Invalid index') { + $this->createIndexIfNotExists($index); + } else { + $this->cli->error('Error truncating index '.$index.': '.$e->getMessage()); + $success = false; + } + } catch (\Throwable $e) { + $this->cli->error('Unexpected error truncating index '.$index.': '.$e->getMessage()); + $success = false; + } + } + + return $success; + } + + /** + * Create index if it doesn't exist + */ + private function createIndexIfNotExists(string $index): void + { + try { + if ($index === 'releases_rt') { + $this->manticoreSearch->table($index)->create([ + 'name' => ['type' => 'string'], + 'searchname' => ['type' => 'string'], + 'fromname' => ['type' => 'string'], + 'filename' => ['type' => 'string'], + 'categories_id' => ['type' => 'integer'], + ]); + $this->cli->info('Created releases_rt index'); + } elseif ($index === 'predb_rt') { + $this->manticoreSearch->table($index)->create([ + 'title' => ['type' => 'string'], + 'filename' => ['type' => 'string'], + 'source' => ['type' => 'string'], + ]); + $this->cli->info('Created predb_rt index'); + } + } catch (\Throwable $e) { + $this->cli->error('Error creating index '.$index.': '.$e->getMessage()); + } + } + + /** + * Optimize the RT indices. + */ + public function optimizeRTIndex(): bool + { + $success = true; + + foreach ($this->config['indexes'] as $index) { + try { + $this->manticoreSearch->table($index)->flush(); + $this->manticoreSearch->table($index)->optimize(); + Log::info("Successfully optimized index: {$index}"); + } catch (ResponseException $e) { + Log::error('Failed to optimize index '.$index.': '.$e->getMessage()); + $success = false; + } catch (\Throwable $e) { + Log::error('Unexpected error optimizing index '.$index.': '.$e->getMessage()); + $success = false; + } + } + + return $success; + } + + /** + * Search releases index. + * + * @param array|string $phrases Search phrases + * @param int $limit Maximum number of results + * @return array Array of release IDs + */ + public function searchReleases(array|string $phrases, int $limit = 1000): array + { + $searchArray = is_array($phrases) ? $phrases : ['searchname' => $phrases]; + + $result = $this->searchIndexes($this->getReleasesIndex(), '', [], $searchArray); + + return ! empty($result) ? ($result['id'] ?? []) : []; + } + + /** + * Search predb index. + * + * @param array|string $searchTerm Search term(s) + * @return array Array of predb records + */ + public function searchPredb(array|string $searchTerm): array + { + $searchString = is_array($searchTerm) ? implode(' ', $searchTerm) : $searchTerm; + + $result = $this->searchIndexes($this->getPredbIndex(), $searchString, ['title', 'filename'], []); + + return $result['data'] ?? []; + } + + public function searchIndexes(string $rt_index, ?string $searchString, array $column = [], array $searchArray = []): array + { + if (empty($rt_index)) { + Log::warning('ManticoreSearch: Index name is required for search'); + + return []; + } + + // Create cache key for search results + $cacheKey = md5(serialize([ + 'index' => $rt_index, + 'search' => $searchString, + 'columns' => $column, + 'array' => $searchArray, + ])); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + // Build query string once so we can retry if needed + $searchExpr = null; + if (! empty($searchArray)) { + $terms = []; + foreach ($searchArray as $key => $value) { + if (! empty($value)) { + $escapedValue = self::escapeString($value); + if (! empty($escapedValue)) { + $terms[] = '@@relaxed @'.$key.' '.$escapedValue; + } + } + } + if (! empty($terms)) { + $searchExpr = implode(' ', $terms); + } else { + return []; + } + } elseif (! empty($searchString)) { + $escapedSearch = self::escapeString($searchString); + if (empty($escapedSearch)) { + return []; + } + + $searchColumns = ''; + if (! empty($column)) { + if (count($column) > 1) { + $searchColumns = '@('.implode(',', $column).')'; + } else { + $searchColumns = '@'.$column[0]; + } + } + + $searchExpr = '@@relaxed '.$searchColumns.' '.$escapedSearch; + } else { + return []; + } + + // Avoid explicit sort for predb_rt to prevent Manticore's "too many sort-by attributes" error + $avoidSortForIndex = ($rt_index === 'predb_rt'); + + try { + // Use a fresh Search instance for every query to avoid parameter accumulation across calls + $query = (new Search($this->manticoreSearch)) + ->setTable($rt_index) + ->option('ranker', 'sph04') + ->maxMatches(10000) + ->limit(10000) + ->stripBadUtf8(true) + ->search($searchExpr); + + if (! $avoidSortForIndex) { + $query->sort('id', 'desc'); + } + + $results = $query->get(); + } catch (ResponseException $e) { + // If we hit Manticore's "too many sort-by attributes" limit, retry once without explicit sorting + if (stripos($e->getMessage(), 'too many sort-by attributes') !== false) { + try { + $query = (new Search($this->manticoreSearch)) + ->setTable($rt_index) + ->option('ranker', 'sph04') + ->maxMatches(10000) + ->limit(10000) + ->stripBadUtf8(true) + ->search($searchExpr); + + $results = $query->get(); + + Log::warning('ManticoreSearch: Retried search without sorting due to sort-by attributes limit', [ + 'index' => $rt_index, + ]); + } catch (ResponseException $e2) { + Log::error('ManticoreSearch searchIndexes ResponseException after retry: '.$e2->getMessage(), [ + 'index' => $rt_index, + 'search' => $searchString, + ]); + + return []; + } + } else { + Log::error('ManticoreSearch searchIndexes ResponseException: '.$e->getMessage(), [ + 'index' => $rt_index, + 'search' => $searchString, + ]); + + return []; + } + } catch (RuntimeException $e) { + Log::error('ManticoreSearch searchIndexes RuntimeException: '.$e->getMessage(), [ + 'index' => $rt_index, + 'search' => $searchString, + ]); + + return []; + } catch (\Throwable $e) { + Log::error('ManticoreSearch searchIndexes unexpected error: '.$e->getMessage(), [ + 'index' => $rt_index, + 'search' => $searchString, + ]); + + return []; + } + + // Parse results and cache + $resultIds = []; + $resultData = []; + foreach ($results as $doc) { + $resultIds[] = $doc->getId(); + $resultData[] = $doc->getData(); + } + + $result = [ + 'id' => $resultIds, + 'data' => $resultData, + ]; + + // Cache results for 5 minutes + Cache::put($cacheKey, $result, now()->addMinutes(5)); + + return $result; + } + + /** + * Get autocomplete suggestions for a search query. + * Searches the releases index and returns matching searchnames. + * + * @param string $query The partial search query + * @param string|null $index Index to search (defaults to releases index) + * @return array + */ + public function autocomplete(string $query, ?string $index = null): array + { + $autocompleteConfig = $this->config['autocomplete'] ?? [ + 'enabled' => true, + 'min_length' => 2, + 'max_results' => 10, + 'cache_minutes' => 10, + ]; + + if (! ($autocompleteConfig['enabled'] ?? true)) { + return []; + } + + $query = trim($query); + $minLength = $autocompleteConfig['min_length'] ?? 2; + if (strlen($query) < $minLength) { + return []; + } + + $index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt'); + $cacheKey = 'manticore:autocomplete:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + $maxResults = $autocompleteConfig['max_results'] ?? 10; + + try { + // Search releases index for matching searchnames + $escapedQuery = self::escapeString($query); + if (empty($escapedQuery)) { + return []; + } + + // Use relaxed search on searchname field + $searchExpr = '@@relaxed @searchname '.$escapedQuery; + + $search = (new Search($this->manticoreSearch)) + ->setTable($index) + ->search($searchExpr) + ->sort('id', 'desc') + ->limit($maxResults * 3) + ->stripBadUtf8(true); + + $results = $search->get(); + + $seen = []; + foreach ($results as $doc) { + $data = $doc->getData(); + $searchname = $data['searchname'] ?? ''; + + if (empty($searchname)) { + continue; + } + + // Create a clean suggestion from the searchname + $suggestion = $this->extractSuggestion($searchname, $query); + + if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) { + $seen[strtolower($suggestion)] = true; + $suggestions[] = [ + 'suggest' => $suggestion, + 'distance' => 0, + 'docs' => 1, + ]; + } + + if (count($suggestions) >= $maxResults) { + break; + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::warning('ManticoreSearch autocomplete error: '.$e->getMessage()); + } + } + + if (! empty($suggestions)) { + $cacheMinutes = (int) ($autocompleteConfig['cache_minutes'] ?? 10); + Cache::put($cacheKey, $suggestions, now()->addMinutes($cacheMinutes)); + } + + return $suggestions; + } + + /** + * Extract a clean suggestion from a searchname. + * + * @param string $searchname The full searchname + * @param string $query The user's query + * @return string|null The extracted suggestion + */ + private function extractSuggestion(string $searchname, string $query): ?string + { + // Clean up the searchname - remove file extensions, quality tags at the end + $clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname); + + // Replace dots and underscores with spaces for readability + $clean = str_replace(['.', '_'], ' ', $clean); + + // Remove multiple spaces + $clean = preg_replace('/\s+/', ' ', $clean); + $clean = trim($clean); + + if (empty($clean)) { + return null; + } + + // If the clean name is reasonable length, use it + if (strlen($clean) <= 80) { + return $clean; + } + + // For very long names, try to extract the relevant part + // Find where the query matches and extract context around it + $pos = stripos($clean, $query); + if ($pos !== false) { + // Get up to 80 chars starting from the match position, or from beginning if match is early + $start = max(0, $pos - 10); + $extracted = substr($clean, $start, 80); + + // Clean up - don't cut mid-word + if ($start > 0) { + $extracted = preg_replace('/^\S*\s/', '', $extracted); + } + $extracted = preg_replace('/\s\S*$/', '', $extracted); + + return trim($extracted); + } + + // Fallback: just truncate + return substr($clean, 0, 80); + } + + /** + * Get spell correction suggestions ("Did you mean?"). + * + * @param string $query The search query to check + * @param string|null $index Index to use for suggestions + * @return array + */ + public function suggest(string $query, ?string $index = null): array + { + $suggestConfig = $this->config['suggest'] ?? [ + 'enabled' => true, + 'max_edits' => 4, + ]; + + if (! ($suggestConfig['enabled'] ?? true)) { + return []; + } + + $query = trim($query); + if (empty($query)) { + return []; + } + + $index = $index ?? ($this->config['indexes']['releases'] ?? 'releases_rt'); + $cacheKey = 'manticore:suggest:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + + try { + // Try native CALL SUGGEST first + $result = $this->manticoreSearch->suggest([ + 'table' => $index, + 'body' => [ + 'query' => $query, + 'options' => [ + 'limit' => 5, + 'max_edits' => $suggestConfig['max_edits'] ?? 4, + ], + ], + ]); + + if (! empty($result) && is_array($result)) { + foreach ($result as $item) { + if (isset($item['suggest']) && $item['suggest'] !== $query) { + $suggestions[] = [ + 'suggest' => $item['suggest'], + 'distance' => $item['distance'] ?? 0, + 'docs' => $item['docs'] ?? 0, + ]; + } + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::debug('ManticoreSearch native suggest failed: '.$e->getMessage()); + } + } + + // If native suggest didn't return results, try a fuzzy search fallback + if (empty($suggestions)) { + $suggestions = $this->suggestFallback($query, $index); + } + + if (! empty($suggestions)) { + Cache::put($cacheKey, $suggestions, now()->addMinutes(5)); + } + + return $suggestions; + } + + /** + * Fallback suggest using similar searchname matches. + * + * @param string $query The search query + * @param string $index Index to search + * @return array + */ + private function suggestFallback(string $query, string $index): array + { + try { + $escapedQuery = self::escapeString($query); + if (empty($escapedQuery)) { + return []; + } + + // Use relaxed search to find partial matches + $searchExpr = '@@relaxed @searchname '.$escapedQuery; + + $search = (new Search($this->manticoreSearch)) + ->setTable($index) + ->search($searchExpr) + ->limit(20) + ->stripBadUtf8(true); + + $results = $search->get(); + + // Extract common terms from the results that differ from the query + $termCounts = []; + foreach ($results as $doc) { + $data = $doc->getData(); + $searchname = $data['searchname'] ?? ''; + + // Extract words from searchname + $words = preg_split('/[\s.\-_]+/', strtolower($searchname)); + foreach ($words as $word) { + if (strlen($word) >= 3 && $word !== strtolower($query)) { + // Check if word is similar to query (within edit distance) + $distance = levenshtein(strtolower($query), $word); + if ($distance > 0 && $distance <= 3) { + if (! isset($termCounts[$word])) { + $termCounts[$word] = ['count' => 0, 'distance' => $distance]; + } + $termCounts[$word]['count']++; + } + } + } + } + + // Sort by count (most common first) + uasort($termCounts, fn ($a, $b) => $b['count'] - $a['count']); + + $suggestions = []; + foreach (array_slice($termCounts, 0, 5, true) as $term => $data) { + $suggestions[] = [ + 'suggest' => $term, + 'distance' => $data['distance'], + 'docs' => $data['count'], + ]; + } + + return $suggestions; + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::debug('ManticoreSearch suggest fallback error: '.$e->getMessage()); + } + + return []; + } + } +} + diff --git a/bootstrap/providers.php b/bootstrap/providers.php index f8a24f937..8787333a7 100644 --- a/bootstrap/providers.php +++ b/bootstrap/providers.php @@ -12,6 +12,7 @@ return [ App\Providers\TvProcessingServiceProvider::class, App\Providers\UserServiceProvider::class, App\Providers\VoltServiceProvider::class, + App\Providers\SearchServiceProvider::class, ]; diff --git a/misc/testing/Dev/test-ReleaseCleaner.php b/misc/testing/Dev/test-ReleaseCleaner.php index ca7813d6a..4ac37b061 100755 --- a/misc/testing/Dev/test-ReleaseCleaner.php +++ b/misc/testing/Dev/test-ReleaseCleaner.php @@ -4,7 +4,7 @@ require_once dirname(__DIR__, 3).DIRECTORY_SEPARATOR.'bootstrap/autoload.php'; use App\Models\Release; use App\Models\UsenetGroup; -use Blacklight\ManticoreSearch; +use App\Services\Search\ManticoreSearchService; use Blacklight\ReleaseCleaning; $message = @@ -52,7 +52,7 @@ if (\count($releases) === 0) { } $releaseCleaner = new ReleaseCleaning; -$manticore = new ManticoreSearch; +$manticore = app(ManticoreSearchService::class); foreach ($releases as $release) { echo '.'; diff --git a/tests/Unit/MediaProcessingServiceTest.php b/tests/Unit/MediaProcessingServiceTest.php index f99024cb3..0f89a5f9c 100644 --- a/tests/Unit/MediaProcessingServiceTest.php +++ b/tests/Unit/MediaProcessingServiceTest.php @@ -5,8 +5,8 @@ namespace Tests\Unit; use App\Models\Release as ReleaseModel; use App\Services\Categorization\CategorizationService; use App\Services\MediaProcessingService; -use Blacklight\ElasticSearchSiteSearch; -use Blacklight\ManticoreSearch; +use App\Services\Search\ElasticSearchService; +use App\Services\Search\ManticoreSearchService; use Blacklight\ReleaseExtra; use Blacklight\ReleaseImage; use FFMpeg\FFMpeg; @@ -51,8 +51,8 @@ class MediaProcessingServiceTest extends TestCase ?MediaInfo $mediaInfo = null, ?ReleaseImage $releaseImage = null, ?ReleaseExtra $releaseExtra = null, - ?ManticoreSearch $manticore = null, - ?ElasticSearchSiteSearch $elastic = null, + ?ManticoreSearchService $manticore = null, + ?ElasticSearchService $elastic = null, ?CategorizationService $categorize = null ): MediaProcessingService { $ffmpeg ??= Mockery::mock(FFMpeg::class); @@ -60,8 +60,8 @@ class MediaProcessingServiceTest extends TestCase $mediaInfo ??= Mockery::mock(MediaInfo::class); $releaseImage ??= Mockery::mock(ReleaseImage::class); $releaseExtra ??= Mockery::mock(ReleaseExtra::class); - $manticore ??= Mockery::mock(ManticoreSearch::class); - $elastic ??= Mockery::mock(ElasticSearchSiteSearch::class); + $manticore ??= Mockery::mock(ManticoreSearchService::class); + $elastic ??= Mockery::mock(ElasticSearchService::class); $categorize ??= Mockery::mock(CategorizationService::class); return new MediaProcessingService($ffmpeg, $ffprobe, $mediaInfo, $releaseImage, $releaseExtra, $manticore, $elastic, $categorize);