From 6ef4093a824621157dfdeee2d0e481fde1428e17 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Wed, 17 Dec 2025 16:48:49 +0100 Subject: [PATCH] Add autosearch for elastic too --- .env.example | 7 + Blacklight/ElasticSearchSiteSearch.php | 352 ++++++++++++++++++ .../Controllers/SearchSuggestController.php | 90 +++-- config/elasticsearch.php | 27 ++ 4 files changed, 439 insertions(+), 37 deletions(-) diff --git a/.env.example b/.env.example index a419ac3c6..824dfea93 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,13 @@ ELASTICSEARCH_USER= ELASTICSEARCH_PASS= ELASTICSEARCH_LOGGING=false ELASTICSEARCH_ENABLED=false +# Elasticsearch autocomplete settings +ELASTICSEARCH_AUTOCOMPLETE_ENABLED=true +ELASTICSEARCH_AUTOCOMPLETE_MIN_LENGTH=2 +ELASTICSEARCH_AUTOCOMPLETE_MAX_RESULTS=10 +ELASTICSEARCH_AUTOCOMPLETE_CACHE=10 +# Elasticsearch suggest/spell correction settings +ELASTICSEARCH_SUGGEST_ENABLED=true NNTP_COMPRESSED_HEADERS=false USE_ALTERNATE_NNTP_SERVER=false diff --git a/Blacklight/ElasticSearchSiteSearch.php b/Blacklight/ElasticSearchSiteSearch.php index c3d1ac6ac..39d197842 100644 --- a/Blacklight/ElasticSearchSiteSearch.php +++ b/Blacklight/ElasticSearchSiteSearch.php @@ -38,6 +38,12 @@ class ElasticSearchSiteSearch private const INDEX_PREDB = 'predb'; + private const AUTOCOMPLETE_CACHE_MINUTES = 10; + + private const AUTOCOMPLETE_MAX_RESULTS = 10; + + private const AUTOCOMPLETE_MIN_LENGTH = 2; + private static ?Client $client = null; private static ?bool $availabilityCache = null; @@ -180,6 +186,352 @@ class ElasticSearchSiteSearch self::$availabilityCacheTime = null; } + /** + * Check if autocomplete is enabled. + */ + public function isAutocompleteEnabled(): bool + { + return config('elasticsearch.autocomplete.enabled', true) && $this->isElasticsearchAvailable(); + } + + /** + * Check if suggest is enabled. + */ + public function isSuggestEnabled(): bool + { + return config('elasticsearch.suggest.enabled', true) && $this->isElasticsearchAvailable(); + } + + /** + * Get the releases index name. + */ + public function getReleasesIndex(): string + { + return self::INDEX_RELEASES; + } + + /** + * Get autocomplete suggestions for a search query. + * Searches the releases index and returns matching searchnames. + * + * @param string $query The partial search query + * @param string|null $index Index to search (defaults to releases index) + * @return array + */ + public function autocomplete(string $query, ?string $index = null): array + { + if (! $this->isAutocompleteEnabled()) { + return []; + } + + $query = trim($query); + $minLength = (int) config('elasticsearch.autocomplete.min_length', self::AUTOCOMPLETE_MIN_LENGTH); + if (strlen($query) < $minLength) { + return []; + } + + $index = $index ?? self::INDEX_RELEASES; + $cacheKey = 'es:autocomplete:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + $maxResults = (int) config('elasticsearch.autocomplete.max_results', self::AUTOCOMPLETE_MAX_RESULTS); + + try { + $client = $this->getClient(); + + // Use a prefix/match query on searchname field + $searchParams = [ + 'index' => $index, + 'body' => [ + 'query' => [ + 'bool' => [ + 'should' => [ + // Prefix match for autocomplete-like behavior + [ + 'match_phrase_prefix' => [ + 'searchname' => [ + 'query' => $query, + 'max_expansions' => 50, + ], + ], + ], + // Also include regular match for better results + [ + 'match' => [ + 'searchname' => [ + 'query' => $query, + 'fuzziness' => 'AUTO', + ], + ], + ], + ], + 'minimum_should_match' => 1, + ], + ], + 'size' => $maxResults * 3, + '_source' => ['searchname'], + 'sort' => [ + '_score' => ['order' => 'desc'], + ], + ], + ]; + + $response = $client->search($searchParams); + + $seen = []; + if (isset($response['hits']['hits'])) { + foreach ($response['hits']['hits'] as $hit) { + $searchname = $hit['_source']['searchname'] ?? ''; + + if (empty($searchname)) { + continue; + } + + // Create a clean suggestion from the searchname + $suggestion = $this->extractSuggestion($searchname, $query); + + if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) { + $seen[strtolower($suggestion)] = true; + $suggestions[] = [ + 'suggest' => $suggestion, + 'distance' => 0, + 'docs' => 1, + ]; + } + + if (count($suggestions) >= $maxResults) { + break; + } + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::warning('ElasticSearch autocomplete error: '.$e->getMessage()); + } + } + + if (! empty($suggestions)) { + $cacheMinutes = (int) config('elasticsearch.autocomplete.cache_minutes', self::AUTOCOMPLETE_CACHE_MINUTES); + Cache::put($cacheKey, $suggestions, now()->addMinutes($cacheMinutes)); + } + + return $suggestions; + } + + /** + * Get spell correction suggestions ("Did you mean?"). + * + * @param string $query The search query to check + * @param string|null $index Index to use for suggestions + * @return array + */ + public function suggest(string $query, ?string $index = null): array + { + if (! $this->isSuggestEnabled()) { + return []; + } + + $query = trim($query); + if (empty($query)) { + return []; + } + + $index = $index ?? self::INDEX_RELEASES; + $cacheKey = 'es:suggest:'.md5($index.$query); + + $cached = Cache::get($cacheKey); + if ($cached !== null) { + return $cached; + } + + $suggestions = []; + + try { + $client = $this->getClient(); + + // Use Elasticsearch suggest API with phrase suggester + $searchParams = [ + 'index' => $index, + 'body' => [ + 'suggest' => [ + 'text' => $query, + 'searchname_suggest' => [ + 'phrase' => [ + 'field' => 'searchname', + 'size' => 5, + 'gram_size' => 3, + 'direct_generator' => [ + [ + 'field' => 'searchname', + 'suggest_mode' => 'popular', + ], + ], + 'highlight' => [ + 'pre_tag' => '', + 'post_tag' => '', + ], + ], + ], + ], + ], + ]; + + $response = $client->search($searchParams); + + if (isset($response['suggest']['searchname_suggest'][0]['options'])) { + foreach ($response['suggest']['searchname_suggest'][0]['options'] as $option) { + $suggestedText = $option['text'] ?? ''; + if (! empty($suggestedText) && strtolower($suggestedText) !== strtolower($query)) { + $suggestions[] = [ + 'suggest' => $suggestedText, + 'distance' => 1, + 'docs' => (int) ($option['freq'] ?? 1), + ]; + } + } + } + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::debug('ElasticSearch native suggest failed: '.$e->getMessage()); + } + } + + // Fallback: if native suggest didn't work, use fuzzy search + if (empty($suggestions)) { + $suggestions = $this->suggestFallback($query, $index); + } + + if (! empty($suggestions)) { + Cache::put($cacheKey, $suggestions, now()->addMinutes(self::CACHE_TTL_MINUTES)); + } + + return $suggestions; + } + + /** + * Fallback suggest using fuzzy search on searchnames. + * + * @param string $query The search query + * @param string $index Index to search + * @return array + */ + private function suggestFallback(string $query, string $index): array + { + try { + $client = $this->getClient(); + + // Use fuzzy match to find similar terms + $searchParams = [ + 'index' => $index, + 'body' => [ + 'query' => [ + 'match' => [ + 'searchname' => [ + 'query' => $query, + 'fuzziness' => 'AUTO', + ], + ], + ], + 'size' => 20, + '_source' => ['searchname'], + ], + ]; + + $response = $client->search($searchParams); + + // Extract common terms from results that differ from the query + $termCounts = []; + if (isset($response['hits']['hits'])) { + foreach ($response['hits']['hits'] as $hit) { + $searchname = $hit['_source']['searchname'] ?? ''; + $words = preg_split('/[\s.\-_]+/', strtolower($searchname)); + + foreach ($words as $word) { + if (strlen($word) >= 3 && $word !== strtolower($query)) { + $distance = levenshtein(strtolower($query), $word); + if ($distance > 0 && $distance <= 3) { + if (! isset($termCounts[$word])) { + $termCounts[$word] = ['count' => 0, 'distance' => $distance]; + } + $termCounts[$word]['count']++; + } + } + } + } + } + + // Sort by count + uasort($termCounts, fn ($a, $b) => $b['count'] - $a['count']); + + $suggestions = []; + foreach (array_slice($termCounts, 0, 5, true) as $term => $data) { + $suggestions[] = [ + 'suggest' => $term, + 'distance' => $data['distance'], + 'docs' => $data['count'], + ]; + } + + return $suggestions; + } catch (\Throwable $e) { + if (config('app.debug')) { + Log::warning('ElasticSearch suggest fallback error: '.$e->getMessage()); + } + + return []; + } + } + + /** + * Extract a clean suggestion from a searchname. + * + * @param string $searchname The full searchname + * @param string $query The user's query + * @return string|null The extracted suggestion + */ + private function extractSuggestion(string $searchname, string $query): ?string + { + // Clean up the searchname - remove file extensions + $clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname); + + // Replace dots and underscores with spaces for readability + $clean = str_replace(['.', '_'], ' ', $clean); + + // Remove multiple spaces + $clean = preg_replace('/\s+/', ' ', $clean); + $clean = trim($clean); + + if (empty($clean)) { + return null; + } + + // If the clean name is reasonable length, use it + if (strlen($clean) <= 80) { + return $clean; + } + + // For very long names, try to extract the relevant part + $pos = stripos($clean, $query); + if ($pos !== false) { + $start = max(0, $pos - 10); + $extracted = substr($clean, $start, 80); + + if ($start > 0) { + $extracted = preg_replace('/^\S*\s/', '', $extracted); + } + $extracted = preg_replace('/\s\S*$/', '', $extracted); + + return trim($extracted); + } + + return substr($clean, 0, 80); + } + /** * Search releases index. * diff --git a/app/Http/Controllers/SearchSuggestController.php b/app/Http/Controllers/SearchSuggestController.php index 207e73868..d933663cc 100644 --- a/app/Http/Controllers/SearchSuggestController.php +++ b/app/Http/Controllers/SearchSuggestController.php @@ -2,6 +2,7 @@ namespace App\Http\Controllers; +use Blacklight\ElasticSearchSiteSearch; use Blacklight\ManticoreSearch; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; @@ -9,11 +10,16 @@ use Illuminate\Support\Facades\RateLimiter; class SearchSuggestController extends Controller { - private ManticoreSearch $manticore; + private ManticoreSearch|ElasticSearchSiteSearch $searchEngine; public function __construct() { - $this->manticore = new ManticoreSearch; + // Use Elasticsearch if enabled, otherwise fall back to ManticoreSearch + if (config('nntmux.elasticsearch_enabled', false)) { + $this->searchEngine = new ElasticSearchSiteSearch; + } else { + $this->searchEngine = new ManticoreSearch; + } } /** @@ -22,8 +28,8 @@ class SearchSuggestController extends Controller public function autocomplete(Request $request): JsonResponse { $query = $request->input('q', ''); - // Use the index from ManticoreSearch config, or allow override via request - $index = $request->input('index') ?? $this->manticore->getReleasesIndex(); + // Use the index from search engine config, or allow override via request + $index = $request->input('index') ?? $this->searchEngine->getReleasesIndex(); // Rate limiting: 60 requests per minute per IP $key = 'autocomplete:'.$request->ip(); @@ -35,7 +41,7 @@ class SearchSuggestController extends Controller } RateLimiter::hit($key, 60); - if (! $this->manticore->isAutocompleteEnabled()) { + if (! $this->searchEngine->isAutocompleteEnabled()) { return response()->json([ 'success' => true, 'suggestions' => [], @@ -50,7 +56,7 @@ class SearchSuggestController extends Controller } try { - $suggestions = $this->manticore->autocomplete($query, $index); + $suggestions = $this->searchEngine->autocomplete($query, $index); return response()->json([ 'success' => true, @@ -72,7 +78,7 @@ class SearchSuggestController extends Controller public function suggest(Request $request): JsonResponse { $query = $request->input('q', ''); - $index = $request->input('index', 'releases_rt'); + $index = $request->input('index') ?? $this->searchEngine->getReleasesIndex(); // Rate limiting: 30 requests per minute per IP $key = 'suggest:'.$request->ip(); @@ -84,11 +90,10 @@ class SearchSuggestController extends Controller } RateLimiter::hit($key, 60); - if (! $this->manticore->isSuggestEnabled()) { + if (! $this->searchEngine->isSuggestEnabled()) { return response()->json([ - 'success' => false, + 'success' => true, 'suggestions' => [], - 'error' => 'Suggest is disabled', ]); } @@ -99,22 +104,29 @@ class SearchSuggestController extends Controller ]); } - $suggestions = $this->manticore->suggest($query, $index); + try { + $suggestions = $this->searchEngine->suggest($query, $index); - // Get the best suggestion (highest doc count) - $bestSuggestion = null; - if (! empty($suggestions)) { - usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']); - $bestSuggestion = $suggestions[0]['suggest'] ?? null; + // Get the best suggestion (highest doc count) + $bestSuggestion = null; + if (! empty($suggestions)) { + usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']); + $bestSuggestion = $suggestions[0]['suggest'] ?? null; + } + + return response()->json([ + 'success' => true, + 'query' => $query, + 'suggestions' => array_map(fn ($s) => $s['suggest'], $suggestions), + 'best' => $bestSuggestion, + ]); + } catch (\Throwable $e) { + return response()->json([ + 'success' => true, + 'query' => $query, + 'suggestions' => [], + ]); } - - return response()->json([ - 'success' => true, - 'query' => $query, - 'suggestions' => array_map(fn ($s) => $s['suggest'], $suggestions), - 'best' => $bestSuggestion, - 'details' => $suggestions, - ]); } /** @@ -123,7 +135,7 @@ class SearchSuggestController extends Controller public function searchAssist(Request $request): JsonResponse { $query = $request->input('q', ''); - $index = $request->input('index', 'releases_rt'); + $index = $request->input('index') ?? $this->searchEngine->getReleasesIndex(); // Rate limiting $key = 'searchassist:'.$request->ip(); @@ -142,19 +154,23 @@ class SearchSuggestController extends Controller 'suggest' => null, ]; - // Get autocomplete suggestions - if ($this->manticore->isAutocompleteEnabled() && strlen($query) >= 2) { - $autocomplete = $this->manticore->autocomplete($query, $index); - $response['autocomplete'] = array_map(fn ($s) => $s['suggest'], $autocomplete); - } - - // Get spell suggestion if query is complete (user stopped typing) - if ($this->manticore->isSuggestEnabled() && ! empty($query)) { - $suggestions = $this->manticore->suggest($query, $index); - if (! empty($suggestions)) { - usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']); - $response['suggest'] = $suggestions[0]['suggest'] ?? null; + try { + // Get autocomplete suggestions + if ($this->searchEngine->isAutocompleteEnabled() && strlen($query) >= 2) { + $autocomplete = $this->searchEngine->autocomplete($query, $index); + $response['autocomplete'] = array_map(fn ($s) => $s['suggest'], $autocomplete); } + + // Get spell suggestion if query is complete (user stopped typing) + if ($this->searchEngine->isSuggestEnabled() && ! empty($query)) { + $suggestions = $this->searchEngine->suggest($query, $index); + if (! empty($suggestions)) { + usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']); + $response['suggest'] = $suggestions[0]['suggest'] ?? null; + } + } + } catch (\Throwable $e) { + // Return empty results on error } return response()->json($response); diff --git a/config/elasticsearch.php b/config/elasticsearch.php index c9aa263e8..9a736f994 100644 --- a/config/elasticsearch.php +++ b/config/elasticsearch.php @@ -197,4 +197,31 @@ return [ ], + /* + |-------------------------------------------------------------------------- + | Autocomplete Settings + |-------------------------------------------------------------------------- + | + | Configure autocomplete behavior for search suggestions. + | + */ + 'autocomplete' => [ + 'enabled' => env('ELASTICSEARCH_AUTOCOMPLETE_ENABLED', true), + 'min_length' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_MIN_LENGTH', 2), + 'max_results' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_MAX_RESULTS', 10), + 'cache_minutes' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_CACHE', 10), + ], + + /* + |-------------------------------------------------------------------------- + | Suggest/Spell Correction Settings + |-------------------------------------------------------------------------- + | + | Configure "Did you mean?" spell correction. + | + */ + 'suggest' => [ + 'enabled' => env('ELASTICSEARCH_SUGGEST_ENABLED', true), + ], + ];