starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-09-02 19:28:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add autosearch for elastic too

Browse Source
This commit is contained in:
DariusIII
2025-12-17 16:48:49 +01:00
parent 1fb2e41b49
commit 6ef4093a82
4 changed files with 439 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+7
View File
@@ -32,6 +32,13 @@ ELASTICSEARCH_USER=
ELASTICSEARCH_PASS=
ELASTICSEARCH_LOGGING=false
ELASTICSEARCH_ENABLED=false
# Elasticsearch autocomplete settings
ELASTICSEARCH_AUTOCOMPLETE_ENABLED=true
ELASTICSEARCH_AUTOCOMPLETE_MIN_LENGTH=2
ELASTICSEARCH_AUTOCOMPLETE_MAX_RESULTS=10
ELASTICSEARCH_AUTOCOMPLETE_CACHE=10
# Elasticsearch suggest/spell correction settings
ELASTICSEARCH_SUGGEST_ENABLED=true
NNTP_COMPRESSED_HEADERS=false
USE_ALTERNATE_NNTP_SERVER=false
Blacklight/ElasticSearchSiteSearch.php
+352
View File
@@ -38,6 +38,12 @@ class ElasticSearchSiteSearch
private const INDEX_PREDB = 'predb';
private const AUTOCOMPLETE_CACHE_MINUTES = 10;
private const AUTOCOMPLETE_MAX_RESULTS = 10;
private const AUTOCOMPLETE_MIN_LENGTH = 2;
private static ?Client $client = null;
private static ?bool $availabilityCache = null;
@@ -180,6 +186,352 @@ class ElasticSearchSiteSearch
self::$availabilityCacheTime = null;
}
/**
* Check if autocomplete is enabled.
*/
public function isAutocompleteEnabled(): bool
{
return config('elasticsearch.autocomplete.enabled', true) && $this->isElasticsearchAvailable();
}
/**
* Check if suggest is enabled.
*/
public function isSuggestEnabled(): bool
{
return config('elasticsearch.suggest.enabled', true) && $this->isElasticsearchAvailable();
}
/**
* Get the releases index name.
*/
public function getReleasesIndex(): string
{
return self::INDEX_RELEASES;
}
/**
* Get autocomplete suggestions for a search query.
* Searches the releases index and returns matching searchnames.
*
* @param string $query The partial search query
* @param string|null $index Index to search (defaults to releases index)
* @return array<array{suggest: string, distance: int, docs: int}>
*/
public function autocomplete(string $query, ?string $index = null): array
{
if (! $this->isAutocompleteEnabled()) {
return [];
}
$query = trim($query);
$minLength = (int) config('elasticsearch.autocomplete.min_length', self::AUTOCOMPLETE_MIN_LENGTH);
if (strlen($query) < $minLength) {
return [];
}
$index = $index ?? self::INDEX_RELEASES;
$cacheKey = 'es:autocomplete:'.md5($index.$query);
$cached = Cache::get($cacheKey);
if ($cached !== null) {
return $cached;
}
$suggestions = [];
$maxResults = (int) config('elasticsearch.autocomplete.max_results', self::AUTOCOMPLETE_MAX_RESULTS);
try {
$client = $this->getClient();
// Use a prefix/match query on searchname field
$searchParams = [
'index' => $index,
'body' => [
'query' => [
'bool' => [
'should' => [
// Prefix match for autocomplete-like behavior
[
'match_phrase_prefix' => [
'searchname' => [
'query' => $query,
'max_expansions' => 50,
],
],
],
// Also include regular match for better results
[
'match' => [
'searchname' => [
'query' => $query,
'fuzziness' => 'AUTO',
],
],
],
],
'minimum_should_match' => 1,
],
],
'size' => $maxResults * 3,
'_source' => ['searchname'],
'sort' => [
'_score' => ['order' => 'desc'],
],
],
];
$response = $client->search($searchParams);
$seen = [];
if (isset($response['hits']['hits'])) {
foreach ($response['hits']['hits'] as $hit) {
$searchname = $hit['_source']['searchname'] ?? '';
if (empty($searchname)) {
continue;
}
// Create a clean suggestion from the searchname
$suggestion = $this->extractSuggestion($searchname, $query);
if (! empty($suggestion) && ! isset($seen[strtolower($suggestion)])) {
$seen[strtolower($suggestion)] = true;
$suggestions[] = [
'suggest' => $suggestion,
'distance' => 0,
'docs' => 1,
];
}
if (count($suggestions) >= $maxResults) {
break;
}
}
}
} catch (\Throwable $e) {
if (config('app.debug')) {
Log::warning('ElasticSearch autocomplete error: '.$e->getMessage());
}
}
if (! empty($suggestions)) {
$cacheMinutes = (int) config('elasticsearch.autocomplete.cache_minutes', self::AUTOCOMPLETE_CACHE_MINUTES);
Cache::put($cacheKey, $suggestions, now()->addMinutes($cacheMinutes));
}
return $suggestions;
}
/**
* Get spell correction suggestions ("Did you mean?").
*
* @param string $query The search query to check
* @param string|null $index Index to use for suggestions
* @return array<array{suggest: string, distance: int, docs: int}>
*/
public function suggest(string $query, ?string $index = null): array
{
if (! $this->isSuggestEnabled()) {
return [];
}
$query = trim($query);
if (empty($query)) {
return [];
}
$index = $index ?? self::INDEX_RELEASES;
$cacheKey = 'es:suggest:'.md5($index.$query);
$cached = Cache::get($cacheKey);
if ($cached !== null) {
return $cached;
}
$suggestions = [];
try {
$client = $this->getClient();
// Use Elasticsearch suggest API with phrase suggester
$searchParams = [
'index' => $index,
'body' => [
'suggest' => [
'text' => $query,
'searchname_suggest' => [
'phrase' => [
'field' => 'searchname',
'size' => 5,
'gram_size' => 3,
'direct_generator' => [
[
'field' => 'searchname',
'suggest_mode' => 'popular',
],
],
'highlight' => [
'pre_tag' => '',
'post_tag' => '',
],
],
],
],
],
];
$response = $client->search($searchParams);
if (isset($response['suggest']['searchname_suggest'][0]['options'])) {
foreach ($response['suggest']['searchname_suggest'][0]['options'] as $option) {
$suggestedText = $option['text'] ?? '';
if (! empty($suggestedText) && strtolower($suggestedText) !== strtolower($query)) {
$suggestions[] = [
'suggest' => $suggestedText,
'distance' => 1,
'docs' => (int) ($option['freq'] ?? 1),
];
}
}
}
} catch (\Throwable $e) {
if (config('app.debug')) {
Log::debug('ElasticSearch native suggest failed: '.$e->getMessage());
}
}
// Fallback: if native suggest didn't work, use fuzzy search
if (empty($suggestions)) {
$suggestions = $this->suggestFallback($query, $index);
}
if (! empty($suggestions)) {
Cache::put($cacheKey, $suggestions, now()->addMinutes(self::CACHE_TTL_MINUTES));
}
return $suggestions;
}
/**
* Fallback suggest using fuzzy search on searchnames.
*
* @param string $query The search query
* @param string $index Index to search
* @return array<array{suggest: string, distance: int, docs: int}>
*/
private function suggestFallback(string $query, string $index): array
{
try {
$client = $this->getClient();
// Use fuzzy match to find similar terms
$searchParams = [
'index' => $index,
'body' => [
'query' => [
'match' => [
'searchname' => [
'query' => $query,
'fuzziness' => 'AUTO',
],
],
],
'size' => 20,
'_source' => ['searchname'],
],
];
$response = $client->search($searchParams);
// Extract common terms from results that differ from the query
$termCounts = [];
if (isset($response['hits']['hits'])) {
foreach ($response['hits']['hits'] as $hit) {
$searchname = $hit['_source']['searchname'] ?? '';
$words = preg_split('/[\s.\-_]+/', strtolower($searchname));
foreach ($words as $word) {
if (strlen($word) >= 3 && $word !== strtolower($query)) {
$distance = levenshtein(strtolower($query), $word);
if ($distance > 0 && $distance <= 3) {
if (! isset($termCounts[$word])) {
$termCounts[$word] = ['count' => 0, 'distance' => $distance];
}
$termCounts[$word]['count']++;
}
}
}
}
}
// Sort by count
uasort($termCounts, fn ($a, $b) => $b['count'] - $a['count']);
$suggestions = [];
foreach (array_slice($termCounts, 0, 5, true) as $term => $data) {
$suggestions[] = [
'suggest' => $term,
'distance' => $data['distance'],
'docs' => $data['count'],
];
}
return $suggestions;
} catch (\Throwable $e) {
if (config('app.debug')) {
Log::warning('ElasticSearch suggest fallback error: '.$e->getMessage());
}
return [];
}
}
/**
* Extract a clean suggestion from a searchname.
*
* @param string $searchname The full searchname
* @param string $query The user's query
* @return string|null The extracted suggestion
*/
private function extractSuggestion(string $searchname, string $query): ?string
{
// Clean up the searchname - remove file extensions
$clean = preg_replace('/\.(mkv|avi|mp4|wmv|nfo|nzb|par2|rar|zip|r\d+)$/i', '', $searchname);
// Replace dots and underscores with spaces for readability
$clean = str_replace(['.', '_'], ' ', $clean);
// Remove multiple spaces
$clean = preg_replace('/\s+/', ' ', $clean);
$clean = trim($clean);
if (empty($clean)) {
return null;
}
// If the clean name is reasonable length, use it
if (strlen($clean) <= 80) {
return $clean;
}
// For very long names, try to extract the relevant part
$pos = stripos($clean, $query);
if ($pos !== false) {
$start = max(0, $pos - 10);
$extracted = substr($clean, $start, 80);
if ($start > 0) {
$extracted = preg_replace('/^\S*\s/', '', $extracted);
}
$extracted = preg_replace('/\s\S*$/', '', $extracted);
return trim($extracted);
}
return substr($clean, 0, 80);
}
/**
* Search releases index.
*
app/Http/Controllers/SearchSuggestController.php
+53 -37
View File
@@ -2,6 +2,7 @@
namespace App\Http\Controllers;
use Blacklight\ElasticSearchSiteSearch;
use Blacklight\ManticoreSearch;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
@@ -9,11 +10,16 @@ use Illuminate\Support\Facades\RateLimiter;
class SearchSuggestController extends Controller
{
private ManticoreSearch $manticore;
private ManticoreSearch|ElasticSearchSiteSearch $searchEngine;
public function __construct()
{
$this->manticore = new ManticoreSearch;
// Use Elasticsearch if enabled, otherwise fall back to ManticoreSearch
if (config('nntmux.elasticsearch_enabled', false)) {
$this->searchEngine = new ElasticSearchSiteSearch;
} else {
$this->searchEngine = new ManticoreSearch;
}
}
/**
@@ -22,8 +28,8 @@ class SearchSuggestController extends Controller
public function autocomplete(Request $request): JsonResponse
{
$query = $request->input('q', '');
// Use the index from ManticoreSearch config, or allow override via request
$index = $request->input('index') ?? $this->manticore->getReleasesIndex();
// Use the index from search engine config, or allow override via request
$index = $request->input('index') ?? $this->searchEngine->getReleasesIndex();
// Rate limiting: 60 requests per minute per IP
$key = 'autocomplete:'.$request->ip();
@@ -35,7 +41,7 @@ class SearchSuggestController extends Controller
}
RateLimiter::hit($key, 60);
if (! $this->manticore->isAutocompleteEnabled()) {
if (! $this->searchEngine->isAutocompleteEnabled()) {
return response()->json([
'success' => true,
'suggestions' => [],
@@ -50,7 +56,7 @@ class SearchSuggestController extends Controller
}
try {
$suggestions = $this->manticore->autocomplete($query, $index);
$suggestions = $this->searchEngine->autocomplete($query, $index);
return response()->json([
'success' => true,
@@ -72,7 +78,7 @@ class SearchSuggestController extends Controller
public function suggest(Request $request): JsonResponse
{
$query = $request->input('q', '');
$index = $request->input('index', 'releases_rt');
$index = $request->input('index') ?? $this->searchEngine->getReleasesIndex();
// Rate limiting: 30 requests per minute per IP
$key = 'suggest:'.$request->ip();
@@ -84,11 +90,10 @@ class SearchSuggestController extends Controller
}
RateLimiter::hit($key, 60);
if (! $this->manticore->isSuggestEnabled()) {
if (! $this->searchEngine->isSuggestEnabled()) {
return response()->json([
'success' => false,
'success' => true,
'suggestions' => [],
'error' => 'Suggest is disabled',
]);
}
@@ -99,22 +104,29 @@ class SearchSuggestController extends Controller
]);
}
$suggestions = $this->manticore->suggest($query, $index);
try {
$suggestions = $this->searchEngine->suggest($query, $index);
// Get the best suggestion (highest doc count)
$bestSuggestion = null;
if (! empty($suggestions)) {
usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']);
$bestSuggestion = $suggestions[0]['suggest'] ?? null;
// Get the best suggestion (highest doc count)
$bestSuggestion = null;
if (! empty($suggestions)) {
usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']);
$bestSuggestion = $suggestions[0]['suggest'] ?? null;
}
return response()->json([
'success' => true,
'query' => $query,
'suggestions' => array_map(fn ($s) => $s['suggest'], $suggestions),
'best' => $bestSuggestion,
]);
} catch (\Throwable $e) {
return response()->json([
'success' => true,
'query' => $query,
'suggestions' => [],
]);
}
return response()->json([
'success' => true,
'query' => $query,
'suggestions' => array_map(fn ($s) => $s['suggest'], $suggestions),
'best' => $bestSuggestion,
'details' => $suggestions,
]);
}
/**
@@ -123,7 +135,7 @@ class SearchSuggestController extends Controller
public function searchAssist(Request $request): JsonResponse
{
$query = $request->input('q', '');
$index = $request->input('index', 'releases_rt');
$index = $request->input('index') ?? $this->searchEngine->getReleasesIndex();
// Rate limiting
$key = 'searchassist:'.$request->ip();
@@ -142,19 +154,23 @@ class SearchSuggestController extends Controller
'suggest' => null,
];
// Get autocomplete suggestions
if ($this->manticore->isAutocompleteEnabled() && strlen($query) >= 2) {
$autocomplete = $this->manticore->autocomplete($query, $index);
$response['autocomplete'] = array_map(fn ($s) => $s['suggest'], $autocomplete);
}
// Get spell suggestion if query is complete (user stopped typing)
if ($this->manticore->isSuggestEnabled() && ! empty($query)) {
$suggestions = $this->manticore->suggest($query, $index);
if (! empty($suggestions)) {
usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']);
$response['suggest'] = $suggestions[0]['suggest'] ?? null;
try {
// Get autocomplete suggestions
if ($this->searchEngine->isAutocompleteEnabled() && strlen($query) >= 2) {
$autocomplete = $this->searchEngine->autocomplete($query, $index);
$response['autocomplete'] = array_map(fn ($s) => $s['suggest'], $autocomplete);
}
// Get spell suggestion if query is complete (user stopped typing)
if ($this->searchEngine->isSuggestEnabled() && ! empty($query)) {
$suggestions = $this->searchEngine->suggest($query, $index);
if (! empty($suggestions)) {
usort($suggestions, fn ($a, $b) => $b['docs'] - $a['docs']);
$response['suggest'] = $suggestions[0]['suggest'] ?? null;
}
}
} catch (\Throwable $e) {
// Return empty results on error
}
return response()->json($response);
config/elasticsearch.php
+27
View File
@@ -197,4 +197,31 @@ return [
],
/*
|--------------------------------------------------------------------------
| Autocomplete Settings
|--------------------------------------------------------------------------
|
| Configure autocomplete behavior for search suggestions.
|
*/
'autocomplete' => [
'enabled' => env('ELASTICSEARCH_AUTOCOMPLETE_ENABLED', true),
'min_length' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_MIN_LENGTH', 2),
'max_results' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_MAX_RESULTS', 10),
'cache_minutes' => (int) env('ELASTICSEARCH_AUTOCOMPLETE_CACHE', 10),
],
/*
|--------------------------------------------------------------------------
| Suggest/Spell Correction Settings
|--------------------------------------------------------------------------
|
| Configure "Did you mean?" spell correction.
|
*/
'suggest' => [
'enabled' => env('ELASTICSEARCH_SUGGEST_ENABLED', true),
],
];