Optimize searches to prevent slow DB queries

app/Services/Releases/ReleaseSearchService.php

@@ -22,6 +22,12 @@ class ReleaseSearchService
 {
     private const CACHE_VERSION_KEY = 'releases:cache_version';
 
+    private const SEARCH_INDEX_BUFFER_PAGES = 10;
+
+    private const SEARCH_INDEX_MIN_CANDIDATES = 250;
+
+    private const SEARCH_INDEX_MAX_CANDIDATES = 2000;
+
     // RAR/ZIP Password indicator.
     public const PASSWD_NONE = 0;
 
@@ -61,8 +67,10 @@ class ReleaseSearchService
             ]);
         }
 
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
+
         // Get search results from index
-        $searchResult = $this->performIndexSearch($searchArr, $limit);
+        $searchResult = $this->performIndexSearch($searchArr, $searchLimit);
 
         if (config('app.debug')) {
             Log::debug('ReleaseSearchService::search after performIndexSearch', [
@@ -145,11 +153,13 @@ class ReleaseSearchService
             ]);
         }
 
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
+
         // Early return if searching with no results
         $searchResult = [];
         if ($searchName !== -1 && $searchName !== '' && $searchName !== null) {
             // Use the unified Search facade with fuzzy fallback
-            $fuzzyResult = Search::searchReleasesWithFuzzy($searchName, $limit);
+            $fuzzyResult = Search::searchReleasesWithFuzzy($searchName, $searchLimit);
             $searchResult = $fuzzyResult['ids'] ?? [];
 
             if (config('app.debug') && ($fuzzyResult['fuzzy'] ?? false)) {
@@ -161,7 +171,7 @@ class ReleaseSearchService
                 if (config('app.debug')) {
                     Log::debug('apiSearch: Falling back to MySQL search');
                 }
-                $searchResult = $this->performMySQLSearch(['searchname' => $searchName], $limit);
+                $searchResult = $this->performMySQLSearch(['searchname' => $searchName], $searchLimit);
             }
 
             if (empty($searchResult)) {
@@ -265,6 +275,7 @@ class ReleaseSearchService
         $shouldCache = ! (isset($siteIdArr['id']) && (int) $siteIdArr['id'] > 0);
         $rawCacheKey = md5(serialize(func_get_args()).'tvSearch');
         $cacheKey = null;
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
         if ($shouldCache) {
             $cacheKey = md5($this->getCacheVersion().$rawCacheKey);
             $cached = Cache::get($cacheKey);
@@ -307,7 +318,7 @@ class ReleaseSearchService
         // Try to get releases directly from search index using external IDs
         $searchResult = [];
         if (! empty($externalIds)) {
-            $searchResult = Search::searchReleasesByExternalId($externalIds, $limit * 2);
+            $searchResult = Search::searchReleasesByExternalId($externalIds, $searchLimit);
 
             if (config('app.debug') && ! empty($searchResult)) {
                 Log::debug('tvSearch: Found releases via search index by external IDs', [
@@ -400,12 +411,12 @@ class ReleaseSearchService
             }
 
             // Use the unified Search facade with fuzzy fallback
-            $fuzzyResult = Search::searchReleasesWithFuzzy(['searchname' => $searchName], $limit);
+            $fuzzyResult = Search::searchReleasesWithFuzzy(['searchname' => $searchName], $searchLimit);
             $searchResult = $fuzzyResult['ids'] ?? [];
 
             // Fall back to MySQL if search engine failed (only if enabled)
             if (empty($searchResult) && config('nntmux.mysql_search_fallback', false) === true) {
-                $searchResult = $this->performMySQLSearch(['searchname' => $searchName], $limit);
+                $searchResult = $this->performMySQLSearch(['searchname' => $searchName], $searchLimit);
             }
 
             if (empty($searchResult)) {
@@ -532,6 +543,8 @@ class ReleaseSearchService
      */
     public function apiTvSearch(array $siteIdArr = [], string $series = '', string $episode = '', string $airDate = '', int $offset = 0, int $limit = 100, string $name = '', array $cat = [-1], int $maxAge = -1, int $minSize = 0, array $excludedCategories = []): mixed
     {
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
+
         // OPTIMIZATION: Try to find releases using search index external IDs first
         $externalIds = [];
         foreach ($siteIdArr as $column => $Id) {
@@ -554,7 +567,7 @@ class ReleaseSearchService
         // Try to get releases directly from search index using external IDs
         $indexSearchResult = [];
         if (! empty($externalIds)) {
-            $indexSearchResult = Search::searchReleasesByExternalId($externalIds, $limit * 2);
+            $indexSearchResult = Search::searchReleasesByExternalId($externalIds, $searchLimit);
 
             if (config('app.debug') && ! empty($indexSearchResult)) {
                 Log::debug('apiTvSearch: Found releases via search index by external IDs', [
@@ -615,12 +628,12 @@ class ReleaseSearchService
         $searchResult = $indexSearchResult; // Use index search result if we have it
         if (empty($searchResult) && ! empty($name)) {
             // Use the unified Search facade with fuzzy fallback
-            $fuzzyResult = Search::searchReleasesWithFuzzy(['searchname' => $name], $limit);
+            $fuzzyResult = Search::searchReleasesWithFuzzy(['searchname' => $name], $searchLimit);
             $searchResult = $fuzzyResult['ids'] ?? [];
 
             // Fall back to MySQL if search engine failed (only if enabled)
             if (empty($searchResult) && config('nntmux.mysql_search_fallback', false) === true) {
-                $searchResult = $this->performMySQLSearch(['searchname' => $name], $limit);
+                $searchResult = $this->performMySQLSearch(['searchname' => $name], $searchLimit);
             }
 
             if (count($searchResult) === 0) {
@@ -677,15 +690,16 @@ class ReleaseSearchService
      */
     public function animeSearch(mixed $aniDbID, int $offset = 0, int $limit = 100, string $name = '', array $cat = [-1], int $maxAge = -1, array $excludedCategories = []): mixed
     {
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
         $searchResult = [];
         if (! empty($name)) {
             // Use the unified Search facade with fuzzy fallback
-            $fuzzyResult = Search::searchReleasesWithFuzzy($name, $limit);
+            $fuzzyResult = Search::searchReleasesWithFuzzy($name, $searchLimit);
             $searchResult = $fuzzyResult['ids'] ?? [];
 
             // Fall back to MySQL if search engine returned no results (only if enabled)
             if (empty($searchResult) && config('nntmux.mysql_search_fallback', false) === true) {
-                $searchResult = $this->performMySQLSearch(['searchname' => $name], $limit);
+                $searchResult = $this->performMySQLSearch(['searchname' => $name], $searchLimit);
             }
 
             if (count($searchResult) === 0) {
@@ -747,6 +761,7 @@ class ReleaseSearchService
      */
     public function moviesSearch(int $imDbId = -1, int $tmDbId = -1, int $traktId = -1, int $offset = 0, int $limit = 100, string $name = '', array $cat = [-1], int $maxAge = -1, int $minSize = 0, array $excludedCategories = []): mixed
     {
+        $searchLimit = $this->determineSearchCandidateLimit($offset, $limit);
         $searchResult = [];
 
         // OPTIMIZATION: If we have external IDs, use the search index to find releases directly
@@ -764,7 +779,7 @@ class ReleaseSearchService
 
         // Use search index for external ID lookups (much faster than database JOINs)
         if (! empty($externalIds)) {
-            $searchResult = Search::searchReleasesByExternalId($externalIds, $limit * 2);
+            $searchResult = Search::searchReleasesByExternalId($externalIds, $searchLimit);
 
             if (config('app.debug') && ! empty($searchResult)) {
                 Log::debug('moviesSearch: Found releases via search index by external IDs', [
@@ -777,12 +792,12 @@ class ReleaseSearchService
         // If no external IDs provided or index search failed, search by name
         if (empty($searchResult) && ! empty($name)) {
             // Use the unified Search facade with fuzzy fallback
-            $fuzzyResult = Search::searchReleasesWithFuzzy($name, $limit);
+            $fuzzyResult = Search::searchReleasesWithFuzzy($name, $searchLimit);
             $searchResult = $fuzzyResult['ids'] ?? [];
 
             // Fall back to MySQL if search engine returned no results (only if enabled)
             if (empty($searchResult) && config('nntmux.mysql_search_fallback', false) === true) {
-                $searchResult = $this->performMySQLSearch(['searchname' => $name], $limit);
+                $searchResult = $this->performMySQLSearch(['searchname' => $name], $searchLimit);
             }
 
             // Only return empty if we were specifically searching by name but found nothing
@@ -977,6 +992,23 @@ class ReleaseSearchService
         return [];
     }
 
+    /**
+     * Search-backed pages still apply SQL filters and ordering after the index lookup,
+     * so fetch a buffered candidate set without handing MySQL thousands of IDs.
+     */
+    private function determineSearchCandidateLimit(int $offset, int $limit): int
+    {
+        $pageSize = max(1, $limit);
+        $requestedRows = max($pageSize, $offset + $pageSize);
+        $bufferedRows = max(
+            $requestedRows,
+            $offset + ($pageSize * self::SEARCH_INDEX_BUFFER_PAGES),
+            self::SEARCH_INDEX_MIN_CANDIDATES
+        );
+
+        return min($bufferedRows, self::SEARCH_INDEX_MAX_CANDIDATES);
+    }
+
     /**
      * Fallback MySQL search when full-text search engines are unavailable
      *

app/Services/Search/Drivers/ManticoreSearchDriver.php

@@ -851,7 +851,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
             }
         }
 
-        $result = $this->searchIndexes($this->getReleasesIndex(), '', [], $searchArray);
+        $result = $this->searchIndexes($this->getReleasesIndex(), '', [], $searchArray, $limit);
 
         return ! empty($result) ? ($result['id'] ?? []) : [];
     }
@@ -965,6 +965,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
             return [];
         }
 
+        $normalizedLimit = $this->normalizeSearchLimit($limit);
         $fuzzyConfig = $this->getFuzzyConfig();
         $distance = $fuzzyConfig['max_distance'] ?? 2;
 
@@ -972,7 +973,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
         $cacheKey = 'manticore:fuzzy:'.md5(serialize([
             'index' => $index,
             'array' => $searchArray,
-            'limit' => $limit,
+            'limit' => $normalizedLimit,
             'distance' => $distance,
         ]));
 
@@ -1025,7 +1026,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
                 ->search($searchExpr)
                 ->option('fuzzy', true)
                 ->option('distance', $distance)
-                ->limit(min($limit, 10000));
+                ->limit($normalizedLimit);
 
             $results = $query->get();
         } catch (ResponseException $e) {
@@ -1039,7 +1040,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
                 ]);
 
                 // Fall back to regular search without fuzzy
-                return $this->searchIndexes($index, '', [], $searchArray);
+                return $this->searchIndexes($index, '', [], $searchArray, $normalizedLimit);
             }
 
             Log::error('ManticoreSearch fuzzySearchIndexes ResponseException: '.$message, [
@@ -1109,7 +1110,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
      * @param  array<string, mixed>  $searchArray
      * @return array<string, mixed>
      */
-    public function searchIndexes(string $rt_index, ?string $searchString, array $column = [], array $searchArray = []): array
+    public function searchIndexes(string $rt_index, ?string $searchString, array $column = [], array $searchArray = [], int $limit = 1000): array
     {
         if (empty($rt_index)) {
             Log::warning('ManticoreSearch: Index name is required for search');
@@ -1117,12 +1118,15 @@ class ManticoreSearchDriver implements SearchDriverInterface
             return [];
         }
 
+        $normalizedLimit = $this->normalizeSearchLimit($limit);
+
         if (config('app.debug')) {
             Log::debug('ManticoreSearch::searchIndexes called', [
                 'rt_index' => $rt_index,
                 'searchString' => $searchString,
                 'column' => $column,
                 'searchArray' => $searchArray,
+                'limit' => $normalizedLimit,
             ]);
         }
 
@@ -1132,6 +1136,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
             'search' => $searchString,
             'columns' => $column,
             'array' => $searchArray,
+            'limit' => $normalizedLimit,
         ]));
 
         $cached = Cache::get($cacheKey);
@@ -1207,8 +1212,8 @@ class ManticoreSearchDriver implements SearchDriverInterface
             $query = (new Search($this->manticoreSearch))
                 ->setTable($rt_index)
                 ->option('ranker', 'sph04')
-                ->maxMatches(10000)
-                ->limit(10000)
+                ->maxMatches($normalizedLimit)
+                ->limit($normalizedLimit)
                 ->stripBadUtf8(true)
                 ->search($searchExpr);
 
@@ -1224,8 +1229,8 @@ class ManticoreSearchDriver implements SearchDriverInterface
                     $query = (new Search($this->manticoreSearch))
                         ->setTable($rt_index)
                         ->option('ranker', 'sph04')
-                        ->maxMatches(10000)
-                        ->limit(10000)
+                        ->maxMatches($normalizedLimit)
+                        ->limit($normalizedLimit)
                         ->stripBadUtf8(true)
                         ->search($searchExpr);
 
@@ -1287,6 +1292,13 @@ class ManticoreSearchDriver implements SearchDriverInterface
         return $result;
     }
 
+    private function normalizeSearchLimit(int $limit): int
+    {
+        $maxMatches = max(1, (int) ($this->config['max_matches'] ?? 10000));
+
+        return max(1, min($limit, $maxMatches));
+    }
+
     /**
      * Get autocomplete suggestions for a search query.
      * Searches the releases index and returns matching searchnames.

tests/Unit/ManticoreSearchQueryTest.php

@@ -6,6 +6,7 @@ use App\Services\Search\Drivers\ManticoreSearchDriver;
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Test;
 use PHPUnit\Framework\TestCase;
+use ReflectionClass;
 
 class ManticoreSearchQueryTest extends TestCase
 {
@@ -168,4 +169,42 @@ class ManticoreSearchQueryTest extends TestCase
             'array with -1 values' => [['searchname' => '-1'], false],
         ];
     }
+
+    #[Test]
+    public function it_passes_release_search_limit_to_search_indexes(): void
+    {
+        $driver = new class extends ManticoreSearchDriver
+        {
+            public int $capturedLimit = 0;
+
+            public function __construct() {}
+
+            public function searchIndexes(string $rt_index, ?string $searchString, array $column = [], array $searchArray = [], int $limit = 1000): array
+            {
+                $this->capturedLimit = $limit;
+
+                return ['id' => [101, 202]];
+            }
+        };
+
+        $result = $driver->searchReleases(['searchname' => 'harry potter'], 75);
+
+        $this->assertSame(75, $driver->capturedLimit);
+        $this->assertSame([101, 202], $result);
+    }
+
+    #[Test]
+    public function it_clamps_search_limit_to_configured_max_matches(): void
+    {
+        $reflection = new ReflectionClass(ManticoreSearchDriver::class);
+        $driver = $reflection->newInstanceWithoutConstructor();
+
+        $configProperty = $reflection->getProperty('config');
+        $configProperty->setValue($driver, ['max_matches' => 500]);
+
+        $method = $reflection->getMethod('normalizeSearchLimit');
+
+        $this->assertSame(500, $method->invoke($driver, 750));
+        $this->assertSame(1, $method->invoke($driver, 0));
+    }
 }

tests/Unit/ReleaseSearchServiceOrderingTest.php

@@ -107,4 +107,27 @@ class ReleaseSearchServiceOrderingTest extends TestCase
 
         $this->assertEquals(['categories_id', 'asc'], $result);
     }
+
+    /**
+     * Test that the search candidate buffer fetches enough rows for page one without exploding.
+     */
+    public function test_determine_search_candidate_limit_buffers_first_page(): void
+    {
+        $reflection = new ReflectionClass(ReleaseSearchService::class);
+        $method = $reflection->getMethod('determineSearchCandidateLimit');
+
+        $this->assertSame(500, $method->invoke($this->service, 0, 50));
+    }
+
+    /**
+     * Test that the search candidate buffer grows with the offset and caps deep pages.
+     */
+    public function test_determine_search_candidate_limit_caps_large_offsets(): void
+    {
+        $reflection = new ReflectionClass(ReleaseSearchService::class);
+        $method = $reflection->getMethod('determineSearchCandidateLimit');
+
+        $this->assertSame(550, $method->invoke($this->service, 50, 50));
+        $this->assertSame(2000, $method->invoke($this->service, 2500, 50));
+    }
 }