diff --git a/.env.example b/.env.example index 0739977ca..df41a9e35 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,10 @@ DB_DATABASE=nntmux COMPOSER_AUTH='{"github-oauth": {"github.com": "YOUR GITHUB TOKEN"}}' SEARCH_DRIVER=manticore +# Increment after rebuilding search indexes to invalidate API v2 cursors. +SEARCH_INDEX_GENERATION=1 +# Lifetime of signed API v2 search cursors. +SEARCH_CURSOR_TTL_MINUTES=15 # ManticoreSearch Configuration MANTICORESEARCH_HOST=localhost diff --git a/app/Http/Controllers/Api/ApiV2Controller.php b/app/Http/Controllers/Api/ApiV2Controller.php index 46b840e4a..7d662d90e 100644 --- a/app/Http/Controllers/Api/ApiV2Controller.php +++ b/app/Http/Controllers/Api/ApiV2Controller.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace App\Http\Controllers\Api; use App\Data\Api\ReleaseData; +use App\Facades\Search; use App\Http\Controllers\BasePageController; use App\Http\Controllers\GetNzbController; use App\Models\Category; @@ -18,6 +19,8 @@ use App\Services\Api\ApiUserResolver; use App\Services\Api\V2\ApiV2Presenter; use App\Services\Releases\ReleaseBrowseService; use App\Services\Releases\ReleaseSearchService; +use App\Services\Search\DTO\SearchCursor; +use App\Services\Search\SearchCursorCodec; use Illuminate\Contracts\Foundation\Application; use Illuminate\Contracts\Routing\ResponseFactory; use Illuminate\Http\JsonResponse; @@ -47,6 +50,11 @@ class ApiV2Controller extends BasePageController private ApiCapabilitiesService $capabilitiesService; + private SearchCursorCodec $cursorCodec; + + /** @var array{enabled: bool, offset: int, limit: int, query_hash: string, cursor: SearchCursor|null} */ + private array $paginationContext = ['enabled' => false, 'offset' => 0, 'limit' => 0, 'query_hash' => '', 'cursor' => null]; + /** * @var array */ @@ -61,6 +69,7 @@ class ApiV2Controller extends BasePageController ?ApiUserResolver $userResolver = null, ?ApiV2Presenter $presenter = null, ?ApiCapabilitiesService $capabilitiesService = null, + ?SearchCursorCodec $cursorCodec = null, ) { $this->releaseSearchService = $releaseSearchService; $this->releaseBrowseService = $releaseBrowseService; @@ -70,6 +79,7 @@ class ApiV2Controller extends BasePageController $this->userResolver = $userResolver ?? app(ApiUserResolver::class); $this->presenter = $presenter ?? app(ApiV2Presenter::class); $this->capabilitiesService = $capabilitiesService ?? app(ApiCapabilitiesService::class); + $this->cursorCodec = $cursorCodec ?? app(SearchCursorCodec::class); } /** @@ -157,7 +167,63 @@ class ApiV2Controller extends BasePageController */ private function buildSearchResponse(iterable $rows, User $user): JsonResponse { - return $this->presenter->search($rows, $user, $this->buildUserStatsResponse($user)); + $rows = is_array($rows) ? $rows : iterator_to_array($rows, false); + $pagination = null; + if ($this->paginationContext['enabled']) { + $total = (int) ($rows[0]->_totalrows ?? 0); + $nextOffset = $this->paginationContext['offset'] + count($rows); + $hasMore = (bool) ($rows[0]->_search_has_more ?? (count($rows) === $this->paginationContext['limit'] && $nextOffset < $total)); + $lastSort = $rows[0]->_search_last_sort ?? []; + $cursorPosition = is_array($lastSort) && count($lastSort) === 2 ? array_values($lastSort) : [$nextOffset]; + $nextCursor = $hasMore ? $this->cursorCodec->encode(new SearchCursor( + sortValues: $cursorPosition, + total: $total, + queryHash: $this->paginationContext['query_hash'], + driver: Search::getCurrentDriver(), + indexGeneration: (string) config('search.index_generation', '1'), + expiresAt: time() + ((int) config('search.cursor_ttl_minutes', 15) * 60), + )) : null; + $pagination = ['next_cursor' => $nextCursor, 'has_more' => $hasMore]; + } + + return $this->presenter->search($rows, $user, $this->buildUserStatsResponse($user), $pagination); + } + + private function resolvePaginationOffset(Request $request, int $limit): int|JsonResponse + { + $offset = $this->queryParameters->offset($request); + if (! $request->has('cursor')) { + $this->paginationContext = ['enabled' => false, 'offset' => $offset, 'limit' => $limit, 'query_hash' => '', 'cursor' => null]; + + return $offset; + } + if ($offset > 0) { + return $this->jsonResponse(['error' => 'cursor cannot be combined with a nonzero offset'], 400); + } + + $query = $request->query(); + unset($query['api_token'], $query['cursor'], $query['offset']); + ksort($query); + $queryHash = hash('sha256', json_encode($query, JSON_THROW_ON_ERROR)); + $token = trim((string) $request->input('cursor', '')); + $cursor = null; + if ($token !== '') { + try { + $cursor = $this->cursorCodec->decode($token); + } catch (\InvalidArgumentException $e) { + return $this->jsonResponse(['error' => $e->getMessage()], 400); + } + if ($cursor->queryHash !== $queryHash + || $cursor->driver !== Search::getCurrentDriver() + || $cursor->indexGeneration !== (string) config('search.index_generation', '1')) { + return $this->jsonResponse(['error' => 'Search cursor does not match this query or index generation.'], 400); + } + $offset = count($cursor->sortValues) === 1 ? max(0, (int) $cursor->sortValues[0]) : 0; + } + + $this->paginationContext = ['enabled' => true, 'offset' => $offset, 'limit' => $limit, 'query_hash' => $queryHash, 'cursor' => $cursor]; + + return $offset; } private function parseMaxAge(Request $request): int|JsonResponse @@ -224,8 +290,11 @@ class ApiV2Controller extends BasePageController if ($searchName === '' && ! imdb_id_is_valid($imdbId) && $tmdbId <= 0 && $traktId <= 0) { return $this->jsonResponse(['error' => 'Specify id (query), imdbid, tmdbid, or traktid'], 400); } - $offset = $this->queryParameters->offset($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $categoryID = $this->queryParameters->categories($request); $maxAge = $this->parseMaxAge($request); if (! is_int($maxAge)) { @@ -242,6 +311,7 @@ class ApiV2Controller extends BasePageController 'tmdbid' => $tmdbId, 'traktid' => $traktId, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'id' => $searchName, 'sort' => $sort, @@ -284,8 +354,11 @@ class ApiV2Controller extends BasePageController return $this->jsonResponse(['error' => 'Incorrect parameter (id must not be empty)'], 400); } - $offset = $this->queryParameters->offset($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $categoryID = $this->queryParameters->categories($request); $maxAge = $this->parseMaxAge($request); if (! is_int($maxAge)) { @@ -311,6 +384,7 @@ class ApiV2Controller extends BasePageController 'id' => $searchName, 'group' => $groupName, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'sort' => $sort, 'category' => $categoryID, @@ -361,8 +435,11 @@ class ApiV2Controller extends BasePageController return $this->jsonResponse(['error' => 'Incorrect parameter (id must not be empty)'], 400); } - $offset = $this->queryParameters->offset($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $categoryID = $this->queryParameters->categories($request); $maxAge = $this->parseMaxAge($request); if (! is_int($maxAge)) { @@ -388,6 +465,7 @@ class ApiV2Controller extends BasePageController 'id' => $searchName, 'group' => $groupName, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'sort' => $sort, 'category' => $categoryID, @@ -441,8 +519,11 @@ class ApiV2Controller extends BasePageController return $this->jsonResponse(['error' => 'Specify id (query), anidbid, or anilistid'], 400); } - $offset = $this->queryParameters->offset($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $categoryID = $this->queryParameters->categories($request); $maxAge = $this->parseMaxAge($request); if (! is_int($maxAge)) { @@ -460,6 +541,7 @@ class ApiV2Controller extends BasePageController 'anidbid' => $anidb, 'anilistid' => $anilist, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'sort' => $sort, 'category' => $categoryID, @@ -493,7 +575,6 @@ class ApiV2Controller extends BasePageController $this->recordApiRequest($user, $request); - $offset = $this->queryParameters->offset($request); $catExclusions = User::getCachedCategoryExclusionById($user->id); $minSize = $request->has('minsize') && $request->input('minsize') > 0 ? $request->input('minsize') : 0; $maxAge = $this->parseMaxAge($request); @@ -510,12 +591,17 @@ class ApiV2Controller extends BasePageController } $categoryID = $this->queryParameters->categories($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $searchName = $request->input('id'); $relData = $this->releaseRowCache->remember('v2', 'search', [ 'id' => $searchName, 'group' => $groupName, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'sort' => $sort, 'category' => $categoryID, @@ -533,7 +619,8 @@ class ApiV2Controller extends BasePageController $catExclusions, $categoryID, $minSize, - $sort + $sort, + $this->paginationContext['cursor'], ); } @@ -598,8 +685,11 @@ class ApiV2Controller extends BasePageController $airDate = str_replace('/', '-', $year[0].'-'.$episode); } - $offset = $this->queryParameters->offset($request); $limit = $this->queryParameters->limit($request); + $offset = $this->resolvePaginationOffset($request, $limit); + if (! is_int($offset)) { + return $offset; + } $categoryID = $this->queryParameters->categories($request); $airDate = $airDate ?? ''; $searchName = $request->input('id') ?? ''; @@ -610,6 +700,7 @@ class ApiV2Controller extends BasePageController 'episode' => $episode, 'air_date' => $airDate, 'offset' => $offset, + 'cursor' => $request->input('cursor'), 'limit' => $limit, 'id' => $searchName, 'category' => $categoryID, diff --git a/app/Models/Release.php b/app/Models/Release.php index df46ec33b..61f690326 100644 --- a/app/Models/Release.php +++ b/app/Models/Release.php @@ -49,6 +49,8 @@ use Illuminate\Support\Facades\DB; * @property float|null $diff_percent Computed column (difference percentage) * @property int|null $releases_id From raw query alias * @property int|null $_totalcount From subquery count + * @property list|null $_search_last_sort Transient search cursor tuple + * @property bool|null $_search_has_more Transient search pagination state */ class Release extends Model { diff --git a/app/Services/Api/ApiCapabilitiesService.php b/app/Services/Api/ApiCapabilitiesService.php index a996eee39..8743df7d7 100644 --- a/app/Services/Api/ApiCapabilitiesService.php +++ b/app/Services/Api/ApiCapabilitiesService.php @@ -54,7 +54,7 @@ final readonly class ApiCapabilitiesService /** @return array */ public function v2(): array { - $data = Cache::remember('api_v2_capabilities', 600, function (): array { + $data = Cache::remember('api_v2_capabilities_cursor_v1', 600, function (): array { return [ 'server' => [ 'title' => config('app.name'), @@ -64,12 +64,12 @@ final readonly class ApiCapabilitiesService ], 'limits' => ['max' => 100, 'default' => 100], 'searching' => [ - 'search' => ['available' => 'yes', 'supportedParams' => 'id,group,minsize,maxsize,maxage,cat,limit,offset,sort'], - 'tv-search' => ['available' => 'yes', 'supportedParams' => 'id,vid,tvdbid,traktid,rid,tvmazeid,imdbid,tmdbid,season,ep,cat,minsize,maxsize,maxage,limit,offset,sort'], - 'movie-search' => ['available' => 'yes', 'supportedParams' => 'id,imdbid,tmdbid,traktid,genre,cat,minsize,maxsize,maxage,limit,offset,sort'], - 'audio-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,sort'], - 'book-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,sort'], - 'anime-search' => ['available' => 'yes', 'supportedParams' => 'id,anidbid,anilistid,cat,minsize,maxsize,maxage,limit,offset,sort'], + 'search' => ['available' => 'yes', 'supportedParams' => 'id,group,minsize,maxsize,maxage,cat,limit,offset,cursor,sort'], + 'tv-search' => ['available' => 'yes', 'supportedParams' => 'id,vid,tvdbid,traktid,rid,tvmazeid,imdbid,tmdbid,season,ep,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'], + 'movie-search' => ['available' => 'yes', 'supportedParams' => 'id,imdbid,tmdbid,traktid,genre,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'], + 'audio-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,cursor,sort'], + 'book-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,cursor,sort'], + 'anime-search' => ['available' => 'yes', 'supportedParams' => 'id,anidbid,anilistid,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'], ], 'categories' => Category::getForApi() ->map(static fn (RootCategory $category): array => CategoryData::fromCategory($category)->toArray()) diff --git a/app/Services/Api/V2/ApiV2Presenter.php b/app/Services/Api/V2/ApiV2Presenter.php index 4708458da..2d4bf3a9f 100644 --- a/app/Services/Api/V2/ApiV2Presenter.php +++ b/app/Services/Api/V2/ApiV2Presenter.php @@ -24,8 +24,9 @@ final class ApiV2Presenter /** * @param iterable $rows * @param array $usage + * @param array{next_cursor: string|null, has_more: bool}|null $pagination */ - public function search(iterable $rows, User $user, array $usage): JsonResponse + public function search(iterable $rows, User $user, array $usage, ?array $pagination = null): JsonResponse { $rows = is_array($rows) ? $rows : iterator_to_array($rows, false); $results = []; @@ -33,11 +34,16 @@ final class ApiV2Presenter $results[] = ReleaseData::toArrayFromRelease($row, $user, url('/details').'/', url('/getnzb')); } - return $this->json(array_merge( + $payload = array_merge( ['Total' => (int) ($rows[0]->_totalrows ?? 0)], $usage, ['results' => $results], - )); + ); + if ($pagination !== null) { + $payload['pagination'] = $pagination; + } + + return $this->json($payload); } public function details(Release|\stdClass $release, User $user): JsonResponse diff --git a/app/Services/Releases/ReleaseSearchService.php b/app/Services/Releases/ReleaseSearchService.php index 8efe640b6..8a8280abb 100644 --- a/app/Services/Releases/ReleaseSearchService.php +++ b/app/Services/Releases/ReleaseSearchService.php @@ -11,6 +11,8 @@ use App\Models\Category; use App\Models\Release; use App\Models\Settings; use App\Models\UsenetGroup; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchCursor; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Arr; use Illuminate\Support\Facades\Cache; @@ -116,7 +118,8 @@ class ReleaseSearchService 'try_fuzzy' => true, ]; - $filtered = Search::searchReleasesFiltered($criteria, $limit, $offset); + $searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset)); + $filtered = $searchPage->legacy(); if ($filtered['ids'] !== []) { $ids = array_map(static fn (int|string $id): int => (int) $id, $filtered['ids']); @@ -214,7 +217,7 @@ class ReleaseSearchService * @param array $excludedCats * @return Collection|mixed */ - public function apiSearch(mixed $searchName, mixed $groupName, int $offset = 0, int $limit = 1000, int $maxAge = -1, array $excludedCats = [], array $cat = [-1], int $minSize = 0, string $orderBy = 'posted_desc'): mixed + public function apiSearch(mixed $searchName, mixed $groupName, int $offset = 0, int $limit = 1000, int $maxAge = -1, array $excludedCats = [], array $cat = [-1], int $minSize = 0, string $orderBy = 'posted_desc', ?SearchCursor $cursor = null): mixed { if (config('app.debug')) { Log::debug('ReleaseSearchService::apiSearch called', [ @@ -258,7 +261,12 @@ class ReleaseSearchService 'try_fuzzy' => true, ]; - $filtered = Search::searchReleasesFiltered($criteria, $limit, $offset); + $criteria['track_total'] = $cursor === null; + $searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset, $cursor)); + $filtered = $searchPage->legacy(); + if ($cursor !== null && $filtered['total'] === 0) { + $filtered['total'] = $cursor->total; + } if ($filtered['ids'] === [] && $hasText && config('nntmux.mysql_search_fallback', false) === true) { return $this->apiSearchLegacyMysql($searchName, $groupName, $offset, $limit, $maxAge, $excludedCats, $cat, $minSize, $orderBy); @@ -304,6 +312,8 @@ class ReleaseSearchService if ($releases->isNotEmpty()) { $releases[0]->_totalrows = $filtered['total']; + $releases[0]->_search_last_sort = $searchPage->lastSortValues; + $releases[0]->_search_has_more = $searchPage->hasMore; } $expiresAt = now()->addMinutes(config('nntmux.cache_expiry_medium')); diff --git a/app/Services/Search/Contracts/SearchServiceInterface.php b/app/Services/Search/Contracts/SearchServiceInterface.php index 6b5586a38..cb61fb08d 100644 --- a/app/Services/Search/Contracts/SearchServiceInterface.php +++ b/app/Services/Search/Contracts/SearchServiceInterface.php @@ -5,6 +5,8 @@ declare(strict_types=1); namespace App\Services\Search\Contracts; use App\Enums\SecondarySearchIndex; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchPage; /** * Interface for full-text search services (ManticoreSearch, Elasticsearch). @@ -352,6 +354,8 @@ interface SearchServiceInterface */ public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array; + public function searchReleasePage(ReleaseSearchQuery $query): SearchPage; + /** * Upsert a metadata document (music, books, games, console, steam, anime). * diff --git a/app/Services/Search/DTO/ReleaseSearchQuery.php b/app/Services/Search/DTO/ReleaseSearchQuery.php new file mode 100644 index 000000000..b5de31b63 --- /dev/null +++ b/app/Services/Search/DTO/ReleaseSearchQuery.php @@ -0,0 +1,98 @@ +|string|null $phrases + * @param list|null $categoryIds + * @param list $excludedCategoryIds + * @param list|null $releaseIds + */ + public function __construct( + public array|string|null $phrases = null, + public ?array $categoryIds = null, + public array $excludedCategoryIds = [], + public ?array $releaseIds = null, + public int $minSize = 0, + public int $maxSize = 0, + public int $maxAgeDays = -1, + public int $minDate = 0, + public int $maxDate = 0, + public ?int $groupId = null, + public bool $passwordAllowRar = false, + public ?int $passwordStatusMin = null, + public string $sortField = 'postdate_ts', + public string $sortDirection = 'desc', + public bool $tryFuzzy = true, + public int $limit = 100, + public int $offset = 0, + public ?SearchCursor $cursor = null, + public bool $trackTotal = true, + ) {} + + /** @param array $criteria */ + public static function fromCriteria(array $criteria, int $limit, int $offset = 0, ?SearchCursor $cursor = null): self + { + return new self( + phrases: is_array($criteria['phrases'] ?? null) || is_string($criteria['phrases'] ?? null) ? $criteria['phrases'] : null, + categoryIds: is_array($criteria['category_ids'] ?? null) ? array_map('intval', $criteria['category_ids']) : null, + excludedCategoryIds: is_array($criteria['excluded_category_ids'] ?? null) ? array_map('intval', $criteria['excluded_category_ids']) : [], + releaseIds: is_array($criteria['release_ids'] ?? null) ? array_map('intval', $criteria['release_ids']) : null, + minSize: (int) ($criteria['min_size'] ?? 0), + maxSize: (int) ($criteria['max_size'] ?? 0), + maxAgeDays: (int) ($criteria['max_age_days'] ?? -1), + minDate: (int) ($criteria['min_date'] ?? 0), + maxDate: (int) ($criteria['max_date'] ?? 0), + groupId: isset($criteria['groups_id']) ? (int) $criteria['groups_id'] : null, + passwordAllowRar: (bool) ($criteria['password_allow_rar'] ?? false), + passwordStatusMin: isset($criteria['password_status_min']) ? (int) $criteria['password_status_min'] : null, + sortField: (string) ($criteria['sort_field'] ?? 'postdate_ts'), + sortDirection: (string) ($criteria['sort_dir'] ?? 'desc'), + tryFuzzy: (bool) ($criteria['try_fuzzy'] ?? true), + limit: max(1, $limit), + offset: max(0, $offset), + cursor: $cursor, + trackTotal: (bool) ($criteria['track_total'] ?? true), + ); + } + + /** @return array */ + public function criteria(): array + { + return [ + 'phrases' => $this->phrases, + 'category_ids' => $this->categoryIds, + 'excluded_category_ids' => $this->excludedCategoryIds, + 'release_ids' => $this->releaseIds, + 'min_size' => max(0, $this->minSize), + 'max_size' => max(0, $this->maxSize), + 'max_age_days' => $this->maxAgeDays, + 'min_date' => max(0, $this->minDate), + 'max_date' => max(0, $this->maxDate), + 'groups_id' => $this->groupId, + 'password_allow_rar' => $this->passwordAllowRar, + 'password_status_min' => $this->passwordStatusMin, + 'sort_field' => $this->normalizedSortField(), + 'sort_dir' => $this->normalizedSortDirection(), + 'try_fuzzy' => $this->tryFuzzy, + 'cursor_sort' => $this->cursor?->sortValues, + 'track_total' => $this->trackTotal, + ]; + } + + public function normalizedSortField(): string + { + return in_array($this->sortField, ['postdate_ts', 'adddate_ts', 'size', 'totalpart', 'grabs', 'categories_id', 'id'], true) + ? $this->sortField + : 'postdate_ts'; + } + + public function normalizedSortDirection(): string + { + return strtolower($this->sortDirection) === 'asc' ? 'asc' : 'desc'; + } +} diff --git a/app/Services/Search/DTO/SearchCursor.php b/app/Services/Search/DTO/SearchCursor.php new file mode 100644 index 000000000..a1888be96 --- /dev/null +++ b/app/Services/Search/DTO/SearchCursor.php @@ -0,0 +1,18 @@ + $sortValues */ + public function __construct( + public array $sortValues, + public int $total, + public string $queryHash, + public string $driver, + public string $indexGeneration, + public int $expiresAt, + ) {} +} diff --git a/app/Services/Search/DTO/SearchPage.php b/app/Services/Search/DTO/SearchPage.php new file mode 100644 index 000000000..0e52bf362 --- /dev/null +++ b/app/Services/Search/DTO/SearchPage.php @@ -0,0 +1,29 @@ + $ids + * @param list $lastSortValues + */ + public function __construct( + public array $ids, + public int $total, + public bool $fuzzy, + public string $driver, + public bool $available = true, + public float $durationMs = 0.0, + public array $lastSortValues = [], + public bool $hasMore = false, + ) {} + + /** @return array{ids: list, total: int, fuzzy: bool} */ + public function legacy(): array + { + return ['ids' => $this->ids, 'total' => $this->total, 'fuzzy' => $this->fuzzy]; + } +} diff --git a/app/Services/Search/Drivers/ElasticSearchDriver.php b/app/Services/Search/Drivers/ElasticSearchDriver.php index 69bc0484d..217445860 100644 --- a/app/Services/Search/Drivers/ElasticSearchDriver.php +++ b/app/Services/Search/Drivers/ElasticSearchDriver.php @@ -14,6 +14,8 @@ use App\Models\Release; use App\Models\SteamApp; use App\Models\Video; use App\Services\Search\Contracts\SearchDriverInterface; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchPage; use App\Services\Search\Support\ElasticsearchClientFactory; use App\Services\Search\Support\ElasticsearchResponseHelper; use App\Support\SecondaryIndexDocuments; @@ -93,6 +95,23 @@ class ElasticSearchDriver implements SearchDriverInterface return $this->isElasticsearchAvailable(); } + public function searchReleasePage(ReleaseSearchQuery $query): SearchPage + { + $startedAt = hrtime(true); + $result = $this->searchReleasesFiltered($query->criteria(), $query->limit, $query->offset); + + return new SearchPage( + ids: array_map(static fn (int|string $id): int => (int) $id, $result['ids']), + total: (int) $result['total'], + fuzzy: (bool) $result['fuzzy'], + driver: $this->getDriverName(), + available: (bool) ($result['available'] ?? true), + durationMs: (hrtime(true) - $startedAt) / 1_000_000, + lastSortValues: $result['last_sort'] ?? [], + hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])), + ); + } + /** * Get or create an Elasticsearch client with proper cURL configuration. * @@ -2859,7 +2878,7 @@ class ElasticSearchDriver implements SearchDriverInterface public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array { if (! $this->isElasticsearchAvailable()) { - return ['ids' => [], 'total' => 0, 'fuzzy' => false]; + return ['ids' => [], 'total' => 0, 'fuzzy' => false, 'available' => false]; } $phrases = $criteria['phrases'] ?? null; @@ -2936,38 +2955,54 @@ class ElasticSearchDriver implements SearchDriverInterface try { $client = $this->getClient(); + $body = [ + 'query' => $query, + 'sort' => [ + [$sortField => ['order' => $order]], + ['id' => ['order' => $order]], + ], + 'size' => max(1, min($limit, self::MAX_RESULTS)), + 'track_total_hits' => (bool) ($criteria['track_total'] ?? true), + '_source' => false, + ]; + $cursorSort = $criteria['cursor_sort'] ?? null; + if (is_array($cursorSort) && count($cursorSort) === 2) { + $body['search_after'] = array_values($cursorSort); + } else { + $body['from'] = max(0, $offset); + } + $response = $client->search([ 'index' => $this->getReleasesIndex(), - 'body' => [ - 'query' => $query, - 'sort' => [ - [$sortField => ['order' => $order]], - ['id' => ['order' => $order]], - ], - 'from' => max(0, $offset), - 'size' => max(1, min($limit, self::MAX_RESULTS)), - 'track_total_hits' => true, - '_source' => false, - ], + 'body' => $body, ]); $ids = []; - foreach ($response['hits']['hits'] ?? [] as $hit) { + $hits = $response['hits']['hits'] ?? []; + foreach ($hits as $hit) { $ids[] = (int) ($hit['_id'] ?? 0); } $total = (int) ($response['hits']['total']['value'] ?? $response['hits']['total'] ?? 0); + $lastHit = $hits === [] ? null : $hits[array_key_last($hits)]; - return ['ids' => $ids, 'total' => $total, 'fuzzy' => $useFuzzy]; + return [ + 'ids' => $ids, + 'total' => $total, + 'fuzzy' => $useFuzzy, + 'available' => true, + 'last_sort' => is_array($lastHit) && is_array($lastHit['sort'] ?? null) ? array_values($lastHit['sort']) : [], + 'has_more' => count($ids) === max(1, min($limit, self::MAX_RESULTS)) && ($total === 0 || count($ids) + $offset < $total), + ]; } catch (\Throwable $e) { Log::error('ElasticSearch searchReleasesFiltered error: '.$e->getMessage()); - return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy]; + return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy, 'available' => false]; } }; if ($hasText) { $first = $run(false); - if ($first['ids'] !== [] || ! $tryFuzzy || ! $this->isFuzzyEnabled()) { + if ($first['ids'] !== [] || ($first['available'] ?? true) === false || ! $tryFuzzy || ! $this->isFuzzyEnabled()) { return $first; } diff --git a/app/Services/Search/Drivers/ManticoreSearchDriver.php b/app/Services/Search/Drivers/ManticoreSearchDriver.php index 343614036..85966dee7 100644 --- a/app/Services/Search/Drivers/ManticoreSearchDriver.php +++ b/app/Services/Search/Drivers/ManticoreSearchDriver.php @@ -15,6 +15,8 @@ use App\Models\Release; use App\Models\SteamApp; use App\Models\Video; use App\Services\Search\Contracts\SearchDriverInterface; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchPage; use App\Services\Search\Support\ManticoreClientFactory; use App\Support\ReleaseSearchIndexDocument; use App\Support\SecondaryIndexDocuments; @@ -31,6 +33,12 @@ use Manticoresearch\Search; */ class ManticoreSearchDriver implements SearchDriverInterface { + private const AVAILABILITY_CHECK_CACHE_TTL = 30; + + private static ?bool $availabilityCache = null; + + private static ?int $availabilityCacheTime = null; + /** * @var array */ @@ -74,11 +82,22 @@ class ManticoreSearchDriver implements SearchDriverInterface */ public function isAvailable(): bool { + $now = time(); + if (self::$availabilityCache !== null && self::$availabilityCacheTime !== null + && ($now - self::$availabilityCacheTime) < self::AVAILABILITY_CHECK_CACHE_TTL) { + return self::$availabilityCache; + } + try { $status = $this->manticoreSearch->nodes()->status(); - return ! empty($status); + self::$availabilityCache = ! empty($status); + self::$availabilityCacheTime = $now; + + return self::$availabilityCache; } catch (\Throwable $e) { + self::$availabilityCache = false; + self::$availabilityCacheTime = $now; if (config('app.debug')) { Log::debug('ManticoreSearch not available: '.$e->getMessage()); } @@ -87,6 +106,23 @@ class ManticoreSearchDriver implements SearchDriverInterface } } + public function searchReleasePage(ReleaseSearchQuery $query): SearchPage + { + $startedAt = hrtime(true); + $result = $this->searchReleasesFiltered($query->criteria(), $query->limit, $query->offset); + + return new SearchPage( + ids: array_map(static fn (int|string $id): int => (int) $id, $result['ids']), + total: (int) $result['total'], + fuzzy: (bool) $result['fuzzy'], + driver: $this->getDriverName(), + available: (bool) ($result['available'] ?? true), + durationMs: (hrtime(true) - $startedAt) / 1_000_000, + lastSortValues: $result['last_sort'] ?? [], + hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])), + ); + } + /** * Check if autocomplete is enabled. */ @@ -2467,7 +2503,7 @@ class ManticoreSearchDriver implements SearchDriverInterface public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array { if (! $this->isAvailable()) { - return ['ids' => [], 'total' => 0, 'fuzzy' => false]; + return ['ids' => [], 'total' => 0, 'fuzzy' => false, 'available' => false]; } $phrases = $criteria['phrases'] ?? null; @@ -2539,7 +2575,7 @@ class ManticoreSearchDriver implements SearchDriverInterface 'criteria' => $criteria, ]); - return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy]; + return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy, 'available' => false]; } $ids = []; @@ -2551,12 +2587,17 @@ class ManticoreSearchDriver implements SearchDriverInterface 'ids' => $ids, 'total' => (int) $results->getTotal(), 'fuzzy' => $useFuzzy, + 'available' => true, + // Manticore uses the compatible offset cursor until the PHP client + // supports a compound search-after predicate without raw SQL. + 'last_sort' => [], + 'has_more' => $offset + count($ids) < (int) $results->getTotal(), ]; }; if ($hasText) { $first = $execute(false); - if ($first['ids'] !== [] || ! $tryFuzzy || ! $this->isFuzzyEnabled() || self::queryHasNegation($phrases)) { + if ($first['ids'] !== [] || ($first['available'] ?? true) === false || ! $tryFuzzy || ! $this->isFuzzyEnabled() || self::queryHasNegation($phrases)) { return $first; } diff --git a/app/Services/Search/SearchCursorCodec.php b/app/Services/Search/SearchCursorCodec.php new file mode 100644 index 000000000..215a11ddf --- /dev/null +++ b/app/Services/Search/SearchCursorCodec.php @@ -0,0 +1,49 @@ +encrypter->encryptString(json_encode([ + 'sort' => $cursor->sortValues, + 'total' => $cursor->total, + 'query' => $cursor->queryHash, + 'driver' => $cursor->driver, + 'generation' => $cursor->indexGeneration, + 'expires' => $cursor->expiresAt, + ], JSON_THROW_ON_ERROR)); + } + + public function decode(string $token): SearchCursor + { + try { + $payload = json_decode($this->encrypter->decryptString($token), true, flags: JSON_THROW_ON_ERROR); + } catch (DecryptException|\JsonException $e) { + throw new InvalidArgumentException('Invalid search cursor.', previous: $e); + } + + if (! is_array($payload) || ! is_array($payload['sort'] ?? null) || (int) ($payload['expires'] ?? 0) < time()) { + throw new InvalidArgumentException('Invalid or expired search cursor.'); + } + + return new SearchCursor( + sortValues: array_values($payload['sort']), + total: (int) ($payload['total'] ?? 0), + queryHash: (string) ($payload['query'] ?? ''), + driver: (string) ($payload['driver'] ?? ''), + indexGeneration: (string) ($payload['generation'] ?? ''), + expiresAt: (int) $payload['expires'], + ); + } +} diff --git a/app/Services/Search/SearchService.php b/app/Services/Search/SearchService.php index 10b549604..49b926d4b 100644 --- a/app/Services/Search/SearchService.php +++ b/app/Services/Search/SearchService.php @@ -9,6 +9,9 @@ use App\Services\Search\Contracts\SearchDriverInterface; use App\Services\Search\Contracts\SearchServiceInterface; use App\Services\Search\Drivers\ElasticSearchDriver; use App\Services\Search\Drivers\ManticoreSearchDriver; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchPage; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Manager; /** @@ -510,6 +513,25 @@ class SearchService extends Manager implements SearchServiceInterface return $this->driver()->searchReleasesFiltered($criteria, $limit, $offset); } + public function searchReleasePage(ReleaseSearchQuery $query): SearchPage + { + $page = $this->driver()->searchReleasePage($query); + Log::debug('Release search completed', [ + 'driver' => $page->driver, + 'available' => $page->available, + 'duration_ms' => round($page->durationMs, 2), + 'result_count' => count($page->ids), + 'total' => $page->total, + 'fuzzy' => $page->fuzzy, + 'pagination_mode' => $query->cursor === null ? 'offset' : 'cursor', + 'offset' => $query->offset, + 'limit' => $query->limit, + 'track_total' => $query->trackTotal, + ]); + + return $page; + } + public function insertSecondary(SecondarySearchIndex $index, int $id, array $document): void { $this->driver()->insertSecondary($index, $id, $document); diff --git a/config/search.php b/config/search.php index baa0af103..1b2b9aa74 100644 --- a/config/search.php +++ b/config/search.php @@ -13,6 +13,10 @@ return [ */ 'default' => env('SEARCH_DRIVER', 'manticore'), + 'index_generation' => env('SEARCH_INDEX_GENERATION', '1'), + + 'cursor_ttl_minutes' => (int) env('SEARCH_CURSOR_TTL_MINUTES', 15), + /* |-------------------------------------------------------------------------- | Search Driver Configurations diff --git a/docs/search-performance.md b/docs/search-performance.md new file mode 100644 index 000000000..8a84dc99c --- /dev/null +++ b/docs/search-performance.md @@ -0,0 +1,36 @@ +# Search performance and index maintenance + +The website, Newznab API, and API v2 share the release search contract in +`app/Services/Search/DTO`. Offset responses keep exact totals. API v2 also accepts an +empty `cursor` parameter to start cursor-mode pagination and returns the next opaque, +signed cursor in `pagination.next_cursor`. + +## Baseline and validation + +Capture warm- and cold-cache measurements for both `SEARCH_DRIVER=manticore` and +`SEARCH_DRIVER=elasticsearch`. Record p50/p95 response time, search backend duration, +SQL query count, peak memory, result count, offset, and whether fuzzy fallback ran. +Use the same query/category/sort matrix before and after a change. Include offsets 0, +1,000, and 5,000 plus API v2 cursor traversal. + +Do not set a performance target until the baseline is recorded. A normal successful +exact search should make one search request; fuzzy search may make a second request +only after an exact zero-result response. + +## Maintenance rebuild + +Search schema changes require a maintenance window. Before rebuilding, enable +`nntmux.mysql_search_fallback` if bounded database fallback is desired while indexes +are empty. + +1. Run `php artisan tmux:stop` and pause queue workers that update search indexes. +2. Recreate Manticore indexes with `php artisan manticore:create-indexes --drop`, or + recreate Elasticsearch indexes with the project's Elasticsearch index command. +3. Repopulate enabled indexes with `php artisan nntmux:populate --manticore --all` or + the equivalent Elasticsearch population flags. +4. Compare source and index document counts and run representative filtered searches. +5. Increment `SEARCH_INDEX_GENERATION` to invalidate outstanding API v2 cursors. +6. Restart queue workers and run `php artisan tmux:start`. + +If count or query validation fails, leave processing stopped, restore/rebuild the +previous schema, and keep database fallback enabled until indexes are healthy. diff --git a/tests/Feature/MovieSearchApiTest.php b/tests/Feature/MovieSearchApiTest.php index ebe043ea8..9d7a139c9 100644 --- a/tests/Feature/MovieSearchApiTest.php +++ b/tests/Feature/MovieSearchApiTest.php @@ -6,6 +6,8 @@ namespace Tests\Feature; use App\Models\Release; use App\Services\Releases\ReleaseSearchService; +use App\Services\Search\DTO\ReleaseSearchQuery; +use App\Services\Search\DTO\SearchPage; use App\Services\Search\SearchService; use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Cache; @@ -140,19 +142,17 @@ final class MovieSearchApiTest extends TestCase { $mock = Mockery::mock(SearchService::class, [$this->app]); $mock->shouldReceive('isAvailable')->andReturn(true); - $mock->shouldReceive('searchReleasesFiltered') + $mock->shouldReceive('searchReleasePage') ->once() - ->withArgs(function (array $criteria, int $limit, int $offset): bool { + ->withArgs(function (ReleaseSearchQuery $query): bool { + $criteria = $query->criteria(); + return ($criteria['phrases'] ?? null) === 'Resurrection 2025' && ($criteria['try_fuzzy'] ?? null) === true - && $limit === 100 - && $offset === 0; + && $query->limit === 100 + && $query->offset === 0; }) - ->andReturn([ - 'ids' => [], - 'total' => 0, - 'fuzzy' => false, - ]); + ->andReturn(new SearchPage([], 0, false, 'manticore')); $this->app->instance(SearchService::class, $mock); diff --git a/tests/Unit/ElasticSearchQueryTest.php b/tests/Unit/ElasticSearchQueryTest.php index 6902b0567..03bde64fe 100644 --- a/tests/Unit/ElasticSearchQueryTest.php +++ b/tests/Unit/ElasticSearchQueryTest.php @@ -54,4 +54,17 @@ final class ElasticSearchQueryTest extends TestCase 'Expected indexSearch/indexSearchApi/indexSearchTMA to use RELEASE_TEXT_FIELDS.' ); } + + #[Test] + public function filtered_release_search_supports_search_after_without_loading_source(): void + { + $driverSource = file_get_contents(__DIR__.'/../../app/Services/Search/Drivers/ElasticSearchDriver.php'); + + $this->assertIsString($driverSource); + $methodSource = strstr($driverSource, 'public function searchReleasesFiltered'); + $this->assertIsString($methodSource); + $this->assertStringContainsString("\$body['search_after']", $methodSource); + $this->assertStringContainsString("'_source' => false", $methodSource); + $this->assertStringContainsString("'track_total_hits' => (bool)", $methodSource); + } } diff --git a/tests/Unit/SearchCursorCodecTest.php b/tests/Unit/SearchCursorCodecTest.php new file mode 100644 index 000000000..7811b8952 --- /dev/null +++ b/tests/Unit/SearchCursorCodecTest.php @@ -0,0 +1,49 @@ +decode($codec->encode($cursor)); + + $this->assertSame($cursor->sortValues, $decoded->sortValues); + $this->assertSame(250, $decoded->total); + $this->assertSame('query-hash', $decoded->queryHash); + $this->assertSame('elasticsearch', $decoded->driver); + $this->assertSame('7', $decoded->indexGeneration); + } + + #[Test] + public function it_rejects_tampered_and_expired_cursors(): void + { + $codec = new SearchCursorCodec(new Encrypter(random_bytes(32), 'AES-256-CBC')); + + $this->expectException(InvalidArgumentException::class); + $codec->decode('tampered'); + } + + #[Test] + public function it_rejects_an_expired_cursor(): void + { + $codec = new SearchCursorCodec(new Encrypter(random_bytes(32), 'AES-256-CBC')); + $token = $codec->encode(new SearchCursor([100], 10, 'hash', 'manticore', '1', time() - 1)); + + $this->expectException(InvalidArgumentException::class); + $codec->decode($token); + } +}