starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve search handling in manticore and elasticsearch

Browse Source
This commit is contained in:
DariusIII
2026-07-14 16:10:02 +02:00
parent 064154ad12
commit efd52fb1d8
19 changed files with 561 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+4
View File
@@ -10,6 +10,10 @@ DB_DATABASE=nntmux
COMPOSER_AUTH='{"github-oauth": {"github.com": "YOUR GITHUB TOKEN"}}'
SEARCH_DRIVER=manticore
# Increment after rebuilding search indexes to invalidate API v2 cursors.
SEARCH_INDEX_GENERATION=1
# Lifetime of signed API v2 search cursors.
SEARCH_CURSOR_TTL_MINUTES=15
# ManticoreSearch Configuration
MANTICORESEARCH_HOST=localhost
app/Http/Controllers/Api/ApiV2Controller.php
+99 -8
View File
@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace App\Http\Controllers\Api;
use App\Data\Api\ReleaseData;
use App\Facades\Search;
use App\Http\Controllers\BasePageController;
use App\Http\Controllers\GetNzbController;
use App\Models\Category;
@@ -18,6 +19,8 @@ use App\Services\Api\ApiUserResolver;
use App\Services\Api\V2\ApiV2Presenter;
use App\Services\Releases\ReleaseBrowseService;
use App\Services\Releases\ReleaseSearchService;
use App\Services\Search\DTO\SearchCursor;
use App\Services\Search\SearchCursorCodec;
use Illuminate\Contracts\Foundation\Application;
use Illuminate\Contracts\Routing\ResponseFactory;
use Illuminate\Http\JsonResponse;
@@ -47,6 +50,11 @@ class ApiV2Controller extends BasePageController
private ApiCapabilitiesService $capabilitiesService;
private SearchCursorCodec $cursorCodec;
/** @var array{enabled: bool, offset: int, limit: int, query_hash: string, cursor: SearchCursor|null} */
private array $paginationContext = ['enabled' => false, 'offset' => 0, 'limit' => 0, 'query_hash' => '', 'cursor' => null];
/**
* @var array<int, object>
*/
@@ -61,6 +69,7 @@ class ApiV2Controller extends BasePageController
?ApiUserResolver $userResolver = null,
?ApiV2Presenter $presenter = null,
?ApiCapabilitiesService $capabilitiesService = null,
?SearchCursorCodec $cursorCodec = null,
) {
$this->releaseSearchService = $releaseSearchService;
$this->releaseBrowseService = $releaseBrowseService;
@@ -70,6 +79,7 @@ class ApiV2Controller extends BasePageController
$this->userResolver = $userResolver ?? app(ApiUserResolver::class);
$this->presenter = $presenter ?? app(ApiV2Presenter::class);
$this->capabilitiesService = $capabilitiesService ?? app(ApiCapabilitiesService::class);
$this->cursorCodec = $cursorCodec ?? app(SearchCursorCodec::class);
}
/**
@@ -157,7 +167,63 @@ class ApiV2Controller extends BasePageController
*/
private function buildSearchResponse(iterable $rows, User $user): JsonResponse
{
return $this->presenter->search($rows, $user, $this->buildUserStatsResponse($user));
$rows = is_array($rows) ? $rows : iterator_to_array($rows, false);
$pagination = null;
if ($this->paginationContext['enabled']) {
$total = (int) ($rows[0]->_totalrows ?? 0);
$nextOffset = $this->paginationContext['offset'] + count($rows);
$hasMore = (bool) ($rows[0]->_search_has_more ?? (count($rows) === $this->paginationContext['limit'] && $nextOffset < $total));
$lastSort = $rows[0]->_search_last_sort ?? [];
$cursorPosition = is_array($lastSort) && count($lastSort) === 2 ? array_values($lastSort) : [$nextOffset];
$nextCursor = $hasMore ? $this->cursorCodec->encode(new SearchCursor(
sortValues: $cursorPosition,
total: $total,
queryHash: $this->paginationContext['query_hash'],
driver: Search::getCurrentDriver(),
indexGeneration: (string) config('search.index_generation', '1'),
expiresAt: time() + ((int) config('search.cursor_ttl_minutes', 15) * 60),
)) : null;
$pagination = ['next_cursor' => $nextCursor, 'has_more' => $hasMore];
}
return $this->presenter->search($rows, $user, $this->buildUserStatsResponse($user), $pagination);
}
private function resolvePaginationOffset(Request $request, int $limit): int|JsonResponse
{
$offset = $this->queryParameters->offset($request);
if (! $request->has('cursor')) {
$this->paginationContext = ['enabled' => false, 'offset' => $offset, 'limit' => $limit, 'query_hash' => '', 'cursor' => null];
return $offset;
}
if ($offset > 0) {
return $this->jsonResponse(['error' => 'cursor cannot be combined with a nonzero offset'], 400);
}
$query = $request->query();
unset($query['api_token'], $query['cursor'], $query['offset']);
ksort($query);
$queryHash = hash('sha256', json_encode($query, JSON_THROW_ON_ERROR));
$token = trim((string) $request->input('cursor', ''));
$cursor = null;
if ($token !== '') {
try {
$cursor = $this->cursorCodec->decode($token);
} catch (\InvalidArgumentException $e) {
return $this->jsonResponse(['error' => $e->getMessage()], 400);
}
if ($cursor->queryHash !== $queryHash
|| $cursor->driver !== Search::getCurrentDriver()
|| $cursor->indexGeneration !== (string) config('search.index_generation', '1')) {
return $this->jsonResponse(['error' => 'Search cursor does not match this query or index generation.'], 400);
}
$offset = count($cursor->sortValues) === 1 ? max(0, (int) $cursor->sortValues[0]) : 0;
}
$this->paginationContext = ['enabled' => true, 'offset' => $offset, 'limit' => $limit, 'query_hash' => $queryHash, 'cursor' => $cursor];
return $offset;
}
private function parseMaxAge(Request $request): int|JsonResponse
@@ -224,8 +290,11 @@ class ApiV2Controller extends BasePageController
if ($searchName === '' && ! imdb_id_is_valid($imdbId) && $tmdbId <= 0 && $traktId <= 0) {
return $this->jsonResponse(['error' => 'Specify id (query), imdbid, tmdbid, or traktid'], 400);
}
$offset = $this->queryParameters->offset($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$categoryID = $this->queryParameters->categories($request);
$maxAge = $this->parseMaxAge($request);
if (! is_int($maxAge)) {
@@ -242,6 +311,7 @@ class ApiV2Controller extends BasePageController
'tmdbid' => $tmdbId,
'traktid' => $traktId,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'id' => $searchName,
'sort' => $sort,
@@ -284,8 +354,11 @@ class ApiV2Controller extends BasePageController
return $this->jsonResponse(['error' => 'Incorrect parameter (id must not be empty)'], 400);
}
$offset = $this->queryParameters->offset($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$categoryID = $this->queryParameters->categories($request);
$maxAge = $this->parseMaxAge($request);
if (! is_int($maxAge)) {
@@ -311,6 +384,7 @@ class ApiV2Controller extends BasePageController
'id' => $searchName,
'group' => $groupName,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'sort' => $sort,
'category' => $categoryID,
@@ -361,8 +435,11 @@ class ApiV2Controller extends BasePageController
return $this->jsonResponse(['error' => 'Incorrect parameter (id must not be empty)'], 400);
}
$offset = $this->queryParameters->offset($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$categoryID = $this->queryParameters->categories($request);
$maxAge = $this->parseMaxAge($request);
if (! is_int($maxAge)) {
@@ -388,6 +465,7 @@ class ApiV2Controller extends BasePageController
'id' => $searchName,
'group' => $groupName,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'sort' => $sort,
'category' => $categoryID,
@@ -441,8 +519,11 @@ class ApiV2Controller extends BasePageController
return $this->jsonResponse(['error' => 'Specify id (query), anidbid, or anilistid'], 400);
}
$offset = $this->queryParameters->offset($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$categoryID = $this->queryParameters->categories($request);
$maxAge = $this->parseMaxAge($request);
if (! is_int($maxAge)) {
@@ -460,6 +541,7 @@ class ApiV2Controller extends BasePageController
'anidbid' => $anidb,
'anilistid' => $anilist,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'sort' => $sort,
'category' => $categoryID,
@@ -493,7 +575,6 @@ class ApiV2Controller extends BasePageController
$this->recordApiRequest($user, $request);
$offset = $this->queryParameters->offset($request);
$catExclusions = User::getCachedCategoryExclusionById($user->id);
$minSize = $request->has('minsize') && $request->input('minsize') > 0 ? $request->input('minsize') : 0;
$maxAge = $this->parseMaxAge($request);
@@ -510,12 +591,17 @@ class ApiV2Controller extends BasePageController
}
$categoryID = $this->queryParameters->categories($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$searchName = $request->input('id');
$relData = $this->releaseRowCache->remember('v2', 'search', [
'id' => $searchName,
'group' => $groupName,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'sort' => $sort,
'category' => $categoryID,
@@ -533,7 +619,8 @@ class ApiV2Controller extends BasePageController
$catExclusions,
$categoryID,
$minSize,
$sort
$sort,
$this->paginationContext['cursor'],
);
}
@@ -598,8 +685,11 @@ class ApiV2Controller extends BasePageController
$airDate = str_replace('/', '-', $year[0].'-'.$episode);
}
$offset = $this->queryParameters->offset($request);
$limit = $this->queryParameters->limit($request);
$offset = $this->resolvePaginationOffset($request, $limit);
if (! is_int($offset)) {
return $offset;
}
$categoryID = $this->queryParameters->categories($request);
$airDate = $airDate ?? '';
$searchName = $request->input('id') ?? '';
@@ -610,6 +700,7 @@ class ApiV2Controller extends BasePageController
'episode' => $episode,
'air_date' => $airDate,
'offset' => $offset,
'cursor' => $request->input('cursor'),
'limit' => $limit,
'id' => $searchName,
'category' => $categoryID,
app/Models/Release.php
+2
View File
@@ -49,6 +49,8 @@ use Illuminate\Support\Facades\DB;
* @property float|null $diff_percent Computed column (difference percentage)
* @property int|null $releases_id From raw query alias
* @property int|null $_totalcount From subquery count
* @property list<int|float|string>|null $_search_last_sort Transient search cursor tuple
* @property bool|null $_search_has_more Transient search pagination state
*/
class Release extends Model
{
app/Services/Api/ApiCapabilitiesService.php
+7 -7
View File
@@ -54,7 +54,7 @@ final readonly class ApiCapabilitiesService
/** @return array<string, mixed> */
public function v2(): array
{
$data = Cache::remember('api_v2_capabilities', 600, function (): array {
$data = Cache::remember('api_v2_capabilities_cursor_v1', 600, function (): array {
return [
'server' => [
'title' => config('app.name'),
@@ -64,12 +64,12 @@ final readonly class ApiCapabilitiesService
],
'limits' => ['max' => 100, 'default' => 100],
'searching' => [
'search' => ['available' => 'yes', 'supportedParams' => 'id,group,minsize,maxsize,maxage,cat,limit,offset,sort'],
'tv-search' => ['available' => 'yes', 'supportedParams' => 'id,vid,tvdbid,traktid,rid,tvmazeid,imdbid,tmdbid,season,ep,cat,minsize,maxsize,maxage,limit,offset,sort'],
'movie-search' => ['available' => 'yes', 'supportedParams' => 'id,imdbid,tmdbid,traktid,genre,cat,minsize,maxsize,maxage,limit,offset,sort'],
'audio-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,sort'],
'book-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,sort'],
'anime-search' => ['available' => 'yes', 'supportedParams' => 'id,anidbid,anilistid,cat,minsize,maxsize,maxage,limit,offset,sort'],
'search' => ['available' => 'yes', 'supportedParams' => 'id,group,minsize,maxsize,maxage,cat,limit,offset,cursor,sort'],
'tv-search' => ['available' => 'yes', 'supportedParams' => 'id,vid,tvdbid,traktid,rid,tvmazeid,imdbid,tmdbid,season,ep,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'],
'movie-search' => ['available' => 'yes', 'supportedParams' => 'id,imdbid,tmdbid,traktid,genre,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'],
'audio-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,cursor,sort'],
'book-search' => ['available' => 'yes', 'supportedParams' => 'id,cat,minsize,maxsize,maxage,group,limit,offset,cursor,sort'],
'anime-search' => ['available' => 'yes', 'supportedParams' => 'id,anidbid,anilistid,cat,minsize,maxsize,maxage,limit,offset,cursor,sort'],
],
'categories' => Category::getForApi()
->map(static fn (RootCategory $category): array => CategoryData::fromCategory($category)->toArray())
app/Services/Api/V2/ApiV2Presenter.php
+9 -3
View File
@@ -24,8 +24,9 @@ final class ApiV2Presenter
/**
* @param iterable<int, Release|\stdClass> $rows
* @param array<string, mixed> $usage
* @param array{next_cursor: string|null, has_more: bool}|null $pagination
*/
public function search(iterable $rows, User $user, array $usage): JsonResponse
public function search(iterable $rows, User $user, array $usage, ?array $pagination = null): JsonResponse
{
$rows = is_array($rows) ? $rows : iterator_to_array($rows, false);
$results = [];
@@ -33,11 +34,16 @@ final class ApiV2Presenter
$results[] = ReleaseData::toArrayFromRelease($row, $user, url('/details').'/', url('/getnzb'));
}
return $this->json(array_merge(
$payload = array_merge(
['Total' => (int) ($rows[0]->_totalrows ?? 0)],
$usage,
['results' => $results],
));
);
if ($pagination !== null) {
$payload['pagination'] = $pagination;
}
return $this->json($payload);
}
public function details(Release|\stdClass $release, User $user): JsonResponse
app/Services/Releases/ReleaseSearchService.php
+13 -3
View File
@@ -11,6 +11,8 @@ use App\Models\Category;
use App\Models\Release;
use App\Models\Settings;
use App\Models\UsenetGroup;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchCursor;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Cache;
@@ -116,7 +118,8 @@ class ReleaseSearchService
'try_fuzzy' => true,
];
$filtered = Search::searchReleasesFiltered($criteria, $limit, $offset);
$searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset));
$filtered = $searchPage->legacy();
if ($filtered['ids'] !== []) {
$ids = array_map(static fn (int|string $id): int => (int) $id, $filtered['ids']);
@@ -214,7 +217,7 @@ class ReleaseSearchService
* @param array<int, int> $excludedCats
* @return Collection|mixed
*/
public function apiSearch(mixed $searchName, mixed $groupName, int $offset = 0, int $limit = 1000, int $maxAge = -1, array $excludedCats = [], array $cat = [-1], int $minSize = 0, string $orderBy = 'posted_desc'): mixed
public function apiSearch(mixed $searchName, mixed $groupName, int $offset = 0, int $limit = 1000, int $maxAge = -1, array $excludedCats = [], array $cat = [-1], int $minSize = 0, string $orderBy = 'posted_desc', ?SearchCursor $cursor = null): mixed
{
if (config('app.debug')) {
Log::debug('ReleaseSearchService::apiSearch called', [
@@ -258,7 +261,12 @@ class ReleaseSearchService
'try_fuzzy' => true,
];
$filtered = Search::searchReleasesFiltered($criteria, $limit, $offset);
$criteria['track_total'] = $cursor === null;
$searchPage = Search::searchReleasePage(ReleaseSearchQuery::fromCriteria($criteria, $limit, $offset, $cursor));
$filtered = $searchPage->legacy();
if ($cursor !== null && $filtered['total'] === 0) {
$filtered['total'] = $cursor->total;
}
if ($filtered['ids'] === [] && $hasText && config('nntmux.mysql_search_fallback', false) === true) {
return $this->apiSearchLegacyMysql($searchName, $groupName, $offset, $limit, $maxAge, $excludedCats, $cat, $minSize, $orderBy);
@@ -304,6 +312,8 @@ class ReleaseSearchService
if ($releases->isNotEmpty()) {
$releases[0]->_totalrows = $filtered['total'];
$releases[0]->_search_last_sort = $searchPage->lastSortValues;
$releases[0]->_search_has_more = $searchPage->hasMore;
}
$expiresAt = now()->addMinutes(config('nntmux.cache_expiry_medium'));
app/Services/Search/Contracts/SearchServiceInterface.php
+4
View File
@@ -5,6 +5,8 @@ declare(strict_types=1);
namespace App\Services\Search\Contracts;
use App\Enums\SecondarySearchIndex;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
/**
* Interface for full-text search services (ManticoreSearch, Elasticsearch).
@@ -352,6 +354,8 @@ interface SearchServiceInterface
*/
public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array;
public function searchReleasePage(ReleaseSearchQuery $query): SearchPage;
/**
* Upsert a metadata document (music, books, games, console, steam, anime).
*
app/Services/Search/DTO/ReleaseSearchQuery.php
+98
View File
@@ -0,0 +1,98 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\DTO;
final readonly class ReleaseSearchQuery
{
/**
* @param array<string, mixed>|string|null $phrases
* @param list<int>|null $categoryIds
* @param list<int> $excludedCategoryIds
* @param list<int>|null $releaseIds
*/
public function __construct(
public array|string|null $phrases = null,
public ?array $categoryIds = null,
public array $excludedCategoryIds = [],
public ?array $releaseIds = null,
public int $minSize = 0,
public int $maxSize = 0,
public int $maxAgeDays = -1,
public int $minDate = 0,
public int $maxDate = 0,
public ?int $groupId = null,
public bool $passwordAllowRar = false,
public ?int $passwordStatusMin = null,
public string $sortField = 'postdate_ts',
public string $sortDirection = 'desc',
public bool $tryFuzzy = true,
public int $limit = 100,
public int $offset = 0,
public ?SearchCursor $cursor = null,
public bool $trackTotal = true,
) {}
/** @param array<string, mixed> $criteria */
public static function fromCriteria(array $criteria, int $limit, int $offset = 0, ?SearchCursor $cursor = null): self
{
return new self(
phrases: is_array($criteria['phrases'] ?? null) || is_string($criteria['phrases'] ?? null) ? $criteria['phrases'] : null,
categoryIds: is_array($criteria['category_ids'] ?? null) ? array_map('intval', $criteria['category_ids']) : null,
excludedCategoryIds: is_array($criteria['excluded_category_ids'] ?? null) ? array_map('intval', $criteria['excluded_category_ids']) : [],
releaseIds: is_array($criteria['release_ids'] ?? null) ? array_map('intval', $criteria['release_ids']) : null,
minSize: (int) ($criteria['min_size'] ?? 0),
maxSize: (int) ($criteria['max_size'] ?? 0),
maxAgeDays: (int) ($criteria['max_age_days'] ?? -1),
minDate: (int) ($criteria['min_date'] ?? 0),
maxDate: (int) ($criteria['max_date'] ?? 0),
groupId: isset($criteria['groups_id']) ? (int) $criteria['groups_id'] : null,
passwordAllowRar: (bool) ($criteria['password_allow_rar'] ?? false),
passwordStatusMin: isset($criteria['password_status_min']) ? (int) $criteria['password_status_min'] : null,
sortField: (string) ($criteria['sort_field'] ?? 'postdate_ts'),
sortDirection: (string) ($criteria['sort_dir'] ?? 'desc'),
tryFuzzy: (bool) ($criteria['try_fuzzy'] ?? true),
limit: max(1, $limit),
offset: max(0, $offset),
cursor: $cursor,
trackTotal: (bool) ($criteria['track_total'] ?? true),
);
}
/** @return array<string, mixed> */
public function criteria(): array
{
return [
'phrases' => $this->phrases,
'category_ids' => $this->categoryIds,
'excluded_category_ids' => $this->excludedCategoryIds,
'release_ids' => $this->releaseIds,
'min_size' => max(0, $this->minSize),
'max_size' => max(0, $this->maxSize),
'max_age_days' => $this->maxAgeDays,
'min_date' => max(0, $this->minDate),
'max_date' => max(0, $this->maxDate),
'groups_id' => $this->groupId,
'password_allow_rar' => $this->passwordAllowRar,
'password_status_min' => $this->passwordStatusMin,
'sort_field' => $this->normalizedSortField(),
'sort_dir' => $this->normalizedSortDirection(),
'try_fuzzy' => $this->tryFuzzy,
'cursor_sort' => $this->cursor?->sortValues,
'track_total' => $this->trackTotal,
];
}
public function normalizedSortField(): string
{
return in_array($this->sortField, ['postdate_ts', 'adddate_ts', 'size', 'totalpart', 'grabs', 'categories_id', 'id'], true)
? $this->sortField
: 'postdate_ts';
}
public function normalizedSortDirection(): string
{
return strtolower($this->sortDirection) === 'asc' ? 'asc' : 'desc';
}
}
app/Services/Search/DTO/SearchCursor.php
+18
View File
@@ -0,0 +1,18 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\DTO;
final readonly class SearchCursor
{
/** @param list<int|float|string> $sortValues */
public function __construct(
public array $sortValues,
public int $total,
public string $queryHash,
public string $driver,
public string $indexGeneration,
public int $expiresAt,
) {}
}
app/Services/Search/DTO/SearchPage.php
+29
View File
@@ -0,0 +1,29 @@
<?php
declare(strict_types=1);
namespace App\Services\Search\DTO;
final readonly class SearchPage
{
/**
* @param list<int> $ids
* @param list<int|float|string> $lastSortValues
*/
public function __construct(
public array $ids,
public int $total,
public bool $fuzzy,
public string $driver,
public bool $available = true,
public float $durationMs = 0.0,
public array $lastSortValues = [],
public bool $hasMore = false,
) {}
/** @return array{ids: list<int>, total: int, fuzzy: bool} */
public function legacy(): array
{
return ['ids' => $this->ids, 'total' => $this->total, 'fuzzy' => $this->fuzzy];
}
}
app/Services/Search/Drivers/ElasticSearchDriver.php
+51 -16
View File
@@ -14,6 +14,8 @@ use App\Models\Release;
use App\Models\SteamApp;
use App\Models\Video;
use App\Services\Search\Contracts\SearchDriverInterface;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use App\Services\Search\Support\ElasticsearchClientFactory;
use App\Services\Search\Support\ElasticsearchResponseHelper;
use App\Support\SecondaryIndexDocuments;
@@ -93,6 +95,23 @@ class ElasticSearchDriver implements SearchDriverInterface
return $this->isElasticsearchAvailable();
}
public function searchReleasePage(ReleaseSearchQuery $query): SearchPage
{
$startedAt = hrtime(true);
$result = $this->searchReleasesFiltered($query->criteria(), $query->limit, $query->offset);
return new SearchPage(
ids: array_map(static fn (int|string $id): int => (int) $id, $result['ids']),
total: (int) $result['total'],
fuzzy: (bool) $result['fuzzy'],
driver: $this->getDriverName(),
available: (bool) ($result['available'] ?? true),
durationMs: (hrtime(true) - $startedAt) / 1_000_000,
lastSortValues: $result['last_sort'] ?? [],
hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])),
);
}
/**
* Get or create an Elasticsearch client with proper cURL configuration.
*
@@ -2859,7 +2878,7 @@ class ElasticSearchDriver implements SearchDriverInterface
public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array
{
if (! $this->isElasticsearchAvailable()) {
return ['ids' => [], 'total' => 0, 'fuzzy' => false];
return ['ids' => [], 'total' => 0, 'fuzzy' => false, 'available' => false];
}
$phrases = $criteria['phrases'] ?? null;
@@ -2936,38 +2955,54 @@ class ElasticSearchDriver implements SearchDriverInterface
try {
$client = $this->getClient();
$body = [
'query' => $query,
'sort' => [
[$sortField => ['order' => $order]],
['id' => ['order' => $order]],
],
'size' => max(1, min($limit, self::MAX_RESULTS)),
'track_total_hits' => (bool) ($criteria['track_total'] ?? true),
'_source' => false,
];
$cursorSort = $criteria['cursor_sort'] ?? null;
if (is_array($cursorSort) && count($cursorSort) === 2) {
$body['search_after'] = array_values($cursorSort);
} else {
$body['from'] = max(0, $offset);
}
$response = $client->search([
'index' => $this->getReleasesIndex(),
'body' => [
'query' => $query,
'sort' => [
[$sortField => ['order' => $order]],
['id' => ['order' => $order]],
],
'from' => max(0, $offset),
'size' => max(1, min($limit, self::MAX_RESULTS)),
'track_total_hits' => true,
'_source' => false,
],
'body' => $body,
]);
$ids = [];
foreach ($response['hits']['hits'] ?? [] as $hit) {
$hits = $response['hits']['hits'] ?? [];
foreach ($hits as $hit) {
$ids[] = (int) ($hit['_id'] ?? 0);
}
$total = (int) ($response['hits']['total']['value'] ?? $response['hits']['total'] ?? 0);
$lastHit = $hits === [] ? null : $hits[array_key_last($hits)];
return ['ids' => $ids, 'total' => $total, 'fuzzy' => $useFuzzy];
return [
'ids' => $ids,
'total' => $total,
'fuzzy' => $useFuzzy,
'available' => true,
'last_sort' => is_array($lastHit) && is_array($lastHit['sort'] ?? null) ? array_values($lastHit['sort']) : [],
'has_more' => count($ids) === max(1, min($limit, self::MAX_RESULTS)) && ($total === 0 || count($ids) + $offset < $total),
];
} catch (\Throwable $e) {
Log::error('ElasticSearch searchReleasesFiltered error: '.$e->getMessage());
return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy];
return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy, 'available' => false];
}
};
if ($hasText) {
$first = $run(false);
if ($first['ids'] !== [] || ! $tryFuzzy || ! $this->isFuzzyEnabled()) {
if ($first['ids'] !== [] || ($first['available'] ?? true) === false || ! $tryFuzzy || ! $this->isFuzzyEnabled()) {
return $first;
}
app/Services/Search/Drivers/ManticoreSearchDriver.php
+45 -4
View File
@@ -15,6 +15,8 @@ use App\Models\Release;
use App\Models\SteamApp;
use App\Models\Video;
use App\Services\Search\Contracts\SearchDriverInterface;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use App\Services\Search\Support\ManticoreClientFactory;
use App\Support\ReleaseSearchIndexDocument;
use App\Support\SecondaryIndexDocuments;
@@ -31,6 +33,12 @@ use Manticoresearch\Search;
*/
class ManticoreSearchDriver implements SearchDriverInterface
{
private const AVAILABILITY_CHECK_CACHE_TTL = 30;
private static ?bool $availabilityCache = null;
private static ?int $availabilityCacheTime = null;
/**
* @var array<string, mixed>
*/
@@ -74,11 +82,22 @@ class ManticoreSearchDriver implements SearchDriverInterface
*/
public function isAvailable(): bool
{
$now = time();
if (self::$availabilityCache !== null && self::$availabilityCacheTime !== null
&& ($now - self::$availabilityCacheTime) < self::AVAILABILITY_CHECK_CACHE_TTL) {
return self::$availabilityCache;
}
try {
$status = $this->manticoreSearch->nodes()->status();
return ! empty($status);
self::$availabilityCache = ! empty($status);
self::$availabilityCacheTime = $now;
return self::$availabilityCache;
} catch (\Throwable $e) {
self::$availabilityCache = false;
self::$availabilityCacheTime = $now;
if (config('app.debug')) {
Log::debug('ManticoreSearch not available: '.$e->getMessage());
}
@@ -87,6 +106,23 @@ class ManticoreSearchDriver implements SearchDriverInterface
}
}
public function searchReleasePage(ReleaseSearchQuery $query): SearchPage
{
$startedAt = hrtime(true);
$result = $this->searchReleasesFiltered($query->criteria(), $query->limit, $query->offset);
return new SearchPage(
ids: array_map(static fn (int|string $id): int => (int) $id, $result['ids']),
total: (int) $result['total'],
fuzzy: (bool) $result['fuzzy'],
driver: $this->getDriverName(),
available: (bool) ($result['available'] ?? true),
durationMs: (hrtime(true) - $startedAt) / 1_000_000,
lastSortValues: $result['last_sort'] ?? [],
hasMore: (bool) ($result['has_more'] ?? (($query->offset + count($result['ids'])) < (int) $result['total'])),
);
}
/**
* Check if autocomplete is enabled.
*/
@@ -2467,7 +2503,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
public function searchReleasesFiltered(array $criteria, int $limit, int $offset = 0): array
{
if (! $this->isAvailable()) {
return ['ids' => [], 'total' => 0, 'fuzzy' => false];
return ['ids' => [], 'total' => 0, 'fuzzy' => false, 'available' => false];
}
$phrases = $criteria['phrases'] ?? null;
@@ -2539,7 +2575,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
'criteria' => $criteria,
]);
return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy];
return ['ids' => [], 'total' => 0, 'fuzzy' => $useFuzzy, 'available' => false];
}
$ids = [];
@@ -2551,12 +2587,17 @@ class ManticoreSearchDriver implements SearchDriverInterface
'ids' => $ids,
'total' => (int) $results->getTotal(),
'fuzzy' => $useFuzzy,
'available' => true,
// Manticore uses the compatible offset cursor until the PHP client
// supports a compound search-after predicate without raw SQL.
'last_sort' => [],
'has_more' => $offset + count($ids) < (int) $results->getTotal(),
];
};
if ($hasText) {
$first = $execute(false);
if ($first['ids'] !== [] || ! $tryFuzzy || ! $this->isFuzzyEnabled() || self::queryHasNegation($phrases)) {
if ($first['ids'] !== [] || ($first['available'] ?? true) === false || ! $tryFuzzy || ! $this->isFuzzyEnabled() || self::queryHasNegation($phrases)) {
return $first;
}
app/Services/Search/SearchCursorCodec.php
+49
View File
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
namespace App\Services\Search;
use App\Services\Search\DTO\SearchCursor;
use Illuminate\Contracts\Encryption\DecryptException;
use Illuminate\Contracts\Encryption\Encrypter;
use InvalidArgumentException;
final readonly class SearchCursorCodec
{
public function __construct(private Encrypter $encrypter) {}
public function encode(SearchCursor $cursor): string
{
return $this->encrypter->encryptString(json_encode([
'sort' => $cursor->sortValues,
'total' => $cursor->total,
'query' => $cursor->queryHash,
'driver' => $cursor->driver,
'generation' => $cursor->indexGeneration,
'expires' => $cursor->expiresAt,
], JSON_THROW_ON_ERROR));
}
public function decode(string $token): SearchCursor
{
try {
$payload = json_decode($this->encrypter->decryptString($token), true, flags: JSON_THROW_ON_ERROR);
} catch (DecryptException|\JsonException $e) {
throw new InvalidArgumentException('Invalid search cursor.', previous: $e);
}
if (! is_array($payload) || ! is_array($payload['sort'] ?? null) || (int) ($payload['expires'] ?? 0) < time()) {
throw new InvalidArgumentException('Invalid or expired search cursor.');
}
return new SearchCursor(
sortValues: array_values($payload['sort']),
total: (int) ($payload['total'] ?? 0),
queryHash: (string) ($payload['query'] ?? ''),
driver: (string) ($payload['driver'] ?? ''),
indexGeneration: (string) ($payload['generation'] ?? ''),
expiresAt: (int) $payload['expires'],
);
}
}
app/Services/Search/SearchService.php
+22
View File
@@ -9,6 +9,9 @@ use App\Services\Search\Contracts\SearchDriverInterface;
use App\Services\Search\Contracts\SearchServiceInterface;
use App\Services\Search\Drivers\ElasticSearchDriver;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Manager;
/**
@@ -510,6 +513,25 @@ class SearchService extends Manager implements SearchServiceInterface
return $this->driver()->searchReleasesFiltered($criteria, $limit, $offset);
}
public function searchReleasePage(ReleaseSearchQuery $query): SearchPage
{
$page = $this->driver()->searchReleasePage($query);
Log::debug('Release search completed', [
'driver' => $page->driver,
'available' => $page->available,
'duration_ms' => round($page->durationMs, 2),
'result_count' => count($page->ids),
'total' => $page->total,
'fuzzy' => $page->fuzzy,
'pagination_mode' => $query->cursor === null ? 'offset' : 'cursor',
'offset' => $query->offset,
'limit' => $query->limit,
'track_total' => $query->trackTotal,
]);
return $page;
}
public function insertSecondary(SecondarySearchIndex $index, int $id, array $document): void
{
$this->driver()->insertSecondary($index, $id, $document);
config/search.php
+4
View File
@@ -13,6 +13,10 @@ return [
*/
'default' => env('SEARCH_DRIVER', 'manticore'),
'index_generation' => env('SEARCH_INDEX_GENERATION', '1'),
'cursor_ttl_minutes' => (int) env('SEARCH_CURSOR_TTL_MINUTES', 15),
/*
|--------------------------------------------------------------------------
| Search Driver Configurations
docs/search-performance.md
+36
View File
@@ -0,0 +1,36 @@
# Search performance and index maintenance
The website, Newznab API, and API v2 share the release search contract in
`app/Services/Search/DTO`. Offset responses keep exact totals. API v2 also accepts an
empty `cursor` parameter to start cursor-mode pagination and returns the next opaque,
signed cursor in `pagination.next_cursor`.
## Baseline and validation
Capture warm- and cold-cache measurements for both `SEARCH_DRIVER=manticore` and
`SEARCH_DRIVER=elasticsearch`. Record p50/p95 response time, search backend duration,
SQL query count, peak memory, result count, offset, and whether fuzzy fallback ran.
Use the same query/category/sort matrix before and after a change. Include offsets 0,
1,000, and 5,000 plus API v2 cursor traversal.
Do not set a performance target until the baseline is recorded. A normal successful
exact search should make one search request; fuzzy search may make a second request
only after an exact zero-result response.
## Maintenance rebuild
Search schema changes require a maintenance window. Before rebuilding, enable
`nntmux.mysql_search_fallback` if bounded database fallback is desired while indexes
are empty.
1. Run `php artisan tmux:stop` and pause queue workers that update search indexes.
2. Recreate Manticore indexes with `php artisan manticore:create-indexes --drop`, or
recreate Elasticsearch indexes with the project's Elasticsearch index command.
3. Repopulate enabled indexes with `php artisan nntmux:populate --manticore --all` or
the equivalent Elasticsearch population flags.
4. Compare source and index document counts and run representative filtered searches.
5. Increment `SEARCH_INDEX_GENERATION` to invalidate outstanding API v2 cursors.
6. Restart queue workers and run `php artisan tmux:start`.
If count or query validation fails, leave processing stopped, restore/rebuild the
previous schema, and keep database fallback enabled until indexes are healthy.
tests/Feature/MovieSearchApiTest.php
+9 -9
View File
@@ -6,6 +6,8 @@ namespace Tests\Feature;
use App\Models\Release;
use App\Services\Releases\ReleaseSearchService;
use App\Services\Search\DTO\ReleaseSearchQuery;
use App\Services\Search\DTO\SearchPage;
use App\Services\Search\SearchService;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Cache;
@@ -140,19 +142,17 @@ final class MovieSearchApiTest extends TestCase
{
$mock = Mockery::mock(SearchService::class, [$this->app]);
$mock->shouldReceive('isAvailable')->andReturn(true);
$mock->shouldReceive('searchReleasesFiltered')
$mock->shouldReceive('searchReleasePage')
->once()
->withArgs(function (array $criteria, int $limit, int $offset): bool {
->withArgs(function (ReleaseSearchQuery $query): bool {
$criteria = $query->criteria();
return ($criteria['phrases'] ?? null) === 'Resurrection 2025'
&& ($criteria['try_fuzzy'] ?? null) === true
&& $limit === 100
&& $offset === 0;
&& $query->limit === 100
&& $query->offset === 0;
})
->andReturn([
'ids' => [],
'total' => 0,
'fuzzy' => false,
]);
->andReturn(new SearchPage([], 0, false, 'manticore'));
$this->app->instance(SearchService::class, $mock);
tests/Unit/ElasticSearchQueryTest.php
+13
View File
@@ -54,4 +54,17 @@ final class ElasticSearchQueryTest extends TestCase
'Expected indexSearch/indexSearchApi/indexSearchTMA to use RELEASE_TEXT_FIELDS.'
);
}
#[Test]
public function filtered_release_search_supports_search_after_without_loading_source(): void
{
$driverSource = file_get_contents(__DIR__.'/../../app/Services/Search/Drivers/ElasticSearchDriver.php');
$this->assertIsString($driverSource);
$methodSource = strstr($driverSource, 'public function searchReleasesFiltered');
$this->assertIsString($methodSource);
$this->assertStringContainsString("\$body['search_after']", $methodSource);
$this->assertStringContainsString("'_source' => false", $methodSource);
$this->assertStringContainsString("'track_total_hits' => (bool)", $methodSource);
}
}
tests/Unit/SearchCursorCodecTest.php
+49
View File
@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
namespace Tests\Unit;
use App\Services\Search\DTO\SearchCursor;
use App\Services\Search\SearchCursorCodec;
use Illuminate\Encryption\Encrypter;
use InvalidArgumentException;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\TestCase;
final class SearchCursorCodecTest extends TestCase
{
#[Test]
public function it_round_trips_an_opaque_signed_cursor(): void
{
$codec = new SearchCursorCodec(new Encrypter(random_bytes(32), 'AES-256-CBC'));
$cursor = new SearchCursor([1710000000, 42], 250, 'query-hash', 'elasticsearch', '7', time() + 60);
$decoded = $codec->decode($codec->encode($cursor));
$this->assertSame($cursor->sortValues, $decoded->sortValues);
$this->assertSame(250, $decoded->total);
$this->assertSame('query-hash', $decoded->queryHash);
$this->assertSame('elasticsearch', $decoded->driver);
$this->assertSame('7', $decoded->indexGeneration);
}
#[Test]
public function it_rejects_tampered_and_expired_cursors(): void
{
$codec = new SearchCursorCodec(new Encrypter(random_bytes(32), 'AES-256-CBC'));
$this->expectException(InvalidArgumentException::class);
$codec->decode('tampered');
}
#[Test]
public function it_rejects_an_expired_cursor(): void
{
$codec = new SearchCursorCodec(new Encrypter(random_bytes(32), 'AES-256-CBC'));
$token = $codec->encode(new SearchCursor([100], 10, 'hash', 'manticore', '1', time() - 1));
$this->expectException(InvalidArgumentException::class);
$codec->decode($token);
}
}