starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix insertion errors

Browse Source
This commit is contained in:
DariusIII
2026-01-01 13:45:49 +01:00
parent d41d35be46
commit e1e034533d
5 changed files with 254 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/NntmuxPopulateSearchIndexes.php
+22 -15
View File
@@ -221,22 +221,22 @@ class NntmuxPopulateSearchIndexes extends Command
$query,
function ($item) {
return [
'id' => (string) $item->id,
'name' => (string) ($item->name ?: ''),
'searchname' => (string) ($item->searchname ?: ''),
'fromname' => (string) ($item->fromname ?: ''),
'categories_id' => (int) ($item->categories_id ?: 0),
'filename' => (string) ($item->filename ?: ''),
'videos_id' => (int) ($item->videos_id ?: 0),
'movieinfo_id' => (int) ($item->movieinfo_id ?: 0),
'id' => (int) $item->id,
'name' => (string) ($item->name ?? ''),
'searchname' => (string) ($item->searchname ?? ''),
'fromname' => (string) ($item->fromname ?? ''),
'categories_id' => (int) ($item->categories_id ?? 0),
'filename' => (string) ($item->filename ?? ''),
'videos_id' => (int) ($item->videos_id ?? 0),
'movieinfo_id' => (int) ($item->movieinfo_id ?? 0),
// Movie external IDs
'imdbid' => (int) ($item->imdbid ?: 0),
'tmdbid' => (int) ($item->tmdbid ?: 0),
'traktid' => (int) ($item->traktid ?: 0),
'imdbid' => (int) ($item->imdbid ?? 0),
'tmdbid' => (int) ($item->tmdbid ?? 0),
'traktid' => (int) ($item->traktid ?? 0),
// TV show external IDs (use video_* for TV shows, fallback to movie IDs)
'tvdb' => (int) ($item->tvdb ?: 0),
'tvmaze' => (int) ($item->tvmaze ?: 0),
'tvrage' => (int) ($item->tvrage ?: 0),
'tvdb' => (int) ($item->tvdb ?? 0),
'tvmaze' => (int) ($item->tvmaze ?? 0),
'tvrage' => (int) ($item->tvrage ?? 0),
];
}
);
@@ -787,7 +787,14 @@ class NntmuxPopulateSearchIndexes extends Command
while ($attempt < $retries) {
try {
Search::bulkInsertReleases($data);
// Use the correct bulk insert method based on index name
if ($indexName === 'releases_rt') {
Search::bulkInsertReleases($data);
} elseif ($indexName === 'predb_rt') {
Search::bulkInsertPredb($data);
} else {
throw new Exception("Unknown index: {$indexName}");
}
break;
} catch (Exception $e) {
$attempt++;
app/Services/Search/Contracts/SearchServiceInterface.php
+8
View File
@@ -162,6 +162,14 @@ interface SearchServiceInterface
*/
public function bulkInsertReleases(array $releases): array;
/**
* Bulk insert multiple predb records into the index.
*
* @param array $predbRecords Array of predb data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertPredb(array $predbRecords): array;
/**
* Delete a predb record from the index.
*
app/Services/Search/Drivers/ElasticSearchDriver.php
+137 -46
View File
@@ -1079,52 +1079,6 @@ class ElasticSearchDriver implements SearchDriverInterface
}
}
/**
* Bulk insert multiple releases into the index.
*
* @param array $releases Array of release data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertReleases(array $releases): array
{
if (empty($releases) || ! $this->isElasticsearchAvailable()) {
return ['success' => 0, 'errors' => 0];
}
$params = ['body' => []];
$validReleases = 0;
foreach ($releases as $release) {
if (empty($release['id'])) {
continue;
}
$params['body'][] = [
'index' => [
'_index' => $this->getReleasesIndex(),
'_id' => $release['id'],
],
];
$document = $this->buildReleaseDocument($release);
$params['body'][] = $document['body'];
$validReleases++;
// Send batch when reaching 500 documents
if ($validReleases % 500 === 0) {
$this->executeBulk($params);
$params = ['body' => []];
}
}
// Send remaining documents
if (! empty($params['body'])) {
$this->executeBulk($params);
}
return ['success' => $validReleases, 'errors' => 0];
}
/**
* Update a release in the index.
*
@@ -1360,6 +1314,143 @@ class ElasticSearchDriver implements SearchDriverInterface
}
}
/**
* Bulk insert multiple releases into the index.
*
* @param array $releases Array of release data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertReleases(array $releases): array
{
if (empty($releases) || ! $this->isElasticsearchAvailable()) {
return ['success' => 0, 'errors' => 0];
}
$success = 0;
$errors = 0;
$params = ['body' => []];
foreach ($releases as $release) {
if (empty($release['id'])) {
$errors++;
continue;
}
$searchNameDotless = $this->createPlainSearchName($release['searchname'] ?? '');
$params['body'][] = [
'index' => [
'_index' => $this->getReleasesIndex(),
'_id' => $release['id'],
],
];
$params['body'][] = [
'id' => $release['id'],
'name' => (string) ($release['name'] ?? ''),
'searchname' => (string) ($release['searchname'] ?? ''),
'plainsearchname' => $searchNameDotless,
'fromname' => (string) ($release['fromname'] ?? ''),
'categories_id' => (int) ($release['categories_id'] ?? 0),
'filename' => (string) ($release['filename'] ?? ''),
];
$success++;
}
if (! empty($params['body'])) {
try {
$client = $this->getClient();
$response = $client->bulk($params);
if (isset($response['errors']) && $response['errors']) {
foreach ($response['items'] as $item) {
if (isset($item['index']['error'])) {
$errors++;
$success--;
if (config('app.debug')) {
Log::error('ElasticSearch bulkInsertReleases error: '.json_encode($item['index']['error']));
}
}
}
}
} catch (\Throwable $e) {
Log::error('ElasticSearch bulkInsertReleases error: '.$e->getMessage());
$errors += $success;
$success = 0;
}
}
return ['success' => $success, 'errors' => $errors];
}
/**
* Bulk insert multiple predb records into the index.
*
* @param array $predbRecords Array of predb data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertPredb(array $predbRecords): array
{
if (empty($predbRecords) || ! $this->isElasticsearchAvailable()) {
return ['success' => 0, 'errors' => 0];
}
$success = 0;
$errors = 0;
$params = ['body' => []];
foreach ($predbRecords as $predb) {
if (empty($predb['id'])) {
$errors++;
continue;
}
$params['body'][] = [
'index' => [
'_index' => $this->getPredbIndex(),
'_id' => $predb['id'],
],
];
$params['body'][] = [
'id' => $predb['id'],
'title' => (string) ($predb['title'] ?? ''),
'filename' => (string) ($predb['filename'] ?? ''),
'source' => (string) ($predb['source'] ?? ''),
];
$success++;
}
if (! empty($params['body'])) {
try {
$client = $this->getClient();
$response = $client->bulk($params);
if (isset($response['errors']) && $response['errors']) {
foreach ($response['items'] as $item) {
if (isset($item['index']['error'])) {
$errors++;
$success--;
if (config('app.debug')) {
Log::error('ElasticSearch bulkInsertPredb error: '.json_encode($item['index']['error']));
}
}
}
}
} catch (\Throwable $e) {
Log::error('ElasticSearch bulkInsertPredb error: '.$e->getMessage());
$errors += $success;
$success = 0;
}
}
return ['success' => $success, 'errors' => $errors];
}
/**
* Check if an index exists.
*
app/Services/Search/Drivers/ManticoreSearchDriver.php
+76 -7
View File
@@ -287,11 +287,11 @@ class ManticoreSearchDriver implements SearchDriverInterface
$documents[] = [
'id' => $release['id'],
'name' => $release['name'] ?? '',
'searchname' => $release['searchname'] ?? '',
'fromname' => $release['fromname'] ?? '',
'name' => (string) ($release['name'] ?? ''),
'searchname' => (string) ($release['searchname'] ?? ''),
'fromname' => (string) ($release['fromname'] ?? ''),
'categories_id' => (int) ($release['categories_id'] ?? 0),
'filename' => $release['filename'] ?? '',
'filename' => (string) ($release['filename'] ?? ''),
// External media IDs for efficient searching
'imdbid' => (int) ($release['imdbid'] ?? 0),
'tmdbid' => (int) ($release['tmdbid'] ?? 0),
@@ -306,6 +306,14 @@ class ManticoreSearchDriver implements SearchDriverInterface
if (! empty($documents)) {
try {
// Log first document for debugging if app.debug is enabled
if (config('app.debug') && ! empty($documents[0])) {
Log::debug('ManticoreSearch bulkInsertReleases sample document', [
'first_doc' => $documents[0],
'total_docs' => count($documents),
]);
}
$this->manticoreSearch->table($this->config['indexes']['releases'])
->replaceDocuments($documents);
$success = count($documents);
@@ -318,6 +326,58 @@ class ManticoreSearchDriver implements SearchDriverInterface
return ['success' => $success, 'errors' => $errors];
}
/**
* Bulk insert multiple predb records into the index.
*
* @param array $predbRecords Array of predb data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertPredb(array $predbRecords): array
{
if (empty($predbRecords)) {
return ['success' => 0, 'errors' => 0];
}
$success = 0;
$errors = 0;
$documents = [];
foreach ($predbRecords as $predb) {
if (empty($predb['id'])) {
$errors++;
continue;
}
$documents[] = [
'id' => $predb['id'],
'title' => (string) ($predb['title'] ?? ''),
'filename' => (string) ($predb['filename'] ?? ''),
'source' => (string) ($predb['source'] ?? ''),
];
}
if (! empty($documents)) {
try {
// Log first document for debugging if app.debug is enabled
if (config('app.debug') && ! empty($documents[0])) {
Log::debug('ManticoreSearch bulkInsertPredb sample document', [
'first_doc' => $documents[0],
'total_docs' => count($documents),
]);
}
$this->manticoreSearch->table($this->config['indexes']['predb'])
->replaceDocuments($documents);
$success = count($documents);
} catch (\Throwable $e) {
Log::error('ManticoreSearch bulkInsertPredb error: '.$e->getMessage());
$errors += count($documents);
}
}
return ['success' => $success, 'errors' => $errors];
}
/**
* Delete release from Manticore RT tables by GUID.
*
@@ -813,7 +873,7 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Check if fuzzy search failed due to missing min_infix_len
// This happens when index was created without proper settings
if (str_contains($message, 'min_infix_len')) {
Log::warning('ManticoreSearch fuzzySearchIndexes: Fuzzy search unavailable - index missing min_infix_len setting. Please recreate the index with: php artisan nntmux:manticore-create --drop', [
Log::warning('ManticoreSearch fuzzySearchIndexes: Fuzzy search unavailable - index missing min_infix_len setting. Please recreate the index with: php artisan manticore:create-indexes --drop', [
'index' => $index,
]);
// Fall back to regular search without fuzzy
@@ -967,6 +1027,13 @@ class ManticoreSearchDriver implements SearchDriverInterface
// Avoid explicit sort for predb_rt to prevent Manticore's "too many sort-by attributes" error
$avoidSortForIndex = ($rt_index === 'predb_rt');
if (config('app.debug')) {
Log::debug('ManticoreSearch::searchIndexes executing query', [
'rt_index' => $rt_index,
'searchExpr' => $searchExpr,
]);
}
try {
// Use a fresh Search instance for every query to avoid parameter accumulation across calls
$query = (new Search($this->manticoreSearch))
@@ -1044,8 +1111,10 @@ class ManticoreSearchDriver implements SearchDriverInterface
'data' => $resultData,
];
// Cache results for 5 minutes
Cache::put($cacheKey, $result, now()->addMinutes($this->config['cache_minutes'] ?? 5));
// Only cache non-empty results to avoid caching temporary failures or empty index states
if (! empty($resultIds)) {
Cache::put($cacheKey, $result, now()->addMinutes($this->config['cache_minutes'] ?? 5));
}
return $result;
}
app/Services/Search/SearchService.php
+11
View File
@@ -254,6 +254,17 @@ class SearchService extends Manager implements SearchServiceInterface
return $this->driver()->bulkInsertReleases($releases);
}
/**
* Bulk insert multiple predb records into the index.
*
* @param array $predbRecords Array of predb data arrays
* @return array Results with 'success' and 'errors' counts
*/
public function bulkInsertPredb(array $predbRecords): array
{
return $this->driver()->bulkInsertPredb($predbRecords);
}
/**
* Delete a predb record from the index.
*