starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update search reconciliation

Browse Source
This commit is contained in:
DariusIII
2026-05-13 13:43:58 +02:00
parent 282070a2af
commit b9dd3d4dac
2 changed files with 118 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/NntmuxSearchDiag.php
+25
View File
@@ -118,6 +118,31 @@ class NntmuxSearchDiag extends Command
return [];
}
// In raw mode (sql($q, true)) the response goes through
// Manticoresearch\Response\SqlToArray::getResponse(), which flattens a SELECT
// result so each row is keyed by its `id` (and the `id` column is removed from
// multi-column rows). Reconstruct row arrays here. The legacy
// ['data' => [...]] shape is retained as a fallback.
$rows = [];
foreach ($response as $key => $value) {
if (\is_int($key) || (\is_string($key) && ctype_digit($key))) {
if (\is_array($value)) {
$row = $value;
if (! isset($row['id'])) {
$row['id'] = (int) $key;
}
$rows[] = $row;
} else {
// Single-column id projection: value mirrors the key.
$rows[] = ['id' => (int) $key];
}
}
}
if ($rows !== []) {
return $rows;
}
if (isset($response['data']) && \is_array($response['data'])) {
return array_values($response['data']);
}
app/Console/Commands/NntmuxSearchReconcile.php
+93 -5
View File
@@ -9,6 +9,7 @@ use App\Models\Release;
use App\Services\Search\Drivers\ManticoreSearchDriver;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Throwable;
class NntmuxSearchReconcile extends Command
@@ -80,6 +81,14 @@ class NntmuxSearchReconcile extends Command
$this->line('Sample ids: '.implode(', ', $sample));
}
// Hardening: if a dry-run with a non-trivial scan reports 100% missing, that almost
// always indicates a response-shape regression in fetchIndexedIds() (see
// Manticoresearch\Response\SqlToArray::getResponse()). Emit a single warning with a
// small probe sample so the regression is obvious without flooding logs.
if ($dryRun && $scanned >= 50 && $missingTotal === $scanned) {
$this->probeAndWarnAllMissing($manticore, $index, \array_slice($missingAll, 0, 5));
}
if ($missingTotal === 0) {
return self::SUCCESS;
}
@@ -145,17 +154,96 @@ class NntmuxSearchReconcile extends Command
return [];
}
if (! \is_array($response) || ! isset($response['data']) || ! \is_array($response['data'])) {
if (! \is_array($response)) {
return [];
}
$requested = array_flip($ids);
$out = [];
foreach ($response['data'] as $row) {
if (isset($row['id'])) {
$out[] = (int) $row['id'];
// In raw mode (sql($q, true)) the response goes through
// Manticoresearch\Response\SqlToArray::getResponse() which FLATTENS a SELECT result
// into [id => id] for single-column projections (or [id => rowArray] for multi-column).
// The old `$response['data']` shape only exists for non-raw mode or pre-4.x clients.
foreach ($response as $key => $value) {
// Top-level numeric keys mirror row ids (single-column id projection).
if (\is_int($key) || (\is_string($key) && ctype_digit($key))) {
$id = (int) $key;
if (isset($requested[$id])) {
$out[$id] = $id;
}
}
// Multi-column rows: value is an array possibly containing 'id'.
if (\is_array($value) && isset($value['id'])) {
$id = (int) $value['id'];
if (isset($requested[$id])) {
$out[$id] = $id;
}
}
// Scalar value that itself is the id (defensive).
if (\is_int($value) || (\is_string($value) && ctype_digit($value))) {
$id = (int) $value;
if (isset($requested[$id])) {
$out[$id] = $id;
}
}
}
return $out;
// Legacy fallback: ['data' => [['id' => N], ...]] (non-raw / older client).
if ($out === [] && isset($response['data']) && \is_array($response['data'])) {
foreach ($response['data'] as $row) {
if (\is_array($row) && isset($row['id'])) {
$id = (int) $row['id'];
if (isset($requested[$id])) {
$out[$id] = $id;
}
}
}
}
return array_values($out);
}
/**
* Emit a single warning when a dry-run scan reports 100% missing. Logs a small sample
* of the raw Manticore response so a future SqlToArray shape change is easy to spot.
*
* @param list<int> $sampleIds
*/
private function probeAndWarnAllMissing(ManticoreSearchDriver $driver, string $index, array $sampleIds): void
{
if ($sampleIds === []) {
return;
}
$list = implode(',', array_map(static fn (int $id): string => (string) $id, $sampleIds));
$safeIndex = str_replace('`', '``', $index);
$sql = "SELECT id FROM `{$safeIndex}` WHERE id IN ({$list})";
$rawShape = 'unavailable';
try {
$response = $driver->manticoreSearch->sql($sql, true);
if (\is_array($response)) {
$rawShape = json_encode([
'top_keys' => \array_slice(array_keys($response), 0, 10),
'has_data_key' => isset($response['data']),
'size' => \count($response),
], JSON_UNESCAPED_SLASHES) ?: 'encode_failed';
} else {
$rawShape = 'non_array:'.\gettype($response);
}
} catch (Throwable $e) {
$rawShape = 'exception:'.$e->getMessage();
}
$message = sprintf(
'nntmux:search-reconcile reported 100%% missing for index "%s" — possible Manticore client response-shape regression. Probe sample ids=%s shape=%s',
$index,
implode(',', $sampleIds),
$rawShape,
);
$this->warn($message);
Log::warning($message);
}
}