From 931fc55d8a6e6bf9b711b1bc6b64793e753237ec Mon Sep 17 00:00:00 2001 From: DariusIII Date: Fri, 14 Aug 2026 12:43:33 +0200 Subject: [PATCH] Update migrations --- .env.example | 9 + .../Commands/ReleasesNormalizeGuids.php | 198 ++++++++++++++++++ .../Commands/ReleasesOptimizePreflight.php | 120 ++++++++++- app/Models/ReleaseComment.php | 2 - .../Nzb/NzbCreationCandidateQuery.php | 26 ++- app/Services/ReleaseProcessingService.php | 15 +- config/nntmux.php | 17 ++ database/factories/ReleaseFactory.php | 4 +- ..._normalize_and_optimize_releases_table.php | 183 ++++++++++++++-- ...ize_releases_claim_tokens_and_pp_index.php | 129 ++++++++++++ .../views/admin/comments/index.blade.php | 10 - tests/Feature/NzbCreationReliabilityTest.php | 64 ++++++ .../Feature/ReleaseFactoryInvariantsTest.php | 102 +++++++++ ...ReleaseSchemaOptimizationMigrationTest.php | 187 +++++++++++++++++ tests/Feature/ReleasesNormalizeGuidsTest.php | 158 ++++++++++++++ .../Feature/ReleasesOptimizePreflightTest.php | 16 ++ ...SchemaOptimizationMigrationMariaDbTest.php | 39 ++++ 17 files changed, 1223 insertions(+), 56 deletions(-) create mode 100644 app/Console/Commands/ReleasesNormalizeGuids.php create mode 100644 database/migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php create mode 100644 tests/Feature/ReleaseFactoryInvariantsTest.php create mode 100644 tests/Feature/ReleaseSchemaOptimizationMigrationTest.php create mode 100644 tests/Feature/ReleasesNormalizeGuidsTest.php diff --git a/.env.example b/.env.example index 6a85ef929..37f4814e9 100644 --- a/.env.example +++ b/.env.example @@ -333,6 +333,15 @@ FORUM_FRONTEND_ENABLED=false STREAM_FORK_OUTPUT=false +# One-off releases table normalization migration (2026_08_13_001652). +# Skip the in-migration preflight only after `php artisan releases:optimize-preflight` passes, +# since it repeats four full scans of the largest table. +RELEASES_OPTIMIZE_SKIP_PREFLIGHT=false +# Bypass the free-space guard that aborts when the data volume cannot hold the rebuilt copy. +RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK=false +# Row batch size for the migration's comment recount and rollback backfills. +RELEASES_OPTIMIZE_CHUNK_SIZE=5000 + # API hot-path caching, asynchronous access metadata, and instrumentation. API_RELEASE_CACHE_TTL=600 API_RELEASE_CACHE_JITTER=60 diff --git a/app/Console/Commands/ReleasesNormalizeGuids.php b/app/Console/Commands/ReleasesNormalizeGuids.php new file mode 100644 index 000000000..5c7a9fa15 --- /dev/null +++ b/app/Console/Commands/ReleasesNormalizeGuids.php @@ -0,0 +1,198 @@ +error('The releases table does not exist.'); + + return self::FAILURE; + } + + $this->dryRun = (bool) $this->option('dry-run'); + if ($this->dryRun) { + $this->info('Dry run: no rows are modified.'); + } + + $leftGuids = $this->syncLeftGuids(); + $invalid = $this->collectInvalidGuidIds(); + $duplicates = $this->collectDuplicateGuidIds(); + + $this->table(['Issue', 'Releases'], [ + ['leftguid out of sync with guid', (string) $leftGuids], + ['guid is not a UUID', (string) count($invalid)], + ['guid duplicated case-insensitively', (string) count($duplicates)], + ]); + + $blocking = array_values(array_unique([...$invalid, ...$duplicates])); + if ($blocking === []) { + $this->info($this->dryRun + ? 'No guid problems found. Re-run without --dry-run to apply the leftguid sync.' + : 'Release guids are consistent.'); + + return self::SUCCESS; + } + + $this->reportBlockingReleases($blocking); + + return self::FAILURE; + } + + /** + * Print the releases that must be resolved by hand before the migration can run. + * + * @param list $ids + */ + private function reportBlockingReleases(array $ids): void + { + $this->error(count($ids).' releases have a guid that blocks the normalization migration.'); + $this->warn( + 'These are not repaired automatically. A guid determines the release NZB path and is published in ' + .'download links, so a new guid orphans the existing NZB file and breaks every link and download_stats ' + .'row that references the old value. Resolve them deliberately: delete the affected releases, or assign ' + .'new guids and relocate the matching NZB files yourself.' + ); + + $sample = array_slice($ids, 0, 20); + $rows = DB::table('releases') + ->select(['id', 'guid', 'name']) + ->whereIn('id', $sample) + ->orderBy('id') + ->get(); + + $this->table(['Release ID', 'Guid', 'Name'], $rows->map(static fn ($row): array => [ + (string) $row->id, + (string) $row->guid, + Str::limit((string) $row->name, 60), + ])->all()); + + if (count($ids) > count($sample)) { + $this->line('… and '.(count($ids) - count($sample)).' more.'); + } + } + + /** + * Point `leftguid` at the first character of `guid` wherever they disagree. + * Safe on its own: no file or index state depends on `leftguid`. + */ + private function syncLeftGuids(): int + { + $chunk = $this->chunkSize(); + $total = 0; + $lastId = 0; + + do { + $rows = DB::table('releases') + ->select(['id', 'guid']) + ->where('id', '>', $lastId) + ->whereRaw('LOWER(leftguid) <> LOWER(SUBSTR(guid, 1, 1))') + ->orderBy('id') + ->limit($chunk) + ->get(); + + foreach ($rows as $row) { + $lastId = max($lastId, (int) $row->id); + $total++; + if (! $this->dryRun) { + DB::table('releases') + ->where('id', (int) $row->id) + ->update(['leftguid' => substr((string) $row->guid, 0, 1)]); + } + } + + if ($rows->count() < $chunk) { + break; + } + } while (true); + + return $total; + } + + /** @return list */ + private function collectInvalidGuidIds(): array + { + $ids = []; + DB::table('releases')->select(['id', 'guid'])->orderBy('id')->chunk($this->chunkSize(), function ($rows) use (&$ids): void { + foreach ($rows as $row) { + if (! is_string($row->guid) || preg_match(self::UUID_PATTERN, $row->guid) !== 1) { + $ids[] = (int) $row->id; + } + } + }); + + return $ids; + } + + /** + * Every release that shares a case-insensitive guid with a lower-id release. + * The lowest id is treated as the owner of the guid and is not reported. + * + * @return list + */ + private function collectDuplicateGuidIds(): array + { + $duplicated = DB::table('releases') + ->selectRaw('LOWER(guid) AS normalized_guid') + ->groupByRaw('LOWER(guid)') + ->havingRaw('COUNT(*) > 1') + ->pluck('normalized_guid'); + + $ids = []; + foreach ($duplicated->chunk(500) as $batch) { + $rows = DB::table('releases') + ->select(['id', 'guid']) + ->whereIn(DB::raw('LOWER(guid)'), $batch->all()) + ->orderBy('id') + ->get(); + + $seen = []; + foreach ($rows as $row) { + $key = strtolower((string) $row->guid); + if (isset($seen[$key])) { + $ids[] = (int) $row->id; + + continue; + } + $seen[$key] = true; + } + } + + sort($ids); + + return $ids; + } + + private function chunkSize(): int + { + return max(100, min(10000, (int) $this->option('chunk'))); + } +} diff --git a/app/Console/Commands/ReleasesOptimizePreflight.php b/app/Console/Commands/ReleasesOptimizePreflight.php index 55a2688d7..66f684623 100644 --- a/app/Console/Commands/ReleasesOptimizePreflight.php +++ b/app/Console/Commands/ReleasesOptimizePreflight.php @@ -206,17 +206,22 @@ class ReleasesOptimizePreflight extends Command return 0; } - return DB::query()->fromSub( - DB::table('releases as r') - ->leftJoin('release_comments as rc', function ($join): void { - $join->on('rc.releases_id', '=', 'r.id')->where('rc.isvisible', 1); - }) - ->select('r.id', 'r.comments') - ->selectRaw('COUNT(rc.id) AS visible_comments') - ->groupBy('r.id', 'r.comments') - ->havingRaw('r.comments <> COUNT(rc.id)'), - 'comment_counter_mismatches', - )->count(); + // Raw SQL with explicit prefixed names: Laravel prefixes query aliases + // too, so `releases as r` would not match a raw `r.comments` reference. + $releases = $this->prefixedTable('releases'); + $comments = $this->prefixedTable('release_comments'); + + return count(DB::select( + "SELECT r.`id` FROM {$releases} r LEFT JOIN {$comments} c + ON c.`releases_id` = r.`id` AND c.`isvisible` = 1 + GROUP BY r.`id`, r.`comments` + HAVING r.`comments` <> COUNT(c.`id`)" + )); + } + + private function prefixedTable(string $name): string + { + return '`'.DB::getTablePrefix().$name.'`'; } private function countNonEmpty(string $table, string $column): int @@ -278,10 +283,103 @@ class ReleasesOptimizePreflight extends Command array_values($report['migration_data']), )); + $this->renderStorageSection($report); + $this->renderDiscardedDataSection($report); + $this->renderIndexSection($report); + if ($report['ok']) { $this->info('Releases optimization preflight passed.'); } else { $this->error('Releases optimization preflight failed. Resolve every blocker before migration.'); } } + + /** + * Print the on-disk footprint and the free space the rebuild needs. + * + * @param array $report + */ + private function renderStorageSection(array $report): void + { + $storage = $report['storage'] ?? []; + if (! is_array($storage) || ($storage['total_bytes'] ?? null) === null) { + $this->line('Storage: not reported for the '.$report['database_driver'].' driver.'); + + return; + } + + $this->table(['Storage', 'Size'], [ + ['Data', self::formatBytes((int) $storage['data_bytes'])], + ['Indexes', self::formatBytes((int) $storage['index_bytes'])], + ['Total', self::formatBytes((int) $storage['total_bytes'])], + ['Free space required', self::formatBytes((int) $storage['required_free_bytes'])], + ]); + $this->warn( + 'The rebuild copies the table, so at least '.self::formatBytes((int) $storage['required_free_bytes']) + .' must be free on the MySQL data volume before you migrate.' + ); + } + + /** + * Print how many values each dropped column is about to lose. + * + * @param array $report + */ + private function renderDiscardedDataSection(array $report): void + { + $discarded = $report['discarded_data'] ?? []; + if (! is_array($discarded) || $discarded === []) { + return; + } + + $rows = []; + foreach ($discarded as $column => $count) { + $rows[] = [$column, $count === null ? 'already dropped' : (string) $count]; + } + + $this->table(['Dropped column', 'Values destroyed'], $rows); + + $destroyed = array_sum(array_map(static fn (?int $count): int => $count ?? 0, $discarded)); + if ($destroyed > 0) { + $this->warn('These '.$destroyed.' values are permanently destroyed by the migration. Back up first.'); + } + } + + /** + * Print the index changes the migration will make. + * + * @param array $report + */ + private function renderIndexSection(array $report): void + { + $indexes = $report['indexes'] ?? []; + if (! is_array($indexes)) { + return; + } + + $removal = $indexes['scheduled_for_removal'] ?? []; + $existing = $indexes['already_present_replacements'] ?? []; + $missing = array_values(array_diff(self::NEW_INDEXES, $existing)); + + $this->table(['Index change', 'Names'], [ + ['Dropped', $removal === [] ? 'none' : implode(', ', $removal)], + ['Created', $missing === [] ? 'none' : implode(', ', $missing)], + ['Already present', $existing === [] ? 'none' : implode(', ', $existing)], + ]); + } + + private static function formatBytes(int $bytes): string + { + $units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + $value = (float) $bytes; + $unit = 0; + while ($value >= 1024.0 && $unit < count($units) - 1) { + $value /= 1024.0; + $unit++; + } + + return $unit === 0 + ? $bytes.' B' + : sprintf('%.2f %s', $value, $units[$unit]); + } } diff --git a/app/Models/ReleaseComment.php b/app/Models/ReleaseComment.php index 24d61258a..b9fb69fe7 100644 --- a/app/Models/ReleaseComment.php +++ b/app/Models/ReleaseComment.php @@ -17,7 +17,6 @@ use Illuminate\Support\Facades\DB; * @property int $releases_id FK to releases.id * @property string $text * @property bool $isvisible - * @property string $text_hash * @property string $username * @property int $users_id * @property Carbon|null $created_at @@ -32,7 +31,6 @@ use Illuminate\Support\Facades\DB; * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereIsvisible($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereReleasesId($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereText($value) - * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereTextHash($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUpdatedAt($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUsername($value) * @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUsersId($value) diff --git a/app/Services/Nzb/NzbCreationCandidateQuery.php b/app/Services/Nzb/NzbCreationCandidateQuery.php index 4ca88b95e..12bb06abb 100644 --- a/app/Services/Nzb/NzbCreationCandidateQuery.php +++ b/app/Services/Nzb/NzbCreationCandidateQuery.php @@ -17,6 +17,10 @@ final class NzbCreationCandidateQuery public const string CLAIM_TOKEN_COLUMN = 'nzb_creation_claim_token'; + private static ?bool $supportsClaims = null; + + private static ?bool $supportsFailureState = null; + /** * @return Builder */ @@ -112,17 +116,31 @@ final class NzbCreationCandidateQuery public static function supportsClaims(): bool { - if (! Schema::hasTable('releases')) { - return false; + if (self::$supportsClaims !== null) { + return self::$supportsClaims; } - return Schema::hasColumn('releases', self::CLAIMED_AT_COLUMN) + if (! Schema::hasTable('releases')) { + return self::$supportsClaims = false; + } + + return self::$supportsClaims = Schema::hasColumn('releases', self::CLAIMED_AT_COLUMN) && Schema::hasColumn('releases', self::CLAIM_TOKEN_COLUMN); } public static function supportsFailureState(): bool { - return Schema::hasTable('release_nzb_creation_failures'); + return self::$supportsFailureState ??= Schema::hasTable('release_nzb_creation_failures'); + } + + /** + * Discard the memoized schema capability flags. Only needed when the schema + * changes inside a single process, such as between tests. + */ + public static function flushCapabilityCache(): void + { + self::$supportsClaims = null; + self::$supportsFailureState = null; } /** diff --git a/app/Services/ReleaseProcessingService.php b/app/Services/ReleaseProcessingService.php index 470f57e48..adbcd618b 100644 --- a/app/Services/ReleaseProcessingService.php +++ b/app/Services/ReleaseProcessingService.php @@ -754,6 +754,10 @@ final class ReleaseProcessingService private function nextNzbCreationAttempt(Release $release): int { + if (! $release->relationLoaded('nzbCreationFailure') && NzbCreationCandidateQuery::supportsFailureState()) { + $release->loadMissing('nzbCreationFailure'); + } + $failureState = $release->relationLoaded('nzbCreationFailure') ? $release->getRelation('nzbCreationFailure') : null; @@ -764,16 +768,13 @@ final class ReleaseProcessingService private function recordNzbCreationRetry(Release $release, NzbCreationResult $result): void { $nextAttempt = $this->nextNzbCreationAttempt($release); - ReleaseNzbCreationFailure::query()->upsert( - [[ - 'releases_id' => (int) $release->id, + $failure = ReleaseNzbCreationFailure::query()->updateOrCreate( + ['releases_id' => (int) $release->id], + [ 'attempts' => $nextAttempt, 'last_error' => mb_substr($result->reason, 0, 1000), - ]], - ['releases_id'], - ['attempts', 'last_error'], + ], ); - $failure = ReleaseNzbCreationFailure::query()->findOrFail((int) $release->id); $release->setRelation('nzbCreationFailure', $failure); Log::channel('nzb_creation')->warning('NZB creation failed; release will be retried', [ diff --git a/config/nntmux.php b/config/nntmux.php index 4f955dec5..0a2dd46bf 100644 --- a/config/nntmux.php +++ b/config/nntmux.php @@ -81,4 +81,21 @@ return [ 'categorization' => [ 'log' => (bool) env('NNTMUX_CATEGORIZATION_LOG', false), ], + + /* + |-------------------------------------------------------------------------- + | Releases table normalization migration + |-------------------------------------------------------------------------- + | + | Knobs for the one-off releases rebuild. Skip the in-migration preflight + | only after `php artisan releases:optimize-preflight` has passed, since the + | checks are four full scans of the largest table. The free-space guard + | aborts before a COPY-algorithm ALTER that would fill the data volume. + | + */ + 'releases_optimize' => [ + 'skip_preflight' => (bool) env('RELEASES_OPTIMIZE_SKIP_PREFLIGHT', false), + 'skip_free_space_check' => (bool) env('RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK', false), + 'chunk_size' => (int) env('RELEASES_OPTIMIZE_CHUNK_SIZE', 5000), + ], ]; diff --git a/database/factories/ReleaseFactory.php b/database/factories/ReleaseFactory.php index 1dc017cb5..15eccc5cc 100644 --- a/database/factories/ReleaseFactory.php +++ b/database/factories/ReleaseFactory.php @@ -3,6 +3,7 @@ namespace Database\Factories; use Illuminate\Database\Eloquent\Factories\Factory; +use Illuminate\Support\Str; class ReleaseFactory extends Factory { @@ -17,7 +18,8 @@ class ReleaseFactory extends Factory 'fromname' => $this->faker->unique()->safeEmail(), 'postdate' => $this->faker->date(), 'adddate' => $this->faker->date(), - 'guid' => $this->faker->sha1(), + 'guid' => Str::uuid()->toString(), + 'leftguid' => fn (array $attributes): string => $attributes['guid'][0], 'categories_id' => '2080', 'nzbstatus' => 1, 'passwordstatus' => 0, diff --git a/database/migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php b/database/migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php index da203693a..2d147f409 100644 --- a/database/migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php +++ b/database/migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php @@ -4,6 +4,7 @@ declare(strict_types=1); use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; @@ -56,6 +57,7 @@ return new class extends Migration } $this->assertReleaseIdentifiersAreSafe(); + $this->assertEnoughFreeSpaceForRebuild(); $this->createSparseTables(); $this->backfillSparseTables(); $this->recountVisibleComments(); @@ -226,13 +228,40 @@ return new class extends Migration return; } - DB::statement( - 'UPDATE '.$this->table('releases').' SET `comments` = (SELECT COUNT(*) FROM ' - .$this->table('release_comments').' WHERE ' - .$this->table('release_comments').'.`releases_id` = ' - .$this->table('releases').'.`id` AND ' - .$this->table('release_comments').'.`isvisible` = 1)' - ); + $chunkSize = $this->chunkSize(); + $releases = $this->table('releases'); + $comments = $this->table('release_comments'); + $lastId = 0; + + do { + // Raw SQL with explicit prefixed names: Laravel also prefixes query + // aliases, so `releases as r` would not match a raw `r.comments`. + $mismatches = DB::select( + "SELECT r.`id` AS id, COUNT(c.`id`) AS visible_comments + FROM {$releases} r LEFT JOIN {$comments} c + ON c.`releases_id` = r.`id` AND c.`isvisible` = 1 + WHERE r.`id` > ? + GROUP BY r.`id`, r.`comments` + HAVING r.`comments` <> COUNT(c.`id`) + ORDER BY r.`id` + LIMIT {$chunkSize}", + [$lastId], + ); + + if ($mismatches === []) { + return; + } + + $idsByCount = []; + foreach ($mismatches as $mismatch) { + $idsByCount[(int) $mismatch->visible_comments][] = (int) $mismatch->id; + $lastId = max($lastId, (int) $mismatch->id); + } + + foreach ($idsByCount as $count => $ids) { + DB::table('releases')->whereIn('id', $ids)->update(['comments' => $count]); + } + } while (count($mismatches) === $chunkSize); } private function rebuildReleasesForMariaDb(): void @@ -367,33 +396,142 @@ return new class extends Migration $table->boolean('proc_sorter')->default(false); $table->boolean('audiostatus')->default(false); $table->index('guid', 'ix_releases_guid'); + $table->index('adddate', 'ix_releases_adddate_only'); + $table->index('videos_id', 'ix_releases_videos_id'); + $table->index('movieinfo_id', 'ix_releases_movieinfo_id'); + $table->index('imdbid', 'ix_releases_imdbid'); + $table->index( + ['passwordstatus', 'categories_id', 'postdate', 'videos_id', 'tv_episodes_id', 'groups_id'], + 'ix_releases_tv_search_covering', + ); + $table->index('passwordstatus', 'ix_releases_passwordstatus'); + $table->index(['haspreview', 'passwordstatus'], 'ix_releases_haspreview_passwordstatus'); + $table->index(['postdate', 'searchname'], 'ix_releases_postdate_searchname'); + $table->index(['predb_id', 'searchname'], 'ix_releases_predb_id_searchname'); + $table->index(['size', 'categories_id', 'passwordstatus'], 'ix_releases_size_cat'); + $table->index( + ['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'additional_pp_claimed_at', 'postdate'], + 'ix_releases_add_pp_claim_queue', + ); + $table->index( + ['nzbstatus', 'groups_id', 'leftguid', 'nzb_creation_claimed_at', 'postdate'], + 'ix_releases_nzb_creation_queue', + ); }); } private function restoreSparseDataToReleases(): void { + $chunkSize = $this->chunkSize(); + if (Schema::hasTable('release_nzb_passwords')) { - DB::table('release_nzb_passwords')->orderBy('releases_id')->chunkById(1000, function ($rows): void { - foreach ($rows as $row) { - DB::table('releases')->where('id', $row->releases_id)->update(['nzb_password' => $row->password]); - } - }, 'releases_id'); + DB::table('release_nzb_passwords')->orderBy('releases_id')->chunkById( + $chunkSize, + fn ($rows) => $this->batchUpdateReleases($rows, ['nzb_password' => 'password']), + 'releases_id', + ); } if (Schema::hasTable('release_nzb_creation_failures')) { - DB::table('release_nzb_creation_failures')->orderBy('releases_id')->chunkById(1000, function ($rows): void { - foreach ($rows as $row) { - DB::table('releases')->where('id', $row->releases_id)->update([ - 'nzb_creation_attempts' => $row->attempts, - 'nzb_creation_last_error' => $row->last_error, - ]); - } - }, 'releases_id'); + DB::table('release_nzb_creation_failures')->orderBy('releases_id')->chunkById( + $chunkSize, + fn ($rows) => $this->batchUpdateReleases($rows, [ + 'nzb_creation_attempts' => 'attempts', + 'nzb_creation_last_error' => 'last_error', + ]), + 'releases_id', + ); } } + /** + * Push one chunk of sparse rows back into `releases` with a single + * `CASE`-driven statement instead of one `UPDATE` per row. + * + * @param Collection $rows + * @param array $columnMap Target `releases` column => source column + */ + private function batchUpdateReleases($rows, array $columnMap): void + { + if ($rows->isEmpty()) { + return; + } + + $setClauses = []; + $bindings = []; + foreach ($columnMap as $target => $source) { + $clause = '`'.$target.'` = CASE `id`'; + foreach ($rows as $row) { + $clause .= ' WHEN ? THEN ?'; + $bindings[] = (int) $row->releases_id; + $bindings[] = $row->{$source}; + } + $setClauses[] = $clause.' ELSE `'.$target.'` END'; + } + + $ids = []; + foreach ($rows as $row) { + $ids[] = (int) $row->releases_id; + } + + DB::update( + 'UPDATE '.$this->table('releases').' SET '.implode(', ', $setClauses) + .' WHERE `id` IN ('.implode(', ', array_fill(0, count($ids), '?')).')', + [...$bindings, ...$ids], + ); + } + + /** + * Abort before the COPY-algorithm rebuild when the data volume cannot hold + * a second copy of the table. Silently skipped when the server's datadir is + * not visible to PHP (remote or containerised MySQL). + */ + private function assertEnoughFreeSpaceForRebuild(): void + { + if (! $this->isMariaDbOrMySql() || (bool) config('nntmux.releases_optimize.skip_free_space_check', false)) { + return; + } + + $sizes = DB::selectOne( + 'SELECT DATA_LENGTH + INDEX_LENGTH AS total_bytes FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [DB::getTablePrefix().'releases'], + ); + $required = (int) ($sizes->total_bytes ?? 0) * 2; + if ($required <= 0) { + return; + } + + $dataDir = (string) (DB::selectOne('SELECT @@datadir AS dir')->dir ?? ''); + if ($dataDir === '' || ! is_dir($dataDir) || ! is_readable($dataDir)) { + return; + } + + $free = @disk_free_space($dataDir); + if ($free === false || $free >= $required) { + return; + } + + throw new RuntimeException(sprintf( + 'Releases rebuild needs about %d bytes free in %s but only %d are available. ' + .'Free up space, use gh-ost/pt-online-schema-change, or set RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK=true to override.', + $required, + $dataDir, + (int) $free, + )); + } + + private function chunkSize(): int + { + return max(100, min(10000, (int) config('nntmux.releases_optimize.chunk_size', 5000))); + } + private function assertReleaseIdentifiersAreSafe(): void { + if ((bool) config('nntmux.releases_optimize.skip_preflight', false)) { + return; + } + $duplicateIds = DB::query()->fromSub( DB::table('releases')->select('id')->groupBy('id')->havingRaw('COUNT(*) > 1'), 'duplicate_release_ids', @@ -427,7 +565,10 @@ return new class extends Migration ]); if ($blockers !== []) { - throw new RuntimeException('Releases optimization preflight failed: '.json_encode($blockers, JSON_THROW_ON_ERROR)); + throw new RuntimeException( + 'Releases optimization preflight failed: '.json_encode($blockers, JSON_THROW_ON_ERROR) + .'. Run `php artisan releases:normalize-guids --dry-run` to see how to resolve them.' + ); } } diff --git a/database/migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php b/database/migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php new file mode 100644 index 000000000..48167385d --- /dev/null +++ b/database/migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php @@ -0,0 +1,129 @@ + min AND size < max` + * ({@see AdditionalCandidateQuery::applyPredicates()}), + * but `size` was not in the index, so each candidate that later failed the + * size filter still cost a primary key lookup. Appending it after the + * existing columns keeps the equality prefix and the `postdate DESC` + * ordering intact while enabling index condition pushdown. + * 2. Narrows both claim tokens to `CHAR(32) ascii`. They only ever hold + * `bin2hex(random_bytes(16))`, so `VARCHAR(64)` utf8mb4 reserved four bytes + * per character for hex digits. + * + * `releases.imdbid` is deliberately left alone: it joins `movieinfo.imdbid`, + * and changing the charset on one side only would force an implicit conversion + * that disables index usage on that join. + */ +return new class extends Migration +{ + private const string PP_CLAIM_INDEX = 'ix_releases_add_pp_claim_queue'; + + /** @var list */ + private const array CLAIM_TOKEN_COLUMNS = [ + 'nzb_creation_claim_token', + 'additional_pp_claim_token', + ]; + + public function up(): void + { + if (! Schema::hasTable('releases')) { + return; + } + + if ($this->isMariaDbOrMySql()) { + $specifications = []; + if ($this->indexExists(self::PP_CLAIM_INDEX)) { + $specifications[] = 'DROP INDEX `'.self::PP_CLAIM_INDEX.'`'; + } + $specifications[] = 'ADD INDEX `'.self::PP_CLAIM_INDEX.'` (`passwordstatus`, `haspreview`, `nzbstatus`,' + .' `leftguid`, `postdate` DESC, `id`, `additional_pp_claimed_at`, `size`)'; + foreach (self::CLAIM_TOKEN_COLUMNS as $column) { + if (Schema::hasColumn('releases', $column)) { + $specifications[] = 'MODIFY `'.$column.'` CHAR(32) CHARACTER SET ascii COLLATE ascii_general_ci NULL'; + } + } + + DB::statement('ALTER TABLE '.$this->table('releases').' '.implode(', ', $specifications)); + + return; + } + + Schema::table('releases', function (Blueprint $table): void { + if ($this->indexExists(self::PP_CLAIM_INDEX)) { + $table->dropIndex(self::PP_CLAIM_INDEX); + } + $table->index( + ['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at', 'size'], + self::PP_CLAIM_INDEX, + ); + }); + } + + public function down(): void + { + if (! Schema::hasTable('releases')) { + return; + } + + if ($this->isMariaDbOrMySql()) { + $specifications = []; + if ($this->indexExists(self::PP_CLAIM_INDEX)) { + $specifications[] = 'DROP INDEX `'.self::PP_CLAIM_INDEX.'`'; + } + $specifications[] = 'ADD INDEX `'.self::PP_CLAIM_INDEX.'` (`passwordstatus`, `haspreview`, `nzbstatus`,' + .' `leftguid`, `postdate` DESC, `id`, `additional_pp_claimed_at`)'; + foreach (self::CLAIM_TOKEN_COLUMNS as $column) { + if (Schema::hasColumn('releases', $column)) { + $specifications[] = 'MODIFY `'.$column.'` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL'; + } + } + + DB::statement('ALTER TABLE '.$this->table('releases').' '.implode(', ', $specifications)); + + return; + } + + Schema::table('releases', function (Blueprint $table): void { + if ($this->indexExists(self::PP_CLAIM_INDEX)) { + $table->dropIndex(self::PP_CLAIM_INDEX); + } + $table->index( + ['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at'], + self::PP_CLAIM_INDEX, + ); + }); + } + + private function indexExists(string $name): bool + { + foreach (Schema::getIndexes('releases') as $index) { + if (($index['name'] ?? null) === $name) { + return true; + } + } + + return false; + } + + private function isMariaDbOrMySql(): bool + { + return in_array(DB::getDriverName(), ['mariadb', 'mysql'], true); + } + + private function table(string $name): string + { + return '`'.DB::getTablePrefix().$name.'`'; + } +}; diff --git a/resources/views/admin/comments/index.blade.php b/resources/views/admin/comments/index.blade.php index e247e9ccc..8c942b0f5 100644 --- a/resources/views/admin/comments/index.blade.php +++ b/resources/views/admin/comments/index.blade.php @@ -70,16 +70,6 @@ Hidden @endif - @if($comment->shared) - - Shared - - @endif - @if($comment->issynced) - - Synced - - @endif diff --git a/tests/Feature/NzbCreationReliabilityTest.php b/tests/Feature/NzbCreationReliabilityTest.php index 367315089..d5037a6a2 100644 --- a/tests/Feature/NzbCreationReliabilityTest.php +++ b/tests/Feature/NzbCreationReliabilityTest.php @@ -78,10 +78,12 @@ class NzbCreationReliabilityTest extends TestCase $this->createSchema(); $this->seedSettings(); + NzbCreationCandidateQuery::flushCapabilityCache(); } protected function tearDown(): void { + NzbCreationCandidateQuery::flushCapabilityCache(); $this->deleteDirectory($this->tempNzbPath); if ($this->databasePath !== '' && file_exists($this->databasePath)) { unlink($this->databasePath); @@ -143,6 +145,8 @@ class NzbCreationReliabilityTest extends TestCase $this->assertSame(1, DB::table('releases')->count()); $this->assertSame(1, (int) DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('attempts')); $this->assertSame('Temporary filesystem failure.', DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('last_error')); + $this->assertNotNull(DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('created_at')); + $this->assertNotNull(DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('updated_at')); $this->assertNull(DB::table('releases')->where('id', 1)->value('nzb_creation_claimed_at')); $this->assertSame(1, DB::table('collections')->count()); $this->assertSame('NZB creation failed; release will be retried', $nzbCreationLogger->warnings[0]['message']); @@ -185,6 +189,56 @@ class NzbCreationReliabilityTest extends TestCase $this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['max_attempts']); } + public function test_repeated_transient_failure_increments_attempts_and_touches_updated_at(): void + { + Log::partialMock(); + Log::shouldReceive('channel')->once()->with('nzb_creation')->andReturn(new RecordingNzbCreationLogger); + + $this->insertRelease(1, 'a'); + $this->insertCbp(200, 2000, 1); + DB::table('release_nzb_creation_failures')->insert([ + 'releases_id' => 1, + 'attempts' => 1, + 'last_error' => 'Earlier failure.', + 'created_at' => '2026-01-01 00:00:00', + 'updated_at' => '2026-01-01 00:00:00', + ]); + + $service = $this->releaseProcessingService( + NzbCreationResult::transient('Another filesystem failure.', [200]) + ); + + $this->assertSame(0, $service->createNZBs(null)); + + $failure = DB::table('release_nzb_creation_failures')->where('releases_id', 1)->first(); + + $this->assertSame(2, (int) $failure->attempts); + $this->assertSame('Another filesystem failure.', $failure->last_error); + $this->assertSame('2026-01-01 00:00:00', $failure->created_at); + $this->assertNotSame('2026-01-01 00:00:00', $failure->updated_at); + } + + public function test_attempt_cap_holds_when_the_failure_relation_was_dropped(): void + { + Log::partialMock(); + $nzbCreationLogger = new RecordingNzbCreationLogger; + Log::shouldReceive('channel')->once()->with('nzb_creation')->andReturn($nzbCreationLogger); + + $this->insertRelease(1, 'a', attempts: 2); + $this->insertCbp(200, 2000, 1); + + $service = (new ReleaseProcessingService( + nzb: new RelationDroppingNzbService(NzbCreationResult::transient('Repeated filesystem failure.', [200])), + releaseManagement: new DatabaseOnlyReleaseManagementService, + collectionCleanupService: app(CollectionCleanupService::class), + ))->setEchoCLI(false); + + $this->assertSame(0, $service->createNZBs(null)); + $this->assertSame(0, DB::table('releases')->count()); + $this->assertSame('Deleting release after NZB creation failure', $nzbCreationLogger->warnings[0]['message']); + $this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['attempt']); + } + public function test_writer_classifies_final_rename_failure_as_transient_without_final_file(): void { $this->insertRelease(1, 'd'); @@ -458,6 +512,16 @@ class FakeNzbCreationService extends NzbService } } +class RelationDroppingNzbService extends FakeNzbCreationService +{ + public function createNzbForRelease(Release $release): NzbCreationResult + { + $release->unsetRelation('nzbCreationFailure'); + + return parent::createNzbForRelease($release); + } +} + class DatabaseOnlyReleaseManagementService extends ReleaseManagementService { /** diff --git a/tests/Feature/ReleaseFactoryInvariantsTest.php b/tests/Feature/ReleaseFactoryInvariantsTest.php new file mode 100644 index 000000000..57d61dff1 --- /dev/null +++ b/tests/Feature/ReleaseFactoryInvariantsTest.php @@ -0,0 +1,102 @@ +databasePath = sys_get_temp_dir().'/nntmux-release-factory.sqlite'; + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')"); + putenv('APP_ENV=testing'); + putenv('DB_CONNECTION=sqlite'); + putenv('DB_DATABASE='.$this->databasePath); + $_ENV['APP_ENV'] = $_SERVER['APP_ENV'] = 'testing'; + $_ENV['DB_CONNECTION'] = $_SERVER['DB_CONNECTION'] = 'sqlite'; + $_ENV['DB_DATABASE'] = $_SERVER['DB_DATABASE'] = $this->databasePath; + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + config(['database.default' => 'sqlite', 'database.connections.sqlite.database' => $this->databasePath]); + DB::purge(); + DB::reconnect(); + DB::statement('CREATE TABLE releases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL DEFAULT "", + searchname VARCHAR(255) NOT NULL DEFAULT "", + fromname VARCHAR(255) NULL, + postdate DATETIME NULL, + adddate DATETIME NULL, + guid CHAR(36) NOT NULL UNIQUE, + leftguid CHAR(1) NOT NULL, + categories_id INTEGER NOT NULL DEFAULT 10, + nzbstatus INTEGER NOT NULL DEFAULT 0, + passwordstatus INTEGER NOT NULL DEFAULT -1, + isrenamed INTEGER NOT NULL DEFAULT 0 + )'); + } + + protected function tearDown(): void + { + DB::disconnect(); + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + parent::tearDown(); + } + + public function test_factory_produces_uuid_guid_with_matching_leftguid(): void + { + for ($iteration = 0; $iteration < 5; $iteration++) { + $attributes = Release::factory()->raw(); + + $this->assertMatchesRegularExpression( + '/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iD', + (string) $attributes['guid'], + 'The factory must produce a UUID guid that fits the char(36) column.', + ); + $this->assertSame(36, mb_strlen((string) $attributes['guid'])); + $this->assertSame( + mb_strtolower(mb_substr((string) $attributes['guid'], 0, 1)), + mb_strtolower((string) $attributes['leftguid']), + 'leftguid must be the first character of guid.', + ); + } + } + + public function test_factory_rows_satisfy_the_release_table_constraints(): void + { + foreach (Release::factory()->count(3)->raw() as $attributes) { + DB::table('releases')->insert($attributes); + } + + $rows = DB::table('releases')->get(['guid', 'leftguid']); + + $this->assertCount(3, $rows); + foreach ($rows as $row) { + $this->assertSame(mb_substr((string) $row->guid, 0, 1), (string) $row->leftguid); + } + $this->assertSame(3, $rows->pluck('guid')->unique()->count()); + } +} diff --git a/tests/Feature/ReleaseSchemaOptimizationMigrationTest.php b/tests/Feature/ReleaseSchemaOptimizationMigrationTest.php new file mode 100644 index 000000000..3386d8e74 --- /dev/null +++ b/tests/Feature/ReleaseSchemaOptimizationMigrationTest.php @@ -0,0 +1,187 @@ +databasePath = sys_get_temp_dir().'/nntmux-releases-migration.sqlite'; + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')"); + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $this->databasePath, + 'nntmux.releases_optimize.chunk_size' => 100, + 'nntmux.releases_optimize.skip_preflight' => false, + ]); + DB::purge(); + DB::reconnect(); + $this->createSchema(); + } + + protected function tearDown(): void + { + DB::disconnect(); + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + parent::tearDown(); + } + + public function test_comment_recount_spans_multiple_chunks_and_skips_correct_rows(): void + { + $releases = []; + $comments = []; + $commentId = 1; + for ($id = 1; $id <= 250; $id++) { + // Every third release already stores the right counter and must not be rewritten. + $alreadyCorrect = $id % 3 === 0; + $releases[] = ['id' => $id, 'guid' => 'g'.$id, 'leftguid' => 'g', 'comments' => $alreadyCorrect ? 1 : 99]; + $comments[] = ['id' => $commentId++, 'releases_id' => $id, 'isvisible' => 1]; + $comments[] = ['id' => $commentId++, 'releases_id' => $id, 'isvisible' => 0]; + } + DB::table('releases')->insert($releases); + DB::table('release_comments')->insert($comments); + + $this->invoke('recountVisibleComments'); + + $this->assertSame(0, DB::table('releases')->where('comments', '<>', 1)->count()); + $this->assertSame(250, DB::table('releases')->where('comments', 1)->count()); + } + + public function test_comment_recount_zeroes_releases_without_visible_comments(): void + { + DB::table('releases')->insert([ + ['id' => 1, 'guid' => 'a', 'leftguid' => 'a', 'comments' => 7], + ['id' => 2, 'guid' => 'b', 'leftguid' => 'b', 'comments' => 0], + ]); + DB::table('release_comments')->insert([['id' => 1, 'releases_id' => 1, 'isvisible' => 0]]); + + $this->invoke('recountVisibleComments'); + + $this->assertSame(0, (int) DB::table('releases')->where('id', 1)->value('comments')); + $this->assertSame(0, (int) DB::table('releases')->where('id', 2)->value('comments')); + } + + public function test_rollback_backfill_restores_sparse_rows_in_batches(): void + { + $releases = []; + $passwords = []; + $failures = []; + for ($id = 1; $id <= 250; $id++) { + $releases[] = ['id' => $id, 'guid' => 'g'.$id, 'leftguid' => 'g', 'comments' => 0]; + $passwords[] = ['releases_id' => $id, 'password' => 'secret-'.$id]; + $failures[] = ['releases_id' => $id, 'attempts' => $id % 5, 'last_error' => 'boom-'.$id]; + } + DB::table('releases')->insert($releases); + DB::table('release_nzb_passwords')->insert($passwords); + DB::table('release_nzb_creation_failures')->insert($failures); + + $this->invoke('restoreSparseDataToReleases'); + + $restored = DB::table('releases')->orderBy('id')->get(); + $this->assertCount(250, $restored); + foreach ($restored as $release) { + $this->assertSame('secret-'.$release->id, $release->nzb_password); + $this->assertSame((int) $release->id % 5, (int) $release->nzb_creation_attempts); + $this->assertSame('boom-'.$release->id, $release->nzb_creation_last_error); + } + } + + public function test_preflight_blocks_bad_identifiers_and_the_opt_out_bypasses_it(): void + { + DB::table('releases')->insert([ + ['id' => 1, 'guid' => 'not-a-uuid', 'leftguid' => 'z', 'comments' => 0], + ]); + + try { + $this->invoke('assertReleaseIdentifiersAreSafe'); + $this->fail('The migration accepted an invalid guid.'); + } catch (RuntimeException $e) { + $this->assertStringContainsString('invalid release GUIDs', $e->getMessage()); + $this->assertStringContainsString('releases:normalize-guids', $e->getMessage()); + } + + config(['nntmux.releases_optimize.skip_preflight' => true]); + $this->invoke('assertReleaseIdentifiersAreSafe'); + $this->assertSame(1, DB::table('releases')->count()); + } + + private function invoke(string $method): void + { + /** @var Migration $migration */ + $migration = require database_path('migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php'); + (new ReflectionMethod($migration, $method))->invoke($migration); + } + + private function createSchema(): void + { + foreach (['release_comments', 'release_nzb_passwords', 'release_nzb_creation_failures', 'releases'] as $table) { + DB::statement('DROP TABLE IF EXISTS '.$table); + } + DB::statement('CREATE TABLE releases ( + id INTEGER PRIMARY KEY, guid VARCHAR(40) NOT NULL, leftguid CHAR(1) NOT NULL, + comments INTEGER NOT NULL DEFAULT 0, nzb_password VARCHAR(255) NULL, + nzb_creation_attempts INTEGER NOT NULL DEFAULT 0, nzb_creation_last_error TEXT NULL + )'); + DB::statement('CREATE TABLE release_comments ( + id INTEGER PRIMARY KEY, releases_id INTEGER NOT NULL, isvisible INTEGER NOT NULL DEFAULT 1 + )'); + DB::statement('CREATE TABLE release_nzb_passwords ( + releases_id INTEGER PRIMARY KEY, password VARCHAR(255) NOT NULL + )'); + DB::statement('CREATE TABLE release_nzb_creation_failures ( + releases_id INTEGER PRIMARY KEY, attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT NULL + )'); + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv("{$key}={$value}"); + $_ENV[$key] = $_SERVER[$key] = $value; + } +} diff --git a/tests/Feature/ReleasesNormalizeGuidsTest.php b/tests/Feature/ReleasesNormalizeGuidsTest.php new file mode 100644 index 000000000..65ffd05c6 --- /dev/null +++ b/tests/Feature/ReleasesNormalizeGuidsTest.php @@ -0,0 +1,158 @@ +databasePath = sys_get_temp_dir().'/nntmux-normalize-guids.sqlite'; + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')"); + $this->setEnvironmentValue('APP_ENV', 'testing'); + $this->setEnvironmentValue('DB_CONNECTION', 'sqlite'); + $this->setEnvironmentValue('DB_DATABASE', $this->databasePath); + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + config([ + 'database.default' => 'sqlite', + 'database.connections.sqlite.database' => $this->databasePath, + ]); + DB::purge(); + DB::reconnect(); + $this->createSchema(); + } + + protected function tearDown(): void + { + DB::disconnect(); + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + parent::tearDown(); + } + + public function test_leftguid_is_resynced_in_place(): void + { + $guid = '01234567-89ab-cdef-0123-456789abcdef'; + $this->insertRelease(1, $guid, 'f'); + + $status = Artisan::call('releases:normalize-guids'); + + $this->assertSame(0, $status); + $this->assertSame('0', DB::table('releases')->where('id', 1)->value('leftguid')); + $this->assertSame($guid, DB::table('releases')->where('id', 1)->value('guid')); + } + + public function test_dry_run_reports_without_writing(): void + { + $this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', 'f'); + $this->insertRelease(2, 'd41d8cd98f00b204e9800998ecf8427e', 'd'); + + $status = Artisan::call('releases:normalize-guids', ['--dry-run' => true]); + $output = Artisan::output(); + + $this->assertSame(1, $status); + $this->assertStringContainsString('Dry run: no rows are modified.', $output); + $this->assertSame('f', DB::table('releases')->where('id', 1)->value('leftguid')); + $this->assertSame('d41d8cd98f00b204e9800998ecf8427e', DB::table('releases')->where('id', 2)->value('guid')); + } + + public function test_invalid_guid_is_reported_and_never_rewritten(): void + { + $this->insertRelease(1, 'd41d8cd98f00b204e9800998ecf8427e', 'd', ['name' => 'Legacy nZEDb release']); + + $status = Artisan::call('releases:normalize-guids'); + $output = Artisan::output(); + + $this->assertSame(1, $status); + $this->assertStringContainsString('1 releases have a guid that blocks the normalization migration.', $output); + $this->assertStringContainsString('d41d8cd98f00b204e9800998ecf8427e', $output); + $this->assertStringContainsString('Legacy nZEDb release', $output); + $this->assertSame('d41d8cd98f00b204e9800998ecf8427e', DB::table('releases')->where('id', 1)->value('guid')); + } + + public function test_case_insensitive_duplicates_report_every_id_but_the_lowest(): void + { + $guid = 'abcdef01-2345-6789-abcd-ef0123456789'; + $this->insertRelease(1, $guid, 'a'); + $this->insertRelease(2, strtoupper($guid), 'A'); + + $status = Artisan::call('releases:normalize-guids'); + $output = Artisan::output(); + + $this->assertSame(1, $status); + $this->assertStringContainsString('1 releases have a guid that blocks', $output); + $this->assertSame($guid, DB::table('releases')->where('id', 1)->value('guid')); + $this->assertSame(strtoupper($guid), DB::table('releases')->where('id', 2)->value('guid')); + } + + public function test_consistent_guids_report_success(): void + { + $this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', '0'); + + $status = Artisan::call('releases:normalize-guids'); + + $this->assertSame(0, $status); + $this->assertStringContainsString('Release guids are consistent.', Artisan::output()); + } + + /** @param array $overrides */ + private function insertRelease(int $id, string $guid, string $leftguid, array $overrides = []): void + { + DB::table('releases')->insert([ + 'id' => $id, + 'guid' => $guid, + 'leftguid' => $leftguid, + 'name' => 'Release '.$id, + ...$overrides, + ]); + } + + private function createSchema(): void + { + DB::statement('DROP TABLE IF EXISTS releases'); + DB::statement('CREATE TABLE releases ( + id INTEGER PRIMARY KEY, + guid VARCHAR(40) NOT NULL, + leftguid CHAR(1) NOT NULL, + name VARCHAR(255) NOT NULL DEFAULT "" + )'); + } + + private function setEnvironmentValue(string $key, ?string $value): void + { + if ($value === null) { + putenv($key); + unset($_ENV[$key], $_SERVER[$key]); + + return; + } + + putenv("{$key}={$value}"); + $_ENV[$key] = $_SERVER[$key] = $value; + } +} diff --git a/tests/Feature/ReleasesOptimizePreflightTest.php b/tests/Feature/ReleasesOptimizePreflightTest.php index dcc5e23b1..031137c0d 100644 --- a/tests/Feature/ReleasesOptimizePreflightTest.php +++ b/tests/Feature/ReleasesOptimizePreflightTest.php @@ -106,6 +106,22 @@ final class ReleasesOptimizePreflightTest extends TestCase } } + public function test_human_report_includes_storage_discarded_data_and_index_sections(): void + { + $this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', '0', ['source' => 2]); + + $status = Artisan::call('releases:optimize-preflight'); + $output = Artisan::output(); + + $this->assertSame(0, $status); + $this->assertStringContainsString('Storage: not reported for the sqlite driver.', $output); + $this->assertStringContainsString('Dropped column', $output); + $this->assertStringContainsString('releases.source', $output); + $this->assertStringContainsString('permanently destroyed', $output); + $this->assertStringContainsString('Index change', $output); + $this->assertStringContainsString('ux_releases_guid', $output); + } + /** @param array $overrides */ private function insertRelease(int $id, string $guid, string $leftguid, array $overrides = []): void { diff --git a/tests/Integration/ReleaseSchemaOptimizationMigrationMariaDbTest.php b/tests/Integration/ReleaseSchemaOptimizationMigrationMariaDbTest.php index 796f21261..516f8f439 100644 --- a/tests/Integration/ReleaseSchemaOptimizationMigrationMariaDbTest.php +++ b/tests/Integration/ReleaseSchemaOptimizationMigrationMariaDbTest.php @@ -113,6 +113,37 @@ final class ReleaseSchemaOptimizationMigrationMariaDbTest extends TestCase $this->assertDatabaseHas('release_files', ['releases_id' => 1, 'name' => 'kept.nzb']); } + #[Test] + public function claim_token_and_pp_index_follow_up_narrows_columns_and_adds_size(): void + { + $this->seedLegacyData(); + $this->migration()->up(); + $followUp = $this->followUpMigration(); + + $followUp->up(); + + $this->assertSame( + ['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at', 'size'], + $this->indexColumns('ix_releases_add_pp_claim_queue'), + ); + foreach (['nzb_creation_claim_token', 'additional_pp_claim_token'] as $column) { + $this->assertSame('char(32)', strtolower($this->columnType($column))); + $this->assertSame('ascii_general_ci', $this->columnCollation($column)); + } + + $token = bin2hex(random_bytes(16)); + DB::table('releases')->where('id', 1)->update(['additional_pp_claim_token' => $token]); + $this->assertSame($token, DB::table('releases')->where('id', 1)->value('additional_pp_claim_token')); + + $followUp->down(); + + $this->assertSame( + ['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at'], + $this->indexColumns('ix_releases_add_pp_claim_queue'), + ); + $this->assertSame('varchar(64)', strtolower($this->columnType('additional_pp_claim_token'))); + } + private function createSchema(): void { $releases = $this->table('releases'); @@ -260,6 +291,14 @@ final class ReleaseSchemaOptimizationMigrationMariaDbTest extends TestCase return $migration; } + private function followUpMigration(): Migration + { + /** @var Migration $migration */ + $migration = require database_path('migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php'); + + return $migration; + } + private function table(string $name): string { return $this->tablePrefix.$name;