mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-28 21:01:22 +00:00
Update migrations
This commit is contained in:
@@ -333,6 +333,15 @@ FORUM_FRONTEND_ENABLED=false
|
||||
|
||||
STREAM_FORK_OUTPUT=false
|
||||
|
||||
# One-off releases table normalization migration (2026_08_13_001652).
|
||||
# Skip the in-migration preflight only after `php artisan releases:optimize-preflight` passes,
|
||||
# since it repeats four full scans of the largest table.
|
||||
RELEASES_OPTIMIZE_SKIP_PREFLIGHT=false
|
||||
# Bypass the free-space guard that aborts when the data volume cannot hold the rebuilt copy.
|
||||
RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK=false
|
||||
# Row batch size for the migration's comment recount and rollback backfills.
|
||||
RELEASES_OPTIMIZE_CHUNK_SIZE=5000
|
||||
|
||||
# API hot-path caching, asynchronous access metadata, and instrumentation.
|
||||
API_RELEASE_CACHE_TTL=600
|
||||
API_RELEASE_CACHE_JITTER=60
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
/**
|
||||
* Reports the guid problems that block the releases normalization migration, and
|
||||
* repairs the one class of problem that can be fixed without side effects.
|
||||
*
|
||||
* `leftguid` is derived data that nothing outside the database depends on, so it is
|
||||
* rewritten in place. Invalid and duplicated `guid` values are only reported: a guid
|
||||
* determines the on-disk NZB path and is published in download links, so replacing one
|
||||
* is a migration of file and link state rather than a database repair.
|
||||
*/
|
||||
#[Signature('releases:normalize-guids
|
||||
{--dry-run : Report the required changes without writing anything}
|
||||
{--chunk=1000 : Rows handled per batch}')]
|
||||
#[Description('Report release guid problems and resync leftguid so the releases normalization migration can run')]
|
||||
class ReleasesNormalizeGuids extends Command
|
||||
{
|
||||
private const string UUID_PATTERN = '/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iD';
|
||||
|
||||
private bool $dryRun = false;
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
if (! Schema::hasTable('releases')) {
|
||||
$this->error('The releases table does not exist.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$this->dryRun = (bool) $this->option('dry-run');
|
||||
if ($this->dryRun) {
|
||||
$this->info('Dry run: no rows are modified.');
|
||||
}
|
||||
|
||||
$leftGuids = $this->syncLeftGuids();
|
||||
$invalid = $this->collectInvalidGuidIds();
|
||||
$duplicates = $this->collectDuplicateGuidIds();
|
||||
|
||||
$this->table(['Issue', 'Releases'], [
|
||||
['leftguid out of sync with guid', (string) $leftGuids],
|
||||
['guid is not a UUID', (string) count($invalid)],
|
||||
['guid duplicated case-insensitively', (string) count($duplicates)],
|
||||
]);
|
||||
|
||||
$blocking = array_values(array_unique([...$invalid, ...$duplicates]));
|
||||
if ($blocking === []) {
|
||||
$this->info($this->dryRun
|
||||
? 'No guid problems found. Re-run without --dry-run to apply the leftguid sync.'
|
||||
: 'Release guids are consistent.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->reportBlockingReleases($blocking);
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the releases that must be resolved by hand before the migration can run.
|
||||
*
|
||||
* @param list<int> $ids
|
||||
*/
|
||||
private function reportBlockingReleases(array $ids): void
|
||||
{
|
||||
$this->error(count($ids).' releases have a guid that blocks the normalization migration.');
|
||||
$this->warn(
|
||||
'These are not repaired automatically. A guid determines the release NZB path and is published in '
|
||||
.'download links, so a new guid orphans the existing NZB file and breaks every link and download_stats '
|
||||
.'row that references the old value. Resolve them deliberately: delete the affected releases, or assign '
|
||||
.'new guids and relocate the matching NZB files yourself.'
|
||||
);
|
||||
|
||||
$sample = array_slice($ids, 0, 20);
|
||||
$rows = DB::table('releases')
|
||||
->select(['id', 'guid', 'name'])
|
||||
->whereIn('id', $sample)
|
||||
->orderBy('id')
|
||||
->get();
|
||||
|
||||
$this->table(['Release ID', 'Guid', 'Name'], $rows->map(static fn ($row): array => [
|
||||
(string) $row->id,
|
||||
(string) $row->guid,
|
||||
Str::limit((string) $row->name, 60),
|
||||
])->all());
|
||||
|
||||
if (count($ids) > count($sample)) {
|
||||
$this->line('… and '.(count($ids) - count($sample)).' more.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point `leftguid` at the first character of `guid` wherever they disagree.
|
||||
* Safe on its own: no file or index state depends on `leftguid`.
|
||||
*/
|
||||
private function syncLeftGuids(): int
|
||||
{
|
||||
$chunk = $this->chunkSize();
|
||||
$total = 0;
|
||||
$lastId = 0;
|
||||
|
||||
do {
|
||||
$rows = DB::table('releases')
|
||||
->select(['id', 'guid'])
|
||||
->where('id', '>', $lastId)
|
||||
->whereRaw('LOWER(leftguid) <> LOWER(SUBSTR(guid, 1, 1))')
|
||||
->orderBy('id')
|
||||
->limit($chunk)
|
||||
->get();
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$lastId = max($lastId, (int) $row->id);
|
||||
$total++;
|
||||
if (! $this->dryRun) {
|
||||
DB::table('releases')
|
||||
->where('id', (int) $row->id)
|
||||
->update(['leftguid' => substr((string) $row->guid, 0, 1)]);
|
||||
}
|
||||
}
|
||||
|
||||
if ($rows->count() < $chunk) {
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
|
||||
return $total;
|
||||
}
|
||||
|
||||
/** @return list<int> */
|
||||
private function collectInvalidGuidIds(): array
|
||||
{
|
||||
$ids = [];
|
||||
DB::table('releases')->select(['id', 'guid'])->orderBy('id')->chunk($this->chunkSize(), function ($rows) use (&$ids): void {
|
||||
foreach ($rows as $row) {
|
||||
if (! is_string($row->guid) || preg_match(self::UUID_PATTERN, $row->guid) !== 1) {
|
||||
$ids[] = (int) $row->id;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return $ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every release that shares a case-insensitive guid with a lower-id release.
|
||||
* The lowest id is treated as the owner of the guid and is not reported.
|
||||
*
|
||||
* @return list<int>
|
||||
*/
|
||||
private function collectDuplicateGuidIds(): array
|
||||
{
|
||||
$duplicated = DB::table('releases')
|
||||
->selectRaw('LOWER(guid) AS normalized_guid')
|
||||
->groupByRaw('LOWER(guid)')
|
||||
->havingRaw('COUNT(*) > 1')
|
||||
->pluck('normalized_guid');
|
||||
|
||||
$ids = [];
|
||||
foreach ($duplicated->chunk(500) as $batch) {
|
||||
$rows = DB::table('releases')
|
||||
->select(['id', 'guid'])
|
||||
->whereIn(DB::raw('LOWER(guid)'), $batch->all())
|
||||
->orderBy('id')
|
||||
->get();
|
||||
|
||||
$seen = [];
|
||||
foreach ($rows as $row) {
|
||||
$key = strtolower((string) $row->guid);
|
||||
if (isset($seen[$key])) {
|
||||
$ids[] = (int) $row->id;
|
||||
|
||||
continue;
|
||||
}
|
||||
$seen[$key] = true;
|
||||
}
|
||||
}
|
||||
|
||||
sort($ids);
|
||||
|
||||
return $ids;
|
||||
}
|
||||
|
||||
private function chunkSize(): int
|
||||
{
|
||||
return max(100, min(10000, (int) $this->option('chunk')));
|
||||
}
|
||||
}
|
||||
@@ -206,17 +206,22 @@ class ReleasesOptimizePreflight extends Command
|
||||
return 0;
|
||||
}
|
||||
|
||||
return DB::query()->fromSub(
|
||||
DB::table('releases as r')
|
||||
->leftJoin('release_comments as rc', function ($join): void {
|
||||
$join->on('rc.releases_id', '=', 'r.id')->where('rc.isvisible', 1);
|
||||
})
|
||||
->select('r.id', 'r.comments')
|
||||
->selectRaw('COUNT(rc.id) AS visible_comments')
|
||||
->groupBy('r.id', 'r.comments')
|
||||
->havingRaw('r.comments <> COUNT(rc.id)'),
|
||||
'comment_counter_mismatches',
|
||||
)->count();
|
||||
// Raw SQL with explicit prefixed names: Laravel prefixes query aliases
|
||||
// too, so `releases as r` would not match a raw `r.comments` reference.
|
||||
$releases = $this->prefixedTable('releases');
|
||||
$comments = $this->prefixedTable('release_comments');
|
||||
|
||||
return count(DB::select(
|
||||
"SELECT r.`id` FROM {$releases} r LEFT JOIN {$comments} c
|
||||
ON c.`releases_id` = r.`id` AND c.`isvisible` = 1
|
||||
GROUP BY r.`id`, r.`comments`
|
||||
HAVING r.`comments` <> COUNT(c.`id`)"
|
||||
));
|
||||
}
|
||||
|
||||
private function prefixedTable(string $name): string
|
||||
{
|
||||
return '`'.DB::getTablePrefix().$name.'`';
|
||||
}
|
||||
|
||||
private function countNonEmpty(string $table, string $column): int
|
||||
@@ -278,10 +283,103 @@ class ReleasesOptimizePreflight extends Command
|
||||
array_values($report['migration_data']),
|
||||
));
|
||||
|
||||
$this->renderStorageSection($report);
|
||||
$this->renderDiscardedDataSection($report);
|
||||
$this->renderIndexSection($report);
|
||||
|
||||
if ($report['ok']) {
|
||||
$this->info('Releases optimization preflight passed.');
|
||||
} else {
|
||||
$this->error('Releases optimization preflight failed. Resolve every blocker before migration.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the on-disk footprint and the free space the rebuild needs.
|
||||
*
|
||||
* @param array<string, mixed> $report
|
||||
*/
|
||||
private function renderStorageSection(array $report): void
|
||||
{
|
||||
$storage = $report['storage'] ?? [];
|
||||
if (! is_array($storage) || ($storage['total_bytes'] ?? null) === null) {
|
||||
$this->line('Storage: not reported for the '.$report['database_driver'].' driver.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->table(['Storage', 'Size'], [
|
||||
['Data', self::formatBytes((int) $storage['data_bytes'])],
|
||||
['Indexes', self::formatBytes((int) $storage['index_bytes'])],
|
||||
['Total', self::formatBytes((int) $storage['total_bytes'])],
|
||||
['Free space required', self::formatBytes((int) $storage['required_free_bytes'])],
|
||||
]);
|
||||
$this->warn(
|
||||
'The rebuild copies the table, so at least '.self::formatBytes((int) $storage['required_free_bytes'])
|
||||
.' must be free on the MySQL data volume before you migrate.'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Print how many values each dropped column is about to lose.
|
||||
*
|
||||
* @param array<string, mixed> $report
|
||||
*/
|
||||
private function renderDiscardedDataSection(array $report): void
|
||||
{
|
||||
$discarded = $report['discarded_data'] ?? [];
|
||||
if (! is_array($discarded) || $discarded === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
$rows = [];
|
||||
foreach ($discarded as $column => $count) {
|
||||
$rows[] = [$column, $count === null ? 'already dropped' : (string) $count];
|
||||
}
|
||||
|
||||
$this->table(['Dropped column', 'Values destroyed'], $rows);
|
||||
|
||||
$destroyed = array_sum(array_map(static fn (?int $count): int => $count ?? 0, $discarded));
|
||||
if ($destroyed > 0) {
|
||||
$this->warn('These '.$destroyed.' values are permanently destroyed by the migration. Back up first.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Print the index changes the migration will make.
|
||||
*
|
||||
* @param array<string, mixed> $report
|
||||
*/
|
||||
private function renderIndexSection(array $report): void
|
||||
{
|
||||
$indexes = $report['indexes'] ?? [];
|
||||
if (! is_array($indexes)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$removal = $indexes['scheduled_for_removal'] ?? [];
|
||||
$existing = $indexes['already_present_replacements'] ?? [];
|
||||
$missing = array_values(array_diff(self::NEW_INDEXES, $existing));
|
||||
|
||||
$this->table(['Index change', 'Names'], [
|
||||
['Dropped', $removal === [] ? 'none' : implode(', ', $removal)],
|
||||
['Created', $missing === [] ? 'none' : implode(', ', $missing)],
|
||||
['Already present', $existing === [] ? 'none' : implode(', ', $existing)],
|
||||
]);
|
||||
}
|
||||
|
||||
private static function formatBytes(int $bytes): string
|
||||
{
|
||||
$units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
$value = (float) $bytes;
|
||||
$unit = 0;
|
||||
while ($value >= 1024.0 && $unit < count($units) - 1) {
|
||||
$value /= 1024.0;
|
||||
$unit++;
|
||||
}
|
||||
|
||||
return $unit === 0
|
||||
? $bytes.' B'
|
||||
: sprintf('%.2f %s', $value, $units[$unit]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ use Illuminate\Support\Facades\DB;
|
||||
* @property int $releases_id FK to releases.id
|
||||
* @property string $text
|
||||
* @property bool $isvisible
|
||||
* @property string $text_hash
|
||||
* @property string $username
|
||||
* @property int $users_id
|
||||
* @property Carbon|null $created_at
|
||||
@@ -32,7 +31,6 @@ use Illuminate\Support\Facades\DB;
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereIsvisible($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereReleasesId($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereText($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereTextHash($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUpdatedAt($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUsername($value)
|
||||
* @method static \Illuminate\Database\Eloquent\Builder|\App\Models\ReleaseComment whereUsersId($value)
|
||||
|
||||
@@ -17,6 +17,10 @@ final class NzbCreationCandidateQuery
|
||||
|
||||
public const string CLAIM_TOKEN_COLUMN = 'nzb_creation_claim_token';
|
||||
|
||||
private static ?bool $supportsClaims = null;
|
||||
|
||||
private static ?bool $supportsFailureState = null;
|
||||
|
||||
/**
|
||||
* @return Builder<Release>
|
||||
*/
|
||||
@@ -112,17 +116,31 @@ final class NzbCreationCandidateQuery
|
||||
|
||||
public static function supportsClaims(): bool
|
||||
{
|
||||
if (! Schema::hasTable('releases')) {
|
||||
return false;
|
||||
if (self::$supportsClaims !== null) {
|
||||
return self::$supportsClaims;
|
||||
}
|
||||
|
||||
return Schema::hasColumn('releases', self::CLAIMED_AT_COLUMN)
|
||||
if (! Schema::hasTable('releases')) {
|
||||
return self::$supportsClaims = false;
|
||||
}
|
||||
|
||||
return self::$supportsClaims = Schema::hasColumn('releases', self::CLAIMED_AT_COLUMN)
|
||||
&& Schema::hasColumn('releases', self::CLAIM_TOKEN_COLUMN);
|
||||
}
|
||||
|
||||
public static function supportsFailureState(): bool
|
||||
{
|
||||
return Schema::hasTable('release_nzb_creation_failures');
|
||||
return self::$supportsFailureState ??= Schema::hasTable('release_nzb_creation_failures');
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard the memoized schema capability flags. Only needed when the schema
|
||||
* changes inside a single process, such as between tests.
|
||||
*/
|
||||
public static function flushCapabilityCache(): void
|
||||
{
|
||||
self::$supportsClaims = null;
|
||||
self::$supportsFailureState = null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -754,6 +754,10 @@ final class ReleaseProcessingService
|
||||
|
||||
private function nextNzbCreationAttempt(Release $release): int
|
||||
{
|
||||
if (! $release->relationLoaded('nzbCreationFailure') && NzbCreationCandidateQuery::supportsFailureState()) {
|
||||
$release->loadMissing('nzbCreationFailure');
|
||||
}
|
||||
|
||||
$failureState = $release->relationLoaded('nzbCreationFailure')
|
||||
? $release->getRelation('nzbCreationFailure')
|
||||
: null;
|
||||
@@ -764,16 +768,13 @@ final class ReleaseProcessingService
|
||||
private function recordNzbCreationRetry(Release $release, NzbCreationResult $result): void
|
||||
{
|
||||
$nextAttempt = $this->nextNzbCreationAttempt($release);
|
||||
ReleaseNzbCreationFailure::query()->upsert(
|
||||
[[
|
||||
'releases_id' => (int) $release->id,
|
||||
$failure = ReleaseNzbCreationFailure::query()->updateOrCreate(
|
||||
['releases_id' => (int) $release->id],
|
||||
[
|
||||
'attempts' => $nextAttempt,
|
||||
'last_error' => mb_substr($result->reason, 0, 1000),
|
||||
]],
|
||||
['releases_id'],
|
||||
['attempts', 'last_error'],
|
||||
],
|
||||
);
|
||||
$failure = ReleaseNzbCreationFailure::query()->findOrFail((int) $release->id);
|
||||
$release->setRelation('nzbCreationFailure', $failure);
|
||||
|
||||
Log::channel('nzb_creation')->warning('NZB creation failed; release will be retried', [
|
||||
|
||||
@@ -81,4 +81,21 @@ return [
|
||||
'categorization' => [
|
||||
'log' => (bool) env('NNTMUX_CATEGORIZATION_LOG', false),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Releases table normalization migration
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Knobs for the one-off releases rebuild. Skip the in-migration preflight
|
||||
| only after `php artisan releases:optimize-preflight` has passed, since the
|
||||
| checks are four full scans of the largest table. The free-space guard
|
||||
| aborts before a COPY-algorithm ALTER that would fill the data volume.
|
||||
|
|
||||
*/
|
||||
'releases_optimize' => [
|
||||
'skip_preflight' => (bool) env('RELEASES_OPTIMIZE_SKIP_PREFLIGHT', false),
|
||||
'skip_free_space_check' => (bool) env('RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK', false),
|
||||
'chunk_size' => (int) env('RELEASES_OPTIMIZE_CHUNK_SIZE', 5000),
|
||||
],
|
||||
];
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace Database\Factories;
|
||||
|
||||
use Illuminate\Database\Eloquent\Factories\Factory;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class ReleaseFactory extends Factory
|
||||
{
|
||||
@@ -17,7 +18,8 @@ class ReleaseFactory extends Factory
|
||||
'fromname' => $this->faker->unique()->safeEmail(),
|
||||
'postdate' => $this->faker->date(),
|
||||
'adddate' => $this->faker->date(),
|
||||
'guid' => $this->faker->sha1(),
|
||||
'guid' => Str::uuid()->toString(),
|
||||
'leftguid' => fn (array $attributes): string => $attributes['guid'][0],
|
||||
'categories_id' => '2080',
|
||||
'nzbstatus' => 1,
|
||||
'passwordstatus' => 0,
|
||||
|
||||
@@ -4,6 +4,7 @@ declare(strict_types=1);
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
@@ -56,6 +57,7 @@ return new class extends Migration
|
||||
}
|
||||
|
||||
$this->assertReleaseIdentifiersAreSafe();
|
||||
$this->assertEnoughFreeSpaceForRebuild();
|
||||
$this->createSparseTables();
|
||||
$this->backfillSparseTables();
|
||||
$this->recountVisibleComments();
|
||||
@@ -226,13 +228,40 @@ return new class extends Migration
|
||||
return;
|
||||
}
|
||||
|
||||
DB::statement(
|
||||
'UPDATE '.$this->table('releases').' SET `comments` = (SELECT COUNT(*) FROM '
|
||||
.$this->table('release_comments').' WHERE '
|
||||
.$this->table('release_comments').'.`releases_id` = '
|
||||
.$this->table('releases').'.`id` AND '
|
||||
.$this->table('release_comments').'.`isvisible` = 1)'
|
||||
);
|
||||
$chunkSize = $this->chunkSize();
|
||||
$releases = $this->table('releases');
|
||||
$comments = $this->table('release_comments');
|
||||
$lastId = 0;
|
||||
|
||||
do {
|
||||
// Raw SQL with explicit prefixed names: Laravel also prefixes query
|
||||
// aliases, so `releases as r` would not match a raw `r.comments`.
|
||||
$mismatches = DB::select(
|
||||
"SELECT r.`id` AS id, COUNT(c.`id`) AS visible_comments
|
||||
FROM {$releases} r LEFT JOIN {$comments} c
|
||||
ON c.`releases_id` = r.`id` AND c.`isvisible` = 1
|
||||
WHERE r.`id` > ?
|
||||
GROUP BY r.`id`, r.`comments`
|
||||
HAVING r.`comments` <> COUNT(c.`id`)
|
||||
ORDER BY r.`id`
|
||||
LIMIT {$chunkSize}",
|
||||
[$lastId],
|
||||
);
|
||||
|
||||
if ($mismatches === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
$idsByCount = [];
|
||||
foreach ($mismatches as $mismatch) {
|
||||
$idsByCount[(int) $mismatch->visible_comments][] = (int) $mismatch->id;
|
||||
$lastId = max($lastId, (int) $mismatch->id);
|
||||
}
|
||||
|
||||
foreach ($idsByCount as $count => $ids) {
|
||||
DB::table('releases')->whereIn('id', $ids)->update(['comments' => $count]);
|
||||
}
|
||||
} while (count($mismatches) === $chunkSize);
|
||||
}
|
||||
|
||||
private function rebuildReleasesForMariaDb(): void
|
||||
@@ -367,33 +396,142 @@ return new class extends Migration
|
||||
$table->boolean('proc_sorter')->default(false);
|
||||
$table->boolean('audiostatus')->default(false);
|
||||
$table->index('guid', 'ix_releases_guid');
|
||||
$table->index('adddate', 'ix_releases_adddate_only');
|
||||
$table->index('videos_id', 'ix_releases_videos_id');
|
||||
$table->index('movieinfo_id', 'ix_releases_movieinfo_id');
|
||||
$table->index('imdbid', 'ix_releases_imdbid');
|
||||
$table->index(
|
||||
['passwordstatus', 'categories_id', 'postdate', 'videos_id', 'tv_episodes_id', 'groups_id'],
|
||||
'ix_releases_tv_search_covering',
|
||||
);
|
||||
$table->index('passwordstatus', 'ix_releases_passwordstatus');
|
||||
$table->index(['haspreview', 'passwordstatus'], 'ix_releases_haspreview_passwordstatus');
|
||||
$table->index(['postdate', 'searchname'], 'ix_releases_postdate_searchname');
|
||||
$table->index(['predb_id', 'searchname'], 'ix_releases_predb_id_searchname');
|
||||
$table->index(['size', 'categories_id', 'passwordstatus'], 'ix_releases_size_cat');
|
||||
$table->index(
|
||||
['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'additional_pp_claimed_at', 'postdate'],
|
||||
'ix_releases_add_pp_claim_queue',
|
||||
);
|
||||
$table->index(
|
||||
['nzbstatus', 'groups_id', 'leftguid', 'nzb_creation_claimed_at', 'postdate'],
|
||||
'ix_releases_nzb_creation_queue',
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private function restoreSparseDataToReleases(): void
|
||||
{
|
||||
$chunkSize = $this->chunkSize();
|
||||
|
||||
if (Schema::hasTable('release_nzb_passwords')) {
|
||||
DB::table('release_nzb_passwords')->orderBy('releases_id')->chunkById(1000, function ($rows): void {
|
||||
foreach ($rows as $row) {
|
||||
DB::table('releases')->where('id', $row->releases_id)->update(['nzb_password' => $row->password]);
|
||||
}
|
||||
}, 'releases_id');
|
||||
DB::table('release_nzb_passwords')->orderBy('releases_id')->chunkById(
|
||||
$chunkSize,
|
||||
fn ($rows) => $this->batchUpdateReleases($rows, ['nzb_password' => 'password']),
|
||||
'releases_id',
|
||||
);
|
||||
}
|
||||
|
||||
if (Schema::hasTable('release_nzb_creation_failures')) {
|
||||
DB::table('release_nzb_creation_failures')->orderBy('releases_id')->chunkById(1000, function ($rows): void {
|
||||
foreach ($rows as $row) {
|
||||
DB::table('releases')->where('id', $row->releases_id)->update([
|
||||
'nzb_creation_attempts' => $row->attempts,
|
||||
'nzb_creation_last_error' => $row->last_error,
|
||||
]);
|
||||
}
|
||||
}, 'releases_id');
|
||||
DB::table('release_nzb_creation_failures')->orderBy('releases_id')->chunkById(
|
||||
$chunkSize,
|
||||
fn ($rows) => $this->batchUpdateReleases($rows, [
|
||||
'nzb_creation_attempts' => 'attempts',
|
||||
'nzb_creation_last_error' => 'last_error',
|
||||
]),
|
||||
'releases_id',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Push one chunk of sparse rows back into `releases` with a single
|
||||
* `CASE`-driven statement instead of one `UPDATE` per row.
|
||||
*
|
||||
* @param Collection<int, object> $rows
|
||||
* @param array<string, string> $columnMap Target `releases` column => source column
|
||||
*/
|
||||
private function batchUpdateReleases($rows, array $columnMap): void
|
||||
{
|
||||
if ($rows->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$setClauses = [];
|
||||
$bindings = [];
|
||||
foreach ($columnMap as $target => $source) {
|
||||
$clause = '`'.$target.'` = CASE `id`';
|
||||
foreach ($rows as $row) {
|
||||
$clause .= ' WHEN ? THEN ?';
|
||||
$bindings[] = (int) $row->releases_id;
|
||||
$bindings[] = $row->{$source};
|
||||
}
|
||||
$setClauses[] = $clause.' ELSE `'.$target.'` END';
|
||||
}
|
||||
|
||||
$ids = [];
|
||||
foreach ($rows as $row) {
|
||||
$ids[] = (int) $row->releases_id;
|
||||
}
|
||||
|
||||
DB::update(
|
||||
'UPDATE '.$this->table('releases').' SET '.implode(', ', $setClauses)
|
||||
.' WHERE `id` IN ('.implode(', ', array_fill(0, count($ids), '?')).')',
|
||||
[...$bindings, ...$ids],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort before the COPY-algorithm rebuild when the data volume cannot hold
|
||||
* a second copy of the table. Silently skipped when the server's datadir is
|
||||
* not visible to PHP (remote or containerised MySQL).
|
||||
*/
|
||||
private function assertEnoughFreeSpaceForRebuild(): void
|
||||
{
|
||||
if (! $this->isMariaDbOrMySql() || (bool) config('nntmux.releases_optimize.skip_free_space_check', false)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$sizes = DB::selectOne(
|
||||
'SELECT DATA_LENGTH + INDEX_LENGTH AS total_bytes FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?',
|
||||
[DB::getTablePrefix().'releases'],
|
||||
);
|
||||
$required = (int) ($sizes->total_bytes ?? 0) * 2;
|
||||
if ($required <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
$dataDir = (string) (DB::selectOne('SELECT @@datadir AS dir')->dir ?? '');
|
||||
if ($dataDir === '' || ! is_dir($dataDir) || ! is_readable($dataDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$free = @disk_free_space($dataDir);
|
||||
if ($free === false || $free >= $required) {
|
||||
return;
|
||||
}
|
||||
|
||||
throw new RuntimeException(sprintf(
|
||||
'Releases rebuild needs about %d bytes free in %s but only %d are available. '
|
||||
.'Free up space, use gh-ost/pt-online-schema-change, or set RELEASES_OPTIMIZE_SKIP_FREE_SPACE_CHECK=true to override.',
|
||||
$required,
|
||||
$dataDir,
|
||||
(int) $free,
|
||||
));
|
||||
}
|
||||
|
||||
private function chunkSize(): int
|
||||
{
|
||||
return max(100, min(10000, (int) config('nntmux.releases_optimize.chunk_size', 5000)));
|
||||
}
|
||||
|
||||
private function assertReleaseIdentifiersAreSafe(): void
|
||||
{
|
||||
if ((bool) config('nntmux.releases_optimize.skip_preflight', false)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$duplicateIds = DB::query()->fromSub(
|
||||
DB::table('releases')->select('id')->groupBy('id')->havingRaw('COUNT(*) > 1'),
|
||||
'duplicate_release_ids',
|
||||
@@ -427,7 +565,10 @@ return new class extends Migration
|
||||
]);
|
||||
|
||||
if ($blockers !== []) {
|
||||
throw new RuntimeException('Releases optimization preflight failed: '.json_encode($blockers, JSON_THROW_ON_ERROR));
|
||||
throw new RuntimeException(
|
||||
'Releases optimization preflight failed: '.json_encode($blockers, JSON_THROW_ON_ERROR)
|
||||
.'. Run `php artisan releases:normalize-guids --dry-run` to see how to resolve them.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use App\Services\AdditionalProcessing\AdditionalCandidateQuery;
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
/**
|
||||
* Follow-up to the releases normalization rebuild.
|
||||
*
|
||||
* 1. Appends `size` to `ix_releases_add_pp_claim_queue`. Every additional
|
||||
* post-processing candidate query filters `size > min AND size < max`
|
||||
* ({@see AdditionalCandidateQuery::applyPredicates()}),
|
||||
* but `size` was not in the index, so each candidate that later failed the
|
||||
* size filter still cost a primary key lookup. Appending it after the
|
||||
* existing columns keeps the equality prefix and the `postdate DESC`
|
||||
* ordering intact while enabling index condition pushdown.
|
||||
* 2. Narrows both claim tokens to `CHAR(32) ascii`. They only ever hold
|
||||
* `bin2hex(random_bytes(16))`, so `VARCHAR(64)` utf8mb4 reserved four bytes
|
||||
* per character for hex digits.
|
||||
*
|
||||
* `releases.imdbid` is deliberately left alone: it joins `movieinfo.imdbid`,
|
||||
* and changing the charset on one side only would force an implicit conversion
|
||||
* that disables index usage on that join.
|
||||
*/
|
||||
return new class extends Migration
|
||||
{
|
||||
private const string PP_CLAIM_INDEX = 'ix_releases_add_pp_claim_queue';
|
||||
|
||||
/** @var list<string> */
|
||||
private const array CLAIM_TOKEN_COLUMNS = [
|
||||
'nzb_creation_claim_token',
|
||||
'additional_pp_claim_token',
|
||||
];
|
||||
|
||||
public function up(): void
|
||||
{
|
||||
if (! Schema::hasTable('releases')) {
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->isMariaDbOrMySql()) {
|
||||
$specifications = [];
|
||||
if ($this->indexExists(self::PP_CLAIM_INDEX)) {
|
||||
$specifications[] = 'DROP INDEX `'.self::PP_CLAIM_INDEX.'`';
|
||||
}
|
||||
$specifications[] = 'ADD INDEX `'.self::PP_CLAIM_INDEX.'` (`passwordstatus`, `haspreview`, `nzbstatus`,'
|
||||
.' `leftguid`, `postdate` DESC, `id`, `additional_pp_claimed_at`, `size`)';
|
||||
foreach (self::CLAIM_TOKEN_COLUMNS as $column) {
|
||||
if (Schema::hasColumn('releases', $column)) {
|
||||
$specifications[] = 'MODIFY `'.$column.'` CHAR(32) CHARACTER SET ascii COLLATE ascii_general_ci NULL';
|
||||
}
|
||||
}
|
||||
|
||||
DB::statement('ALTER TABLE '.$this->table('releases').' '.implode(', ', $specifications));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Schema::table('releases', function (Blueprint $table): void {
|
||||
if ($this->indexExists(self::PP_CLAIM_INDEX)) {
|
||||
$table->dropIndex(self::PP_CLAIM_INDEX);
|
||||
}
|
||||
$table->index(
|
||||
['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at', 'size'],
|
||||
self::PP_CLAIM_INDEX,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
if (! Schema::hasTable('releases')) {
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->isMariaDbOrMySql()) {
|
||||
$specifications = [];
|
||||
if ($this->indexExists(self::PP_CLAIM_INDEX)) {
|
||||
$specifications[] = 'DROP INDEX `'.self::PP_CLAIM_INDEX.'`';
|
||||
}
|
||||
$specifications[] = 'ADD INDEX `'.self::PP_CLAIM_INDEX.'` (`passwordstatus`, `haspreview`, `nzbstatus`,'
|
||||
.' `leftguid`, `postdate` DESC, `id`, `additional_pp_claimed_at`)';
|
||||
foreach (self::CLAIM_TOKEN_COLUMNS as $column) {
|
||||
if (Schema::hasColumn('releases', $column)) {
|
||||
$specifications[] = 'MODIFY `'.$column.'` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL';
|
||||
}
|
||||
}
|
||||
|
||||
DB::statement('ALTER TABLE '.$this->table('releases').' '.implode(', ', $specifications));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Schema::table('releases', function (Blueprint $table): void {
|
||||
if ($this->indexExists(self::PP_CLAIM_INDEX)) {
|
||||
$table->dropIndex(self::PP_CLAIM_INDEX);
|
||||
}
|
||||
$table->index(
|
||||
['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at'],
|
||||
self::PP_CLAIM_INDEX,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private function indexExists(string $name): bool
|
||||
{
|
||||
foreach (Schema::getIndexes('releases') as $index) {
|
||||
if (($index['name'] ?? null) === $name) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function isMariaDbOrMySql(): bool
|
||||
{
|
||||
return in_array(DB::getDriverName(), ['mariadb', 'mysql'], true);
|
||||
}
|
||||
|
||||
private function table(string $name): string
|
||||
{
|
||||
return '`'.DB::getTablePrefix().$name.'`';
|
||||
}
|
||||
};
|
||||
@@ -70,16 +70,6 @@
|
||||
<i class="fas fa-eye-slash mr-1"></i>Hidden
|
||||
</span>
|
||||
@endif
|
||||
@if($comment->shared)
|
||||
<span class="px-2 py-1 inline-flex text-xs leading-5 font-semibold rounded-full bg-blue-100 dark:bg-blue-900 text-blue-800 dark:text-blue-200">
|
||||
<i class="fas fa-share-alt mr-1"></i>Shared
|
||||
</span>
|
||||
@endif
|
||||
@if($comment->issynced)
|
||||
<span class="px-2 py-1 inline-flex text-xs leading-5 font-semibold rounded-full bg-purple-100 dark:bg-purple-900 text-purple-800 dark:text-purple-200">
|
||||
<i class="fas fa-sync mr-1"></i>Synced
|
||||
</span>
|
||||
@endif
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-500 dark:text-gray-400">
|
||||
|
||||
@@ -78,10 +78,12 @@ class NzbCreationReliabilityTest extends TestCase
|
||||
|
||||
$this->createSchema();
|
||||
$this->seedSettings();
|
||||
NzbCreationCandidateQuery::flushCapabilityCache();
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
NzbCreationCandidateQuery::flushCapabilityCache();
|
||||
$this->deleteDirectory($this->tempNzbPath);
|
||||
if ($this->databasePath !== '' && file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
@@ -143,6 +145,8 @@ class NzbCreationReliabilityTest extends TestCase
|
||||
$this->assertSame(1, DB::table('releases')->count());
|
||||
$this->assertSame(1, (int) DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('attempts'));
|
||||
$this->assertSame('Temporary filesystem failure.', DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('last_error'));
|
||||
$this->assertNotNull(DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('created_at'));
|
||||
$this->assertNotNull(DB::table('release_nzb_creation_failures')->where('releases_id', 1)->value('updated_at'));
|
||||
$this->assertNull(DB::table('releases')->where('id', 1)->value('nzb_creation_claimed_at'));
|
||||
$this->assertSame(1, DB::table('collections')->count());
|
||||
$this->assertSame('NZB creation failed; release will be retried', $nzbCreationLogger->warnings[0]['message']);
|
||||
@@ -185,6 +189,56 @@ class NzbCreationReliabilityTest extends TestCase
|
||||
$this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['max_attempts']);
|
||||
}
|
||||
|
||||
public function test_repeated_transient_failure_increments_attempts_and_touches_updated_at(): void
|
||||
{
|
||||
Log::partialMock();
|
||||
Log::shouldReceive('channel')->once()->with('nzb_creation')->andReturn(new RecordingNzbCreationLogger);
|
||||
|
||||
$this->insertRelease(1, 'a');
|
||||
$this->insertCbp(200, 2000, 1);
|
||||
DB::table('release_nzb_creation_failures')->insert([
|
||||
'releases_id' => 1,
|
||||
'attempts' => 1,
|
||||
'last_error' => 'Earlier failure.',
|
||||
'created_at' => '2026-01-01 00:00:00',
|
||||
'updated_at' => '2026-01-01 00:00:00',
|
||||
]);
|
||||
|
||||
$service = $this->releaseProcessingService(
|
||||
NzbCreationResult::transient('Another filesystem failure.', [200])
|
||||
);
|
||||
|
||||
$this->assertSame(0, $service->createNZBs(null));
|
||||
|
||||
$failure = DB::table('release_nzb_creation_failures')->where('releases_id', 1)->first();
|
||||
|
||||
$this->assertSame(2, (int) $failure->attempts);
|
||||
$this->assertSame('Another filesystem failure.', $failure->last_error);
|
||||
$this->assertSame('2026-01-01 00:00:00', $failure->created_at);
|
||||
$this->assertNotSame('2026-01-01 00:00:00', $failure->updated_at);
|
||||
}
|
||||
|
||||
public function test_attempt_cap_holds_when_the_failure_relation_was_dropped(): void
|
||||
{
|
||||
Log::partialMock();
|
||||
$nzbCreationLogger = new RecordingNzbCreationLogger;
|
||||
Log::shouldReceive('channel')->once()->with('nzb_creation')->andReturn($nzbCreationLogger);
|
||||
|
||||
$this->insertRelease(1, 'a', attempts: 2);
|
||||
$this->insertCbp(200, 2000, 1);
|
||||
|
||||
$service = (new ReleaseProcessingService(
|
||||
nzb: new RelationDroppingNzbService(NzbCreationResult::transient('Repeated filesystem failure.', [200])),
|
||||
releaseManagement: new DatabaseOnlyReleaseManagementService,
|
||||
collectionCleanupService: app(CollectionCleanupService::class),
|
||||
))->setEchoCLI(false);
|
||||
|
||||
$this->assertSame(0, $service->createNZBs(null));
|
||||
$this->assertSame(0, DB::table('releases')->count());
|
||||
$this->assertSame('Deleting release after NZB creation failure', $nzbCreationLogger->warnings[0]['message']);
|
||||
$this->assertSame(3, $nzbCreationLogger->warnings[0]['context']['attempt']);
|
||||
}
|
||||
|
||||
public function test_writer_classifies_final_rename_failure_as_transient_without_final_file(): void
|
||||
{
|
||||
$this->insertRelease(1, 'd');
|
||||
@@ -458,6 +512,16 @@ class FakeNzbCreationService extends NzbService
|
||||
}
|
||||
}
|
||||
|
||||
class RelationDroppingNzbService extends FakeNzbCreationService
|
||||
{
|
||||
public function createNzbForRelease(Release $release): NzbCreationResult
|
||||
{
|
||||
$release->unsetRelation('nzbCreationFailure');
|
||||
|
||||
return parent::createNzbForRelease($release);
|
||||
}
|
||||
}
|
||||
|
||||
class DatabaseOnlyReleaseManagementService extends ReleaseManagementService
|
||||
{
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Feature;
|
||||
|
||||
use App\Models\Release;
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use PDO;
|
||||
use Tests\TestCase;
|
||||
|
||||
final class ReleaseFactoryInvariantsTest extends TestCase
|
||||
{
|
||||
private string $databasePath = '';
|
||||
|
||||
public function createApplication()
|
||||
{
|
||||
$this->databasePath = sys_get_temp_dir().'/nntmux-release-factory.sqlite';
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
$pdo = new PDO('sqlite:'.$this->databasePath);
|
||||
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
|
||||
$pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')");
|
||||
putenv('APP_ENV=testing');
|
||||
putenv('DB_CONNECTION=sqlite');
|
||||
putenv('DB_DATABASE='.$this->databasePath);
|
||||
$_ENV['APP_ENV'] = $_SERVER['APP_ENV'] = 'testing';
|
||||
$_ENV['DB_CONNECTION'] = $_SERVER['DB_CONNECTION'] = 'sqlite';
|
||||
$_ENV['DB_DATABASE'] = $_SERVER['DB_DATABASE'] = $this->databasePath;
|
||||
$app = require __DIR__.'/../../bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
|
||||
return $app;
|
||||
}
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
config(['database.default' => 'sqlite', 'database.connections.sqlite.database' => $this->databasePath]);
|
||||
DB::purge();
|
||||
DB::reconnect();
|
||||
DB::statement('CREATE TABLE releases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(255) NOT NULL DEFAULT "",
|
||||
searchname VARCHAR(255) NOT NULL DEFAULT "",
|
||||
fromname VARCHAR(255) NULL,
|
||||
postdate DATETIME NULL,
|
||||
adddate DATETIME NULL,
|
||||
guid CHAR(36) NOT NULL UNIQUE,
|
||||
leftguid CHAR(1) NOT NULL,
|
||||
categories_id INTEGER NOT NULL DEFAULT 10,
|
||||
nzbstatus INTEGER NOT NULL DEFAULT 0,
|
||||
passwordstatus INTEGER NOT NULL DEFAULT -1,
|
||||
isrenamed INTEGER NOT NULL DEFAULT 0
|
||||
)');
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
DB::disconnect();
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
public function test_factory_produces_uuid_guid_with_matching_leftguid(): void
|
||||
{
|
||||
for ($iteration = 0; $iteration < 5; $iteration++) {
|
||||
$attributes = Release::factory()->raw();
|
||||
|
||||
$this->assertMatchesRegularExpression(
|
||||
'/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iD',
|
||||
(string) $attributes['guid'],
|
||||
'The factory must produce a UUID guid that fits the char(36) column.',
|
||||
);
|
||||
$this->assertSame(36, mb_strlen((string) $attributes['guid']));
|
||||
$this->assertSame(
|
||||
mb_strtolower(mb_substr((string) $attributes['guid'], 0, 1)),
|
||||
mb_strtolower((string) $attributes['leftguid']),
|
||||
'leftguid must be the first character of guid.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public function test_factory_rows_satisfy_the_release_table_constraints(): void
|
||||
{
|
||||
foreach (Release::factory()->count(3)->raw() as $attributes) {
|
||||
DB::table('releases')->insert($attributes);
|
||||
}
|
||||
|
||||
$rows = DB::table('releases')->get(['guid', 'leftguid']);
|
||||
|
||||
$this->assertCount(3, $rows);
|
||||
foreach ($rows as $row) {
|
||||
$this->assertSame(mb_substr((string) $row->guid, 0, 1), (string) $row->leftguid);
|
||||
}
|
||||
$this->assertSame(3, $rows->pluck('guid')->unique()->count());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Feature;
|
||||
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use PDO;
|
||||
use ReflectionMethod;
|
||||
use RuntimeException;
|
||||
use Tests\Integration\ReleaseSchemaOptimizationMigrationMariaDbTest;
|
||||
use Tests\TestCase;
|
||||
|
||||
/**
|
||||
* Portable coverage for the hardening added to the releases normalization
|
||||
* migration: the chunked comment recount, the batched rollback backfill, and
|
||||
* the preflight opt-out. The full schema rebuild is MariaDB-only and lives in
|
||||
* {@see ReleaseSchemaOptimizationMigrationMariaDbTest}.
|
||||
*/
|
||||
final class ReleaseSchemaOptimizationMigrationTest extends TestCase
|
||||
{
|
||||
private string $databasePath = '';
|
||||
|
||||
public function createApplication()
|
||||
{
|
||||
$this->databasePath = sys_get_temp_dir().'/nntmux-releases-migration.sqlite';
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
$pdo = new PDO('sqlite:'.$this->databasePath);
|
||||
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
|
||||
$pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')");
|
||||
$this->setEnvironmentValue('APP_ENV', 'testing');
|
||||
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
|
||||
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
|
||||
|
||||
$app = require __DIR__.'/../../bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
|
||||
return $app;
|
||||
}
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
config([
|
||||
'database.default' => 'sqlite',
|
||||
'database.connections.sqlite.database' => $this->databasePath,
|
||||
'nntmux.releases_optimize.chunk_size' => 100,
|
||||
'nntmux.releases_optimize.skip_preflight' => false,
|
||||
]);
|
||||
DB::purge();
|
||||
DB::reconnect();
|
||||
$this->createSchema();
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
DB::disconnect();
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
public function test_comment_recount_spans_multiple_chunks_and_skips_correct_rows(): void
|
||||
{
|
||||
$releases = [];
|
||||
$comments = [];
|
||||
$commentId = 1;
|
||||
for ($id = 1; $id <= 250; $id++) {
|
||||
// Every third release already stores the right counter and must not be rewritten.
|
||||
$alreadyCorrect = $id % 3 === 0;
|
||||
$releases[] = ['id' => $id, 'guid' => 'g'.$id, 'leftguid' => 'g', 'comments' => $alreadyCorrect ? 1 : 99];
|
||||
$comments[] = ['id' => $commentId++, 'releases_id' => $id, 'isvisible' => 1];
|
||||
$comments[] = ['id' => $commentId++, 'releases_id' => $id, 'isvisible' => 0];
|
||||
}
|
||||
DB::table('releases')->insert($releases);
|
||||
DB::table('release_comments')->insert($comments);
|
||||
|
||||
$this->invoke('recountVisibleComments');
|
||||
|
||||
$this->assertSame(0, DB::table('releases')->where('comments', '<>', 1)->count());
|
||||
$this->assertSame(250, DB::table('releases')->where('comments', 1)->count());
|
||||
}
|
||||
|
||||
public function test_comment_recount_zeroes_releases_without_visible_comments(): void
|
||||
{
|
||||
DB::table('releases')->insert([
|
||||
['id' => 1, 'guid' => 'a', 'leftguid' => 'a', 'comments' => 7],
|
||||
['id' => 2, 'guid' => 'b', 'leftguid' => 'b', 'comments' => 0],
|
||||
]);
|
||||
DB::table('release_comments')->insert([['id' => 1, 'releases_id' => 1, 'isvisible' => 0]]);
|
||||
|
||||
$this->invoke('recountVisibleComments');
|
||||
|
||||
$this->assertSame(0, (int) DB::table('releases')->where('id', 1)->value('comments'));
|
||||
$this->assertSame(0, (int) DB::table('releases')->where('id', 2)->value('comments'));
|
||||
}
|
||||
|
||||
public function test_rollback_backfill_restores_sparse_rows_in_batches(): void
|
||||
{
|
||||
$releases = [];
|
||||
$passwords = [];
|
||||
$failures = [];
|
||||
for ($id = 1; $id <= 250; $id++) {
|
||||
$releases[] = ['id' => $id, 'guid' => 'g'.$id, 'leftguid' => 'g', 'comments' => 0];
|
||||
$passwords[] = ['releases_id' => $id, 'password' => 'secret-'.$id];
|
||||
$failures[] = ['releases_id' => $id, 'attempts' => $id % 5, 'last_error' => 'boom-'.$id];
|
||||
}
|
||||
DB::table('releases')->insert($releases);
|
||||
DB::table('release_nzb_passwords')->insert($passwords);
|
||||
DB::table('release_nzb_creation_failures')->insert($failures);
|
||||
|
||||
$this->invoke('restoreSparseDataToReleases');
|
||||
|
||||
$restored = DB::table('releases')->orderBy('id')->get();
|
||||
$this->assertCount(250, $restored);
|
||||
foreach ($restored as $release) {
|
||||
$this->assertSame('secret-'.$release->id, $release->nzb_password);
|
||||
$this->assertSame((int) $release->id % 5, (int) $release->nzb_creation_attempts);
|
||||
$this->assertSame('boom-'.$release->id, $release->nzb_creation_last_error);
|
||||
}
|
||||
}
|
||||
|
||||
public function test_preflight_blocks_bad_identifiers_and_the_opt_out_bypasses_it(): void
|
||||
{
|
||||
DB::table('releases')->insert([
|
||||
['id' => 1, 'guid' => 'not-a-uuid', 'leftguid' => 'z', 'comments' => 0],
|
||||
]);
|
||||
|
||||
try {
|
||||
$this->invoke('assertReleaseIdentifiersAreSafe');
|
||||
$this->fail('The migration accepted an invalid guid.');
|
||||
} catch (RuntimeException $e) {
|
||||
$this->assertStringContainsString('invalid release GUIDs', $e->getMessage());
|
||||
$this->assertStringContainsString('releases:normalize-guids', $e->getMessage());
|
||||
}
|
||||
|
||||
config(['nntmux.releases_optimize.skip_preflight' => true]);
|
||||
$this->invoke('assertReleaseIdentifiersAreSafe');
|
||||
$this->assertSame(1, DB::table('releases')->count());
|
||||
}
|
||||
|
||||
private function invoke(string $method): void
|
||||
{
|
||||
/** @var Migration $migration */
|
||||
$migration = require database_path('migrations/2026_08_13_001652_normalize_and_optimize_releases_table.php');
|
||||
(new ReflectionMethod($migration, $method))->invoke($migration);
|
||||
}
|
||||
|
||||
private function createSchema(): void
|
||||
{
|
||||
foreach (['release_comments', 'release_nzb_passwords', 'release_nzb_creation_failures', 'releases'] as $table) {
|
||||
DB::statement('DROP TABLE IF EXISTS '.$table);
|
||||
}
|
||||
DB::statement('CREATE TABLE releases (
|
||||
id INTEGER PRIMARY KEY, guid VARCHAR(40) NOT NULL, leftguid CHAR(1) NOT NULL,
|
||||
comments INTEGER NOT NULL DEFAULT 0, nzb_password VARCHAR(255) NULL,
|
||||
nzb_creation_attempts INTEGER NOT NULL DEFAULT 0, nzb_creation_last_error TEXT NULL
|
||||
)');
|
||||
DB::statement('CREATE TABLE release_comments (
|
||||
id INTEGER PRIMARY KEY, releases_id INTEGER NOT NULL, isvisible INTEGER NOT NULL DEFAULT 1
|
||||
)');
|
||||
DB::statement('CREATE TABLE release_nzb_passwords (
|
||||
releases_id INTEGER PRIMARY KEY, password VARCHAR(255) NOT NULL
|
||||
)');
|
||||
DB::statement('CREATE TABLE release_nzb_creation_failures (
|
||||
releases_id INTEGER PRIMARY KEY, attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT NULL
|
||||
)');
|
||||
}
|
||||
|
||||
private function setEnvironmentValue(string $key, ?string $value): void
|
||||
{
|
||||
if ($value === null) {
|
||||
putenv($key);
|
||||
unset($_ENV[$key], $_SERVER[$key]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
putenv("{$key}={$value}");
|
||||
$_ENV[$key] = $_SERVER[$key] = $value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Feature;
|
||||
|
||||
use Illuminate\Contracts\Console\Kernel;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use PDO;
|
||||
use Tests\TestCase;
|
||||
|
||||
final class ReleasesNormalizeGuidsTest extends TestCase
|
||||
{
|
||||
private string $databasePath = '';
|
||||
|
||||
public function createApplication()
|
||||
{
|
||||
$this->databasePath = sys_get_temp_dir().'/nntmux-normalize-guids.sqlite';
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
$pdo = new PDO('sqlite:'.$this->databasePath);
|
||||
$pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)');
|
||||
$pdo->exec("INSERT INTO settings VALUES ('categorizeforeign', '0'), ('catwebdl', '0')");
|
||||
$this->setEnvironmentValue('APP_ENV', 'testing');
|
||||
$this->setEnvironmentValue('DB_CONNECTION', 'sqlite');
|
||||
$this->setEnvironmentValue('DB_DATABASE', $this->databasePath);
|
||||
|
||||
$app = require __DIR__.'/../../bootstrap/app.php';
|
||||
$app->make(Kernel::class)->bootstrap();
|
||||
|
||||
return $app;
|
||||
}
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
|
||||
config([
|
||||
'database.default' => 'sqlite',
|
||||
'database.connections.sqlite.database' => $this->databasePath,
|
||||
]);
|
||||
DB::purge();
|
||||
DB::reconnect();
|
||||
$this->createSchema();
|
||||
}
|
||||
|
||||
protected function tearDown(): void
|
||||
{
|
||||
DB::disconnect();
|
||||
if (file_exists($this->databasePath)) {
|
||||
unlink($this->databasePath);
|
||||
}
|
||||
parent::tearDown();
|
||||
}
|
||||
|
||||
public function test_leftguid_is_resynced_in_place(): void
|
||||
{
|
||||
$guid = '01234567-89ab-cdef-0123-456789abcdef';
|
||||
$this->insertRelease(1, $guid, 'f');
|
||||
|
||||
$status = Artisan::call('releases:normalize-guids');
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertSame('0', DB::table('releases')->where('id', 1)->value('leftguid'));
|
||||
$this->assertSame($guid, DB::table('releases')->where('id', 1)->value('guid'));
|
||||
}
|
||||
|
||||
public function test_dry_run_reports_without_writing(): void
|
||||
{
|
||||
$this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', 'f');
|
||||
$this->insertRelease(2, 'd41d8cd98f00b204e9800998ecf8427e', 'd');
|
||||
|
||||
$status = Artisan::call('releases:normalize-guids', ['--dry-run' => true]);
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(1, $status);
|
||||
$this->assertStringContainsString('Dry run: no rows are modified.', $output);
|
||||
$this->assertSame('f', DB::table('releases')->where('id', 1)->value('leftguid'));
|
||||
$this->assertSame('d41d8cd98f00b204e9800998ecf8427e', DB::table('releases')->where('id', 2)->value('guid'));
|
||||
}
|
||||
|
||||
public function test_invalid_guid_is_reported_and_never_rewritten(): void
|
||||
{
|
||||
$this->insertRelease(1, 'd41d8cd98f00b204e9800998ecf8427e', 'd', ['name' => 'Legacy nZEDb release']);
|
||||
|
||||
$status = Artisan::call('releases:normalize-guids');
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(1, $status);
|
||||
$this->assertStringContainsString('1 releases have a guid that blocks the normalization migration.', $output);
|
||||
$this->assertStringContainsString('d41d8cd98f00b204e9800998ecf8427e', $output);
|
||||
$this->assertStringContainsString('Legacy nZEDb release', $output);
|
||||
$this->assertSame('d41d8cd98f00b204e9800998ecf8427e', DB::table('releases')->where('id', 1)->value('guid'));
|
||||
}
|
||||
|
||||
public function test_case_insensitive_duplicates_report_every_id_but_the_lowest(): void
|
||||
{
|
||||
$guid = 'abcdef01-2345-6789-abcd-ef0123456789';
|
||||
$this->insertRelease(1, $guid, 'a');
|
||||
$this->insertRelease(2, strtoupper($guid), 'A');
|
||||
|
||||
$status = Artisan::call('releases:normalize-guids');
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(1, $status);
|
||||
$this->assertStringContainsString('1 releases have a guid that blocks', $output);
|
||||
$this->assertSame($guid, DB::table('releases')->where('id', 1)->value('guid'));
|
||||
$this->assertSame(strtoupper($guid), DB::table('releases')->where('id', 2)->value('guid'));
|
||||
}
|
||||
|
||||
public function test_consistent_guids_report_success(): void
|
||||
{
|
||||
$this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', '0');
|
||||
|
||||
$status = Artisan::call('releases:normalize-guids');
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertStringContainsString('Release guids are consistent.', Artisan::output());
|
||||
}
|
||||
|
||||
/** @param array<string, mixed> $overrides */
|
||||
private function insertRelease(int $id, string $guid, string $leftguid, array $overrides = []): void
|
||||
{
|
||||
DB::table('releases')->insert([
|
||||
'id' => $id,
|
||||
'guid' => $guid,
|
||||
'leftguid' => $leftguid,
|
||||
'name' => 'Release '.$id,
|
||||
...$overrides,
|
||||
]);
|
||||
}
|
||||
|
||||
private function createSchema(): void
|
||||
{
|
||||
DB::statement('DROP TABLE IF EXISTS releases');
|
||||
DB::statement('CREATE TABLE releases (
|
||||
id INTEGER PRIMARY KEY,
|
||||
guid VARCHAR(40) NOT NULL,
|
||||
leftguid CHAR(1) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL DEFAULT ""
|
||||
)');
|
||||
}
|
||||
|
||||
private function setEnvironmentValue(string $key, ?string $value): void
|
||||
{
|
||||
if ($value === null) {
|
||||
putenv($key);
|
||||
unset($_ENV[$key], $_SERVER[$key]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
putenv("{$key}={$value}");
|
||||
$_ENV[$key] = $_SERVER[$key] = $value;
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,22 @@ final class ReleasesOptimizePreflightTest extends TestCase
|
||||
}
|
||||
}
|
||||
|
||||
public function test_human_report_includes_storage_discarded_data_and_index_sections(): void
|
||||
{
|
||||
$this->insertRelease(1, '01234567-89ab-cdef-0123-456789abcdef', '0', ['source' => 2]);
|
||||
|
||||
$status = Artisan::call('releases:optimize-preflight');
|
||||
$output = Artisan::output();
|
||||
|
||||
$this->assertSame(0, $status);
|
||||
$this->assertStringContainsString('Storage: not reported for the sqlite driver.', $output);
|
||||
$this->assertStringContainsString('Dropped column', $output);
|
||||
$this->assertStringContainsString('releases.source', $output);
|
||||
$this->assertStringContainsString('permanently destroyed', $output);
|
||||
$this->assertStringContainsString('Index change', $output);
|
||||
$this->assertStringContainsString('ux_releases_guid', $output);
|
||||
}
|
||||
|
||||
/** @param array<string, mixed> $overrides */
|
||||
private function insertRelease(int $id, string $guid, string $leftguid, array $overrides = []): void
|
||||
{
|
||||
|
||||
@@ -113,6 +113,37 @@ final class ReleaseSchemaOptimizationMigrationMariaDbTest extends TestCase
|
||||
$this->assertDatabaseHas('release_files', ['releases_id' => 1, 'name' => 'kept.nzb']);
|
||||
}
|
||||
|
||||
#[Test]
|
||||
public function claim_token_and_pp_index_follow_up_narrows_columns_and_adds_size(): void
|
||||
{
|
||||
$this->seedLegacyData();
|
||||
$this->migration()->up();
|
||||
$followUp = $this->followUpMigration();
|
||||
|
||||
$followUp->up();
|
||||
|
||||
$this->assertSame(
|
||||
['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at', 'size'],
|
||||
$this->indexColumns('ix_releases_add_pp_claim_queue'),
|
||||
);
|
||||
foreach (['nzb_creation_claim_token', 'additional_pp_claim_token'] as $column) {
|
||||
$this->assertSame('char(32)', strtolower($this->columnType($column)));
|
||||
$this->assertSame('ascii_general_ci', $this->columnCollation($column));
|
||||
}
|
||||
|
||||
$token = bin2hex(random_bytes(16));
|
||||
DB::table('releases')->where('id', 1)->update(['additional_pp_claim_token' => $token]);
|
||||
$this->assertSame($token, DB::table('releases')->where('id', 1)->value('additional_pp_claim_token'));
|
||||
|
||||
$followUp->down();
|
||||
|
||||
$this->assertSame(
|
||||
['passwordstatus', 'haspreview', 'nzbstatus', 'leftguid', 'postdate', 'id', 'additional_pp_claimed_at'],
|
||||
$this->indexColumns('ix_releases_add_pp_claim_queue'),
|
||||
);
|
||||
$this->assertSame('varchar(64)', strtolower($this->columnType('additional_pp_claim_token')));
|
||||
}
|
||||
|
||||
private function createSchema(): void
|
||||
{
|
||||
$releases = $this->table('releases');
|
||||
@@ -260,6 +291,14 @@ final class ReleaseSchemaOptimizationMigrationMariaDbTest extends TestCase
|
||||
return $migration;
|
||||
}
|
||||
|
||||
private function followUpMigration(): Migration
|
||||
{
|
||||
/** @var Migration $migration */
|
||||
$migration = require database_path('migrations/2026_08_14_121946_optimize_releases_claim_tokens_and_pp_index.php');
|
||||
|
||||
return $migration;
|
||||
}
|
||||
|
||||
private function table(string $name): string
|
||||
{
|
||||
return $this->tablePrefix.$name;
|
||||
|
||||
Reference in New Issue
Block a user