Update cbp handling

.env.example

@@ -237,6 +237,15 @@ IMAGE_FETCH_TIMEOUT=30
 IMAGE_FETCH_MAX_REDIRECTS=5
 MAX_PAGER_RESULTS=125000
 ECHOCLI=true
+
+# Collections/binaries/parts processing limits. Lower values reduce peak memory
+# and lock duration; higher values may improve throughput on larger DB servers.
+CBP_HEADER_CHUNK_SIZE=500
+CBP_SQL_CHUNK_SIZE=500
+CBP_RECONCILE_BATCH_SIZE=500
+CBP_NZB_STREAM_ROWS=5000
+# Permit php artisan migrate to run the resumable CBP hash/key rewrite. Enable only during maintenance.
+CBP_STORAGE_MIGRATION_EXECUTE=false
 RENAME_PAR2=false
 ADD_PAR2=false
 RENAME_MUSIC_MEDIAINFO=true

app/Console/Commands/OptimizeCbpStorage.php

@@ -0,0 +1,454 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Services\XrefService;
+use App\Support\Utf8;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+final class OptimizeCbpStorage extends Command
+{
+    protected $signature = 'cbp:optimize-storage
+                            {--execute : Apply the reported storage changes}
+                            {--batch= : Override the bounded reconciliation batch size}';
+
+    protected $description = 'Audit or resumably optimize collections, binaries, and parts storage';
+
+    public function handle(XrefService $xrefService): int
+    {
+        if (! \in_array(DB::getDriverName(), ['mysql', 'mariadb'], true)) {
+            $this->error('CBP storage optimization requires MySQL 8 or MariaDB. SQLite is test-only.');
+
+            return self::FAILURE;
+        }
+
+        foreach (['collections', 'binaries', 'parts'] as $table) {
+            if (! Schema::hasTable($table)) {
+                $this->error("Required table {$table} does not exist.");
+
+                return self::FAILURE;
+            }
+        }
+
+        $this->reportPreflight();
+        if (! (bool) $this->option('execute')) {
+            $this->newLine();
+            $this->warn('Dry-run only. Stop processing, verify a backup, then rerun with --execute.');
+
+            return self::SUCCESS;
+        }
+
+        $batchSize = max(1, min(5000, (int) ($this->option('batch') ?: config('nntmux.cbp.reconcile_batch_size', 500))));
+        $this->ensureCheckpointTable();
+
+        try {
+            $this->step('01_additive_schema', fn () => $this->prepareAdditiveSchema());
+            $this->step('02_collection_groups', fn () => $this->backfillCollectionGroups($xrefService, $batchSize));
+            $this->step('03_hash_columns', fn () => $this->prepareHashColumns());
+            $this->step('04_binary_hashes', fn () => $this->populateBinaryHashes($batchSize));
+            $this->step('05_binary_map', fn () => $this->buildBinaryMap());
+            $this->step('06_parts_shadow', fn () => $this->buildPartsShadow());
+            $this->step('07_swap_parts', fn () => $this->swapParts());
+            $this->step('08_merge_binaries', fn () => $this->mergeBinaries());
+            $this->step('09_collection_hash', fn () => $this->convertCollectionHash());
+            $this->step('10_constraints', fn () => $this->installConstraintsAndIndexes());
+            $this->step('11_aggregates', fn () => $this->rebuildAggregates($batchSize));
+            $this->step('12_finalize', fn () => $this->finalizeOptimization());
+        } catch (\Throwable $e) {
+            $this->error('Optimization stopped: '.$e->getMessage());
+            $this->warn('Completed checkpoints are retained; fix the cause and rerun the same command.');
+
+            return self::FAILURE;
+        }
+
+        $this->info('CBP storage optimization completed. Validate the preflight again before restarting workers.');
+
+        return self::SUCCESS;
+    }
+
+    private function reportPreflight(): void
+    {
+        $duplicateParts = (int) DB::scalar(
+            'SELECT COALESCE(SUM(n - 1), 0) FROM (
+                SELECT COUNT(*) AS n FROM parts GROUP BY binaries_id, partnumber HAVING COUNT(*) > 1
+             ) duplicate_parts'
+        );
+        $duplicateBinaries = (int) DB::scalar(
+            'SELECT COALESCE(SUM(n - 1), 0) FROM (
+                SELECT COUNT(*) AS n FROM binaries GROUP BY collections_id, binaryhash HAVING COUNT(*) > 1
+             ) duplicate_binaries'
+        );
+        $aggregateDrift = (int) DB::scalar(
+            'SELECT COUNT(*) FROM binaries b
+             LEFT JOIN (
+                SELECT binaries_id, COUNT(*) AS currentparts, COALESCE(SUM(size), 0) AS partsize
+                FROM parts GROUP BY binaries_id
+             ) p ON p.binaries_id = b.id
+             WHERE b.currentparts <> COALESCE(p.currentparts, 0)
+                OR b.partsize <> COALESCE(p.partsize, 0)'
+        );
+        $orphanParts = (int) DB::scalar(
+            'SELECT COUNT(*) FROM parts p LEFT JOIN binaries b ON b.id = p.binaries_id WHERE b.id IS NULL'
+        );
+        $orphanBinaries = (int) DB::scalar(
+            'SELECT COUNT(*) FROM binaries b LEFT JOIN collections c ON c.id = b.collections_id WHERE c.id IS NULL'
+        );
+        $invalidMessageIds = (int) DB::scalar(
+            "SELECT COUNT(*) FROM parts WHERE messageid = '' OR messageid REGEXP '[^ -~]'"
+        );
+        $bytes = (int) DB::scalar(
+            "SELECT COALESCE(SUM(data_length + index_length), 0)
+             FROM information_schema.TABLES
+             WHERE table_schema = DATABASE() AND table_name IN ('collections', 'binaries', 'parts')"
+        );
+        $freeBytes = (int) DB::scalar(
+            "SELECT COALESCE(SUM(data_free), 0)
+             FROM information_schema.TABLES
+             WHERE table_schema = DATABASE() AND table_name IN ('collections', 'binaries', 'parts')"
+        );
+        $cascadeForeignKeys = (int) DB::scalar(
+            "SELECT COUNT(*) FROM information_schema.REFERENTIAL_CONSTRAINTS
+             WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME IN ('binaries', 'parts')
+               AND DELETE_RULE = 'CASCADE'"
+        );
+
+        $this->table(['Check', 'Result'], [
+            ['Duplicate binary/part identities', number_format($duplicateBinaries)],
+            ['Duplicate binary part numbers', number_format($duplicateParts)],
+            ['Binary aggregate drift', number_format($aggregateDrift)],
+            ['Orphan parts / binaries', number_format($orphanParts).' / '.number_format($orphanBinaries)],
+            ['Required cascading foreign keys', $cascadeForeignKeys.'/2 healthy'],
+            ['Empty or non-ASCII message IDs', number_format($invalidMessageIds)],
+            ['Current CBP table+index bytes', number_format($bytes)],
+            ['Reported reusable table bytes', number_format($freeBytes)],
+            ['Conservative additional disk required', number_format(max($bytes, 1))],
+        ]);
+        $this->line('Proposed: normalized group table, compact hashes, corrected identities, deterministic part merge, authoritative aggregates, cascade FKs, and hot-path indexes.');
+    }
+
+    private function ensureCheckpointTable(): void
+    {
+        DB::statement(
+            'CREATE TABLE IF NOT EXISTS cbp_optimization_checkpoints (
+                step_name VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+                completed_at DATETIME NOT NULL,
+                PRIMARY KEY (step_name)
+             ) ENGINE=InnoDB'
+        );
+    }
+
+    private function step(string $name, callable $operation): void
+    {
+        if (DB::table('cbp_optimization_checkpoints')->where('step_name', $name)->exists()) {
+            $this->line("Skipping completed step {$name}.");
+
+            return;
+        }
+
+        $this->info("Running {$name}...");
+        $operation();
+        DB::table('cbp_optimization_checkpoints')->insert([
+            'step_name' => $name,
+            'completed_at' => now(),
+        ]);
+    }
+
+    private function prepareAdditiveSchema(): void
+    {
+        if (! Schema::hasColumn('collections', 'last_seen_at')) {
+            DB::statement('ALTER TABLE collections ADD last_seen_at DATETIME NULL AFTER dateadded');
+        }
+        DB::statement(
+            'CREATE TABLE IF NOT EXISTS collection_groups (
+                collections_id INT UNSIGNED NOT NULL,
+                group_name VARCHAR(255) NOT NULL,
+                PRIMARY KEY (collections_id, group_name)
+             ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci'
+        );
+    }
+
+    private function backfillCollectionGroups(XrefService $xrefService, int $batchSize): void
+    {
+        $lastId = 0;
+        do {
+            $rows = DB::select(
+                'SELECT c.id, c.xref, g.name AS group_name
+                 FROM collections c
+                 LEFT JOIN usenet_groups g ON g.id = c.groups_id
+                 WHERE c.id > ? ORDER BY c.id LIMIT '.$batchSize,
+                [$lastId]
+            );
+            $values = [];
+            $bindings = [];
+            foreach ($rows as $row) {
+                $lastId = (int) $row->id;
+                $groups = $xrefService->extractGroupNames((string) $row->xref);
+                if ($groups === [] && (string) $row->group_name !== '') {
+                    $groups = [(string) $row->group_name];
+                }
+                foreach ($groups as $group) {
+                    $values[] = '(?, ?)';
+                    $bindings[] = $lastId;
+                    $bindings[] = $group;
+                }
+            }
+            if ($values !== []) {
+                DB::statement(
+                    'INSERT IGNORE INTO collection_groups (collections_id, group_name) VALUES '.implode(',', $values),
+                    $bindings
+                );
+            }
+        } while (\count($rows) === $batchSize);
+    }
+
+    private function prepareHashColumns(): void
+    {
+        if (! Schema::hasColumn('binaries', 'cbp_hash')) {
+            DB::statement('ALTER TABLE binaries ADD cbp_hash BINARY(16) NULL AFTER binaryhash');
+        }
+        if (! Schema::hasColumn('collections', 'cbp_hash')) {
+            DB::statement('ALTER TABLE collections ADD cbp_hash BINARY(20) NULL AFTER collectionhash');
+        }
+    }
+
+    private function populateBinaryHashes(int $batchSize): void
+    {
+        $lastId = 0;
+        do {
+            $rows = DB::select(
+                'SELECT b.id, b.name, b.filenumber, c.fromname
+                 FROM binaries b INNER JOIN collections c ON c.id = b.collections_id
+                 WHERE b.id > ? AND b.cbp_hash IS NULL ORDER BY b.id LIMIT '.$batchSize,
+                [$lastId]
+            );
+            foreach ($rows as $row) {
+                $lastId = (int) $row->id;
+                $hash = (int) $row->filenumber > 0
+                    ? md5('file:'.(int) $row->filenumber, true)
+                    : md5('subject:'.$this->normalizeIdentity((string) $row->name)."\0poster:".$this->normalizeIdentity((string) $row->fromname), true);
+                DB::update('UPDATE binaries SET cbp_hash = ? WHERE id = ?', [$hash, $lastId]);
+            }
+        } while (\count($rows) === $batchSize);
+    }
+
+    private function buildBinaryMap(): void
+    {
+        DB::statement('DROP TABLE IF EXISTS cbp_binary_map');
+        DB::statement(
+            'CREATE TABLE cbp_binary_map (
+                old_id BIGINT UNSIGNED NOT NULL,
+                keep_id BIGINT UNSIGNED NOT NULL,
+                PRIMARY KEY (old_id), KEY ix_cbp_binary_map_keep (keep_id)
+             ) ENGINE=InnoDB'
+        );
+        DB::statement(
+            'INSERT INTO cbp_binary_map (old_id, keep_id)
+             SELECT id, MIN(id) OVER (PARTITION BY collections_id, cbp_hash) FROM binaries'
+        );
+
+        $binaryCount = (int) DB::table('binaries')->count();
+        $mappedCount = (int) DB::table('cbp_binary_map')->count();
+        if ($mappedCount !== $binaryCount) {
+            throw new \RuntimeException("Binary map is incomplete ({$mappedCount}/{$binaryCount}); the source tables were not changed.");
+        }
+    }
+
+    private function buildPartsShadow(): void
+    {
+        DB::statement('DROP TABLE IF EXISTS parts_cbp_new');
+        DB::statement(
+            'CREATE TABLE parts_cbp_new (
+                binaries_id BIGINT UNSIGNED NOT NULL,
+                messageid VARCHAR(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT \'\',
+                number BIGINT UNSIGNED NOT NULL DEFAULT 0,
+                partnumber INT UNSIGNED NOT NULL DEFAULT 0,
+                size INT UNSIGNED NOT NULL DEFAULT 0,
+                PRIMARY KEY (binaries_id, partnumber),
+                KEY ix_parts_number (number)
+             ) ENGINE=InnoDB'
+        );
+        DB::statement(
+            'INSERT INTO parts_cbp_new (binaries_id, messageid, number, partnumber, size)
+             SELECT binaries_id, messageid, number, partnumber, size
+             FROM (
+                SELECT m.keep_id AS binaries_id, p.messageid, p.number, p.partnumber, p.size,
+                       ROW_NUMBER() OVER (
+                         PARTITION BY m.keep_id, p.partnumber
+                         ORDER BY (p.messageid <> \'\') DESC, p.size DESC, p.number ASC
+                       ) AS preference
+                FROM parts p INNER JOIN cbp_binary_map m ON m.old_id = p.binaries_id
+             ) ranked WHERE preference = 1'
+        );
+
+        $unmappedParts = (int) DB::scalar(
+            'SELECT COUNT(*) FROM parts p
+             LEFT JOIN cbp_binary_map m ON m.old_id = p.binaries_id
+             WHERE m.old_id IS NULL'
+        );
+        if ($unmappedParts > 0) {
+            throw new \RuntimeException("{$unmappedParts} parts are missing a binary-map entry; the source tables were not changed.");
+        }
+
+        $expectedParts = (int) DB::scalar(
+            'SELECT COUNT(*) FROM (
+                SELECT m.keep_id, p.partnumber
+                FROM parts p INNER JOIN cbp_binary_map m ON m.old_id = p.binaries_id
+                GROUP BY m.keep_id, p.partnumber
+             ) retained_parts'
+        );
+        $retainedParts = (int) DB::table('parts_cbp_new')->count();
+        if ($retainedParts !== $expectedParts) {
+            throw new \RuntimeException("Parts shadow is incomplete ({$retainedParts}/{$expectedParts}); the source tables were not changed.");
+        }
+    }
+
+    private function swapParts(): void
+    {
+        if (! Schema::hasTable('parts_cbp_new') && Schema::hasTable('parts_cbp_pre_optimize')) {
+            return;
+        }
+        if (! Schema::hasTable('parts_cbp_new')) {
+            throw new \RuntimeException('parts_cbp_new is missing; rerun the parts shadow step.');
+        }
+        $this->dropForeignKeyIfExists('parts', 'FK_binaries');
+        DB::statement('DROP TABLE IF EXISTS parts_cbp_pre_optimize');
+        DB::statement('RENAME TABLE parts TO parts_cbp_pre_optimize, parts_cbp_new TO parts');
+    }
+
+    private function mergeBinaries(): void
+    {
+        if (! Schema::hasColumn('binaries', 'cbp_hash')) {
+            return;
+        }
+        DB::statement(
+            'UPDATE binaries keep_binary
+             INNER JOIN (
+                SELECT m.keep_id, MAX(b.totalparts) AS totalparts
+                FROM cbp_binary_map m INNER JOIN binaries b ON b.id = m.old_id
+                GROUP BY m.keep_id
+             ) totals ON totals.keep_id = keep_binary.id
+             SET keep_binary.totalparts = totals.totalparts'
+        );
+        DB::statement(
+            'DELETE b FROM binaries b INNER JOIN cbp_binary_map m ON m.old_id = b.id WHERE m.old_id <> m.keep_id'
+        );
+        DB::statement('UPDATE binaries SET binaryhash = cbp_hash');
+
+        $this->dropIndexIfExists('binaries', 'ux_collection_id_filenumber');
+        $this->dropIndexIfExists('binaries', 'ix_binaries_binaryhash');
+        $this->dropIndexIfExists('binaries', 'ix_binaries_collection_hash');
+        $this->dropIndexIfExists('binaries', 'ux_binaries_collection_hash');
+        DB::statement('ALTER TABLE binaries MODIFY binaryhash BINARY(16) NOT NULL');
+        DB::statement('ALTER TABLE binaries DROP COLUMN cbp_hash');
+        DB::statement('CREATE UNIQUE INDEX ux_binaries_collection_hash ON binaries (collections_id, binaryhash)');
+        if (! $this->indexExists('binaries', 'ix_binaries_collection_filenumber')) {
+            DB::statement('CREATE INDEX ix_binaries_collection_filenumber ON binaries (collections_id, filenumber)');
+        }
+    }
+
+    private function convertCollectionHash(): void
+    {
+        if (! Schema::hasColumn('collections', 'cbp_hash')) {
+            return;
+        }
+        DB::statement(
+            "UPDATE collections SET cbp_hash = CASE
+                WHEN OCTET_LENGTH(collectionhash) = 20 THEN CAST(collectionhash AS BINARY)
+                WHEN OCTET_LENGTH(collectionhash) = 40 AND collectionhash REGEXP '^[0-9A-Fa-f]{40}$' THEN UNHEX(collectionhash)
+                ELSE UNHEX(SHA1(collectionhash)) END
+             WHERE cbp_hash IS NULL"
+        );
+        $this->dropIndexIfExists('collections', 'ix_collection_collectionhash');
+        DB::statement('ALTER TABLE collections DROP COLUMN collectionhash');
+        DB::statement('ALTER TABLE collections CHANGE cbp_hash collectionhash BINARY(20) NOT NULL');
+        DB::statement('CREATE UNIQUE INDEX ix_collection_collectionhash ON collections (collectionhash)');
+    }
+
+    private function installConstraintsAndIndexes(): void
+    {
+        if (! $this->indexExists('collections', 'ix_collections_group_filecheck_seen_id')) {
+            DB::statement('CREATE INDEX ix_collections_group_filecheck_seen_id ON collections (groups_id, filecheck, last_seen_at, id)');
+        }
+        $this->dropForeignKeyIfExists('collection_groups', 'fk_collection_groups_collection');
+        DB::statement(
+            'ALTER TABLE collection_groups ADD CONSTRAINT fk_collection_groups_collection
+             FOREIGN KEY (collections_id) REFERENCES collections(id) ON DELETE CASCADE ON UPDATE CASCADE'
+        );
+        $this->dropForeignKeyIfExists('parts', 'FK_binaries');
+        DB::statement(
+            'ALTER TABLE parts ADD CONSTRAINT FK_binaries
+             FOREIGN KEY (binaries_id) REFERENCES binaries(id) ON DELETE CASCADE ON UPDATE CASCADE'
+        );
+    }
+
+    private function rebuildAggregates(int $batchSize): void
+    {
+        $lastId = 0;
+        do {
+            $ids = DB::table('binaries')->where('id', '>', $lastId)->orderBy('id')->limit($batchSize)->pluck('id')->map(static fn ($id): int => (int) $id)->all();
+            if ($ids === []) {
+                break;
+            }
+            $lastId = (int) end($ids);
+            $placeholders = implode(',', array_fill(0, \count($ids), '?'));
+            DB::update(
+                "UPDATE binaries b LEFT JOIN (
+                    SELECT binaries_id, COUNT(*) currentparts, COALESCE(SUM(size), 0) partsize
+                    FROM parts WHERE binaries_id IN ({$placeholders}) GROUP BY binaries_id
+                 ) p ON p.binaries_id = b.id
+                 SET b.currentparts = COALESCE(p.currentparts, 0),
+                     b.partsize = COALESCE(p.partsize, 0),
+                     b.partcheck = CASE WHEN COALESCE(p.currentparts, 0) >= b.totalparts THEN 1 ELSE 0 END
+                 WHERE b.id IN ({$placeholders})",
+                [...$ids, ...$ids]
+            );
+        } while (\count($ids) === $batchSize);
+
+        DB::update(
+            'UPDATE collections c LEFT JOIN (
+                SELECT collections_id, COALESCE(SUM(partsize), 0) filesize FROM binaries GROUP BY collections_id
+             ) b ON b.collections_id = c.id SET c.filesize = COALESCE(b.filesize, 0)'
+        );
+    }
+
+    private function finalizeOptimization(): void
+    {
+        DB::statement('DROP TABLE IF EXISTS parts_cbp_pre_optimize');
+        DB::statement('DROP TABLE IF EXISTS cbp_binary_map');
+        DB::statement('ANALYZE TABLE collections, binaries, parts, collection_groups');
+    }
+
+    private function normalizeIdentity(string $value): string
+    {
+        return mb_strtolower(trim(preg_replace('/\s+/u', ' ', Utf8::clean($value)) ?? ''));
+    }
+
+    private function indexExists(string $table, string $index): bool
+    {
+        return DB::select("SHOW INDEX FROM `{$table}` WHERE Key_name = ?", [$index]) !== [];
+    }
+
+    private function dropIndexIfExists(string $table, string $index): void
+    {
+        if ($this->indexExists($table, $index)) {
+            DB::statement("ALTER TABLE `{$table}` DROP INDEX `{$index}`");
+        }
+    }
+
+    private function dropForeignKeyIfExists(string $table, string $constraint): void
+    {
+        $exists = DB::select(
+            'SELECT CONSTRAINT_NAME FROM information_schema.TABLE_CONSTRAINTS
+             WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = ?
+               AND CONSTRAINT_NAME = ? AND CONSTRAINT_TYPE = \'FOREIGN KEY\'',
+            [$table, $constraint]
+        );
+        if ($exists !== []) {
+            DB::statement("ALTER TABLE `{$table}` DROP FOREIGN KEY `{$constraint}`");
+        }
+    }
+}

app/Models/Collection.php

@@ -16,7 +16,7 @@ use Illuminate\Database\Eloquent\Model;
  * @property string $xref
  * @property int $totalfiles
  * @property int $groups_id
- * @property string $collectionhash
+ * @property string $collectionhash Raw 20-byte SHA-1 digest
  * @property int $collection_regexes_id FK to collection_regexes.id
  * @property string|null $dateadded
  * @property string $added

app/Services/Binaries/BinariesConfig.php

@@ -21,20 +21,17 @@ final readonly class BinariesConfig
         public int $newGroupDaysToScan = 3,
         public int $partRepairLimit = 15000,
         public int $partRepairMaxTries = 3,
-        public int $partsChunkSize = 5000,
-        public int $binariesUpdateChunkSize = 1000,
         public bool $echoCli = false,
         // Number of headers processed (and bulk-inserted) at a time inside
         // HeaderStorageService. This MUST stay small because each chunk
         // produces multi-row INSERTs/SELECTs whose binding count and SQL size
-        // grow linearly with the value. Defaulting it to partsChunkSize
-        // (which used to only control single-row part flushes) caused MySQL
+        // grow linearly with the value. Large unbounded chunks caused MySQL
         // and PHP to allocate hundreds of MB per scan and run out of RAM.
         public int $headerChunkSize = 500,
-        // Hard upper bound applied internally to bulk SELECT/INSERT/UPDATE
-        // operations regardless of caller-provided chunk size, so a
-        // misconfiguration cannot blow up server memory.
-        public int $bulkSqlChunkSize = 500,
+        // One shared hard upper bound for every bulk SELECT/INSERT/UPDATE.
+        public int $sqlChunkSize = 500,
+        public int $reconcileBatchSize = 500,
+        public int $nzbStreamRows = 5000,
     ) {}
 
     /**
@@ -51,11 +48,11 @@ final readonly class BinariesConfig
             newGroupDaysToScan: self::getSettingInt('newgroupdaystoscan', 3),
             partRepairLimit: self::getSettingInt('maxpartrepair', 15000),
             partRepairMaxTries: self::getSettingInt('partrepairmaxtries', 3),
-            partsChunkSize: max(100, (int) config('nntmux.parts_chunk_size', 5000)),
-            binariesUpdateChunkSize: max(100, min(1000, (int) config('nntmux.binaries_update_chunk_size', 1000))),
             echoCli: (bool) config('nntmux.echocli'),
-            headerChunkSize: max(50, min(2000, (int) config('nntmux.header_chunk_size', 500))),
-            bulkSqlChunkSize: max(50, min(1000, (int) config('nntmux.bulk_sql_chunk_size', 500))),
+            headerChunkSize: max(50, min(2000, (int) config('nntmux.cbp.header_chunk_size', 500))),
+            sqlChunkSize: max(50, min(1000, (int) config('nntmux.cbp.sql_chunk_size', 500))),
+            reconcileBatchSize: max(50, min(2000, (int) config('nntmux.cbp.reconcile_batch_size', 500))),
+            nzbStreamRows: max(500, min(20000, (int) config('nntmux.cbp.nzb_stream_rows', 5000))),
         );
     }
 

app/Services/Binaries/BinariesService.php

@@ -62,9 +62,7 @@ class BinariesService
 
     private int $headersBlackListed = 0;
 
-    /**
-     * @var array<int, int|string>
-     */
+    /** @var array<int, true> */
     private array $headersReceived = [];
 
     public function __construct(
@@ -80,7 +78,7 @@ class BinariesService
         $this->missedPartHandler = $missedPartHandler ?? new MissedPartHandler(
             $this->config->partRepairLimit,
             $this->config->partRepairMaxTries,
-            $this->config->bulkSqlChunkSize
+            $this->config->sqlChunkSize
         );
         $this->nntp = $nntp;
         $this->startUpdate = Carbon::now();
@@ -297,47 +295,68 @@ class BinariesService
         // Extract article range info
         $returnArray = $this->headerParser->getArticleRange($headers);
 
-        // Parse and filter headers
-        $this->headerParser->reset();
-        $parseResult = $this->headerParser->parse($headers, $groupMySQL['name'], $partRepair, $missingParts);
+        // Parse and store one bounded chunk at a time. The previous flow built
+        // a second full-size parsed header array before storage, doubling peak
+        // memory for large XOVER responses.
+        $this->startUpdate = Carbon::now();  // Reset before storage begins
+        $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true);
 
-        $this->headersReceived = $parseResult['received'] ?? [];
-        $this->notYEnc = $parseResult['notYEnc'];
-        $this->headersBlackListed = $parseResult['blacklisted'];
+        $headersNotInserted = [];
+        $repairedNumbers = [];
+        $missingPartSet = $missingParts === null
+            ? null
+            : array_fill_keys(array_map('intval', $missingParts), true);
+        $totalHeaders = \count($headers);
+        for ($offset = 0; $offset < $totalHeaders; $offset += $this->config->headerChunkSize) {
+            $rawChunk = array_slice($headers, $offset, $this->config->headerChunkSize);
+            $this->headerParser->reset();
+            $parseResult = $this->headerParser->parse($rawChunk, $groupMySQL['name'], $partRepair, $missingPartSet);
 
-        // Update blacklist last_activity
-        $this->headerParser->flushBlacklistUpdates();
+            foreach ($parseResult['received'] ?? [] as $number) {
+                $this->headersReceived[(int) $number] = true;
+            }
+            foreach ($parseResult['repaired'] ?? [] as $number) {
+                $repairedNumbers[(int) $number] = true;
+            }
+            $this->notYEnc += (int) $parseResult['notYEnc'];
+            $this->headersBlackListed += (int) $parseResult['blacklisted'];
+            $this->headerParser->flushBlacklistUpdates();
 
+            if ($parseResult['headers'] !== []) {
+                try {
+                    $headersNotInserted = array_merge(
+                        $headersNotInserted,
+                        $this->headerStorage->store($parseResult['headers'], $groupMySQL, $addToPartRepair)
+                    );
+                } catch (\Throwable $e) {
+                    $this->logError('storeHeaders failed: '.$e->getMessage());
+                    foreach ($parseResult['headers'] as $failedHeader) {
+                        if (isset($failedHeader['Number'])) {
+                            $headersNotInserted[] = $failedHeader['Number'];
+                        }
+                    }
+                }
+            }
+
+            unset($rawChunk, $parseResult);
+        }
         unset($headers);
 
         if ($this->config->echoCli && ! $partRepair) {
             $this->outputHeaderInitial();
         }
 
-        // Store headers
-        $this->startUpdate = Carbon::now();  // Reset before storage begins
-        $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true);
-
-        $headersNotInserted = [];
-        if (! empty($parseResult['headers'])) {
-            try {
-                $headersNotInserted = $this->headerStorage->store($parseResult['headers'], $groupMySQL, $addToPartRepair);
-            } catch (\Throwable $e) {
-                $this->logError('storeHeaders failed: '.$e->getMessage());
-            }
-        }
-
         $this->startPR = Carbon::now();
         $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true);
 
         // Handle repaired parts
-        if ($partRepair && ! empty($parseResult['repaired'])) {
-            $this->missedPartHandler->removeRepairedParts($parseResult['repaired'], $groupMySQL['id']);
+        if ($partRepair && $repairedNumbers !== []) {
+            $this->missedPartHandler->removeRepairedParts(array_keys($repairedNumbers), $groupMySQL['id']);
         }
 
         // Handle part repair tracking
         if ($addToPartRepair) {
-            $this->handlePartRepairTracking($headersNotInserted, $parseResult['headers']);
+            $this->handlePartRepairTracking($headersNotInserted);
         }
 
         $this->outputHeaderDuration();
@@ -737,9 +756,8 @@ class BinariesService
 
     /**
      * @param  array<int, int|string>  $headersNotInserted
-     * @param  array<int, array<string, mixed>>  $parsedHeaders
      */
-    private function handlePartRepairTracking(array $headersNotInserted, array $parsedHeaders): void
+    private function handlePartRepairTracking(array $headersNotInserted): void
     {
         $notInsertedCount = \count($headersNotInserted);
         if ($notInsertedCount > 0) {
@@ -747,21 +765,29 @@ class BinariesService
             $this->log($notInsertedCount.' articles failed to insert!', __FUNCTION__, 'warning');
         }
 
-        // Check for missing headers in range
-        $expectedCount = $this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1;
-        if ($expectedCount > \count($this->headersReceived)) {
-            $rangeNotReceived = array_values(array_diff(range($this->first, $this->last), $this->headersReceived));
-            $notReceivedCount = \count($rangeNotReceived);
-
-            if ($notReceivedCount > 0) {
-                $this->missedPartHandler->addMissingParts($rangeNotReceived, $this->groupMySQL['id']);
-
-                if ($this->config->echoCli) {
-                    cli()->alternate(
-                        'Server did not return '.$notReceivedCount.' articles from '.$this->groupMySQL['name'].'.'
-                    );
-                }
+        // Find missing article numbers without materialising range($first, $last)
+        // and a second array_diff() copy. Flush bounded batches directly.
+        $missing = [];
+        $notReceivedCount = 0;
+        for ($number = $this->first; $number <= $this->last; $number++) {
+            if (isset($this->headersReceived[$number])) {
+                continue;
             }
+            $missing[] = $number;
+            $notReceivedCount++;
+            if (\count($missing) >= $this->config->sqlChunkSize) {
+                $this->missedPartHandler->addMissingParts($missing, $this->groupMySQL['id']);
+                $missing = [];
+            }
+        }
+        if ($missing !== []) {
+            $this->missedPartHandler->addMissingParts($missing, $this->groupMySQL['id']);
+        }
+
+        if ($notReceivedCount > 0 && $this->config->echoCli) {
+            cli()->alternate(
+                'Server did not return '.$notReceivedCount.' articles from '.$this->groupMySQL['name'].'.'
+            );
         }
     }
 

app/Services/Binaries/BinaryHandler.php

@@ -19,10 +19,7 @@ final class BinaryHandler
      * generated SQL string and PDO parameter list bounded regardless of
      * how many headers the caller passes in.
      */
-    private const MAX_SQL_ROWS_PER_STATEMENT = 500;
-
-    /** @var array<int, array{Size: int, Parts: int}> Pending binary updates */
-    private array $binariesUpdate = [];
+    private int $sqlChunkSize;
 
     /** @var array<int, true> IDs of binaries created in this batch */
     private array $insertedBinaryIds = [];
@@ -30,16 +27,21 @@ final class BinaryHandler
     /** @var array<string, array{CollectionID: int, BinaryID: int}> Processed articles */
     private array $articles = [];
 
-    public function __construct() {}
+    private ?\Throwable $lastException = null;
+
+    public function __construct(int $sqlChunkSize = 500)
+    {
+        $this->sqlChunkSize = max(1, min(1000, $sqlChunkSize));
+    }
 
     /**
      * Reset state for a new batch.
      */
     public function reset(): void
     {
-        $this->binariesUpdate = [];
         $this->insertedBinaryIds = [];
         $this->articles = [];
+        $this->lastException = null;
     }
 
     /**
@@ -54,18 +56,14 @@ final class BinaryHandler
         int $groupId,
         int $fileNumber
     ): ?int {
-        $articleKey = $this->articleKey($collectionId, $header['matches'][1]);
+        $hash = $this->identityHash($header, $fileNumber);
+        $articleKey = $this->binaryLookupKey($hash, $collectionId);
 
         // Return cached if already processed
         if (isset($this->articles[$articleKey])) {
-            $binaryId = $this->articles[$articleKey]['BinaryID'];
-            $this->binariesUpdate[$binaryId]['Size'] += $header['Bytes'];
-            $this->binariesUpdate[$binaryId]['Parts']++;
-
-            return $binaryId;
+            return $this->articles[$articleKey]['BinaryID'];
         }
 
-        $hash = md5((string) ($header['matches'][1] ?? '').(string) ($header['From'] ?? '').(string) $groupId);
         $driver = DB::getDriverName();
 
         try {
@@ -78,7 +76,6 @@ final class BinaryHandler
             );
 
             if ($binaryId > 0) {
-                $this->binariesUpdate[$binaryId] = ['Size' => 0, 'Parts' => 0];
                 $this->articles[$articleKey] = [
                     'CollectionID' => $collectionId,
                     'BinaryID' => $binaryId,
@@ -87,6 +84,7 @@ final class BinaryHandler
                 return $binaryId;
             }
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Binary insert failed: '.$e->getMessage());
             }
@@ -106,18 +104,15 @@ final class BinaryHandler
         $resolved = [];
         $pending = [];
         $indexesByArticleKey = [];
-        $extraUpdatesByArticleKey = [];
-
         foreach ($records as $index => $record) {
             $header = $record['header'];
             $collectionId = (int) $record['collection_id'];
             $fileNumber = (int) $record['file_number'];
-            $articleKey = $this->articleKey($collectionId, $header['matches'][1]);
+            $hash = $this->identityHash($header, $fileNumber);
+            $articleKey = $this->binaryLookupKey($hash, $collectionId);
 
             if (isset($this->articles[$articleKey])) {
                 $binaryId = $this->articles[$articleKey]['BinaryID'];
-                $this->binariesUpdate[$binaryId]['Size'] += (int) $header['Bytes'];
-                $this->binariesUpdate[$binaryId]['Parts']++;
                 $resolved[$index] = $binaryId;
 
                 continue;
@@ -125,20 +120,15 @@ final class BinaryHandler
 
             $indexesByArticleKey[$articleKey][] = $index;
             if (isset($pending[$articleKey])) {
-                $extraUpdatesByArticleKey[$articleKey]['Size'] = ($extraUpdatesByArticleKey[$articleKey]['Size'] ?? 0) + (int) $header['Bytes'];
-                $extraUpdatesByArticleKey[$articleKey]['Parts'] = ($extraUpdatesByArticleKey[$articleKey]['Parts'] ?? 0) + 1;
-
                 continue;
             }
 
-            $hash = md5((string) ($header['matches'][1] ?? '').(string) ($header['From'] ?? '').(string) $groupId);
             $pending[$articleKey] = [
                 'hash' => $hash,
                 'name' => Utf8::clean($header['matches'][1]),
                 'collections_id' => $collectionId,
                 'totalparts' => (int) $header['matches'][3],
                 'filenumber' => $fileNumber,
-                'partsize' => (int) $header['Bytes'],
             ];
         }
 
@@ -149,8 +139,6 @@ final class BinaryHandler
         try {
             $result = $this->bulkInsertAndResolve($pending);
             $idsByKey = $result['ids'];
-            $existingKeys = $result['existing'];
-            $driver = DB::getDriverName();
             foreach ($pending as $articleKey => $row) {
                 $lookupKey = $this->binaryLookupKey($row['hash'], (int) $row['collections_id']);
                 $binaryId = $idsByKey[$lookupKey] ?? 0;
@@ -158,16 +146,6 @@ final class BinaryHandler
                     continue;
                 }
 
-                $this->binariesUpdate[$binaryId] = $this->binariesUpdate[$binaryId] ?? ['Size' => 0, 'Parts' => 0];
-                if ($driver === 'sqlite' && isset($existingKeys[$lookupKey])) {
-                    $this->binariesUpdate[$binaryId]['Size'] += (int) $row['partsize'];
-                    $this->binariesUpdate[$binaryId]['Parts']++;
-                }
-                if (isset($extraUpdatesByArticleKey[$articleKey])) {
-                    $this->binariesUpdate[$binaryId]['Size'] += $extraUpdatesByArticleKey[$articleKey]['Size'];
-                    $this->binariesUpdate[$binaryId]['Parts'] += $extraUpdatesByArticleKey[$articleKey]['Parts'];
-                }
-
                 $this->articles[$articleKey] = [
                     'CollectionID' => (int) $row['collections_id'],
                     'BinaryID' => $binaryId,
@@ -178,6 +156,7 @@ final class BinaryHandler
                 }
             }
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Bulk binary insert failed: '.$e->getMessage());
             }
@@ -251,7 +230,7 @@ final class BinaryHandler
     /** @param  list<array<string, mixed>>  $rows */
     private function bulkInsertBinariesSqlite(array $rows): void
     {
-        foreach (array_chunk($rows, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk($rows, $this->sqlChunkSize) as $chunk) {
             $insertRows = [];
             foreach ($chunk as $row) {
                 $insertRows[] = [
@@ -259,9 +238,9 @@ final class BinaryHandler
                     'name' => $row['name'],
                     'collections_id' => $row['collections_id'],
                     'totalparts' => $row['totalparts'],
-                    'currentparts' => 1,
+                    'currentparts' => 0,
                     'filenumber' => $row['filenumber'],
-                    'partsize' => $row['partsize'],
+                    'partsize' => 0,
                 ];
             }
 
@@ -272,26 +251,25 @@ final class BinaryHandler
     /** @param  list<array<string, mixed>>  $rows */
     private function bulkInsertBinariesMysql(array $rows): void
     {
-        foreach (array_chunk($rows, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk($rows, $this->sqlChunkSize) as $chunk) {
             $placeholders = [];
             $bindings = [];
             foreach ($chunk as $row) {
-                $placeholders[] = '(UNHEX(?), ?, ?, ?, 1, ?, ?)';
+                $placeholders[] = '(UNHEX(?), ?, ?, ?, 0, ?, 0)';
                 array_push(
                     $bindings,
                     $row['hash'],
                     $row['name'],
                     $row['collections_id'],
                     $row['totalparts'],
-                    $row['filenumber'],
-                    $row['partsize']
+                    $row['filenumber']
                 );
             }
 
             DB::statement(
                 'INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) VALUES '
                 .implode(',', $placeholders)
-                .' ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + VALUES(partsize)',
+                .' ON DUPLICATE KEY UPDATE totalparts = GREATEST(totalparts, VALUES(totalparts)), id = LAST_INSERT_ID(id)',
                 $bindings
             );
         }
@@ -323,7 +301,7 @@ final class BinaryHandler
         $results = [];
         foreach ($rowsByCollection as $collectionId => $hashes) {
             $hashes = array_values(array_unique($hashes));
-            foreach (array_chunk($hashes, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+            foreach (array_chunk($hashes, $this->sqlChunkSize) as $chunk) {
                 $placeholders = implode(',', array_fill(0, \count($chunk), $driver === 'sqlite' ? '?' : 'UNHEX(?)'));
                 $bindings = $chunk;
                 $bindings[] = $collectionId;
@@ -360,13 +338,11 @@ final class BinaryHandler
     ): int {
         $name = Utf8::clean($header['matches'][1]);
         $totalParts = (int) $header['matches'][3];
-        $partSize = (int) $header['Bytes'];
-
         if ($driver === 'sqlite') {
-            return $this->insertBinarySqlite($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize);
+            return $this->insertBinarySqlite($hash, $name, $collectionId, $totalParts, $fileNumber);
         }
 
-        return $this->insertBinaryMysql($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize);
+        return $this->insertBinaryMysql($hash, $name, $collectionId, $totalParts, $fileNumber);
     }
 
     private function insertBinarySqlite(
@@ -374,17 +350,16 @@ final class BinaryHandler
         string $name,
         int $collectionId,
         int $totalParts,
-        int $fileNumber,
-        int $partSize
+        int $fileNumber
     ): int {
         $affected = DB::table('binaries')->insertOrIgnore([
             'binaryhash' => $hash,
             'name' => $name,
             'collections_id' => $collectionId,
             'totalparts' => $totalParts,
-            'currentparts' => 1,
+            'currentparts' => 0,
             'filenumber' => $fileNumber,
-            'partsize' => $partSize,
+            'partsize' => 0,
         ]);
 
         if ($affected > 0 && ($lastId = (int) DB::connection()->getPdo()->lastInsertId()) > 0) {
@@ -406,16 +381,15 @@ final class BinaryHandler
         string $name,
         int $collectionId,
         int $totalParts,
-        int $fileNumber,
-        int $partSize
+        int $fileNumber
     ): int {
 
         $sql = 'INSERT INTO binaries '
             .'(binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) '
-            .'VALUES (UNHEX(?), ?, ?, ?, 1, ?, ?) '
-            .'ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + VALUES(partsize)';
+            .'VALUES (UNHEX(?), ?, ?, ?, 0, ?, 0) '
+            .'ON DUPLICATE KEY UPDATE totalparts = GREATEST(totalparts, VALUES(totalparts)), id = LAST_INSERT_ID(id)';
 
-        DB::statement($sql, [$hash, $name, $collectionId, $totalParts, $fileNumber, $partSize]);
+        DB::statement($sql, [$hash, $name, $collectionId, $totalParts, $fileNumber]);
 
         $lastId = (int) DB::connection()->getPdo()->lastInsertId();
         if ($lastId > 0) {
@@ -433,24 +407,56 @@ final class BinaryHandler
     }
 
     /**
-     * Flush accumulated size/parts updates to the database.
+     * Rebuild aggregates from stored parts for the binaries touched by a chunk.
+     * This is deliberately idempotent: ignored duplicate part inserts cannot
+     * inflate currentparts or partsize.
+     *
+     * @param  list<int>  $binaryIds
      */
-    public function flushUpdates(int $chunkSize = 1000): bool
+    public function refreshAggregates(array $binaryIds, int $chunkSize = 500): bool
     {
-        $updates = $this->getPendingUpdates();
-        if (empty($updates)) {
+        $binaryIds = array_values(array_unique(array_map('intval', $binaryIds)));
+        if ($binaryIds === []) {
             return true;
         }
 
-        $driver = DB::getDriverName();
-
         try {
-            if ($driver === 'sqlite') {
-                return $this->flushUpdatesSqlite($updates); // @phpstan-ignore argument.type
+            if (DB::getDriverName() === 'sqlite') {
+                foreach ($binaryIds as $binaryId) {
+                    $aggregate = DB::selectOne(
+                        'SELECT COUNT(*) AS currentparts, COALESCE(SUM(size), 0) AS partsize FROM parts WHERE binaries_id = ?',
+                        [$binaryId]
+                    );
+                    DB::update(
+                        'UPDATE binaries SET currentparts = ?, partsize = ?, partcheck = CASE WHEN ? >= totalparts THEN 1 ELSE 0 END WHERE id = ?',
+                        [(int) $aggregate->currentparts, (int) $aggregate->partsize, (int) $aggregate->currentparts, $binaryId]
+                    );
+                }
+
+                return true;
             }
 
-            return $this->flushUpdatesMysql($updates, $chunkSize); // @phpstan-ignore argument.type
+            $chunkSize = max(1, min($chunkSize, $this->sqlChunkSize));
+            foreach (array_chunk($binaryIds, $chunkSize) as $chunk) {
+                $placeholders = implode(',', array_fill(0, \count($chunk), '?'));
+                DB::update(
+                    "UPDATE binaries b
+                     INNER JOIN (
+                         SELECT p.binaries_id, COUNT(*) AS currentparts, COALESCE(SUM(p.size), 0) AS partsize
+                         FROM parts p
+                         WHERE p.binaries_id IN ({$placeholders})
+                         GROUP BY p.binaries_id
+                     ) a ON a.binaries_id = b.id
+                     SET b.currentparts = a.currentparts,
+                         b.partsize = a.partsize,
+                         b.partcheck = CASE WHEN a.currentparts >= b.totalparts THEN 1 ELSE 0 END",
+                    $chunk
+                );
+            }
+
+            return true;
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Binaries aggregate update failed: '.$e->getMessage());
             }
@@ -459,54 +465,6 @@ final class BinaryHandler
         }
     }
 
-    /**
-     * @param  array<string, mixed>  $updates
-     */
-    private function flushUpdatesSqlite(array $updates): bool
-    {
-        foreach ($updates as $row) {
-            DB::statement(
-                'UPDATE binaries SET partsize = partsize + ?, currentparts = currentparts + ? WHERE id = ?',
-                [$row['partsize'], $row['currentparts'], $row['id']]
-            );
-        }
-
-        return true;
-    }
-
-    /**
-     * @param  array<string, mixed>  $updates
-     */
-    private function flushUpdatesMysql(array $updates, int $chunkSize): bool
-    {
-        // Clamp the caller-provided chunk size to our hard upper bound so an
-        // unusually large config value cannot produce a 100 KB+ UNION ALL
-        // query that exhausts memory on either side of the connection.
-        $chunkSize = max(1, min($chunkSize, self::MAX_SQL_ROWS_PER_STATEMENT));
-
-        foreach (array_chunk($updates, $chunkSize) as $chunk) {
-            $selects = [];
-            $bindings = [];
-
-            foreach ($chunk as $row) {
-                $selects[] = 'SELECT ? AS id, ? AS partsize, ? AS currentparts';
-                $bindings[] = $row['id'];
-                $bindings[] = $row['partsize'];
-                $bindings[] = $row['currentparts'];
-            }
-
-            $sql = 'UPDATE binaries b INNER JOIN ('
-                .implode(' UNION ALL ', $selects)
-                .') u ON u.id = b.id '
-                .'SET b.partsize = b.partsize + u.partsize, '
-                .'b.currentparts = b.currentparts + u.currentparts';
-
-            DB::statement($sql, $bindings);
-        }
-
-        return true;
-    }
-
     /**
      * Check if article is already processed.
      */
@@ -515,11 +473,6 @@ final class BinaryHandler
         return isset($this->articles[$articleKey]);
     }
 
-    private function articleKey(int $collectionId, string $name): string
-    {
-        return $collectionId.':'.$name;
-    }
-
     /**
      * Get IDs created in this batch.
      *
@@ -530,24 +483,21 @@ final class BinaryHandler
         return array_keys($this->insertedBinaryIds);
     }
 
-    /**
-     * Get pending binary updates that haven't been flushed.
-     *
-     * @return list<array<string, int>>
-     */
-    private function getPendingUpdates(): array
+    public function getLastException(): ?\Throwable
     {
-        $rows = [];
-        foreach ($this->binariesUpdate as $binaryId => $binary) {
-            if (($binary['Size'] ?? 0) > 0 || ($binary['Parts'] ?? 0) > 0) {
-                $rows[] = [
-                    'id' => $binaryId,
-                    'partsize' => $binary['Size'],
-                    'currentparts' => $binary['Parts'],
-                ];
-            }
+        return $this->lastException;
+    }
+
+    /** @param array<string, mixed> $header */
+    private function identityHash(array $header, int $fileNumber): string
+    {
+        if ($fileNumber > 0) {
+            return md5('file:'.$fileNumber);
         }
 
-        return $rows;
+        $subject = preg_replace('/\s+/u', ' ', Utf8::clean((string) ($header['matches'][1] ?? ''))) ?? '';
+        $poster = preg_replace('/\s+/u', ' ', Utf8::clean((string) ($header['From'] ?? ''))) ?? '';
+
+        return md5('subject:'.mb_strtolower(trim($subject))."\0poster:".mb_strtolower(trim($poster)));
     }
 }

app/Services/Binaries/CollectionHandler.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 
 namespace App\Services\Binaries;
 
+use App\Enums\CollectionFileCheckStatus;
 use App\Models\Collection;
 use App\Services\CollectionsCleaningService;
 use App\Services\XrefService;
@@ -21,7 +22,7 @@ final class CollectionHandler
      * (multi-row INSERT, IN(...) lookup, etc.). Keeps the generated SQL
      * and PDO parameter count bounded regardless of caller batch size.
      */
-    private const MAX_SQL_ROWS_PER_STATEMENT = 500;
+    private int $sqlChunkSize;
 
     private CollectionsCleaningService $collectionsCleaning;
 
@@ -36,18 +37,19 @@ final class CollectionHandler
     /** @var array<string, true> Collection hashes touched in this batch */
     private array $batchCollectionHashes = [];
 
-    /** @var array<string, string|null> Cached collection xrefs by collection key */
-    private array $existingXrefs = [];
-
     /** @var array<string, int> Cached collection IDs by collectionhash (populated by bulk prefetch) */
     private array $existingIdsByHash = [];
 
+    private ?\Throwable $lastException = null;
+
     public function __construct(
         ?CollectionsCleaningService $collectionsCleaning = null,
-        ?XrefService $xrefService = null
+        ?XrefService $xrefService = null,
+        int $sqlChunkSize = 500,
     ) {
         $this->collectionsCleaning = $collectionsCleaning ?? new CollectionsCleaningService;
         $this->xrefService = $xrefService ?? new XrefService;
+        $this->sqlChunkSize = max(1, min(1000, $sqlChunkSize));
     }
 
     /**
@@ -58,8 +60,8 @@ final class CollectionHandler
         $this->collectionIds = [];
         $this->insertedCollectionIds = [];
         $this->batchCollectionHashes = [];
-        $this->existingXrefs = [];
         $this->existingIdsByHash = [];
+        $this->lastException = null;
     }
 
     /**
@@ -87,20 +89,14 @@ final class CollectionHandler
             return $this->collectionIds[$collectionKey];
         }
 
-        $collectionHash = sha1($collectionKey);
+        $collectionHash = sha1($collectionKey, true);
         $this->batchCollectionHashes[$collectionHash] = true;
 
         $headerDate = is_numeric($header['Date']) ? (int) $header['Date'] : strtotime($header['Date']);
         $now = now()->timestamp;
         $unixtime = min($headerDate, $now) ?: $now;
 
-        if (! array_key_exists($collectionKey, $this->existingXrefs)) {
-            $this->existingXrefs[$collectionKey] = Collection::whereCollectionhash($collectionHash)->value('xref');
-        }
-        $existingXref = $this->existingXrefs[$collectionKey];
         $headerTokens = $this->xrefService->extractTokens($header['Xref'] ?? '');
-        $newTokens = $this->xrefService->diffNewTokens($existingXref, $header['Xref'] ?? '');
-        $finalXrefAppend = implode(' ', $newTokens);
 
         $subject = substr(Utf8::clean($header['matches'][1]), 0, 255);
         $fromName = Utf8::clean($header['From']);
@@ -114,7 +110,6 @@ final class CollectionHandler
                 $fromName,
                 $unixtime,
                 $headerTokens,
-                $finalXrefAppend,
                 $groupId,
                 $totalFiles,
                 $collectionHash,
@@ -124,10 +119,17 @@ final class CollectionHandler
 
             if ($collectionId > 0) {
                 $this->collectionIds[$collectionKey] = $collectionId;
+                $this->insertCollectionGroups([
+                    $collectionKey => array_fill_keys(
+                        $this->xrefService->extractGroupNames($header['Xref'] ?? '') ?: [$groupName],
+                        true
+                    ),
+                ]);
 
                 return $collectionId;
             }
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Collection insert failed: '.$e->getMessage());
             }
@@ -153,7 +155,8 @@ final class CollectionHandler
         $resolved = [];
         $pending = [];
         $indexByCollectionKey = [];
-        $xrefsToPrefetch = [];
+        $hashesToPrefetch = [];
+        $groupNamesByCollectionKey = [];
 
         foreach ($headers as $index => $header) {
             $totalFiles = (int) ($totalFilesByIndex[$index] ?? 0);
@@ -163,6 +166,14 @@ final class CollectionHandler
             );
 
             $collectionKey = $collMatch['name'].$totalFiles;
+            $groupNames = $this->xrefService->extractGroupNames($header['Xref'] ?? '');
+            if ($groupNames === []) {
+                $groupNames = [$groupName];
+            }
+            foreach ($groupNames as $xrefGroupName) {
+                $groupNamesByCollectionKey[$collectionKey][$xrefGroupName] = true;
+            }
+
             if (isset($this->collectionIds[$collectionKey])) {
                 $resolved[$index] = $this->collectionIds[$collectionKey];
 
@@ -174,10 +185,10 @@ final class CollectionHandler
                 continue;
             }
 
-            $collectionHash = sha1($collectionKey);
+            $collectionHash = sha1($collectionKey, true);
             $this->batchCollectionHashes[$collectionHash] = true;
 
-            $xrefsToPrefetch[$collectionKey] = $collectionHash;
+            $hashesToPrefetch[$collectionKey] = $collectionHash;
 
             $headerDate = is_numeric($header['Date']) ? (int) $header['Date'] : strtotime($header['Date']);
             $now = now()->timestamp;
@@ -189,7 +200,6 @@ final class CollectionHandler
                 'fromname' => Utf8::clean($header['From']),
                 'unixtime' => $unixtime,
                 'xref' => implode(' ', $headerTokens),
-                'header_xref' => $header['Xref'] ?? '',
                 'groups_id' => $groupId,
                 'totalfiles' => $totalFiles,
                 'collectionhash' => $collectionHash,
@@ -199,18 +209,12 @@ final class CollectionHandler
         }
 
         if ($pending === []) {
+            $this->insertCollectionGroups($groupNamesByCollectionKey);
+
             return $resolved;
         }
 
-        $this->prefetchExistingCollections($xrefsToPrefetch);
-        foreach ($pending as $collectionKey => &$row) {
-            $row['xref_append'] = implode(' ', $this->xrefService->diffNewTokens(
-                $this->existingXrefs[$collectionKey],
-                $row['header_xref']
-            ));
-            unset($row['header_xref']);
-        }
-        unset($row);
+        $this->prefetchExistingCollections($hashesToPrefetch);
 
         try {
             $idsByHash = $this->bulkInsertAndResolve($pending);
@@ -225,7 +229,9 @@ final class CollectionHandler
                     $resolved[$index] = $collectionId;
                 }
             }
+            $this->insertCollectionGroups($groupNamesByCollectionKey);
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Bulk collection insert failed: '.$e->getMessage());
             }
@@ -236,7 +242,7 @@ final class CollectionHandler
 
     /**
      * Prefetch existing collections in one round-trip. Populates both
-     * $existingXrefs (keyed by collectionKey) and $existingIdsByHash so the
+     * $existingIdsByHash so the
      * subsequent bulkInsertAndResolve() can skip its existence-check SELECT
      * and only re-query for the freshly inserted rows.
      *
@@ -246,7 +252,7 @@ final class CollectionHandler
     {
         $missing = array_filter(
             $collectionHashByKey,
-            fn (string $hash, string $collectionKey): bool => ! array_key_exists($collectionKey, $this->existingXrefs),
+            fn (string $hash): bool => ! isset($this->existingIdsByHash[$hash]),
             ARRAY_FILTER_USE_BOTH
         );
 
@@ -255,24 +261,22 @@ final class CollectionHandler
         }
 
         $rows = [];
-        foreach (array_chunk(array_values($missing), self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
-            foreach (Collection::query()
-                ->whereIn('collectionhash', $chunk)
-                ->select(['id', 'collectionhash', 'xref'])
-                ->get() as $row) {
+        foreach (array_chunk(array_values($missing), $this->sqlChunkSize) as $chunk) {
+            $placeholders = implode(',', array_fill(0, \count($chunk), '?'));
+            $existingRows = DB::select(
+                "SELECT id, collectionhash FROM collections WHERE collectionhash IN ({$placeholders})",
+                $chunk
+            );
+            foreach ($existingRows as $row) {
                 $rows[(string) $row->collectionhash] = [
                     'id' => (int) $row->id,
-                    'xref' => (string) $row->xref,
                 ];
             }
         }
 
         foreach ($missing as $collectionKey => $hash) {
             if (isset($rows[$hash])) {
-                $this->existingXrefs[$collectionKey] = $rows[$hash]['xref'];
                 $this->existingIdsByHash[$hash] = $rows[$hash]['id'];
-            } else {
-                $this->existingXrefs[$collectionKey] = null;
             }
         }
     }
@@ -300,7 +304,7 @@ final class CollectionHandler
         if (DB::getDriverName() === 'sqlite') {
             $this->bulkInsertCollectionsSqlite($rowsByCollectionKey);
         } else {
-            $this->bulkInsertCollectionsMysql($rowsByCollectionKey, $existingHashes);
+            $this->bulkInsertCollectionsMysql($rowsByCollectionKey);
         }
 
         // Only resolve ids for the hashes we couldn't satisfy from the
@@ -344,18 +348,17 @@ final class CollectionHandler
             ];
         }
 
-        foreach (array_chunk($rows, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk($rows, $this->sqlChunkSize) as $chunk) {
             DB::table('collections')->insertOrIgnore($chunk);
         }
     }
 
     /**
      * @param  array<string, array<string, mixed>>  $rowsByCollectionKey
-     * @param  array<string, true>  $existingHashes
      */
-    private function bulkInsertCollectionsMysql(array $rowsByCollectionKey, array $existingHashes): void
+    private function bulkInsertCollectionsMysql(array $rowsByCollectionKey): void
     {
-        foreach (array_chunk(array_values($rowsByCollectionKey), self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk(array_values($rowsByCollectionKey), $this->sqlChunkSize) as $chunk) {
             $placeholders = [];
             $bindings = [];
             foreach ($chunk as $row) {
@@ -386,50 +389,8 @@ final class CollectionHandler
             );
         }
 
-        $this->batchAppendXrefs($rowsByCollectionKey, $existingHashes);
-    }
-
-    /**
-     * Append xref tokens for every existing collection in a chunk in a single
-     * UPDATE...JOIN per sub-chunk instead of N standalone UPDATEs (one per row).
-     * Same UNION ALL shape as BinaryHandler::flushUpdatesMysql().
-     *
-     * @param  array<string, array<string, mixed>>  $rowsByCollectionKey
-     * @param  array<string, true>  $existingHashes
-     */
-    private function batchAppendXrefs(array $rowsByCollectionKey, array $existingHashes): void
-    {
-        $updates = [];
-        foreach ($rowsByCollectionKey as $row) {
-            if (($row['xref_append'] ?? '') !== '' && isset($existingHashes[$row['collectionhash']])) {
-                $updates[] = [
-                    'collectionhash' => $row['collectionhash'],
-                    'xref_append' => $row['xref_append'],
-                ];
-            }
-        }
-
-        if ($updates === []) {
-            return;
-        }
-
-        foreach (array_chunk($updates, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
-            $selects = [];
-            $bindings = [];
-            foreach ($chunk as $u) {
-                $selects[] = 'SELECT ? AS collectionhash, ? AS xref_append';
-                $bindings[] = $u['collectionhash'];
-                $bindings[] = $u['xref_append'];
-            }
-
-            $sql = 'UPDATE collections c INNER JOIN ('
-                .implode(' UNION ALL ', $selects)
-                .') u ON u.collectionhash = c.collectionhash '
-                .'SET c.xref = CONCAT(c.xref, ?, u.xref_append)';
-            $bindings[] = "\n";
-
-            DB::statement($sql, $bindings);
-        }
+        // Cross-post groups are normalized in collection_groups. Avoid
+        // rewriting the wide collections row for every new article number.
     }
 
     /**
@@ -443,17 +404,56 @@ final class CollectionHandler
         }
 
         $resolved = [];
-        foreach (array_chunk($hashes, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
-            $resolved += Collection::query()
-                ->whereIn('collectionhash', $chunk)
-                ->pluck('id', 'collectionhash')
-                ->mapWithKeys(static fn (int|string $id, string $hash): array => [$hash => (int) $id])
-                ->all();
+        foreach (array_chunk($hashes, $this->sqlChunkSize) as $chunk) {
+            $placeholders = implode(',', array_fill(0, \count($chunk), '?'));
+            foreach (DB::select(
+                "SELECT id, collectionhash FROM collections WHERE collectionhash IN ({$placeholders})",
+                $chunk
+            ) as $row) {
+                $resolved[(string) $row->collectionhash] = (int) $row->id;
+            }
         }
 
         return $resolved;
     }
 
+    /**
+     * @param  array<string, array<string, true>>  $groupNamesByCollectionKey
+     */
+    private function insertCollectionGroups(array $groupNamesByCollectionKey): void
+    {
+        $rows = [];
+        foreach ($groupNamesByCollectionKey as $collectionKey => $groupNames) {
+            $collectionId = $this->collectionIds[$collectionKey] ?? null;
+            if ($collectionId === null) {
+                continue;
+            }
+            foreach (array_keys($groupNames) as $groupName) {
+                $rows[] = ['collections_id' => $collectionId, 'group_name' => $groupName];
+            }
+        }
+
+        foreach (array_chunk($rows, $this->sqlChunkSize) as $chunk) {
+            if (DB::getDriverName() === 'sqlite') {
+                DB::table('collection_groups')->insertOrIgnore($chunk);
+
+                continue;
+            }
+
+            $values = [];
+            $bindings = [];
+            foreach ($chunk as $row) {
+                $values[] = '(?, ?)';
+                $bindings[] = $row['collections_id'];
+                $bindings[] = $row['group_name'];
+            }
+            DB::statement(
+                'INSERT IGNORE INTO collection_groups (collections_id, group_name) VALUES '.implode(',', $values),
+                $bindings
+            );
+        }
+    }
+
     /**
      * @param  array<string, mixed>  $headerTokens
      */
@@ -463,7 +463,6 @@ final class CollectionHandler
         string $fromName,
         int $unixtime,
         array $headerTokens,
-        string $finalXrefAppend,
         int $groupId,
         int $totalFiles,
         string $collectionHash,
@@ -489,7 +488,6 @@ final class CollectionHandler
             $fromName,
             $unixtime,
             $headerTokens,
-            $finalXrefAppend,
             $groupId,
             $totalFiles,
             $collectionHash,
@@ -542,7 +540,6 @@ final class CollectionHandler
         string $fromName,
         int $unixtime,
         array $headerTokens,
-        string $finalXrefAppend,
         int $groupId,
         int $totalFiles,
         string $collectionHash,
@@ -569,14 +566,8 @@ final class CollectionHandler
             $batchNoise,
         ];
 
-        if ($finalXrefAppend !== '') {
-            $insertSql .= ', xref = CONCAT(xref, "\\n", ?)';
-            $bindings[] = $finalXrefAppend;
-        }
-
-        // affectingStatement so we can distinguish a brand-new insert
-        // (rowCount = 1) from a duplicate-key hit (rowCount = 0 with no xref
-        // append, or 2 when xref was actually appended). LAST_INSERT_ID(id) in
+        // affectingStatement distinguishes a brand-new insert (rowCount = 1)
+        // from a duplicate-key hit (rowCount = 0). LAST_INSERT_ID(id) in
         // the ODKU clause makes lastInsertId() return the existing row id
         // even on a duplicate, so we can't rely on lastInsertId() alone.
         $affected = (int) DB::affectingStatement($insertSql, $bindings);
@@ -593,6 +584,105 @@ final class CollectionHandler
         return (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0);
     }
 
+    /**
+     * Refresh size and readiness for only the collections touched by a header
+     * chunk. Binary and part aggregates have already been refreshed, so this
+     * replaces broad table-wide completion scans on the normal ingestion path.
+     *
+     * @param  list<int>  $collectionIds
+     */
+    public function refreshAggregates(array $collectionIds, int $chunkSize = 500): bool
+    {
+        $collectionIds = array_values(array_unique(array_map('intval', $collectionIds)));
+        if ($collectionIds === []) {
+            return true;
+        }
+
+        $inProgressStatuses = [
+            CollectionFileCheckStatus::Default->value,
+            CollectionFileCheckStatus::CompleteCollection->value,
+            CollectionFileCheckStatus::CompleteParts->value,
+            CollectionFileCheckStatus::TempComplete->value,
+            CollectionFileCheckStatus::ZeroPart->value,
+        ];
+
+        try {
+            $chunkSize = max(1, min($chunkSize, $this->sqlChunkSize));
+            if (DB::getDriverName() === 'sqlite') {
+                foreach ($collectionIds as $collectionId) {
+                    $aggregate = DB::selectOne(
+                        'SELECT COUNT(*) AS currentfiles,
+                                COALESCE(SUM(CASE WHEN partcheck = 1 THEN 1 ELSE 0 END), 0) AS completefiles,
+                                COALESCE(SUM(partsize), 0) AS filesize
+                         FROM binaries WHERE collections_id = ?',
+                        [$collectionId]
+                    );
+                    $collection = DB::selectOne(
+                        'SELECT totalfiles, filecheck FROM collections WHERE id = ?',
+                        [$collectionId]
+                    );
+                    if ($collection === null) {
+                        continue;
+                    }
+
+                    $currentFiles = (int) $aggregate->currentfiles;
+                    $totalFiles = (int) $collection->totalfiles;
+                    $ready = \in_array((int) $collection->filecheck, $inProgressStatuses, true)
+                        && $totalFiles > 0
+                        && ($currentFiles === $totalFiles || $currentFiles === $totalFiles + 1)
+                        && (int) $aggregate->completefiles >= $totalFiles;
+
+                    DB::update(
+                        'UPDATE collections SET filesize = ?, last_seen_at = ?, filecheck = ? WHERE id = ?',
+                        [
+                            (int) $aggregate->filesize,
+                            now()->format('Y-m-d H:i:s'),
+                            $ready ? CollectionFileCheckStatus::Sized->value : (int) $collection->filecheck,
+                            $collectionId,
+                        ]
+                    );
+                }
+
+                return true;
+            }
+
+            foreach (array_chunk($collectionIds, $chunkSize) as $chunk) {
+                $idPlaceholders = implode(',', array_fill(0, \count($chunk), '?'));
+                $statusPlaceholders = implode(',', array_fill(0, \count($inProgressStatuses), '?'));
+                DB::update(
+                    "UPDATE collections c
+                     INNER JOIN (
+                         SELECT b.collections_id,
+                                COUNT(*) AS currentfiles,
+                                SUM(CASE WHEN b.partcheck = 1 THEN 1 ELSE 0 END) AS completefiles,
+                                COALESCE(SUM(b.partsize), 0) AS filesize
+                         FROM binaries b
+                         WHERE b.collections_id IN ({$idPlaceholders})
+                         GROUP BY b.collections_id
+                     ) a ON a.collections_id = c.id
+                     SET c.filesize = a.filesize,
+                         c.last_seen_at = NOW(),
+                         c.filecheck = CASE
+                             WHEN c.filecheck IN ({$statusPlaceholders})
+                              AND c.totalfiles > 0
+                              AND a.currentfiles IN (c.totalfiles, c.totalfiles + 1)
+                              AND a.completefiles >= c.totalfiles
+                             THEN ? ELSE c.filecheck END",
+                    [...$chunk, ...$inProgressStatuses, CollectionFileCheckStatus::Sized->value]
+                );
+            }
+
+            return true;
+        } catch (\Throwable $e) {
+            $this->lastException = $e;
+            if (config('app.debug') === true) {
+                Log::error('Collection aggregate refresh failed: '.$e->getMessage());
+            }
+
+            return false;
+        }
+    }
+
     /**
      * Get IDs created in this batch.
      *
@@ -603,6 +693,11 @@ final class CollectionHandler
         return array_keys($this->insertedCollectionIds);
     }
 
+    public function getLastException(): ?\Throwable
+    {
+        return $this->lastException;
+    }
+
     /**
      * Get all collection IDs processed this batch.
      *

app/Services/Binaries/HeaderParser.php

@@ -49,6 +49,11 @@ final class HeaderParser
         $parsed = [];
         $headersRepaired = [];
         $receivedNumbers = [];
+        $missingPartSet = $missingParts === null
+            ? null
+            : (array_is_list($missingParts)
+                ? array_fill_keys(array_map('intval', $missingParts), true)
+                : $missingParts);
 
         foreach ($headers as $header) {
             // Check if we got the article
@@ -59,8 +64,8 @@ final class HeaderParser
             $receivedNumbers[] = $header['Number'];
 
             // For part repair, only process missing parts
-            if ($partRepair && $missingParts !== null) {
-                if (! \in_array($header['Number'], $missingParts, true)) {
+            if ($partRepair && $missingPartSet !== null) {
+                if (! isset($missingPartSet[(int) $header['Number']])) {
                     continue;
                 }
                 $headersRepaired[] = $header['Number'];
@@ -92,14 +97,11 @@ final class HeaderParser
                 $header['Bytes'] = $header[':bytes'] ?? 0;
             }
 
-            $parsed[] = [
-                'header' => $header,
-                'repaired' => $partRepair,
-            ];
+            $parsed[] = $header;
         }
 
         return [
-            'headers' => array_column($parsed, 'header'),
+            'headers' => $parsed,
             'repaired' => $headersRepaired,
             'received' => $receivedNumbers,
             'notYEnc' => $this->notYEnc,

app/Services/Binaries/HeaderStorageService.php

@@ -4,6 +4,8 @@ declare(strict_types=1);
 
 namespace App\Services\Binaries;
 
+use Illuminate\Database\QueryException;
+
 /**
  * Orchestrates the header storage process.
  *
@@ -12,6 +14,8 @@ namespace App\Services\Binaries;
  */
 final class HeaderStorageService
 {
+    private const int LOCK_RETRY_MAX = 5;
+
     private CollectionHandler $collectionHandler;
 
     private BinaryHandler $binaryHandler;
@@ -23,6 +27,8 @@ final class HeaderStorageService
     /** @var array<int, int|string> Article numbers that failed to insert */
     private array $failedInserts = [];
 
+    private ?\Throwable $lastStorageException = null;
+
     public function __construct(
         ?CollectionHandler $collectionHandler = null,
         ?BinaryHandler $binaryHandler = null,
@@ -30,10 +36,10 @@ final class HeaderStorageService
         ?BinariesConfig $config = null
     ) {
         $this->config = $config ?? BinariesConfig::fromSettings();
-        $this->collectionHandler = $collectionHandler ?? new CollectionHandler;
-        $this->binaryHandler = $binaryHandler ?? new BinaryHandler;
+        $this->collectionHandler = $collectionHandler ?? new CollectionHandler(sqlChunkSize: $this->config->sqlChunkSize);
+        $this->binaryHandler = $binaryHandler ?? new BinaryHandler($this->config->sqlChunkSize);
         $this->partHandler = $partHandler ?? new PartHandler(
-            $this->config->partsChunkSize,
+            $this->config->sqlChunkSize,
             true
         );
     }
@@ -54,17 +60,13 @@ final class HeaderStorageService
 
         $this->failedInserts = [];
 
-        // Use the dedicated header chunk size, NOT partsChunkSize. The latter
-        // controls single-row part flushes and is normally much larger; using
-        // it here forces every collection/binary bulk INSERT and OR-clause
-        // SELECT to scale to thousands of rows per chunk, which exhausts PHP
-        // and MySQL memory.
+        // Header and SQL chunking are configured independently, but both are
+        // clamped by BinariesConfig so generated statements remain bounded.
         $chunkSize = max(1, $this->config->headerChunkSize);
 
         // Walk the array with offset slicing instead of array_chunk() so we
         // don't materialize every chunk simultaneously in memory.
         $total = \count($headers);
-        $headers = array_values($headers);
         for ($offset = 0; $offset < $total; $offset += $chunkSize) {
             $chunk = \array_slice($headers, $offset, $chunkSize);
             $this->storeChunk($chunk, $groupMySQL, $addToPartRepair);
@@ -82,6 +84,30 @@ final class HeaderStorageService
      */
     private function storeChunk(array $headers, array $groupMySQL, bool $addToPartRepair): void
     {
+        $attempt = 0;
+        do {
+            $failedInsertCount = \count($this->failedInserts);
+            if ($this->storeChunkAttempt($headers, $groupMySQL, $addToPartRepair)) {
+                return;
+            }
+
+            $attempt++;
+            if ($attempt >= self::LOCK_RETRY_MAX || ! $this->isTransientLockError($this->lastStorageException)) {
+                return;
+            }
+
+            $this->failedInserts = array_slice($this->failedInserts, 0, $failedInsertCount);
+            usleep((min(500, 20 * $attempt) + random_int(0, 25)) * 1000);
+        } while (true);
+    }
+
+    /**
+     * @param  array<int, array<string, mixed>>  $headers
+     * @param  array<string, mixed>  $groupMySQL
+     */
+    private function storeChunkAttempt(array $headers, array $groupMySQL, bool $addToPartRepair): bool
+    {
+        $this->lastStorageException = null;
         $this->collectionHandler->reset();
         $this->binaryHandler->reset();
         $this->partHandler->reset();
@@ -113,13 +139,26 @@ final class HeaderStorageService
 
         // Flush binary aggregate updates
         if (! $transaction->hasErrors()) {
-            if (! $this->binaryHandler->flushUpdates($this->config->binariesUpdateChunkSize)) {
+            if (! $this->binaryHandler->refreshAggregates(
+                $this->partHandler->getTouchedBinaryIds(),
+                $this->config->sqlChunkSize
+            )) {
+                $transaction->markError();
+            }
+            if (! $transaction->hasErrors() && ! $this->collectionHandler->refreshAggregates(
+                $this->collectionHandler->getAllIds(),
+                $this->config->sqlChunkSize
+            )) {
                 $transaction->markError();
             }
         }
 
         // Finish transaction
         if (! $transaction->finish()) {
+            $this->lastStorageException = $transaction->getLastException()
+                ?? $this->partHandler->getLastException()
+                ?? $this->binaryHandler->getLastException()
+                ?? $this->collectionHandler->getLastException();
             if ($addToPartRepair) {
                 $this->failedInserts = array_merge(
                     $this->failedInserts,
@@ -128,13 +167,32 @@ final class HeaderStorageService
                 );
             }
 
-            return;
+            return false;
         }
 
         $this->failedInserts = array_merge(
             $this->failedInserts,
             $this->partHandler->getFailedNumbers()
         );
+
+        return true;
+    }
+
+    private function isTransientLockError(?\Throwable $exception): bool
+    {
+        if ($exception === null) {
+            return false;
+        }
+        if ($exception instanceof QueryException) {
+            $driverCode = (int) ($exception->errorInfo[1] ?? 0);
+            if ($exception->getCode() === '40001' || \in_array($driverCode, [1205, 1213], true)) {
+                return true;
+            }
+        }
+
+        return str_contains($exception->getMessage(), 'Deadlock found')
+            || str_contains($exception->getMessage(), 'Lock wait timeout exceeded')
+            || str_contains($exception->getMessage(), 'database is locked');
     }
 
     /**
@@ -198,9 +256,6 @@ final class HeaderStorageService
     private function extractFileNumberAndTotal(array $header): array
     {
         $fileCount = $this->getFileCount($header['matches'][1]);
-        if ($fileCount[1] === 0 && $fileCount[3] === 0) {
-            $fileCount = $this->getFileCount($header['matches'][0]);
-        }
 
         return [(int) $fileCount[1], (int) $fileCount[3]];
     }

app/Services/Binaries/HeaderStorageTransaction.php

@@ -22,6 +22,8 @@ final class HeaderStorageTransaction
 
     private bool $hadErrors = false;
 
+    private ?\Throwable $lastException = null;
+
     public function __construct(
         CollectionHandler $collectionHandler,
         BinaryHandler $binaryHandler
@@ -46,6 +48,7 @@ final class HeaderStorageTransaction
     {
         DB::beginTransaction();
         $this->hadErrors = false;
+        $this->lastException = null;
     }
 
     /**
@@ -80,6 +83,7 @@ final class HeaderStorageTransaction
 
             return true;
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             $this->rollbackAndCleanup();
 
             if (config('app.debug') === true) {
@@ -90,6 +94,11 @@ final class HeaderStorageTransaction
         }
     }
 
+    public function getLastException(): ?\Throwable
+    {
+        return $this->lastException;
+    }
+
     /**
      * Perform rollback and cleanup any orphaned data.
      */

app/Services/Binaries/PartHandler.php

@@ -9,34 +9,33 @@ use Illuminate\Support\Facades\Log;
 
 /**
  * Handles part record creation during header storage.
+ *
+ * @phpstan-type PartRow array{binaries_id: int, number: int|string, messageid: string, partnumber: int, size: int|string}
  */
 final class PartHandler
 {
-    /**
-     * Hard upper bound on rows packed into a single SQL statement
-     * (multi-row INSERT, OR-clause SELECT). The public chunkSize controls
-     * when we flush; this constant guarantees the actual SQL we emit is
-     * never large enough to blow up PHP/MySQL memory.
-     */
-    private const MAX_SQL_ROWS_PER_STATEMENT = 500;
-
-    /** @var array<string, mixed> Pending parts to insert */
+    /** @var list<PartRow> Pending parts to insert */
     private array $parts = [];
 
-    /** @var array<string, mixed> Part numbers successfully inserted */
+    /** @var list<int|string> Part numbers successfully inserted */
     private array $insertedPartNumbers = [];
 
-    /** @var array<string, mixed> Part numbers that failed to insert */
+    /** @var list<int|string> Part numbers that failed to insert */
     private array $failedPartNumbers = [];
 
+    /** @var array<int, true> Binary ids whose stored parts must be re-aggregated */
+    private array $touchedBinaryIds = [];
+
     private int $chunkSize;
 
+    private ?\Throwable $lastException = null;
+
     /** @phpstan-ignore property.onlyWritten */
     private bool $addToPartRepair;
 
-    public function __construct(int $chunkSize = 5000, bool $addToPartRepair = true)
+    public function __construct(int $chunkSize = 500, bool $addToPartRepair = true)
     {
-        $this->chunkSize = max(100, $chunkSize);
+        $this->chunkSize = max(1, $chunkSize);
         $this->addToPartRepair = $addToPartRepair;
     }
 
@@ -48,6 +47,8 @@ final class PartHandler
         $this->parts = [];
         $this->insertedPartNumbers = [];
         $this->failedPartNumbers = [];
+        $this->touchedBinaryIds = [];
+        $this->lastException = null;
     }
 
     /**
@@ -66,11 +67,21 @@ final class PartHandler
      */
     public function addPart(int $binaryId, array $header): bool
     {
+        $partNumber = (int) ($header['matches'][2] ?? 0);
+        $messageId = trim((string) ($header['Message-ID'] ?? ''));
+        if ($partNumber <= 0 || $messageId === '' || strlen($messageId) > 255 || preg_match('/^[\x20-\x7E]+$/D', $messageId) !== 1) {
+            if (isset($header['Number'])) {
+                $this->failedPartNumbers[] = $header['Number'];
+            }
+
+            return false;
+        }
+
         $this->parts[] = [
             'binaries_id' => $binaryId,
             'number' => $header['Number'],
-            'messageid' => $header['Message-ID'],
-            'partnumber' => $header['matches'][2],
+            'messageid' => $messageId,
+            'partnumber' => $partNumber,
             'size' => $header['Bytes'],
         ];
 
@@ -91,10 +102,16 @@ final class PartHandler
             return true;
         }
 
-        $insertedCount = $this->insertChunk($this->parts);
+        $pendingParts = $this->parts;
+        $parts = $this->deduplicateParts($pendingParts);
+        foreach ($parts as $part) {
+            $this->touchedBinaryIds[(int) $part['binaries_id']] = true;
+        }
+
+        $insertedCount = $this->insertChunk($parts);
 
         if ($insertedCount === null) {
-            foreach ($this->parts as $part) {
+            foreach ($pendingParts as $part) {
                 $this->failedPartNumbers[] = $part['number'];
             }
 
@@ -103,8 +120,8 @@ final class PartHandler
             return false;
         }
 
-        if ($insertedCount === \count($this->parts)) {
-            foreach ($this->parts as $part) {
+        if ($insertedCount === \count($parts)) {
+            foreach ($parts as $part) {
                 $this->insertedPartNumbers[] = $part['number'];
             }
 
@@ -113,9 +130,9 @@ final class PartHandler
             return true;
         }
 
-        $existingKeys = $this->existingPartKeys($this->parts);
-        foreach ($this->parts as $part) {
-            $key = $this->partKey((int) $part['binaries_id'], (int) $part['number']);
+        $existingKeys = $this->existingPartKeys($parts);
+        foreach ($parts as $part) {
+            $key = $this->partKey((int) $part['binaries_id'], (int) $part['partnumber']);
             if (! isset($existingKeys[$key])) {
                 $this->failedPartNumbers[] = $part['number'];
             }
@@ -127,7 +144,7 @@ final class PartHandler
     }
 
     /**
-     * @param  array<string, mixed>  $parts
+     * @param  list<PartRow>  $parts
      */
     private function insertChunk(array $parts): ?int
     {
@@ -135,7 +152,7 @@ final class PartHandler
         $totalInserted = 0;
 
         try {
-            foreach (array_chunk($parts, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+            foreach (array_chunk($parts, $this->chunkSize) as $chunk) {
                 $placeholders = [];
                 $bindings = [];
 
@@ -157,6 +174,7 @@ final class PartHandler
 
             return $totalInserted;
         } catch (\Throwable $e) {
+            $this->lastException = $e;
             if (config('app.debug') === true) {
                 Log::error('Parts chunk insert failed: '.$e->getMessage());
             }
@@ -166,7 +184,7 @@ final class PartHandler
     }
 
     /**
-     * @param  array<string, mixed>  $parts
+     * @param  list<PartRow>  $parts
      * @return array<string, true>
      */
     private function existingPartKeys(array $parts): array
@@ -175,49 +193,95 @@ final class PartHandler
             return [];
         }
 
-        // Deduplicate (binaries_id, number) pairs so we don't bind the same
+        // Deduplicate (binaries_id, partnumber) pairs so we don't bind the same
         // tuple twice when a chunk contains repeats.
         $uniquePairs = [];
         foreach ($parts as $part) {
             $bid = (int) $part['binaries_id'];
-            $num = (int) $part['number'];
-            $uniquePairs[$bid.':'.$num] = [$bid, $num];
+            $partNumber = (int) $part['partnumber'];
+            $uniquePairs[$bid.':'.$partNumber] = [$bid, $partNumber];
         }
 
         $keys = [];
-        // Single tuple-IN per sub-chunk: (binaries_id, number) IN ((?,?),...).
+        // Single tuple-IN per sub-chunk: (binaries_id, partnumber) IN ((?,?),...).
         // Both MySQL and SQLite (3.15+) support this row-constructor form,
         // which lets one SELECT replace the previous "one SELECT per binary".
-        foreach (array_chunk(array_values($uniquePairs), self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk(array_values($uniquePairs), $this->chunkSize) as $chunk) {
             $tuples = implode(',', array_fill(0, \count($chunk), '(?,?)'));
             $bindings = [];
-            foreach ($chunk as [$bid, $num]) {
+            foreach ($chunk as [$bid, $partNumber]) {
                 $bindings[] = $bid;
-                $bindings[] = $num;
+                $bindings[] = $partNumber;
             }
 
             $rows = DB::select(
-                "SELECT binaries_id, number FROM parts WHERE (binaries_id, number) IN ({$tuples})",
+                "SELECT binaries_id, partnumber FROM parts WHERE (binaries_id, partnumber) IN ({$tuples})",
                 $bindings
             );
 
             foreach ($rows as $row) {
-                $keys[$this->partKey((int) $row->binaries_id, (int) $row->number)] = true;
+                $keys[$this->partKey((int) $row->binaries_id, (int) $row->partnumber)] = true;
             }
         }
 
         return $keys;
     }
 
-    private function partKey(int $binaryId, int $number): string
+    private function partKey(int $binaryId, int $partNumber): string
     {
-        return $binaryId.':'.$number;
+        return $binaryId.':'.$partNumber;
+    }
+
+    /**
+     * Choose one deterministic segment for each (binary, part number).
+     * Prefer a non-empty message id, then the larger payload, then the lower
+     * article number. This matches the storage migration's duplicate policy.
+     *
+     * @param  list<PartRow>  $parts
+     * @return list<PartRow>
+     */
+    private function deduplicateParts(array $parts): array
+    {
+        $deduplicated = [];
+        foreach ($parts as $part) {
+            $key = $this->partKey((int) $part['binaries_id'], (int) $part['partnumber']);
+            if (! isset($deduplicated[$key]) || $this->shouldPrefer($part, $deduplicated[$key])) {
+                $deduplicated[$key] = $part;
+            }
+        }
+
+        return array_values($deduplicated);
+    }
+
+    /**
+     * @param  PartRow  $candidate
+     * @param  PartRow  $current
+     */
+    private function shouldPrefer(array $candidate, array $current): bool
+    {
+        $candidateHasMessageId = trim((string) $candidate['messageid']) !== '';
+        $currentHasMessageId = trim((string) $current['messageid']) !== '';
+        if ($candidateHasMessageId !== $currentHasMessageId) {
+            return $candidateHasMessageId;
+        }
+
+        if ((int) $candidate['size'] !== (int) $current['size']) {
+            return (int) $candidate['size'] > (int) $current['size'];
+        }
+
+        return (int) $candidate['number'] < (int) $current['number'];
+    }
+
+    /** @return list<int> */
+    public function getTouchedBinaryIds(): array
+    {
+        return array_keys($this->touchedBinaryIds);
     }
 
     /**
      * Get numbers of successfully inserted parts.
      *
-     * @return array<string, mixed>
+     * @return list<int|string>
      */
     public function getInsertedNumbers(): array
     {
@@ -227,13 +291,18 @@ final class PartHandler
     /**
      * Get numbers of failed part inserts.
      *
-     * @return array<string, mixed>
+     * @return list<int|string>
      */
     public function getFailedNumbers(): array
     {
         return $this->failedPartNumbers;
     }
 
+    public function getLastException(): ?\Throwable
+    {
+        return $this->lastException;
+    }
+
     /**
      * Check if there are pending parts waiting to be flushed.
      */

app/Services/CollectionCleanupService.php

@@ -5,14 +5,13 @@ declare(strict_types=1);
 namespace App\Services;
 
 use App\Models\Settings;
+use App\Services\Binaries\BinariesConfig;
 use Illuminate\Database\QueryException;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Str;
 
 class CollectionCleanupService
 {
-    private const MAX_SQL_ROWS_PER_STATEMENT = 500;
-
     /**
      * Maximum number of retries for a lock-related DB error before giving up.
      */
@@ -31,7 +30,14 @@ class CollectionCleanupService
      */
     private const LOCK_DRIVER_CODES = [1213, 1205];
 
-    public function __construct() {}
+    private ?bool $cascadeDeleteReady = null;
+
+    private readonly BinariesConfig $binariesConfig;
+
+    public function __construct(?BinariesConfig $binariesConfig = null)
+    {
+        $this->binariesConfig = $binariesConfig ?? BinariesConfig::fromSettings();
+    }
 
     /**
      * Deletes finished/old collections, cleans orphans, and removes collections missed after NZB creation.
@@ -61,7 +67,7 @@ class CollectionCleanupService
             $ids = DB::table('collections')
                 ->where('dateadded', '<', $cutoff)
                 ->orderBy('id')
-                ->limit(self::MAX_SQL_ROWS_PER_STATEMENT)
+                ->limit($this->sqlChunkSize())
                 ->pluck('id')
                 ->all();
 
@@ -72,7 +78,7 @@ class CollectionCleanupService
             $affected = $this->deleteCollectionsAndDescendants($ids, 'Cleanup', $echoCLI);
 
             $batchDeleted += $affected;
-            if ($affected < self::MAX_SQL_ROWS_PER_STATEMENT) {
+            if ($affected < $this->sqlChunkSize()) {
                 break;
             }
             // Brief pause to reduce pressure on the lock manager in busy systems.
@@ -147,7 +153,7 @@ class CollectionCleanupService
     {
         $deleted = 0;
         $maxBatches = 20; // hard cap per cycle; bounded backlog drain
-        $batchSize = self::MAX_SQL_ROWS_PER_STATEMENT;
+        $batchSize = $this->sqlChunkSize();
 
         for ($i = 0; $i < $maxBatches; $i++) {
             $ids = DB::table('collections as c')
@@ -196,7 +202,7 @@ class CollectionCleanupService
     {
         $deleted = 0;
         $maxBatches = 20;
-        $batchSize = self::MAX_SQL_ROWS_PER_STATEMENT;
+        $batchSize = $this->sqlChunkSize();
 
         for ($i = 0; $i < $maxBatches; $i++) {
             $ids = DB::table('collections as c')
@@ -240,11 +246,18 @@ class CollectionCleanupService
 
         $deletedCollections = 0;
 
-        foreach (array_chunk($collectionIds, self::MAX_SQL_ROWS_PER_STATEMENT) as $chunk) {
+        foreach (array_chunk($collectionIds, $this->sqlChunkSize()) as $chunk) {
             $placeholders = implode(',', array_fill(0, count($chunk), '?'));
             $deletedCollections += $this->retryOnLockError(
                 fn (): int => DB::transaction(
                     function () use ($chunk, $placeholders): int {
+                        if ($this->cascadeDeleteReady()) {
+                            return (int) DB::affectingStatement(
+                                "DELETE FROM collections WHERE id IN ({$placeholders})",
+                                $chunk
+                            );
+                        }
+
                         DB::statement(
                             "DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN ({$placeholders}))",
                             $chunk
@@ -268,6 +281,45 @@ class CollectionCleanupService
         return $deletedCollections;
     }
 
+    private function sqlChunkSize(): int
+    {
+        return $this->binariesConfig->sqlChunkSize;
+    }
+
+    /**
+     * Production deletes can safely target only the parent table when both
+     * descendant foreign keys cascade. SQLite fixtures and incomplete legacy
+     * schemas deliberately retain the explicit descendant-delete fallback.
+     */
+    private function cascadeDeleteReady(): bool
+    {
+        if ($this->cascadeDeleteReady !== null) {
+            return $this->cascadeDeleteReady;
+        }
+        if (DB::getDriverName() === 'sqlite') {
+            return $this->cascadeDeleteReady = false;
+        }
+
+        try {
+            $rows = DB::select(
+                "SELECT TABLE_NAME, DELETE_RULE
+                 FROM information_schema.REFERENTIAL_CONSTRAINTS
+                 WHERE CONSTRAINT_SCHEMA = DATABASE()
+                   AND TABLE_NAME IN ('binaries', 'parts')"
+            );
+            $cascades = [];
+            foreach ($rows as $row) {
+                if (strtoupper((string) $row->DELETE_RULE) === 'CASCADE') {
+                    $cascades[(string) $row->TABLE_NAME] = true;
+                }
+            }
+
+            return $this->cascadeDeleteReady = isset($cascades['binaries'], $cascades['parts']);
+        } catch (\Throwable) {
+            return $this->cascadeDeleteReady = false;
+        }
+    }
+
     /**
      * Run a DB write inside a bounded retry loop that only swallows transient
      * InnoDB lock errors (deadlock 1213, lock wait timeout 1205). Any other

app/Services/Nzb/NzbService.php

@@ -4,17 +4,17 @@ declare(strict_types=1);
 
 namespace App\Services\Nzb;
 
-use App\Models\Binary;
 use App\Models\Collection;
-use App\Models\Part;
 use App\Models\Release;
 use App\Models\Settings;
+use App\Services\Binaries\BinariesConfig;
 use App\Services\CollectionCleanupService;
 use App\Support\Data\NzbCreationResult;
 use Illuminate\Database\QueryException;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\File;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Schema;
 use Illuminate\Support\Str;
 use Throwable;
 
@@ -62,7 +62,9 @@ class NzbService
 
     public function __construct(
         private readonly CollectionCleanupService $collectionCleanupService,
+        ?BinariesConfig $binariesConfig = null,
     ) {
+        $this->binariesConfig = $binariesConfig ?? BinariesConfig::fromSettings();
         try {
             $nzbSplitLevel = (int) Settings::settingValue('nzbsplitlevel');
         } catch (QueryException $e) {
@@ -82,6 +84,8 @@ class NzbService
         );
     }
 
+    private readonly BinariesConfig $binariesConfig;
+
     /**
      * Write an NZB file for a release.
      *
@@ -98,7 +102,9 @@ class NzbService
             $collections = Collection::whereReleasesId($release->id)
                 ->join('usenet_groups', 'collections.groups_id', '=', 'usenet_groups.id')
                 ->select(['collections.*', DB::raw('UNIX_TIMESTAMP(collections.date) AS udate'), 'usenet_groups.name as groupname'])
-                ->get();
+                ->orderBy('collections.id')
+                ->get()
+                ->keyBy('id');
         } catch (Throwable $e) {
             return NzbCreationResult::transient('Failed to load release collections: '.$e->getMessage());
         }
@@ -107,29 +113,30 @@ class NzbService
             return NzbCreationResult::deterministic('Release has no collections to write into an NZB.');
         }
 
-        // Pre-load every binary and part for the release in two flat queries
-        // and group them in PHP, instead of issuing one binaries SELECT per
-        // collection and one parts SELECT per binary (1 + N + N*M queries).
         $collectionIds = $collections->pluck('id')->map(static fn (mixed $id): int => (int) $id)->all();
         try {
-            $binariesByCollection = Binary::query()
-                ->whereIn('collections_id', $collectionIds)
-                ->orderBy('name')
-                ->get(['id', 'collections_id', 'name', 'totalparts'])
-                ->groupBy('collections_id');
+            $emptyCollection = DB::selectOne(
+                'SELECT c.id FROM collections c
+                 LEFT JOIN binaries b ON b.collections_id = c.id
+                 WHERE c.releases_id = ? GROUP BY c.id HAVING COUNT(b.id) = 0 LIMIT 1',
+                [$release->id]
+            );
+            if ($emptyCollection !== null) {
+                return NzbCreationResult::deterministic("Collection {$emptyCollection->id} has no binaries.", $collectionIds);
+            }
 
-            $allBinaryIds = $binariesByCollection->flatten(1)->pluck('id')->all();
-            $partsByBinary = $allBinaryIds === []
-                ? collect()
-                : Part::query()
-                    ->whereIn('binaries_id', $allBinaryIds)
-                    // distinct() preserves the dedup the previous per-binary
-                    // query had, in case two rows share (messageid, size,
-                    // partnumber) for the same binary (defensive).
-                    ->distinct()
-                    ->orderBy('partnumber')
-                    ->get(['binaries_id', 'messageid', 'size', 'partnumber'])
-                    ->groupBy('binaries_id');
+            $emptyBinary = DB::selectOne(
+                'SELECT b.id FROM binaries b
+                 INNER JOIN collections c ON c.id = b.collections_id
+                 LEFT JOIN parts p ON p.binaries_id = b.id
+                 WHERE c.releases_id = ? GROUP BY b.id HAVING COUNT(p.partnumber) = 0 LIMIT 1',
+                [$release->id]
+            );
+            if ($emptyBinary !== null) {
+                return NzbCreationResult::deterministic("Binary {$emptyBinary->id} has no parts.", $collectionIds);
+            }
+
+            $groupsByCollection = $this->loadCollectionGroups((int) $release->id, $collections);
         } catch (Throwable $e) {
             return NzbCreationResult::transient('Failed to load NZB binaries or parts: '.$e->getMessage(), $collectionIds);
         }
@@ -173,55 +180,64 @@ class NzbService
                 return NzbCreationResult::transient("Failed to write NZB header to temporary file: {$tempPath}", $collectionIds, $path);
             }
 
-            foreach ($collections as $collection) {
-                $binaries = $binariesByCollection->get($collection->id);
-                if ($binaries === null || $binaries->isEmpty()) {
-                    return NzbCreationResult::deterministic("Collection {$collection->id} has no binaries.", $collectionIds, $path);
-                }
-
-                $groups = $this->groupsFromXref((string) $collection->xref);
-                if ($groups === []) {
-                    return NzbCreationResult::deterministic("Collection {$collection->id} has no valid xref groups.", $collectionIds, $path);
-                }
-
-                $poster = $collection->fromname;
-
-                foreach ($binaries as $binary) {
-                    $parts = $partsByBinary->get($binary->id);
-                    if ($parts === null || $parts->isEmpty()) {
-                        return NzbCreationResult::deterministic("Binary {$binary->id} has no parts.", $collectionIds, $path);
-                    }
-
-                    $subject = $this->buildBinarySubject($binary->name, $binary->totalparts);
-                    $XMLWriter->startElement('file');
-                    $XMLWriter->writeAttribute('poster', (string) $poster);
-                    $XMLWriter->writeAttribute('date', (string) $collection->udate);
-                    $XMLWriter->writeAttribute('subject', (string) $subject);
-                    $XMLWriter->startElement('groups');
-                    foreach ($groups as $group) {
-                        $XMLWriter->writeElement('group', $group);
-                    }
-                    $XMLWriter->endElement(); // groups
-                    $XMLWriter->startElement('segments');
-                    foreach ($parts as $part) {
-                        $messageId = $this->normalizeSegmentMessageId($part->messageid);
-                        if ($messageId === '') {
-                            return NzbCreationResult::deterministic("Part {$part->partnumber} for binary {$binary->id} has an empty message ID.", $collectionIds, $path);
+            $cursor = ['collection_id' => 0, 'name' => '', 'binary_id' => 0, 'partnumber' => 0];
+            $openBinaryId = 0;
+            do {
+                $page = $this->loadNzbRowPage((int) $release->id, $cursor);
+                foreach ($page as $row) {
+                    $binaryId = (int) $row->binary_id;
+                    if ($binaryId !== $openBinaryId) {
+                        if ($openBinaryId > 0) {
+                            $XMLWriter->endElement(); // segments
+                            $XMLWriter->endElement(); // file
+                            if (! $this->flushXmlWriter($XMLWriter, $gz)) {
+                                return NzbCreationResult::transient("Failed to write NZB file entry to temporary file: {$tempPath}", $collectionIds, $path);
+                            }
                         }
 
-                        $XMLWriter->startElement('segment');
-                        $XMLWriter->writeAttribute('bytes', (string) $part->size);
-                        $XMLWriter->writeAttribute('number', (string) $part->partnumber);
-                        $XMLWriter->text($messageId);
-                        $XMLWriter->endElement();
-                    }
-                    $XMLWriter->endElement(); // segments
-                    $XMLWriter->endElement(); // file
+                        $collection = $collections->get((int) $row->collection_id);
+                        $groups = $groupsByCollection[(int) $row->collection_id] ?? [];
+                        if ($collection === null || $groups === []) {
+                            return NzbCreationResult::deterministic("Collection {$row->collection_id} has no valid cross-post groups.", $collectionIds, $path);
+                        }
 
-                    if (! $this->flushXmlWriter($XMLWriter, $gz)) {
-                        return NzbCreationResult::transient("Failed to write NZB file entry to temporary file: {$tempPath}", $collectionIds, $path);
+                        $subject = $this->buildBinarySubject((string) $row->binary_name, (int) $row->totalparts);
+                        $XMLWriter->startElement('file');
+                        $XMLWriter->writeAttribute('poster', (string) $collection->fromname);
+                        $XMLWriter->writeAttribute('date', (string) $collection->udate);
+                        $XMLWriter->writeAttribute('subject', (string) $subject);
+                        $XMLWriter->startElement('groups');
+                        foreach ($groups as $group) {
+                            $XMLWriter->writeElement('group', $group);
+                        }
+                        $XMLWriter->endElement(); // groups
+                        $XMLWriter->startElement('segments');
+                        $openBinaryId = $binaryId;
                     }
+
+                    $messageId = $this->normalizeSegmentMessageId($row->messageid);
+                    if ($messageId === '') {
+                        return NzbCreationResult::deterministic("Part {$row->partnumber} for binary {$binaryId} has an empty message ID.", $collectionIds, $path);
+                    }
+
+                    $XMLWriter->startElement('segment');
+                    $XMLWriter->writeAttribute('bytes', (string) $row->size);
+                    $XMLWriter->writeAttribute('number', (string) $row->partnumber);
+                    $XMLWriter->text($messageId);
+                    $XMLWriter->endElement();
+
+                    $cursor = [
+                        'collection_id' => (int) $row->collection_id,
+                        'name' => (string) $row->binary_name,
+                        'binary_id' => $binaryId,
+                        'partnumber' => (int) $row->partnumber,
+                    ];
                 }
+            } while (\count($page) === $this->binariesConfig->nzbStreamRows);
+
+            if ($openBinaryId > 0) {
+                $XMLWriter->endElement(); // segments
+                $XMLWriter->endElement(); // file
             }
 
             $XMLWriter->writeComment($this->siteCommentString);
@@ -279,6 +295,67 @@ class NzbService
         return NzbCreationResult::success($path, $collectionIds);
     }
 
+    /**
+     * @param  \Illuminate\Support\Collection<int|string, mixed>  $collections
+     * @return array<int, list<string>>
+     */
+    private function loadCollectionGroups(int $releaseId, \Illuminate\Support\Collection $collections): array
+    {
+        $groups = [];
+        if (Schema::hasTable('collection_groups')) {
+            $rows = DB::select(
+                'SELECT cg.collections_id, cg.group_name
+                 FROM collection_groups cg INNER JOIN collections c ON c.id = cg.collections_id
+                 WHERE c.releases_id = ? ORDER BY cg.collections_id, cg.group_name',
+                [$releaseId]
+            );
+            foreach ($rows as $row) {
+                $groups[(int) $row->collections_id][] = (string) $row->group_name;
+            }
+        }
+
+        foreach ($collections as $collection) {
+            $collectionId = (int) $collection->id;
+            if (($groups[$collectionId] ?? []) === []) {
+                $groups[$collectionId] = $this->groupsFromXref((string) $collection->xref);
+            }
+        }
+
+        return $groups;
+    }
+
+    /**
+     * @param  array{collection_id: int, name: string, binary_id: int, partnumber: int}  $cursor
+     * @return list<object>
+     */
+    private function loadNzbRowPage(int $releaseId, array $cursor): array
+    {
+        $limit = $this->binariesConfig->nzbStreamRows;
+
+        return DB::select(
+            'SELECT b.collections_id AS collection_id, b.id AS binary_id, b.name AS binary_name,
+                    b.totalparts, p.messageid, p.size, p.partnumber
+             FROM binaries b
+             INNER JOIN collections c ON c.id = b.collections_id
+             INNER JOIN parts p ON p.binaries_id = b.id
+             WHERE c.releases_id = ? AND (
+                b.collections_id > ? OR
+                (b.collections_id = ? AND b.name > ?) OR
+                (b.collections_id = ? AND b.name = ? AND b.id > ?) OR
+                (b.collections_id = ? AND b.name = ? AND b.id = ? AND p.partnumber > ?)
+             )
+             ORDER BY b.collections_id, b.name, b.id, p.partnumber
+             LIMIT '.$limit,
+            [
+                $releaseId,
+                $cursor['collection_id'],
+                $cursor['collection_id'], $cursor['name'],
+                $cursor['collection_id'], $cursor['name'], $cursor['binary_id'],
+                $cursor['collection_id'], $cursor['name'], $cursor['binary_id'], $cursor['partnumber'],
+            ]
+        );
+    }
+
     /**
      * Build a folder path on the hard drive where the NZB file will be stored.
      *

app/Services/RegexService.php

@@ -181,7 +181,7 @@ class RegexService
         $rows = DB::select(
             'SELECT
                     b.name, b.totalparts, b.currentparts, HEX(b.binaryhash) AS binaryhash,
-                    c.fromname, c.collectionhash
+                    c.fromname, HEX(c.collectionhash) AS collectionhash
                 FROM binaries b
                 INNER JOIN collections c ON c.id = b.collections_id'
         );

app/Services/ReleaseCreationService.php

@@ -10,7 +10,6 @@ use App\Models\Collection;
 use App\Models\Predb;
 use App\Models\Release;
 use App\Models\ReleaseRegex;
-use App\Models\ReleasesGroups;
 use App\Models\UsenetGroup;
 use App\Services\Categorization\CategorizationService;
 use App\Services\Nzb\NzbService;
@@ -18,6 +17,7 @@ use App\Services\Releases\ReleaseDuplicateFinder;
 use App\Support\Utf8;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Schema;
 use Illuminate\Support\Str;
 
 class ReleaseCreationService
@@ -56,6 +56,7 @@ class ReleaseCreationService
             ->join('usenet_groups', 'usenet_groups.id', '=', 'collections.groups_id')
             ->limit($limit);
         $collections = $collectionsQuery->get();
+        $releaseGroupIds = $this->loadReleaseGroupIds($collections);
 
         if ($echoCLI && $collections->count() > 0) {
             cli()->primary(\count($collections).' Collections ready to be converted to releases.', true);
@@ -137,38 +138,12 @@ class ReleaseCreationService
                         'naming_regex_id' => $namingRegexId,
                     ]);
 
-                    if (preg_match_all('#(\S+):\S+#', $collection->xref, $hits)) {
-                        foreach ($hits[1] as $grp) {
-                            $grpTmp = UsenetGroup::isValidGroup($grp);
-                            if ($grpTmp !== false) {
-                                $xrefGrpID = UsenetGroup::getIDByName($grpTmp);
-                                if ($xrefGrpID === '') {
-                                    $xrefGrpID = UsenetGroup::addGroup([
-                                        'name' => $grpTmp,
-                                        'description' => 'Added by Release processing',
-                                        'backfill_target' => 1,
-                                        'first_record' => 0,
-                                        'last_record' => 0,
-                                        'active' => 0,
-                                        'backfill' => 0,
-                                        'minfilestoformrelease' => '',
-                                        'minsizetoformrelease' => '',
-                                    ]);
-                                }
-
-                                $relGroupsChk = ReleasesGroups::query()->where([
-                                    ['releases_id', '=', $releaseID],
-                                    ['groups_id', '=', $xrefGrpID],
-                                ])->first();
-
-                                if ($relGroupsChk === null) {
-                                    ReleasesGroups::query()->insert([
-                                        'releases_id' => $releaseID,
-                                        'groups_id' => $xrefGrpID,
-                                    ]);
-                                }
-                            }
-                        }
+                    $groupRows = [];
+                    foreach ($releaseGroupIds[(int) $collection->id] ?? [] as $groupId) {
+                        $groupRows[] = ['releases_id' => $releaseID, 'groups_id' => $groupId];
+                    }
+                    if ($groupRows !== []) {
+                        DB::table('releases_groups')->insertOrIgnore($groupRows);
                     }
 
                     $returnCount++;
@@ -215,4 +190,75 @@ class ReleaseCreationService
 
         return ['added' => $returnCount, 'dupes' => $duplicate];
     }
+
+    /**
+     * Resolve all normalized cross-post groups for the selected collection
+     * batch up front, avoiding release/group membership N+1 queries.
+     *
+     * @param  \Illuminate\Support\Collection<int, Collection>  $collections
+     * @return array<int, list<int>>
+     */
+    private function loadReleaseGroupIds(\Illuminate\Support\Collection $collections): array
+    {
+        $namesByCollection = [];
+        $collectionIds = $collections->pluck('id')->map(static fn ($id): int => (int) $id)->all();
+        if (Schema::hasTable('collection_groups')) {
+            foreach (array_chunk($collectionIds, 500) as $chunk) {
+                foreach (DB::table('collection_groups')->whereIn('collections_id', $chunk)->get() as $row) {
+                    $namesByCollection[(int) $row->collections_id][(string) $row->group_name] = true;
+                }
+            }
+        }
+
+        foreach ($collections as $collection) {
+            $collectionId = (int) $collection->id;
+            if (($namesByCollection[$collectionId] ?? []) === []) {
+                $fallback = (new XrefService)->extractGroupNames((string) $collection->xref);
+                if ($fallback === []) {
+                    $fallback = [(string) $collection->gname];
+                }
+                foreach ($fallback as $name) {
+                    $namesByCollection[$collectionId][$name] = true;
+                }
+            }
+        }
+
+        $allNames = [];
+        foreach ($namesByCollection as $names) {
+            $allNames += $names;
+        }
+        $idsByName = UsenetGroup::query()
+            ->whereIn('name', array_keys($allNames))
+            ->pluck('id', 'name')
+            ->map(static fn ($id): int => (int) $id)
+            ->all();
+        foreach (array_keys($allNames) as $name) {
+            $validName = UsenetGroup::isValidGroup($name);
+            if ($validName === false || isset($idsByName[$validName])) {
+                continue;
+            }
+            $idsByName[$validName] = (int) UsenetGroup::addGroup([
+                'name' => $validName,
+                'description' => 'Added by Release processing',
+                'backfill_target' => 1,
+                'first_record' => 0,
+                'last_record' => 0,
+                'active' => 0,
+                'backfill' => 0,
+                'minfilestoformrelease' => '',
+                'minsizetoformrelease' => '',
+            ]);
+        }
+
+        $resolved = [];
+        foreach ($namesByCollection as $collectionId => $names) {
+            foreach (array_keys($names) as $name) {
+                if (isset($idsByName[$name])) {
+                    $resolved[$collectionId][] = $idsByName[$name];
+                }
+            }
+        }
+
+        return $resolved;
+    }
 }

app/Services/ReleaseProcessingService.php

@@ -5,13 +5,13 @@ declare(strict_types=1);
 namespace App\Services;
 
 use App\Enums\CollectionFileCheckStatus;
-use App\Enums\FileCompletionStatus;
 use App\Models\Category;
 use App\Models\Collection;
 use App\Models\MusicInfo;
 use App\Models\Release;
 use App\Models\Settings;
 use App\Models\UsenetGroup;
+use App\Services\Binaries\BinariesConfig;
 use App\Services\Categorization\CategorizationService;
 use App\Services\NNTP\NNTPService;
 use App\Services\Nzb\NzbCreationCandidateQuery;
@@ -25,10 +25,11 @@ use App\Support\Data\ReleaseCreationResult;
 use App\Support\Data\ReleaseDeleteStats;
 use App\Support\ReleaseSearchIndexSync;
 use DateTimeInterface;
-use Illuminate\Database\QueryException;
+use Illuminate\Database\Query\Builder;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Schema;
 use Throwable;
 
 /**
@@ -74,6 +75,8 @@ final class ReleaseProcessingService
 
     private readonly ?PostProcessService $postProcessService;
 
+    private readonly BinariesConfig $binariesConfig;
+
     public function __construct(
         ?NzbService $nzb = null,
         ?ReleaseCleaningService $releaseCleaning = null,
@@ -82,6 +85,7 @@ final class ReleaseProcessingService
         ?ReleaseCreationService $releaseCreationService = null,
         ?CollectionCleanupService $collectionCleanupService = null,
         ?PostProcessService $postProcessService = null,
+        ?BinariesConfig $binariesConfig = null,
     ) {
         $this->echoCLI = (bool) config('nntmux.echocli');
 
@@ -99,6 +103,7 @@ final class ReleaseProcessingService
                 app(ReleaseDuplicateFinder::class)
             );
         $this->postProcessService = $postProcessService;
+        $this->binariesConfig = $binariesConfig ?? BinariesConfig::fromSettings();
 
         $this->settings = $this->loadSettings();
         $this->validateSettings();
@@ -368,15 +373,8 @@ final class ReleaseProcessingService
         $this->outputSubHeader('Finding Complete Collections');
 
         $normalizedGroupId = $this->normalizeGroupId($groupID);
-        $whereSql = $this->buildGroupWhereSql($normalizedGroupId, 'c');
-
         $this->processStuckCollections($normalizedGroupId ?? 0);
-        $this->runCollectionFileCheckStage1($normalizedGroupId ?? 0);
-        $this->runCollectionFileCheckStage2($normalizedGroupId ?? 0);
-        $this->runCollectionFileCheckStage3($whereSql);
-        $this->runCollectionFileCheckStage4($whereSql);
-        $this->runCollectionFileCheckStage5($normalizedGroupId ?? 0);
-        $this->runCollectionFileCheckStage6($whereSql);
+        $this->reconcileIncompleteCollections($normalizedGroupId);
 
         $count = $this->countCompleteCollections($normalizedGroupId);
         $this->outputStat('Complete collections found', $count);
@@ -394,34 +392,192 @@ final class ReleaseProcessingService
         $this->outputSubHeader('Calculating Collection Sizes');
 
         $updated = 0;
-        DB::transaction(function () use ($groupID, &$updated): void {
-            $normalizedGroupId = $this->normalizeGroupId($groupID);
-            $whereSql = $normalizedGroupId !== null
-                ? " AND c.groups_id = {$normalizedGroupId} "
-                : ' ';
-
-            $sql = <<<SQL
-                UPDATE collections c
-                SET c.filesize = (
-                    SELECT COALESCE(SUM(b.partsize), 0)
-                    FROM binaries b
-                    WHERE b.collections_id = c.id
-                ),
-                c.filecheck = ?
-                WHERE c.filecheck = ?
-                AND c.filesize = 0{$whereSql}
-            SQL;
-
-            $updated = DB::update($sql, [
-                CollectionFileCheckStatus::Sized->value,
-                CollectionFileCheckStatus::CompleteParts->value,
+        $lastId = 0;
+        $normalizedGroupId = $this->normalizeGroupId($groupID);
+        do {
+            $query = Collection::query()
+                ->where('id', '>', $lastId)
+                ->where('filecheck', CollectionFileCheckStatus::CompleteParts->value)
+                ->when($normalizedGroupId !== null, static fn ($q) => $q->where('groups_id', $normalizedGroupId))
+                ->orderBy('id')
+                ->limit($this->binariesConfig->reconcileBatchSize);
+            $ids = $query->pluck('id')->map(static fn ($id): int => (int) $id)->all();
+            if ($ids === []) {
+                break;
+            }
+            $lastId = (int) end($ids);
+            $updated += Collection::query()->whereIn('id', $ids)->update([
+                'filecheck' => CollectionFileCheckStatus::Sized->value,
             ]);
-        }, 10);
+        } while (\count($ids) === $this->binariesConfig->reconcileBatchSize);
 
         $this->outputStat('Collections sized', $updated);
         $this->outputElapsedTime($startTime);
     }
 
+    /**
+     * Reconcile only a bounded keyset page at a time. Stored parts are the
+     * authority for binary counts/sizes; binary aggregates are then the
+     * authority for collection readiness and filesize.
+     */
+    private function reconcileIncompleteCollections(?int $groupId): void
+    {
+        $lastId = 0;
+        $statuses = [
+            CollectionFileCheckStatus::Default->value,
+            CollectionFileCheckStatus::CompleteCollection->value,
+            CollectionFileCheckStatus::CompleteParts->value,
+            CollectionFileCheckStatus::TempComplete->value,
+            CollectionFileCheckStatus::ZeroPart->value,
+            10,
+        ];
+
+        do {
+            $query = Collection::query()
+                ->where('id', '>', $lastId)
+                ->where(function ($query) use ($statuses): void {
+                    $query->where('filecheck', CollectionFileCheckStatus::CompleteParts->value);
+                    if (Schema::hasColumn('collections', 'last_seen_at')) {
+                        $query->orWhere(function ($stale) use ($statuses): void {
+                            $stale->whereIn('filecheck', array_values(array_diff(
+                                $statuses,
+                                [CollectionFileCheckStatus::CompleteParts->value]
+                            )))->whereRaw(
+                                'COALESCE(last_seen_at, dateadded, added) < ?',
+                                [now()->subHours($this->settings->collectionDelayTime)]
+                            );
+                        });
+                    } else {
+                        $query->orWhereIn('filecheck', array_values(array_diff(
+                            $statuses,
+                            [CollectionFileCheckStatus::CompleteParts->value]
+                        )));
+                    }
+                })
+                ->when($groupId !== null, static fn ($q) => $q->where('groups_id', $groupId))
+                ->orderBy('id')
+                ->limit($this->binariesConfig->reconcileBatchSize);
+            $ids = $query->pluck('id')->map(static fn ($id): int => (int) $id)->all();
+            if ($ids === []) {
+                break;
+            }
+            $lastId = (int) end($ids);
+            $this->reconcileCollectionIds($ids, $statuses);
+        } while (\count($ids) === $this->binariesConfig->reconcileBatchSize);
+    }
+
+    /**
+     * @param  list<int>  $collectionIds
+     * @param  list<int>  $statuses
+     */
+    private function reconcileCollectionIds(array $collectionIds, array $statuses): void
+    {
+        if (DB::getDriverName() === 'sqlite') {
+            $this->reconcileCollectionIdsSqlite($collectionIds, $statuses);
+
+            return;
+        }
+
+        $idPlaceholders = implode(',', array_fill(0, \count($collectionIds), '?'));
+        $statusPlaceholders = implode(',', array_fill(0, \count($statuses), '?'));
+
+        DB::transaction(function () use ($collectionIds, $idPlaceholders, $statuses, $statusPlaceholders): void {
+            DB::update(
+                "UPDATE binaries b
+                 LEFT JOIN (
+                    SELECT p.binaries_id, COUNT(*) currentparts, COALESCE(SUM(p.size), 0) partsize
+                    FROM parts p INNER JOIN binaries selected ON selected.id = p.binaries_id
+                    WHERE selected.collections_id IN ({$idPlaceholders}) GROUP BY p.binaries_id
+                 ) p ON p.binaries_id = b.id
+                 SET b.currentparts = COALESCE(p.currentparts, 0),
+                     b.partsize = COALESCE(p.partsize, 0),
+                     b.partcheck = CASE WHEN COALESCE(p.currentparts, 0) >= b.totalparts THEN 1 ELSE 0 END
+                 WHERE b.collections_id IN ({$idPlaceholders})",
+                [...$collectionIds, ...$collectionIds]
+            );
+
+            DB::update(
+                "UPDATE collections c
+                 LEFT JOIN (
+                    SELECT b.collections_id, COUNT(*) currentfiles,
+                           COALESCE(SUM(CASE WHEN b.partcheck = 1 THEN 1 ELSE 0 END), 0) completefiles,
+                           COALESCE(SUM(b.partsize), 0) filesize
+                    FROM binaries b WHERE b.collections_id IN ({$idPlaceholders}) GROUP BY b.collections_id
+                 ) a ON a.collections_id = c.id
+                 SET c.filesize = COALESCE(a.filesize, 0),
+                     c.totalfiles = CASE
+                        WHEN c.dateadded < DATE_SUB(NOW(), INTERVAL ? HOUR)
+                         AND c.filecheck IN (0, 1, 10)
+                        THEN COALESCE(a.currentfiles, 0) ELSE c.totalfiles END,
+                     c.filecheck = CASE
+                        WHEN c.totalfiles > 0
+                         AND COALESCE(a.currentfiles, 0) IN (c.totalfiles, c.totalfiles + 1)
+                         AND COALESCE(a.completefiles, 0) >= c.totalfiles THEN ?
+                        WHEN c.dateadded < DATE_SUB(NOW(), INTERVAL ? HOUR)
+                         AND c.filecheck IN (0, 1, 10) THEN ?
+                        ELSE c.filecheck END
+                 WHERE c.id IN ({$idPlaceholders}) AND c.filecheck IN ({$statusPlaceholders})",
+                [
+                    ...$collectionIds,
+                    $this->settings->collectionDelayTime,
+                    CollectionFileCheckStatus::CompleteParts->value,
+                    $this->settings->collectionDelayTime,
+                    CollectionFileCheckStatus::CompleteParts->value,
+                    ...$collectionIds,
+                    ...$statuses,
+                ]
+            );
+        }, self::MAX_RETRIES);
+    }
+
+    /**
+     * @param  list<int>  $collectionIds
+     * @param  list<int>  $statuses
+     */
+    private function reconcileCollectionIdsSqlite(array $collectionIds, array $statuses): void
+    {
+        DB::transaction(function () use ($collectionIds, $statuses): void {
+            foreach ($collectionIds as $collectionId) {
+                $binaryIds = DB::table('binaries')->where('collections_id', $collectionId)->pluck('id')->all();
+                foreach ($binaryIds as $binaryId) {
+                    $aggregate = DB::selectOne(
+                        'SELECT COUNT(*) currentparts, COALESCE(SUM(size), 0) partsize FROM parts WHERE binaries_id = ?',
+                        [$binaryId]
+                    );
+                    DB::update(
+                        'UPDATE binaries SET currentparts = ?, partsize = ?, partcheck = CASE WHEN ? >= totalparts THEN 1 ELSE 0 END WHERE id = ?',
+                        [(int) $aggregate->currentparts, (int) $aggregate->partsize, (int) $aggregate->currentparts, $binaryId]
+                    );
+                }
+
+                $collection = DB::table('collections')->where('id', $collectionId)->first();
+                if ($collection === null || ! \in_array((int) $collection->filecheck, $statuses, true)) {
+                    continue;
+                }
+                $aggregate = DB::selectOne(
+                    'SELECT COUNT(*) currentfiles,
+                            COALESCE(SUM(CASE WHEN partcheck = 1 THEN 1 ELSE 0 END), 0) completefiles,
+                            COALESCE(SUM(partsize), 0) filesize
+                     FROM binaries WHERE collections_id = ?',
+                    [$collectionId]
+                );
+                $stale = strtotime((string) $collection->dateadded) < now()->subHours($this->settings->collectionDelayTime)->timestamp
+                    && \in_array((int) $collection->filecheck, [0, 1, 10], true);
+                $totalFiles = $stale ? (int) $aggregate->currentfiles : (int) $collection->totalfiles;
+                $ready = $totalFiles > 0
+                    && \in_array((int) $aggregate->currentfiles, [$totalFiles, $totalFiles + 1], true)
+                    && (int) $aggregate->completefiles >= $totalFiles;
+                DB::table('collections')->where('id', $collectionId)->update([
+                    'filesize' => (int) $aggregate->filesize,
+                    'totalfiles' => $totalFiles,
+                    'filecheck' => $ready || $stale
+                        ? CollectionFileCheckStatus::CompleteParts->value
+                        : (int) $collection->filecheck,
+                ]);
+            }
+        }, self::MAX_RETRIES);
+    }
+
     /**
      * Delete collections that don't meet size/file count requirements.
      *
@@ -445,24 +601,18 @@ final class ReleaseProcessingService
             ->where('c.filecheck', CollectionFileCheckStatus::Sized->value)
             ->where('c.filesize', '>', 0)
             ->groupBy('c.id')
-            ->havingRaw("COUNT(b.id) = SUM(CASE WHEN b.name REGEXP '\\\\.(vol[0-9]+\\\\+[0-9]+\\\\.par2|par2)' THEN 1 ELSE 0 END)")
-            ->pluck('c.id')
-            ->all();
-
-        if ($par2OnlyCollectionIds !== []) {
-            $stats['par2Only'] += $this->collectionCleanupService->deleteCollectionsAndDescendants(
-                $par2OnlyCollectionIds,
-                'Par2-only cleanup',
-                $this->echoCLI
-            );
+            ->havingRaw("COUNT(b.id) = SUM(CASE WHEN b.name REGEXP '\\\\.(vol[0-9]+\\\\+[0-9]+\\\\.par2|par2)' THEN 1 ELSE 0 END)");
+        if ($normalizedGroupId !== null) {
+            $par2OnlyCollectionIds->where('c.groups_id', $normalizedGroupId);
         }
+        $stats['par2Only'] += $this->deleteCollectionQueryInBatches($par2OnlyCollectionIds, 'Par2-only cleanup');
 
         foreach ($groupIDs as $grpID) {
             $groupSettings = UsenetGroup::getGroupByID($grpID['id']);
             $groupMinSize = (int) ($groupSettings['minsizetoformrelease'] ?? 0);
             $groupMinFiles = (int) ($groupSettings['minfilestoformrelease'] ?? 0);
 
-            if (! $this->hasSizedCollections()) {
+            if (! $this->hasSizedCollections((int) $grpID['id'])) {
                 continue;
             }
 
@@ -470,43 +620,28 @@ final class ReleaseProcessingService
             if ($effectiveMinSize > 0) {
                 $ids = Collection::query()
                     ->where('filecheck', CollectionFileCheckStatus::Sized->value)
+                    ->where('groups_id', (int) $grpID['id'])
                     ->where('filesize', '>', 0)
-                    ->where('filesize', '<', $effectiveMinSize)
-                    ->pluck('id')
-                    ->all();
-                $stats['minSize'] += $this->collectionCleanupService->deleteCollectionsAndDescendants(
-                    $ids,
-                    'Min-size cleanup',
-                    $this->echoCLI
-                );
+                    ->where('filesize', '<', $effectiveMinSize);
+                $stats['minSize'] += $this->deleteCollectionQueryInBatches($ids, 'Min-size cleanup');
             }
 
             if ($this->settings->maxSizeToFormRelease > 0) {
                 $ids = Collection::query()
                     ->where('filecheck', CollectionFileCheckStatus::Sized->value)
-                    ->where('filesize', '>', $this->settings->maxSizeToFormRelease)
-                    ->pluck('id')
-                    ->all();
-                $stats['maxSize'] += $this->collectionCleanupService->deleteCollectionsAndDescendants(
-                    $ids,
-                    'Max-size cleanup',
-                    $this->echoCLI
-                );
+                    ->where('groups_id', (int) $grpID['id'])
+                    ->where('filesize', '>', $this->settings->maxSizeToFormRelease);
+                $stats['maxSize'] += $this->deleteCollectionQueryInBatches($ids, 'Max-size cleanup');
             }
 
             $effectiveMinFiles = max($groupMinFiles, $this->settings->minFilesToFormRelease);
             if ($effectiveMinFiles > 0) {
                 $ids = Collection::query()
                     ->where('filecheck', CollectionFileCheckStatus::Sized->value)
+                    ->where('groups_id', (int) $grpID['id'])
                     ->where('filesize', '>', 0)
-                    ->where('totalfiles', '<', $effectiveMinFiles)
-                    ->pluck('id')
-                    ->all();
-                $stats['minFiles'] += $this->collectionCleanupService->deleteCollectionsAndDescendants(
-                    $ids,
-                    'Min-files cleanup',
-                    $this->echoCLI
-                );
+                    ->where('totalfiles', '<', $effectiveMinFiles);
+                $stats['minFiles'] += $this->deleteCollectionQueryInBatches($ids, 'Min-files cleanup');
             }
         }
 
@@ -823,14 +958,41 @@ final class ReleaseProcessingService
         return $query->count('id');
     }
 
-    private function hasSizedCollections(): bool
+    private function hasSizedCollections(int $groupId): bool
     {
         return Collection::query()
             ->where('filecheck', CollectionFileCheckStatus::Sized->value)
+            ->where('groups_id', $groupId)
             ->where('filesize', '>', 0)
             ->exists();
     }
 
+    /** @param Builder|\Illuminate\Database\Eloquent\Builder<Collection> $query */
+    private function deleteCollectionQueryInBatches(
+        Builder|\Illuminate\Database\Eloquent\Builder $query,
+        string $label,
+    ): int {
+        $deleted = 0;
+        $idColumn = $query instanceof \Illuminate\Database\Eloquent\Builder ? 'id' : 'c.id';
+        do {
+            $ids = (clone $query)->orderBy($idColumn)->limit(self::BATCH_SIZE)->pluck($idColumn)->all();
+            if ($ids === []) {
+                break;
+            }
+            $affected = $this->collectionCleanupService->deleteCollectionsAndDescendants(
+                array_map('intval', $ids),
+                $label,
+                $this->echoCLI
+            );
+            $deleted += $affected;
+            if ($affected < self::BATCH_SIZE) {
+                break;
+            }
+        } while (true);
+
+        return $deleted;
+    }
+
     private function normalizeGroupId(int|string|null $groupID): ?int
     {
         if ($groupID === null || $groupID === '') {
@@ -846,240 +1008,6 @@ final class ReleaseProcessingService
         return $groupInfo !== null ? (int) $groupInfo['id'] : null;
     }
 
-    private function buildGroupWhereSql(?int $groupID, string $alias = 'c'): string
-    {
-        return $groupID !== null ? " AND {$alias}.groups_id = {$groupID} " : ' ';
-    }
-
-    // ========================================================================
-    // Collection Processing Stages
-    // ========================================================================
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage1(int $groupID): void
-    {
-        DB::transaction(static function () use ($groupID): void {
-            $collectionsQuery = Collection::query()
-                ->select(['collections.id'])
-                ->join('binaries', 'binaries.collections_id', '=', 'collections.id')
-                ->where('collections.totalfiles', '>', 0)
-                ->where('collections.filecheck', '=', CollectionFileCheckStatus::Default->value)
-                ->groupBy(['binaries.collections_id', 'collections.totalfiles', 'collections.id'])
-                ->havingRaw('COUNT(binaries.id) IN (collections.totalfiles, collections.totalfiles + 1)');
-
-            if ($groupID !== 0) {
-                $collectionsQuery->where('collections.groups_id', $groupID);
-            }
-
-            Collection::query()
-                ->joinSub($collectionsQuery, 'r', static fn ($join) => $join->on('collections.id', '=', 'r.id'))
-                ->update(['collections.filecheck' => CollectionFileCheckStatus::CompleteCollection->value]);
-        }, 10);
-    }
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage2(int $groupID): void
-    {
-        DB::transaction(static function () use ($groupID): void {
-            $collectionsQuery = Collection::query()
-                ->select(['collections.id'])
-                ->join('binaries', 'binaries.collections_id', '=', 'collections.id')
-                ->where('binaries.filenumber', '=', 0)
-                ->where('collections.totalfiles', '>', 0)
-                ->where('collections.filecheck', '=', CollectionFileCheckStatus::CompleteCollection->value)
-                ->groupBy(['collections.id']);
-
-            if ($groupID !== 0) {
-                $collectionsQuery->where('collections.groups_id', $groupID);
-            }
-
-            Collection::query()
-                ->joinSub($collectionsQuery, 'r', static fn ($join) => $join->on('collections.id', '=', 'r.id'))
-                ->update(['collections.filecheck' => CollectionFileCheckStatus::ZeroPart->value]);
-        }, 10);
-
-        $this->updateCollectionsFilecheckInChunks(
-            $groupID,
-            CollectionFileCheckStatus::CompleteCollection->value,
-            CollectionFileCheckStatus::TempComplete->value
-        );
-    }
-
-    /**
-     * Update collections filecheck in small chunks to reduce deadlock risk.
-     * Retries on deadlock (1213) with exponential backoff.
-     *
-     * @throws Throwable
-     */
-    private function updateCollectionsFilecheckInChunks(int $groupID, int $fromStatus, int $toStatus): void
-    {
-        $attempt = 0;
-        $maxAttempts = self::MAX_RETRIES + 1;
-
-        while ($attempt < $maxAttempts) {
-            try {
-                $updated = 0;
-                do {
-                    $ids = Collection::query()
-                        ->where('filecheck', $fromStatus)
-                        ->when($groupID !== 0, static fn ($q) => $q->where('groups_id', $groupID))
-                        ->orderBy('id')
-                        ->limit(self::BATCH_SIZE)
-                        ->pluck('id')
-                        ->all();
-
-                    if ($ids === []) {
-                        break;
-                    }
-
-                    DB::transaction(static function () use ($ids, $toStatus): void {
-                        Collection::query()
-                            ->whereIn('id', $ids)
-                            ->update(['filecheck' => $toStatus]);
-                    }, 10);
-
-                    $updated = \count($ids);
-                } while ($updated === self::BATCH_SIZE);
-
-                return;
-            } catch (QueryException $e) {
-                $isDeadlock = ($e->errorInfo[1] ?? 0) === 1213
-                    || str_contains($e->getMessage(), 'Deadlock');
-
-                if ($isDeadlock && $attempt < $maxAttempts - 1) {
-                    $attempt++;
-                    usleep(self::RETRY_BASE_DELAY_US * (2 ** $attempt));
-                } else {
-                    throw $e;
-                }
-            }
-        }
-    }
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage3(string $whereSql): void
-    {
-        DB::transaction(static function () use ($whereSql): void {
-            $sql = <<<SQL
-                UPDATE binaries b
-                INNER JOIN (
-                    SELECT b.id
-                    FROM binaries b
-                    INNER JOIN collections c ON c.id = b.collections_id
-                    WHERE c.filecheck = ?
-                    AND b.partcheck = ? {$whereSql}
-                    AND b.currentparts = b.totalparts
-                    GROUP BY b.id, b.totalparts
-                ) r ON b.id = r.id
-                SET b.partcheck = ?
-            SQL;
-
-            DB::update($sql, [
-                CollectionFileCheckStatus::TempComplete->value,
-                FileCompletionStatus::Incomplete->value,
-                FileCompletionStatus::Complete->value,
-            ]);
-        }, 10);
-
-        DB::transaction(static function () use ($whereSql): void {
-            $sql = <<<SQL
-                UPDATE binaries b
-                INNER JOIN (
-                    SELECT b.id
-                    FROM binaries b
-                    INNER JOIN collections c ON c.id = b.collections_id
-                    WHERE c.filecheck = ?
-                    AND b.partcheck = ? {$whereSql}
-                    AND b.currentparts >= (b.totalparts + 1)
-                    GROUP BY b.id, b.totalparts
-                ) r ON b.id = r.id
-                SET b.partcheck = ?
-            SQL;
-
-            DB::update($sql, [
-                CollectionFileCheckStatus::ZeroPart->value,
-                FileCompletionStatus::Incomplete->value,
-                FileCompletionStatus::Complete->value,
-            ]);
-        }, 10);
-    }
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage4(string $whereSql): void
-    {
-        DB::transaction(static function () use ($whereSql): void {
-            $sql = <<<SQL
-                UPDATE collections c
-                INNER JOIN (
-                    SELECT c.id
-                    FROM collections c
-                    INNER JOIN binaries b ON c.id = b.collections_id
-                    WHERE b.partcheck = ? AND c.filecheck IN (?, ?) {$whereSql}
-                    GROUP BY b.collections_id, c.totalfiles, c.id
-                    HAVING COUNT(b.id) >= c.totalfiles
-                ) r ON c.id = r.id
-                SET filecheck = ?
-            SQL;
-
-            DB::update($sql, [
-                FileCompletionStatus::Complete->value,
-                CollectionFileCheckStatus::TempComplete->value,
-                CollectionFileCheckStatus::ZeroPart->value,
-                CollectionFileCheckStatus::CompleteParts->value,
-            ]);
-        }, 10);
-    }
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage5(int $groupId): void
-    {
-        DB::transaction(static function () use ($groupId): void {
-            $query = Collection::query()
-                ->whereIn('filecheck', [
-                    CollectionFileCheckStatus::TempComplete->value,
-                    CollectionFileCheckStatus::ZeroPart->value,
-                ]);
-
-            if ($groupId !== 0) {
-                $query->where('groups_id', $groupId);
-            }
-
-            $query->update(['filecheck' => CollectionFileCheckStatus::CompleteCollection->value]);
-        }, 10);
-    }
-
-    /**
-     * @throws Throwable
-     */
-    private function runCollectionFileCheckStage6(string $whereSql): void
-    {
-        DB::transaction(function () use ($whereSql): void {
-            $sql = <<<SQL
-                UPDATE collections c
-                SET filecheck = ?, totalfiles = (SELECT COUNT(b.id) FROM binaries b WHERE b.collections_id = c.id)
-                WHERE c.dateadded < NOW() - INTERVAL ? HOUR
-                AND c.filecheck IN (?, ?, 10){$whereSql}
-            SQL;
-
-            DB::update($sql, [
-                CollectionFileCheckStatus::CompleteParts->value,
-                $this->settings->collectionDelayTime,
-                CollectionFileCheckStatus::Default->value,
-                CollectionFileCheckStatus::CompleteCollection->value,
-            ]);
-        }, 10);
-    }
-
     /**
      * @throws Throwable
      */
@@ -1128,9 +1056,20 @@ final class ReleaseProcessingService
         do {
             try {
                 $query = DB::table('collections')
-                    ->where('added', '<', $cutoff)
+                    ->whereIn('filecheck', [
+                        CollectionFileCheckStatus::Default->value,
+                        CollectionFileCheckStatus::CompleteCollection->value,
+                        CollectionFileCheckStatus::TempComplete->value,
+                        CollectionFileCheckStatus::ZeroPart->value,
+                        10,
+                    ])
                     ->orderBy('id')
                     ->limit(self::BATCH_SIZE);
+                if (Schema::hasColumn('collections', 'last_seen_at')) {
+                    $query->whereRaw('COALESCE(last_seen_at, dateadded, added) < ?', [$cutoff]);
+                } else {
+                    $query->where('added', '<', $cutoff);
+                }
                 if ($groupID !== 0) {
                     $query->where('groups_id', '=', $groupID);
                 }

app/Services/XrefService.php

@@ -45,4 +45,18 @@ class XrefService
 
         return array_values(array_diff($incoming, $existing));
     }
+
+    /** @return list<string> */
+    public function extractGroupNames(?string $xref): array
+    {
+        $groups = [];
+        foreach ($this->extractTokens($xref) as $token) {
+            $group = preg_replace('/:\d+$/', '', (string) $token);
+            if ($group !== null && $group !== '') {
+                $groups[$group] = true;
+            }
+        }
+
+        return array_keys($groups);
+    }
 }

config/nntmux.php

@@ -18,6 +18,18 @@ return [
     'multiprocessing_max_child_time' => env('NN_MULTIPROCESSING_MAX_CHILD_TIME', 1800),
     'concurrency_timeout' => env('NN_CONCURRENCY_TIMEOUT'),
     'stream_fork_output' => env('STREAM_FORK_OUTPUT', false),
+    'cbp' => [
+        // Bound the amount of header data retained and written in one transaction.
+        'header_chunk_size' => (int) env('CBP_HEADER_CHUNK_SIZE', 500),
+        // Hard limit for rows represented by a generated raw SQL statement.
+        'sql_chunk_size' => (int) env('CBP_SQL_CHUNK_SIZE', 500),
+        // Number of stale collections reconciled by one release-processing batch.
+        'reconcile_batch_size' => (int) env('CBP_RECONCILE_BATCH_SIZE', 500),
+        // Maximum flattened binary/part rows loaded while streaming an NZB.
+        'nzb_stream_rows' => (int) env('CBP_NZB_STREAM_ROWS', 5000),
+        // Explicit maintenance-window approval for the destructive hash/key migration.
+        'storage_migration_execute' => (bool) env('CBP_STORAGE_MIGRATION_EXECUTE', false),
+    ],
     'purge_inactive_users' => env('PURGE_INACTIVE_USERS', false),
     'purge_inactive_users_days' => env('PURGE_INACTIVE_USERS_DAYS', 180),
     'mysql_search_fallback' => env('MYSQL_SEARCH_FALLBACK', false), // Disable MySQL LIKE fallback when Manticore/Elasticsearch return no results

database/migrations/2026_08_03_000000_prepare_cbp_optimized_storage.php

@@ -0,0 +1,69 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        if (Schema::hasTable('collections') && ! Schema::hasColumn('collections', 'last_seen_at')) {
+            Schema::table('collections', function (Blueprint $table): void {
+                $table->dateTime('last_seen_at')->nullable()->after('dateadded');
+            });
+        }
+
+        if (! Schema::hasTable('collection_groups')) {
+            Schema::create('collection_groups', function (Blueprint $table): void {
+                $table->unsignedInteger('collections_id');
+                $table->string('group_name', 255);
+                $table->primary(['collections_id', 'group_name']);
+                $table->foreign('collections_id', 'fk_collection_groups_collection')
+                    ->references('id')->on('collections')
+                    ->cascadeOnUpdate()->cascadeOnDelete();
+            });
+        }
+
+        if (Schema::hasTable('collections') && ! $this->indexExists('collections', 'ix_collections_group_filecheck_seen_id')) {
+            Schema::table('collections', function (Blueprint $table): void {
+                $table->index(
+                    ['groups_id', 'filecheck', 'last_seen_at', 'id'],
+                    'ix_collections_group_filecheck_seen_id'
+                );
+            });
+        }
+    }
+
+    public function down(): void
+    {
+        if (Schema::hasTable('collection_groups')) {
+            Schema::drop('collection_groups');
+        }
+
+        if (Schema::hasTable('collections') && $this->indexExists('collections', 'ix_collections_group_filecheck_seen_id')) {
+            Schema::table('collections', function (Blueprint $table): void {
+                $table->dropIndex('ix_collections_group_filecheck_seen_id');
+            });
+        }
+
+        if (Schema::hasTable('collections') && Schema::hasColumn('collections', 'last_seen_at')) {
+            Schema::table('collections', function (Blueprint $table): void {
+                $table->dropColumn('last_seen_at');
+            });
+        }
+    }
+
+    private function indexExists(string $table, string $index): bool
+    {
+        if (DB::getDriverName() === 'sqlite') {
+            return DB::select(
+                "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND name = ?",
+                [$table, $index]
+            ) !== [];
+        }
+
+        return DB::select("SHOW INDEX FROM `{$table}` WHERE Key_name = ?", [$index]) !== [];
+    }
+};

database/migrations/2026_08_03_000001_finalize_cbp_binary_hash_storage.php

@@ -0,0 +1,175 @@
+<?php
+
+declare(strict_types=1);
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        if (DB::getDriverName() === 'sqlite' || ! $this->hasCbpTables()) {
+            return;
+        }
+
+        $issues = $this->schemaIssues();
+        if ($issues === []) {
+            return;
+        }
+
+        if (! config('nntmux.cbp.storage_migration_execute', false)) {
+            throw new RuntimeException(
+                "The CBP binary-hash storage migration requires an approved maintenance window.\n".
+                'Stop ingestion and queue workers, verify a backup, run `php artisan cbp:optimize-storage`, '.
+                'then set CBP_STORAGE_MIGRATION_EXECUTE=true and rerun `php artisan migrate`.'."\n".
+                'Pending changes: '.implode('; ', $issues)
+            );
+        }
+
+        $collectionCount = (int) DB::table('collections')->count();
+        $exitCode = Artisan::call('cbp:optimize-storage', [
+            '--execute' => true,
+            '--batch' => (int) config('nntmux.cbp.reconcile_batch_size', 500),
+        ]);
+
+        if ($exitCode !== 0) {
+            throw new RuntimeException(
+                "The resumable CBP storage optimizer failed. Fix the reported error and rerun the migration.\n".
+                Artisan::output()
+            );
+        }
+
+        $optimizedCollectionCount = (int) DB::table('collections')->count();
+        if ($optimizedCollectionCount !== $collectionCount) {
+            throw new RuntimeException(
+                "CBP storage optimization changed the collection row count ({$collectionCount} to {$optimizedCollectionCount}). Restore the backup before retrying."
+            );
+        }
+
+        $issues = $this->schemaIssues();
+        if ($issues !== []) {
+            throw new RuntimeException(
+                'CBP storage optimization finished without satisfying the migration contract: '.implode('; ', $issues)
+            );
+        }
+    }
+
+    public function down(): void
+    {
+        if (DB::getDriverName() === 'sqlite' || ! $this->hasCbpTables()) {
+            return;
+        }
+
+        throw new RuntimeException(
+            'The CBP binary-hash migration cannot be rolled back: duplicate binaries and parts were merged. Restore the pre-migration backup instead.'
+        );
+    }
+
+    private function hasCbpTables(): bool
+    {
+        return Schema::hasTable('collections')
+            && Schema::hasTable('binaries')
+            && Schema::hasTable('parts');
+    }
+
+    /** @return list<string> */
+    private function schemaIssues(): array
+    {
+        $issues = [];
+
+        if (! $this->binaryColumnIs('collections', 'collectionhash', 20)) {
+            $issues[] = 'collections.collectionhash must be BINARY(20)';
+        }
+        if (! $this->binaryColumnIs('binaries', 'binaryhash', 16)) {
+            $issues[] = 'binaries.binaryhash must be BINARY(16)';
+        }
+        if (! $this->indexMatches('binaries', 'ux_binaries_collection_hash', ['collections_id', 'binaryhash'], true)) {
+            $issues[] = 'binaries requires UNIQUE (collections_id, binaryhash)';
+        }
+        if (! $this->indexMatches('binaries', 'ix_binaries_collection_filenumber', ['collections_id', 'filenumber'], false)) {
+            $issues[] = 'binaries requires a non-unique (collections_id, filenumber) lookup index';
+        }
+        if ($this->hasUniqueIndexForColumns('binaries', ['collections_id', 'filenumber'])) {
+            $issues[] = 'the legacy unique (collections_id, filenumber) constraint must be removed';
+        }
+        if (! $this->indexMatches('parts', 'PRIMARY', ['binaries_id', 'partnumber'], true)) {
+            $issues[] = 'parts primary key must be (binaries_id, partnumber)';
+        }
+        if (! $this->indexMatches('parts', 'ix_parts_number', ['number'], false)) {
+            $issues[] = 'parts requires a non-unique number lookup index';
+        }
+        if (! $this->messageIdUsesAsciiBinaryCollation()) {
+            $issues[] = 'parts.messageid must use an ASCII binary collation';
+        }
+
+        return $issues;
+    }
+
+    private function binaryColumnIs(string $table, string $column, int $length): bool
+    {
+        $definition = DB::selectOne(
+            'SELECT DATA_TYPE AS data_type, CHARACTER_MAXIMUM_LENGTH AS max_length
+             FROM information_schema.COLUMNS
+             WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?',
+            [$table, $column]
+        );
+
+        return $definition !== null
+            && strtolower((string) $definition->data_type) === 'binary'
+            && (int) $definition->max_length === $length;
+    }
+
+    /** @param list<string> $columns */
+    private function indexMatches(string $table, string $name, array $columns, bool $unique): bool
+    {
+        $rows = DB::select(
+            'SELECT COLUMN_NAME AS column_name, NON_UNIQUE AS non_unique
+             FROM information_schema.STATISTICS
+             WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?
+             ORDER BY SEQ_IN_INDEX',
+            [$table, $name]
+        );
+
+        if ($rows === []) {
+            return false;
+        }
+
+        return array_map(static fn (object $row): string => (string) $row->column_name, $rows) === $columns
+            && ((int) $rows[0]->non_unique === ($unique ? 0 : 1));
+    }
+
+    /** @param list<string> $columns */
+    private function hasUniqueIndexForColumns(string $table, array $columns): bool
+    {
+        $rows = DB::select(
+            'SELECT INDEX_NAME AS index_name, COLUMN_NAME AS column_name
+             FROM information_schema.STATISTICS
+             WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND NON_UNIQUE = 0
+             ORDER BY INDEX_NAME, SEQ_IN_INDEX',
+            [$table]
+        );
+        $indexes = [];
+        foreach ($rows as $row) {
+            $indexes[(string) $row->index_name][] = (string) $row->column_name;
+        }
+
+        return in_array($columns, $indexes, true);
+    }
+
+    private function messageIdUsesAsciiBinaryCollation(): bool
+    {
+        $definition = DB::selectOne(
+            'SELECT CHARACTER_SET_NAME AS character_set_name, COLLATION_NAME AS collation_name
+             FROM information_schema.COLUMNS
+             WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?',
+            ['parts', 'messageid']
+        );
+
+        return $definition !== null
+            && strtolower((string) $definition->character_set_name) === 'ascii'
+            && str_ends_with(strtolower((string) $definition->collation_name), '_bin');
+    }
+};

database/schema/mariadb-schema.sql

@@ -66,7 +66,7 @@ DROP TABLE IF EXISTS `binaries`;
 
 CREATE TABLE `binaries` (
   `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
-  `binaryhash` blob NOT NULL DEFAULT '0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0',
+  `binaryhash` binary(16) NOT NULL,
   `name` varchar(1000) NOT NULL DEFAULT '',
   `collections_id` int(10) unsigned NOT NULL DEFAULT 0,
   `filenumber` int(10) unsigned NOT NULL DEFAULT 0,
@@ -75,8 +75,8 @@ CREATE TABLE `binaries` (
   `partcheck` tinyint(1) NOT NULL DEFAULT 0,
   `partsize` bigint(20) unsigned NOT NULL DEFAULT 0,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `ux_collection_id_filenumber` (`collections_id`,`filenumber`),
-  KEY `ix_binaries_binaryhash` (`binaryhash`(3072)),
+  UNIQUE KEY `ux_binaries_collection_hash` (`collections_id`,`binaryhash`),
+  KEY `ix_binaries_collection_filenumber` (`collections_id`,`filenumber`),
   KEY `ix_binaries_collection` (`collections_id`),
   KEY `ix_binaries_partcheck` (`partcheck`),
   CONSTRAINT `FK_Collections` FOREIGN KEY (`collections_id`) REFERENCES `collections` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
@@ -190,9 +190,10 @@ CREATE TABLE `collections` (
   `xref` varchar(2000) NOT NULL DEFAULT '',
   `totalfiles` int(10) unsigned NOT NULL DEFAULT 0,
   `groups_id` int(10) unsigned NOT NULL DEFAULT 0,
-  `collectionhash` varchar(255) NOT NULL DEFAULT '0',
+  `collectionhash` binary(20) NOT NULL,
   `collection_regexes_id` int(11) NOT NULL DEFAULT 0 COMMENT 'FK to collection_regexes.id',
   `dateadded` datetime DEFAULT NULL,
+  `last_seen_at` datetime DEFAULT NULL,
   `added` timestamp NOT NULL DEFAULT current_timestamp(),
   `filecheck` tinyint(1) NOT NULL DEFAULT 0,
   `filesize` bigint(20) unsigned NOT NULL DEFAULT 0,
@@ -205,7 +206,17 @@ CREATE TABLE `collections` (
   KEY `groups_id` (`groups_id`),
   KEY `ix_collection_dateadded` (`dateadded`),
   KEY `ix_collection_filecheck` (`filecheck`),
-  KEY `ix_collection_releaseid` (`releases_id`)
+  KEY `ix_collection_releaseid` (`releases_id`),
+  KEY `ix_collections_group_filecheck_seen_id` (`groups_id`,`filecheck`,`last_seen_at`,`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
+
+DROP TABLE IF EXISTS `collection_groups`;
+
+CREATE TABLE `collection_groups` (
+  `collections_id` int(10) unsigned NOT NULL,
+  `group_name` varchar(255) NOT NULL,
+  PRIMARY KEY (`collections_id`,`group_name`),
+  CONSTRAINT `fk_collection_groups_collection` FOREIGN KEY (`collections_id`) REFERENCES `collections` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
 
 DROP TABLE IF EXISTS `consoleinfo`;
@@ -576,11 +587,12 @@ DROP TABLE IF EXISTS `parts`;
 
 CREATE TABLE `parts` (
   `binaries_id` bigint(20) unsigned NOT NULL DEFAULT 0,
-  `messageid` varchar(255) NOT NULL DEFAULT '',
+  `messageid` varchar(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
   `number` bigint(20) unsigned NOT NULL DEFAULT 0,
   `partnumber` int(10) unsigned NOT NULL DEFAULT 0,
   `size` int(10) unsigned NOT NULL DEFAULT 0,
-  PRIMARY KEY (`binaries_id`,`number`),
+  PRIMARY KEY (`binaries_id`,`partnumber`),
+  KEY `ix_parts_number` (`number`),
   CONSTRAINT `FK_binaries` FOREIGN KEY (`binaries_id`) REFERENCES `binaries` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
 

database/schema/mysql-schema.sql

@@ -66,7 +66,7 @@ DROP TABLE IF EXISTS `binaries`;
 
 CREATE TABLE `binaries` (
   `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
-  `binaryhash` blob NOT NULL DEFAULT '0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0',
+  `binaryhash` binary(16) NOT NULL,
   `name` varchar(1000) NOT NULL DEFAULT '',
   `collections_id` int(10) unsigned NOT NULL DEFAULT 0,
   `filenumber` int(10) unsigned NOT NULL DEFAULT 0,
@@ -75,8 +75,8 @@ CREATE TABLE `binaries` (
   `partcheck` tinyint(1) NOT NULL DEFAULT 0,
   `partsize` bigint(20) unsigned NOT NULL DEFAULT 0,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `ux_collection_id_filenumber` (`collections_id`,`filenumber`),
-  KEY `ix_binaries_binaryhash` (`binaryhash`(3072)),
+  UNIQUE KEY `ux_binaries_collection_hash` (`collections_id`,`binaryhash`),
+  KEY `ix_binaries_collection_filenumber` (`collections_id`,`filenumber`),
   KEY `ix_binaries_collection` (`collections_id`),
   KEY `ix_binaries_partcheck` (`partcheck`),
   CONSTRAINT `FK_Collections` FOREIGN KEY (`collections_id`) REFERENCES `collections` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
@@ -190,9 +190,10 @@ CREATE TABLE `collections` (
   `xref` varchar(2000) NOT NULL DEFAULT '',
   `totalfiles` int(10) unsigned NOT NULL DEFAULT 0,
   `groups_id` int(10) unsigned NOT NULL DEFAULT 0,
-  `collectionhash` varchar(255) NOT NULL DEFAULT '0',
+  `collectionhash` binary(20) NOT NULL,
   `collection_regexes_id` int(11) NOT NULL DEFAULT 0 COMMENT 'FK to collection_regexes.id',
   `dateadded` datetime DEFAULT NULL,
+  `last_seen_at` datetime DEFAULT NULL,
   `added` timestamp NOT NULL DEFAULT current_timestamp(),
   `filecheck` tinyint(1) NOT NULL DEFAULT 0,
   `filesize` bigint(20) unsigned NOT NULL DEFAULT 0,
@@ -205,7 +206,17 @@ CREATE TABLE `collections` (
   KEY `groups_id` (`groups_id`),
   KEY `ix_collection_dateadded` (`dateadded`),
   KEY `ix_collection_filecheck` (`filecheck`),
-  KEY `ix_collection_releaseid` (`releases_id`)
+  KEY `ix_collection_releaseid` (`releases_id`),
+  KEY `ix_collections_group_filecheck_seen_id` (`groups_id`,`filecheck`,`last_seen_at`,`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
+
+DROP TABLE IF EXISTS `collection_groups`;
+
+CREATE TABLE `collection_groups` (
+  `collections_id` int(10) unsigned NOT NULL,
+  `group_name` varchar(255) NOT NULL,
+  PRIMARY KEY (`collections_id`,`group_name`),
+  CONSTRAINT `fk_collection_groups_collection` FOREIGN KEY (`collections_id`) REFERENCES `collections` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
 
 DROP TABLE IF EXISTS `consoleinfo`;
@@ -576,11 +587,12 @@ DROP TABLE IF EXISTS `parts`;
 
 CREATE TABLE `parts` (
   `binaries_id` bigint(20) unsigned NOT NULL DEFAULT 0,
-  `messageid` varchar(255) NOT NULL DEFAULT '',
+  `messageid` varchar(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
   `number` bigint(20) unsigned NOT NULL DEFAULT 0,
   `partnumber` int(10) unsigned NOT NULL DEFAULT 0,
   `size` int(10) unsigned NOT NULL DEFAULT 0,
-  PRIMARY KEY (`binaries_id`,`number`),
+  PRIMARY KEY (`binaries_id`,`partnumber`),
+  KEY `ix_parts_number` (`number`),
   CONSTRAINT `FK_binaries` FOREIGN KEY (`binaries_id`) REFERENCES `binaries` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci;
 

tests/Feature/BinariesQueryCountTest.php

@@ -11,9 +11,8 @@ use Tests\TestCase;
  * Regression tests guarding the SQL fan-out of the CBP write path.
  *
  * The collections/binaries/parts handlers were rewritten to consolidate
- * redundant SELECTs (prefetchExistingXrefs + existingHashes + resolveIdsByHash
- * → 1 query, existingBinaryKeys + resolveBinaryIds → 1 query) and to batch
- * per-row UPDATEs into single JOIN..UNION ALL statements. These tests assert
+ * redundant SELECTs and derives aggregates from stored parts in bounded raw
+ * SQL updates. These tests assert
  * the resulting per-chunk query count stays within a sane ceiling so the old
  * fan-out cannot silently come back.
  */
@@ -58,9 +57,18 @@ class BinariesQueryCountTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -70,6 +78,7 @@ class BinariesQueryCountTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -79,7 +88,7 @@ class BinariesQueryCountTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -127,8 +136,8 @@ class BinariesQueryCountTest extends TestCase
         });
 
         // Ceiling rationale (per chunk, 5 distinct collections, 5 binaries):
-        //   collections: 1 prefetch + 1 INSERT + 1 resolve-new = 3
-        //   binaries:    1 prefetch + 1 INSERT + 1 resolve-new = 3
+        //   collections: 1 prefetch + 1 INSERT + 1 resolve-new + 1 aggregate = 4
+        //   binaries:    1 prefetch + 1 INSERT + 1 resolve-new + 1 aggregate = 4
         //   parts:       1 INSERT (chunk fits in MAX_SQL_ROWS_PER_STATEMENT)
         // Allow a small headroom so the regression test isn't brittle.
         $this->assertLessThanOrEqual(8, $counts['collections'] ?? 0,
@@ -164,7 +173,7 @@ class BinariesQueryCountTest extends TestCase
 
         // Second pass — exact same headers. The prefetch should hit every row,
         // so no resolve-new SELECT should fire; existing rows skip the xref
-        // append UPDATE because the tokens already match.
+        // append UPDATE; normalized collection_groups inserts are idempotent.
         $counts = $this->countQueriesPerTable(static function () use ($headers): void {
             $freshHarness = new TestBinariesHarness;
             $freshHarness->publicStoreHeaders($headers);
@@ -172,8 +181,8 @@ class BinariesQueryCountTest extends TestCase
 
         // With nothing new to insert, we expect at most:
         //   collections: 1 prefetch + 1 INSERT (no-op via ODKU id=LAST_INSERT_ID(id))
-        //   binaries:    1 prefetch + 1 INSERT (currentparts/partsize incremented by ODKU)
-        //   parts:       1 INSERT IGNORE (no rows actually persist)
+        //   binaries:    1 prefetch + 1 INSERT + 1 authoritative aggregate update
+        //   parts:       1 INSERT IGNORE + 1 identity lookup (no rows actually persist)
         $this->assertLessThanOrEqual(4, $counts['collections'] ?? 0);
         $this->assertLessThanOrEqual(4, $counts['binaries'] ?? 0);
         $this->assertLessThanOrEqual(2, $counts['parts'] ?? 0);

tests/Feature/BinariesStorageInternalsTest.php

@@ -53,7 +53,7 @@ class BinariesStorageInternalsTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         $handler = new PartHandler(100);
@@ -79,8 +79,17 @@ class BinariesStorageInternalsTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
+        DB::statement('CREATE TABLE parts (
+            binaries_id INT,
+            number INT,
+            messageid VARCHAR(255),
+            partnumber INT,
+            size INT,
+            UNIQUE(binaries_id, partnumber)
+        )');
 
         $handler = new BinaryHandler;
         $first = $this->parsedHeader(251, 1, 'Aggregate.Release', 100);
@@ -89,7 +98,11 @@ class BinariesStorageInternalsTest extends TestCase
         $binaryId = $handler->getOrCreateBinary($first, 1, 1, 0);
         $this->assertNotNull($binaryId);
         $this->assertSame($binaryId, $handler->getOrCreateBinary($second, 1, 1, 0));
-        $this->assertTrue($handler->flushUpdates());
+        DB::table('parts')->insert([
+            ['binaries_id' => $binaryId, 'number' => 251, 'messageid' => '<251@example>', 'partnumber' => 1, 'size' => 100],
+            ['binaries_id' => $binaryId, 'number' => 252, 'messageid' => '<252@example>', 'partnumber' => 2, 'size' => 50],
+        ]);
+        $this->assertTrue($handler->refreshAggregates([$binaryId]));
 
         $binary = DB::table('binaries')->where('id', $binaryId)->first();
         $this->assertSame(2, (int) $binary->currentparts);
@@ -128,7 +141,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables('CHECK(size < 500)');
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 2, headerChunkSize: 2));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 2, headerChunkSize: 2));
         $failed = $service->store([
             $this->parsedHeader(301, 1, 'Chunk.One', 100),
             $this->parsedHeader(302, 2, 'Chunk.One', 100),
@@ -149,7 +162,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
         $failed = $service->store([
             $this->parsedHeader(401, 1, 'Batch.Release', 150),
             $this->parsedHeader(402, 2, 'Batch.Release', 175),
@@ -169,7 +182,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
         $this->assertSame([], $service->store([
             $this->parsedHeader(501, 1, 'Existing.Batch.Release', 100),
         ], ['id' => 1, 'name' => 'alt.test'], true));
@@ -187,24 +200,106 @@ class BinariesStorageInternalsTest extends TestCase
         $this->assertSame(250, (int) $binary->partsize);
     }
 
+    public function test_three_identical_ingestions_leave_authoritative_counts_unchanged(): void
+    {
+        $this->createHeaderStorageTables();
+
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $headers = [
+            $this->parsedHeader(551, 1, 'Idempotent.Release', 125),
+            $this->parsedHeader(552, 2, 'Idempotent.Release', 175),
+        ];
+
+        for ($attempt = 0; $attempt < 3; $attempt++) {
+            $this->assertSame([], $service->store($headers, ['id' => 1, 'name' => 'alt.test'], true));
+        }
+
+        $binary = DB::table('binaries')->first();
+        $collection = DB::table('collections')->first();
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(2, (int) $binary->currentparts);
+        $this->assertSame(300, (int) $binary->partsize);
+        $this->assertSame(300, (int) $collection->filesize);
+    }
+
+    public function test_duplicate_partnumber_prefers_non_empty_message_then_size_then_article_number(): void
+    {
+        DB::statement('CREATE TABLE parts (
+            binaries_id INT,
+            number INT,
+            messageid VARCHAR(255),
+            partnumber INT,
+            size INT,
+            UNIQUE(binaries_id, partnumber)
+        )');
+
+        $handler = new PartHandler(10);
+        $small = $this->parsedHeader(900, 1, 'Preference.Release', 100);
+        $large = $this->parsedHeader(901, 1, 'Preference.Release', 200);
+        $this->assertTrue($handler->addPart(7, $small));
+        $this->assertTrue($handler->addPart(7, $large));
+        $this->assertTrue($handler->flush());
+
+        $part = DB::table('parts')->first();
+        $this->assertSame(901, (int) $part->number);
+        $this->assertSame(200, (int) $part->size);
+    }
+
+    public function test_invalid_message_id_rolls_back_the_header_chunk(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $header = $this->parsedHeader(910, 1, 'Invalid.Message.Release', 100);
+        $header['Message-ID'] = "<invalid\u{00E9}@example>";
+
+        $this->assertSame([910], $service->store([$header], ['id' => 1, 'name' => 'alt.test'], true));
+        $this->assertSame(0, DB::table('collections')->count());
+        $this->assertSame(0, DB::table('binaries')->count());
+        $this->assertSame(0, DB::table('parts')->count());
+    }
+
+    public function test_zero_file_number_uses_subject_and_poster_identity(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $first = $this->parsedHeader(920, 1, 'Unknown.File.Release', 100);
+        $second = $this->parsedHeader(921, 1, 'Unknown.File.Release', 100);
+        $second['From'] = 'another-poster@example.com';
+
+        $this->assertSame([], $service->store([$first, $second], ['id' => 1, 'name' => 'alt.test'], true));
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(2, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+    }
+
+    public function test_cross_post_groups_are_normalized_and_deduplicated(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $header = $this->parsedHeader(930, 1, 'Cross.Post.Release', 100);
+        $header['Xref'] = 'news.example alt.binaries.one:930 alt.binaries.two:931 alt.binaries.one:930';
+
+        $this->assertSame([], $service->store([$header], ['id' => 1, 'name' => 'alt.binaries.one'], true));
+        $this->assertSame(
+            ['alt.binaries.one', 'alt.binaries.two'],
+            DB::table('collection_groups')->orderBy('group_name')->pluck('group_name')->all()
+        );
+    }
+
     public function test_header_storage_does_not_merge_same_subject_across_different_collections(): void
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
-        $failed = $service->store([
-            $this->parsedHeaderWithTotal(601, 1, 2, 'Same.Subject', 100),
-            $this->parsedHeaderWithTotal(602, 1, 3, 'Same.Subject', 200),
-        ], ['id' => 1, 'name' => 'alt.test'], true);
+        $handler = new BinaryHandler;
+        $header = $this->parsedHeader(601, 1, 'Same.Subject', 100);
+        $firstId = $handler->getOrCreateBinary($header, 10, 1, 0);
+        $secondId = $handler->getOrCreateBinary($header, 20, 1, 0);
 
-        $binaries = DB::table('binaries')->orderBy('partsize')->get();
-
-        $this->assertSame([], $failed);
-        $this->assertSame(2, DB::table('collections')->count());
+        $this->assertNotNull($firstId);
+        $this->assertNotNull($secondId);
+        $this->assertNotSame($firstId, $secondId);
         $this->assertSame(2, DB::table('binaries')->count());
-        $this->assertSame(2, DB::table('parts')->count());
-        $this->assertSame([100, 200], $binaries->pluck('partsize')->map(static fn ($value): int => (int) $value)->all());
-        $this->assertSame([1, 1], $binaries->pluck('currentparts')->map(static fn ($value): int => (int) $value)->all());
     }
 
     private function rawHeader(int $number, string $subject): array
@@ -252,9 +347,18 @@ class BinariesStorageInternalsTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT \'\'
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY,
             binaryhash BLOB,
@@ -264,6 +368,7 @@ class BinariesStorageInternalsTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -273,7 +378,7 @@ class BinariesStorageInternalsTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT '.$partSizeConstraint.',
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE collection_regexes (

tests/Feature/BinariesStoreHeadersMixedTest.php

@@ -48,9 +48,18 @@ class BinariesStoreHeadersMixedTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -60,6 +69,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -70,7 +80,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -117,7 +127,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
         $group = ['id' => 3, 'name' => 'alt.mixed'];
 
         // Chunk of 2: first flush succeeds, second flush fails
-        config(['nntmux.parts_chunk_size' => 2]);
+        config(['nntmux.cbp.sql_chunk_size' => 2]);
 
         $headers = [
             $this->makeHeader(4001, 1, 5, 101),

tests/Feature/BinariesStoreHeadersTest.php

@@ -49,9 +49,18 @@ class BinariesStoreHeadersTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -61,6 +70,7 @@ class BinariesStoreHeadersTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -71,7 +81,7 @@ class BinariesStoreHeadersTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -159,7 +169,7 @@ class BinariesStoreHeadersTest extends TestCase
         $harness = new TestBinariesHarness;
         $group = ['id' => 2, 'name' => 'alt.rollback'];
 
-        config(['nntmux.parts_chunk_size' => 2]); // force flush earlier
+        config(['nntmux.cbp.sql_chunk_size' => 2]); // force flush earlier
 
         $headers = [
             $this->makeHeader(2001, 1, 3, 111),
@@ -185,7 +195,7 @@ class BinariesStoreHeadersTest extends TestCase
         $group = ['id' => 3, 'name' => 'alt.mixed'];
 
         // Chunk of 2: first flush succeeds, second flush fails
-        config(['nntmux.parts_chunk_size' => 2]);
+        config(['nntmux.cbp.sql_chunk_size' => 2]);
 
         $headers = [
             $this->makeHeader(4001, 1, 5, 101),

tests/Feature/NzbCreationReliabilityTest.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace Tests\Feature;
 
 use App\Models\Release;
+use App\Services\Binaries\BinariesConfig;
 use App\Services\CollectionCleanupService;
 use App\Services\Nzb\NzbCreationCandidateQuery;
 use App\Services\Nzb\NzbService;
@@ -199,6 +200,30 @@ class NzbCreationReliabilityTest extends TestCase
         $this->assertSame(0, (int) DB::table('releases')->where('id', 1)->value('nzbstatus'));
     }
 
+    public function test_writer_streams_multiple_keyset_pages_in_segment_order(): void
+    {
+        $this->insertRelease(1, 'h');
+        $this->insertWritableCbp(200, 2000, 1);
+        DB::table('parts')->where('binaries_id', 2000)->delete();
+        DB::table('parts')->insert([
+            ['binaries_id' => 2000, 'messageid' => '<part02@example.test>', 'partnumber' => 2, 'size' => 200],
+            ['binaries_id' => 2000, 'messageid' => '<part01@example.test>', 'partnumber' => 1, 'size' => 100],
+            ['binaries_id' => 2000, 'messageid' => '<part03@example.test>', 'partnumber' => 3, 'size' => 300],
+        ]);
+        $release = Release::query()->findOrFail(1);
+        $release->setRelation('category', (object) ['title' => 'Misc', 'parent' => (object) ['title' => 'Other']]);
+
+        $nzb = new NzbService(app(CollectionCleanupService::class), new BinariesConfig(nzbStreamRows: 2));
+        $result = $nzb->createNzbForRelease($release);
+
+        $this->assertTrue($result->success, $result->reason);
+        $xml = unzipGzipFile((string) $result->path);
+        $this->assertIsString($xml);
+        $this->assertLessThan(strpos($xml, 'part02@example.test'), strpos($xml, 'part01@example.test'));
+        $this->assertLessThan(strpos($xml, 'part03@example.test'), strpos($xml, 'part02@example.test'));
+        $this->assertStringContainsString('<group>alt.test</group>', $xml);
+    }
+
     public function test_stale_temporary_nzb_cleanup_deletes_only_old_temp_files(): void
     {
         $directory = $this->tempNzbPath.'/a';

tests/Integration/CbpMariaDbIngestionTest.php

@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Integration;
+
+use App\Services\Binaries\BinariesConfig;
+use App\Services\Binaries\CollectionHandler;
+use App\Services\Binaries\HeaderStorageService;
+use App\Services\CollectionsCleaningService;
+use Illuminate\Contracts\Console\Kernel;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
+use Tests\TestCase;
+
+final class CbpMariaDbIngestionTest extends TestCase
+{
+    public function createApplication()
+    {
+        $database = getenv('CBP_INTEGRATION_DB_DATABASE');
+        if ($database === false || $database === '') {
+            return parent::createApplication();
+        }
+
+        putenv('DB_CONNECTION=mariadb');
+        putenv('DB_DATABASE='.$database);
+        putenv('DB_HOST=mariadb');
+        putenv('DB_USERNAME='.(string) getenv('CBP_INTEGRATION_DB_USERNAME'));
+        putenv('DB_PASSWORD='.(string) getenv('CBP_INTEGRATION_DB_PASSWORD'));
+        $_ENV['DB_CONNECTION'] = $_SERVER['DB_CONNECTION'] = 'mariadb';
+        $_ENV['DB_DATABASE'] = $_SERVER['DB_DATABASE'] = $database;
+        $_ENV['DB_HOST'] = $_SERVER['DB_HOST'] = 'mariadb';
+        $_ENV['DB_USERNAME'] = $_SERVER['DB_USERNAME'] = (string) getenv('CBP_INTEGRATION_DB_USERNAME');
+        $_ENV['DB_PASSWORD'] = $_SERVER['DB_PASSWORD'] = (string) getenv('CBP_INTEGRATION_DB_PASSWORD');
+
+        $app = require __DIR__.'/../../bootstrap/app.php';
+        $app->make(Kernel::class)->bootstrap();
+
+        return $app;
+    }
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        if (! \in_array(DB::getDriverName(), ['mysql', 'mariadb'], true)) {
+            $this->markTestSkipped('MariaDB/MySQL integration test.');
+        }
+
+        foreach (['cbp_optimization_checkpoints', 'cbp_binary_map', 'parts_cbp_new', 'parts_cbp_pre_optimize'] as $table) {
+            DB::statement("DROP TABLE IF EXISTS {$table}");
+        }
+        DB::statement('CREATE TABLE usenet_groups (id INT UNSIGNED PRIMARY KEY, name VARCHAR(255) NOT NULL) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collection_regexes (id INT PRIMARY KEY, group_regex VARCHAR(255), regex VARCHAR(255), status TINYINT DEFAULT 1, ordinal INT DEFAULT 0) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collections (
+            id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+            subject VARCHAR(255) NOT NULL, fromname VARCHAR(255) NOT NULL, date DATETIME NULL,
+            xref VARCHAR(2000) NOT NULL DEFAULT \'\', groups_id INT UNSIGNED NOT NULL,
+            totalfiles INT UNSIGNED NOT NULL DEFAULT 0, collectionhash BINARY(20) NOT NULL,
+            collection_regexes_id INT NOT NULL DEFAULT 0, dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL, filecheck TINYINT NOT NULL DEFAULT 0,
+            filesize BIGINT UNSIGNED NOT NULL DEFAULT 0, noise CHAR(32) NOT NULL DEFAULT \'\',
+            UNIQUE KEY ix_collection_collectionhash (collectionhash),
+            KEY ix_collections_group_filecheck_seen_id (groups_id, filecheck, last_seen_at, id)
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT UNSIGNED NOT NULL, group_name VARCHAR(255) NOT NULL,
+            PRIMARY KEY (collections_id, group_name),
+            CONSTRAINT fk_test_collection_groups FOREIGN KEY (collections_id) REFERENCES collections(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE binaries (
+            id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, binaryhash BINARY(16) NOT NULL,
+            name VARCHAR(1000) NOT NULL, collections_id INT UNSIGNED NOT NULL,
+            filenumber INT UNSIGNED NOT NULL DEFAULT 0, totalparts INT UNSIGNED NOT NULL DEFAULT 0,
+            currentparts INT UNSIGNED NOT NULL DEFAULT 0, partcheck TINYINT NOT NULL DEFAULT 0,
+            partsize BIGINT UNSIGNED NOT NULL DEFAULT 0,
+            UNIQUE KEY ux_binaries_collection_hash (collections_id, binaryhash),
+            KEY ix_binaries_collection_filenumber (collections_id, filenumber),
+            CONSTRAINT fk_test_binaries FOREIGN KEY (collections_id) REFERENCES collections(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE parts (
+            binaries_id BIGINT UNSIGNED NOT NULL,
+            messageid VARCHAR(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+            number BIGINT UNSIGNED NOT NULL, partnumber INT UNSIGNED NOT NULL, size INT UNSIGNED NOT NULL,
+            PRIMARY KEY (binaries_id, partnumber), KEY ix_parts_number (number),
+            CONSTRAINT fk_test_parts FOREIGN KEY (binaries_id) REFERENCES binaries(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.binaries.test']);
+    }
+
+    protected function tearDown(): void
+    {
+        if (\in_array(DB::getDriverName(), ['mysql', 'mariadb'], true)) {
+            DB::statement('SET FOREIGN_KEY_CHECKS=0');
+            foreach (['parts', 'parts_cbp_new', 'parts_cbp_pre_optimize', 'cbp_binary_map', 'cbp_optimization_checkpoints', 'binaries', 'collection_groups', 'collections', 'collection_regexes', 'usenet_groups'] as $table) {
+                DB::statement("DROP TABLE IF EXISTS {$table}");
+            }
+            DB::statement('SET FOREIGN_KEY_CHECKS=1');
+        }
+        parent::tearDown();
+    }
+
+    public function test_reingestion_is_idempotent_and_hot_lookups_use_indexes(): void
+    {
+        $service = new HeaderStorageService(
+            new CollectionHandler(new class extends CollectionsCleaningService
+            {
+                public function collectionsCleaner(string $subject, string $groupName = ''): array
+                {
+                    return ['id' => 0, 'name' => $subject];
+                }
+            }),
+            config: new BinariesConfig(headerChunkSize: 2, sqlChunkSize: 2),
+        );
+        $headers = [$this->header(1001, 1, 125), $this->header(1002, 2, 175)];
+
+        for ($attempt = 0; $attempt < 3; $attempt++) {
+            $this->assertSame([], $service->store($headers, ['id' => 1, 'name' => 'alt.binaries.test']));
+        }
+
+        $binary = DB::table('binaries')->first();
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(2, (int) $binary->currentparts);
+        $this->assertSame(300, (int) $binary->partsize);
+        $this->assertSame(300, (int) DB::table('collections')->value('filesize'));
+
+        $plan = DB::selectOne(
+            'EXPLAIN FORMAT=JSON SELECT id FROM binaries WHERE collections_id = ? AND binaryhash = ?',
+            [1, $binary->binaryhash]
+        );
+        $json = strtolower((string) array_values((array) $plan)[0]);
+        $this->assertStringContainsString('ux_binaries_collection_hash', $json);
+
+        $this->assertSame(0, Artisan::call('cbp:optimize-storage'));
+        $this->assertStringContainsString('Dry-run only', Artisan::output());
+
+        $this->restoreLegacyStorageShape();
+        $this->assertSame(2, DB::table('parts')->count());
+        config()->set('nntmux.cbp.reconcile_batch_size', 2);
+
+        $migration = require database_path('migrations/2026_08_03_000001_finalize_cbp_binary_hash_storage.php');
+        config()->set('nntmux.cbp.storage_migration_execute', false);
+        try {
+            $migration->up();
+            $this->fail('The legacy storage migration ran without explicit maintenance approval.');
+        } catch (\RuntimeException $exception) {
+            $this->assertStringContainsString('CBP_STORAGE_MIGRATION_EXECUTE=true', $exception->getMessage());
+        }
+        $this->assertSame(2, DB::table('parts')->count());
+
+        config()->set('nntmux.cbp.storage_migration_execute', true);
+        $migration->up();
+
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(300, (int) DB::table('binaries')->value('partsize'));
+    }
+
+    private function restoreLegacyStorageShape(): void
+    {
+        DB::statement('ALTER TABLE parts DROP FOREIGN KEY fk_test_parts');
+        DB::statement('ALTER TABLE parts DROP PRIMARY KEY, DROP INDEX ix_parts_number, ADD PRIMARY KEY (binaries_id, number)');
+        DB::statement('ALTER TABLE parts MODIFY messageid VARCHAR(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL');
+
+        DB::statement('ALTER TABLE binaries DROP INDEX ux_binaries_collection_hash');
+        DB::statement('ALTER TABLE binaries MODIFY binaryhash BLOB NOT NULL');
+        DB::statement('CREATE INDEX ix_binaries_binaryhash ON binaries (binaryhash(16))');
+        DB::statement('CREATE UNIQUE INDEX ux_collection_id_filenumber ON binaries (collections_id, filenumber)');
+
+        DB::statement('ALTER TABLE collections DROP INDEX ix_collection_collectionhash');
+        DB::statement('ALTER TABLE collections MODIFY collectionhash BLOB NOT NULL');
+        DB::statement('CREATE UNIQUE INDEX ix_collection_collectionhash ON collections (collectionhash(20))');
+    }
+
+    /** @return array<string, mixed> */
+    private function header(int $number, int $part, int $bytes): array
+    {
+        return [
+            'Number' => $number,
+            'Subject' => "Integration.Release ({$part}/2)",
+            'From' => 'poster@example.test',
+            'Date' => time(),
+            'Bytes' => $bytes,
+            'Message-ID' => "<{$number}@example.test>",
+            'Xref' => "news.example alt.binaries.test:{$number}",
+            'matches' => [
+                0 => "Integration.Release ({$part}/2)",
+                1 => 'Integration.Release',
+                2 => $part,
+                3 => 2,
+            ],
+        ];
+    }
+}

tests/Support/TestBinariesHarness.php

@@ -33,8 +33,7 @@ class TestBinariesHarness extends BinariesService
             newGroupDaysToScan: 3,
             partRepairLimit: 15000,
             partRepairMaxTries: 3,
-            partsChunkSize: 5000,
-            binariesUpdateChunkSize: 1000,
+            sqlChunkSize: 500,
             echoCli: false
         );