Update cbp handling

tests/Feature/BinariesQueryCountTest.php

@@ -11,9 +11,8 @@ use Tests\TestCase;
  * Regression tests guarding the SQL fan-out of the CBP write path.
  *
  * The collections/binaries/parts handlers were rewritten to consolidate
- * redundant SELECTs (prefetchExistingXrefs + existingHashes + resolveIdsByHash
- * → 1 query, existingBinaryKeys + resolveBinaryIds → 1 query) and to batch
- * per-row UPDATEs into single JOIN..UNION ALL statements. These tests assert
+ * redundant SELECTs and derives aggregates from stored parts in bounded raw
+ * SQL updates. These tests assert
  * the resulting per-chunk query count stays within a sane ceiling so the old
  * fan-out cannot silently come back.
  */
@@ -58,9 +57,18 @@ class BinariesQueryCountTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -70,6 +78,7 @@ class BinariesQueryCountTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -79,7 +88,7 @@ class BinariesQueryCountTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -127,8 +136,8 @@ class BinariesQueryCountTest extends TestCase
         });
 
         // Ceiling rationale (per chunk, 5 distinct collections, 5 binaries):
-        //   collections: 1 prefetch + 1 INSERT + 1 resolve-new = 3
-        //   binaries:    1 prefetch + 1 INSERT + 1 resolve-new = 3
+        //   collections: 1 prefetch + 1 INSERT + 1 resolve-new + 1 aggregate = 4
+        //   binaries:    1 prefetch + 1 INSERT + 1 resolve-new + 1 aggregate = 4
         //   parts:       1 INSERT (chunk fits in MAX_SQL_ROWS_PER_STATEMENT)
         // Allow a small headroom so the regression test isn't brittle.
         $this->assertLessThanOrEqual(8, $counts['collections'] ?? 0,
@@ -164,7 +173,7 @@ class BinariesQueryCountTest extends TestCase
 
         // Second pass — exact same headers. The prefetch should hit every row,
         // so no resolve-new SELECT should fire; existing rows skip the xref
-        // append UPDATE because the tokens already match.
+        // append UPDATE; normalized collection_groups inserts are idempotent.
         $counts = $this->countQueriesPerTable(static function () use ($headers): void {
             $freshHarness = new TestBinariesHarness;
             $freshHarness->publicStoreHeaders($headers);
@@ -172,8 +181,8 @@ class BinariesQueryCountTest extends TestCase
 
         // With nothing new to insert, we expect at most:
         //   collections: 1 prefetch + 1 INSERT (no-op via ODKU id=LAST_INSERT_ID(id))
-        //   binaries:    1 prefetch + 1 INSERT (currentparts/partsize incremented by ODKU)
-        //   parts:       1 INSERT IGNORE (no rows actually persist)
+        //   binaries:    1 prefetch + 1 INSERT + 1 authoritative aggregate update
+        //   parts:       1 INSERT IGNORE + 1 identity lookup (no rows actually persist)
         $this->assertLessThanOrEqual(4, $counts['collections'] ?? 0);
         $this->assertLessThanOrEqual(4, $counts['binaries'] ?? 0);
         $this->assertLessThanOrEqual(2, $counts['parts'] ?? 0);

tests/Feature/BinariesStorageInternalsTest.php

@@ -53,7 +53,7 @@ class BinariesStorageInternalsTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         $handler = new PartHandler(100);
@@ -79,8 +79,17 @@ class BinariesStorageInternalsTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
+        DB::statement('CREATE TABLE parts (
+            binaries_id INT,
+            number INT,
+            messageid VARCHAR(255),
+            partnumber INT,
+            size INT,
+            UNIQUE(binaries_id, partnumber)
+        )');
 
         $handler = new BinaryHandler;
         $first = $this->parsedHeader(251, 1, 'Aggregate.Release', 100);
@@ -89,7 +98,11 @@ class BinariesStorageInternalsTest extends TestCase
         $binaryId = $handler->getOrCreateBinary($first, 1, 1, 0);
         $this->assertNotNull($binaryId);
         $this->assertSame($binaryId, $handler->getOrCreateBinary($second, 1, 1, 0));
-        $this->assertTrue($handler->flushUpdates());
+        DB::table('parts')->insert([
+            ['binaries_id' => $binaryId, 'number' => 251, 'messageid' => '<251@example>', 'partnumber' => 1, 'size' => 100],
+            ['binaries_id' => $binaryId, 'number' => 252, 'messageid' => '<252@example>', 'partnumber' => 2, 'size' => 50],
+        ]);
+        $this->assertTrue($handler->refreshAggregates([$binaryId]));
 
         $binary = DB::table('binaries')->where('id', $binaryId)->first();
         $this->assertSame(2, (int) $binary->currentparts);
@@ -128,7 +141,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables('CHECK(size < 500)');
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 2, headerChunkSize: 2));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 2, headerChunkSize: 2));
         $failed = $service->store([
             $this->parsedHeader(301, 1, 'Chunk.One', 100),
             $this->parsedHeader(302, 2, 'Chunk.One', 100),
@@ -149,7 +162,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
         $failed = $service->store([
             $this->parsedHeader(401, 1, 'Batch.Release', 150),
             $this->parsedHeader(402, 2, 'Batch.Release', 175),
@@ -169,7 +182,7 @@ class BinariesStorageInternalsTest extends TestCase
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
         $this->assertSame([], $service->store([
             $this->parsedHeader(501, 1, 'Existing.Batch.Release', 100),
         ], ['id' => 1, 'name' => 'alt.test'], true));
@@ -187,24 +200,106 @@ class BinariesStorageInternalsTest extends TestCase
         $this->assertSame(250, (int) $binary->partsize);
     }
 
+    public function test_three_identical_ingestions_leave_authoritative_counts_unchanged(): void
+    {
+        $this->createHeaderStorageTables();
+
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $headers = [
+            $this->parsedHeader(551, 1, 'Idempotent.Release', 125),
+            $this->parsedHeader(552, 2, 'Idempotent.Release', 175),
+        ];
+
+        for ($attempt = 0; $attempt < 3; $attempt++) {
+            $this->assertSame([], $service->store($headers, ['id' => 1, 'name' => 'alt.test'], true));
+        }
+
+        $binary = DB::table('binaries')->first();
+        $collection = DB::table('collections')->first();
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(2, (int) $binary->currentparts);
+        $this->assertSame(300, (int) $binary->partsize);
+        $this->assertSame(300, (int) $collection->filesize);
+    }
+
+    public function test_duplicate_partnumber_prefers_non_empty_message_then_size_then_article_number(): void
+    {
+        DB::statement('CREATE TABLE parts (
+            binaries_id INT,
+            number INT,
+            messageid VARCHAR(255),
+            partnumber INT,
+            size INT,
+            UNIQUE(binaries_id, partnumber)
+        )');
+
+        $handler = new PartHandler(10);
+        $small = $this->parsedHeader(900, 1, 'Preference.Release', 100);
+        $large = $this->parsedHeader(901, 1, 'Preference.Release', 200);
+        $this->assertTrue($handler->addPart(7, $small));
+        $this->assertTrue($handler->addPart(7, $large));
+        $this->assertTrue($handler->flush());
+
+        $part = DB::table('parts')->first();
+        $this->assertSame(901, (int) $part->number);
+        $this->assertSame(200, (int) $part->size);
+    }
+
+    public function test_invalid_message_id_rolls_back_the_header_chunk(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $header = $this->parsedHeader(910, 1, 'Invalid.Message.Release', 100);
+        $header['Message-ID'] = "<invalid\u{00E9}@example>";
+
+        $this->assertSame([910], $service->store([$header], ['id' => 1, 'name' => 'alt.test'], true));
+        $this->assertSame(0, DB::table('collections')->count());
+        $this->assertSame(0, DB::table('binaries')->count());
+        $this->assertSame(0, DB::table('parts')->count());
+    }
+
+    public function test_zero_file_number_uses_subject_and_poster_identity(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $first = $this->parsedHeader(920, 1, 'Unknown.File.Release', 100);
+        $second = $this->parsedHeader(921, 1, 'Unknown.File.Release', 100);
+        $second['From'] = 'another-poster@example.com';
+
+        $this->assertSame([], $service->store([$first, $second], ['id' => 1, 'name' => 'alt.test'], true));
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(2, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+    }
+
+    public function test_cross_post_groups_are_normalized_and_deduplicated(): void
+    {
+        $this->createHeaderStorageTables();
+        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
+        $header = $this->parsedHeader(930, 1, 'Cross.Post.Release', 100);
+        $header['Xref'] = 'news.example alt.binaries.one:930 alt.binaries.two:931 alt.binaries.one:930';
+
+        $this->assertSame([], $service->store([$header], ['id' => 1, 'name' => 'alt.binaries.one'], true));
+        $this->assertSame(
+            ['alt.binaries.one', 'alt.binaries.two'],
+            DB::table('collection_groups')->orderBy('group_name')->pluck('group_name')->all()
+        );
+    }
+
     public function test_header_storage_does_not_merge_same_subject_across_different_collections(): void
     {
         $this->createHeaderStorageTables();
 
-        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(partsChunkSize: 10));
-        $failed = $service->store([
-            $this->parsedHeaderWithTotal(601, 1, 2, 'Same.Subject', 100),
-            $this->parsedHeaderWithTotal(602, 1, 3, 'Same.Subject', 200),
-        ], ['id' => 1, 'name' => 'alt.test'], true);
+        $handler = new BinaryHandler;
+        $header = $this->parsedHeader(601, 1, 'Same.Subject', 100);
+        $firstId = $handler->getOrCreateBinary($header, 10, 1, 0);
+        $secondId = $handler->getOrCreateBinary($header, 20, 1, 0);
 
-        $binaries = DB::table('binaries')->orderBy('partsize')->get();
-
-        $this->assertSame([], $failed);
-        $this->assertSame(2, DB::table('collections')->count());
+        $this->assertNotNull($firstId);
+        $this->assertNotNull($secondId);
+        $this->assertNotSame($firstId, $secondId);
         $this->assertSame(2, DB::table('binaries')->count());
-        $this->assertSame(2, DB::table('parts')->count());
-        $this->assertSame([100, 200], $binaries->pluck('partsize')->map(static fn ($value): int => (int) $value)->all());
-        $this->assertSame([1, 1], $binaries->pluck('currentparts')->map(static fn ($value): int => (int) $value)->all());
     }
 
     private function rawHeader(int $number, string $subject): array
@@ -252,9 +347,18 @@ class BinariesStorageInternalsTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT \'\'
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY,
             binaryhash BLOB,
@@ -264,6 +368,7 @@ class BinariesStorageInternalsTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -273,7 +378,7 @@ class BinariesStorageInternalsTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT '.$partSizeConstraint.',
-            UNIQUE(binaries_id, number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE collection_regexes (

tests/Feature/BinariesStoreHeadersMixedTest.php

@@ -48,9 +48,18 @@ class BinariesStoreHeadersMixedTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -60,6 +69,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -70,7 +80,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -117,7 +127,7 @@ class BinariesStoreHeadersMixedTest extends TestCase
         $group = ['id' => 3, 'name' => 'alt.mixed'];
 
         // Chunk of 2: first flush succeeds, second flush fails
-        config(['nntmux.parts_chunk_size' => 2]);
+        config(['nntmux.cbp.sql_chunk_size' => 2]);
 
         $headers = [
             $this->makeHeader(4001, 1, 5, 101),

tests/Feature/BinariesStoreHeadersTest.php

@@ -49,9 +49,18 @@ class BinariesStoreHeadersTest extends TestCase
             collectionhash VARCHAR(40) UNIQUE,
             collection_regexes_id INT,
             dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL,
+            filecheck INT DEFAULT 0,
+            filesize INT DEFAULT 0,
             noise VARCHAR(64) DEFAULT ""
         )');
 
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT,
+            group_name VARCHAR(255),
+            UNIQUE(collections_id, group_name)
+        )');
+
         DB::statement('CREATE TABLE binaries (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             binaryhash BLOB,
@@ -61,6 +70,7 @@ class BinariesStoreHeadersTest extends TestCase
             currentparts INT,
             filenumber INT,
             partsize INT,
+            partcheck INT DEFAULT 0,
             UNIQUE(binaryhash, collections_id)
         )');
 
@@ -71,7 +81,7 @@ class BinariesStoreHeadersTest extends TestCase
             messageid VARCHAR(255),
             partnumber INT,
             size INT,
-            UNIQUE(number)
+            UNIQUE(binaries_id, partnumber)
         )');
 
         DB::statement('CREATE TABLE missed_parts (
@@ -159,7 +169,7 @@ class BinariesStoreHeadersTest extends TestCase
         $harness = new TestBinariesHarness;
         $group = ['id' => 2, 'name' => 'alt.rollback'];
 
-        config(['nntmux.parts_chunk_size' => 2]); // force flush earlier
+        config(['nntmux.cbp.sql_chunk_size' => 2]); // force flush earlier
 
         $headers = [
             $this->makeHeader(2001, 1, 3, 111),
@@ -185,7 +195,7 @@ class BinariesStoreHeadersTest extends TestCase
         $group = ['id' => 3, 'name' => 'alt.mixed'];
 
         // Chunk of 2: first flush succeeds, second flush fails
-        config(['nntmux.parts_chunk_size' => 2]);
+        config(['nntmux.cbp.sql_chunk_size' => 2]);
 
         $headers = [
             $this->makeHeader(4001, 1, 5, 101),

tests/Feature/NzbCreationReliabilityTest.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace Tests\Feature;
 
 use App\Models\Release;
+use App\Services\Binaries\BinariesConfig;
 use App\Services\CollectionCleanupService;
 use App\Services\Nzb\NzbCreationCandidateQuery;
 use App\Services\Nzb\NzbService;
@@ -199,6 +200,30 @@ class NzbCreationReliabilityTest extends TestCase
         $this->assertSame(0, (int) DB::table('releases')->where('id', 1)->value('nzbstatus'));
     }
 
+    public function test_writer_streams_multiple_keyset_pages_in_segment_order(): void
+    {
+        $this->insertRelease(1, 'h');
+        $this->insertWritableCbp(200, 2000, 1);
+        DB::table('parts')->where('binaries_id', 2000)->delete();
+        DB::table('parts')->insert([
+            ['binaries_id' => 2000, 'messageid' => '<part02@example.test>', 'partnumber' => 2, 'size' => 200],
+            ['binaries_id' => 2000, 'messageid' => '<part01@example.test>', 'partnumber' => 1, 'size' => 100],
+            ['binaries_id' => 2000, 'messageid' => '<part03@example.test>', 'partnumber' => 3, 'size' => 300],
+        ]);
+        $release = Release::query()->findOrFail(1);
+        $release->setRelation('category', (object) ['title' => 'Misc', 'parent' => (object) ['title' => 'Other']]);
+
+        $nzb = new NzbService(app(CollectionCleanupService::class), new BinariesConfig(nzbStreamRows: 2));
+        $result = $nzb->createNzbForRelease($release);
+
+        $this->assertTrue($result->success, $result->reason);
+        $xml = unzipGzipFile((string) $result->path);
+        $this->assertIsString($xml);
+        $this->assertLessThan(strpos($xml, 'part02@example.test'), strpos($xml, 'part01@example.test'));
+        $this->assertLessThan(strpos($xml, 'part03@example.test'), strpos($xml, 'part02@example.test'));
+        $this->assertStringContainsString('<group>alt.test</group>', $xml);
+    }
+
     public function test_stale_temporary_nzb_cleanup_deletes_only_old_temp_files(): void
     {
         $directory = $this->tempNzbPath.'/a';

tests/Integration/CbpMariaDbIngestionTest.php

@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Integration;
+
+use App\Services\Binaries\BinariesConfig;
+use App\Services\Binaries\CollectionHandler;
+use App\Services\Binaries\HeaderStorageService;
+use App\Services\CollectionsCleaningService;
+use Illuminate\Contracts\Console\Kernel;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
+use Tests\TestCase;
+
+final class CbpMariaDbIngestionTest extends TestCase
+{
+    public function createApplication()
+    {
+        $database = getenv('CBP_INTEGRATION_DB_DATABASE');
+        if ($database === false || $database === '') {
+            return parent::createApplication();
+        }
+
+        putenv('DB_CONNECTION=mariadb');
+        putenv('DB_DATABASE='.$database);
+        putenv('DB_HOST=mariadb');
+        putenv('DB_USERNAME='.(string) getenv('CBP_INTEGRATION_DB_USERNAME'));
+        putenv('DB_PASSWORD='.(string) getenv('CBP_INTEGRATION_DB_PASSWORD'));
+        $_ENV['DB_CONNECTION'] = $_SERVER['DB_CONNECTION'] = 'mariadb';
+        $_ENV['DB_DATABASE'] = $_SERVER['DB_DATABASE'] = $database;
+        $_ENV['DB_HOST'] = $_SERVER['DB_HOST'] = 'mariadb';
+        $_ENV['DB_USERNAME'] = $_SERVER['DB_USERNAME'] = (string) getenv('CBP_INTEGRATION_DB_USERNAME');
+        $_ENV['DB_PASSWORD'] = $_SERVER['DB_PASSWORD'] = (string) getenv('CBP_INTEGRATION_DB_PASSWORD');
+
+        $app = require __DIR__.'/../../bootstrap/app.php';
+        $app->make(Kernel::class)->bootstrap();
+
+        return $app;
+    }
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        if (! \in_array(DB::getDriverName(), ['mysql', 'mariadb'], true)) {
+            $this->markTestSkipped('MariaDB/MySQL integration test.');
+        }
+
+        foreach (['cbp_optimization_checkpoints', 'cbp_binary_map', 'parts_cbp_new', 'parts_cbp_pre_optimize'] as $table) {
+            DB::statement("DROP TABLE IF EXISTS {$table}");
+        }
+        DB::statement('CREATE TABLE usenet_groups (id INT UNSIGNED PRIMARY KEY, name VARCHAR(255) NOT NULL) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collection_regexes (id INT PRIMARY KEY, group_regex VARCHAR(255), regex VARCHAR(255), status TINYINT DEFAULT 1, ordinal INT DEFAULT 0) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collections (
+            id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+            subject VARCHAR(255) NOT NULL, fromname VARCHAR(255) NOT NULL, date DATETIME NULL,
+            xref VARCHAR(2000) NOT NULL DEFAULT \'\', groups_id INT UNSIGNED NOT NULL,
+            totalfiles INT UNSIGNED NOT NULL DEFAULT 0, collectionhash BINARY(20) NOT NULL,
+            collection_regexes_id INT NOT NULL DEFAULT 0, dateadded DATETIME NULL,
+            last_seen_at DATETIME NULL, filecheck TINYINT NOT NULL DEFAULT 0,
+            filesize BIGINT UNSIGNED NOT NULL DEFAULT 0, noise CHAR(32) NOT NULL DEFAULT \'\',
+            UNIQUE KEY ix_collection_collectionhash (collectionhash),
+            KEY ix_collections_group_filecheck_seen_id (groups_id, filecheck, last_seen_at, id)
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE collection_groups (
+            collections_id INT UNSIGNED NOT NULL, group_name VARCHAR(255) NOT NULL,
+            PRIMARY KEY (collections_id, group_name),
+            CONSTRAINT fk_test_collection_groups FOREIGN KEY (collections_id) REFERENCES collections(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE binaries (
+            id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, binaryhash BINARY(16) NOT NULL,
+            name VARCHAR(1000) NOT NULL, collections_id INT UNSIGNED NOT NULL,
+            filenumber INT UNSIGNED NOT NULL DEFAULT 0, totalparts INT UNSIGNED NOT NULL DEFAULT 0,
+            currentparts INT UNSIGNED NOT NULL DEFAULT 0, partcheck TINYINT NOT NULL DEFAULT 0,
+            partsize BIGINT UNSIGNED NOT NULL DEFAULT 0,
+            UNIQUE KEY ux_binaries_collection_hash (collections_id, binaryhash),
+            KEY ix_binaries_collection_filenumber (collections_id, filenumber),
+            CONSTRAINT fk_test_binaries FOREIGN KEY (collections_id) REFERENCES collections(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::statement('CREATE TABLE parts (
+            binaries_id BIGINT UNSIGNED NOT NULL,
+            messageid VARCHAR(255) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+            number BIGINT UNSIGNED NOT NULL, partnumber INT UNSIGNED NOT NULL, size INT UNSIGNED NOT NULL,
+            PRIMARY KEY (binaries_id, partnumber), KEY ix_parts_number (number),
+            CONSTRAINT fk_test_parts FOREIGN KEY (binaries_id) REFERENCES binaries(id) ON DELETE CASCADE
+        ) ENGINE=InnoDB');
+        DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.binaries.test']);
+    }
+
+    protected function tearDown(): void
+    {
+        if (\in_array(DB::getDriverName(), ['mysql', 'mariadb'], true)) {
+            DB::statement('SET FOREIGN_KEY_CHECKS=0');
+            foreach (['parts', 'parts_cbp_new', 'parts_cbp_pre_optimize', 'cbp_binary_map', 'cbp_optimization_checkpoints', 'binaries', 'collection_groups', 'collections', 'collection_regexes', 'usenet_groups'] as $table) {
+                DB::statement("DROP TABLE IF EXISTS {$table}");
+            }
+            DB::statement('SET FOREIGN_KEY_CHECKS=1');
+        }
+        parent::tearDown();
+    }
+
+    public function test_reingestion_is_idempotent_and_hot_lookups_use_indexes(): void
+    {
+        $service = new HeaderStorageService(
+            new CollectionHandler(new class extends CollectionsCleaningService
+            {
+                public function collectionsCleaner(string $subject, string $groupName = ''): array
+                {
+                    return ['id' => 0, 'name' => $subject];
+                }
+            }),
+            config: new BinariesConfig(headerChunkSize: 2, sqlChunkSize: 2),
+        );
+        $headers = [$this->header(1001, 1, 125), $this->header(1002, 2, 175)];
+
+        for ($attempt = 0; $attempt < 3; $attempt++) {
+            $this->assertSame([], $service->store($headers, ['id' => 1, 'name' => 'alt.binaries.test']));
+        }
+
+        $binary = DB::table('binaries')->first();
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(2, (int) $binary->currentparts);
+        $this->assertSame(300, (int) $binary->partsize);
+        $this->assertSame(300, (int) DB::table('collections')->value('filesize'));
+
+        $plan = DB::selectOne(
+            'EXPLAIN FORMAT=JSON SELECT id FROM binaries WHERE collections_id = ? AND binaryhash = ?',
+            [1, $binary->binaryhash]
+        );
+        $json = strtolower((string) array_values((array) $plan)[0]);
+        $this->assertStringContainsString('ux_binaries_collection_hash', $json);
+
+        $this->assertSame(0, Artisan::call('cbp:optimize-storage'));
+        $this->assertStringContainsString('Dry-run only', Artisan::output());
+
+        $this->restoreLegacyStorageShape();
+        $this->assertSame(2, DB::table('parts')->count());
+        config()->set('nntmux.cbp.reconcile_batch_size', 2);
+
+        $migration = require database_path('migrations/2026_08_03_000001_finalize_cbp_binary_hash_storage.php');
+        config()->set('nntmux.cbp.storage_migration_execute', false);
+        try {
+            $migration->up();
+            $this->fail('The legacy storage migration ran without explicit maintenance approval.');
+        } catch (\RuntimeException $exception) {
+            $this->assertStringContainsString('CBP_STORAGE_MIGRATION_EXECUTE=true', $exception->getMessage());
+        }
+        $this->assertSame(2, DB::table('parts')->count());
+
+        config()->set('nntmux.cbp.storage_migration_execute', true);
+        $migration->up();
+
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame(300, (int) DB::table('binaries')->value('partsize'));
+    }
+
+    private function restoreLegacyStorageShape(): void
+    {
+        DB::statement('ALTER TABLE parts DROP FOREIGN KEY fk_test_parts');
+        DB::statement('ALTER TABLE parts DROP PRIMARY KEY, DROP INDEX ix_parts_number, ADD PRIMARY KEY (binaries_id, number)');
+        DB::statement('ALTER TABLE parts MODIFY messageid VARCHAR(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL');
+
+        DB::statement('ALTER TABLE binaries DROP INDEX ux_binaries_collection_hash');
+        DB::statement('ALTER TABLE binaries MODIFY binaryhash BLOB NOT NULL');
+        DB::statement('CREATE INDEX ix_binaries_binaryhash ON binaries (binaryhash(16))');
+        DB::statement('CREATE UNIQUE INDEX ux_collection_id_filenumber ON binaries (collections_id, filenumber)');
+
+        DB::statement('ALTER TABLE collections DROP INDEX ix_collection_collectionhash');
+        DB::statement('ALTER TABLE collections MODIFY collectionhash BLOB NOT NULL');
+        DB::statement('CREATE UNIQUE INDEX ix_collection_collectionhash ON collections (collectionhash(20))');
+    }
+
+    /** @return array<string, mixed> */
+    private function header(int $number, int $part, int $bytes): array
+    {
+        return [
+            'Number' => $number,
+            'Subject' => "Integration.Release ({$part}/2)",
+            'From' => 'poster@example.test',
+            'Date' => time(),
+            'Bytes' => $bytes,
+            'Message-ID' => "<{$number}@example.test>",
+            'Xref' => "news.example alt.binaries.test:{$number}",
+            'matches' => [
+                0 => "Integration.Release ({$part}/2)",
+                1 => 'Integration.Release',
+                2 => $part,
+                3 => 2,
+            ],
+        ];
+    }
+}

tests/Support/TestBinariesHarness.php

@@ -33,8 +33,7 @@ class TestBinariesHarness extends BinariesService
             newGroupDaysToScan: 3,
             partRepairLimit: 15000,
             partRepairMaxTries: 3,
-            partsChunkSize: 5000,
-            binariesUpdateChunkSize: 1000,
+            sqlChunkSize: 500,
             echoCli: false
         );