Update chunk headers insert

app/Services/Binaries/BinariesService.php

@@ -7,7 +7,6 @@ namespace App\Services\Binaries;
 use App\Models\Settings;
 use App\Models\UsenetGroup;
 use App\Services\NNTP\NNTPService;
-use Carbon\CarbonInterface;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
@@ -41,13 +40,13 @@ class BinariesService
 
     private float $timeInsert = 0;
 
-    private CarbonInterface $startLoop;
+    private Carbon $startLoop;
 
-    private CarbonInterface $startCleaning;
+    private Carbon $startCleaning;
 
-    private CarbonInterface $startPR;
+    private Carbon $startPR;
 
-    private CarbonInterface $startUpdate;
+    private Carbon $startUpdate;
 
     // Scan state
     /**
@@ -83,7 +82,7 @@ class BinariesService
             $this->config->partRepairMaxTries
         );
         $this->nntp = $nntp;
-        $this->startUpdate = now();
+        $this->startUpdate = Carbon::now();
     }
 
     /**
@@ -265,7 +264,7 @@ class BinariesService
      */
     public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', ?array $missingParts = null): array
     {
-        $this->startLoop = now();
+        $this->startLoop = Carbon::now();
         $this->groupMySQL = $groupMySQL;
         $this->last = $last;
         $this->first = $first;
@@ -286,7 +285,7 @@ class BinariesService
             return $returnArray;
         }
 
-        $this->startCleaning = now();
+        $this->startCleaning = Carbon::now();
         $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true);
 
         $msgCount = \count($headers);
@@ -315,7 +314,7 @@ class BinariesService
         }
 
         // Store headers
-        $this->startUpdate = now();  // Reset before storage begins
+        $this->startUpdate = Carbon::now();  // Reset before storage begins
         $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true);
 
         $headersNotInserted = [];
@@ -327,7 +326,7 @@ class BinariesService
             }
         }
 
-        $this->startPR = now();
+        $this->startPR = Carbon::now();
         $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true);
 
         // Handle repaired parts

app/Services/Binaries/HeaderStorageService.php

@@ -52,12 +52,33 @@ final class HeaderStorageService
             return [];
         }
 
-        // Reset all handlers
+        $this->failedInserts = [];
+
+        $chunkSize = max(1, $this->config->partsChunkSize);
+        foreach (array_chunk($headers, $chunkSize) as $chunk) {
+            $this->storeChunk($chunk, $groupMySQL, $addToPartRepair);
+        }
+
+        return array_values(array_unique($this->failedInserts));
+    }
+
+    /**
+     * Store one bounded header chunk inside its own transaction.
+     *
+     * @param  array<string, mixed>  $headers
+     * @param  array<string, mixed>  $groupMySQL
+     */
+    private function storeChunk(array $headers, array $groupMySQL, bool $addToPartRepair): void
+    {
         $this->collectionHandler->reset();
         $this->binaryHandler->reset();
         $this->partHandler->reset();
         $this->partHandler->setAddToPartRepair($addToPartRepair);
-        $this->failedInserts = [];
+
+        $chunkNumbers = array_values(array_filter(array_map(
+            static fn (array $header): mixed => $header['Number'] ?? null,
+            $headers
+        )));
 
         // Create transaction
         $transaction = new HeaderStorageTransaction(
@@ -93,21 +114,21 @@ final class HeaderStorageService
 
         // Finish transaction
         if (! $transaction->finish()) {
-            // All failed
             if ($addToPartRepair) {
-                return array_unique(array_merge(
+                $this->failedInserts = array_merge(
                     $this->failedInserts,
+                    $chunkNumbers,
                     $this->partHandler->getFailedNumbers()
-                ));
+                );
             }
 
-            return [];
+            return;
         }
 
-        return array_unique(array_merge(
+        $this->failedInserts = array_merge(
             $this->failedInserts,
             $this->partHandler->getFailedNumbers()
-        ));
+        );
     }
 
     /**

tests/Feature/BinariesStorageInternalsTest.php

@@ -3,11 +3,14 @@
 namespace Tests\Feature;
 
 use App\Services\Binaries\BinaryHandler;
+use App\Services\Binaries\BinariesConfig;
 use App\Services\Binaries\CollectionHandler;
 use App\Services\Binaries\HeaderParser;
+use App\Services\Binaries\HeaderStorageService;
 use App\Services\Binaries\HeaderStorageTransaction;
 use App\Services\Binaries\PartHandler;
 use App\Services\BlacklistService;
+use App\Services\CollectionsCleaningService;
 use Illuminate\Support\Facades\DB;
 use Tests\TestCase;
 
@@ -93,6 +96,36 @@ class BinariesStorageInternalsTest extends TestCase
         $this->assertSame(0, DB::table('parts')->where('binaries_id', 2)->count());
     }
 
+    public function test_header_storage_commits_successful_chunks_and_reports_failed_chunk_numbers(): void
+    {
+        $this->createHeaderStorageTables('CHECK(size < 500)');
+
+        $collectionHandler = new CollectionHandler(new class extends CollectionsCleaningService
+        {
+            public function __construct() {}
+
+            public function collectionsCleaner(string $subject, string $groupName = ''): array
+            {
+                return ['id' => 0, 'name' => $subject];
+            }
+        });
+        $service = new HeaderStorageService($collectionHandler, config: new BinariesConfig(partsChunkSize: 2));
+        $failed = $service->store([
+            $this->parsedHeader(301, 1, 'Chunk.One', 100),
+            $this->parsedHeader(302, 2, 'Chunk.One', 100),
+            $this->parsedHeader(303, 1, 'Chunk.Two', 999),
+            $this->parsedHeader(304, 2, 'Chunk.Two', 999),
+        ], ['id' => 1, 'name' => 'alt.test'], true);
+
+        sort($failed);
+
+        $this->assertSame([303, 304], $failed);
+        $this->assertSame(1, DB::table('collections')->count());
+        $this->assertSame(1, DB::table('binaries')->count());
+        $this->assertSame(2, DB::table('parts')->count());
+        $this->assertSame([301, 302], DB::table('parts')->orderBy('number')->pluck('number')->all());
+    }
+
     private function rawHeader(int $number, string $subject): array
     {
         return [
@@ -106,12 +139,13 @@ class BinariesStorageInternalsTest extends TestCase
         ];
     }
 
-    private function parsedHeader(int $number, int $partNumber): array
+    private function parsedHeader(int $number, int $partNumber, string $subjectBase = 'Example.Release', int $bytes = 100): array
     {
-        $header = $this->rawHeader($number, 'Example.Release ('.$partNumber.'/2)');
+        $header = $this->rawHeader($number, $subjectBase.' ('.$partNumber.'/2)');
+        $header['Bytes'] = $bytes;
         $header['matches'] = [
             0 => $header['Subject'],
-            1 => 'Example.Release',
+            1 => $subjectBase,
             2 => $partNumber,
             3 => 2,
         ];
@@ -119,6 +153,52 @@ class BinariesStorageInternalsTest extends TestCase
         return $header;
     }
 
+    private function createHeaderStorageTables(string $partSizeConstraint = ''): void
+    {
+        DB::statement('CREATE TABLE collections (
+            id INTEGER PRIMARY KEY,
+            subject VARCHAR(255),
+            fromname VARCHAR(255),
+            date DATETIME NULL,
+            xref TEXT DEFAULT "",
+            groups_id INT,
+            totalfiles INT,
+            collectionhash VARCHAR(40) UNIQUE,
+            collection_regexes_id INT,
+            dateadded DATETIME NULL,
+            noise VARCHAR(64) DEFAULT ""
+        )');
+
+        DB::statement('CREATE TABLE binaries (
+            id INTEGER PRIMARY KEY,
+            binaryhash BLOB,
+            name VARCHAR(255),
+            collections_id INT,
+            totalparts INT,
+            currentparts INT,
+            filenumber INT,
+            partsize INT,
+            UNIQUE(binaryhash, collections_id)
+        )');
+
+        DB::statement('CREATE TABLE parts (
+            binaries_id INT,
+            number INT,
+            messageid VARCHAR(255),
+            partnumber INT,
+            size INT '.$partSizeConstraint.',
+            UNIQUE(binaries_id, number)
+        )');
+
+        DB::statement('CREATE TABLE collection_regexes (
+            id INTEGER PRIMARY KEY,
+            group_regex VARCHAR(255),
+            regex VARCHAR(255),
+            status INT DEFAULT 1,
+            ordinal INT DEFAULT 0
+        )');
+    }
+
     private function setPrivateProperty(object $object, string $property, mixed $value): void
     {
         $reflection = new \ReflectionProperty($object, $property);