From 445294e9e7644fe628f9769b073699e6b6603c5b Mon Sep 17 00:00:00 2001 From: DariusIII Date: Tue, 28 Apr 2026 14:58:29 +0200 Subject: [PATCH] Update chunk headers insert --- app/Services/Binaries/BinariesService.php | 19 ++-- .../Binaries/HeaderStorageService.php | 37 ++++++-- .../Feature/BinariesStorageInternalsTest.php | 86 ++++++++++++++++++- 3 files changed, 121 insertions(+), 21 deletions(-) diff --git a/app/Services/Binaries/BinariesService.php b/app/Services/Binaries/BinariesService.php index e2ee3d2f6..680c5a8be 100644 --- a/app/Services/Binaries/BinariesService.php +++ b/app/Services/Binaries/BinariesService.php @@ -7,7 +7,6 @@ namespace App\Services\Binaries; use App\Models\Settings; use App\Models\UsenetGroup; use App\Services\NNTP\NNTPService; -use Carbon\CarbonInterface; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; @@ -41,13 +40,13 @@ class BinariesService private float $timeInsert = 0; - private CarbonInterface $startLoop; + private Carbon $startLoop; - private CarbonInterface $startCleaning; + private Carbon $startCleaning; - private CarbonInterface $startPR; + private Carbon $startPR; - private CarbonInterface $startUpdate; + private Carbon $startUpdate; // Scan state /** @@ -83,7 +82,7 @@ class BinariesService $this->config->partRepairMaxTries ); $this->nntp = $nntp; - $this->startUpdate = now(); + $this->startUpdate = Carbon::now(); } /** @@ -265,7 +264,7 @@ class BinariesService */ public function scan(array $groupMySQL, int $first, int $last, string $type = 'update', ?array $missingParts = null): array { - $this->startLoop = now(); + $this->startLoop = Carbon::now(); $this->groupMySQL = $groupMySQL; $this->last = $last; $this->first = $first; @@ -286,7 +285,7 @@ class BinariesService return $returnArray; } - $this->startCleaning = now(); + $this->startCleaning = Carbon::now(); $this->timeHeaders = $this->startCleaning->diffInSeconds($this->startLoop, true); $msgCount = \count($headers); @@ -315,7 +314,7 @@ class BinariesService } // Store headers - $this->startUpdate = now(); // Reset before storage begins + $this->startUpdate = Carbon::now(); // Reset before storage begins $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning, true); $headersNotInserted = []; @@ -327,7 +326,7 @@ class BinariesService } } - $this->startPR = now(); + $this->startPR = Carbon::now(); $this->timeInsert = $this->startPR->diffInSeconds($this->startUpdate, true); // Handle repaired parts diff --git a/app/Services/Binaries/HeaderStorageService.php b/app/Services/Binaries/HeaderStorageService.php index afe792451..f63dcdf06 100644 --- a/app/Services/Binaries/HeaderStorageService.php +++ b/app/Services/Binaries/HeaderStorageService.php @@ -52,12 +52,33 @@ final class HeaderStorageService return []; } - // Reset all handlers + $this->failedInserts = []; + + $chunkSize = max(1, $this->config->partsChunkSize); + foreach (array_chunk($headers, $chunkSize) as $chunk) { + $this->storeChunk($chunk, $groupMySQL, $addToPartRepair); + } + + return array_values(array_unique($this->failedInserts)); + } + + /** + * Store one bounded header chunk inside its own transaction. + * + * @param array $headers + * @param array $groupMySQL + */ + private function storeChunk(array $headers, array $groupMySQL, bool $addToPartRepair): void + { $this->collectionHandler->reset(); $this->binaryHandler->reset(); $this->partHandler->reset(); $this->partHandler->setAddToPartRepair($addToPartRepair); - $this->failedInserts = []; + + $chunkNumbers = array_values(array_filter(array_map( + static fn (array $header): mixed => $header['Number'] ?? null, + $headers + ))); // Create transaction $transaction = new HeaderStorageTransaction( @@ -93,21 +114,21 @@ final class HeaderStorageService // Finish transaction if (! $transaction->finish()) { - // All failed if ($addToPartRepair) { - return array_unique(array_merge( + $this->failedInserts = array_merge( $this->failedInserts, + $chunkNumbers, $this->partHandler->getFailedNumbers() - )); + ); } - return []; + return; } - return array_unique(array_merge( + $this->failedInserts = array_merge( $this->failedInserts, $this->partHandler->getFailedNumbers() - )); + ); } /** diff --git a/tests/Feature/BinariesStorageInternalsTest.php b/tests/Feature/BinariesStorageInternalsTest.php index f3732c505..3aeb9769a 100644 --- a/tests/Feature/BinariesStorageInternalsTest.php +++ b/tests/Feature/BinariesStorageInternalsTest.php @@ -3,11 +3,14 @@ namespace Tests\Feature; use App\Services\Binaries\BinaryHandler; +use App\Services\Binaries\BinariesConfig; use App\Services\Binaries\CollectionHandler; use App\Services\Binaries\HeaderParser; +use App\Services\Binaries\HeaderStorageService; use App\Services\Binaries\HeaderStorageTransaction; use App\Services\Binaries\PartHandler; use App\Services\BlacklistService; +use App\Services\CollectionsCleaningService; use Illuminate\Support\Facades\DB; use Tests\TestCase; @@ -93,6 +96,36 @@ class BinariesStorageInternalsTest extends TestCase $this->assertSame(0, DB::table('parts')->where('binaries_id', 2)->count()); } + public function test_header_storage_commits_successful_chunks_and_reports_failed_chunk_numbers(): void + { + $this->createHeaderStorageTables('CHECK(size < 500)'); + + $collectionHandler = new CollectionHandler(new class extends CollectionsCleaningService + { + public function __construct() {} + + public function collectionsCleaner(string $subject, string $groupName = ''): array + { + return ['id' => 0, 'name' => $subject]; + } + }); + $service = new HeaderStorageService($collectionHandler, config: new BinariesConfig(partsChunkSize: 2)); + $failed = $service->store([ + $this->parsedHeader(301, 1, 'Chunk.One', 100), + $this->parsedHeader(302, 2, 'Chunk.One', 100), + $this->parsedHeader(303, 1, 'Chunk.Two', 999), + $this->parsedHeader(304, 2, 'Chunk.Two', 999), + ], ['id' => 1, 'name' => 'alt.test'], true); + + sort($failed); + + $this->assertSame([303, 304], $failed); + $this->assertSame(1, DB::table('collections')->count()); + $this->assertSame(1, DB::table('binaries')->count()); + $this->assertSame(2, DB::table('parts')->count()); + $this->assertSame([301, 302], DB::table('parts')->orderBy('number')->pluck('number')->all()); + } + private function rawHeader(int $number, string $subject): array { return [ @@ -106,12 +139,13 @@ class BinariesStorageInternalsTest extends TestCase ]; } - private function parsedHeader(int $number, int $partNumber): array + private function parsedHeader(int $number, int $partNumber, string $subjectBase = 'Example.Release', int $bytes = 100): array { - $header = $this->rawHeader($number, 'Example.Release ('.$partNumber.'/2)'); + $header = $this->rawHeader($number, $subjectBase.' ('.$partNumber.'/2)'); + $header['Bytes'] = $bytes; $header['matches'] = [ 0 => $header['Subject'], - 1 => 'Example.Release', + 1 => $subjectBase, 2 => $partNumber, 3 => 2, ]; @@ -119,6 +153,52 @@ class BinariesStorageInternalsTest extends TestCase return $header; } + private function createHeaderStorageTables(string $partSizeConstraint = ''): void + { + DB::statement('CREATE TABLE collections ( + id INTEGER PRIMARY KEY, + subject VARCHAR(255), + fromname VARCHAR(255), + date DATETIME NULL, + xref TEXT DEFAULT "", + groups_id INT, + totalfiles INT, + collectionhash VARCHAR(40) UNIQUE, + collection_regexes_id INT, + dateadded DATETIME NULL, + noise VARCHAR(64) DEFAULT "" + )'); + + DB::statement('CREATE TABLE binaries ( + id INTEGER PRIMARY KEY, + binaryhash BLOB, + name VARCHAR(255), + collections_id INT, + totalparts INT, + currentparts INT, + filenumber INT, + partsize INT, + UNIQUE(binaryhash, collections_id) + )'); + + DB::statement('CREATE TABLE parts ( + binaries_id INT, + number INT, + messageid VARCHAR(255), + partnumber INT, + size INT '.$partSizeConstraint.', + UNIQUE(binaries_id, number) + )'); + + DB::statement('CREATE TABLE collection_regexes ( + id INTEGER PRIMARY KEY, + group_regex VARCHAR(255), + regex VARCHAR(255), + status INT DEFAULT 1, + ordinal INT DEFAULT 0 + )'); + } + private function setPrivateProperty(object $object, string $property, mixed $value): void { $reflection = new \ReflectionProperty($object, $property);