newznab-tmux-NNTmux/tests/Feature/BinariesStorageInternalsTest.php

<?php

namespace Tests\Feature;

use App\Services\Binaries\BinariesConfig;
use App\Services\Binaries\BinaryHandler;
use App\Services\Binaries\CollectionHandler;
use App\Services\Binaries\HeaderParser;
use App\Services\Binaries\HeaderStorageService;
use App\Services\Binaries\HeaderStorageTransaction;
use App\Services\Binaries\PartHandler;
use App\Services\BlacklistService;
use App\Services\CollectionsCleaningService;
use Illuminate\Support\Facades\DB;
use Tests\TestCase;

class BinariesStorageInternalsTest extends TestCase
{
    protected function setUp(): void
    {
        parent::setUp();
        config(['database.default' => 'sqlite', 'database.connections.sqlite.database' => ':memory:']);
        DB::purge();
        DB::reconnect();
    }

    public function test_header_parser_excludes_usenet_index_posts_and_returns_received_numbers(): void
    {
        $parser = new HeaderParser(new class extends BlacklistService
        {
            public function isBlackListed(array $msg, string $groupName): bool
            {
                return false;
            }
        });

        $result = $parser->parse([
            $this->rawHeader(101, 'Example.Release (1/2)'),
            $this->rawHeader(102, 'Usenet Index Post Example.Release (1/1)'),
            ['Subject' => 'Missing number (1/1)'],
        ], 'alt.test');

        $this->assertSame([101, 102], $result['received']);
        $this->assertCount(1, $result['headers']);
        $this->assertSame(1, $result['notYEnc']);
    }

    public function test_part_handler_ignored_duplicate_is_not_reported_failed(): void
    {
        DB::statement('CREATE TABLE parts (
            binaries_id INT,
            number INT,
            messageid VARCHAR(255),
            partnumber INT,
            size INT,
            UNIQUE(binaries_id, partnumber)
        )');

        $handler = new PartHandler(100);
        $this->assertTrue($handler->addPart(1, $this->parsedHeader(201, 1)));
        $this->assertTrue($handler->flush());
        $this->assertSame([201], $handler->getInsertedNumbers());
        $this->assertSame([], $handler->getFailedNumbers());

        $this->assertTrue($handler->addPart(1, $this->parsedHeader(201, 1)));
        $this->assertTrue($handler->flush());
        $this->assertSame([], $handler->getFailedNumbers());
        $this->assertSame(1, DB::table('parts')->count());
    }

    public function test_binary_handler_flushes_cached_article_aggregate_updates(): void
    {
        DB::statement('CREATE TABLE binaries (
            id INTEGER PRIMARY KEY,
            binaryhash BLOB,
            name VARCHAR(255),
            collections_id INT,
            totalparts INT,
            currentparts INT,
            filenumber INT,
            partsize INT,
            partcheck INT DEFAULT 0,
            UNIQUE(binaryhash, collections_id)
        )');
        DB::statement('CREATE TABLE parts (
            binaries_id INT,
            number INT,
            messageid VARCHAR(255),
            partnumber INT,
            size INT,
            UNIQUE(binaries_id, partnumber)
        )');

        $handler = new BinaryHandler;
        $first = $this->parsedHeader(251, 1, 'Aggregate.Release', 100);
        $second = $this->parsedHeader(252, 2, 'Aggregate.Release', 50);

        $binaryId = $handler->getOrCreateBinary($first, 1, 1, 0);
        $this->assertNotNull($binaryId);
        $this->assertSame($binaryId, $handler->getOrCreateBinary($second, 1, 1, 0));
        DB::table('parts')->insert([
            ['binaries_id' => $binaryId, 'number' => 251, 'messageid' => '<251@example>', 'partnumber' => 1, 'size' => 100],
            ['binaries_id' => $binaryId, 'number' => 252, 'messageid' => '<252@example>', 'partnumber' => 2, 'size' => 50],
        ]);
        $this->assertTrue($handler->refreshAggregates([$binaryId]));

        $binary = DB::table('binaries')->where('id', $binaryId)->first();
        $this->assertSame(2, (int) $binary->currentparts);
        $this->assertSame(150, (int) $binary->partsize);
    }

    public function test_sqlite_rollback_cleanup_keeps_unrelated_parts_with_same_article_number(): void
    {
        DB::statement('CREATE TABLE collections (id INTEGER PRIMARY KEY, collectionhash VARCHAR(40), noise VARCHAR(64))');
        DB::statement('CREATE TABLE binaries (id INTEGER PRIMARY KEY, collections_id INT)');
        DB::statement('CREATE TABLE parts (binaries_id INT, number INT, messageid VARCHAR(255), UNIQUE(binaries_id, number))');

        DB::table('collections')->insert(['id' => 1, 'collectionhash' => 'keep', 'noise' => '']);
        DB::table('binaries')->insert(['id' => 1, 'collections_id' => 1]);
        DB::table('parts')->insert(['binaries_id' => 1, 'number' => 777, 'messageid' => '<keep@example>']);

        $collectionHandler = new CollectionHandler;
        $binaryHandler = new BinaryHandler;
        $partHandler = new PartHandler;
        $transaction = new HeaderStorageTransaction($collectionHandler, $binaryHandler, $partHandler);

        $transaction->begin();
        DB::table('collections')->insert(['id' => 2, 'collectionhash' => 'rollback', 'noise' => $transaction->getBatchNoise()]);
        DB::table('binaries')->insert(['id' => 2, 'collections_id' => 2]);
        DB::table('parts')->insert(['binaries_id' => 2, 'number' => 777, 'messageid' => '<rollback@example>']);
        $this->setPrivateProperty($collectionHandler, 'insertedCollectionIds', [2 => true]);
        $this->setPrivateProperty($binaryHandler, 'insertedBinaryIds', [2 => true]);
        $transaction->markError();
        $this->assertFalse($transaction->finish());

        $this->assertSame(1, DB::table('parts')->where('binaries_id', 1)->where('number', 777)->count());
        $this->assertSame(0, DB::table('parts')->where('binaries_id', 2)->count());
    }

    public function test_header_storage_commits_successful_chunks_and_reports_failed_chunk_numbers(): void
    {
        $this->createHeaderStorageTables('CHECK(size < 500)');

        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 2, headerChunkSize: 2));
        $failed = $service->store([
            $this->parsedHeader(301, 1, 'Chunk.One', 100),
            $this->parsedHeader(302, 2, 'Chunk.One', 100),
            $this->parsedHeader(303, 1, 'Chunk.Two', 999),
            $this->parsedHeader(304, 2, 'Chunk.Two', 999),
        ], ['id' => 1, 'name' => 'alt.test'], true);

        sort($failed);

        $this->assertSame([303, 304], $failed);
        $this->assertSame(1, DB::table('collections')->count());
        $this->assertSame(1, DB::table('binaries')->count());
        $this->assertSame(2, DB::table('parts')->count());
        $this->assertSame([301, 302], DB::table('parts')->orderBy('number')->pluck('number')->all());
    }

    public function test_header_storage_batch_reuses_collection_and_binary_for_parts(): void
    {
        $this->createHeaderStorageTables();

        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $failed = $service->store([
            $this->parsedHeader(401, 1, 'Batch.Release', 150),
            $this->parsedHeader(402, 2, 'Batch.Release', 175),
        ], ['id' => 1, 'name' => 'alt.test'], true);

        $binary = DB::table('binaries')->first();

        $this->assertSame([], $failed);
        $this->assertSame(1, DB::table('collections')->count());
        $this->assertSame(1, DB::table('binaries')->count());
        $this->assertSame(2, DB::table('parts')->count());
        $this->assertSame(2, (int) $binary->currentparts);
        $this->assertSame(325, (int) $binary->partsize);
    }

    public function test_header_storage_batch_updates_binary_that_exists_before_chunk(): void
    {
        $this->createHeaderStorageTables();

        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $this->assertSame([], $service->store([
            $this->parsedHeader(501, 1, 'Existing.Batch.Release', 100),
        ], ['id' => 1, 'name' => 'alt.test'], true));

        $this->assertSame([], $service->store([
            $this->parsedHeader(502, 2, 'Existing.Batch.Release', 150),
        ], ['id' => 1, 'name' => 'alt.test'], true));

        $binary = DB::table('binaries')->first();

        $this->assertSame(1, DB::table('collections')->count());
        $this->assertSame(1, DB::table('binaries')->count());
        $this->assertSame(2, DB::table('parts')->count());
        $this->assertSame(2, (int) $binary->currentparts);
        $this->assertSame(250, (int) $binary->partsize);
    }

    public function test_three_identical_ingestions_leave_authoritative_counts_unchanged(): void
    {
        $this->createHeaderStorageTables();

        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $headers = [
            $this->parsedHeader(551, 1, 'Idempotent.Release', 125),
            $this->parsedHeader(552, 2, 'Idempotent.Release', 175),
        ];

        for ($attempt = 0; $attempt < 3; $attempt++) {
            $this->assertSame([], $service->store($headers, ['id' => 1, 'name' => 'alt.test'], true));
        }

        $binary = DB::table('binaries')->first();
        $collection = DB::table('collections')->first();
        $this->assertSame(1, DB::table('binaries')->count());
        $this->assertSame(2, DB::table('parts')->count());
        $this->assertSame(2, (int) $binary->currentparts);
        $this->assertSame(300, (int) $binary->partsize);
        $this->assertSame(300, (int) $collection->filesize);
    }

    public function test_duplicate_partnumber_prefers_non_empty_message_then_size_then_article_number(): void
    {
        DB::statement('CREATE TABLE parts (
            binaries_id INT,
            number INT,
            messageid VARCHAR(255),
            partnumber INT,
            size INT,
            UNIQUE(binaries_id, partnumber)
        )');

        $handler = new PartHandler(10);
        $small = $this->parsedHeader(900, 1, 'Preference.Release', 100);
        $large = $this->parsedHeader(901, 1, 'Preference.Release', 200);
        $this->assertTrue($handler->addPart(7, $small));
        $this->assertTrue($handler->addPart(7, $large));
        $this->assertTrue($handler->flush());

        $part = DB::table('parts')->first();
        $this->assertSame(901, (int) $part->number);
        $this->assertSame(200, (int) $part->size);
    }

    public function test_invalid_message_id_rolls_back_the_header_chunk(): void
    {
        $this->createHeaderStorageTables();
        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $header = $this->parsedHeader(910, 1, 'Invalid.Message.Release', 100);
        $header['Message-ID'] = "<invalid\u{00E9}@example>";

        $this->assertSame([910], $service->store([$header], ['id' => 1, 'name' => 'alt.test'], true));
        $this->assertSame(0, DB::table('collections')->count());
        $this->assertSame(0, DB::table('binaries')->count());
        $this->assertSame(0, DB::table('parts')->count());
    }

    public function test_zero_file_number_uses_subject_and_poster_identity(): void
    {
        $this->createHeaderStorageTables();
        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $first = $this->parsedHeader(920, 1, 'Unknown.File.Release', 100);
        $second = $this->parsedHeader(921, 1, 'Unknown.File.Release', 100);
        $second['From'] = 'another-poster@example.com';

        $this->assertSame([], $service->store([$first, $second], ['id' => 1, 'name' => 'alt.test'], true));
        $this->assertSame(1, DB::table('collections')->count());
        $this->assertSame(2, DB::table('binaries')->count());
        $this->assertSame(2, DB::table('parts')->count());
    }

    public function test_cross_post_groups_are_normalized_and_deduplicated(): void
    {
        $this->createHeaderStorageTables();
        $service = new HeaderStorageService($this->deterministicCollectionHandler(), config: new BinariesConfig(sqlChunkSize: 10));
        $header = $this->parsedHeader(930, 1, 'Cross.Post.Release', 100);
        $header['Xref'] = 'news.example alt.binaries.one:930 alt.binaries.two:931 alt.binaries.one:930';

        $this->assertSame([], $service->store([$header], ['id' => 1, 'name' => 'alt.binaries.one'], true));
        $this->assertSame(
            ['alt.binaries.one', 'alt.binaries.two'],
            DB::table('collection_groups')->orderBy('group_name')->pluck('group_name')->all()
        );
    }

    public function test_header_storage_does_not_merge_same_subject_across_different_collections(): void
    {
        $this->createHeaderStorageTables();

        $handler = new BinaryHandler;
        $header = $this->parsedHeader(601, 1, 'Same.Subject', 100);
        $firstId = $handler->getOrCreateBinary($header, 10, 1, 0);
        $secondId = $handler->getOrCreateBinary($header, 20, 1, 0);

        $this->assertNotNull($firstId);
        $this->assertNotNull($secondId);
        $this->assertNotSame($firstId, $secondId);
        $this->assertSame(2, DB::table('binaries')->count());
    }

    private function rawHeader(int $number, string $subject): array
    {
        return [
            'Number' => $number,
            'Subject' => $subject,
            'From' => 'poster@example.com',
            'Date' => time(),
            'Bytes' => 100,
            'Message-ID' => '<msg'.$number.'@example.com>',
            'Xref' => 'news.example.com group:'.$number,
        ];
    }

    private function parsedHeader(int $number, int $partNumber, string $subjectBase = 'Example.Release', int $bytes = 100): array
    {
        return $this->parsedHeaderWithTotal($number, $partNumber, 2, $subjectBase, $bytes);
    }

    private function parsedHeaderWithTotal(int $number, int $partNumber, int $totalParts, string $subjectBase, int $bytes = 100): array
    {
        $header = $this->rawHeader($number, $subjectBase.' ('.$partNumber.'/'.$totalParts.')');
        $header['Bytes'] = $bytes;
        $header['matches'] = [
            0 => $header['Subject'],
            1 => $subjectBase,
            2 => $partNumber,
            3 => $totalParts,
        ];

        return $header;
    }

    private function createHeaderStorageTables(string $partSizeConstraint = ''): void
    {
        DB::statement('CREATE TABLE collections (
            id INTEGER PRIMARY KEY,
            subject VARCHAR(255),
            fromname VARCHAR(255),
            date DATETIME NULL,
            xref TEXT DEFAULT \'\',
            groups_id INT,
            totalfiles INT,
            collectionhash VARCHAR(40) UNIQUE,
            collection_regexes_id INT,
            dateadded DATETIME NULL,
            last_seen_at DATETIME NULL,
            filecheck INT DEFAULT 0,
            filesize INT DEFAULT 0,
            noise VARCHAR(64) DEFAULT \'\'
        )');

        DB::statement('CREATE TABLE collection_groups (
            collections_id INT,
            group_name VARCHAR(255),
            UNIQUE(collections_id, group_name)
        )');

        DB::statement('CREATE TABLE binaries (
            id INTEGER PRIMARY KEY,
            binaryhash BLOB,
            name VARCHAR(255),
            collections_id INT,
            totalparts INT,
            currentparts INT,
            filenumber INT,
            partsize INT,
            partcheck INT DEFAULT 0,
            UNIQUE(binaryhash, collections_id)
        )');

        DB::statement('CREATE TABLE parts (
            binaries_id INT,
            number INT,
            messageid VARCHAR(255),
            partnumber INT,
            size INT '.$partSizeConstraint.',
            UNIQUE(binaries_id, partnumber)
        )');

        DB::statement('CREATE TABLE collection_regexes (
            id INTEGER PRIMARY KEY,
            group_regex VARCHAR(255),
            regex VARCHAR(255),
            status INT DEFAULT 1,
            ordinal INT DEFAULT 0
        )');
    }

    private function deterministicCollectionHandler(): CollectionHandler
    {
        return new CollectionHandler(new class extends CollectionsCleaningService
        {
            public function __construct()
            {
                parent::__construct();
            }

            public function collectionsCleaner(string $subject, string $groupName = ''): array
            {
                return ['id' => 0, 'name' => $subject];
            }
        });
    }

    private function setPrivateProperty(object $object, string $property, mixed $value): void
    {
        $reflection = new \ReflectionProperty($object, $property);
        $reflection->setValue($object, $value);
    }
}