diff --git a/app/Facades/Search.php b/app/Facades/Search.php index 237c197ce..562febfa1 100644 --- a/app/Facades/Search.php +++ b/app/Facades/Search.php @@ -17,6 +17,7 @@ use Illuminate\Support\Facades\Facade; * @method static void insertRelease(array $parameters) * @method static void updateRelease(int|string $releaseID) * @method static void deleteRelease(int $id) + * @method static void deleteReleases(iterable $ids) * @method static void insertPredb(array $parameters) * @method static void updatePreDb(array $parameters) * @method static void deletePreDb(int $id) diff --git a/app/Services/ReleaseRemoverService.php b/app/Services/ReleaseRemoverService.php index ee52334d7..1673542fb 100644 --- a/app/Services/ReleaseRemoverService.php +++ b/app/Services/ReleaseRemoverService.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace App\Services; use App\Enums\BlacklistConstants; -use App\Facades\Search; use App\Models\Category; use App\Models\Settings; use App\Services\Nzb\NzbService; @@ -13,12 +12,17 @@ use App\Services\Releases\ReleaseManagementService; use Carbon\Carbon; use Exception; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; /** * Handles removing of various unwanted releases. */ class ReleaseRemoverService { + private const int BATCH_SIZE = 500; + + private const int BATCH_PAUSE_US = 10000; + // Crap removal types private const string TYPE_BLACKLIST = 'blacklist'; @@ -72,11 +76,6 @@ class ReleaseRemoverService protected ReleaseManagementService $releaseManagement; - /** - * @var array - */ - protected array $result = []; - private NzbService $nzb; private ReleaseImageService $releaseImage; @@ -586,40 +585,6 @@ class ReleaseRemoverService )); } - /** - * Get group IDs clause for a regex group name pattern. - * - * @return string|null Group IDs clause or null if no groups found - */ - private function getGroupIDsClause(string $groupname): ?string - { - if (strtolower($groupname) === 'alt.binaries.*') { - return ''; - } - - $groupIDs = DB::select( - 'SELECT id FROM usenet_groups WHERE name REGEXP '.escapeString($groupname) - ); - - if (empty($groupIDs)) { - return null; - } - - $ids = collect($groupIDs)->pluck('id')->implode(','); - - return ' AND r.groups_id IN ('.$ids.') '; - } - - /** - * Perform search using configured search engine. - * - * @return array - */ - private function performSearch(string $regexMatch): array - { - return Search::searchReleases($regexMatch, 100); - } - /** * Remove releases using the site blacklist regexes. * @@ -650,77 +615,33 @@ class ReleaseRemoverService return true; } - foreach ($regexList as $regex) { - $this->processBlacklistRegex($regex); + $rules = $this->compileBlacklistRules($regexList); + if ($rules === []) { + return true; } - return true; - } + $this->processPhpMatchedCandidates( + 'SELECT r.id, r.guid, r.searchname, r.fromname, r.groups_id FROM releases r WHERE 1=1 '.$this->crapTime, + function (object $release) use ($rules): ?string { + foreach ($rules as $rule) { + if (! $this->ruleAppliesToGroup($rule, (int) $release->groups_id)) { + continue; + } - /** - * Process a single blacklist regex. - * - * @throws Exception - */ - private function processBlacklistRegex(object $regex): void - { - $dbRegex = escapeString($regex->regex); - $regexMatch = ($this->crapTime === '') ? $this->extractSrchFromRegx($dbRegex) : ''; + $value = (int) $rule->msgcol === BlacklistConstants::BLACKLIST_FIELD_SUBJECT + ? (string) $release->searchname + : (string) $release->fromname; - [$regexSQL, $opTypeName] = $this->buildBlacklistRegexSQL((int) $regex->msgcol, $dbRegex); + if (preg_match($rule->pattern, $value) === 1) { + return 'Blacklist ['.$rule->id.']'; + } + } - if ($regexSQL === '') { - return; - } - - $groupID = $this->getGroupIDsClause($regex->groupname); - if ($groupID === null) { - return; - } - - $this->method = 'Blacklist ['.$regex->id.']'; - $this->logBlacklistOperation($opTypeName, $regexMatch); - - $searchResult = ($opTypeName === 'Subject') ? $this->performSearch($regexMatch) : ''; - - $this->query = sprintf( - 'SELECT r.guid, r.searchname, r.id FROM releases r %s %s %s %s', - $regexSQL, - ! empty($searchResult) ? ' WHERE r.id IN ('.implode(',', $searchResult).')' : '', - $groupID, - $this->crapTime + return null; + } ); - if ($this->checkSelectQuery()) { - $this->deleteReleases(); - } - } - - /** - * Build the regex SQL and operation type name for blacklist. - * - * @return list - */ - private function buildBlacklistRegexSQL(int $msgcol, string $dbRegex): array - { - return match ($msgcol) { - BlacklistConstants::BLACKLIST_FIELD_SUBJECT => [sprintf('WHERE r.searchname REGEXP %s', $dbRegex), 'Subject'], - BlacklistConstants::BLACKLIST_FIELD_FROM => ['WHERE r.fromname REGEXP '.$dbRegex, 'Poster'], - default => ['', ''], - }; - } - - /** - * Log blacklist operation details. - */ - private function logBlacklistOperation(string $opTypeName, string $regexMatch): void - { - cli()->header(sprintf( - 'Finding crap releases for %s: Using only REGEXP method against release %s.%s', - $this->method, - $opTypeName, - PHP_EOL - ), true); + return true; } /** @@ -746,48 +667,155 @@ class ReleaseRemoverService return true; } - foreach ($allRegex as $regex) { - $this->processBlacklistFilesRegex($regex); + $rules = $this->compileBlacklistRules($allRegex); + if ($rules === []) { + return true; } + $this->processPhpMatchedCandidates( + 'SELECT r.id, r.guid, r.searchname, r.groups_id FROM releases r '. + 'WHERE EXISTS (SELECT 1 FROM release_files rf WHERE rf.releases_id = r.id) '.$this->crapTime, + function (object $release) use ($rules): ?string { + foreach ($rules as $rule) { + if (! $this->ruleAppliesToGroup($rule, (int) $release->groups_id)) { + continue; + } + + foreach ($release->file_names as $name) { + if (preg_match($rule->pattern, (string) $name) === 1) { + return 'Blacklist Files '.$rule->id; + } + } + } + + return null; + }, + true + ); + return true; } /** - * Process a single blacklist files regex. - * - * @throws Exception + * @param array $rules + * @return array */ - private function processBlacklistFilesRegex(object $regex): void + private function compileBlacklistRules(array $rules): array { - $regexSQL = sprintf( - 'JOIN release_files rf ON r.id = rf.releases_id WHERE rf.name REGEXP %s', - escapeString($regex->regex) - ); + $groups = DB::table('usenet_groups')->pluck('name', 'id'); + $compiled = []; - $groupID = $this->getGroupIDsClause($regex->groupname); - if ($groupID === null) { - return; + foreach ($rules as $rule) { + $pattern = '/'.$rule->regex.'/i'; + if (@preg_match($pattern, '') === false) { + Log::warning('Skipping invalid release-removal blacklist regex', [ + 'blacklist_id' => $rule->id, + 'regex' => $rule->regex, + ]); + + continue; + } + + $rule->pattern = $pattern; + $rule->group_ids = null; + + if (strtolower((string) $rule->groupname) !== 'alt.binaries.*') { + $groupPattern = '/'.$rule->groupname.'/i'; + if (@preg_match($groupPattern, '') === false) { + Log::warning('Skipping blacklist with invalid group regex', [ + 'blacklist_id' => $rule->id, + 'group_regex' => $rule->groupname, + ]); + + continue; + } + + $rule->group_ids = $groups + ->filter(static fn (string $name): bool => preg_match($groupPattern, $name) === 1) + ->keys() + ->map(static fn (int|string $id): int => (int) $id) + ->all(); + + if ($rule->group_ids === []) { + continue; + } + } + + $compiled[] = $rule; } - $this->method = 'Blacklist Files '.$regex->id; + return $compiled; + } - cli()->header(sprintf( - 'Finding crap releases for %s: Using only REGEXP method against release filenames.%s', - $this->method, - PHP_EOL - ), true); + private function ruleAppliesToGroup(object $rule, int $groupId): bool + { + return $rule->group_ids === null || in_array($groupId, $rule->group_ids, true); + } - $this->query = sprintf( - 'SELECT DISTINCT r.id, r.guid, r.searchname FROM releases r %s %s %s', - $regexSQL, - $groupID, - $this->crapTime - ); + /** + * @param callable(object): ?string $matcher + */ + private function processPhpMatchedCandidates(string $candidateSql, callable $matcher, bool $includeFiles = false): void + { + $lastId = 0; - if ($this->checkSelectQuery()) { - $this->deleteReleases(); - } + do { + $candidates = DB::select( + 'SELECT candidates.* FROM ('.$this->cleanSpaces($candidateSql).') candidates '. + 'WHERE candidates.id > ? ORDER BY candidates.id ASC LIMIT '.self::BATCH_SIZE, + [$lastId] + ); + + if ($candidates === []) { + break; + } + + $lastId = (int) end($candidates)->id; + $matches = collect(); + + if ($includeFiles) { + $fileNames = DB::table('release_files') + ->whereIn('releases_id', array_column($candidates, 'id')) + ->select(['releases_id', 'name']) + ->get() + ->groupBy('releases_id'); + + foreach ($candidates as $candidate) { + $candidate->file_names = $fileNames + ->get((int) $candidate->id, collect()) + ->pluck('name'); + } + } + + foreach ($candidates as $candidate) { + $method = $matcher($candidate); + if ($method === null) { + continue; + } + + $candidate->removal_method = $method; + $matches->push($candidate); + } + + foreach ($matches as $release) { + if ($this->echoCLI) { + cli()->primary( + ($this->delete ? 'Deleting: ' : 'Would be deleting: ').$release->removal_method.': '.$release->searchname, + true + ); + } + } + + if ($this->delete && $matches->isNotEmpty()) { + $this->releaseManagement->deleteBatch($matches, $this->nzb, $this->releaseImage); + } + + $this->deletedCount += $matches->count(); + + if (count($candidates) === self::BATCH_SIZE) { + usleep(self::BATCH_PAUSE_US); + } + } while (true); } /** @@ -934,20 +962,41 @@ class ReleaseRemoverService */ protected function deleteReleases(): bool { - $deletedCount = 0; - foreach ($this->result as $release) { - if ($this->delete) { - $this->releaseManagement->deleteSingleWithService(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage); - if ($this->echoCLI) { - cli()->primary('Deleting: '.$this->method.': '.$release->searchname, true); - } - } elseif ($this->echoCLI) { - cli()->primary('Would be deleting: '.$this->method.': '.$release->searchname, true); - } - $deletedCount++; - } + $lastId = 0; - $this->deletedCount += $deletedCount; + do { + $batch = DB::select( + 'SELECT candidates.id, candidates.guid, candidates.searchname FROM ('.$this->cleanSpaces($this->query).') candidates '. + 'WHERE candidates.id > ? ORDER BY candidates.id ASC LIMIT '.self::BATCH_SIZE, + [$lastId] + ); + + if ($batch === []) { + break; + } + + $batch = collect($batch)->unique('id')->values(); + $lastId = (int) $batch->max('id'); + + foreach ($batch as $release) { + if ($this->echoCLI) { + cli()->primary( + ($this->delete ? 'Deleting: ' : 'Would be deleting: ').$this->method.': '.$release->searchname, + true + ); + } + } + + if ($this->delete) { + $this->releaseManagement->deleteBatch($batch, $this->nzb, $this->releaseImage); + } + + $this->deletedCount += $batch->count(); + + if ($batch->count() === self::BATCH_SIZE) { + usleep(self::BATCH_PAUSE_US); + } + } while (true); return true; } @@ -959,7 +1008,9 @@ class ReleaseRemoverService */ protected function checkSelectQuery(): bool { - $result = DB::select($this->cleanSpaces($this->query)); + $result = DB::select( + 'SELECT 1 FROM ('.$this->cleanSpaces($this->query).') candidates LIMIT 1' + ); if (empty($result)) { $this->error = ''; if ($this->method === 'userCriteria') { @@ -968,7 +1019,6 @@ class ReleaseRemoverService return false; } - $this->result = $result; return true; } @@ -1199,71 +1249,4 @@ class ReleaseRemoverService return false; } - - /** - * Extract search terms from a regex pattern for fulltext search optimization. - * - * @return array - */ - protected function extractSrchFromRegx(string $dbRegex = ''): array|string - { - $patterns = [ - ['offset' => 2, 'length' => 17, 'match' => 'brazilian|chinese', 'search' => 'brazilian', 'useLastParen' => false], - ['offset' => 7, 'length' => 11, 'match' => 'bl|cz|de|es', 'search' => 'bl|cz', 'useLastParen' => false, 'wrapQuotes' => true], - ['offset' => 8, 'length' => 5, 'match' => '19|20', 'search' => 'bl|cz', 'useLastParen' => true, 'wrapQuotes' => true], - ['offset' => 7, 'length' => 14, 'match' => 'chinese.subbed', 'search' => 'chinese', 'useLastParen' => true, 'cleanChars' => true], - ['offset' => 8, 'length' => 2, 'match' => '4u', 'search' => '4u', 'useLastParen' => false, 'replace4u' => true], - ['offset' => 8, 'length' => 5, 'match' => 'bd|dl', 'search' => 'bd|dl', 'useLastParen' => true, 'replaceBdDl' => true], - ['offset' => 7, 'length' => 9, 'match' => 'imageset|', 'search' => 'imageset', 'useLastParen' => false], - ['offset' => 1, 'length' => 9, 'match' => 'hdnectar|', 'stripQuotes' => true], - ['offset' => 1, 'length' => 10, 'match' => 'Passworded', 'stripQuotes' => true], - ]; - - foreach ($patterns as $pattern) { - if (substr($dbRegex, $pattern['offset'], $pattern['length']) !== $pattern['match']) { - continue; - } - - // Handle simple quote stripping patterns - if (! empty($pattern['stripQuotes'])) { - return str_replace('\'', '', $dbRegex); - } - - $searchPos = strpos($dbRegex, $pattern['search']); - if ($searchPos === false) { - continue; - } - - $parenPos = ! empty($pattern['useLastParen']) - ? strrpos($dbRegex, ')') - : strpos($dbRegex, ')'); - - $extracted = substr($dbRegex, $searchPos, $parenPos - $searchPos); - - // Apply specific transformations - if (! empty($pattern['wrapQuotes'])) { - return '"'.str_replace('|', '" "', $extracted).'"'; - } - - if (! empty($pattern['cleanChars'])) { - return str_replace( - ['-', '(', ')', '.', '?', 'nl subed|bed|s'], - ['', '', '', ' ', '', 'nlsubs|nlsubbed|nlsubed'], - $extracted - ); - } - - if (! empty($pattern['replace4u'])) { - return str_replace(['4u.nl', 'nov[ a]*rip'], ['"4u" "nl"', 'nova'], $extracted); - } - - if (! empty($pattern['replaceBdDl'])) { - return str_replace(['bd|dl)mux', '\\', ']', '['], ['bdmux|dlmux', '', '', ''], $extracted); - } - - return $extracted; - } - - return ''; - } } diff --git a/app/Services/Releases/ReleaseManagementService.php b/app/Services/Releases/ReleaseManagementService.php index 0b3c19ae9..9199f6677 100644 --- a/app/Services/Releases/ReleaseManagementService.php +++ b/app/Services/Releases/ReleaseManagementService.php @@ -14,6 +14,7 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Log; +use Throwable; /** * Service for managing releases (delete, update, export). @@ -87,6 +88,62 @@ class ReleaseManagementService $this->deleteSingle($identifiers, $nzb, $releaseImage); } + /** + * Delete a bounded set of releases while batching search and database work. + * + * @param iterable> $releases + */ + public function deleteBatch(iterable $releases, NzbService $nzb, ReleaseImageService $releaseImage): int + { + $rows = collect($releases) + ->map(static fn (object|array $release): array => [ + 'id' => (int) data_get($release, 'id'), + 'guid' => (string) data_get($release, 'guid'), + ]) + ->filter(static fn (array $release): bool => $release['id'] > 0 && $release['guid'] !== '') + ->unique('id') + ->values(); + + if ($rows->isEmpty()) { + return 0; + } + + foreach ($rows as $release) { + try { + $nzb->deleteNzb($release['guid']); + $releaseImage->delete($release['guid']); + } catch (Throwable $e) { + Log::error('Release batch filesystem cleanup failed', [ + 'release_id' => $release['id'], + 'guid' => $release['guid'], + 'error' => $e->getMessage(), + ]); + } + } + + $ids = $rows->pluck('id')->all(); + + try { + Search::deleteReleases($ids); + } catch (Throwable $e) { + Log::error('Release batch search cleanup failed', [ + 'release_ids' => $ids, + 'error' => $e->getMessage(), + ]); + } + + try { + return Release::query()->whereIn('id', $ids)->delete(); + } catch (Throwable $e) { + Log::error('Release batch database cleanup failed', [ + 'release_ids' => $ids, + 'error' => $e->getMessage(), + ]); + + return 0; + } + } + /** * @return bool|int */ @@ -157,7 +214,7 @@ class ReleaseManagementService try { Search::updateRelease($intId); - } catch (\Throwable $e) { + } catch (Throwable $e) { Log::error('ReleaseManagementService: Failed to sync release to search index after category change', [ 'release_id' => $intId, 'error' => $e->getMessage(), diff --git a/app/Services/Search/Contracts/SearchServiceInterface.php b/app/Services/Search/Contracts/SearchServiceInterface.php index cb61fb08d..9b6a3a068 100644 --- a/app/Services/Search/Contracts/SearchServiceInterface.php +++ b/app/Services/Search/Contracts/SearchServiceInterface.php @@ -78,6 +78,13 @@ interface SearchServiceInterface */ public function deleteRelease(int $id): void; + /** + * Delete multiple releases from the index in one request. + * + * @param iterable $ids + */ + public function deleteReleases(iterable $ids): void; + /** * Insert a predb record into the search index. * diff --git a/app/Services/Search/Drivers/ElasticSearchDriver.php b/app/Services/Search/Drivers/ElasticSearchDriver.php index 217445860..e2548c37c 100644 --- a/app/Services/Search/Drivers/ElasticSearchDriver.php +++ b/app/Services/Search/Drivers/ElasticSearchDriver.php @@ -1256,8 +1256,18 @@ class ElasticSearchDriver implements SearchDriverInterface */ public function deleteRelease(int $id): void { - if (empty($id) || ! $this->isElasticsearchAvailable()) { - if (empty($id)) { + $this->deleteReleases([$id]); + } + + public function deleteReleases(iterable $ids): void + { + $ids = array_values(array_unique(array_filter( + array_map('intval', is_array($ids) ? $ids : iterator_to_array($ids)), + static fn (int $id): bool => $id > 0 + ))); + + if ($ids === [] || ! $this->isElasticsearchAvailable()) { + if ($ids === []) { Log::warning('ElasticSearch: Cannot delete release without ID'); } @@ -1266,30 +1276,31 @@ class ElasticSearchDriver implements SearchDriverInterface try { $client = $this->getClient(); - $client->delete([ - 'index' => $this->getReleasesIndex(), - 'id' => $id, - ]); + $body = []; + foreach ($ids as $id) { + $body[] = ['delete' => ['_index' => $this->getReleasesIndex(), '_id' => $id]]; + } + $client->bulk(['body' => $body]); } catch (\Throwable $e) { if (ElasticsearchResponseHelper::isNotFound($e)) { if (config('app.debug')) { - Log::debug('ElasticSearch deleteRelease: document not found', ['release_id' => $id]); + Log::debug('ElasticSearch deleteReleases: document not found', ['release_ids' => $ids]); } return; } if ($e instanceof ElasticsearchException) { - Log::error('ElasticSearch deleteRelease error: '.$e->getMessage(), [ - 'release_id' => $id, + Log::error('ElasticSearch deleteReleases error: '.$e->getMessage(), [ + 'release_ids' => $ids, ]); return; } - Log::error('ElasticSearch deleteRelease unexpected error: '.$e->getMessage(), [ - 'release_id' => $id, + Log::error('ElasticSearch deleteReleases unexpected error: '.$e->getMessage(), [ + 'release_ids' => $ids, ]); } } diff --git a/app/Services/Search/Drivers/ManticoreSearchDriver.php b/app/Services/Search/Drivers/ManticoreSearchDriver.php index e0c97c6cb..d20799655 100644 --- a/app/Services/Search/Drivers/ManticoreSearchDriver.php +++ b/app/Services/Search/Drivers/ManticoreSearchDriver.php @@ -345,18 +345,32 @@ class ManticoreSearchDriver implements SearchDriverInterface */ public function deleteRelease(int $id): void { - if (empty($id)) { + $this->deleteReleases([$id]); + } + + public function deleteReleases(iterable $ids): void + { + $ids = array_values(array_unique(array_filter( + array_map('intval', is_array($ids) ? $ids : iterator_to_array($ids)), + static fn (int $id): bool => $id > 0 + ))); + + if ($ids === []) { Log::warning('ManticoreSearch: Cannot delete release without ID'); return; } try { - $this->manticoreSearch->table($this->config['indexes']['releases']) - ->deleteDocument($id); + $index = str_replace('`', '``', $this->getReleasesIndex()); + $this->manticoreSearch->sql(sprintf( + 'DELETE FROM `%s` WHERE id IN (%s)', + $index, + implode(',', $ids) + )); } catch (ResponseException $e) { - Log::error('ManticoreSearch deleteRelease error: '.$e->getMessage(), [ - 'id' => $id, + Log::error('ManticoreSearch deleteReleases error: '.$e->getMessage(), [ + 'ids' => $ids, ]); } } diff --git a/app/Services/Search/SearchService.php b/app/Services/Search/SearchService.php index 49b926d4b..6d7d5dd9b 100644 --- a/app/Services/Search/SearchService.php +++ b/app/Services/Search/SearchService.php @@ -164,7 +164,12 @@ class SearchService extends Manager implements SearchServiceInterface */ public function deleteRelease(int $id): void { - $this->driver()->deleteRelease($id); + $this->deleteReleases([$id]); + } + + public function deleteReleases(iterable $ids): void + { + $this->driver()->deleteReleases($ids); } /** diff --git a/database/migrations/2026_07_15_000000_add_releases_adddate_id_index.php b/database/migrations/2026_07_15_000000_add_releases_adddate_id_index.php new file mode 100644 index 000000000..103e958a3 --- /dev/null +++ b/database/migrations/2026_07_15_000000_add_releases_adddate_id_index.php @@ -0,0 +1,45 @@ +indexExists()) { + return; + } + + Schema::table('releases', function (Blueprint $table): void { + $table->index(['adddate', 'id'], self::INDEX); + }); + } + + public function down(): void + { + if (! $this->indexExists()) { + return; + } + + Schema::table('releases', function (Blueprint $table): void { + $table->dropIndex(self::INDEX); + }); + } + + private function indexExists(): bool + { + if (DB::getDriverName() === 'sqlite') { + return collect(DB::select("PRAGMA index_list('releases')")) + ->contains(static fn (object $index): bool => $index->name === self::INDEX); + } + + return DB::select('SHOW INDEX FROM `releases` WHERE Key_name = ?', [self::INDEX]) !== []; + } +}; diff --git a/tests/Feature/ReleaseRemoverBatchingTest.php b/tests/Feature/ReleaseRemoverBatchingTest.php new file mode 100644 index 000000000..9f3a58911 --- /dev/null +++ b/tests/Feature/ReleaseRemoverBatchingTest.php @@ -0,0 +1,210 @@ +databasePath = sys_get_temp_dir().'/nntmux-release-remover-test.sqlite'; + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + + $pdo = new PDO('sqlite:'.$this->databasePath); + $pdo->exec('CREATE TABLE settings (name VARCHAR PRIMARY KEY, value TEXT NULL)'); + $pdo->exec("INSERT INTO settings (name, value) VALUES + ('categorizeforeign', '0'), ('catwebdl', '0'), ('innerfileblacklist', '')"); + + putenv('APP_ENV=testing'); + putenv('DB_CONNECTION=sqlite'); + putenv('DB_DATABASE='.$this->databasePath); + $_ENV['APP_ENV'] = $_SERVER['APP_ENV'] = 'testing'; + $_ENV['DB_CONNECTION'] = $_SERVER['DB_CONNECTION'] = 'sqlite'; + $_ENV['DB_DATABASE'] = $_SERVER['DB_DATABASE'] = $this->databasePath; + + $app = require __DIR__.'/../../bootstrap/app.php'; + $app->make(Kernel::class)->bootstrap(); + + return $app; + } + + protected function setUp(): void + { + parent::setUp(); + + config()->set('nntmux.echocli', false); + config()->set('database.default', 'sqlite'); + config()->set('database.connections.sqlite.database', $this->databasePath); + DB::purge(); + DB::reconnect(); + + Schema::dropIfExists('release_files'); + Schema::dropIfExists('releases'); + Schema::dropIfExists('binaryblacklist'); + Schema::dropIfExists('usenet_groups'); + + Schema::create('usenet_groups', function (Blueprint $table): void { + $table->increments('id'); + $table->string('name'); + }); + Schema::create('binaryblacklist', function (Blueprint $table): void { + $table->increments('id'); + $table->string('groupname'); + $table->text('regex'); + $table->unsignedTinyInteger('status'); + $table->unsignedTinyInteger('optype'); + $table->unsignedTinyInteger('msgcol'); + }); + Schema::create('releases', function (Blueprint $table): void { + $table->increments('id'); + $table->string('guid', 40); + $table->string('searchname'); + $table->string('fromname')->nullable(); + $table->unsignedInteger('groups_id'); + $table->dateTime('adddate')->nullable(); + }); + Schema::create('release_files', function (Blueprint $table): void { + $table->unsignedInteger('releases_id'); + $table->string('name'); + }); + } + + protected function tearDown(): void + { + parent::tearDown(); + + if (file_exists($this->databasePath)) { + unlink($this->databasePath); + } + } + + public function test_blacklist_removal_is_not_limited_to_one_hundred_search_results(): void + { + DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.binaries.test']); + DB::table('binaryblacklist')->insert([ + 'groupname' => 'alt.binaries.*', + 'regex' => '^blocked-', + 'status' => BlacklistConstants::BLACKLIST_ENABLED, + 'optype' => BlacklistConstants::OPTYPE_BLACKLIST, + 'msgcol' => BlacklistConstants::BLACKLIST_FIELD_SUBJECT, + ]); + + DB::table('releases')->insert(collect(range(1, 125))->map(static fn (int $id): array => [ + 'id' => $id, + 'guid' => str_pad((string) $id, 40, '0', STR_PAD_LEFT), + 'searchname' => 'blocked-'.$id, + 'fromname' => 'poster', + 'groups_id' => 1, + 'adddate' => now(), + ])->all()); + + $management = Mockery::mock(ReleaseManagementService::class); + $management->shouldReceive('deleteBatch') + ->once() + ->withArgs(static fn ($releases): bool => $releases->count() === 125) + ->andReturn(125); + + $service = new ReleaseRemoverService( + $management, + Mockery::mock(NzbService::class), + Mockery::mock(ReleaseImageService::class) + ); + + self::assertTrue($service->removeCrap(true, 'full', 'blacklist')); + } + + public function test_invalid_blacklist_regex_does_not_prevent_valid_rules_from_running(): void + { + DB::table('usenet_groups')->insert(['id' => 1, 'name' => 'alt.binaries.test']); + DB::table('binaryblacklist')->insert([ + [ + 'groupname' => 'alt.binaries.*', + 'regex' => '[invalid', + 'status' => 1, + 'optype' => 1, + 'msgcol' => 1, + ], + [ + 'groupname' => 'alt.binaries.*', + 'regex' => '^blocked$', + 'status' => 1, + 'optype' => 1, + 'msgcol' => 1, + ], + ]); + DB::table('releases')->insert([ + 'id' => 1, + 'guid' => str_repeat('a', 40), + 'searchname' => 'blocked', + 'fromname' => 'poster', + 'groups_id' => 1, + 'adddate' => now(), + ]); + + $management = Mockery::mock(ReleaseManagementService::class); + $management->shouldReceive('deleteBatch')->once()->andReturn(1); + + $service = new ReleaseRemoverService( + $management, + Mockery::mock(NzbService::class), + Mockery::mock(ReleaseImageService::class) + ); + + self::assertTrue($service->removeCrap(true, 'full', 'blacklist')); + } + + public function test_release_management_batches_search_and_database_deletion(): void + { + DB::table('releases')->insert([ + [ + 'id' => 1, + 'guid' => str_repeat('a', 40), + 'searchname' => 'one', + 'fromname' => 'poster', + 'groups_id' => 1, + 'adddate' => now(), + ], + [ + 'id' => 2, + 'guid' => str_repeat('b', 40), + 'searchname' => 'two', + 'fromname' => 'poster', + 'groups_id' => 1, + 'adddate' => now(), + ], + ]); + + $nzb = Mockery::mock(NzbService::class); + $nzb->shouldReceive('deleteNzb')->twice()->andReturnTrue(); + $images = Mockery::mock(ReleaseImageService::class); + $images->shouldReceive('delete')->twice(); + Search::shouldReceive('deleteReleases')->once()->with([1, 2]); + + $deleted = (new ReleaseManagementService)->deleteBatch([ + (object) ['id' => 1, 'guid' => str_repeat('a', 40)], + (object) ['id' => 2, 'guid' => str_repeat('b', 40)], + ], $nzb, $images); + + self::assertSame(2, $deleted); + self::assertSame(0, DB::table('releases')->count()); + } +}