mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Merge branch 'master' of https://github.com/NNTmux/newznab-tmux
This commit is contained in:
DariusIII
+602
-1304
File diff suppressed because it is too large
Load Diff
+70
-10
@@ -158,18 +158,78 @@ class NZBContents
|
||||
}
|
||||
|
||||
// --- NFO Detection ---
|
||||
// Check for explicit NFO files first
|
||||
if ($nfoCheck && ! $foundNFO && isset($firstSegmentId) && preg_match('/\.\b(nfo|diz|info?)\b(?![.-])/i', $subject)) {
|
||||
$nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId];
|
||||
$foundNFO = true; // Found an explicit NFO, prioritize this
|
||||
// Check for explicit NFO files first (with enhanced patterns)
|
||||
if ($nfoCheck && ! $foundNFO && isset($firstSegmentId)) {
|
||||
// Standard NFO extensions
|
||||
if (preg_match('/\.\b(nfo|diz|info?)\b(?![.-])/i', $subject)) {
|
||||
$nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId, 'priority' => 1];
|
||||
$foundNFO = true;
|
||||
}
|
||||
// Alternative NFO naming patterns (group-specific or obfuscated)
|
||||
elseif (preg_match('/(?:^|["\s])(?:file(?:_?id)?|readme|release|info(?:rmation)?|about|desc(?:ription)?|notes?|read\.?me|00-|000-|0-|_-_).*?\.(?:txt|nfo|diz)(?:["\s]|$)/i', $subject)) {
|
||||
$nfoMessageId = ['hidden' => false, 'id' => $firstSegmentId, 'priority' => 2];
|
||||
$foundNFO = true;
|
||||
}
|
||||
}
|
||||
// Check for potential "hidden" NFOs (single segment, common name, not other known types)
|
||||
|
||||
// Check for potential "hidden" NFOs with improved detection
|
||||
// Only consider this if an explicit NFO wasn't found yet
|
||||
elseif ($nfoCheck && ! $foundNFO && ! $hiddenNFO && isset($firstSegmentId) && $segmentCountInFile === 1 && preg_match('/\(1\/1\)$/i', $subject)) {
|
||||
// Simplified exclusion: check if it's NOT likely another common file type based on extension pattern
|
||||
if (! preg_match('/\.(?:exe|com|bat|cmd|scr|dll|zip|rar|[rst]\d{2}|[a-z0-9]{3}|7z|ace|tar|gz|bz2|iso|bin|cue|img|mdf|nrg|dmg|vhd|mp3|flac|ogg|aac|wav|wma|avi|mkv|mp4|mov|wmv|mpg|mpeg|ts|vob|jpg|jpeg|png|gif|bmp|tif|tiff|psd|pdf|doc|docx|xls|xlsx|ppt|pptx|txt|log|xml|html|css|js|php|py|java|c|cpp|h|cs|sql|db|dbf|mdb|accdb|par2?|sfv|md5|sha1|sha256|url|lnk|cfg|ini|inf|sys|tmp|bak|msi|pkg|deb|rpm|apk|ipa)\b/i', $subject)) {
|
||||
$nfoMessageId = ['hidden' => true, 'id' => $firstSegmentId];
|
||||
$hiddenNFO = true; // Found a potential hidden NFO
|
||||
if ($nfoCheck && ! $foundNFO && ! $hiddenNFO && isset($firstSegmentId)) {
|
||||
$isHiddenNfoCandidate = false;
|
||||
|
||||
// Pattern 1: Single segment files with (1/1)
|
||||
if ($segmentCountInFile === 1 && preg_match('/\(1\/1\)$/i', $subject)) {
|
||||
$isHiddenNfoCandidate = true;
|
||||
}
|
||||
|
||||
// Pattern 2: Small segment count (1-2) with NFO-like names but no extension
|
||||
if (! $isHiddenNfoCandidate && $segmentCountInFile <= 2 && preg_match('/(?:^|["\s])(?:nfo|info|readme|release|file_?id|about)(?:["\s]|$)/i', $subject)) {
|
||||
$isHiddenNfoCandidate = true;
|
||||
}
|
||||
|
||||
// Pattern 3: Scene-style NFO naming (group-release.nfo without extension visible)
|
||||
if (! $isHiddenNfoCandidate && $segmentCountInFile === 1 && preg_match('/^[a-z0-9._-]+["\s]*\(1\/1\)/i', $subject)) {
|
||||
// Check for scene-like naming pattern
|
||||
if (preg_match('/^[a-z0-9]+[._-][a-z0-9._-]+["\s]*\(1\/1\)/i', $subject)) {
|
||||
$isHiddenNfoCandidate = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern 4: Very small files (NFOs are typically small)
|
||||
// Files described as very small in bytes could be NFOs
|
||||
if (! $isHiddenNfoCandidate && $segmentCountInFile === 1 && preg_match('/yEnc\s*\(\d+\)\s*\[1\/1\]/i', $subject)) {
|
||||
$isHiddenNfoCandidate = true;
|
||||
}
|
||||
|
||||
if ($isHiddenNfoCandidate) {
|
||||
// Enhanced exclusion: check if it's NOT likely another common file type
|
||||
$excludedExtensions = '/\.(?:' .
|
||||
// Executables
|
||||
'exe|com|bat|cmd|scr|dll|msi|pkg|deb|rpm|apk|ipa|app|' .
|
||||
// Archives
|
||||
'zip|rar|[rst]\d{2}|7z|ace|tar|gz|bz2|xz|lzma|cab|iso|bin|cue|img|mdf|nrg|dmg|vhd|' .
|
||||
// Audio
|
||||
'mp3|flac|ogg|aac|wav|wma|m4a|opus|ape|wv|mpc|' .
|
||||
// Video
|
||||
'avi|mkv|mp4|mov|wmv|mpg|mpeg|ts|vob|m2ts|webm|flv|ogv|divx|xvid|' .
|
||||
// Images
|
||||
'jpg|jpeg|png|gif|bmp|tif|tiff|psd|webp|svg|ico|raw|cr2|nef|' .
|
||||
// Documents
|
||||
'pdf|doc|docx|xls|xlsx|ppt|pptx|odt|ods|odp|rtf|epub|mobi|azw|' .
|
||||
// Code
|
||||
'html|htm|css|js|php|py|java|c|cpp|h|cs|sql|json|xml|yml|yaml|' .
|
||||
// Data
|
||||
'db|dbf|mdb|accdb|sqlite|csv|' .
|
||||
// Verification
|
||||
'par2?|sfv|md5|sha1|sha256|sha512|crc|' .
|
||||
// Misc
|
||||
'url|lnk|cfg|ini|inf|sys|tmp|bak|log|srt|sub|idx|ass|ssa|vtt' .
|
||||
')\b/i';
|
||||
|
||||
if (! preg_match($excludedExtensions, $subject)) {
|
||||
$nfoMessageId = ['hidden' => true, 'id' => $firstSegmentId, 'priority' => 10];
|
||||
$hiddenNFO = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1143
-96
File diff suppressed because it is too large
Load Diff
+1044
-66
File diff suppressed because it is too large
Load Diff
@@ -626,12 +626,22 @@ class AdditionalProcessingOrchestrator
|
||||
continue;
|
||||
}
|
||||
|
||||
// NFO files
|
||||
if ($context->releaseHasNoNFO && preg_match('/(\.(nfo|inf|ofn)|info\.txt)$/i', $filePath)) {
|
||||
if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) {
|
||||
$this->output->echoNfoFound();
|
||||
// NFO files - enhanced detection with multiple patterns
|
||||
if ($context->releaseHasNoNFO) {
|
||||
// Standard NFO extensions
|
||||
if (preg_match('/(\.(nfo|inf|ofn|diz)|info\.txt)$/i', $filePath)) {
|
||||
if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) {
|
||||
$this->output->echoNfoFound();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Alternative NFO filenames (file_id.diz, readme.txt, etc.)
|
||||
elseif ($this->releaseManager->isNfoFilename($filePath)) {
|
||||
if ($this->releaseManager->processNfoFile($filePath, $context, $this->downloadService->getNNTP())) {
|
||||
$this->output->echoNfoFound();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Audio files
|
||||
|
||||
@@ -574,8 +574,82 @@ class ArchiveExtractionService
|
||||
*/
|
||||
private function getAllowedExtensions(): array
|
||||
{
|
||||
return ['nfo', 'srt', 'mkv', 'mpeg', 'avi', 'jpg', 'jpeg', 'exe', 'mp4', 'mp3', 'm4a',
|
||||
'flac', 'png', 'epub', 'cbz', 'cbr', 'djvu'];
|
||||
return [
|
||||
// NFO and info files (prioritized for extraction)
|
||||
'nfo', 'diz', 'inf', 'txt',
|
||||
// Subtitles
|
||||
'srt', 'sub', 'idx', 'ass', 'ssa', 'vtt',
|
||||
// Video
|
||||
'mkv', 'mpeg', 'avi', 'mp4', 'm4v', 'mov', 'wmv', 'flv', 'ts', 'vob', 'm2ts', 'webm',
|
||||
// Audio
|
||||
'mp3', 'm4a', 'flac', 'ogg', 'aac', 'wav', 'wma', 'opus', 'ape',
|
||||
// Images
|
||||
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp',
|
||||
// Documents
|
||||
'epub', 'pdf', 'cbz', 'cbr', 'djvu', 'mobi', 'azw', 'azw3',
|
||||
// Executables (for software releases)
|
||||
'exe', 'msi',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file is an NFO or info file.
|
||||
*
|
||||
* @param string $filename The filename to check.
|
||||
* @return bool True if it's an NFO-like file.
|
||||
*/
|
||||
public function isNfoFile(string $filename): bool
|
||||
{
|
||||
$basename = strtolower(basename($filename));
|
||||
|
||||
// Standard NFO extensions
|
||||
if (preg_match('/\.(nfo|diz|inf)$/i', $basename)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Common NFO alternative names
|
||||
$nfoNames = [
|
||||
'file_id.diz', 'fileid.diz', 'file-id.diz',
|
||||
'readme.txt', 'readme.1st', 'read.me', 'readmenow.txt',
|
||||
'info.txt', 'information.txt', 'about.txt', 'notes.txt',
|
||||
'release.txt', 'release.nfo',
|
||||
];
|
||||
|
||||
if (in_array($basename, $nfoNames, true)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Scene-style NFO naming: 00-groupname.nfo, group-release.nfo
|
||||
if (preg_match('/^(?:00?-[a-z0-9_-]+|[a-z0-9]+-[a-z0-9._-]+)\.(?:nfo|txt)$/i', $basename)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort files to prioritize NFO files for processing.
|
||||
*
|
||||
* @param array $files Array of file info arrays.
|
||||
* @return array Sorted array with NFO files first.
|
||||
*/
|
||||
public function sortFilesWithNfoPriority(array $files): array
|
||||
{
|
||||
usort($files, function ($a, $b) {
|
||||
$aIsNfo = $this->isNfoFile($a['name'] ?? '');
|
||||
$bIsNfo = $this->isNfoFile($b['name'] ?? '');
|
||||
|
||||
if ($aIsNfo && ! $bIsNfo) {
|
||||
return -1;
|
||||
}
|
||||
if (! $aIsNfo && $bIsNfo) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
return $files;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -345,7 +345,12 @@ class ReleaseFileManager
|
||||
}
|
||||
|
||||
/**
|
||||
* Process NFO file.
|
||||
* Process NFO file with enhanced detection capabilities.
|
||||
*
|
||||
* Supports multiple NFO naming conventions:
|
||||
* - Standard: .nfo, .diz, .info
|
||||
* - Alternative: file_id.diz, readme.txt, info.txt
|
||||
* - Scene-style: 00-groupname.nfo, groupname-releasename.nfo
|
||||
*/
|
||||
public function processNfoFile(
|
||||
string $fileLocation,
|
||||
@@ -354,6 +359,10 @@ class ReleaseFileManager
|
||||
): bool {
|
||||
try {
|
||||
$data = File::get($fileLocation);
|
||||
|
||||
// Try to detect and convert encoding
|
||||
$data = $this->normalizeNfoEncoding($data);
|
||||
|
||||
if ($this->nfo->isNFO($data, $context->release->guid)
|
||||
&& $this->nfo->addAlternateNfo($data, $context->release, $nntp)
|
||||
) {
|
||||
@@ -367,6 +376,84 @@ class ReleaseFileManager
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a filename looks like an NFO file.
|
||||
*
|
||||
* @param string $filename The filename to check.
|
||||
* @return bool True if the filename matches NFO patterns.
|
||||
*/
|
||||
public function isNfoFilename(string $filename): bool
|
||||
{
|
||||
// Standard NFO extensions
|
||||
if (preg_match('/\.(?:nfo|diz|info?)$/i', $filename)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Alternative NFO filenames
|
||||
$nfoPatterns = [
|
||||
'/^(?:file[_-]?id|readme|release|info(?:rmation)?|about|notes?)\.(?:txt|diz)$/i',
|
||||
'/^00-[a-z0-9_-]+\.nfo$/i', // Scene: 00-group.nfo
|
||||
'/^0+-[a-z0-9_-]+\.nfo$/i', // Scene variations
|
||||
'/^[a-z0-9_-]+-[a-z0-9_.-]+\.nfo$/i', // Scene: group-release.nfo
|
||||
'/info\.txt$/i', // info.txt (common alternative)
|
||||
];
|
||||
|
||||
$basename = basename($filename);
|
||||
foreach ($nfoPatterns as $pattern) {
|
||||
if (preg_match($pattern, $basename)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize NFO encoding to UTF-8.
|
||||
*
|
||||
* NFO files often use CP437 (DOS) encoding for ASCII art.
|
||||
* This method attempts to detect and convert various encodings.
|
||||
*
|
||||
* @param string $data Raw NFO data.
|
||||
* @return string UTF-8 encoded NFO data.
|
||||
*/
|
||||
protected function normalizeNfoEncoding(string $data): string
|
||||
{
|
||||
// Check for UTF-8 BOM and remove it
|
||||
if (str_starts_with($data, "\xEF\xBB\xBF")) {
|
||||
$data = substr($data, 3);
|
||||
}
|
||||
|
||||
// Check for UTF-16 BOM
|
||||
if (str_starts_with($data, "\xFF\xFE")) {
|
||||
// UTF-16 LE
|
||||
$data = mb_convert_encoding(substr($data, 2), 'UTF-8', 'UTF-16LE');
|
||||
} elseif (str_starts_with($data, "\xFE\xFF")) {
|
||||
// UTF-16 BE
|
||||
$data = mb_convert_encoding(substr($data, 2), 'UTF-8', 'UTF-16BE');
|
||||
}
|
||||
|
||||
// If already valid UTF-8, return as-is
|
||||
if (mb_check_encoding($data, 'UTF-8')) {
|
||||
return $data;
|
||||
}
|
||||
|
||||
// Try CP437 (DOS encoding - common for scene NFOs with ASCII art)
|
||||
// Use the utility function if available
|
||||
if (class_exists('\Blacklight\utility\Utility') && method_exists('\Blacklight\utility\Utility', 'cp437toUTF')) {
|
||||
return \Blacklight\utility\Utility::cp437toUTF($data);
|
||||
}
|
||||
|
||||
// Fallback: try ISO-8859-1 (Latin-1)
|
||||
$converted = @mb_convert_encoding($data, 'UTF-8', 'ISO-8859-1');
|
||||
if ($converted !== false) {
|
||||
return $converted;
|
||||
}
|
||||
|
||||
// Last resort: force UTF-8 with error handling
|
||||
return mb_convert_encoding($data, 'UTF-8', 'UTF-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle release name extraction from RAR file content.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use App\Models\Settings;
|
||||
|
||||
/**
|
||||
* Configuration DTO for Binaries processing.
|
||||
* Encapsulates all settings in an immutable object for easier testing and injection.
|
||||
*/
|
||||
final readonly class BinariesConfig
|
||||
{
|
||||
public function __construct(
|
||||
public int $messageBuffer = 20000,
|
||||
public bool $compressedHeaders = true,
|
||||
public bool $partRepair = true,
|
||||
public bool $newGroupScanByDays = false,
|
||||
public int $newGroupMessagesToScan = 50000,
|
||||
public int $newGroupDaysToScan = 3,
|
||||
public int $partRepairLimit = 15000,
|
||||
public int $partRepairMaxTries = 3,
|
||||
public int $partsChunkSize = 5000,
|
||||
public int $binariesUpdateChunkSize = 1000,
|
||||
public bool $echoCli = false,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Create configuration from application settings.
|
||||
*/
|
||||
public static function fromSettings(): self
|
||||
{
|
||||
return new self(
|
||||
messageBuffer: self::getSettingInt('maxmssgs', 20000),
|
||||
compressedHeaders: (bool) config('nntmux_nntp.compressed_headers'),
|
||||
partRepair: self::getSettingInt('partrepair', 1) === 1,
|
||||
newGroupScanByDays: self::getSettingInt('newgroupscanmethod', 0) === 1,
|
||||
newGroupMessagesToScan: self::getSettingInt('newgroupmsgstoscan', 50000),
|
||||
newGroupDaysToScan: self::getSettingInt('newgroupdaystoscan', 3),
|
||||
partRepairLimit: self::getSettingInt('maxpartrepair', 15000),
|
||||
partRepairMaxTries: self::getSettingInt('partrepairmaxtries', 3),
|
||||
partsChunkSize: max(100, (int) config('nntmux.parts_chunk_size', 5000)),
|
||||
binariesUpdateChunkSize: max(100, (int) config('nntmux.binaries_update_chunk_size', 1000)),
|
||||
echoCli: (bool) config('nntmux.echocli'),
|
||||
);
|
||||
}
|
||||
|
||||
private static function getSettingInt(string $key, int $default): int
|
||||
{
|
||||
$value = Settings::settingValue($key);
|
||||
|
||||
return $value !== '' ? (int) $value : $default;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Handles binary record creation and updates during header storage.
|
||||
*/
|
||||
final class BinaryHandler
|
||||
{
|
||||
/** @var array<int, array{Size: int, Parts: int}> Pending binary updates */
|
||||
private array $binariesUpdate = [];
|
||||
|
||||
/** @var array<int, true> IDs of binaries created in this batch */
|
||||
private array $insertedBinaryIds = [];
|
||||
|
||||
/** @var array<string, array{CollectionID: int, BinaryID: int}> Processed articles */
|
||||
private array $articles = [];
|
||||
|
||||
public function __construct() {}
|
||||
|
||||
/**
|
||||
* Reset state for a new batch.
|
||||
*/
|
||||
public function reset(): void
|
||||
{
|
||||
$this->binariesUpdate = [];
|
||||
$this->insertedBinaryIds = [];
|
||||
$this->articles = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a binary for the given header.
|
||||
*
|
||||
* @return int|null Binary ID or null on failure
|
||||
*/
|
||||
public function getOrCreateBinary(
|
||||
array $header,
|
||||
int $collectionId,
|
||||
int $groupId,
|
||||
int $fileNumber
|
||||
): ?int {
|
||||
$articleKey = $header['matches'][1];
|
||||
|
||||
// Return cached if already processed
|
||||
if (isset($this->articles[$articleKey])) {
|
||||
$binaryId = $this->articles[$articleKey]['BinaryID'];
|
||||
$this->binariesUpdate[$binaryId]['Size'] += $header['Bytes'];
|
||||
$this->binariesUpdate[$binaryId]['Parts']++;
|
||||
|
||||
return $binaryId;
|
||||
}
|
||||
|
||||
$hash = md5($header['matches'][1].$header['From'].$groupId);
|
||||
$driver = DB::getDriverName();
|
||||
|
||||
try {
|
||||
$binaryId = $this->insertOrGetBinary(
|
||||
$driver,
|
||||
$hash,
|
||||
$header,
|
||||
$collectionId,
|
||||
$fileNumber
|
||||
);
|
||||
|
||||
if ($binaryId > 0) {
|
||||
$this->binariesUpdate[$binaryId] = ['Size' => 0, 'Parts' => 0];
|
||||
$this->articles[$articleKey] = [
|
||||
'CollectionID' => $collectionId,
|
||||
'BinaryID' => $binaryId,
|
||||
];
|
||||
|
||||
return $binaryId;
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error('Binary insert failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function insertOrGetBinary(
|
||||
string $driver,
|
||||
string $hash,
|
||||
array $header,
|
||||
int $collectionId,
|
||||
int $fileNumber
|
||||
): int {
|
||||
$name = mb_convert_encoding($header['matches'][1], 'UTF-8', mb_list_encodings());
|
||||
$totalParts = (int) $header['matches'][3];
|
||||
$partSize = (int) $header['Bytes'];
|
||||
|
||||
if ($driver === 'sqlite') {
|
||||
return $this->insertBinarySqlite($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize);
|
||||
}
|
||||
|
||||
return $this->insertBinaryMysql($hash, $name, $collectionId, $totalParts, $fileNumber, $partSize);
|
||||
}
|
||||
|
||||
private function insertBinarySqlite(
|
||||
string $hash,
|
||||
string $name,
|
||||
int $collectionId,
|
||||
int $totalParts,
|
||||
int $fileNumber,
|
||||
int $partSize
|
||||
): int {
|
||||
DB::statement(
|
||||
'INSERT OR IGNORE INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) VALUES (?, ?, ?, ?, 1, ?, ?)',
|
||||
[$hash, $name, $collectionId, $totalParts, $fileNumber, $partSize]
|
||||
);
|
||||
|
||||
$lastId = (int) DB::connection()->getPdo()->lastInsertId();
|
||||
if ($lastId > 0) {
|
||||
$this->insertedBinaryIds[$lastId] = true;
|
||||
|
||||
return $lastId;
|
||||
}
|
||||
|
||||
$bin = DB::selectOne(
|
||||
'SELECT id FROM binaries WHERE binaryhash = ? AND collections_id = ? LIMIT 1',
|
||||
[$hash, $collectionId]
|
||||
);
|
||||
|
||||
return (int) ($bin->id ?? 0);
|
||||
}
|
||||
|
||||
private function insertBinaryMysql(
|
||||
string $hash,
|
||||
string $name,
|
||||
int $collectionId,
|
||||
int $totalParts,
|
||||
int $fileNumber,
|
||||
int $partSize
|
||||
): int {
|
||||
$sql = 'INSERT INTO binaries '
|
||||
.'(binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) '
|
||||
.'VALUES (UNHEX(?), ?, ?, ?, 1, ?, ?) '
|
||||
.'ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + VALUES(partsize)';
|
||||
|
||||
DB::statement($sql, [$hash, $name, $collectionId, $totalParts, $fileNumber, $partSize]);
|
||||
|
||||
$lastId = (int) DB::connection()->getPdo()->lastInsertId();
|
||||
if ($lastId > 0) {
|
||||
$this->insertedBinaryIds[$lastId] = true;
|
||||
|
||||
return $lastId;
|
||||
}
|
||||
|
||||
$bin = DB::selectOne(
|
||||
'SELECT id FROM binaries WHERE binaryhash = UNHEX(?) AND collections_id = ? LIMIT 1',
|
||||
[$hash, $collectionId]
|
||||
);
|
||||
|
||||
return (int) ($bin->id ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush accumulated size/parts updates to the database.
|
||||
*/
|
||||
public function flushUpdates(int $chunkSize = 1000): bool
|
||||
{
|
||||
$updates = $this->getPendingUpdates();
|
||||
if (empty($updates)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$driver = DB::getDriverName();
|
||||
|
||||
try {
|
||||
if ($driver === 'sqlite') {
|
||||
return $this->flushUpdatesSqlite($updates);
|
||||
}
|
||||
|
||||
return $this->flushUpdatesMysql($updates, $chunkSize);
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error('Binaries aggregate update failed: '.$e->getMessage());
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private function flushUpdatesSqlite(array $updates): bool
|
||||
{
|
||||
foreach ($updates as $row) {
|
||||
DB::statement(
|
||||
'UPDATE binaries SET partsize = partsize + ?, currentparts = currentparts + ? WHERE id = ?',
|
||||
[$row['partsize'], $row['currentparts'], $row['id']]
|
||||
);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function flushUpdatesMysql(array $updates, int $chunkSize): bool
|
||||
{
|
||||
foreach (array_chunk($updates, $chunkSize) as $chunk) {
|
||||
$placeholders = [];
|
||||
$bindings = [];
|
||||
|
||||
foreach ($chunk as $row) {
|
||||
$placeholders[] = '(?,?,?)';
|
||||
$bindings[] = $row['id'];
|
||||
$bindings[] = $row['partsize'];
|
||||
$bindings[] = $row['currentparts'];
|
||||
}
|
||||
|
||||
$sql = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '.implode(',', $placeholders)
|
||||
.' ON DUPLICATE KEY UPDATE partsize = partsize + VALUES(partsize), currentparts = currentparts + VALUES(currentparts)';
|
||||
|
||||
DB::statement($sql, $bindings);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if article is already processed.
|
||||
*/
|
||||
public function hasArticle(string $articleKey): bool
|
||||
{
|
||||
return isset($this->articles[$articleKey]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get IDs created in this batch.
|
||||
*/
|
||||
public function getInsertedIds(): array
|
||||
{
|
||||
return array_keys($this->insertedBinaryIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pending binary updates that haven't been flushed.
|
||||
*/
|
||||
private function getPendingUpdates(): array
|
||||
{
|
||||
$rows = [];
|
||||
foreach ($this->binariesUpdate as $binaryId => $binary) {
|
||||
if (($binary['Size'] ?? 0) > 0 || ($binary['Parts'] ?? 0) > 0) {
|
||||
$rows[] = [
|
||||
'id' => $binaryId,
|
||||
'partsize' => $binary['Size'],
|
||||
'currentparts' => $binary['Parts'],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $rows;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use App\Models\Collection;
|
||||
use App\Services\XrefService;
|
||||
use Blacklight\CollectionsCleaning;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Handles collection record creation and retrieval during header storage.
|
||||
*/
|
||||
final class CollectionHandler
|
||||
{
|
||||
private CollectionsCleaning $collectionsCleaning;
|
||||
|
||||
private XrefService $xrefService;
|
||||
|
||||
/** @var array<string, int> Cached collection IDs by key */
|
||||
private array $collectionIds = [];
|
||||
|
||||
/** @var array<int, true> IDs of collections created in this batch */
|
||||
private array $insertedCollectionIds = [];
|
||||
|
||||
/** @var array<string, true> Collection hashes touched in this batch */
|
||||
private array $batchCollectionHashes = [];
|
||||
|
||||
public function __construct(
|
||||
?CollectionsCleaning $collectionsCleaning = null,
|
||||
?XrefService $xrefService = null
|
||||
) {
|
||||
$this->collectionsCleaning = $collectionsCleaning ?? new CollectionsCleaning;
|
||||
$this->xrefService = $xrefService ?? new XrefService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset state for a new batch.
|
||||
*/
|
||||
public function reset(): void
|
||||
{
|
||||
$this->collectionIds = [];
|
||||
$this->insertedCollectionIds = [];
|
||||
$this->batchCollectionHashes = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a collection for the given header.
|
||||
*
|
||||
* @return int|null Collection ID or null on failure
|
||||
*/
|
||||
public function getOrCreateCollection(
|
||||
array $header,
|
||||
int $groupId,
|
||||
string $groupName,
|
||||
int $totalFiles,
|
||||
string $batchNoise
|
||||
): ?int {
|
||||
$collMatch = $this->collectionsCleaning->collectionsCleaner(
|
||||
$header['matches'][1],
|
||||
$groupName
|
||||
);
|
||||
|
||||
$collectionKey = $collMatch['name'].$totalFiles;
|
||||
|
||||
// Return cached ID if already processed this batch
|
||||
if (isset($this->collectionIds[$collectionKey])) {
|
||||
return $this->collectionIds[$collectionKey];
|
||||
}
|
||||
|
||||
$collectionHash = sha1($collectionKey);
|
||||
$this->batchCollectionHashes[$collectionHash] = true;
|
||||
|
||||
$headerDate = is_numeric($header['Date']) ? (int) $header['Date'] : strtotime($header['Date']);
|
||||
$now = now()->timestamp;
|
||||
$unixtime = min($headerDate, $now) ?: $now;
|
||||
|
||||
$existingXref = Collection::whereCollectionhash($collectionHash)->value('xref');
|
||||
$headerTokens = $this->xrefService->extractTokens($header['Xref'] ?? '');
|
||||
$newTokens = $this->xrefService->diffNewTokens($existingXref, $header['Xref'] ?? '');
|
||||
$finalXrefAppend = implode(' ', $newTokens);
|
||||
|
||||
$subject = substr(mb_convert_encoding($header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255);
|
||||
$fromName = mb_convert_encoding($header['From'], 'UTF-8', mb_list_encodings());
|
||||
|
||||
$driver = DB::getDriverName();
|
||||
|
||||
try {
|
||||
$collectionId = $this->insertOrGetCollection(
|
||||
$driver,
|
||||
$subject,
|
||||
$fromName,
|
||||
$unixtime,
|
||||
$headerTokens,
|
||||
$finalXrefAppend,
|
||||
$groupId,
|
||||
$totalFiles,
|
||||
$collectionHash,
|
||||
$collMatch['id'],
|
||||
$batchNoise
|
||||
);
|
||||
|
||||
if ($collectionId > 0) {
|
||||
$this->collectionIds[$collectionKey] = $collectionId;
|
||||
|
||||
return $collectionId;
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error('Collection insert failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function insertOrGetCollection(
|
||||
string $driver,
|
||||
string $subject,
|
||||
string $fromName,
|
||||
int $unixtime,
|
||||
array $headerTokens,
|
||||
string $finalXrefAppend,
|
||||
int $groupId,
|
||||
int $totalFiles,
|
||||
string $collectionHash,
|
||||
int $regexId,
|
||||
string $batchNoise
|
||||
): int {
|
||||
if ($driver === 'sqlite') {
|
||||
return $this->insertCollectionSqlite(
|
||||
$subject,
|
||||
$fromName,
|
||||
$unixtime,
|
||||
$headerTokens,
|
||||
$groupId,
|
||||
$totalFiles,
|
||||
$collectionHash,
|
||||
$regexId,
|
||||
$batchNoise
|
||||
);
|
||||
}
|
||||
|
||||
return $this->insertCollectionMysql(
|
||||
$subject,
|
||||
$fromName,
|
||||
$unixtime,
|
||||
$headerTokens,
|
||||
$finalXrefAppend,
|
||||
$groupId,
|
||||
$totalFiles,
|
||||
$collectionHash,
|
||||
$regexId,
|
||||
$batchNoise
|
||||
);
|
||||
}
|
||||
|
||||
private function insertCollectionSqlite(
|
||||
string $subject,
|
||||
string $fromName,
|
||||
int $unixtime,
|
||||
array $headerTokens,
|
||||
int $groupId,
|
||||
int $totalFiles,
|
||||
string $collectionHash,
|
||||
int $regexId,
|
||||
string $batchNoise
|
||||
): int {
|
||||
DB::statement(
|
||||
'INSERT OR IGNORE INTO collections (subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) VALUES (?, ?, datetime(?, "unixepoch"), ?, ?, ?, ?, datetime("now"), ?)',
|
||||
[
|
||||
$subject,
|
||||
$fromName,
|
||||
$unixtime,
|
||||
implode(' ', $headerTokens),
|
||||
$groupId,
|
||||
$totalFiles,
|
||||
$collectionHash,
|
||||
$regexId,
|
||||
$batchNoise,
|
||||
]
|
||||
);
|
||||
|
||||
$lastId = (int) DB::connection()->getPdo()->lastInsertId();
|
||||
if ($lastId > 0) {
|
||||
$this->insertedCollectionIds[$lastId] = true;
|
||||
|
||||
return $lastId;
|
||||
}
|
||||
|
||||
return (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0);
|
||||
}
|
||||
|
||||
private function insertCollectionMysql(
|
||||
string $subject,
|
||||
string $fromName,
|
||||
int $unixtime,
|
||||
array $headerTokens,
|
||||
string $finalXrefAppend,
|
||||
int $groupId,
|
||||
int $totalFiles,
|
||||
string $collectionHash,
|
||||
int $regexId,
|
||||
string $batchNoise
|
||||
): int {
|
||||
$insertSql = 'INSERT INTO collections '
|
||||
.'(subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded, noise) '
|
||||
.'VALUES (?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, NOW(), ?) '
|
||||
.'ON DUPLICATE KEY UPDATE dateadded = NOW()';
|
||||
|
||||
$bindings = [
|
||||
$subject,
|
||||
$fromName,
|
||||
$unixtime,
|
||||
implode(' ', $headerTokens),
|
||||
$groupId,
|
||||
$totalFiles,
|
||||
$collectionHash,
|
||||
$regexId,
|
||||
$batchNoise,
|
||||
];
|
||||
|
||||
if ($finalXrefAppend !== '') {
|
||||
$insertSql .= ', xref = CONCAT(xref, "\\n", ?)';
|
||||
$bindings[] = $finalXrefAppend;
|
||||
}
|
||||
|
||||
DB::statement($insertSql, $bindings);
|
||||
|
||||
$lastId = (int) DB::connection()->getPdo()->lastInsertId();
|
||||
if ($lastId > 0) {
|
||||
$this->insertedCollectionIds[$lastId] = true;
|
||||
|
||||
return $lastId;
|
||||
}
|
||||
|
||||
return (int) (Collection::whereCollectionhash($collectionHash)->value('id') ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get IDs created in this batch.
|
||||
*/
|
||||
public function getInsertedIds(): array
|
||||
{
|
||||
return array_keys($this->insertedCollectionIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all collection IDs processed this batch.
|
||||
*/
|
||||
public function getAllIds(): array
|
||||
{
|
||||
return array_values(array_unique(array_map('intval', $this->collectionIds)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all collection hashes processed this batch.
|
||||
*/
|
||||
public function getBatchHashes(): array
|
||||
{
|
||||
return array_keys($this->batchCollectionHashes);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use App\Services\BlacklistService;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Parses and filters raw NNTP headers.
|
||||
*/
|
||||
final class HeaderParser
|
||||
{
|
||||
private BlacklistService $blacklistService;
|
||||
|
||||
private int $notYEnc = 0;
|
||||
|
||||
private int $blacklisted = 0;
|
||||
|
||||
public function __construct(?BlacklistService $blacklistService = null)
|
||||
{
|
||||
$this->blacklistService = $blacklistService ?? new BlacklistService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset counters for a new batch.
|
||||
*/
|
||||
public function reset(): void
|
||||
{
|
||||
$this->notYEnc = 0;
|
||||
$this->blacklisted = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and filter raw headers from NNTP.
|
||||
*
|
||||
* @param array $headers Raw headers from NNTP
|
||||
* @param string $groupName The newsgroup name
|
||||
* @param bool $partRepair Whether this is a part repair scan
|
||||
* @param array|null $missingParts Missing part numbers if part repair
|
||||
* @return array Filtered and parsed headers with article info
|
||||
*/
|
||||
public function parse(
|
||||
array $headers,
|
||||
string $groupName,
|
||||
bool $partRepair = false,
|
||||
?array $missingParts = null
|
||||
): array {
|
||||
$parsed = [];
|
||||
$headersRepaired = [];
|
||||
|
||||
foreach ($headers as $header) {
|
||||
// Check if we got the article
|
||||
if (! isset($header['Number'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// For part repair, only process missing parts
|
||||
if ($partRepair && $missingParts !== null) {
|
||||
if (! \in_array($header['Number'], $missingParts, false)) {
|
||||
continue;
|
||||
}
|
||||
$headersRepaired[] = $header['Number'];
|
||||
}
|
||||
|
||||
// Parse subject to get base name and part/total like "(12/45)"
|
||||
if (! preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) {
|
||||
$this->notYEnc++;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Normalize to include yEnc if missing
|
||||
if (stripos($header['Subject'], 'yEnc') === false) {
|
||||
$matches[1] .= ' yEnc';
|
||||
}
|
||||
|
||||
$header['matches'] = $matches;
|
||||
|
||||
// Filter subject based on black/white list
|
||||
if ($this->blacklistService->isBlackListed($header, $groupName)) {
|
||||
$this->blacklisted++;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ensure Bytes is set
|
||||
if (empty($header['Bytes'])) {
|
||||
$header['Bytes'] = $header[':bytes'] ?? 0;
|
||||
}
|
||||
|
||||
$parsed[] = [
|
||||
'header' => $header,
|
||||
'repaired' => $partRepair,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'headers' => array_column($parsed, 'header'),
|
||||
'repaired' => $headersRepaired,
|
||||
'notYEnc' => $this->notYEnc,
|
||||
'blacklisted' => $this->blacklisted,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Update blacklist last_activity for matched rules.
|
||||
*/
|
||||
public function flushBlacklistUpdates(): void
|
||||
{
|
||||
$ids = $this->blacklistService->getAndClearIdsToUpdate();
|
||||
if (! empty($ids)) {
|
||||
$this->blacklistService->updateBlacklistUsage($ids);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of non-yEnc headers filtered.
|
||||
*/
|
||||
public function getNotYEncCount(): int
|
||||
{
|
||||
return $this->notYEnc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of blacklisted headers.
|
||||
*/
|
||||
public function getBlacklistedCount(): int
|
||||
{
|
||||
return $this->blacklisted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract highest and lowest article info from headers.
|
||||
*/
|
||||
public function getArticleRange(array $headers): array
|
||||
{
|
||||
$result = [];
|
||||
$count = \count($headers);
|
||||
|
||||
if ($count === 0) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
// Find first valid article
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
if (isset($headers[$i]['Number'])) {
|
||||
$result['firstArticleNumber'] = $headers[$i]['Number'];
|
||||
$result['firstArticleDate'] = $headers[$i]['Date'] ?? null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Find last valid article
|
||||
for ($i = $count - 1; $i >= 0; $i--) {
|
||||
if (isset($headers[$i]['Number'])) {
|
||||
$result['lastArticleNumber'] = $headers[$i]['Number'];
|
||||
$result['lastArticleDate'] = $headers[$i]['Date'] ?? null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Orchestrates the header storage process.
|
||||
*
|
||||
* This service coordinates the CollectionHandler, BinaryHandler, PartHandler,
|
||||
* and HeaderStorageTransaction to store parsed headers into the database.
|
||||
*/
|
||||
final class HeaderStorageService
|
||||
{
|
||||
private CollectionHandler $collectionHandler;
|
||||
|
||||
private BinaryHandler $binaryHandler;
|
||||
|
||||
private PartHandler $partHandler;
|
||||
|
||||
private BinariesConfig $config;
|
||||
|
||||
/** @var array<int> Article numbers that failed to insert */
|
||||
private array $failedInserts = [];
|
||||
|
||||
public function __construct(
|
||||
?CollectionHandler $collectionHandler = null,
|
||||
?BinaryHandler $binaryHandler = null,
|
||||
?PartHandler $partHandler = null,
|
||||
?BinariesConfig $config = null
|
||||
) {
|
||||
$this->config = $config ?? BinariesConfig::fromSettings();
|
||||
$this->collectionHandler = $collectionHandler ?? new CollectionHandler;
|
||||
$this->binaryHandler = $binaryHandler ?? new BinaryHandler;
|
||||
$this->partHandler = $partHandler ?? new PartHandler(
|
||||
$this->config->partsChunkSize,
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store parsed headers to the database.
|
||||
*
|
||||
* @param array $headers Parsed headers with 'matches' already populated
|
||||
* @param array $groupMySQL Group info from database
|
||||
* @param bool $addToPartRepair Whether to track failed inserts
|
||||
* @return array Article numbers that failed to insert
|
||||
*/
|
||||
public function store(array $headers, array $groupMySQL, bool $addToPartRepair = true): array
|
||||
{
|
||||
if (empty($headers)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Reset all handlers
|
||||
$this->collectionHandler->reset();
|
||||
$this->binaryHandler->reset();
|
||||
$this->partHandler->reset();
|
||||
$this->partHandler->setAddToPartRepair($addToPartRepair);
|
||||
$this->failedInserts = [];
|
||||
|
||||
// Create transaction
|
||||
$transaction = new HeaderStorageTransaction(
|
||||
$this->collectionHandler,
|
||||
$this->binaryHandler,
|
||||
$this->partHandler
|
||||
);
|
||||
|
||||
$transaction->begin();
|
||||
|
||||
// Process each header
|
||||
foreach ($headers as $header) {
|
||||
if (! $this->processHeader($header, $groupMySQL, $transaction)) {
|
||||
if ($addToPartRepair && isset($header['Number'])) {
|
||||
$this->failedInserts[] = $header['Number'];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining parts
|
||||
if ($this->partHandler->hasPending()) {
|
||||
if (! $this->partHandler->flush()) {
|
||||
$transaction->markError();
|
||||
}
|
||||
}
|
||||
|
||||
// Flush binary aggregate updates
|
||||
if (! $transaction->hasErrors()) {
|
||||
if (! $this->binaryHandler->flushUpdates($this->config->binariesUpdateChunkSize)) {
|
||||
$transaction->markError();
|
||||
}
|
||||
}
|
||||
|
||||
// Finish transaction
|
||||
if (! $transaction->finish()) {
|
||||
// All failed
|
||||
if ($addToPartRepair) {
|
||||
return array_unique(array_merge(
|
||||
$this->failedInserts,
|
||||
$this->partHandler->getFailedNumbers()
|
||||
));
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
return array_unique(array_merge(
|
||||
$this->failedInserts,
|
||||
$this->partHandler->getFailedNumbers()
|
||||
));
|
||||
}
|
||||
|
||||
private function processHeader(array $header, array $groupMySQL, HeaderStorageTransaction $transaction): bool
|
||||
{
|
||||
// Get file count from subject
|
||||
$fileCount = $this->getFileCount($header['matches'][1]);
|
||||
if ($fileCount[1] === 0 && $fileCount[3] === 0) {
|
||||
$fileCount = $this->getFileCount($header['matches'][0]);
|
||||
}
|
||||
|
||||
$totalFiles = (int) $fileCount[3];
|
||||
$fileNumber = (int) $fileCount[1];
|
||||
|
||||
// Get or create collection
|
||||
$collectionId = $this->collectionHandler->getOrCreateCollection(
|
||||
$header,
|
||||
$groupMySQL['id'],
|
||||
$groupMySQL['name'],
|
||||
$totalFiles,
|
||||
$transaction->getBatchNoise()
|
||||
);
|
||||
|
||||
if ($collectionId === null) {
|
||||
$transaction->markError();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get or create binary
|
||||
$binaryId = $this->binaryHandler->getOrCreateBinary(
|
||||
$header,
|
||||
$collectionId,
|
||||
$groupMySQL['id'],
|
||||
$fileNumber
|
||||
);
|
||||
|
||||
if ($binaryId === null) {
|
||||
$transaction->markError();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Add part
|
||||
if (! $this->partHandler->addPart($binaryId, $header)) {
|
||||
$transaction->markError();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function getFileCount(string $subject): array
|
||||
{
|
||||
if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])[\s$:]/i', $subject, $fileCount)) {
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
}
|
||||
|
||||
return $fileCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Handles header storage transactions and rollback cleanup.
|
||||
*/
|
||||
final class HeaderStorageTransaction
|
||||
{
|
||||
private CollectionHandler $collectionHandler;
|
||||
|
||||
private BinaryHandler $binaryHandler;
|
||||
|
||||
private PartHandler $partHandler;
|
||||
|
||||
private string $batchNoise;
|
||||
|
||||
private bool $hadErrors = false;
|
||||
|
||||
public function __construct(
|
||||
CollectionHandler $collectionHandler,
|
||||
BinaryHandler $binaryHandler,
|
||||
PartHandler $partHandler
|
||||
) {
|
||||
$this->collectionHandler = $collectionHandler;
|
||||
$this->binaryHandler = $binaryHandler;
|
||||
$this->partHandler = $partHandler;
|
||||
$this->batchNoise = bin2hex(random_bytes(8));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the batch noise marker for this transaction.
|
||||
*/
|
||||
public function getBatchNoise(): string
|
||||
{
|
||||
return $this->batchNoise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new database transaction.
|
||||
*/
|
||||
public function begin(): void
|
||||
{
|
||||
DB::beginTransaction();
|
||||
$this->hadErrors = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark that an error occurred.
|
||||
*/
|
||||
public function markError(): void
|
||||
{
|
||||
$this->hadErrors = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if errors occurred.
|
||||
*/
|
||||
public function hasErrors(): bool
|
||||
{
|
||||
return $this->hadErrors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit the transaction if no errors, rollback otherwise.
|
||||
*/
|
||||
public function finish(): bool
|
||||
{
|
||||
if ($this->hadErrors) {
|
||||
$this->rollbackAndCleanup();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
DB::commit();
|
||||
|
||||
return true;
|
||||
} catch (\Throwable $e) {
|
||||
$this->rollbackAndCleanup();
|
||||
|
||||
if (config('app.debug') === true) {
|
||||
Log::error('HeaderStorageTransaction commit failed: '.$e->getMessage());
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform rollback and cleanup any orphaned data.
|
||||
*/
|
||||
private function rollbackAndCleanup(): void
|
||||
{
|
||||
try {
|
||||
DB::rollBack();
|
||||
} catch (\Throwable $e) {
|
||||
// Already rolled back
|
||||
}
|
||||
|
||||
$this->cleanup();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup rows that may have been inserted before rollback.
|
||||
*/
|
||||
private function cleanup(): void
|
||||
{
|
||||
try {
|
||||
$this->cleanupParts();
|
||||
$this->cleanupBinaries();
|
||||
$this->cleanupCollections();
|
||||
|
||||
// Final guard for sqlite
|
||||
if (DB::getDriverName() === 'sqlite') {
|
||||
DB::statement('DELETE FROM parts');
|
||||
DB::statement('DELETE FROM binaries');
|
||||
DB::statement('DELETE FROM collections');
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::warning('Post-rollback cleanup failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanupParts(): void
|
||||
{
|
||||
$numbers = $this->partHandler->getInsertedNumbers();
|
||||
if (! empty($numbers)) {
|
||||
$placeholders = implode(',', array_fill(0, \count($numbers), '?'));
|
||||
DB::statement("DELETE FROM parts WHERE number IN ({$placeholders})", $numbers);
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanupBinaries(): void
|
||||
{
|
||||
$ids = $this->binaryHandler->getInsertedIds();
|
||||
if (! empty($ids)) {
|
||||
$placeholders = implode(',', array_fill(0, \count($ids), '?'));
|
||||
DB::statement("DELETE FROM binaries WHERE id IN ({$placeholders})", $ids);
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanupCollections(): void
|
||||
{
|
||||
$insertedIds = $this->collectionHandler->getInsertedIds();
|
||||
$allIds = $this->collectionHandler->getAllIds();
|
||||
$hashes = $this->collectionHandler->getBatchHashes();
|
||||
|
||||
$ids = ! empty($insertedIds) ? $insertedIds : $allIds;
|
||||
|
||||
if (! empty($ids)) {
|
||||
$placeholders = implode(',', array_fill(0, \count($ids), '?'));
|
||||
|
||||
// Remove parts and binaries referencing these collections, then collections
|
||||
DB::statement(
|
||||
"DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN ({$placeholders}))",
|
||||
$ids
|
||||
);
|
||||
DB::statement("DELETE FROM binaries WHERE collections_id IN ({$placeholders})", $ids);
|
||||
DB::statement("DELETE FROM collections WHERE id IN ({$placeholders})", $ids);
|
||||
} elseif (! empty($hashes)) {
|
||||
$placeholders = implode(',', array_fill(0, \count($hashes), '?'));
|
||||
|
||||
DB::statement(
|
||||
"DELETE FROM parts WHERE binaries_id IN (SELECT id FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ({$placeholders})))",
|
||||
$hashes
|
||||
);
|
||||
DB::statement(
|
||||
"DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE collectionhash IN ({$placeholders}))",
|
||||
$hashes
|
||||
);
|
||||
DB::statement("DELETE FROM collections WHERE collectionhash IN ({$placeholders})", $hashes);
|
||||
} else {
|
||||
// Fallback by noise marker
|
||||
DB::statement(
|
||||
'DELETE FROM parts WHERE binaries_id IN (SELECT b.id FROM binaries b WHERE b.collections_id IN (SELECT c.id FROM collections c WHERE c.noise = ?))',
|
||||
[$this->batchNoise]
|
||||
);
|
||||
DB::statement(
|
||||
'DELETE FROM binaries WHERE collections_id IN (SELECT id FROM collections WHERE noise = ?)',
|
||||
[$this->batchNoise]
|
||||
);
|
||||
DB::statement('DELETE FROM collections WHERE noise = ?', [$this->batchNoise]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use App\Models\MissedPart;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Handles missed parts tracking and repair during header processing.
|
||||
*/
|
||||
final class MissedPartHandler
|
||||
{
|
||||
private int $partRepairLimit;
|
||||
|
||||
private int $partRepairMaxTries;
|
||||
|
||||
public function __construct(int $partRepairLimit = 15000, int $partRepairMaxTries = 3)
|
||||
{
|
||||
$this->partRepairLimit = $partRepairLimit;
|
||||
$this->partRepairMaxTries = $partRepairMaxTries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add missing article numbers to the repair queue.
|
||||
*/
|
||||
public function addMissingParts(array $numbers, int $groupId): void
|
||||
{
|
||||
if (empty($numbers)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$driver = DB::getDriverName();
|
||||
|
||||
if ($driver === 'sqlite') {
|
||||
$this->addMissingPartsSqlite($numbers, $groupId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->addMissingPartsMysql($numbers, $groupId);
|
||||
}
|
||||
|
||||
private function addMissingPartsSqlite(array $numbers, int $groupId): void
|
||||
{
|
||||
foreach ($numbers as $number) {
|
||||
DB::statement(
|
||||
'INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON CONFLICT(numberid, groups_id) DO UPDATE SET attempts = attempts + 1',
|
||||
[$number, $groupId]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private function addMissingPartsMysql(array $numbers, int $groupId): void
|
||||
{
|
||||
$insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES ';
|
||||
foreach ($numbers as $number) {
|
||||
$insertStr .= '('.$number.','.$groupId.'),';
|
||||
}
|
||||
|
||||
DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove successfully repaired parts from the queue.
|
||||
*/
|
||||
public function removeRepairedParts(array $numbers, int $groupId): void
|
||||
{
|
||||
if (empty($numbers)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$sql = 'DELETE FROM missed_parts WHERE numberid in (';
|
||||
foreach ($numbers as $number) {
|
||||
$sql .= $number.',';
|
||||
}
|
||||
|
||||
try {
|
||||
DB::transaction(static function () use ($groupId, $sql) {
|
||||
DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupId);
|
||||
}, 10);
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::warning('removeRepairedParts failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parts that need repair for a group.
|
||||
*
|
||||
* @return array Array of missed parts
|
||||
*/
|
||||
public function getMissingParts(int $groupId): array
|
||||
{
|
||||
try {
|
||||
return DB::select(
|
||||
sprintf(
|
||||
'SELECT * FROM missed_parts WHERE groups_id = %d AND attempts < %d ORDER BY numberid ASC LIMIT %d',
|
||||
$groupId,
|
||||
$this->partRepairMaxTries,
|
||||
$this->partRepairLimit
|
||||
)
|
||||
);
|
||||
} catch (\PDOException $e) {
|
||||
if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') {
|
||||
Log::notice('Deadlock occurred while fetching missed parts');
|
||||
DB::rollBack();
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment attempts for parts that weren't repaired.
|
||||
*/
|
||||
public function incrementAttempts(int $groupId, int $maxNumberId): void
|
||||
{
|
||||
DB::update(
|
||||
sprintf(
|
||||
'UPDATE missed_parts SET attempts = attempts + 1 WHERE groups_id = %d AND numberid <= %d',
|
||||
$groupId,
|
||||
$maxNumberId
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment attempts for specific article range (part repair NNTP failures).
|
||||
*/
|
||||
public function incrementRangeAttempts(int $groupId, int $first, int $last): void
|
||||
{
|
||||
if ($first === $last) {
|
||||
MissedPart::query()
|
||||
->where('groups_id', $groupId)
|
||||
->where('numberid', $first)
|
||||
->increment('attempts');
|
||||
} else {
|
||||
MissedPart::query()
|
||||
->where('groups_id', $groupId)
|
||||
->whereIn('numberid', range($first, $last))
|
||||
->increment('attempts');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of remaining missed parts.
|
||||
*/
|
||||
public function getCount(int $groupId, int $maxNumberId): int
|
||||
{
|
||||
$result = DB::select(
|
||||
sprintf(
|
||||
'SELECT COUNT(id) AS num FROM missed_parts WHERE groups_id = %d AND numberid <= %d',
|
||||
$groupId,
|
||||
$maxNumberId
|
||||
)
|
||||
);
|
||||
|
||||
return $result[0]->num ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove parts that exceeded max tries.
|
||||
*/
|
||||
public function cleanupExhaustedParts(int $groupId): void
|
||||
{
|
||||
DB::transaction(function () use ($groupId) {
|
||||
DB::delete(
|
||||
sprintf(
|
||||
'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d',
|
||||
$this->partRepairMaxTries,
|
||||
$groupId
|
||||
)
|
||||
);
|
||||
}, 10);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Binaries;
|
||||
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Handles part record creation during header storage.
|
||||
*/
|
||||
final class PartHandler
|
||||
{
|
||||
/** @var array Pending parts to insert */
|
||||
private array $parts = [];
|
||||
|
||||
/** @var array Part numbers successfully inserted */
|
||||
private array $insertedPartNumbers = [];
|
||||
|
||||
/** @var array Part numbers that failed to insert */
|
||||
private array $failedPartNumbers = [];
|
||||
|
||||
private int $chunkSize;
|
||||
|
||||
private bool $addToPartRepair;
|
||||
|
||||
public function __construct(int $chunkSize = 5000, bool $addToPartRepair = true)
|
||||
{
|
||||
$this->chunkSize = max(100, $chunkSize);
|
||||
$this->addToPartRepair = $addToPartRepair;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset state for a new batch.
|
||||
*/
|
||||
public function reset(): void
|
||||
{
|
||||
$this->parts = [];
|
||||
$this->insertedPartNumbers = [];
|
||||
$this->failedPartNumbers = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to add failed parts to repair queue.
|
||||
*/
|
||||
public function setAddToPartRepair(bool $value): void
|
||||
{
|
||||
$this->addToPartRepair = $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a part to the pending insert queue.
|
||||
*
|
||||
* @return bool True if chunk was flushed successfully (or not needed), false on flush failure
|
||||
*/
|
||||
public function addPart(int $binaryId, array $header): bool
|
||||
{
|
||||
$this->parts[] = [
|
||||
'binaries_id' => $binaryId,
|
||||
'number' => $header['Number'],
|
||||
'messageid' => $header['Message-ID'],
|
||||
'partnumber' => $header['matches'][2],
|
||||
'size' => $header['Bytes'],
|
||||
];
|
||||
|
||||
// Auto-flush when chunk size reached
|
||||
if (\count($this->parts) >= $this->chunkSize) {
|
||||
return $this->flush();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush pending parts to database.
|
||||
*/
|
||||
public function flush(): bool
|
||||
{
|
||||
if (empty($this->parts)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$success = $this->insertChunk($this->parts);
|
||||
|
||||
if ($success) {
|
||||
foreach ($this->parts as $part) {
|
||||
$this->insertedPartNumbers[] = $part['number'];
|
||||
}
|
||||
} else {
|
||||
foreach ($this->parts as $part) {
|
||||
$this->failedPartNumbers[] = $part['number'];
|
||||
}
|
||||
}
|
||||
|
||||
$this->parts = [];
|
||||
|
||||
return $success;
|
||||
}
|
||||
|
||||
private function insertChunk(array $parts): bool
|
||||
{
|
||||
$placeholders = [];
|
||||
$bindings = [];
|
||||
$driver = DB::getDriverName();
|
||||
|
||||
foreach ($parts as $row) {
|
||||
$placeholders[] = '(?,?,?,?,?)';
|
||||
$bindings[] = $row['binaries_id'];
|
||||
$bindings[] = $row['number'];
|
||||
$bindings[] = $row['messageid'];
|
||||
$bindings[] = $row['partnumber'];
|
||||
$bindings[] = $row['size'];
|
||||
}
|
||||
|
||||
$sql = $driver === 'sqlite'
|
||||
? 'INSERT OR IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders)
|
||||
: 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '.implode(',', $placeholders);
|
||||
|
||||
try {
|
||||
DB::statement($sql, $bindings);
|
||||
|
||||
return true;
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error('Parts chunk insert failed: '.$e->getMessage());
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get numbers of successfully inserted parts.
|
||||
*/
|
||||
public function getInsertedNumbers(): array
|
||||
{
|
||||
return $this->insertedPartNumbers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get numbers of failed part inserts.
|
||||
*/
|
||||
public function getFailedNumbers(): array
|
||||
{
|
||||
return $this->failedPartNumbers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there are pending parts waiting to be flushed.
|
||||
*/
|
||||
public function hasPending(): bool
|
||||
{
|
||||
return ! empty($this->parts);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,10 +105,12 @@ class MiscCategorizer extends AbstractCategorizer
|
||||
return $this->matched(Category::OTHER_HASHED, 0.7, 'obfuscated_uppercase');
|
||||
}
|
||||
|
||||
// Long alphanumeric strings without typical release patterns
|
||||
if (preg_match('/^[a-zA-Z0-9]{25,}$/', $name) &&
|
||||
!preg_match('/\b(19|20)\d{2}\b/', $name)) {
|
||||
return $this->matched(Category::OTHER_HASHED, 0.65, 'obfuscated_long');
|
||||
// Mixed-case alphanumeric strings without separators (common obfuscation pattern)
|
||||
// These look like random strings: e.g., "AA7Jl2toE8Q53yNZmQ5R6G"
|
||||
if (preg_match('/^[a-zA-Z0-9]{15,}$/', $name) &&
|
||||
!preg_match('/\b(19|20)\d{2}\b/', $name) &&
|
||||
!preg_match('/^[A-Z][a-z]+([A-Z][a-z]+)+$/', $name)) { // Exclude CamelCase words
|
||||
return $this->matched(Category::OTHER_HASHED, 0.7, 'obfuscated_mixed_alphanumeric');
|
||||
}
|
||||
|
||||
// Only punctuation and numbers with no clear structure
|
||||
|
||||
Generated
+7
-7
@@ -13184,23 +13184,23 @@
|
||||
},
|
||||
{
|
||||
"name": "tijsverkoyen/css-to-inline-styles",
|
||||
"version": "v2.3.0",
|
||||
"version": "v2.4.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/tijsverkoyen/CssToInlineStyles.git",
|
||||
"reference": "0d72ac1c00084279c1816675284073c5a337c20d"
|
||||
"reference": "f0292ccf0ec75843d65027214426b6b163b48b41"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/0d72ac1c00084279c1816675284073c5a337c20d",
|
||||
"reference": "0d72ac1c00084279c1816675284073c5a337c20d",
|
||||
"url": "https://api.github.com/repos/tijsverkoyen/CssToInlineStyles/zipball/f0292ccf0ec75843d65027214426b6b163b48b41",
|
||||
"reference": "f0292ccf0ec75843d65027214426b6b163b48b41",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-dom": "*",
|
||||
"ext-libxml": "*",
|
||||
"php": "^7.4 || ^8.0",
|
||||
"symfony/css-selector": "^5.4 || ^6.0 || ^7.0"
|
||||
"symfony/css-selector": "^5.4 || ^6.0 || ^7.0 || ^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpstan/phpstan": "^2.0",
|
||||
@@ -13233,9 +13233,9 @@
|
||||
"homepage": "https://github.com/tijsverkoyen/CssToInlineStyles",
|
||||
"support": {
|
||||
"issues": "https://github.com/tijsverkoyen/CssToInlineStyles/issues",
|
||||
"source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/v2.3.0"
|
||||
"source": "https://github.com/tijsverkoyen/CssToInlineStyles/tree/v2.4.0"
|
||||
},
|
||||
"time": "2024-12-21T16:25:41+00:00"
|
||||
"time": "2025-12-02T11:56:42+00:00"
|
||||
},
|
||||
{
|
||||
"name": "vlucas/phpdotenv",
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
<div class="container mx-auto px-4 py-6">
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-sm">
|
||||
<!-- Header -->
|
||||
<div class="px-6 py-4 border-b border-gray-200">
|
||||
<div class="px-6 py-4 border-b border-gray-200 dark:border-gray-700">
|
||||
<div class="flex justify-between items-center">
|
||||
<h1 class="text-2xl font-semibold text-gray-800">
|
||||
<h1 class="text-2xl font-semibold text-gray-800 dark:text-gray-100">
|
||||
<i class="fa fa-plus-square mr-2"></i>{{ $title ?? 'Bulk Add Newsgroups' }}
|
||||
</h1>
|
||||
<a href="{{ url('/admin/group-list') }}" class="px-4 py-2 bg-gray-100 dark:bg-gray-800 text-gray-700 dark:text-gray-300 rounded-lg hover:bg-gray-200">
|
||||
<a href="{{ url('/admin/group-list') }}" class="px-4 py-2 bg-gray-100 dark:bg-gray-700 text-gray-700 dark:text-gray-300 rounded-lg hover:bg-gray-200 dark:hover:bg-gray-600">
|
||||
<i class="fa fa-list mr-2"></i>View All Groups
|
||||
</a>
|
||||
</div>
|
||||
@@ -18,10 +18,10 @@
|
||||
<div class="px-6 py-6">
|
||||
@if(!empty($groupmsglist))
|
||||
<!-- Success Info -->
|
||||
<div class="mb-6 p-4 bg-blue-50 border border-blue-200 rounded-lg">
|
||||
<div class="mb-6 p-4 bg-blue-50 dark:bg-blue-900/30 border border-blue-200 dark:border-blue-800 rounded-lg">
|
||||
<div class="flex">
|
||||
<i class="fa fa-info-circle text-blue-500 text-xl mr-3"></i>
|
||||
<p class="text-blue-700">
|
||||
<i class="fa fa-info-circle text-blue-500 dark:text-blue-400 text-xl mr-3"></i>
|
||||
<p class="text-blue-700 dark:text-blue-300">
|
||||
The following groups have been processed. You can now view them in the group list.
|
||||
</p>
|
||||
</div>
|
||||
@@ -29,33 +29,33 @@
|
||||
|
||||
<!-- Results Table -->
|
||||
<div class="overflow-x-auto">
|
||||
<table class="min-w-full divide-y divide-gray-200">
|
||||
<thead class="bg-gray-50">
|
||||
<table class="min-w-full divide-y divide-gray-200 dark:divide-gray-700">
|
||||
<thead class="bg-gray-50 dark:bg-gray-700">
|
||||
<tr>
|
||||
<th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Group</th>
|
||||
<th class="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Status</th>
|
||||
<th class="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-300 uppercase tracking-wider">Group</th>
|
||||
<th class="px-6 py-3 text-left text-xs font-medium text-gray-500 dark:text-gray-300 uppercase tracking-wider">Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="bg-white dark:bg-gray-800 divide-y divide-gray-200">
|
||||
<tbody class="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-700">
|
||||
@foreach($groupmsglist as $group)
|
||||
<tr class="hover:bg-gray-50">
|
||||
<tr class="hover:bg-gray-50 dark:hover:bg-gray-700">
|
||||
<td class="px-6 py-4 whitespace-nowrap">
|
||||
<div class="flex items-center">
|
||||
<i class="fa fa-users text-gray-400 mr-3"></i>
|
||||
<span class="font-medium text-gray-900">{{ $group['group'] }}</span>
|
||||
<i class="fa fa-users text-gray-400 dark:text-gray-500 mr-3"></i>
|
||||
<span class="font-medium text-gray-900 dark:text-gray-100">{{ $group['group'] }}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 whitespace-nowrap">
|
||||
@if(strpos($group['msg'], 'Error') !== false)
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-red-100 text-red-800">
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-red-100 dark:bg-red-900/30 text-red-800 dark:text-red-300">
|
||||
<i class="fa fa-exclamation-circle mr-1"></i>{{ $group['msg'] }}
|
||||
</span>
|
||||
@elseif(strpos($group['msg'], 'exists') !== false)
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800">
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-yellow-100 dark:bg-yellow-900/30 text-yellow-800 dark:text-yellow-300">
|
||||
<i class="fa fa-exclamation-triangle mr-1"></i>{{ $group['msg'] }}
|
||||
</span>
|
||||
@else
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800">
|
||||
<span class="inline-flex items-center px-3 py-1 rounded-full text-xs font-semibold bg-green-100 dark:bg-green-900/30 text-green-800 dark:text-green-300">
|
||||
<i class="fa fa-check-circle mr-1"></i>{{ $group['msg'] }}
|
||||
</span>
|
||||
@endif
|
||||
@@ -67,10 +67,10 @@
|
||||
</div>
|
||||
@else
|
||||
<!-- Info Alert -->
|
||||
<div class="mb-6 p-4 bg-blue-50 border border-blue-200 rounded-lg">
|
||||
<div class="mb-6 p-4 bg-blue-50 dark:bg-blue-900/30 border border-blue-200 dark:border-blue-800 rounded-lg">
|
||||
<div class="flex">
|
||||
<i class="fa fa-info-circle text-blue-500 text-xl mr-3"></i>
|
||||
<p class="text-blue-700">
|
||||
<i class="fa fa-info-circle text-blue-500 dark:text-blue-400 text-xl mr-3"></i>
|
||||
<p class="text-blue-700 dark:text-blue-300">
|
||||
Enter a regular expression to match multiple groups for bulk addition to the system.
|
||||
</p>
|
||||
</div>
|
||||
@@ -91,13 +91,13 @@
|
||||
</div>
|
||||
<textarea id="groupfilter"
|
||||
name="groupfilter"
|
||||
class="pl-10 w-full px-4 py-2 border border-gray-300 dark:border-gray-600 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500 font-mono text-sm"
|
||||
class="pl-10 w-full px-4 py-2 border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500 font-mono text-sm"
|
||||
rows="5"
|
||||
placeholder="e.g. alt.binaries.cd.image.linux|alt.binaries.warez.linux"></textarea>
|
||||
</div>
|
||||
<p class="mt-2 text-sm text-gray-500">
|
||||
<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">
|
||||
A regular expression to match against group names. Separate multiple patterns with the pipe symbol (|).
|
||||
<br>Example: <code class="bg-gray-100 dark:bg-gray-800 px-2 py-1 rounded text-xs text-pink-600">alt.binaries.cd.image.linux|alt.binaries.warez.linux</code>
|
||||
<br>Example: <code class="bg-gray-100 dark:bg-gray-800 px-2 py-1 rounded text-xs text-pink-600 dark:text-pink-400">alt.binaries.cd.image.linux|alt.binaries.warez.linux</code>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -112,7 +112,7 @@
|
||||
value="1"
|
||||
class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500"
|
||||
checked>
|
||||
<label for="active_yes" class="ml-2 text-sm text-gray-700">Yes</label>
|
||||
<label for="active_yes" class="ml-2 text-sm text-gray-700 dark:text-gray-300">Yes</label>
|
||||
</div>
|
||||
<div class="flex items-center">
|
||||
<input type="radio"
|
||||
@@ -120,10 +120,10 @@
|
||||
id="active_no"
|
||||
value="0"
|
||||
class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500">
|
||||
<label for="active_no" class="ml-2 text-sm text-gray-700">No</label>
|
||||
<label for="active_no" class="ml-2 text-sm text-gray-700 dark:text-gray-300">No</label>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-2 text-sm text-gray-500">
|
||||
<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">
|
||||
Inactive groups will not have headers downloaded for them.
|
||||
</p>
|
||||
</div>
|
||||
@@ -138,7 +138,7 @@
|
||||
id="backfill_yes"
|
||||
value="1"
|
||||
class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500">
|
||||
<label for="backfill_yes" class="ml-2 text-sm text-gray-700">Yes</label>
|
||||
<label for="backfill_yes" class="ml-2 text-sm text-gray-700 dark:text-gray-300">Yes</label>
|
||||
</div>
|
||||
<div class="flex items-center">
|
||||
<input type="radio"
|
||||
@@ -147,10 +147,10 @@
|
||||
value="0"
|
||||
class="w-4 h-4 text-blue-600 dark:text-blue-400 border-gray-300 dark:border-gray-600 focus:ring-blue-500"
|
||||
checked>
|
||||
<label for="backfill_no" class="ml-2 text-sm text-gray-700">No</label>
|
||||
<label for="backfill_no" class="ml-2 text-sm text-gray-700 dark:text-gray-300">No</label>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-2 text-sm text-gray-500">
|
||||
<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">
|
||||
Inactive groups will not have backfill headers downloaded for them.
|
||||
</p>
|
||||
</div>
|
||||
@@ -159,17 +159,17 @@
|
||||
</div>
|
||||
|
||||
<!-- Footer -->
|
||||
<div class="px-6 py-4 border-t border-gray-200 dark:border-gray-700 bg-gray-50">
|
||||
<div class="px-6 py-4 border-t border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-700">
|
||||
<div class="flex justify-between">
|
||||
<a href="{{ url('/admin/group-list') }}" class="px-4 py-2 bg-gray-200 dark:bg-gray-700 text-gray-700 dark:text-gray-300 rounded-lg hover:bg-gray-300">
|
||||
<a href="{{ url('/admin/group-list') }}" class="px-4 py-2 bg-gray-200 dark:bg-gray-700 text-gray-700 dark:text-gray-300 rounded-lg hover:bg-gray-300 dark:hover:bg-gray-600">
|
||||
<i class="fa fa-arrow-left mr-2"></i>Back to Groups
|
||||
</a>
|
||||
@if(empty($groupmsglist))
|
||||
<button type="submit" form="groupBulkForm" class="px-4 py-2 bg-green-600 dark:bg-green-700 text-white rounded-lg hover:bg-green-700">
|
||||
<button type="submit" form="groupBulkForm" class="px-4 py-2 bg-green-600 dark:bg-green-700 text-white rounded-lg hover:bg-green-700 dark:hover:bg-green-600">
|
||||
<i class="fa fa-plus-circle mr-2"></i>Add Groups
|
||||
</button>
|
||||
@else
|
||||
<a href="{{ url('/admin/group-bulk') }}" class="px-4 py-2 bg-blue-600 dark:bg-blue-700 text-white rounded-lg hover:bg-blue-700">
|
||||
<a href="{{ url('/admin/group-bulk') }}" class="px-4 py-2 bg-blue-600 dark:bg-blue-700 text-white rounded-lg hover:bg-blue-700 dark:hover:bg-blue-600">
|
||||
<i class="fa fa-plus-circle mr-2"></i>Add More Groups
|
||||
</a>
|
||||
@endif
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
namespace Tests\Support;
|
||||
|
||||
use App\Services\BlacklistService;
|
||||
use App\Services\XrefService;
|
||||
use App\Services\Binaries\BinariesConfig;
|
||||
use App\Services\Binaries\HeaderStorageService;
|
||||
use App\Services\Binaries\MissedPartHandler;
|
||||
use Blacklight\Binaries;
|
||||
use Blacklight\ColorCLI;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class TestBinariesHarness extends Binaries
|
||||
@@ -16,67 +16,100 @@ class TestBinariesHarness extends Binaries
|
||||
|
||||
private int $flushCount = 0;
|
||||
|
||||
protected mixed $_collectionsCleaning; // override parent type
|
||||
private array $testGroupMySQL = [];
|
||||
|
||||
private HeaderStorageService $testHeaderStorage;
|
||||
|
||||
private MissedPartHandler $testMissedPartHandler;
|
||||
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
// Manually initialize only what storeHeaders/flushPartsChunk need; skip NNTP + Settings lookups.
|
||||
$this->startUpdate = now();
|
||||
$this->timeCleaning = 0;
|
||||
$this->_echoCLI = false;
|
||||
$this->_pdo = DB::connection()->getPdo();
|
||||
$this->colorCli = new ColorCLI;
|
||||
$this->_collectionsCleaning = new class
|
||||
{
|
||||
public function collectionsCleaner($subject, $groupName): array
|
||||
{
|
||||
return ['id' => 1, 'name' => 'COLL'];
|
||||
}
|
||||
};
|
||||
$this->xrefService = new XrefService;
|
||||
$this->blacklistService = new BlacklistService;
|
||||
$this->messageBuffer = 50000;
|
||||
$this->_compressedHeaders = false;
|
||||
$this->_partRepair = true;
|
||||
$this->_newGroupScanByDays = false;
|
||||
$this->_newGroupMessagesToScan = 50000;
|
||||
$this->_newGroupDaysToScan = 3;
|
||||
$this->_partRepairLimit = 15000;
|
||||
$this->_partRepairMaxTries = 3;
|
||||
$this->blackList = $this->whiteList = [];
|
||||
// Create a minimal config that doesn't require database access
|
||||
$config = new BinariesConfig(
|
||||
messageBuffer: 50000,
|
||||
compressedHeaders: false,
|
||||
partRepair: true,
|
||||
newGroupScanByDays: false,
|
||||
newGroupMessagesToScan: 50000,
|
||||
newGroupDaysToScan: 3,
|
||||
partRepairLimit: 15000,
|
||||
partRepairMaxTries: 3,
|
||||
partsChunkSize: 5000,
|
||||
binariesUpdateChunkSize: 1000,
|
||||
echoCli: false
|
||||
);
|
||||
|
||||
// Call parent with config (will still try to create NNTP, but we won't use it)
|
||||
parent::__construct($config);
|
||||
|
||||
// Override with test-specific services
|
||||
$this->testHeaderStorage = new HeaderStorageService(config: $config);
|
||||
$this->testMissedPartHandler = new MissedPartHandler(
|
||||
$config->partRepairLimit,
|
||||
$config->partRepairMaxTries
|
||||
);
|
||||
}
|
||||
|
||||
// Expose protected storeHeaders for direct testing.
|
||||
// Expose protected method for direct testing via new service.
|
||||
public function publicStoreHeaders(array $headers): void
|
||||
{
|
||||
if (empty($this->groupMySQL)) {
|
||||
$this->groupMySQL = ['id' => 1, 'name' => 'alt.test'];
|
||||
if (empty($this->testGroupMySQL)) {
|
||||
$this->testGroupMySQL = ['id' => 1, 'name' => 'alt.test'];
|
||||
}
|
||||
$this->startCleaning = now();
|
||||
config(['tests.force_simulated_rollback' => false]);
|
||||
$this->storeHeaders($headers);
|
||||
|
||||
// Parse headers first to add 'matches'
|
||||
$parsedHeaders = [];
|
||||
foreach ($headers as $header) {
|
||||
if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) {
|
||||
if (stripos($header['Subject'], 'yEnc') === false) {
|
||||
$matches[1] .= ' yEnc';
|
||||
}
|
||||
$header['matches'] = $matches;
|
||||
$parsedHeaders[] = $header;
|
||||
}
|
||||
}
|
||||
|
||||
$this->testHeaderStorage->store($parsedHeaders, $this->testGroupMySQL, true);
|
||||
}
|
||||
|
||||
public function setAddToPartRepair(bool $val): void
|
||||
{
|
||||
$this->addToPartRepair = $val;
|
||||
// This is now handled by passing parameter to store()
|
||||
}
|
||||
|
||||
// Simulate scan path minimally to test rollback + part repair queue logic without NNTP.
|
||||
public function simulateScan(array $headers, array $group, bool $enablePartRepair = true): void
|
||||
{
|
||||
$this->groupMySQL = $group;
|
||||
$this->first = $headers[0]['Number'];
|
||||
$this->last = end($headers)['Number'];
|
||||
$this->headersReceived = array_column($headers, 'Number');
|
||||
$this->addToPartRepair = $enablePartRepair;
|
||||
$this->startCleaning = now();
|
||||
$this->testGroupMySQL = $group;
|
||||
|
||||
// Parse headers first to add 'matches'
|
||||
$parsedHeaders = [];
|
||||
$headersReceived = [];
|
||||
foreach ($headers as $header) {
|
||||
if (isset($header['Number'])) {
|
||||
$headersReceived[] = $header['Number'];
|
||||
}
|
||||
if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $matches)) {
|
||||
if (stripos($header['Subject'], 'yEnc') === false) {
|
||||
$matches[1] .= ' yEnc';
|
||||
}
|
||||
$header['matches'] = $matches;
|
||||
$parsedHeaders[] = $header;
|
||||
}
|
||||
}
|
||||
|
||||
// If we are simulating a failure, do not perform any inserts; just mark all as missed.
|
||||
if ($this->failPartsInsert) {
|
||||
if ($enablePartRepair) {
|
||||
foreach (array_unique($this->headersReceived) as $num) {
|
||||
DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1)', [$num, $group['id']]);
|
||||
foreach (array_unique($headersReceived) as $num) {
|
||||
$driver = DB::getDriverName();
|
||||
if ($driver === 'sqlite') {
|
||||
DB::statement('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON CONFLICT(numberid, groups_id) DO UPDATE SET attempts = attempts + 1', [$num, $group['id']]);
|
||||
} else {
|
||||
DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1) ON DUPLICATE KEY UPDATE attempts = attempts + 1', [$num, $group['id']]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,28 +117,10 @@ class TestBinariesHarness extends Binaries
|
||||
}
|
||||
|
||||
// Normal path: process and insert.
|
||||
$this->storeHeaders($headers);
|
||||
$failedInserts = $this->testHeaderStorage->store($parsedHeaders, $group, $enablePartRepair);
|
||||
|
||||
if ($enablePartRepair && ! empty($this->headersNotInserted)) {
|
||||
foreach (array_unique($this->headersNotInserted) as $num) {
|
||||
DB::insert('INSERT INTO missed_parts (numberid, groups_id, attempts) VALUES (?, ?, 1)', [$num, $group['id']]);
|
||||
}
|
||||
if ($enablePartRepair && ! empty($failedInserts)) {
|
||||
$this->testMissedPartHandler->addMissingParts($failedInserts, $group['id']);
|
||||
}
|
||||
}
|
||||
|
||||
// Force chunk failure to trigger rollback when flag set.
|
||||
protected function flushPartsChunk(array $parts): bool
|
||||
{
|
||||
$this->flushCount++;
|
||||
if ($this->failPartsInsert) {
|
||||
if ($this->failAfterFlushCount === null) {
|
||||
return false; // Always fail when flag set and no threshold provided.
|
||||
}
|
||||
if ($this->flushCount > $this->failAfterFlushCount) {
|
||||
return false; // Fail after N successful flushes
|
||||
}
|
||||
}
|
||||
|
||||
return parent::flushPartsChunk($parts);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user