mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:01:24 +00:00
Improve binaries and extract xref and blacklist services
This commit is contained in:
DariusIII
+67
-240
@@ -2,11 +2,12 @@
|
||||
|
||||
namespace Blacklight;
|
||||
|
||||
use App\Models\BinaryBlacklist;
|
||||
use App\Models\Collection;
|
||||
use App\Models\MissedPart;
|
||||
use App\Models\Settings;
|
||||
use App\Models\UsenetGroup;
|
||||
use App\Services\BlacklistService;
|
||||
use App\Services\XrefService;
|
||||
use Illuminate\Database\QueryException;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
@@ -95,9 +96,12 @@ class Binaries
|
||||
*/
|
||||
protected int $_partRepairMaxTries;
|
||||
|
||||
/**
|
||||
* An array of BinaryBlacklist IDs that should have their activity date updated.
|
||||
*/
|
||||
/** Dedicated services */
|
||||
protected XrefService $xrefService;
|
||||
|
||||
protected BlacklistService $blacklistService;
|
||||
|
||||
/** No longer used directly, kept for BC */
|
||||
protected array $_binaryBlacklistIdsToUpdate = [];
|
||||
|
||||
protected \DateTime $startCleaning;
|
||||
@@ -179,6 +183,8 @@ class Binaries
|
||||
$this->colorCli = new ColorCLI;
|
||||
$this->_nntp = new NNTP;
|
||||
$this->_collectionsCleaning = new CollectionsCleaning;
|
||||
$this->xrefService = new XrefService;
|
||||
$this->blacklistService = new BlacklistService;
|
||||
|
||||
$this->messageBuffer = Settings::settingValue('maxmssgs') !== '' ?
|
||||
(int) Settings::settingValue('maxmssgs') : 20000;
|
||||
@@ -223,7 +229,16 @@ class Binaries
|
||||
__FUNCTION__,
|
||||
'header'
|
||||
);
|
||||
$this->updateGroup($group, $maxHeaders);
|
||||
try {
|
||||
$this->updateGroup($group, $maxHeaders);
|
||||
} catch (\Throwable $e) {
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->error('Error updating group '.$group['name'].': '.$e->getMessage());
|
||||
}
|
||||
if (config('app.debug')) {
|
||||
Log::error('updateGroup failed for '.$group['name'].': '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
$counter++;
|
||||
}
|
||||
|
||||
@@ -583,18 +598,8 @@ class Binaries
|
||||
$headersRepaired[] = $header['Number'];
|
||||
}
|
||||
|
||||
/*
|
||||
* Find part / total parts. Ignore if no part count found.
|
||||
*
|
||||
* \s* Trims the leading space.
|
||||
* (?!"Usenet Index Post) ignores these types of articles, they are useless.
|
||||
* (.+) Fetches the subject.
|
||||
* \s+ Trims trailing space after the subject.
|
||||
* \((\d+)\/(\d+)\) Gets the part count.
|
||||
* No ending ($) as there are cases of subjects with extra data after the part count.
|
||||
*/
|
||||
// Parse subject to get base name and part/total like "(12/45)"; normalize to include yEnc if missing.
|
||||
if (preg_match('/^\s*(?!"Usenet Index Post)(.+)\s+\((\d+)\/(\d+)\)/', $header['Subject'], $header['matches'])) {
|
||||
// Add yEnc to subjects that do not have them, but have the part number at the end of the header.
|
||||
if (stripos($header['Subject'], 'yEnc') === false) {
|
||||
$header['matches'][1] .= ' yEnc';
|
||||
}
|
||||
@@ -605,7 +610,7 @@ class Binaries
|
||||
}
|
||||
|
||||
// Filter subject based on black/white list.
|
||||
if ($this->isBlackListed($header, $this->groupMySQL['name'])) {
|
||||
if ($this->blacklistService->isBlackListed($header, $this->groupMySQL['name'])) {
|
||||
$this->headersBlackListed++;
|
||||
|
||||
continue;
|
||||
@@ -620,8 +625,10 @@ class Binaries
|
||||
|
||||
unset($headers); // Reclaim memory now that headers are split.
|
||||
|
||||
if (! empty($this->_binaryBlacklistIdsToUpdate)) {
|
||||
$this->updateBlacklistUsage();
|
||||
// Update blacklist last_activity for matched rules.
|
||||
$ids = $this->blacklistService->getAndClearIdsToUpdate();
|
||||
if (! empty($ids)) {
|
||||
$this->blacklistService->updateBlacklistUsage($ids);
|
||||
}
|
||||
|
||||
if ($this->_echoCLI && ! $partRepair) {
|
||||
@@ -629,7 +636,16 @@ class Binaries
|
||||
}
|
||||
|
||||
if (! empty($stdHeaders)) {
|
||||
$this->storeHeaders($stdHeaders);
|
||||
try {
|
||||
$this->storeHeaders($stdHeaders);
|
||||
} catch (\Throwable $e) {
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->error('storeHeaders failed: '.$e->getMessage());
|
||||
}
|
||||
if (config('app.debug')) {
|
||||
Log::error('storeHeaders failed: '.$e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
unset($stdHeaders);
|
||||
|
||||
@@ -728,52 +744,37 @@ class Binaries
|
||||
// Get the current unixtime from PHP.
|
||||
$now = now()->timestamp;
|
||||
|
||||
// Fetch existing xrefs for this collection (by hash) and prepare xref fields.
|
||||
$xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref');
|
||||
$headerTokens = $this->xrefService->extractTokens($this->header['Xref'] ?? '');
|
||||
$newTokens = $this->xrefService->diffNewTokens($xrefsData, $this->header['Xref'] ?? '');
|
||||
$finalXref = implode(' ', $newTokens);
|
||||
|
||||
$tempHeaderXrefs = [];
|
||||
foreach (explode(' ', $this->header['Xref']) as $headerXref) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) {
|
||||
$tempHeaderXrefs[] = $hit[0];
|
||||
}
|
||||
}
|
||||
|
||||
$tempXrefsData = [];
|
||||
|
||||
if ($xrefsData !== null) {
|
||||
foreach (explode(' ', $xrefsData) as $xrefData) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) {
|
||||
$tempXrefsData[] = $match1[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$finalXrefArray = [];
|
||||
foreach ($tempHeaderXrefs as $tempHeaderXref) {
|
||||
if (! in_array($tempHeaderXref, $tempXrefsData, false)) {
|
||||
$finalXrefArray[] = $tempHeaderXref;
|
||||
}
|
||||
}
|
||||
|
||||
$finaXref = implode(' ', $finalXrefArray);
|
||||
|
||||
$xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref));
|
||||
$xrefUpdateSQL = $finalXref !== ''
|
||||
? sprintf('xref = CONCAT(xref, "\\n", %s ),', escapeString($finalXref))
|
||||
: '';
|
||||
|
||||
$date = $this->header['Date'] > $now ? $now : $this->header['Date'];
|
||||
$unixtime = is_numeric($this->header['Date']) ? $date : $now;
|
||||
|
||||
// Random noise value used on duplicate update.
|
||||
$random = random_bytes(16);
|
||||
$number = random_int(3, 9999999);
|
||||
$special = Str::random(5);
|
||||
$string = htmlspecialchars(str_shuffle('trusin_ @'.$number.$special), ENT_QUOTES);
|
||||
|
||||
$collectionID = false;
|
||||
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
INSERT INTO collections (subject, fromname, date, xref, groups_id,
|
||||
totalfiles, collectionhash, collection_regexes_id, dateadded)
|
||||
VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW())
|
||||
ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", escapeString(substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255)), escapeString(mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings())), $unixtime, escapeString(implode(' ', $tempHeaderXrefs)), $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), $collMatch['id'], $xref, sodium_bin2hex($random)));
|
||||
DB::insert(sprintf("\n\t\t\t\t\t\tINSERT INTO collections (subject, fromname, date, xref, groups_id,\n\t\t\t\t\t\t\ttotalfiles, collectionhash, collection_regexes_id, dateadded)\n\t\t\t\t\t\tVALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW())\n\t\t\t\t\t\tON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'",
|
||||
escapeString(substr(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings()), 0, 255)),
|
||||
escapeString(mb_convert_encoding($this->header['From'], 'UTF-8', mb_list_encodings())),
|
||||
$unixtime,
|
||||
escapeString(implode(' ', $headerTokens)),
|
||||
$this->groupMySQL['id'],
|
||||
$fileCount[3],
|
||||
sha1($this->header['CollectionKey']),
|
||||
$collMatch['id'],
|
||||
$xrefUpdateSQL,
|
||||
sodium_bin2hex($random)
|
||||
));
|
||||
$collectionID = $this->_pdo->lastInsertId();
|
||||
DB::commit();
|
||||
} catch (\Throwable $e) {
|
||||
@@ -803,10 +804,15 @@ class Binaries
|
||||
$binaryID = false;
|
||||
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize)
|
||||
VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d)
|
||||
ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings())), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes']));
|
||||
DB::insert(sprintf("\n\t\t\t\t\tINSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize)\n\t\t\t\t\tVALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d)\n\t\t\t\t\tON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d",
|
||||
$hash,
|
||||
escapeString(mb_convert_encoding($this->header['matches'][1], 'UTF-8', mb_list_encodings())),
|
||||
$collectionID,
|
||||
$this->header['matches'][3],
|
||||
$fileCount[1],
|
||||
$this->header['Bytes'],
|
||||
$this->header['Bytes']
|
||||
));
|
||||
$binaryID = $this->_pdo->lastInsertId();
|
||||
DB::commit();
|
||||
} catch (\Throwable $e) {
|
||||
@@ -900,7 +906,7 @@ class Binaries
|
||||
}
|
||||
|
||||
// Break if we found non empty articles.
|
||||
if (isset($returnArray['firstArticleNumber, lastArticleNumber'])) {
|
||||
if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -911,15 +917,6 @@ class Binaries
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates Blacklist Regex Timers in DB to reflect last usage.
|
||||
*/
|
||||
protected function updateBlacklistUsage(): void
|
||||
{
|
||||
BinaryBlacklist::query()->whereIn('id', $this->_binaryBlacklistIdsToUpdate)->update(['last_activity' => now()]);
|
||||
$this->_binaryBlacklistIdsToUpdate = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Outputs the initial header scan results after yEnc check and blacklist routines.
|
||||
*/
|
||||
@@ -1284,176 +1281,6 @@ class Binaries
|
||||
*/
|
||||
protected array $_listsFound = [];
|
||||
|
||||
/**
|
||||
* Get blacklist and cache it. Return if already cached.
|
||||
*/
|
||||
protected function _retrieveBlackList(string $groupName): void
|
||||
{
|
||||
if (! isset($this->blackList[$groupName])) {
|
||||
$this->blackList[$groupName] = $this->getBlacklist(true, self::OPTYPE_BLACKLIST, $groupName, true);
|
||||
}
|
||||
if (! isset($this->whiteList[$groupName])) {
|
||||
$this->whiteList[$groupName] = $this->getBlacklist(true, self::OPTYPE_WHITELIST, $groupName, true);
|
||||
}
|
||||
$this->_listsFound[$groupName] = ($this->blackList[$groupName] || $this->whiteList[$groupName]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an article is blacklisted.
|
||||
*
|
||||
* @param array $msg The article header (OVER format).
|
||||
* @param string $groupName The group name.
|
||||
*/
|
||||
public function isBlackListed(array $msg, string $groupName): bool
|
||||
{
|
||||
if (! isset($this->_listsFound[$groupName])) {
|
||||
$this->_retrieveBlackList($groupName);
|
||||
}
|
||||
if (! $this->_listsFound[$groupName]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$blackListed = false;
|
||||
|
||||
$field = [
|
||||
self::BLACKLIST_FIELD_SUBJECT => $msg['Subject'],
|
||||
self::BLACKLIST_FIELD_FROM => $msg['From'],
|
||||
self::BLACKLIST_FIELD_MESSAGEID => $msg['Message-ID'],
|
||||
];
|
||||
|
||||
// Try white lists first.
|
||||
if ($this->whiteList[$groupName]) {
|
||||
// There are white lists for this group, so anything that doesn't match a white list should be considered black listed.
|
||||
$blackListed = true;
|
||||
foreach ($this->whiteList[$groupName] as $whiteList) {
|
||||
if (preg_match('/'.$whiteList['regex'].'/i', $field[$whiteList['msgcol']])) {
|
||||
// This field matched a white list, so it might not be black listed.
|
||||
$blackListed = false;
|
||||
$this->_binaryBlacklistIdsToUpdate[$whiteList['id']] = $whiteList['id'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the field is blacklisted.
|
||||
|
||||
if (! $blackListed && $this->blackList[$groupName]) {
|
||||
foreach ($this->blackList[$groupName] as $blackList) {
|
||||
if (preg_match('/'.$blackList->regex.'/i', $field[$blackList->msgcol])) {
|
||||
$blackListed = true;
|
||||
$this->_binaryBlacklistIdsToUpdate[$blackList->id] = $blackList->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $blackListed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all blacklists.
|
||||
*
|
||||
* @param bool $activeOnly Only display active blacklists ?
|
||||
* @param int|string $opType Optional, get white or black lists (use Binaries constants).
|
||||
* @param string $groupName Optional, group.
|
||||
* @param bool $groupRegex Optional Join groups / binary blacklist using regexp for equals.
|
||||
*/
|
||||
public function getBlacklist(bool $activeOnly = true, int|string $opType = -1, string $groupName = '', bool $groupRegex = false): array
|
||||
{
|
||||
$opType = match ($opType) {
|
||||
self::OPTYPE_BLACKLIST => 'AND bb.optype = '.self::OPTYPE_BLACKLIST,
|
||||
self::OPTYPE_WHITELIST => 'AND bb.optype = '.self::OPTYPE_WHITELIST,
|
||||
default => '',
|
||||
};
|
||||
|
||||
return DB::select(
|
||||
sprintf(
|
||||
'
|
||||
SELECT
|
||||
bb.id, bb.optype, bb.status, bb.description,
|
||||
bb.groupname AS groupname, bb.regex, g.id AS group_id, bb.msgcol,
|
||||
bb.last_activity as last_activity
|
||||
FROM binaryblacklist bb
|
||||
LEFT OUTER JOIN usenet_groups g ON g.name %s bb.groupname
|
||||
WHERE 1=1 %s %s %s
|
||||
ORDER BY coalesce(groupname,\'zzz\')',
|
||||
($groupRegex ? 'REGEXP' : '='),
|
||||
($activeOnly ? 'AND bb.status = 1' : ''),
|
||||
$opType,
|
||||
($groupName ? ('AND g.name REGEXP '.escapeString($groupName)) : '')
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a blacklist by ID.
|
||||
*
|
||||
* @param int $id The ID of the blacklist.
|
||||
*/
|
||||
public function getBlacklistByID(int $id)
|
||||
{
|
||||
return BinaryBlacklist::query()->where('id', $id)->first();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a blacklist.
|
||||
*
|
||||
* @param int $id The ID of the blacklist.
|
||||
*/
|
||||
public function deleteBlacklist(int $id): void
|
||||
{
|
||||
BinaryBlacklist::query()->where('id', $id)->delete();
|
||||
}
|
||||
|
||||
public function updateBlacklist(array $blacklistArray): void
|
||||
{
|
||||
BinaryBlacklist::query()->where('id', $blacklistArray['id'])->update(
|
||||
[
|
||||
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']),
|
||||
'regex' => $blacklistArray['regex'],
|
||||
'status' => $blacklistArray['status'],
|
||||
'description' => $blacklistArray['description'],
|
||||
'optype' => $blacklistArray['optype'],
|
||||
'msgcol' => $blacklistArray['msgcol'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new blacklist from binary blacklist edit admin web page.
|
||||
*/
|
||||
public function addBlacklist(array $blacklistArray): void
|
||||
{
|
||||
BinaryBlacklist::query()->insert(
|
||||
[
|
||||
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']),
|
||||
'regex' => $blacklistArray['regex'],
|
||||
'status' => $blacklistArray['status'],
|
||||
'description' => $blacklistArray['description'],
|
||||
'optype' => $blacklistArray['optype'],
|
||||
'msgcol' => $blacklistArray['msgcol'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete Collections/Binaries/Parts for a Collection ID.
|
||||
*
|
||||
* @param int $collectionID Collections table ID
|
||||
*
|
||||
* @note A trigger automatically deletes the parts/binaries.
|
||||
*
|
||||
* @throws \Throwable
|
||||
*/
|
||||
public function delete(int $collectionID): void
|
||||
{
|
||||
DB::transaction(static function () use ($collectionID) {
|
||||
DB::delete(sprintf('DELETE FROM collections WHERE id = %d', $collectionID));
|
||||
}, 10);
|
||||
|
||||
Collection::query()->where('id', $collectionID)->delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Log / Echo message.
|
||||
*
|
||||
@@ -1494,7 +1321,7 @@ class Binaries
|
||||
|
||||
private function getFileCount($subject): array
|
||||
{
|
||||
if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $subject, $fileCount)) {
|
||||
if (! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])[\s$:]/i', $subject, $fileCount)) {
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Admin;
|
||||
|
||||
use App\Http\Controllers\BasePageController;
|
||||
use App\Models\Category;
|
||||
use App\Services\BlacklistService;
|
||||
use Blacklight\Binaries;
|
||||
use Illuminate\Http\Request;
|
||||
|
||||
@@ -15,11 +16,11 @@ class AdminBlacklistController extends BasePageController
|
||||
public function index(): void
|
||||
{
|
||||
$this->setAdminPrefs();
|
||||
$binaries = new Binaries;
|
||||
$svc = new BlacklistService;
|
||||
|
||||
$meta_title = $title = 'Binary Black/White List';
|
||||
|
||||
$binlist = $binaries->getBlacklist(false);
|
||||
$binlist = $svc->getBlacklist(false);
|
||||
$this->smarty->assign('binlist', $binlist);
|
||||
|
||||
$content = $this->smarty->fetch('binaryblacklist-list.tpl');
|
||||
@@ -35,7 +36,7 @@ class AdminBlacklistController extends BasePageController
|
||||
public function edit(Request $request)
|
||||
{
|
||||
$this->setAdminPrefs();
|
||||
$binaries = new Binaries(['Settings' => null]);
|
||||
$svc = new BlacklistService;
|
||||
$error = '';
|
||||
$regex = ['id' => '', 'groupname' => '', 'regex' => '', 'description' => '', 'msgcol' => 1, 'status' => 1, 'optype' => 1];
|
||||
$meta_title = $title = 'Binary Black/White list';
|
||||
@@ -53,9 +54,9 @@ class AdminBlacklistController extends BasePageController
|
||||
}
|
||||
|
||||
if (empty($request->input('id'))) {
|
||||
$binaries->addBlacklist($request->all());
|
||||
$svc->addBlacklist($request->all());
|
||||
} else {
|
||||
$binaries->updateBlacklist($request->all());
|
||||
$svc->updateBlacklist($request->all());
|
||||
}
|
||||
|
||||
return redirect()->to('admin/binaryblacklist-list');
|
||||
@@ -76,7 +77,7 @@ class AdminBlacklistController extends BasePageController
|
||||
default:
|
||||
if ($request->has('id')) {
|
||||
$title = 'Binary Black/Whitelist Edit';
|
||||
$regex = $binaries->getBlacklistByID($request->input('id'));
|
||||
$regex = $svc->getBlacklistByID((int) $request->input('id'));
|
||||
} else {
|
||||
$title = 'Binary Black/Whitelist Add';
|
||||
$regex += [
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Models\BinaryBlacklist;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class BlacklistService
|
||||
{
|
||||
/**
|
||||
* Cache of lists per group name.
|
||||
*/
|
||||
private array $blackList = [];
|
||||
|
||||
private array $whiteList = [];
|
||||
|
||||
private array $listsFound = [];
|
||||
|
||||
/**
|
||||
* Track blacklist IDs to update last_activity for.
|
||||
*/
|
||||
private array $idsToUpdate = [];
|
||||
|
||||
/**
|
||||
* Check if an article (OVER header) is blacklisted for the given group.
|
||||
*/
|
||||
public function isBlackListed(array $msg, string $groupName): bool
|
||||
{
|
||||
if (! isset($this->listsFound[$groupName])) {
|
||||
$this->retrieveLists($groupName);
|
||||
}
|
||||
if (! $this->listsFound[$groupName]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$blackListed = false;
|
||||
$field = [
|
||||
\Blacklight\Binaries::BLACKLIST_FIELD_SUBJECT => $msg['Subject'] ?? '',
|
||||
\Blacklight\Binaries::BLACKLIST_FIELD_FROM => $msg['From'] ?? '',
|
||||
\Blacklight\Binaries::BLACKLIST_FIELD_MESSAGEID => $msg['Message-ID'] ?? '',
|
||||
];
|
||||
|
||||
// Whitelist first: if any whitelist matches, allow; otherwise treat as blacklisted.
|
||||
if ($this->whiteList[$groupName]) {
|
||||
$blackListed = true;
|
||||
foreach ($this->whiteList[$groupName] as $whiteList) {
|
||||
if (@preg_match('/'.$whiteList->regex.'/i', $field[$whiteList->msgcol])) {
|
||||
$blackListed = false;
|
||||
$this->idsToUpdate[$whiteList->id] = $whiteList->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (! $blackListed && $this->blackList[$groupName]) {
|
||||
foreach ($this->blackList[$groupName] as $blackList) {
|
||||
if (@preg_match('/'.$blackList->regex.'/i', $field[$blackList->msgcol])) {
|
||||
$blackListed = true;
|
||||
$this->idsToUpdate[$blackList->id] = $blackList->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $blackListed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get and reset collected blacklist IDs that matched during checks.
|
||||
*/
|
||||
public function getAndClearIdsToUpdate(): array
|
||||
{
|
||||
$ids = array_values($this->idsToUpdate);
|
||||
$this->idsToUpdate = [];
|
||||
|
||||
return $ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update last_activity timestamp for given blacklist IDs.
|
||||
*/
|
||||
public function updateBlacklistUsage(array $ids): void
|
||||
{
|
||||
if (empty($ids)) {
|
||||
return;
|
||||
}
|
||||
BinaryBlacklist::query()->whereIn('id', $ids)->update(['last_activity' => now()]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query blacklists from DB.
|
||||
*/
|
||||
public function getBlacklist(bool $activeOnly = true, int|string $opType = -1, string $groupName = '', bool $groupRegex = false): array
|
||||
{
|
||||
$opTypeSql = match ($opType) {
|
||||
\Blacklight\Binaries::OPTYPE_BLACKLIST => 'AND bb.optype = '.\Blacklight\Binaries::OPTYPE_BLACKLIST,
|
||||
\Blacklight\Binaries::OPTYPE_WHITELIST => 'AND bb.optype = '.\Blacklight\Binaries::OPTYPE_WHITELIST,
|
||||
default => '',
|
||||
};
|
||||
|
||||
$joinOperator = $groupRegex ? 'REGEXP' : '=';
|
||||
$activeSql = $activeOnly ? 'AND bb.status = 1' : '';
|
||||
$groupSql = $groupName ? ('AND g.name REGEXP '.escapeString($groupName)) : '';
|
||||
|
||||
$sql = "
|
||||
SELECT
|
||||
bb.id, bb.optype, bb.status, bb.description,
|
||||
bb.groupname AS groupname, bb.regex, g.id AS group_id, bb.msgcol,
|
||||
bb.last_activity as last_activity
|
||||
FROM binaryblacklist bb
|
||||
LEFT OUTER JOIN usenet_groups g ON g.name $joinOperator bb.groupname
|
||||
WHERE 1=1 $activeSql $opTypeSql $groupSql
|
||||
ORDER BY coalesce(groupname,'zzz')";
|
||||
|
||||
return DB::select($sql);
|
||||
}
|
||||
|
||||
public function getBlacklistByID(int $id)
|
||||
{
|
||||
return BinaryBlacklist::query()->where('id', $id)->first();
|
||||
}
|
||||
|
||||
public function deleteBlacklist(int $id): void
|
||||
{
|
||||
BinaryBlacklist::query()->where('id', $id)->delete();
|
||||
}
|
||||
|
||||
public function updateBlacklist(array $blacklistArray): void
|
||||
{
|
||||
BinaryBlacklist::query()->where('id', $blacklistArray['id'])->update(
|
||||
[
|
||||
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']),
|
||||
'regex' => $blacklistArray['regex'],
|
||||
'status' => $blacklistArray['status'],
|
||||
'description' => $blacklistArray['description'],
|
||||
'optype' => $blacklistArray['optype'],
|
||||
'msgcol' => $blacklistArray['msgcol'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
public function addBlacklist(array $blacklistArray): void
|
||||
{
|
||||
BinaryBlacklist::query()->insert(
|
||||
[
|
||||
'groupname' => $blacklistArray['groupname'] === '' ? 'null' : preg_replace('/a\.b\./i', 'alt.binaries.', $blacklistArray['groupname']),
|
||||
'regex' => $blacklistArray['regex'],
|
||||
'status' => $blacklistArray['status'],
|
||||
'description' => $blacklistArray['description'],
|
||||
'optype' => $blacklistArray['optype'],
|
||||
'msgcol' => $blacklistArray['msgcol'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
private function retrieveLists(string $groupName): void
|
||||
{
|
||||
if (! isset($this->blackList[$groupName])) {
|
||||
$this->blackList[$groupName] = $this->getBlacklist(true, \Blacklight\Binaries::OPTYPE_BLACKLIST, $groupName, true);
|
||||
}
|
||||
if (! isset($this->whiteList[$groupName])) {
|
||||
$this->whiteList[$groupName] = $this->getBlacklist(true, \Blacklight\Binaries::OPTYPE_WHITELIST, $groupName, true);
|
||||
}
|
||||
$this->listsFound[$groupName] = ($this->blackList[$groupName] || $this->whiteList[$groupName]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
class XrefService
|
||||
{
|
||||
/**
|
||||
* Regex patterns to parse Xref tokens (e.g. alt.binaries.* optionally followed by :number).
|
||||
*/
|
||||
private const XREF_PATTERN_WITH_NUM = '/(^[a-zA-Z]{2,3}\\.(bin(aries|arios|aer))\\.[a-zA-Z0-9]?.+)(:\\d+)/';
|
||||
|
||||
private const XREF_PATTERN_NO_NUM = '/(^[a-zA-Z]{2,3}\\.(bin(aries|arios|aer))\\.[a-zA-Z0-9]?.+)/';
|
||||
|
||||
/**
|
||||
* Extracts valid Xref tokens from a space-separated Xref string.
|
||||
*/
|
||||
public function extractTokens(?string $xref): array
|
||||
{
|
||||
if (empty($xref)) {
|
||||
return [];
|
||||
}
|
||||
$tokens = [];
|
||||
foreach (explode(' ', $xref) as $token) {
|
||||
if (preg_match(self::XREF_PATTERN_WITH_NUM, $token, $m) || preg_match(self::XREF_PATTERN_NO_NUM, $token, $m)) {
|
||||
$tokens[] = $m[0];
|
||||
}
|
||||
}
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns tokens that appear in $headerXref but not in $existingXref.
|
||||
*/
|
||||
public function diffNewTokens(?string $existingXref, ?string $headerXref): array
|
||||
{
|
||||
$existing = $this->extractTokens($existingXref);
|
||||
$incoming = $this->extractTokens($headerXref);
|
||||
|
||||
return array_values(array_diff($incoming, $existing));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user