starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove code that handles requestids

Browse Source
This commit is contained in:
DariusIII
2017-12-19 11:14:51 +01:00
parent 678181d298
commit 220fabb7d8
12 changed files with 10 additions and 1065 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Changelog
+1
View File
@@ -1,4 +1,5 @@
2017-12-19 DariusIII
* Chg: Remove code that handles requestids
* Chg: Update ParHash model and query in ReleaseFiles
* Chg: Possible fix for Carbon error
2017-12-18 DariusIII
database/migrations/2017_10_12_124141_create_releases_table.php
-2
View File
@@ -54,7 +54,6 @@ class CreateReleasesTable extends Migration
$table->boolean('iscategorized')->default(0);
$table->boolean('isrenamed')->default(0);
$table->boolean('ishashed')->default(0);
$table->boolean('isrequestid')->default(0);
$table->boolean('proc_pp')->default(0);
$table->boolean('proc_sorter')->default(0);
$table->boolean('proc_par2')->default(0);
@@ -74,7 +73,6 @@ hash16k processed');
$table->index(['haspreview', 'passwordstatus'], 'ix_releases_haspreview_passwordstatus');
$table->index(['nfostatus', 'size'], 'ix_releases_nfostatus');
$table->index(['dehashstatus', 'ishashed'], 'ix_releases_dehashstatus');
$table->index(['adddate', 'reqidstatus', 'isrequestid'], 'ix_releases_reqidstatus');
});
}
database/migrations/2017_10_12_135623_create_triggers.php
-4
View File
@@ -15,8 +15,6 @@ class CreateTriggers extends Migration
BEGIN
IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}'
THEN SET NEW.ishashed = 1;
ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\['
THEN SET NEW.isrequestid = 1;
END IF;
END;
@@ -24,8 +22,6 @@ CREATE TRIGGER check_update BEFORE UPDATE ON releases FOR EACH ROW
BEGIN
IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}'
THEN SET NEW.ishashed = 1;
ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\['
THEN SET NEW.isrequestid = 1;
END IF;
END;
misc/update/multiprocessing/.do_not_run/switch.php
-9
View File
@@ -17,7 +17,6 @@ use nntmux\Groups;
use nntmux\Nfo;
use nntmux\NNTP;
use nntmux\processing\ProcessReleasesMultiGroup;
use nntmux\RequestIDLocal;
// Are we coming from python or php ? $options[0] => (string): python|php
// The type of process we want to do: $options[1] => (string): releases
@@ -179,14 +178,6 @@ switch ($options[1]) {
}
break;
// Process all local requestid for a single group.
// $options[2] => (int)groupid, group to work on
case 'requestid':
if (is_numeric($options[2])) {
(new RequestIDLocal(['Echo' => true]))->lookupRequestIDs(['GroupID' => $options[2], 'limit' => 5000]);
}
break;
/* Update a single group's article headers.
*
* $options[2] => (string) Group name.
nntmux/RequestID.php
-245
View File
@@ -1,245 +0,0 @@
<?php
namespace nntmux;
use nntmux\db\DB;
use GuzzleHttp\Client;
/**
* Class RequestID.
*/
abstract class RequestID
{
// Request id.
const REQID_OLD = -4; // We rechecked the web a second time and didn't find a title so don't process it again.
const REQID_NONE = -3; // The Request id was not found locally or via web lookup.
const REQID_ZERO = -2; // The Request id was 0.
const REQID_NOLL = -1; // Request id was not found via local lookup.
const REQID_UPROC = 0; // Release has not been processed.
const REQID_FOUND = 1; // Request id found and release was updated.
const IS_REQID_TRUE = 1; // releases.isrequestid is 1
const IS_REQID_FALSE = 0; // releases.isrequestid is 0
/**
* @var Groups
*/
public $groups;
/**
* @var Client
*/
public $client;
/**
* @param array $options Class instances / Echo to cli?
*/
public function __construct(array $options = [])
{
$defaults = [
'Echo' => true,
'Categorize' => null,
'ConsoleTools' => null,
'Groups' => null,
'Settings' => null,
'SphinxSearch' => null,
];
$options += $defaults;
$this->echoOutput = ($options['Echo'] && NN_ECHOCLI);
$this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB());
$this->category = ($options['Categorize'] instanceof Categorize ? $options['Categorize'] : new Categorize(['Settings' => $this->pdo]));
$this->groups = ($options['Groups'] instanceof Groups ? $options['Groups'] : new Groups(['Settings' => $this->pdo]));
$this->consoleTools = ($options['ConsoleTools'] instanceof ConsoleTools ? $options['ConsoleTools'] : new ConsoleTools(['ColorCLI' => $this->pdo->log]));
$this->sphinx = ($options['SphinxSearch'] instanceof SphinxSearch ? $options['SphinxSearch'] : new SphinxSearch());
$this->client = new Client();
}
/**
* Look up request id's for releases.
*
* @param array $options
*
* @return int Quantity of releases matched to a request id.
*/
public function lookupRequestIDs(array $options = [])
{
$curOptions = [
'charGUID' => '',
'GroupID' => '',
'limit' => '',
'show' => 1,
'time' => 0,
];
$curOptions = array_replace($curOptions, $options);
$startTime = time();
$renamed = 0;
$this->_charGUID = $curOptions['charGUID'];
$this->_groupID = $curOptions['GroupID'];
$this->_show = $curOptions['show'];
$this->_maxTime = $curOptions['time'];
$this->_limit = $curOptions['limit'];
$this->_getReleases();
if ($this->_releases !== false && $this->_releases->rowCount() > 0) {
$this->_totalReleases = $this->_releases->rowCount();
ColorCLI::doEcho(ColorCLI::primary('Processing '.$this->_totalReleases." releases for RequestID's."));
$renamed = $this->_processReleases();
if ($this->echoOutput) {
echo ColorCLI::header(
"\nRenamed ".number_format($renamed).' releases in '.$this->consoleTools->convertTime(time() - $startTime).'.'
);
}
} elseif ($this->echoOutput) {
ColorCLI::doEcho(ColorCLI::primary("No RequestID's to process."));
}
return $renamed;
}
/**
* Fetch releases with requestid's from MySQL.
*/
protected function _getReleases()
{
}
/**
* Process releases for requestid's.
*
* @return int How many did we rename?
*/
protected function _processReleases()
{
}
/**
* No request id was found, update the release.
*
* @param int $releaseID
* @param int $status
*/
protected function _requestIdNotFound($releaseID, $status)
{
if ($releaseID == 0) {
return;
}
$this->pdo->queryExec(
sprintf('
UPDATE releases SET reqidstatus = %d WHERE id = %d',
$status, $releaseID
)
);
}
/**
* Get a new title / pre id for a release.
*
* @return array|bool
*/
protected function _getNewTitle()
{
}
/**
* Find a RequestID in a usenet subject.
*
* @return int
*/
protected function _siftReqId()
{
$requestID = [];
switch (true) {
case preg_match('/\[\s*#?scnzb@?efnet\s*\]\[(\d+)\]/', $this->_release['name'], $requestID):
case preg_match('/\[\s*(\d+)\s*\]/', $this->_release['name'], $requestID):
case preg_match('/^REQ\s*(\d{4,6})/i', $this->_release['name'], $requestID):
case preg_match('/^(\d{4,6})-\d{1}\[/', $this->_release['name'], $requestID):
case preg_match('/(\d{4,6}) -/', $this->_release['name'], $requestID):
if ((int) $requestID[1] > 0) {
return (int) $requestID[1];
}
}
return self::REQID_ZERO;
}
/**
* @var bool Echo to CLI?
*/
protected $echoOutput;
/**
* @var Categorize
*/
protected $category;
/**
* @var \nntmux\db\DB
*/
protected $pdo;
/**
* @var ConsoleTools
*/
protected $consoleTools;
/**
* @var ColorCLI
*/
protected $colorCLI;
/**
* The found request id for the release.
* @var int
*/
protected $_requestID = self::REQID_ZERO;
/**
* The title found from a request id lookup.
* @var bool|string|array
*/
protected $_newTitle = false;
/**
* Releases with potential Request id's we can work on.
* @var \PDOStatement|bool
*/
protected $_releases;
/**
* Total amount of releases we will be working on.
* @var int
*/
protected $_totalReleases;
/**
* Release we are currently working on.
* @var array
*/
protected $_release;
/**
* @var int To show the result or not.
*/
protected $_show = 0;
/**
* GroupID, which is optional, to limit query results.
* @var string
*/
protected $_groupID;
/**
* First character of a release GUID, which is optional, to limit query results.
* @var string
*/
protected $_charGUID;
protected $_limit;
protected $_maxTime;
}
nntmux/RequestIDLocal.php
-305
View File
@@ -1,305 +0,0 @@
<?php
namespace nntmux;
/**
* Attempts to find a PRE name for a release using a request id from our local pre database,
* or internet request id database using a Standalone -- more intensive methods.
*
* Class RequestIDLocal
*/
class RequestIDLocal extends RequestID
{
/**
* @param array $options Class instances / Echo to cli?
*/
public function __construct(array $options = [])
{
parent::__construct($options);
}
/**
* Fetch releases with requestid's from MySQL.
*/
protected function _getReleases()
{
$query =
'SELECT r.id, r.name, r.fromname, r.categories_id, r.reqidstatus, g.name AS groupname, g.id as gid
FROM releases r
LEFT JOIN groups g ON r.groups_id = g.id
WHERE r.nzbstatus = 1
AND r.predb_id = 0
AND r.isrequestID = 1';
$query .= ($this->_charGUID === '' ? '' : ' AND r.leftguid = '.$this->pdo->escapeString($this->_charGUID));
$query .= ($this->_groupID === '' ? '' : ' AND r.groups_id = '.$this->_groupID);
$query .= ($this->_maxTime === 0 ? '' : sprintf(' AND r.adddate > NOW() - INTERVAL %d HOUR', $this->_maxTime));
switch ($this->_limit) {
case 'full':
$query .= sprintf(
' AND r.reqidstatus in (%d, %d, %d)',
self::REQID_UPROC,
self::REQID_NOLL,
self::REQID_NONE
);
break;
case is_numeric($this->_limit):
$query .= sprintf(
' AND r.reqidstatus in (%d, %d, %d) ORDER BY r.postdate DESC LIMIT %d',
self::REQID_UPROC,
self::REQID_NOLL,
self::REQID_NONE,
$this->_limit
);
break;
case 'all':
default:
break;
}
$this->_releases = $this->pdo->queryDirect($query);
}
/**
* Process releases for requestid's.
*
* @return int How many did we rename?
*/
protected function _processReleases()
{
$renamed = $checked = 0;
if ($this->_releases instanceof \Traversable) {
foreach ($this->_releases as $this->_release) {
$this->_requestID = $this->_siftReqId();
// Do a local lookup using multiple possible methods
$this->_newTitle = $this->_getNewTitle();
if ($this->_newTitle !== false && isset($this->_newTitle['title'])) {
$this->_updateRelease();
$renamed++;
} else {
$this->_requestIdNotFound($this->_release['id'], ($this->_release['reqidstatus'] === self::REQID_UPROC ? self::REQID_NOLL : self::REQID_NONE));
}
if ($this->echoOutput && $this->_show === 0) {
$this->consoleTools->overWritePrimary(
'Checked Releases: ['.number_format($checked).'] '.
$this->consoleTools->percentString(++$checked, $this->_totalReleases)
);
}
}
}
return $renamed;
}
/**
* Get a new title / pre id for a release.
*
* @return array|bool
*/
protected function _getNewTitle()
{
if ($this->_requestID === -2) {
return $this->_multiLookup();
}
$check = $this->pdo->queryDirect(
sprintf(
'SELECT id, title FROM predb WHERE requestid = %d AND groups_id = %d',
$this->_requestID,
$this->_release['gid']
)
);
if ($check instanceof \Traversable) {
if ($check->rowCount() === 1) {
foreach ($check as $row) {
if (preg_match('/s\d+/i', $row['title']) && ! preg_match('/s\d+e\d+/i', $row['title'])) {
return false;
}
return ['title' => $row['title'], 'id' => $row['id']];
}
} else {
//Prevents multiple releases with the same request id/group from being renamed to the same Pre.
return $this->_multiLookup();
}
} else {
$result = $this->_singleAltLookup();
if (is_array($result) && is_numeric($result['id']) && $result['title'] !== '') {
return $result;
} else {
return $this->_multiLookup();
}
}
return false;
}
/**
* Sub function that attempts to match RequestID Releases
* by preg_matching the title from the usenet name.
*
* @return array|bool
*/
protected function _multiLookup()
{
$regex1 =
'/^\[\s*\d+\s*\][ -]+(\[(ISO|FULL|PART|MP3|0DAY|android)\][ -]+)?\[(alt-?bin| ?#?a[a-z0-9. -]+)((@?ef{1,2})?net)? ?\]'.
'[ -]+(\[(ISO|FULL|PART|MP3|0DAY|android)\][ -]+)?(\[\s*\d+\s*\][ -]+)?(\[\d+\/\d+\][ -]+)?(\"|\[)\s*'.
'(?P<title>.+?)(\.+(vol\d+\+\d+\.)?(-cd\d\.)?(avi|jpg|nzb|m3u|mkv|par2|part\d+|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)?\s*(\"|\])'.
'[ -]*(\[\d+\/\d+\][ -]*)?((\"\s*(?P<filename1>.+?)([-.]sample)?([-.]cd(\d|[ab]))?(\.+(vol\d+\+\d+\.)?([-.]d\d\.)?([-.]part\d+)?'.
'(avi|jpg|nzb|m3u|mkv|par2|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)?\s*\")| - (?P<filename2>.+?) (yEnc|\(\d+\/\d+\)))?.*/i';
$regex2 =
'/^\[\s*\d+\s*\].*'.
'\"\s*(?P<title>.+?)(\.+(vol\d+\+\d+\.)?(-cd\d\.)?'.
'(avi|jpg|nzb|m3u|mkv|par2|part\d+|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)\s*\".*/i';
$matches = [];
switch (true) {
case preg_match($regex1, $this->_release['name'], $matches):
case preg_match($regex2, $this->_release['name'], $matches):
$check = $this->pdo->queryOneRow(
sprintf(
'SELECT id, title FROM predb WHERE title = %s OR filename = %s %s',
$this->pdo->escapeString($matches['title']),
$this->pdo->escapeString($matches['title']),
(
isset($matches['filename1']) && $matches['filename1'] !== ''
? 'OR filename = '.$this->pdo->escapeString($matches['filename1'])
:
(
isset($matches['filename2']) && $matches['filename2'] !== ''
? 'OR filename = '.$this->pdo->escapeString($matches['filename2'])
: ''
)
)
)
);
if ($check !== false) {
return ['title' => $check['title'], 'id' => $check['id']];
}
continue;
default:
return false;
}
return false;
}
private $groupIDCache = [];
/**
* Attempts to remap the release groups_id by extracting the new group name from the release usenet name.
*
* @return array|bool
*/
protected function _singleAltLookup()
{
switch (true) {
case $this->_release['name'] === 'alt.binaries.etc':
$groupName = 'alt.binaries.teevee';
break;
case strpos($this->_release['name'], 'teevee') !== false:
$groupName = 'alt.binaries.teevee';
break;
case strpos($this->_release['name'], 'moovee') !== false:
$groupName = 'alt.binaries.moovee';
break;
case strpos($this->_release['name'], 'erotica') !== false:
$groupName = 'alt.binaries.erotica';
break;
case strpos($this->_release['name'], 'foreign') !== false:
$groupName = 'alt.binaries.mom';
break;
case strpos($this->_release['name'], 'inner-sanctum') !== false:
$groupName = 'alt.binaries.inner-sanctum';
break;
case strpos($this->_release['name'], 'sounds.flac') !== false:
$groupName = 'alt.binaries.sounds.flac';
break;
case strpos($this->_release['name'], 'scnzb') !== false:
$groupName = 'alt.binaries.boneless';
break;
case strpos($this->_release['name'], 'hdtv.x264') !== false:
$groupName = 'alt.binaries.hdtv.x264';
break;
default:
return false;
}
if (isset($this->groupIDCache[$groupName])) {
$groupID = $this->groupIDCache[$groupName];
} else {
$groupID = $this->groups->getIDByName($groupName);
}
$check = $this->pdo->queryOneRow(
sprintf('
SELECT id, title FROM predb WHERE requestid = %d AND groups_id = %d',
$this->_requestID,
($groupID === '' ? 0 : $groupID)
)
);
if ($check !== false) {
return ['title' => $check['title'], 'id' => $check['id']];
}
return false;
}
/**
* Updates release information when a proper Request id match is found.
*/
protected function _updateRelease()
{
$determinedCat = $this->category->determineCategory($this->_release['gid'], $this->_newTitle['title'], $this->_release['fromname']);
if ($determinedCat === $this->_release['categories_id']) {
$newTitle = $this->pdo->escapeString($this->_newTitle['title']);
$this->pdo->queryExec(
sprintf('
UPDATE releases
SET predb_id = %d, reqidstatus = %d, isrenamed = 1, iscategorized = 1, searchname = %s
WHERE id = %d',
$this->_newTitle['id'],
self::REQID_FOUND,
$newTitle,
$this->_release['id']
)
);
$this->sphinx->updateRelease($this->_release['id'], $this->pdo);
} else {
$newTitle = $this->pdo->escapeString($this->_newTitle['title']);
$this->pdo->queryExec(
sprintf('
UPDATE releases SET
videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfo_id = NULL, consoleinfo_id = NULL,
bookinfo_id = NULL, anidbid = NULL, predb_id = %d, reqidstatus = %d, isrenamed = 1,
iscategorized = 1, searchname = %s, categories_id = %d
WHERE id = %d',
$this->_newTitle['id'],
self::REQID_FOUND,
$newTitle,
$determinedCat,
$this->_release['id']
)
);
$this->sphinx->updateRelease($this->_release['id'], $this->pdo);
}
if ($this->_show === 1 && $this->_release['name'] !== $this->_newTitle['title']) {
NameFixer::echoChangedReleaseName(
[
'new_name' => $this->_newTitle['title'],
'old_name' => $this->_release['name'],
'new_category' => $this->category->getNameByID($determinedCat),
'old_category' => $this->category->getNameByID($this->_release['categories_id']),
'group' => $this->_release['groupname'],
'releases_id' => $this->_release['id'],
'method' => 'RequestIDLocal',
]
);
}
}
}
nntmux/RequestIDWeb.php
-330
View File
@@ -1,330 +0,0 @@
<?php
namespace nntmux;
use App\Models\Settings;
use GuzzleHttp\Exception\RequestException;
/**
* Attempts to find a PRE name for a release using a request id from our local pre database,
* or internet request id database.
*
* Class RequestIDWeb
*/
class RequestIDWeb extends RequestID
{
const MAX_WEB_LOOKUPS = 75; // Please don't exceed this, not to be to harsh on the Request id server.
/**
* The id of the PRE entry the found request id belongs to.
* @var bool|int
*/
protected $_preDbID = false;
/**
* @var int
*/
protected $_request_hours;
/**
* Construct.
*
* @param array $options Class instances / Echo to cli?
*/
public function __construct(array $options = [])
{
parent::__construct($options);
$this->_request_hours = (Settings::settingValue('..request_hours') != '') ? (int) Settings::settingValue('..request_hours') : 1;
}
/**
* Get all results from the releases table that have request id's to be processed.
*/
protected function _getReleases()
{
$this->_releases = $this->pdo->queryDirect(
sprintf('
SELECT r.id, r.name, r.searchname, r.fromname, g.name AS groupname, r.groups_id, r.categories_id
FROM releases r
LEFT JOIN groups g ON r.groups_id = g.id
WHERE r.nzbstatus = 1
AND r.predb_id = 0
AND r.isrequestid = 1
AND (
r.reqidstatus = %d
OR (r.reqidstatus = %d AND r.adddate < NOW() - INTERVAL %d HOUR)
)
%s %s %s
ORDER BY r.postdate+0 DESC
LIMIT %d',
self::REQID_NOLL,
self::REQID_NONE,
$this->_request_hours,
(empty($this->_groupID) ? '' : ('AND r.groups_id = '.$this->_groupID)),
$this->_getReqIdGroups(),
($this->_maxTime === 0 ? '' : sprintf(' AND r.adddate > NOW() - INTERVAL %d HOUR', $this->_maxTime)),
(empty($this->_limit) || $this->_limit > 1000 ? 1000 : $this->_limit)
)
);
}
/**
* Create "AND" part of query for request id groups.
* Less load on the request id web server, by limiting results.
*
* @return string
*/
protected function _getReqIdGroups()
{
return
"AND g.name IN (
'alt.binaries.boneless',
'alt.binaries.cd.image',
'alt.binaries.console.ps3',
'alt.binaries.erotica',
'alt.binaries.games.nintendods',
'alt.binaries.games.wii',
'alt.binaries.games.xbox360',
'alt.binaries.inner-sanctum',
'alt.binaries.mom',
'alt.binaries.moovee',
'alt.binaries.movies.divx',
'alt.binaries.sony.psp',
'alt.binaries.sounds.mp3.complete_cd',
'alt.binaries.sounds.flac',
'alt.binaries.teevee',
'alt.binaries.warez',".
// Extra groups we will need to remap later, etc is teevee for example.
"'alt.binaries.etc'
)";
}
/**
* Process releases for requestid's.
*
* @return int How many did we rename?
*/
protected function _processReleases()
{
// Array to store results.
$requestArray = [];
if ($this->_releases instanceof \Traversable) {
// Loop all the results.
foreach ($this->_releases as $release) {
$this->_release['name'] = $release['name'];
// Try to find a request id for the release.
$requestId = $this->_siftReqId();
// If there's none, update the release and continue.
if ($requestId === self::REQID_ZERO) {
$this->_requestIdNotFound($release['id'], self::REQID_NONE);
if ($this->echoOutput) {
echo '-';
}
continue;
}
// Change etc to teevee.
if ($release['groupname'] === 'alt.binaries.etc') {
$release['groupname'] = 'alt.binaries.teevee';
}
// Send the release id so we can track the return data.
$requestArray[$release['id']] = [
'reqid' => $requestId,
'ident' => $release['id'],
'group' => $release['groupname'],
'sname' => $release['searchname'],
'fromname' => $release['fromname'],
];
}
}
// Check if we requests to send to the web.
if (count($requestArray) < 1) {
return 0;
}
// Mock array for isset check on server.
$requestArray[0] = ['ident' => 0, 'group' => 'none', 'reqid' => 0];
// Do a web lookup.
try {
$returnXml = $this->client->request('POST', Settings::settingValue('..request_url'),
['Link' => 'data='.serialize($requestArray)]
)->getBody();
} catch (RequestException $e) {
if ($e->hasResponse()) {
if ($e->getCode() === 404) {
ColorCLI::doEcho(ColorCLI::notice('Data not available on server'));
} elseif ($e->getCode() === 503) {
ColorCLI::doEcho(ColorCLI::notice('Service unavailable'));
} else {
ColorCLI::doEcho(ColorCLI::notice('Unable to fetch data, server responded with code: '.$e->getCode()));
}
}
}
$renamed = 0;
// Change the release titles and insert the PRE's if they don't exist.
if (isset($returnXml) && $returnXml !== false) {
$returnXml = @simplexml_load_string($returnXml);
if ($returnXml !== false) {
// Store the returned identifiers so we can check which releases we didn't find a request id.
$returnedIdentifiers = [];
$groupIDArray = [];
foreach ($returnXml->request as $result) {
if (isset($result['name'], $result['ident']) && (int) $result['ident'] > 0) {
$this->_newTitle['title'] = (string) $result['name'];
$this->_requestID = (int) $result['reqid'];
$this->_release['id'] = (int) $result['ident'];
// Buffer groupid queries.
$this->_release['groupname'] = $requestArray[(int) $result['ident']]['group'];
if (isset($groupIDarray[$this->_release['groupname']])) {
$this->_release['groups_id'] = $groupIDArray[$this->_release['groupname']];
} else {
$this->_release['groups_id'] = $this->groups->getIDByName($this->_release['groupname']);
$groupIDArray[$this->_release['groupname']] = $this->_release['groups_id'];
}
$this->_release['gid'] = $this->_release['groups_id'];
$this->_release['fromname'] = $requestArray[(string) $result['ident']]['fromname'];
$this->_release['searchname'] = $requestArray[(int) $result['ident']]['sname'];
$this->_insertIntoPreDB();
if ($this->_preDbID === false) {
$this->_preDbID = 0;
}
$this->_newTitle['id'] = $this->_preDbID;
$this->_updateRelease();
$renamed++;
if ($this->echoOutput) {
echo '+';
}
$returnedIdentifiers[] = (string) $result['ident'];
}
}
// Check if the WEB didn't send back some titles, update the release.
if (count($returnedIdentifiers) > 0) {
foreach ($returnedIdentifiers as $identifier) {
if (array_key_exists($identifier, $requestArray)) {
unset($requestArray[$identifier]);
}
}
}
unset($requestArray[0]);
foreach ($requestArray as $request) {
$addDate = $this->pdo->queryOneRow(
sprintf(
'SELECT UNIX_TIMESTAMP(adddate) AS adddate FROM releases WHERE id = %d', $request['ident']
)
);
$status = self::REQID_NONE;
if ($addDate !== false && ! empty($addDate['adddate'])) {
if ((bool) (intval((time() - (int) $addDate['adddate']) / 3600) > $this->_request_hours)) {
$status = self::REQID_OLD;
}
} else {
$status = self::REQID_OLD;
}
$this->_requestIdNotFound(
$request['ident'],
$status
);
if ($this->echoOutput) {
echo '-';
}
}
}
}
return $renamed;
}
/**
* If we found a request id on the internet, check if our PRE database has it, insert it if not.
*/
protected function _insertIntoPreDB()
{
$dupeCheck = $this->pdo->queryOneRow(
sprintf('
SELECT id AS predb_id, requestid, groups_id
FROM predb
WHERE title = %s',
$this->pdo->escapeString($this->_newTitle['title'])
)
);
if ($dupeCheck === false) {
$this->_preDbID = (int) $this->pdo->queryInsert(
sprintf('
INSERT INTO predb (title, source, requestid, groups_id, predate)
VALUES (%s, %s, %d, %d, NOW())',
$this->pdo->escapeString($this->_newTitle['title']),
$this->pdo->escapeString('requestWEB'),
$this->_requestID,
$this->_release['groups_id']
)
);
} else {
$this->_preDbID = $dupeCheck['predb_id'];
$this->pdo->queryExec(
sprintf('
UPDATE predb
SET requestid = %d, groups_id = %d
WHERE id = %d',
$this->_requestID,
$this->_release['groups_id'],
$this->_preDbID
)
);
}
}
/**
* If we found a PRE name, update the releases name and reset post processing.
*/
protected function _updateRelease()
{
$determinedCategory = $this->category->determineCategory($this->_release['groups_id'], $this->_newTitle['title'], $this->_release['fromname']);
$newTitle = $this->pdo->escapeString($this->_newTitle['title']);
$this->pdo->queryExec(
sprintf('
UPDATE releases
SET videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfo_id = NULL, consoleinfo_id = NULL, bookinfo_id = NULL, anidbid = NULL,
reqidstatus = %d, isrenamed = 1, proc_files = 1, searchname = %s, categories_id = %d,
predb_id = %d
WHERE id = %d',
self::REQID_FOUND,
$newTitle,
$determinedCategory,
$this->_preDbID,
$this->_release['id']
)
);
$this->sphinx->updateRelease($this->_release['id'], $this->pdo);
if ($this->echoOutput) {
NameFixer::echoChangedReleaseName([
'new_name' => $this->_newTitle['title'],
'old_name' => $this->_release['searchname'],
'new_category' => $this->category->getNameByID($determinedCategory),
'old_category' => '',
'group' => $this->_release['groupname'],
'releases_id' => $this->_release['id'],
'method' => 'RequestID->updateRelease<web>',
]
);
}
}
}
nntmux/Tmux.php
+2 -13
View File
@@ -269,8 +269,6 @@ class Tmux
(%2\$s 'nzbthreads') AS nzbthreads,
(%2\$s 'tmpunrarpath') AS tmpunrar,
(%2\$s 'compressedheaders') AS compressed,
(%2\$s 'book_reqids') AS book_reqids,
(%2\$s 'request_hours') AS request_hours,
(%2\$s 'maxsizetopostprocess') AS maxsize_pp,
(%2\$s 'minsizetopostprocess') AS minsize_pp",
$tmuxstr,
@@ -462,7 +460,6 @@ class Tmux
/**
* @param $qry
* @param $bookreqids
* @param int $request_hours
* @param string $db_name
* @param string $ppmax
* @param string $ppmin
@@ -470,7 +467,7 @@ class Tmux
* @return bool|string
* @throws \Exception
*/
public function proc_query($qry, $bookreqids, $request_hours, $db_name, $ppmax = '', $ppmin = '')
public function proc_query($qry, $bookreqids, $db_name, $ppmax = '', $ppmin = '')
{
switch ((int) $qry) {
case 1:
@@ -491,8 +488,6 @@ class Tmux
OR (ishashed = 1 AND dehashstatus BETWEEN -6 AND 0)) AND categories_id IN (%s),1,0)) AS processrenames,
SUM(IF(isrenamed = %d,1,0)) AS renamed,
SUM(IF(nzbstatus = %1$d AND nfostatus = %20$d,1,0)) AS nfo,
SUM(IF(nzbstatus = %1$d AND isrequestid = %d AND predb_id = 0 AND ((reqidstatus = %d) OR (reqidstatus = %d) OR (reqidstatus = %d AND adddate > NOW() - INTERVAL %s HOUR)),1,0)) AS requestid_inprogress,
SUM(IF(predb_id > 0 AND nzbstatus = %1$d AND isrequestid = %28$d AND reqidstatus = %d,1,0)) AS requestid_matched,
SUM(IF(predb_id > 0,1,0)) AS predb_matched,
COUNT(DISTINCT(predb_id)) AS distinct_predb_matched
FROM releases r',
@@ -525,12 +520,6 @@ class Tmux
MiscSorter::PROC_SORTER_NONE,
Category::getCategoryOthersGroup(),
NameFixer::IS_RENAMED_DONE,
RequestID::IS_REQID_TRUE,
RequestID::REQID_UPROC,
RequestID::REQID_NOLL,
RequestID::REQID_NONE,
RequestID::REQID_FOUND,
$request_hours
);
case 2:
@@ -623,7 +612,7 @@ class Tmux
/**
* @throws \RuntimeException
*/
public function startRunning()
public function startRunning(): void
{
if ($this->isRunning() === false) {
TmuxModel::query()->where('setting', '=', 'running')->update(['value' => 1]);
nntmux/TmuxOutput.php
-14
View File
@@ -248,20 +248,6 @@ class TmuxOutput extends Tmux
$this->runVar['counts']['percent']['predb_matched']
)
);
$buffer .= sprintf(
$this->tmpMasks[4],
'RequestID',
sprintf(
'%s(%s)',
number_format($this->runVar['counts']['now']['requestid_inprogress']),
$this->runVar['counts']['diff']['requestid_inprogress']
),
sprintf(
'%s(%d%%)',
number_format($this->runVar['counts']['now']['requestid_matched']),
$this->runVar['counts']['percent']['requestid_matched']
)
);
$buffer .= sprintf(
$this->tmpMasks[4],
'Renames',
nntmux/libraries/Forking.php
-35
View File
@@ -9,7 +9,6 @@ use nntmux\db\DB;
use Carbon\Carbon;
use App\Models\Tmux;
use nntmux\ColorCLI;
use nntmux\RequestID;
use App\Models\Settings;
use nntmux\processing\PostProcess;
@@ -256,10 +255,6 @@ class Forking extends \fork_daemon
$maxProcesses = $this->postProcessTvMainMethod();
break;
case 'request_id':
$maxProcesses = $this->requestIDMainMethod();
break;
case 'safe_backfill':
$maxProcesses = $this->safeBackfillMainMethod();
break;
@@ -973,36 +968,6 @@ class Forking extends \fork_daemon
$postProcess->processXXX();
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////// All requestID code goes here ////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
* @return null|string
* @throws \Exception
*/
private function requestIDMainMethod()
{
$this->register_child_run([0 => $this, 1 => 'requestIDChildWorker']);
$this->work = $this->pdo->query(
sprintf(
'
SELECT DISTINCT(g.id)
FROM groups g
INNER JOIN releases r ON r.groups_id = g.id
WHERE (g.active = 1 OR g.backfill = 1)
AND r.nzbstatus = %d
AND r.predb_id = 0
AND r.isrequestid = 1
AND r.reqidstatus = %d',
NZB::NZB_ADDED,
RequestID::REQID_UPROC
)
);
return (int) Settings::settingValue('..reqidthreads');
}
/**
* @param $groups
* @param string $identifier
nntmux/processing/ProcessReleases.php
+6 -101
View File
@@ -16,8 +16,6 @@ use App\Models\Release;
use App\Models\Settings;
use nntmux\ConsoleTools;
use nntmux\ReleaseImage;
use nntmux\RequestIDWeb;
use nntmux\RequestIDLocal;
use nntmux\ReleaseCleaning;
use App\Models\ReleaseRegex;
use App\Models\ReleasesGroups;
@@ -62,11 +60,6 @@ class ProcessReleases
*/
public $completion;
/**
* @var int
*/
public $processRequestIDs;
/**
* @var bool
*/
@@ -210,35 +203,12 @@ class ProcessReleases
$this->processCollectionSizes($groupID);
$this->deleteUnwantedCollections($groupID);
//$DIR = NN_MISC;
$totalReleasesAdded = 0;
do {
$releasesCount = $this->createReleases($groupID);
$totalReleasesAdded += $releasesCount['added'];
$nzbFilesAdded = $this->createNZBs($groupID);
// requestid lookups disabled because they are no longer being posted
/*if ($this->processRequestIDs === 0) {
$this->processRequestIDs($groupID, 5000, true);
} elseif ($this->processRequestIDs === 1) {
$this->processRequestIDs($groupID, 5000, true);
$this->processRequestIDs($groupID, 1000, false);
} elseif ($this->processRequestIDs === 2) {
$requestIDTime = time();
if ($this->echoCLI) {
ColorCLI::doEcho(ColorCLI::header('Process Releases -> Request ID Threaded lookup.'));
}
passthru("${DIR}update/multiprocessing/requestid.php");
if ($this->echoCLI) {
ColorCLI::doEcho(
ColorCLI::primary(
"\nReleases updated in ".
$this->consoleTools->convertTime(time() - $requestIDTime)
)
);
}
} */
$this->categorizeReleases($categorize, $groupID);
$this->postProcessReleases($postProcess, $nntp);
@@ -628,14 +598,13 @@ class ProcessReleases
$collection['gname']
);
if (is_array($cleanedName)) {
if (\is_array($cleanedName)) {
$properName = $cleanedName['properlynamed'];
$preID = $cleanerName['predb'] ?? false;
$isReqID = $cleanerName['requestid'] ?? false;
$cleanedName = $cleanedName['cleansubject'];
} else {
$properName = true;
$isReqID = $preID = false;
$preID = false;
}
if ($preID === false && $cleanedName !== '') {
@@ -660,7 +629,6 @@ class ProcessReleases
'size' => $collection['filesize'],
'categories_id' => $categorize->determineCategory($collection['groups_id'], $cleanedName),
'isrenamed' => $properName === true ? 1 : 0,
'reqidstatus' => $isReqID === true ? 1 : 0,
'predb_id' => $preID === false ? 0 : $preID,
'nzbstatus' => NZB::NZB_NONE,
]
@@ -780,6 +748,7 @@ class ProcessReleases
* @param int|string $groupID (optional)
*
* @return int
* @throws \RuntimeException
*/
public function createNZBs($groupID): int
{
@@ -836,70 +805,6 @@ class ProcessReleases
return $nzbCount;
}
/**
* Process RequestID's.
*
* @param int|string $groupID
* @param int $limit
* @param bool $local
*
* @void
* @throws \Exception
*/
public function processRequestIDs($groupID = '', $limit = 5000, $local = true): void
{
if ($local === false && (int) Settings::settingValue('..lookup_reqids') === 0) {
return;
}
$startTime = time();
if ($this->echoCLI) {
ColorCLI::doEcho(
ColorCLI::header(
sprintf(
'Process Releases -> Request ID %s lookup -- limit %s',
($local === true ? 'local' : 'web'),
$limit
)
)
);
}
if ($local === true) {
$foundRequestIDs = (
new RequestIDLocal(
[
'Echo' => $this->echoCLI,
'ConsoleTools' => $this->consoleTools,
'Groups' => $this->groups,
'Settings' => $this->pdo,
]
)
)->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]);
} else {
$foundRequestIDs = (
new RequestIDWeb(
[
'Echo' => $this->echoCLI,
'ConsoleTools' => $this->consoleTools,
'Groups' => $this->groups,
'Settings' => $this->pdo,
]
)
)->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]);
}
if ($this->echoCLI) {
ColorCLI::doEcho(
ColorCLI::primary(
number_format($foundRequestIDs).
' releases updated in '.
$this->consoleTools->convertTime(time() - $startTime)
),
true
);
}
}
/**
* Categorize releases.
*
@@ -1377,7 +1282,7 @@ class ProcessReleases
// Disabled categories.
$disabledCategories = $category->getDisabledIDs();
if (count($disabledCategories) > 0) {
if (\count($disabledCategories) > 0) {
foreach ($disabledCategories as $disabledCategory) {
$releases = $this->pdo->queryDirect(
sprintf('SELECT SQL_NO_CACHE id, guid FROM releases WHERE categories_id = %d', (int) $disabledCategory['id'])
@@ -1428,7 +1333,7 @@ class ProcessReleases
// Disabled music genres.
$genrelist = $genres->getDisabledIDs();
if (count($genrelist) > 0) {
if (\count($genrelist) > 0) {
foreach ($genrelist as $genre) {
$releases = $this->pdo->queryDirect(
sprintf(
@@ -1821,7 +1726,7 @@ class ProcessReleases
$where
)
);
if ($this->echoCLI && is_object($obj) && $obj->rowCount()) {
if ($this->echoCLI && \is_object($obj) && $obj->rowCount()) {
ColorCLI::doEcho(
ColorCLI::primary('Deleted '.$obj->rowCount().' broken/stuck collections.')
);
resources/db/schema/mysql-ddl.sql
+1 -7
View File
@@ -758,7 +758,6 @@ CREATE TABLE releases (
iscategorized TINYINT(1) NOT NULL DEFAULT '0',
isrenamed TINYINT(1) NOT NULL DEFAULT '0',
ishashed TINYINT(1) NOT NULL DEFAULT '0',
isrequestid TINYINT(1) NOT NULL DEFAULT '0',
proc_pp TINYINT(1) NOT NULL DEFAULT '0',
proc_sorter TINYINT(1) NOT NULL DEFAULT '0',
proc_par2 TINYINT(1) NOT NULL DEFAULT '0',
@@ -789,8 +788,7 @@ processed',
INDEX ix_releases_haspreview_passwordstatus (haspreview,passwordstatus),
INDEX ix_releases_passwordstatus (passwordstatus),
INDEX ix_releases_nfostatus (nfostatus,size),
INDEX ix_releases_dehashstatus (dehashstatus,ishashed),
INDEX ix_releases_reqidstatus (adddate,reqidstatus,isrequestid)
INDEX ix_releases_dehashstatus (dehashstatus,ishashed)
)
ENGINE = InnoDB
DEFAULT CHARSET = utf8
@@ -1445,8 +1443,6 @@ CREATE TRIGGER check_insert BEFORE INSERT ON releases FOR EACH ROW
BEGIN
IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}'
THEN SET NEW.ishashed = 1;
ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\['
THEN SET NEW.isrequestid = 1;
END IF;
END; $$
@@ -1454,8 +1450,6 @@ CREATE TRIGGER check_update BEFORE UPDATE ON releases FOR EACH ROW
BEGIN
IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}'
THEN SET NEW.ishashed = 1;
ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\['
THEN SET NEW.isrequestid = 1;
END IF;
END; $$