diff --git a/Changelog b/Changelog index c5d96f2d6..ee790a85c 100755 --- a/Changelog +++ b/Changelog @@ -1,4 +1,5 @@ 2017-12-19 DariusIII + * Chg: Remove code that handles requestids * Chg: Update ParHash model and query in ReleaseFiles * Chg: Possible fix for Carbon error 2017-12-18 DariusIII diff --git a/database/migrations/2017_10_12_124141_create_releases_table.php b/database/migrations/2017_10_12_124141_create_releases_table.php index d37c0fb1f..bd3b63638 100644 --- a/database/migrations/2017_10_12_124141_create_releases_table.php +++ b/database/migrations/2017_10_12_124141_create_releases_table.php @@ -54,7 +54,6 @@ class CreateReleasesTable extends Migration $table->boolean('iscategorized')->default(0); $table->boolean('isrenamed')->default(0); $table->boolean('ishashed')->default(0); - $table->boolean('isrequestid')->default(0); $table->boolean('proc_pp')->default(0); $table->boolean('proc_sorter')->default(0); $table->boolean('proc_par2')->default(0); @@ -74,7 +73,6 @@ hash16k processed'); $table->index(['haspreview', 'passwordstatus'], 'ix_releases_haspreview_passwordstatus'); $table->index(['nfostatus', 'size'], 'ix_releases_nfostatus'); $table->index(['dehashstatus', 'ishashed'], 'ix_releases_dehashstatus'); - $table->index(['adddate', 'reqidstatus', 'isrequestid'], 'ix_releases_reqidstatus'); }); } diff --git a/database/migrations/2017_10_12_135623_create_triggers.php b/database/migrations/2017_10_12_135623_create_triggers.php index 29a0e40da..08f1c4a15 100644 --- a/database/migrations/2017_10_12_135623_create_triggers.php +++ b/database/migrations/2017_10_12_135623_create_triggers.php @@ -15,8 +15,6 @@ class CreateTriggers extends Migration BEGIN IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}' THEN SET NEW.ishashed = 1; - ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\[' - THEN SET NEW.isrequestid = 1; END IF; END; @@ -24,8 +22,6 @@ CREATE TRIGGER check_update BEFORE UPDATE ON releases FOR EACH ROW BEGIN IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}' THEN SET NEW.ishashed = 1; - ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\[' - THEN SET NEW.isrequestid = 1; END IF; END; diff --git a/misc/update/multiprocessing/.do_not_run/switch.php b/misc/update/multiprocessing/.do_not_run/switch.php index 0a528addc..c2713a541 100644 --- a/misc/update/multiprocessing/.do_not_run/switch.php +++ b/misc/update/multiprocessing/.do_not_run/switch.php @@ -17,7 +17,6 @@ use nntmux\Groups; use nntmux\Nfo; use nntmux\NNTP; use nntmux\processing\ProcessReleasesMultiGroup; -use nntmux\RequestIDLocal; // Are we coming from python or php ? $options[0] => (string): python|php // The type of process we want to do: $options[1] => (string): releases @@ -179,14 +178,6 @@ switch ($options[1]) { } break; - // Process all local requestid for a single group. - // $options[2] => (int)groupid, group to work on - case 'requestid': - if (is_numeric($options[2])) { - (new RequestIDLocal(['Echo' => true]))->lookupRequestIDs(['GroupID' => $options[2], 'limit' => 5000]); - } - break; - /* Update a single group's article headers. * * $options[2] => (string) Group name. diff --git a/nntmux/RequestID.php b/nntmux/RequestID.php deleted file mode 100755 index 594cfb776..000000000 --- a/nntmux/RequestID.php +++ /dev/null @@ -1,245 +0,0 @@ - true, - 'Categorize' => null, - 'ConsoleTools' => null, - 'Groups' => null, - 'Settings' => null, - 'SphinxSearch' => null, - ]; - $options += $defaults; - - $this->echoOutput = ($options['Echo'] && NN_ECHOCLI); - $this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB()); - $this->category = ($options['Categorize'] instanceof Categorize ? $options['Categorize'] : new Categorize(['Settings' => $this->pdo])); - $this->groups = ($options['Groups'] instanceof Groups ? $options['Groups'] : new Groups(['Settings' => $this->pdo])); - $this->consoleTools = ($options['ConsoleTools'] instanceof ConsoleTools ? $options['ConsoleTools'] : new ConsoleTools(['ColorCLI' => $this->pdo->log])); - $this->sphinx = ($options['SphinxSearch'] instanceof SphinxSearch ? $options['SphinxSearch'] : new SphinxSearch()); - $this->client = new Client(); - } - - /** - * Look up request id's for releases. - * - * @param array $options - * - * @return int Quantity of releases matched to a request id. - */ - public function lookupRequestIDs(array $options = []) - { - $curOptions = [ - 'charGUID' => '', - 'GroupID' => '', - 'limit' => '', - 'show' => 1, - 'time' => 0, - ]; - $curOptions = array_replace($curOptions, $options); - - $startTime = time(); - $renamed = 0; - - $this->_charGUID = $curOptions['charGUID']; - $this->_groupID = $curOptions['GroupID']; - $this->_show = $curOptions['show']; - $this->_maxTime = $curOptions['time']; - $this->_limit = $curOptions['limit']; - - $this->_getReleases(); - - if ($this->_releases !== false && $this->_releases->rowCount() > 0) { - $this->_totalReleases = $this->_releases->rowCount(); - ColorCLI::doEcho(ColorCLI::primary('Processing '.$this->_totalReleases." releases for RequestID's.")); - $renamed = $this->_processReleases(); - if ($this->echoOutput) { - echo ColorCLI::header( - "\nRenamed ".number_format($renamed).' releases in '.$this->consoleTools->convertTime(time() - $startTime).'.' - ); - } - } elseif ($this->echoOutput) { - ColorCLI::doEcho(ColorCLI::primary("No RequestID's to process.")); - } - - return $renamed; - } - - /** - * Fetch releases with requestid's from MySQL. - */ - protected function _getReleases() - { - } - - /** - * Process releases for requestid's. - * - * @return int How many did we rename? - */ - protected function _processReleases() - { - } - - /** - * No request id was found, update the release. - * - * @param int $releaseID - * @param int $status - */ - protected function _requestIdNotFound($releaseID, $status) - { - if ($releaseID == 0) { - return; - } - - $this->pdo->queryExec( - sprintf(' - UPDATE releases SET reqidstatus = %d WHERE id = %d', - $status, $releaseID - ) - ); - } - - /** - * Get a new title / pre id for a release. - * - * @return array|bool - */ - protected function _getNewTitle() - { - } - - /** - * Find a RequestID in a usenet subject. - * - * @return int - */ - protected function _siftReqId() - { - $requestID = []; - switch (true) { - case preg_match('/\[\s*#?scnzb@?efnet\s*\]\[(\d+)\]/', $this->_release['name'], $requestID): - case preg_match('/\[\s*(\d+)\s*\]/', $this->_release['name'], $requestID): - case preg_match('/^REQ\s*(\d{4,6})/i', $this->_release['name'], $requestID): - case preg_match('/^(\d{4,6})-\d{1}\[/', $this->_release['name'], $requestID): - case preg_match('/(\d{4,6}) -/', $this->_release['name'], $requestID): - if ((int) $requestID[1] > 0) { - return (int) $requestID[1]; - } - } - - return self::REQID_ZERO; - } - - /** - * @var bool Echo to CLI? - */ - protected $echoOutput; - - /** - * @var Categorize - */ - protected $category; - - /** - * @var \nntmux\db\DB - */ - protected $pdo; - - /** - * @var ConsoleTools - */ - protected $consoleTools; - - /** - * @var ColorCLI - */ - protected $colorCLI; - - /** - * The found request id for the release. - * @var int - */ - protected $_requestID = self::REQID_ZERO; - - /** - * The title found from a request id lookup. - * @var bool|string|array - */ - protected $_newTitle = false; - - /** - * Releases with potential Request id's we can work on. - * @var \PDOStatement|bool - */ - protected $_releases; - - /** - * Total amount of releases we will be working on. - * @var int - */ - protected $_totalReleases; - - /** - * Release we are currently working on. - * @var array - */ - protected $_release; - - /** - * @var int To show the result or not. - */ - protected $_show = 0; - - /** - * GroupID, which is optional, to limit query results. - * @var string - */ - protected $_groupID; - - /** - * First character of a release GUID, which is optional, to limit query results. - * @var string - */ - protected $_charGUID; - - protected $_limit; - - protected $_maxTime; -} diff --git a/nntmux/RequestIDLocal.php b/nntmux/RequestIDLocal.php deleted file mode 100755 index 9f7ade614..000000000 --- a/nntmux/RequestIDLocal.php +++ /dev/null @@ -1,305 +0,0 @@ -_charGUID === '' ? '' : ' AND r.leftguid = '.$this->pdo->escapeString($this->_charGUID)); - $query .= ($this->_groupID === '' ? '' : ' AND r.groups_id = '.$this->_groupID); - $query .= ($this->_maxTime === 0 ? '' : sprintf(' AND r.adddate > NOW() - INTERVAL %d HOUR', $this->_maxTime)); - - switch ($this->_limit) { - case 'full': - $query .= sprintf( - ' AND r.reqidstatus in (%d, %d, %d)', - self::REQID_UPROC, - self::REQID_NOLL, - self::REQID_NONE - ); - break; - case is_numeric($this->_limit): - $query .= sprintf( - ' AND r.reqidstatus in (%d, %d, %d) ORDER BY r.postdate DESC LIMIT %d', - self::REQID_UPROC, - self::REQID_NOLL, - self::REQID_NONE, - $this->_limit - ); - break; - case 'all': - default: - break; - } - $this->_releases = $this->pdo->queryDirect($query); - } - - /** - * Process releases for requestid's. - * - * @return int How many did we rename? - */ - protected function _processReleases() - { - $renamed = $checked = 0; - if ($this->_releases instanceof \Traversable) { - foreach ($this->_releases as $this->_release) { - $this->_requestID = $this->_siftReqId(); - - // Do a local lookup using multiple possible methods - $this->_newTitle = $this->_getNewTitle(); - - if ($this->_newTitle !== false && isset($this->_newTitle['title'])) { - $this->_updateRelease(); - $renamed++; - } else { - $this->_requestIdNotFound($this->_release['id'], ($this->_release['reqidstatus'] === self::REQID_UPROC ? self::REQID_NOLL : self::REQID_NONE)); - } - - if ($this->echoOutput && $this->_show === 0) { - $this->consoleTools->overWritePrimary( - 'Checked Releases: ['.number_format($checked).'] '. - $this->consoleTools->percentString(++$checked, $this->_totalReleases) - ); - } - } - } - - return $renamed; - } - - /** - * Get a new title / pre id for a release. - * - * @return array|bool - */ - protected function _getNewTitle() - { - if ($this->_requestID === -2) { - return $this->_multiLookup(); - } - - $check = $this->pdo->queryDirect( - sprintf( - 'SELECT id, title FROM predb WHERE requestid = %d AND groups_id = %d', - $this->_requestID, - $this->_release['gid'] - ) - ); - - if ($check instanceof \Traversable) { - if ($check->rowCount() === 1) { - foreach ($check as $row) { - if (preg_match('/s\d+/i', $row['title']) && ! preg_match('/s\d+e\d+/i', $row['title'])) { - return false; - } - - return ['title' => $row['title'], 'id' => $row['id']]; - } - } else { - //Prevents multiple releases with the same request id/group from being renamed to the same Pre. - return $this->_multiLookup(); - } - } else { - $result = $this->_singleAltLookup(); - if (is_array($result) && is_numeric($result['id']) && $result['title'] !== '') { - return $result; - } else { - return $this->_multiLookup(); - } - } - - return false; - } - - /** - * Sub function that attempts to match RequestID Releases - * by preg_matching the title from the usenet name. - * - * @return array|bool - */ - protected function _multiLookup() - { - $regex1 = - '/^\[\s*\d+\s*\][ -]+(\[(ISO|FULL|PART|MP3|0DAY|android)\][ -]+)?\[(alt-?bin| ?#?a[a-z0-9. -]+)((@?ef{1,2})?net)? ?\]'. - '[ -]+(\[(ISO|FULL|PART|MP3|0DAY|android)\][ -]+)?(\[\s*\d+\s*\][ -]+)?(\[\d+\/\d+\][ -]+)?(\"|\[)\s*'. - '(?P.+?)(\.+(vol\d+\+\d+\.)?(-cd\d\.)?(avi|jpg|nzb|m3u|mkv|par2|part\d+|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)?\s*(\"|\])'. - '[ -]*(\[\d+\/\d+\][ -]*)?((\"\s*(?P<filename1>.+?)([-.]sample)?([-.]cd(\d|[ab]))?(\.+(vol\d+\+\d+\.)?([-.]d\d\.)?([-.]part\d+)?'. - '(avi|jpg|nzb|m3u|mkv|par2|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)?\s*\")| - (?P<filename2>.+?) (yEnc|\(\d+\/\d+\)))?.*/i'; - - $regex2 = - '/^\[\s*\d+\s*\].*'. - '\"\s*(?P<title>.+?)(\.+(vol\d+\+\d+\.)?(-cd\d\.)?'. - '(avi|jpg|nzb|m3u|mkv|par2|part\d+|nfo|sample|sfv|rar|r?\d{1,3}|\d+|zip)*)\s*\".*/i'; - - $matches = []; - switch (true) { - case preg_match($regex1, $this->_release['name'], $matches): - case preg_match($regex2, $this->_release['name'], $matches): - $check = $this->pdo->queryOneRow( - sprintf( - 'SELECT id, title FROM predb WHERE title = %s OR filename = %s %s', - $this->pdo->escapeString($matches['title']), - $this->pdo->escapeString($matches['title']), - ( - isset($matches['filename1']) && $matches['filename1'] !== '' - ? 'OR filename = '.$this->pdo->escapeString($matches['filename1']) - : - ( - isset($matches['filename2']) && $matches['filename2'] !== '' - ? 'OR filename = '.$this->pdo->escapeString($matches['filename2']) - : '' - ) - ) - ) - ); - if ($check !== false) { - return ['title' => $check['title'], 'id' => $check['id']]; - } - continue; - default: - return false; - } - - return false; - } - - private $groupIDCache = []; - - /** - * Attempts to remap the release groups_id by extracting the new group name from the release usenet name. - * - * @return array|bool - */ - protected function _singleAltLookup() - { - switch (true) { - case $this->_release['name'] === 'alt.binaries.etc': - $groupName = 'alt.binaries.teevee'; - break; - case strpos($this->_release['name'], 'teevee') !== false: - $groupName = 'alt.binaries.teevee'; - break; - case strpos($this->_release['name'], 'moovee') !== false: - $groupName = 'alt.binaries.moovee'; - break; - case strpos($this->_release['name'], 'erotica') !== false: - $groupName = 'alt.binaries.erotica'; - break; - case strpos($this->_release['name'], 'foreign') !== false: - $groupName = 'alt.binaries.mom'; - break; - case strpos($this->_release['name'], 'inner-sanctum') !== false: - $groupName = 'alt.binaries.inner-sanctum'; - break; - case strpos($this->_release['name'], 'sounds.flac') !== false: - $groupName = 'alt.binaries.sounds.flac'; - break; - case strpos($this->_release['name'], 'scnzb') !== false: - $groupName = 'alt.binaries.boneless'; - break; - case strpos($this->_release['name'], 'hdtv.x264') !== false: - $groupName = 'alt.binaries.hdtv.x264'; - break; - default: - return false; - } - if (isset($this->groupIDCache[$groupName])) { - $groupID = $this->groupIDCache[$groupName]; - } else { - $groupID = $this->groups->getIDByName($groupName); - } - $check = $this->pdo->queryOneRow( - sprintf(' - SELECT id, title FROM predb WHERE requestid = %d AND groups_id = %d', - $this->_requestID, - ($groupID === '' ? 0 : $groupID) - ) - ); - if ($check !== false) { - return ['title' => $check['title'], 'id' => $check['id']]; - } - - return false; - } - - /** - * Updates release information when a proper Request id match is found. - */ - protected function _updateRelease() - { - $determinedCat = $this->category->determineCategory($this->_release['gid'], $this->_newTitle['title'], $this->_release['fromname']); - if ($determinedCat === $this->_release['categories_id']) { - $newTitle = $this->pdo->escapeString($this->_newTitle['title']); - $this->pdo->queryExec( - sprintf(' - UPDATE releases - SET predb_id = %d, reqidstatus = %d, isrenamed = 1, iscategorized = 1, searchname = %s - WHERE id = %d', - $this->_newTitle['id'], - self::REQID_FOUND, - $newTitle, - $this->_release['id'] - ) - ); - $this->sphinx->updateRelease($this->_release['id'], $this->pdo); - } else { - $newTitle = $this->pdo->escapeString($this->_newTitle['title']); - $this->pdo->queryExec( - sprintf(' - UPDATE releases SET - videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfo_id = NULL, consoleinfo_id = NULL, - bookinfo_id = NULL, anidbid = NULL, predb_id = %d, reqidstatus = %d, isrenamed = 1, - iscategorized = 1, searchname = %s, categories_id = %d - WHERE id = %d', - $this->_newTitle['id'], - self::REQID_FOUND, - $newTitle, - $determinedCat, - $this->_release['id'] - ) - ); - $this->sphinx->updateRelease($this->_release['id'], $this->pdo); - } - - if ($this->_show === 1 && $this->_release['name'] !== $this->_newTitle['title']) { - NameFixer::echoChangedReleaseName( - [ - 'new_name' => $this->_newTitle['title'], - 'old_name' => $this->_release['name'], - 'new_category' => $this->category->getNameByID($determinedCat), - 'old_category' => $this->category->getNameByID($this->_release['categories_id']), - 'group' => $this->_release['groupname'], - 'releases_id' => $this->_release['id'], - 'method' => 'RequestIDLocal', - ] - ); - } - } -} diff --git a/nntmux/RequestIDWeb.php b/nntmux/RequestIDWeb.php deleted file mode 100755 index 47665af86..000000000 --- a/nntmux/RequestIDWeb.php +++ /dev/null @@ -1,330 +0,0 @@ -<?php - -namespace nntmux; - -use App\Models\Settings; -use GuzzleHttp\Exception\RequestException; - -/** - * Attempts to find a PRE name for a release using a request id from our local pre database, - * or internet request id database. - * - * Class RequestIDWeb - */ -class RequestIDWeb extends RequestID -{ - const MAX_WEB_LOOKUPS = 75; // Please don't exceed this, not to be to harsh on the Request id server. - - /** - * The id of the PRE entry the found request id belongs to. - * @var bool|int - */ - protected $_preDbID = false; - - /** - * @var int - */ - protected $_request_hours; - - /** - * Construct. - * - * @param array $options Class instances / Echo to cli? - */ - public function __construct(array $options = []) - { - parent::__construct($options); - $this->_request_hours = (Settings::settingValue('..request_hours') != '') ? (int) Settings::settingValue('..request_hours') : 1; - } - - /** - * Get all results from the releases table that have request id's to be processed. - */ - protected function _getReleases() - { - $this->_releases = $this->pdo->queryDirect( - sprintf(' - SELECT r.id, r.name, r.searchname, r.fromname, g.name AS groupname, r.groups_id, r.categories_id - FROM releases r - LEFT JOIN groups g ON r.groups_id = g.id - WHERE r.nzbstatus = 1 - AND r.predb_id = 0 - AND r.isrequestid = 1 - AND ( - r.reqidstatus = %d - OR (r.reqidstatus = %d AND r.adddate < NOW() - INTERVAL %d HOUR) - ) - %s %s %s - ORDER BY r.postdate+0 DESC - LIMIT %d', - self::REQID_NOLL, - self::REQID_NONE, - $this->_request_hours, - (empty($this->_groupID) ? '' : ('AND r.groups_id = '.$this->_groupID)), - $this->_getReqIdGroups(), - ($this->_maxTime === 0 ? '' : sprintf(' AND r.adddate > NOW() - INTERVAL %d HOUR', $this->_maxTime)), - (empty($this->_limit) || $this->_limit > 1000 ? 1000 : $this->_limit) - ) - ); - } - - /** - * Create "AND" part of query for request id groups. - * Less load on the request id web server, by limiting results. - * - * @return string - */ - protected function _getReqIdGroups() - { - return - "AND g.name IN ( - 'alt.binaries.boneless', - 'alt.binaries.cd.image', - 'alt.binaries.console.ps3', - 'alt.binaries.erotica', - 'alt.binaries.games.nintendods', - 'alt.binaries.games.wii', - 'alt.binaries.games.xbox360', - 'alt.binaries.inner-sanctum', - 'alt.binaries.mom', - 'alt.binaries.moovee', - 'alt.binaries.movies.divx', - 'alt.binaries.sony.psp', - 'alt.binaries.sounds.mp3.complete_cd', - 'alt.binaries.sounds.flac', - 'alt.binaries.teevee', - 'alt.binaries.warez',". - - // Extra groups we will need to remap later, etc is teevee for example. - "'alt.binaries.etc' - )"; - } - - /** - * Process releases for requestid's. - * - * @return int How many did we rename? - */ - protected function _processReleases() - { - // Array to store results. - $requestArray = []; - - if ($this->_releases instanceof \Traversable) { - // Loop all the results. - foreach ($this->_releases as $release) { - $this->_release['name'] = $release['name']; - // Try to find a request id for the release. - $requestId = $this->_siftReqId(); - - // If there's none, update the release and continue. - if ($requestId === self::REQID_ZERO) { - $this->_requestIdNotFound($release['id'], self::REQID_NONE); - if ($this->echoOutput) { - echo '-'; - } - continue; - } - - // Change etc to teevee. - if ($release['groupname'] === 'alt.binaries.etc') { - $release['groupname'] = 'alt.binaries.teevee'; - } - - // Send the release id so we can track the return data. - $requestArray[$release['id']] = [ - 'reqid' => $requestId, - 'ident' => $release['id'], - 'group' => $release['groupname'], - 'sname' => $release['searchname'], - 'fromname' => $release['fromname'], - ]; - } - } - - // Check if we requests to send to the web. - if (count($requestArray) < 1) { - return 0; - } - - // Mock array for isset check on server. - $requestArray[0] = ['ident' => 0, 'group' => 'none', 'reqid' => 0]; - - // Do a web lookup. - try { - $returnXml = $this->client->request('POST', Settings::settingValue('..request_url'), - ['Link' => 'data='.serialize($requestArray)] - )->getBody(); - } catch (RequestException $e) { - if ($e->hasResponse()) { - if ($e->getCode() === 404) { - ColorCLI::doEcho(ColorCLI::notice('Data not available on server')); - } elseif ($e->getCode() === 503) { - ColorCLI::doEcho(ColorCLI::notice('Service unavailable')); - } else { - ColorCLI::doEcho(ColorCLI::notice('Unable to fetch data, server responded with code: '.$e->getCode())); - } - } - } - - $renamed = 0; - // Change the release titles and insert the PRE's if they don't exist. - if (isset($returnXml) && $returnXml !== false) { - $returnXml = @simplexml_load_string($returnXml); - if ($returnXml !== false) { - - // Store the returned identifiers so we can check which releases we didn't find a request id. - $returnedIdentifiers = []; - - $groupIDArray = []; - foreach ($returnXml->request as $result) { - if (isset($result['name'], $result['ident']) && (int) $result['ident'] > 0) { - $this->_newTitle['title'] = (string) $result['name']; - $this->_requestID = (int) $result['reqid']; - $this->_release['id'] = (int) $result['ident']; - - // Buffer groupid queries. - $this->_release['groupname'] = $requestArray[(int) $result['ident']]['group']; - if (isset($groupIDarray[$this->_release['groupname']])) { - $this->_release['groups_id'] = $groupIDArray[$this->_release['groupname']]; - } else { - $this->_release['groups_id'] = $this->groups->getIDByName($this->_release['groupname']); - $groupIDArray[$this->_release['groupname']] = $this->_release['groups_id']; - } - $this->_release['gid'] = $this->_release['groups_id']; - - $this->_release['fromname'] = $requestArray[(string) $result['ident']]['fromname']; - - $this->_release['searchname'] = $requestArray[(int) $result['ident']]['sname']; - - $this->_insertIntoPreDB(); - if ($this->_preDbID === false) { - $this->_preDbID = 0; - } - $this->_newTitle['id'] = $this->_preDbID; - $this->_updateRelease(); - $renamed++; - if ($this->echoOutput) { - echo '+'; - } - $returnedIdentifiers[] = (string) $result['ident']; - } - } - - // Check if the WEB didn't send back some titles, update the release. - if (count($returnedIdentifiers) > 0) { - foreach ($returnedIdentifiers as $identifier) { - if (array_key_exists($identifier, $requestArray)) { - unset($requestArray[$identifier]); - } - } - } - - unset($requestArray[0]); - foreach ($requestArray as $request) { - $addDate = $this->pdo->queryOneRow( - sprintf( - 'SELECT UNIX_TIMESTAMP(adddate) AS adddate FROM releases WHERE id = %d', $request['ident'] - ) - ); - - $status = self::REQID_NONE; - if ($addDate !== false && ! empty($addDate['adddate'])) { - if ((bool) (intval((time() - (int) $addDate['adddate']) / 3600) > $this->_request_hours)) { - $status = self::REQID_OLD; - } - } else { - $status = self::REQID_OLD; - } - - $this->_requestIdNotFound( - $request['ident'], - $status - ); - if ($this->echoOutput) { - echo '-'; - } - } - } - } - - return $renamed; - } - - /** - * If we found a request id on the internet, check if our PRE database has it, insert it if not. - */ - protected function _insertIntoPreDB() - { - $dupeCheck = $this->pdo->queryOneRow( - sprintf(' - SELECT id AS predb_id, requestid, groups_id - FROM predb - WHERE title = %s', - $this->pdo->escapeString($this->_newTitle['title']) - ) - ); - - if ($dupeCheck === false) { - $this->_preDbID = (int) $this->pdo->queryInsert( - sprintf(' - INSERT INTO predb (title, source, requestid, groups_id, predate) - VALUES (%s, %s, %d, %d, NOW())', - $this->pdo->escapeString($this->_newTitle['title']), - $this->pdo->escapeString('requestWEB'), - $this->_requestID, - $this->_release['groups_id'] - ) - ); - } else { - $this->_preDbID = $dupeCheck['predb_id']; - $this->pdo->queryExec( - sprintf(' - UPDATE predb - SET requestid = %d, groups_id = %d - WHERE id = %d', - $this->_requestID, - $this->_release['groups_id'], - $this->_preDbID - ) - ); - } - } - - /** - * If we found a PRE name, update the releases name and reset post processing. - */ - protected function _updateRelease() - { - $determinedCategory = $this->category->determineCategory($this->_release['groups_id'], $this->_newTitle['title'], $this->_release['fromname']); - $newTitle = $this->pdo->escapeString($this->_newTitle['title']); - $this->pdo->queryExec( - sprintf(' - UPDATE releases - SET videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfo_id = NULL, consoleinfo_id = NULL, bookinfo_id = NULL, anidbid = NULL, - reqidstatus = %d, isrenamed = 1, proc_files = 1, searchname = %s, categories_id = %d, - predb_id = %d - WHERE id = %d', - self::REQID_FOUND, - $newTitle, - $determinedCategory, - $this->_preDbID, - $this->_release['id'] - ) - ); - $this->sphinx->updateRelease($this->_release['id'], $this->pdo); - - if ($this->echoOutput) { - NameFixer::echoChangedReleaseName([ - 'new_name' => $this->_newTitle['title'], - 'old_name' => $this->_release['searchname'], - 'new_category' => $this->category->getNameByID($determinedCategory), - 'old_category' => '', - 'group' => $this->_release['groupname'], - 'releases_id' => $this->_release['id'], - 'method' => 'RequestID->updateRelease<web>', - ] - ); - } - } -} diff --git a/nntmux/Tmux.php b/nntmux/Tmux.php index cceaaf379..bfdb0b715 100755 --- a/nntmux/Tmux.php +++ b/nntmux/Tmux.php @@ -269,8 +269,6 @@ class Tmux (%2\$s 'nzbthreads') AS nzbthreads, (%2\$s 'tmpunrarpath') AS tmpunrar, (%2\$s 'compressedheaders') AS compressed, - (%2\$s 'book_reqids') AS book_reqids, - (%2\$s 'request_hours') AS request_hours, (%2\$s 'maxsizetopostprocess') AS maxsize_pp, (%2\$s 'minsizetopostprocess') AS minsize_pp", $tmuxstr, @@ -462,7 +460,6 @@ class Tmux /** * @param $qry * @param $bookreqids - * @param int $request_hours * @param string $db_name * @param string $ppmax * @param string $ppmin @@ -470,7 +467,7 @@ class Tmux * @return bool|string * @throws \Exception */ - public function proc_query($qry, $bookreqids, $request_hours, $db_name, $ppmax = '', $ppmin = '') + public function proc_query($qry, $bookreqids, $db_name, $ppmax = '', $ppmin = '') { switch ((int) $qry) { case 1: @@ -491,8 +488,6 @@ class Tmux OR (ishashed = 1 AND dehashstatus BETWEEN -6 AND 0)) AND categories_id IN (%s),1,0)) AS processrenames, SUM(IF(isrenamed = %d,1,0)) AS renamed, SUM(IF(nzbstatus = %1$d AND nfostatus = %20$d,1,0)) AS nfo, - SUM(IF(nzbstatus = %1$d AND isrequestid = %d AND predb_id = 0 AND ((reqidstatus = %d) OR (reqidstatus = %d) OR (reqidstatus = %d AND adddate > NOW() - INTERVAL %s HOUR)),1,0)) AS requestid_inprogress, - SUM(IF(predb_id > 0 AND nzbstatus = %1$d AND isrequestid = %28$d AND reqidstatus = %d,1,0)) AS requestid_matched, SUM(IF(predb_id > 0,1,0)) AS predb_matched, COUNT(DISTINCT(predb_id)) AS distinct_predb_matched FROM releases r', @@ -525,12 +520,6 @@ class Tmux MiscSorter::PROC_SORTER_NONE, Category::getCategoryOthersGroup(), NameFixer::IS_RENAMED_DONE, - RequestID::IS_REQID_TRUE, - RequestID::REQID_UPROC, - RequestID::REQID_NOLL, - RequestID::REQID_NONE, - RequestID::REQID_FOUND, - $request_hours ); case 2: @@ -623,7 +612,7 @@ class Tmux /** * @throws \RuntimeException */ - public function startRunning() + public function startRunning(): void { if ($this->isRunning() === false) { TmuxModel::query()->where('setting', '=', 'running')->update(['value' => 1]); diff --git a/nntmux/TmuxOutput.php b/nntmux/TmuxOutput.php index c3e7a01b7..66ecb6c67 100755 --- a/nntmux/TmuxOutput.php +++ b/nntmux/TmuxOutput.php @@ -248,20 +248,6 @@ class TmuxOutput extends Tmux $this->runVar['counts']['percent']['predb_matched'] ) ); - $buffer .= sprintf( - $this->tmpMasks[4], - 'RequestID', - sprintf( - '%s(%s)', - number_format($this->runVar['counts']['now']['requestid_inprogress']), - $this->runVar['counts']['diff']['requestid_inprogress'] - ), - sprintf( - '%s(%d%%)', - number_format($this->runVar['counts']['now']['requestid_matched']), - $this->runVar['counts']['percent']['requestid_matched'] - ) - ); $buffer .= sprintf( $this->tmpMasks[4], 'Renames', diff --git a/nntmux/libraries/Forking.php b/nntmux/libraries/Forking.php index 848ae826e..c4c760018 100755 --- a/nntmux/libraries/Forking.php +++ b/nntmux/libraries/Forking.php @@ -9,7 +9,6 @@ use nntmux\db\DB; use Carbon\Carbon; use App\Models\Tmux; use nntmux\ColorCLI; -use nntmux\RequestID; use App\Models\Settings; use nntmux\processing\PostProcess; @@ -256,10 +255,6 @@ class Forking extends \fork_daemon $maxProcesses = $this->postProcessTvMainMethod(); break; - case 'request_id': - $maxProcesses = $this->requestIDMainMethod(); - break; - case 'safe_backfill': $maxProcesses = $this->safeBackfillMainMethod(); break; @@ -973,36 +968,6 @@ class Forking extends \fork_daemon $postProcess->processXXX(); } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////// All requestID code goes here //////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * @return null|string - * @throws \Exception - */ - private function requestIDMainMethod() - { - $this->register_child_run([0 => $this, 1 => 'requestIDChildWorker']); - $this->work = $this->pdo->query( - sprintf( - ' - SELECT DISTINCT(g.id) - FROM groups g - INNER JOIN releases r ON r.groups_id = g.id - WHERE (g.active = 1 OR g.backfill = 1) - AND r.nzbstatus = %d - AND r.predb_id = 0 - AND r.isrequestid = 1 - AND r.reqidstatus = %d', - NZB::NZB_ADDED, - RequestID::REQID_UPROC - ) - ); - - return (int) Settings::settingValue('..reqidthreads'); - } - /** * @param $groups * @param string $identifier diff --git a/nntmux/processing/ProcessReleases.php b/nntmux/processing/ProcessReleases.php index fca5a9a21..240204477 100755 --- a/nntmux/processing/ProcessReleases.php +++ b/nntmux/processing/ProcessReleases.php @@ -16,8 +16,6 @@ use App\Models\Release; use App\Models\Settings; use nntmux\ConsoleTools; use nntmux\ReleaseImage; -use nntmux\RequestIDWeb; -use nntmux\RequestIDLocal; use nntmux\ReleaseCleaning; use App\Models\ReleaseRegex; use App\Models\ReleasesGroups; @@ -62,11 +60,6 @@ class ProcessReleases */ public $completion; - /** - * @var int - */ - public $processRequestIDs; - /** * @var bool */ @@ -210,35 +203,12 @@ class ProcessReleases $this->processCollectionSizes($groupID); $this->deleteUnwantedCollections($groupID); - //$DIR = NN_MISC; - $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); $totalReleasesAdded += $releasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); - // requestid lookups disabled because they are no longer being posted - /*if ($this->processRequestIDs === 0) { - $this->processRequestIDs($groupID, 5000, true); - } elseif ($this->processRequestIDs === 1) { - $this->processRequestIDs($groupID, 5000, true); - $this->processRequestIDs($groupID, 1000, false); - } elseif ($this->processRequestIDs === 2) { - $requestIDTime = time(); - if ($this->echoCLI) { - ColorCLI::doEcho(ColorCLI::header('Process Releases -> Request ID Threaded lookup.')); - } - passthru("${DIR}update/multiprocessing/requestid.php"); - if ($this->echoCLI) { - ColorCLI::doEcho( - ColorCLI::primary( - "\nReleases updated in ". - $this->consoleTools->convertTime(time() - $requestIDTime) - ) - ); - } - } */ $this->categorizeReleases($categorize, $groupID); $this->postProcessReleases($postProcess, $nntp); @@ -628,14 +598,13 @@ class ProcessReleases $collection['gname'] ); - if (is_array($cleanedName)) { + if (\is_array($cleanedName)) { $properName = $cleanedName['properlynamed']; $preID = $cleanerName['predb'] ?? false; - $isReqID = $cleanerName['requestid'] ?? false; $cleanedName = $cleanedName['cleansubject']; } else { $properName = true; - $isReqID = $preID = false; + $preID = false; } if ($preID === false && $cleanedName !== '') { @@ -660,7 +629,6 @@ class ProcessReleases 'size' => $collection['filesize'], 'categories_id' => $categorize->determineCategory($collection['groups_id'], $cleanedName), 'isrenamed' => $properName === true ? 1 : 0, - 'reqidstatus' => $isReqID === true ? 1 : 0, 'predb_id' => $preID === false ? 0 : $preID, 'nzbstatus' => NZB::NZB_NONE, ] @@ -780,6 +748,7 @@ class ProcessReleases * @param int|string $groupID (optional) * * @return int + * @throws \RuntimeException */ public function createNZBs($groupID): int { @@ -836,70 +805,6 @@ class ProcessReleases return $nzbCount; } - /** - * Process RequestID's. - * - * @param int|string $groupID - * @param int $limit - * @param bool $local - * - * @void - * @throws \Exception - */ - public function processRequestIDs($groupID = '', $limit = 5000, $local = true): void - { - if ($local === false && (int) Settings::settingValue('..lookup_reqids') === 0) { - return; - } - - $startTime = time(); - if ($this->echoCLI) { - ColorCLI::doEcho( - ColorCLI::header( - sprintf( - 'Process Releases -> Request ID %s lookup -- limit %s', - ($local === true ? 'local' : 'web'), - $limit - ) - ) - ); - } - - if ($local === true) { - $foundRequestIDs = ( - new RequestIDLocal( - [ - 'Echo' => $this->echoCLI, - 'ConsoleTools' => $this->consoleTools, - 'Groups' => $this->groups, - 'Settings' => $this->pdo, - ] - ) - )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); - } else { - $foundRequestIDs = ( - new RequestIDWeb( - [ - 'Echo' => $this->echoCLI, - 'ConsoleTools' => $this->consoleTools, - 'Groups' => $this->groups, - 'Settings' => $this->pdo, - ] - ) - )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); - } - if ($this->echoCLI) { - ColorCLI::doEcho( - ColorCLI::primary( - number_format($foundRequestIDs). - ' releases updated in '. - $this->consoleTools->convertTime(time() - $startTime) - ), - true - ); - } - } - /** * Categorize releases. * @@ -1377,7 +1282,7 @@ class ProcessReleases // Disabled categories. $disabledCategories = $category->getDisabledIDs(); - if (count($disabledCategories) > 0) { + if (\count($disabledCategories) > 0) { foreach ($disabledCategories as $disabledCategory) { $releases = $this->pdo->queryDirect( sprintf('SELECT SQL_NO_CACHE id, guid FROM releases WHERE categories_id = %d', (int) $disabledCategory['id']) @@ -1428,7 +1333,7 @@ class ProcessReleases // Disabled music genres. $genrelist = $genres->getDisabledIDs(); - if (count($genrelist) > 0) { + if (\count($genrelist) > 0) { foreach ($genrelist as $genre) { $releases = $this->pdo->queryDirect( sprintf( @@ -1821,7 +1726,7 @@ class ProcessReleases $where ) ); - if ($this->echoCLI && is_object($obj) && $obj->rowCount()) { + if ($this->echoCLI && \is_object($obj) && $obj->rowCount()) { ColorCLI::doEcho( ColorCLI::primary('Deleted '.$obj->rowCount().' broken/stuck collections.') ); diff --git a/resources/db/schema/mysql-ddl.sql b/resources/db/schema/mysql-ddl.sql index d45825cb3..009a34387 100755 --- a/resources/db/schema/mysql-ddl.sql +++ b/resources/db/schema/mysql-ddl.sql @@ -758,7 +758,6 @@ CREATE TABLE releases ( iscategorized TINYINT(1) NOT NULL DEFAULT '0', isrenamed TINYINT(1) NOT NULL DEFAULT '0', ishashed TINYINT(1) NOT NULL DEFAULT '0', - isrequestid TINYINT(1) NOT NULL DEFAULT '0', proc_pp TINYINT(1) NOT NULL DEFAULT '0', proc_sorter TINYINT(1) NOT NULL DEFAULT '0', proc_par2 TINYINT(1) NOT NULL DEFAULT '0', @@ -789,8 +788,7 @@ processed', INDEX ix_releases_haspreview_passwordstatus (haspreview,passwordstatus), INDEX ix_releases_passwordstatus (passwordstatus), INDEX ix_releases_nfostatus (nfostatus,size), - INDEX ix_releases_dehashstatus (dehashstatus,ishashed), - INDEX ix_releases_reqidstatus (adddate,reqidstatus,isrequestid) + INDEX ix_releases_dehashstatus (dehashstatus,ishashed) ) ENGINE = InnoDB DEFAULT CHARSET = utf8 @@ -1445,8 +1443,6 @@ CREATE TRIGGER check_insert BEFORE INSERT ON releases FOR EACH ROW BEGIN IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}' THEN SET NEW.ishashed = 1; - ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\[' - THEN SET NEW.isrequestid = 1; END IF; END; $$ @@ -1454,8 +1450,6 @@ CREATE TRIGGER check_update BEFORE UPDATE ON releases FOR EACH ROW BEGIN IF NEW.searchname REGEXP '[a-fA-F0-9]{32}' OR NEW.name REGEXP '[a-fA-F0-9]{32}' THEN SET NEW.ishashed = 1; - ELSEIF NEW.name REGEXP '^\\[ ?([[:digit:]]{4,6}) ?\\]|^REQ\\s*([[:digit:]]{4,6})|^([[:digit:]]{4,6})-[[:digit:]]{1}\\s?\\[' - THEN SET NEW.isrequestid = 1; END IF; END; $$