From 29f69c588001fa2a90a9275168dfbbc216bdbe38 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Wed, 25 May 2016 22:33:18 +0200 Subject: [PATCH] Speed up UID renaming, remove Srr filename matching. --- misc/testing/Releases/fixReleaseNames.php | 14 +- .../nix/multiprocessing/fixrelnames.php | 3 +- misc/update/nix/tmux/bin/groupfixrelnames.php | 117 +++++++-- newznab/NameFixer.php | 232 +++++------------- newznab/libraries/Forking.php | 10 +- 5 files changed, 154 insertions(+), 222 deletions(-) diff --git a/misc/testing/Releases/fixReleaseNames.php b/misc/testing/Releases/fixReleaseNames.php index 664d5893c..123765173 100755 --- a/misc/testing/Releases/fixReleaseNames.php +++ b/misc/testing/Releases/fixReleaseNames.php @@ -69,15 +69,9 @@ if (isset($argv[1]) && isset($argv[2]) && isset($argv[3]) && isset($argv[4])) { $namefixer->fixNamesWithPar2(2, $update, $other, $setStatus, $show, $nntp); break; case 9: - $namefixer->fixNamesWithSrr(1, $update, $other, $setStatus, $show); - break; - case 10: - $namefixer->fixNamesWithSrr(2, $update, $other, $setStatus, $show); - break; - case 11: $namefixer->fixNamesWithMedia(1, $update, $other, $setStatus, $show); break; - case 12: + case 10: $namefixer->fixNamesWithMedia(2, $update, $other, $setStatus, $show); break; default : @@ -98,8 +92,6 @@ if (isset($argv[1]) && isset($argv[2]) && isset($argv[3]) && isset($argv[4])) { . "php $argv[0] 6 false other no ...: Fix release names in misc categories using File Name.\n" . "php $argv[0] 7 false other no ...: Fix release names in misc categories using Par2 Files in the past 6 hours.\n" . "php $argv[0] 8 false other no ...: Fix release names in misc categories using Par2 Files.\n" - . "php $argv[0] 9 false other no ...: Fix release names in misc categories using SRR Files in the past 6 hours.\n" - . "php $argv[0] 10 false other no ...: Fix release names in misc categories using SRR Files.\n" - . "php $argv[0] 11 false other no ...: Fix release names in misc categories using UID in the past 6 hours.\n" - . "php $argv[0] 12 false other no ...: Fix release names in misc categories using UID.\n")); + . "php $argv[0] 9 false other no ...: Fix release names in misc categories using UID in the past 6 hours.\n" + . "php $argv[0] 10 false other no ...: Fix release names in misc categories using UID.\n")); } diff --git a/misc/update/nix/multiprocessing/fixrelnames.php b/misc/update/nix/multiprocessing/fixrelnames.php index dc0b6ac81..e257a9f8f 100644 --- a/misc/update/nix/multiprocessing/fixrelnames.php +++ b/misc/update/nix/multiprocessing/fixrelnames.php @@ -1,10 +1,9 @@ Attempt to fix release name using the nfo.' . PHP_EOL . 'filename => Attempt to fix release name using the filenames.' . PHP_EOL . - 'srr => Attempt to fix release name using the srr file.' . PHP_EOL . 'uid => Attempt to fix release name using the mediainfo uid.' . PHP_EOL . 'md5 => Attempt to fix release name using the MD5.' . PHP_EOL . 'par2 => Attempt to fix release name using the par2.' . PHP_EOL . diff --git a/misc/update/nix/tmux/bin/groupfixrelnames.php b/misc/update/nix/tmux/bin/groupfixrelnames.php index ed66563f4..5b8fc00c5 100644 --- a/misc/update/nix/tmux/bin/groupfixrelnames.php +++ b/misc/update/nix/tmux/bin/groupfixrelnames.php @@ -1,13 +1,13 @@ escapeString($guidChar), + $namefixer::PROC_NFO_NONE, $maxperrun ) ); - if ($releases instanceof Traversable) { + if ($releases instanceof \Traversable) { foreach ($releases as $release) { if (preg_match('/^=newz\[NZB\]=\w+/', $release['textstring'])) { $namefixer->done = $namefixer->matched = false; @@ -58,13 +59,64 @@ if (!isset($argv[1])) { } break; case $pieces[0] === 'filename' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun): - $namefixer->fixNamesWithFiles(1, 1, 1, 1, 1, $guidChar, $maxperrun); - break; - case $pieces[0] === 'srr' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun): - $namefixer->fixNamesWithSrr(1, 1, 1, 1, 1, $guidChar, $maxperrun); + $releases = $pdo->queryDirect( + sprintf(' + SELECT rf.name AS textstring, rf.releases_id AS fileid, + r.id AS releases_id, r.name, r.searchname, r.categories_id, r.groups_id + FROM releases r + INNER JOIN release_files rf ON r.id = rf.releases_id + WHERE r.leftguid = %s + AND r.nzbstatus = 1 + AND r.proc_files = %d + AND r.predb_id = 0 + ORDER BY r.id ASC + LIMIT %s', + $pdo->escapeString($guidChar), + $namefixer::PROC_FILES_NONE, + $maxperrun + ) + ); + + if ($releases instanceof \Traversable) { + foreach ($releases as $release) { + $namefixer->done = $namefixer->matched = false; + if ($namefixer->checkName($release, true, 'Filenames, ', 1, 1) !== true) { + echo '.'; + } + $namefixer->checked++; + } + } break; case $pieces[0] === 'uid' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun): - $namefixer->fixNamesWithMedia(2, 1, 2, 1, 1, $guidChar, $maxperrun); + $releases = $pdo->queryDirect( + sprintf(' + SELECT + r.id AS releases_id, r.size AS relsize, r.groups_id, r.categories_id, + r.name, r.name AS textstring, r.predb_id, r.searchname, + HEX(ru.uniqueid) AS uid + FROM releases r + LEFT JOIN release_unique ru ON ru.releases_id = r.id + WHERE ru.releases_id IS NOT NULL + AND r.leftguid = %s + AND r.nzbstatus = 1 + AND r.predb_id = 0 + AND r.proc_uid = %d + ORDER BY r.id DESC + LIMIT %d', + $pdo->escapeString($guidChar), + $namefixer::PROC_UID_NONE, + $maxperrun + ) + ); + if ($releases instanceof \Traversable) { + foreach ($releases as $release) { + $namefixer->done = $namefixer->matched = false; + if ($namefixer->uidCheck($release, true, 'UID, ', 1, 1) === false) { + echo '.'; + } + $namefixer->checked++; + } + } break; case $pieces[0] === 'md5' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun): $releases = $pdo->queryDirect( @@ -77,15 +129,15 @@ if (!isset($argv[1])) { AND nzbstatus = 1 AND r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0 - AND r.predb_id < 1 + AND r.predb_id = 1 ORDER BY r.dehashstatus DESC, r.id ASC - LIMIT %s', + LIMIT %s', $pdo->escapeString($guidChar), $maxperrun ) ); - if ($releases instanceof Traversable) { + if ($releases instanceof \Traversable) { foreach ($releases as $release) { if (preg_match('/[a-fA-F0-9]{32,40}/i', $release['name'], $matches)) { $namefixer->matchPredbHash($matches[0], $release, 1, 1, true, 1); @@ -105,16 +157,17 @@ if (!isset($argv[1])) { FROM releases r WHERE r.leftguid = %s AND r.nzbstatus = 1 - AND r.proc_par2 = 0 - AND r.predb_id < 1 + AND r.proc_par2 = %d + AND r.predb_id = 0 ORDER BY r.id ASC - LIMIT %s', + LIMIT %s', $pdo->escapeString($guidChar), + $namefixer::PROC_PAR2_NONE, $maxperrun ) ); - if ($releases instanceof Traversable) { + if ($releases instanceof \Traversable) { $nntp = new NNTP(['Settings' => $pdo]); if (($pdo->getSetting('alternate_nntp') == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { exit($pdo->log->error("Unable to connect to usenet.")); @@ -122,10 +175,10 @@ if (!isset($argv[1])) { $Nfo = new Nfo(['Settings' => $pdo, 'Echo' => true]); $nzbcontents = new NZBContents( - array( + [ 'Echo' => true, 'NNTP' => $nntp, 'Nfo' => $Nfo, 'Settings' => $pdo, 'PostProcess' => new PostProcess(['Settings' => $pdo, 'Nfo' => $Nfo, 'NameFixer' => $namefixer]) - ) + ] ); foreach ($releases as $release) { $res = $nzbcontents->checkPAR2($release['guid'], $release['releases_id'], $release['groups_id'], 1, 1); @@ -143,17 +196,19 @@ if (!isset($argv[1])) { WHERE r.leftguid = %s AND r.nzbstatus = 1 AND r.nfostatus = 1 - AND r.proc_sorter = 0 - AND r.isrenamed = 0 - AND r.predb_id < 1 + AND r.proc_sorter = %d + AND r.isrenamed = %d + AND r.predb_id = 0 ORDER BY r.id DESC LIMIT %s', $pdo->escapeString($guidChar), + MiscSorter::PROC_SORTER_NONE, + $namefixer::IS_RENAMED_NONE, $maxperrun ) ); - if ($releases instanceof Traversable) { + if ($releases instanceof \Traversable) { $sorter = new MiscSorter(true, $pdo); foreach ($releases as $release) { $res = $sorter->nfosorter(null, $release['releases_id']); @@ -176,7 +231,7 @@ if (!isset($argv[1])) { ) ); - if ($pres instanceof Traversable) { + if ($pres instanceof \Traversable) { foreach ($pres as $pre) { $namefixer->done = $namefixer->matched = false; $ftmatched = $searched = 0; @@ -190,7 +245,15 @@ if (!isset($argv[1])) { $searched = $pre['searched'] - 1; echo "."; } - $pdo->queryExec(sprintf("UPDATE predb SET searched = %d WHERE id = %d", $searched, $pre['predb_id'])); + $pdo->queryExec( + sprintf(" + UPDATE predb + SET searched = %d + WHERE id = %d", + $searched, + $pre['predb_id'] + ) + ); $namefixer->checked++; } } diff --git a/newznab/NameFixer.php b/newznab/NameFixer.php index eef0200a9..2648a8184 100755 --- a/newznab/NameFixer.php +++ b/newznab/NameFixer.php @@ -19,8 +19,6 @@ class NameFixer const PROC_FILES_DONE = 1; const PROC_PAR2_NONE = 0; const PROC_PAR2_DONE = 1; - const PROC_SRR_NONE = 0; - const PROC_SRR_DONE = 1; const PROC_UID_NONE = 0; const PROC_UID_DONE = 1; @@ -327,86 +325,6 @@ class NameFixer } } - /** - * Attempts to fix release names using the SRR file extension. - * - * @param int $time 1: 24 hours, 2: no time limit - * @param boolean $echo 1: change the name, anything else: preview of what could have been changed. - * @param int $cats 1: other categories, 2: all categories - * @param $nameStatus - * @param $show - * @param string $guidChar - * @param string $limit - */ - public function fixNamesWithSrr($time, $echo, $cats, $nameStatus, $show, $guidChar = '', $limit = '') - { - $type = 'Srr, '; - $guid = ($guidChar === '') ? '' : ('AND rel.leftguid = ' . $this->pdo->escapeString($guidChar)); - $queryLimit = ($limit === '') ? '' : $limit; - if ($guid === '') { - $this->_echoStartMessage($time, 'srr files'); - } - - $preId = false; - if ($cats === 3) { - $query = - sprintf(' - SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - STRAIGHT_JOIN release_files rf ON rf.releases_id = rel.id - WHERE rel.nzbstatus = %d - AND rel.predb_id < 1', - NZB::NZB_ADDED - ); - $cats = 2; - $preId = true; - } else { - $query = - sprintf(' - SELECT rf.name AS textstring, rel.categories_id, rel.name, rel.searchname, rel.groups_id, - rf.releases_id AS fileid, rel.id AS releases_id - FROM releases rel - STRAIGHT_JOIN release_files rf ON rf.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) - AND rf.name %s - AND rel.proc_srr = %d - %s', - self::IS_RENAMED_NONE, - Category::OTHER_MISC, - Category::OTHER_HASHED, - $this->pdo->likeString('.srr', true, false), - self::PROC_SRR_NONE, - $guid - ); - } - - $releases = $this->_getReleases($time, $cats, $query, $queryLimit); - if ($releases instanceof \Traversable && $releases !== false) { - - $total = $releases->rowCount(); - if ($total > 0) { - $this->_totalReleases = $total; - if ($guid === '') { - echo $this->pdo->log->primary(number_format($total) . ' srr files to process.'); - } - - foreach ($releases as $release) { - $this->done = $this->matched = false; - $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); - $this->checked++; - $this->_echoRenamed($show); - } - - $this->_echoFoundCount($echo, ' files'); - } elseif ($guid === '') { - echo $this->pdo->log->info('Nothing to fix.'); - } else { - echo '.'; - } - } - } - /** * Attempts to fix release names using the Par2 File. * @@ -484,58 +402,57 @@ class NameFixer * @param int $cats 1: other categories, 2: all categories * @param $nameStatus * @param $show - * @param string $guidChar - * @param int $limit */ - public function fixNamesWithMedia($time, $echo, $cats, $nameStatus, $show, $guidChar = '', $limit = '') + public function fixNamesWithMedia($time, $echo, $cats, $nameStatus, $show) { $type = 'UID, '; - $guid = ($guidChar === '') ? '' : ('AND rel.leftguid = ' . $this->pdo->escapeString($guidChar)); - $queryLimit = ($limit === '') ? '' : $limit; - if ($guid === '') { - $this->_echoStartMessage($time, 'mediainfo Unique_IDs'); - } - // Only select releases we haven't checked here before + $this->_echoStartMessage($time, 'mediainfo Unique_IDs'); + + // Re-check all releases we haven't matched to a PreDB if ($cats === 3) { $query = sprintf(' - SELECT rel.id AS releases_id, rel.name AS textstring - FROM releases rel - INNER JOIN release_unique ru ON ru.releases_id = rel.id - WHERE rel.nzbstatus = %d - AND rel.predb_id < 1', + SELECT + rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.categories_id, + rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, + HEX(ru.uniqueid) AS uid + FROM releases rel + LEFT JOIN release_unique ru ON ru.releases_id = rel.id + WHERE ru.releases_id IS NOT NULL + AND rel.nzbstatus = %d + AND rel.predb_id = 0', NZB::NZB_ADDED ); + $cats = 2; + // Otherwise check only releases we haven't renamed and checked uid before in Misc categories } else { $query = sprintf(' SELECT rel.id AS releases_id, rel.size AS relsize, rel.groups_id, rel.categories_id, - rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, ru.releases_id, + rel.name, rel.name AS textstring, rel.predb_id, rel.searchname, HEX(ru.uniqueid) AS uid FROM releases rel - INNER JOIN release_unique ru ON ru.releases_id = rel.id - WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) + LEFT JOIN release_unique ru ON ru.releases_id = rel.id + WHERE ru.releases_id IS NOT NULL + AND rel.nzbstatus = %d + AND rel.isrenamed = 0 + AND rel.categories_id IN (%d, %d) AND rel.proc_uid = %d - %s - ORDER BY rel.id DESC - LIMIT %d', + %s', + NZB::NZB_ADDED, self::IS_RENAMED_NONE, Category::OTHER_MISC, Category::OTHER_HASHED, - self::PROC_UID_NONE, - $guid, - $queryLimit + self::PROC_UID_NONE ); } - $releases = $this->pdo->queryDirect($query); + $releases = $this->_getReleases($time, $cats, $query); if ($releases instanceof \Traversable && $releases !== false) { $total = $releases->rowCount(); if ($total > 0) { $this->_totalReleases = $total; - if ($guid === '') { - echo $this->pdo->log->primary(number_format($total) . ' unique ids to process.'); - } + echo $this->pdo->log->primary(number_format($total) . ' unique ids to process.'); foreach ($releases as $rel) { $this->checked++; $this->done = $this->matched = false; @@ -543,10 +460,8 @@ class NameFixer $this->_echoRenamed($show); } $this->_echoFoundCount($echo, ' UID\'s'); - } elseif ($guid === '') { - echo $this->pdo->log->info('Nothing to fix.'); } else { - echo '.'; + echo $this->pdo->log->info('Nothing to fix.'); } } } @@ -678,13 +593,6 @@ class NameFixer } } - if ($type === "Srr, ") { - $newName = ucwords($newName); - if (preg_match('/(.+?)\.(srr)?$/i', $name, $match)) { - $newName = $match[1]; - } - } - $this->fixed++; $newName = explode("\\", $newName); @@ -753,9 +661,6 @@ class NameFixer case "sorter, ": $status = "isrenamed = 1, iscategorized = 1, proc_sorter = 1,"; break; - case "Srr, ": - $status = "isrenamed = 1, iscategorized = 1, proc_srr = 1,"; - break; case "UID, ": $status = "isrenamed = 1, iscategorized = 1, proc_uid = 1,"; break; @@ -1131,9 +1036,6 @@ class NameFixer case "PAR2, ": $this->fileCheck($release, $echo, $type, $namestatus, $show); break; - case "Srr, ": - $this->srrCheck($release, $echo, $type, $namestatus, $show); - break; case "UID, ": $this->uidCheck($release, $echo, $type, $namestatus, $show); break; @@ -1166,9 +1068,6 @@ class NameFixer case "PAR2, ": $this->_updateSingleColumn('proc_par2', self::PROC_FILES_DONE, $release['releases_id']); break; - case "Srr, ": - $this->_updateSingleColumn('proc_srr', self::PROC_SRR_DONE, $release['releases_id']); - break; case "UID, ": $this->_updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release['releases_id']); break; @@ -1711,64 +1610,49 @@ class NameFixer } } - /** - * Look for a name based on srr filename. - * - * @param $release - * @param boolean $echo - * @param string $type - * @param $namestatus - * @param $show - */ - public function srrCheck($release, $echo, $type, $namestatus, $show) - { - $result = []; - - if ($this->done === false && $this->relid !== $release["releases_id"]) { - if (preg_match('/(.+?)\.(srr)$/i', $release["textstring"], $result)) { - $this->updateRelease($release, $result["1"], $method = "srrCheck: Srr filename", $echo, $type, $namestatus, $show); - } - } - } - /** * Look for a name based on mediainfo xml Unique_ID. * - * @param $release - * @param $release - * @param boolean $echo - * @param string $type - * @param $namestatus - * @param $show + * @param array $release The release to be matched + * @param boolean $echo Should we show CLI output + * @param string $type The rename type + * @param int $namestatus Should we rename the release if match is found + * @param int $show Should we show the rename results + * + * @return bool Whether or not we matched the release */ - protected function uidCheck($release, $echo, $type, $namestatus, $show) + public function uidCheck($release, $echo, $type, $namestatus, $show) { if ($this->done === false && $this->relid !== $release["releases_id"]) { - $result = $this->pdo->queryOneRow(" + $result = $this->pdo->queryDirect(" SELECT r.id AS releases_id, r.size AS relsize, r.name AS textstring, r.searchname, r.predb_id FROM release_unique ru - LEFT JOIN releases r ON ru.releases_id = r.id - WHERE ru.uniqueid = UNHEX('{$release['uid']}') + STRAIGHT_JOIN releases r ON ru.releases_id = r.id + WHERE ru.uniqueid = UNHEX({$this->pdo->escapeString($release['uid'])}) AND ru.releases_id != {$release['releases_id']} AND (r.predb_id > 0 OR r.anidbid > 0)" ); - if ($result !== false) { - $floor = floor((1 - $result['relsize'] / $release['relsize']) * 100); - if ($floor <= 5 && $floor >= -5) { - $this->updateRelease( - $release, - $result['searchname'], - $method = "uidCheck: Unique_ID", - $echo, - $type, - $namestatus, - $show, - $result['predb_id'] - ); - } else { - $this->_updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release['releases_id']); + + if ($result instanceof \Traversable) { + foreach ($result AS $res) { + $floor = round(($res['relsize'] - $release['relsize']) / $res['relsize'] * 100, 1); + if ($floor >= -5 && $floor <= 5) { + $this->updateRelease( + $release, + $res['searchname'], + $method = "uidCheck: Unique_ID", + $echo, + $type, + $namestatus, + $show, + $res['predb_id'] + ); + return true; + } } } } + $this->_updateSingleColumn('proc_uid', self::PROC_UID_DONE, $release['releases_id']); + return false; } -} +} \ No newline at end of file diff --git a/newznab/libraries/Forking.php b/newznab/libraries/Forking.php index 4f3f1f9e2..c924ddd23 100644 --- a/newznab/libraries/Forking.php +++ b/newznab/libraries/Forking.php @@ -147,7 +147,6 @@ class Forking extends \fork_daemon case 'fixRelNames_nfo': case 'fixRelNames_filename': - case 'fixRelNames_srr': case 'fixRelNames_uid': case 'fixRelNames_md5': case 'fixRelNames_par2': @@ -524,14 +523,9 @@ class Forking extends \fork_daemon $where = "r.proc_files = 0"; break; - case "srr": - $join = "STRAIGHT_JOIN release_files rf ON rf.releases_id = r.id"; - $where = "r.proc_srr = 0"; - break; - case "uid": - $join = "STRAIGHT_JOIN release_unique ru ON ru.releases_id = r.id"; - $where = "r.proc_uid = 0"; + $join = "LEFT JOIN release_unique ru ON ru.releases_id = r.id"; + $where = "ru.releases_id IS NOT NULL AND r.nzbstatus = 1 AND r.predb_id = 0 AND r.proc_uid = 0"; break; case "par2":