From d280719b42ca839bdacb6e088bf4777d886f92a1 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Sat, 14 Jan 2017 23:44:00 +0100 Subject: [PATCH] Bring branch up-to-date --- .../multiprocessing/.do_not_run/switch.php | 22 +- misc/update/nix/tmux/monitor.php | 8 +- misc/update/nix/tmux/run.php | 2 +- misc/update/update_releases.php | 45 +- nntmux/NZBMultiGroup.php | 4 +- nntmux/processing/ProcessReleases.php | 645 +++++++++--------- .../processing/ProcessReleasesMultiGroup.php | 149 +--- 7 files changed, 404 insertions(+), 471 deletions(-) diff --git a/misc/update/nix/multiprocessing/.do_not_run/switch.php b/misc/update/nix/multiprocessing/.do_not_run/switch.php index db1de920d..af3978a19 100644 --- a/misc/update/nix/multiprocessing/.do_not_run/switch.php +++ b/misc/update/nix/multiprocessing/.do_not_run/switch.php @@ -160,10 +160,12 @@ switch ($options[1]) { collectionCheck($pdo, $options[2]); } - processReleases($releases, $mgrreleases, $options[2]); - + processReleases($releases, $options[2]); } else { + // Run MGR once after all other release updates for standard groups + processReleases(new ProcessReleasesMultiGroup(['Settings' => $pdo]), ''); + // Run functions that run on releases table after all others completed. $groupCount = rtrim($options[2], '_'); if (!is_numeric($groupCount)) { @@ -223,7 +225,8 @@ switch ($options[1]) { $backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal'); // Create releases. - processReleases(new ProcessReleases(['Settings' => $pdo]), new ProcessReleasesMultiGroup(['Settings' => $pdo]), $options[2]); + processReleases(new ProcessReleases(['Settings' => $pdo]), $options[2]); + processReleases(new ProcessReleasesMultiGroup(['Settings' => $pdo]), $options[2]); // Post process the releases. (new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start($options[2]); @@ -278,30 +281,23 @@ switch ($options[1]) { /** * Create / process releases for a groupID. * - * @param ProcessReleases $releases - * @param ProcessReleasesMultiGroup $mgrreleases + * @param ProcessReleases|ProcessReleasesMultiGroup $releases * @param int $groupID */ -function processReleases($releases, $mgrreleases, $groupID) +function processReleases($releases, $groupID) { $releaseCreationLimit = (Settings::value('..maxnzbsprocessed') != '' ? (int)Settings::value('..maxnzbsprocessed') : 1000); $releases->processIncompleteCollections($groupID); $releases->processCollectionSizes($groupID); $releases->deleteUnwantedCollections($groupID); - $mgrreleases->processIncompleteMgrCollections($groupID); - $mgrreleases->processMgrCollectionSizes($groupID); - $mgrreleases->deleteUnwantedMgrCollections($groupID); do { - $mgrReleasesCount = $mgrreleases->createMGRReleases($groupID); - $mgrFilesAdded = $mgrreleases->createMGRNzbs($groupID); $releasesCount = $releases->createReleases($groupID); $nzbFilesAdded = $releases->createNZBs($groupID); // This loops as long as the number of releases or nzbs added was >= the limit (meaning there are more waiting to be created) - } while (($releasesCount['added'] + $releasesCount['dupes'] + $mgrReleasesCount['added'] + $mgrReleasesCount['dupes']) >= $releaseCreationLimit || $nzbFilesAdded + $mgrFilesAdded >= $releaseCreationLimit); + } while (($releasesCount['added'] + $releasesCount['dupes'] >= $releaseCreationLimit || $nzbFilesAdded >= $releaseCreationLimit)); $releases->deleteCollections($groupID); - $mgrreleases->deleteMgrCollections($groupID); } /** diff --git a/misc/update/nix/tmux/monitor.php b/misc/update/nix/tmux/monitor.php index 54f5f76f8..600061c7e 100644 --- a/misc/update/nix/tmux/monitor.php +++ b/misc/update/nix/tmux/monitor.php @@ -217,7 +217,7 @@ while ($runVar['counts']['iterations'] > 0) { $stamp = 'UNIX_TIMESTAMP(MIN(dateadded))'; switch (true) { - case strpos($tbl, 'collections_') !== false: + case strpos($tbl, 'collections') !== false: $runVar['counts']['now']['collections_table'] += getTableRowCount($psTableRowCount, $tbl); $added = $pdo->queryOneRow(sprintf('SELECT %s AS dateadded FROM %s', $stamp, $tbl)); @@ -227,17 +227,17 @@ while ($runVar['counts']['iterations'] > 0) { $age = $added['dateadded']; } break; - case strpos($tbl, 'binaries_') !== false: + case strpos($tbl, 'binaries') !== false: $runVar['counts']['now']['binaries_table'] += getTableRowCount($psTableRowCount, $tbl); break; // This case must come before the 'parts_' one. - case strpos($tbl, 'missed_parts_') !== false: + case strpos($tbl, 'missed_parts') !== false: $runVar['counts']['now']['missed_parts_table'] += getTableRowCount($psTableRowCount, $tbl); break; - case strpos($tbl, 'parts_') !== false: + case strpos($tbl, 'parts') !== false: $runVar['counts']['now']['parts_table'] += getTableRowCount($psTableRowCount, $tbl); break; diff --git a/misc/update/nix/tmux/run.php b/misc/update/nix/tmux/run.php index 52b2e76c1..be1f667bf 100644 --- a/misc/update/nix/tmux/run.php +++ b/misc/update/nix/tmux/run.php @@ -67,7 +67,7 @@ if ($tablepergroup == 1) { $ran = 0; foreach ($tables as $row) { $tbl = $row['name']; - if (preg_match('/collections_\d+/', $tbl)) { + if (preg_match('/(multigroup\_)?collections(_\d+)?/', $tbl)) { $run = $pdo->queryExec('UPDATE ' . $tbl . ' SET dateadded = now() WHERE dateadded < now() - INTERVAL ' . $delaytimet . ' HOUR'); diff --git a/misc/update/update_releases.php b/misc/update/update_releases.php index d817dbfa9..ca4ceee59 100755 --- a/misc/update/update_releases.php +++ b/misc/update/update_releases.php @@ -7,6 +7,7 @@ use nntmux\ConsoleTools; use nntmux\NNTP; use nntmux\db\DB; use nntmux\processing\ProcessReleases; +use nntmux\processing\ProcessReleasesMultiGroup; $category = new Category(); $pdo = new DB(); @@ -18,6 +19,7 @@ if (isset($argv[2]) && $argv[2] === 'true') { exit($pdo->log->error("Unable to connect to usenet.")); } } + if (Settings::value('..tablepergroup') === 1) { exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update/nix/multiprocessing/releases.php")); } @@ -26,33 +28,46 @@ $groupName = isset($argv[3]) ? $argv[3] : ''; if (isset($argv[1]) && isset($argv[2])) { $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); $releases = new ProcessReleases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]); + $prmg = new ProcessReleasesMultiGroup(['Settings' => $pdo]); if ($argv[1] == 1 && $argv[2] == 'true') { $releases->processReleases(1, 1, $groupName, $nntp, true); + if ($groupName = '') { + $prmg->processReleases(1, 1, 'mgr', $nntp, true); + } } else if ($argv[1] == 1 && $argv[2] == 'false') { $releases->processReleases(1, 2, $groupName, $nntp, true); + if ($groupName = '') { + $prmg->processReleases(1, 1, 'mgr', $nntp, true); + } } else if ($argv[1] == 2 && $argv[2] == 'true') { $releases->processReleases(2, 1, $groupName, $nntp, true); + if ($groupName = '') { + $prmg->processReleases(1, 1, 'mgr', $nntp, true); + } } else if ($argv[1] == 2 && $argv[2] == 'false') { $releases->processReleases(2, 2, $groupName, $nntp, true); + if ($groupName = '') { + $prmg->processReleases(1, 1, 'mgr', $nntp, true); + } } else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) { echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient."); $releases->resetCategorize(); } else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) { echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient."); - $timestart = time(); + $timestart = TIME(); $relcount = $releases->categorizeRelease('name', "WHERE iscategorized = 0 AND categories_id = " . Category::OTHER_MISC); - $time = $consoletools->convertTime(time() - $timestart); + $time = $consoletools->convertTime(TIME() - $timestart); echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject."); } else if ($argv[1] == 6 && $argv[2] == 'true') { echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient."); - $timestart = time(); + $timestart = TIME(); $relcount = $releases->categorizeRelease('searchname', ''); $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); - $time = $consoletools->convertTime(time() - $timestart); + $time = $consoletools->convertTime(TIME() - $timestart); echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); } else if ($argv[1] == 6 && $argv[2] == 'false') { echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient."); - $timestart = time(); + $timestart = TIME(); $relcount = $releases->categorizeRelease('searchname', sprintf("WHERE categories_id IN (%s, %s, %s, %s, %s, %s)", Category::GAME_OTHER, @@ -63,20 +78,20 @@ if (isset($argv[1]) && isset($argv[2])) { Category::OTHER_MISC )); $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); - $time = $consoletools->convertTime(time() - $timestart); + $time = $consoletools->convertTime(TIME() - $timestart); echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); } else { exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments.")); } } else { exit($pdo->log->error("\nWrong set of arguments.\n" - . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n" - . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n" - . "\nYou must pass a second argument whether to post process or not, true or false\n" - . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n" - . "\nExtra commands::\n" - . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n" - . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n" - . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n" - . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n")); + . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n" + . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n" + . "\nYou must pass a second argument whether to post process or not, true or false\n" + . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n" + . "\nExtra commands::\n" + . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n" + . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n" + . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n" + . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n")); } diff --git a/nntmux/NZBMultiGroup.php b/nntmux/NZBMultiGroup.php index 2bf4ac199..388589057 100644 --- a/nntmux/NZBMultiGroup.php +++ b/nntmux/NZBMultiGroup.php @@ -23,8 +23,10 @@ class NZBMultiGroup extends NZB * Initiate class vars when writing NZB's. * * @access public + * + * @param int $groupID */ - public function initiateForMgrWrite() + public function initiateForWrite($groupID) { $this->_tableNames = [ 'cName' => 'multigroup_collections', diff --git a/nntmux/processing/ProcessReleases.php b/nntmux/processing/ProcessReleases.php index 05a2a8c7f..ce0bb9e62 100755 --- a/nntmux/processing/ProcessReleases.php +++ b/nntmux/processing/ProcessReleases.php @@ -4,21 +4,19 @@ namespace nntmux\processing; use app\models\MultigroupPosters; use app\models\ReleasesGroups; use app\models\Settings; -use nntmux\db\DB; -use nntmux\Groups; -use nntmux\ConsoleTools; -use nntmux\Releases; -use nntmux\ReleaseCleaning; -use nntmux\ReleaseImage; -use nntmux\NZB; use nntmux\Categorize; use nntmux\Category; +use nntmux\ConsoleTools; +use nntmux\Genres; +use nntmux\Groups; +use nntmux\NZB; +use nntmux\PreDb; +use nntmux\ReleaseCleaning; +use nntmux\ReleaseImage; +use nntmux\Releases; use nntmux\RequestIDLocal; use nntmux\RequestIDWeb; -use nntmux\PreDb; -use nntmux\Genres; -use nntmux\NNTP; -use nntmux\utility\Utility; +use nntmux\db\DB; class ProcessReleases { @@ -35,7 +33,7 @@ class ProcessReleases const FILE_COMPLETE = 1; // We have all the parts for the file (binaries table partcheck column). /** - * @var Groups + * @var \nntmux\Groups */ public $groups; @@ -80,33 +78,43 @@ class ProcessReleases public $pdo; /** - * @var ConsoleTools + * @var \nntmux\ConsoleTools */ public $consoleTools; /** - * @var NZB + * @var \nntmux\NZB */ public $nzb; /** - * @var ReleaseCleaning + * @var \nntmux\ReleaseCleaning */ public $releaseCleaning; /** - * @var Releases + * @var \nntmux\Releases */ public $releases; /** - * @var ReleaseImage + * @var \nntmux\ReleaseImage */ public $releaseImage; /** - * @var int Time (hours) to wait before creating a stuck collection into a release. - */ + * @var array $tables List of table names to be using for method calls. + */ + protected $tables = []; + + /** + * @var string $fromNamesQuery + */ + protected $fromNamesQuery; + + /** + * @var int Time (hours) to wait before delete a stuck/broken collection. + */ private $collectionTimeout; /** @@ -137,14 +145,18 @@ class ProcessReleases $this->releaseImage = ($options['ReleaseImage'] instanceof ReleaseImage ? $options['ReleaseImage'] : new ReleaseImage($this->pdo)); $this->tablePerGroup = (Settings::value('..tablepergroup') == 0 ? false : true); - $this->collectionDelayTime = (Settings::value('..delaytime') != '' ? (int)Settings::value('..delaytime') : 2); - $this->crossPostTime = (Settings::value('..crossposttime') != '' ? (int)Settings::value('..crossposttime') : 2); - $this->releaseCreationLimit = (Settings::value('..maxnzbsprocessed') != '' ? (int)Settings::value('..maxnzbsprocessed') : 1000); - $this->completion = (Settings::value('..completionpercent') != '' ? (int)Settings::value('..completionpercent') : 0); - $this->processRequestIDs = (int)Settings::value('..lookup_reqids'); + $dummy = Settings::value('..delaytime'); + $this->collectionDelayTime = ($dummy != '' ? (int)$dummy : 2); + $dummy = Settings::value('..crossposttime'); + $this->crossPostTime = ($dummy != '' ? (int)$dummy : 2); + $dummy = Settings::value('..maxnzbsprocessed'); + $this->releaseCreationLimit = ($dummy != '' ? (int)$dummy : 1000); + $dummy = Settings::value('..releasecompletion'); + $this->completion = ($dummy != '' ? (int)$dummy : 0); + $this->processRequestIDs = (int)Settings::value('lookup_reqids'); if ($this->completion > 100) { $this->completion = 100; - echo $this->pdo->log->error(PHP_EOL . 'You have an invalid setting for completion. It must be lower than 100.'); + echo $this->pdo->log->error(PHP_EOL . 'You have an invalid setting for completion. It cannot be higher than 100.'); } $this->collectionTimeout = intval(Settings::value('indexer.processing.collection_timeout')); } @@ -155,7 +167,7 @@ class ProcessReleases * @param int $categorize * @param int $postProcess * @param string $groupName (optional) - * @param NNTP $nntp + * @param \nntmux\NNTP $nntp * @param bool $echooutput * * @return int @@ -165,7 +177,7 @@ class ProcessReleases $this->echoCLI = ($echooutput && NN_ECHOCLI); $groupID = ''; - if (!empty($groupName)) { + if (!empty($groupName) && $groupName !== 'mgr') { $groupInfo = $this->groups->getByName($groupName); $groupID = $groupInfo['id']; } @@ -177,7 +189,10 @@ class ProcessReleases if (!file_exists(Settings::value('..nzbpath'))) { if ($this->echoCLI) { - $this->pdo->log->doEcho($this->pdo->log->error('Bad or missing nzb directory - ' . Settings::value('..nzbpath')), true); + $this->pdo->log->doEcho( + $this->pdo->log->error('Bad or missing nzb directory - ' . Settings::value('..nzbpath')), + true + ); } return 0; @@ -187,21 +202,14 @@ class ProcessReleases $this->processCollectionSizes($groupID); $this->deleteUnwantedCollections($groupID); - (new ProcessReleasesMultiGroup(['Settings' => $this->pdo]))->processIncompleteMgrCollections($groupID); - (new ProcessReleasesMultiGroup(['Settings' => $this->pdo]))->processMgrCollectionSizes($groupID); - (new ProcessReleasesMultiGroup(['Settings' => $this->pdo]))->deleteUnwantedMgrCollections($groupID); - $DIR = NN_MISC; $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); - $mgrReleasesCount = (new ProcessReleasesMultiGroup(['Settings' => $this->pdo]))->createMGRReleases($groupID); - $totalReleasesAdded += $releasesCount['added'] += $mgrReleasesCount['added']; + $totalReleasesAdded += $releasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); - $mgrFilesAdded = (new ProcessReleasesMultiGroup(['Settings' => $this->pdo]))->createMGRNZBs($groupID); - $this->deleteCollections($groupID); if ($this->processRequestIDs === 0) { $this->processRequestIDs($groupID, 5000, true); } else if ($this->processRequestIDs === 1) { @@ -210,9 +218,9 @@ class ProcessReleases } else if ($this->processRequestIDs === 2) { $requestIDTime = time(); if ($this->echoCLI) { - $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request id Threaded lookup.")); + $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request ID Threaded lookup.")); } - passthru("{$DIR}update/nix/multiprocessing/requestid.php"); + passthru("${DIR}update/nix/multiprocessing/requestid.php"); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( @@ -225,14 +233,19 @@ class ProcessReleases $this->categorizeReleases($categorize, $groupID); $this->postProcessReleases($postProcess, $nntp); + $this->deleteCollections($groupID); - // This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately - } while (($releasesCount['added'] + $releasesCount['dupes']+ $mgrReleasesCount['added'] + $mgrReleasesCount['dupes']) >= $this->releaseCreationLimit || $nzbFilesAdded + $mgrFilesAdded >= $this->releaseCreationLimit); + // This loops as long as the number of releases or nzbs added was >= the limit (meaning there are more waiting to be created) + } while ( + ($releasesCount['added'] + $releasesCount['dupes']) >= $this->releaseCreationLimit + || $nzbFilesAdded >= $this->releaseCreationLimit + ); - - - $this->deletedReleasesByGroup($groupID); - $this->deleteReleases(); + // Only run if non-mgr as mgr is not specific to group + if ($groupName !== 'mgr') { + $this->deletedReleasesByGroup($groupID); + $this->deleteReleases(); + } //Print amount of added releases and time it took. if ($this->echoCLI && $this->tablePerGroup === false) { @@ -264,7 +277,7 @@ class ProcessReleases public function resetCategorize($where = '') { $this->pdo->queryExec( - sprintf('UPDATE releases SET categories_id = %d, iscategorized = 0 %s', \Category::OTHER_MISC, $where) + sprintf('UPDATE releases SET categories_id = %d, iscategorized = 0 %s', Category::OTHER_MISC, $where) ); } @@ -281,13 +294,26 @@ class ProcessReleases { $cat = new Categorize(['Settings' => $this->pdo]); $categorized = $total = 0; - $releases = $this->pdo->queryDirect(sprintf('SELECT id, fromname, %s, groups_id FROM releases %s', $type, $where)); + $releases = $this->pdo->queryDirect( + sprintf(' + SELECT id, fromname, %s, groups_id + FROM releases %s', + $type, + $where + ) + ); if ($releases && $releases->rowCount()) { $total = $releases->rowCount(); foreach ($releases as $release) { $catId = $cat->determineCategory($release['groups_id'], $release[$type], $release['fromname']); $this->pdo->queryExec( - sprintf('UPDATE releases SET categories_id = %d, iscategorized = 1 WHERE id = %d', $catId, $release['id']) + sprintf(' + UPDATE releases + SET categories_id = %d, iscategorized = 1 + WHERE id = %d', + $catId, + $release['id'] + ) ); $categorized++; if ($this->echoCLI) { @@ -304,23 +330,9 @@ class ProcessReleases } public function processIncompleteCollections($groupID) - { - $tableNames = $this->initiateTableNames($groupID); - $this->processIncompleteCollectionsMain($groupID, $tableNames); - } - - /** - * Find complete collections to be processed by processCollectionSizes. - * - * @param int $groupID - * @param $tableNames - * - * @void - * @access public - */ - public function processIncompleteCollectionsMain($groupID, $tableNames) { $startTime = time(); + $this->initiateTableNames($groupID); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Attempting to find complete collections.")); @@ -328,19 +340,21 @@ class ProcessReleases $where = (!empty($groupID) ? ' AND c.group_id = ' . $groupID . ' ' : ' '); - $this->processStuckCollections($tableNames, $where); - $this->collectionFileCheckStage1($tableNames, $where); - $this->collectionFileCheckStage2($tableNames, $where); - $this->collectionFileCheckStage3($tableNames, $where); - $this->collectionFileCheckStage4($tableNames, $where); - $this->collectionFileCheckStage5($tableNames, $where); - $this->collectionFileCheckStage6($tableNames, $where); + $this->processStuckCollections($where); + $this->collectionFileCheckStage1($where); + $this->collectionFileCheckStage2($where); + $this->collectionFileCheckStage3($where); + $this->collectionFileCheckStage4($where); + $this->collectionFileCheckStage5($where); + $this->collectionFileCheckStage6($where); if ($this->echoCLI) { $count = $this->pdo->queryOneRow( - sprintf( - 'SELECT COUNT(id) AS complete FROM %s c WHERE filecheck = %d %s', - $tableNames['cname'], + sprintf(' + SELECT COUNT(c.id) AS complete + FROM %s c + WHERE c.filecheck = %d %s', + $this->tables['cname'], self::COLLFC_COMPPART, $where ) @@ -355,43 +369,31 @@ class ProcessReleases } public function processCollectionSizes($groupID) - { - $tableNames = $this->initiateTableNames($groupID); - $this->processCollectionSizesMain($groupID, $tableNames); - } - - /** - * Calculate the total size of a complete collection (COLLFC_COMPPART) in bytes. - * Set the collection to (COLLFC_SIZED) - * - * @param string|int $groupID (optional) - * - * @param $tableNames - * - * @void - * @access public - */ - public function processCollectionSizesMain($groupID, $tableNames) { $startTime = time(); + $this->initiateTableNames($groupID); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Calculating collection sizes (in bytes).")); } - // Get the total size in bytes of the collection for collections where filecheck = 2. $checked = $this->pdo->queryExec( - sprintf( - 'UPDATE %s c - SET filesize = (SELECT COALESCE(SUM(b.partsize), 0) FROM %s b WHERE b.collection_id = c.id), - filecheck = %d + sprintf(' + UPDATE %s c + SET c.filesize = + ( + SELECT COALESCE(SUM(b.partsize), 0) + FROM %s b + WHERE b.collection_id = c.id + ), + c.filecheck = %d WHERE c.filecheck = %d AND c.filesize = 0 %s', - $tableNames['cname'], - $tableNames['bname'], + $this->tables['cname'], + $this->tables['bname'], self::COLLFC_SIZED, self::COLLFC_COMPPART, - (!empty($groupID) ? ' AND c.group_id = ' . $groupID : '') + (!empty($groupID) ? ' AND c.group_id = ' . $groupID : ' ') ) ); if ($checked !== false && $this->echoCLI) { @@ -405,24 +407,9 @@ class ProcessReleases } public function deleteUnwantedCollections($groupID) - { - $tableNames = $this->initiateTableNames($groupID); - $this->deleteUnwantedCollectionsMain($groupID, $tableNames); - } - - /** - * Delete unwanted collections based on size/file count using admin settings. - * - * @param int|string $groupID (optional) - * - * @param $tableNames - * - * @void - * @access public - */ - public function deleteUnwantedCollectionsMain($groupID, $tableNames) { $startTime = time(); + $this->initiateTableNames($groupID); if ($this->echoCLI) { $this->pdo->log->doEcho( @@ -459,9 +446,12 @@ class ProcessReleases } if ($this->pdo->queryOneRow( - sprintf( - 'SELECT SQL_NO_CACHE id FROM %s c WHERE c.filecheck = %d AND c.filesize > 0 %s LIMIT 1', - $tableNames['cname'], + sprintf(' + SELECT SQL_NO_CACHE id + FROM %s c + WHERE c.filecheck = %d + AND c.filesize > 0 %s', + $this->tables['cname'], self::COLLFC_SIZED, $this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '' ) @@ -478,9 +468,9 @@ class ProcessReleases AND c.filesize > 0 AND GREATEST(%d, %d) > 0 AND c.filesize < GREATEST(%d, %d)', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], self::COLLFC_SIZED, $this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '', $groupMinSizeSetting, @@ -502,9 +492,9 @@ class ProcessReleases LEFT JOIN %s p ON b.id = p.binaryid WHERE c.filecheck = %d %s AND c.filesize > %d', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], self::COLLFC_SIZED, $this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '', $maxSizeSetting @@ -518,14 +508,14 @@ class ProcessReleases $deleteQuery = $this->pdo->queryExec( sprintf(' DELETE c, b, p FROM %s c - LEFT JOIN %s b ON (c.id=b.collection_id) - LEFT JOIN %s p ON (b.id=p.binaryid) + LEFT JOIN %s b ON c.id = b.collection_id + LEFT JOIN %s p ON b.id = p.binaryid WHERE c.filecheck = %d %s AND GREATEST(%d, %d) > 0 AND c.totalfiles < GREATEST(%d, %d)', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], self::COLLFC_SIZED, $this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '', $groupMinFilesSetting, @@ -553,32 +543,36 @@ class ProcessReleases } } + /** + * @param $groupID + * + * @void + */ protected function initiateTableNames($groupID) { - return $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $this->tables = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); } - public function createReleases($groupID) - { - $tableNames = $this->initiateTableNames($groupID); - return $this->createReleasesMain($groupID, $tableNames); - } - - /** - * Create releases from complete collections. + * Form fromNamesQuery for creating NZBs * + * @void + */ + protected function formFromNamesQuery() + { + $posters = MultigroupPosters::commaSeparatedList(); + $this->fromNamesQuery = sprintf("AND r.fromname NOT IN('%s')", $posters); + } + + /** * @param int|string $groupID (optional) * - * @param array $tableNames - * * @return array - * @access public */ - public function createReleasesMain($groupID, $tableNames) + public function createReleases($groupID) { $startTime = time(); - $group = $tableNames; + $this->initiateTableNames($groupID); $categorize = new Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; @@ -591,16 +585,14 @@ class ProcessReleases $collections = $this->pdo->queryDirect( sprintf(' - SELECT SQL_NO_CACHE %s.*, groups.name AS gname - FROM %s - INNER JOIN groups ON %s.group_id = groups.id - WHERE %s %s.filecheck = %d - AND filesize > 0 LIMIT %d', - $group['cname'], - $group['cname'], - $group['cname'], - (!empty($groupID) ? ' group_id = ' . $groupID . ' AND ' : ' '), - $group['cname'], + SELECT SQL_NO_CACHE c.*, g.name AS gname + FROM %s c + INNER JOIN groups g ON c.group_id = g.id + WHERE %s c.filecheck = %d + AND c.filesize > 0 + LIMIT %d', + $this->tables['cname'], + (!empty($groupID) ? ' c.group_id = ' . $groupID . ' AND ' : ' '), self::COLLFC_SIZED, $this->releaseCreationLimit ) @@ -632,8 +624,7 @@ class ProcessReleases FROM releases WHERE name = %s AND fromname = %s - AND size BETWEEN '%s' - AND '%s'", + AND size BETWEEN '%s' AND '%s'", $cleanRelName, $fromName, ($collection['filesize'] * .99), @@ -677,7 +668,7 @@ class ProcessReleases 'postdate' => $this->pdo->escapeString($collection['date']), 'fromname' => $fromName, 'size' => $collection['filesize'], - 'categories_id' => $categorize->determineCategory($collection['group_id'], $cleanedName, $fromName), + 'categories_id' => $categorize->determineCategory($collection['group_id'], $cleanedName), 'isrenamed' => ($properName === true ? 1 : 0), 'reqidstatus' => ($isReqID === true ? 1 : 0), 'predb_id' => ($preID === false ? 0 : $preID), @@ -692,7 +683,7 @@ class ProcessReleases UPDATE %s SET filecheck = %d, releaseid = %d WHERE id = %d', - $group['cname'], + $this->tables['cname'], self::COLLFC_INSERTED, $releaseID, $collection['id'] @@ -722,13 +713,27 @@ class ProcessReleases ); } - $relGroups = ReleasesGroups::create( + $relGroupsChk = ReleasesGroups::find('first', [ - 'releases_id' => $releaseID, - 'groups_id' => $xrefGrpID, + 'conditions' => + [ + 'releases_id' => $releaseID, + 'groups_id' => $xrefGrpID, + ], + 'fields' => ['releases_id'], + 'limit' => 1, ] ); - $relGroups->save(); + + if ($relGroupsChk === null) { + $relGroups = ReleasesGroups::create( + [ + 'releases_id' => $releaseID, + 'groups_id' => $xrefGrpID, + ] + ); + $relGroups->save(); + } } } } @@ -743,11 +748,14 @@ class ProcessReleases // The release was already in the DB, so delete the collection. $this->pdo->queryExec( sprintf(' - DELETE c, b, p FROM %s c - INNER JOIN %s b ON(c.id=b.collection_id) - STRAIGHT_JOIN %s p ON(b.id=p.binaryid) + DELETE c, b, p + FROM %s c + INNER JOIN %s b ON c.id = b.collection_id + STRAIGHT_JOIN %s p ON b.id = p.binaryid WHERE c.collectionhash = %s', - $group['cname'], $group['bname'], $group['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], $this->pdo->escapeString($collection['collectionhash']) ) ); @@ -783,34 +791,23 @@ class ProcessReleases public function createNZBs($groupID) { $startTime = time(); + $this->formFromnamesQuery(); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create the NZB, delete collections/binaries/parts.")); } - $list = []; - $posters = MultigroupPosters::find('all', - [ - 'fields' => ['poster'], - 'order' => ['poster' => 'ASC'], - ] - ); - - foreach ($posters as $poster) { - $list[] = $poster->poster; - } - $posters = implode("','", $list); $releases = $this->pdo->queryDirect( sprintf(" - SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, + SELECT SQL_NO_CACHE + CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, r.name, r.id, r.guid FROM releases r INNER JOIN categories c ON r.categories_id = c.id INNER JOIN categories cp ON cp.id = c.parentid - WHERE %s nzbstatus = 0 - AND r.fromname NOT IN('%s')", + WHERE %s nzbstatus = 0 %s", (!empty($groupID) ? ' r.groups_id = ' . $groupID . ' AND ' : ' '), - $posters + $this->fromNamesQuery ) ); @@ -867,7 +864,7 @@ class ProcessReleases $this->pdo->log->doEcho( $this->pdo->log->header( sprintf( - "Process Releases -> Request id %s lookup -- limit %s", + "Process Releases -> Request ID %s lookup -- limit %s", ($local === true ? 'local' : 'web'), $limit ) @@ -877,17 +874,25 @@ class ProcessReleases if ($local === true) { $foundRequestIDs = ( - new RequestIDLocal( - ['Echo' => $this->echoCLI, 'ConsoleTools' => $this->consoleTools, - 'Groups' => $this->groups, 'Settings' => $this->pdo] - ) + new RequestIDLocal( + [ + 'Echo' => $this->echoCLI, + 'ConsoleTools' => $this->consoleTools, + 'Groups' => $this->groups, + 'Settings' => $this->pdo, + ] + ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } else { $foundRequestIDs = ( - new RequestIDWeb( - ['Echo' => $this->echoCLI, 'ConsoleTools' => $this->consoleTools, - 'Groups' => $this->groups, 'Settings' => $this->pdo] - ) + new RequestIDWeb( + [ + 'Echo' => $this->echoCLI, + 'ConsoleTools' => $this->consoleTools, + 'Groups' => $this->groups, + 'Settings' => $this->pdo, + ] + ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } if ($this->echoCLI) { @@ -929,9 +934,9 @@ class ProcessReleases $this->categorizeRelease( $type, (!empty($groupID) - ? 'WHERE categories_id = ' . Category::OTHER_MISC . ' AND iscategorized = 0 AND groups_id = ' . $groupID - : 'WHERE categories_id = ' . Category::OTHER_MISC . ' AND iscategorized = 0') - ); + ? 'WHERE categories_id = ' . Category::OTHER_MISC . ' AND iscategorized = 0 AND group_id = ' . $groupID + : 'WHERE categories_id = ' . Category::OTHER_MISC . ' AND iscategorized = 0') + ); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary($this->consoleTools->convertTime(time() - $startTime)), true); @@ -942,7 +947,7 @@ class ProcessReleases * Post-process releases. * * @param int $postProcess - * @param NNTP $nntp + * @param \nntmux\NNTP $nntp * * @void * @access public @@ -964,24 +969,9 @@ class ProcessReleases } public function deleteCollections($groupID) - { - $tableNames = $this->initiateTableNames($groupID); - $this->deleteCollectionsMain($groupID, $tableNames); - } - - /** - * Delete collections (complete/incomplete/old/etc). - * - * @param int|string $groupID (optional) - * - * @param $tableNames - * - * @void - * @access public - */ - public function deleteCollectionsMain($groupID, $tableNames) { $startTime = time(); + $this->initiateTableNames($groupID); $deletedCount = 0; @@ -998,14 +988,15 @@ class ProcessReleases $deleted = 0; $deleteQuery = $this->pdo->queryExec( - sprintf( - 'DELETE c, b, p FROM %s c - LEFT JOIN %s b ON (c.id=b.collection_id) - LEFT JOIN %s p ON (b.id=p.binaryid) + sprintf(' + DELETE c, b, p + FROM %s c + LEFT JOIN %s b ON c.id = b.collection_id + LEFT JOIN %s p ON b.id = p.binaryid WHERE (c.dateadded < NOW() - INTERVAL %d HOUR) %s', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], Settings::value('..partretentionhours'), (!empty($groupID) && $this->tablePerGroup === false ? ' AND c.group_id = ' . $groupID : '') ) @@ -1027,7 +1018,7 @@ class ProcessReleases // Cleanup orphaned collections, binaries and parts // this really shouldn't happen, but just incase - so we only run 1/200 of the time - if (mt_rand(0, 200) <= 1 ) { + if (mt_rand(0, 200) <= 1) { // CBP collection orphaned with no binaries or parts. if ($this->echoCLI) { echo ( @@ -1038,14 +1029,15 @@ class ProcessReleases $deleted = 0; $deleteQuery = $this->pdo->queryExec( - sprintf( - 'DELETE c, b, p FROM %s c - LEFT JOIN %s b ON (c.id=b.collection_id) - LEFT JOIN %s p ON (b.id=p.binaryid) + sprintf(' + DELETE c, b, p + FROM %s c + LEFT JOIN %s b ON c.id = b.collection_id + LEFT JOIN %s p ON b.id = p.binaryid WHERE (b.id IS NULL OR p.binaryid IS NULL) %s', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], (!empty($groupID) && $this->tablePerGroup === false ? ' AND c.group_id = ' . $groupID : '') ) ); @@ -1074,10 +1066,14 @@ class ProcessReleases $deleteQuery = $this->pdo->queryExec( sprintf( 'DELETE b, p FROM %s b - LEFT JOIN %s p ON(b.id=p.binaryid) - LEFT JOIN %s c ON(b.collection_id=c.id) - WHERE (p.binaryid IS NULL OR c.id IS NULL) AND b.id < %d ', - $tableNames['bname'], $tableNames['pname'], $tableNames['cname'], $this->maxQueryFormulator($tableNames['bname'], 20000) + LEFT JOIN %s p ON b.id = p.binaryid + LEFT JOIN %s c ON b.collection_id = c.id + WHERE (p.binaryid IS NULL OR c.id IS NULL) + AND b.id < %d', + $this->tables['bname'], + $this->tables['pname'], + $this->tables['cname'], + $this->maxQueryFormulator($this->tables['bname'], 20000) ) ); @@ -1102,9 +1098,15 @@ class ProcessReleases } $deleted = 0; $deleteQuery = $this->pdo->queryExec( - sprintf( - 'DELETE p FROM %s p LEFT JOIN %s b ON (p.binaryid=b.id) WHERE b.id IS NULL AND p.binaryid < %d', - $tableNames['pname'], $tableNames['bname'], $this->maxQueryFormulator($tableNames['bname'], 20000) + sprintf(' + DELETE p + FROM %s p + LEFT JOIN %s b ON p.binaryid = b.id + WHERE b.id IS NULL + AND p.binaryid < %d', + $this->tables['pname'], + $this->tables['bname'], + $this->maxQueryFormulator($this->tables['bname'], 20000) ) ); if ($deleteQuery !== false) { @@ -1136,7 +1138,7 @@ class ProcessReleases FROM %s c INNER JOIN releases r ON r.id = c.releaseid WHERE r.nzbstatus = 1', - $tableNames['cname'] + $this->tables['cname'] ) ); @@ -1150,9 +1152,9 @@ class ProcessReleases LEFT JOIN %s b ON(c.id=b.collection_id) LEFT JOIN %s p ON(b.id=p.binaryid) WHERE c.id = %d', - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], $collection['id'] ) ); @@ -1205,7 +1207,7 @@ class ProcessReleases foreach ($groupIDs as $groupID) { $releases = $this->pdo->queryDirect( sprintf(" - SELECT r.guid, r.id + SELECT SQL_NO_CACHE r.guid, r.id FROM releases r INNER JOIN groups g ON g.id = r.groups_id WHERE r.groups_id = %d @@ -1226,7 +1228,7 @@ class ProcessReleases if ($maxSizeSetting > 0) { $releases = $this->pdo->queryDirect( sprintf(' - SELECT id, guid + SELECT SQL_NO_CACHE id, guid FROM releases WHERE groups_id = %d AND size > %d', @@ -1244,7 +1246,7 @@ class ProcessReleases $releases = $this->pdo->queryDirect( sprintf(" - SELECT r.id, r.guid + SELECT SQL_NO_CACHE r.id, r.guid FROM releases r INNER JOIN groups g ON g.id = r.groups_id WHERE r.groups_id = %d @@ -1300,7 +1302,7 @@ class ProcessReleases if (Settings::value('..releaseretentiondays') != 0) { $releases = $this->pdo->queryDirect( sprintf( - 'SELECT id, guid FROM releases WHERE postdate < (NOW() - INTERVAL %d DAY)', + 'SELECT SQL_NO_CACHE id, guid FROM releases WHERE postdate < (NOW() - INTERVAL %d DAY)', Settings::value('..releaseretentiondays') ) ); @@ -1316,7 +1318,7 @@ class ProcessReleases if (Settings::value('..deletepasswordedrelease') == 1) { $releases = $this->pdo->queryDirect( sprintf( - 'SELECT id, guid FROM releases WHERE passwordstatus = %d', + 'SELECT SQL_NO_CACHE id, guid FROM releases WHERE passwordstatus = %d', Releases::PASSWD_RAR ) ); @@ -1332,7 +1334,7 @@ class ProcessReleases if (Settings::value('..deletepossiblerelease') == 1) { $releases = $this->pdo->queryDirect( sprintf( - 'SELECT id, guid FROM releases WHERE passwordstatus = %d', + 'SELECT SQL_NO_CACHE id, guid FROM releases WHERE passwordstatus = %d', Releases::PASSWD_POTENTIAL ) ); @@ -1362,7 +1364,7 @@ class ProcessReleases if ($this->completion > 0) { $releases = $this->pdo->queryDirect( - sprintf('SELECT id, guid FROM releases WHERE completion < %d AND completion > 0', $this->completion) + sprintf('SELECT SQL_NO_CACHE id, guid FROM releases WHERE completion < %d AND completion > 0', $this->completion) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { @@ -1377,7 +1379,7 @@ class ProcessReleases if (count($disabledCategories) > 0) { foreach ($disabledCategories as $disabledCategory) { $releases = $this->pdo->queryDirect( - sprintf('SELECT id, guid FROM releases WHERE categories_id = %d', $disabledCategory['id']) + sprintf('SELECT SQL_NO_CACHE id, guid FROM releases WHERE categories_id = %d', $disabledCategory['id']) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { @@ -1390,7 +1392,7 @@ class ProcessReleases // Delete smaller than category minimum sizes. $categories = $this->pdo->queryDirect(' - SELECT c.id AS id, + SELECT SQL_NO_CACHE c.id AS id, CASE WHEN c.minsizetoformrelease = 0 THEN cp.minsizetoformrelease ELSE c.minsizetoformrelease END AS minsize FROM categories c INNER JOIN categories cp ON cp.id = c.parentid @@ -1402,10 +1404,11 @@ class ProcessReleases if ($category['minsize'] > 0) { $releases = $this->pdo->queryDirect( sprintf(' - SELECT r.id, r.guid + SELECT SQL_NO_CACHE r.id, r.guid FROM releases r WHERE r.categories_id = %d - AND r.size < %d LIMIT 1000', + AND r.size < %d + LIMIT 1000', $category['id'], $category['minsize'] ) @@ -1426,10 +1429,14 @@ class ProcessReleases foreach ($genrelist as $genre) { $releases = $this->pdo->queryDirect( sprintf(' - SELECT id, guid + SELECT SQL_NO_CACHE id, guid FROM releases - INNER JOIN (SELECT id AS mid FROM musicinfo WHERE musicinfo.genres_id = %d) mi - ON musicinfo_id = mid', + INNER JOIN + ( + SELECT id AS mid + FROM musicinfo + WHERE musicinfo.genre_id = %d + ) mi ON musicinfo_id = mid', $genre['id'] ) ); @@ -1446,7 +1453,7 @@ class ProcessReleases if (Settings::value('..miscotherretentionhours') > 0) { $releases = $this->pdo->queryDirect( sprintf(' - SELECT id, guid + SELECT SQL_NO_CACHE id, guid FROM releases WHERE categories_id = %d AND adddate <= NOW() - INTERVAL %d HOUR', @@ -1466,7 +1473,7 @@ class ProcessReleases if (Settings::value('..mischashedretentionhours') > 0) { $releases = $this->pdo->queryDirect( sprintf(' - SELECT id, guid + SELECT SQL_NO_CACHE id, guid FROM releases WHERE categories_id = %d AND adddate <= NOW() - INTERVAL %d HOUR', @@ -1534,9 +1541,15 @@ class ProcessReleases * @return string * @access private */ - protected function maxQueryFormulator($groupName, $difference) + private function maxQueryFormulator($groupName, $difference) { - $maxID = $this->pdo->queryOneRow(sprintf('SELECT IFNULL(MAX(id),0) AS max FROM %s', $groupName )); + $maxID = $this->pdo->queryOneRow( + sprintf(' + SELECT IFNULL(MAX(id),0) AS max + FROM %s', + $groupName + ) + ); return empty($maxID['max']) || $maxID['max'] < $difference ? 0 : $maxID['max'] - $difference; } @@ -1546,27 +1559,31 @@ class ProcessReleases * This means the the binary table has the same count as the file count in the subject, but * the collection might not be complete yet since we might not have all the articles in the parts table. * - * @param array $tableNames * @param string $where * * @void - * @access private + * @access private */ - private function collectionFileCheckStage1(array &$tableNames, &$where) + private function collectionFileCheckStage1(&$where) { + $this->pdo->queryExec( sprintf(' - UPDATE %s c INNER JOIN - (SELECT c.id FROM %s c + UPDATE %s c + INNER JOIN + ( + SELECT c.id + FROM %s c INNER JOIN %s b ON b.collection_id = c.id - WHERE c.totalfiles > 0 AND c.filecheck = %d %s + WHERE c.totalfiles > 0 + AND c.filecheck = %d %s GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) IN (c.totalfiles, c.totalfiles + 1) - ) - r ON c.id = r.id SET filecheck = %d', - $tableNames['cname'], - $tableNames['cname'], - $tableNames['bname'], + ) r ON c.id = r.id + SET filecheck = %d', + $this->tables['cname'], + $this->tables['cname'], + $this->tables['bname'], self::COLLFC_DEFAULT, $where, self::COLLFC_COMPCOLL @@ -1582,27 +1599,30 @@ class ProcessReleases * at 0 then you would never get a complete collection if it starts with 1 and if it starts, you can end up creating * a incomplete collection, since you assumed it was complete. * - * @param array $tableNames * @param string $where + * * @void * @access private */ - private function collectionFileCheckStage2(array &$tableNames, &$where) + private function collectionFileCheckStage2(&$where) { $this->pdo->queryExec( sprintf(' - UPDATE %s c INNER JOIN - (SELECT c.id FROM %s c + UPDATE %s c + INNER JOIN + ( + SELECT c.id + FROM %s c INNER JOIN %s b ON b.collection_id = c.id WHERE b.filenumber = 0 AND c.totalfiles > 0 AND c.filecheck = %d %s GROUP BY c.id - ) - r ON c.id = r.id SET c.filecheck = %d', - $tableNames['cname'], - $tableNames['cname'], - $tableNames['bname'], + ) r ON c.id = r.id + SET c.filecheck = %d', + $this->tables['cname'], + $this->tables['cname'], + $this->tables['bname'], self::COLLFC_COMPCOLL, $where, self::COLLFC_ZEROPART @@ -1613,7 +1633,7 @@ class ProcessReleases UPDATE %s c SET filecheck = %d WHERE filecheck = %d %s', - $tableNames['cname'], + $this->tables['cname'], self::COLLFC_TEMPCOMP, self::COLLFC_COMPCOLL, $where @@ -1625,27 +1645,31 @@ class ProcessReleases * Check if the files (binaries table) in a complete collection has all the parts. * If we have all the parts, set binaries table partcheck to FILE_COMPLETE. * - * @param array $tableNames * @param string $where * - * * @void * @access private */ - private function collectionFileCheckStage3(array &$tableNames, $where) + private function collectionFileCheckStage3($where) { + $this->pdo->queryExec( sprintf(' - UPDATE %s b INNER JOIN - (SELECT b.id FROM %s b + UPDATE %s b + INNER JOIN + ( + SELECT b.id + FROM %s b INNER JOIN %s c ON c.id = b.collection_id - WHERE c.filecheck = %d AND b.partcheck = %d %s + WHERE c.filecheck = %d + AND b.partcheck = %d %s AND b.currentparts = b.totalparts - GROUP BY b.id, b.totalparts) - r ON b.id = r.id SET b.partcheck = %d', - $tableNames['bname'], - $tableNames['bname'], - $tableNames['cname'], + GROUP BY b.id, b.totalparts + ) r ON b.id = r.id + SET b.partcheck = %d', + $this->tables['bname'], + $this->tables['bname'], + $this->tables['cname'], self::COLLFC_TEMPCOMP, self::FILE_INCOMPLETE, $where, @@ -1654,16 +1678,21 @@ class ProcessReleases ); $this->pdo->queryExec( sprintf(' - UPDATE %s b INNER JOIN - (SELECT b.id FROM %s b + UPDATE %s b + INNER JOIN + ( + SELECT b.id + FROM %s b INNER JOIN %s c ON c.id = b.collection_id - WHERE c.filecheck = %d AND b.partcheck = %d %s + WHERE c.filecheck = %d + AND b.partcheck = %d %s AND b.currentparts >= (b.totalparts + 1) - GROUP BY b.id, b.totalparts) - r ON b.id = r.id SET b.partcheck = %d', - $tableNames['bname'], - $tableNames['bname'], - $tableNames['cname'], + GROUP BY b.id, b.totalparts + ) r ON b.id = r.id + SET b.partcheck = %d', + $this->tables['bname'], + $this->tables['bname'], + $this->tables['cname'], self::COLLFC_ZEROPART, self::FILE_INCOMPLETE, $where, @@ -1677,13 +1706,12 @@ class ProcessReleases * Set collections filecheck column to COLLFC_COMPPART. * This means the collection is complete. * - * @param array $tableNames * @param string $where * * @void * @access private */ - private function collectionFileCheckStage4(array &$tableNames, &$where) + private function collectionFileCheckStage4(&$where) { $this->pdo->queryExec( @@ -1694,9 +1722,9 @@ class ProcessReleases WHERE b.partcheck = 1 AND c.filecheck IN (%d, %d) %s GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) >= c.totalfiles) r ON c.id = r.id SET filecheck = %d', - $tableNames['cname'], - $tableNames['cname'], - $tableNames['bname'], + $this->tables['cname'], + $this->tables['cname'], + $this->tables['bname'], self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART, $where, @@ -1709,20 +1737,20 @@ class ProcessReleases * If not all files (binaries table) had their parts on the previous stage, * reset the collection filecheck column to COLLFC_COMPCOLL so we reprocess them next time. * - * @param array $tableNames * @param string $where * * @void * @access private */ - private function collectionFileCheckStage5(array &$tableNames, &$where) + private function collectionFileCheckStage5(&$where) { + $this->pdo->queryExec( sprintf(' UPDATE %s c SET filecheck = %d WHERE filecheck IN (%d, %d) %s', - $tableNames['cname'], + $this->tables['cname'], self::COLLFC_COMPCOLL, self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART, @@ -1735,22 +1763,22 @@ class ProcessReleases * If a collection did not have the file count (ie: [00/12]) or the collection is incomplete after * $this->collectionDelayTime hours, set the collection to complete to create it into a release/nzb. * - * @param array $tableNames * @param string $where * * @void * @access private */ - private function collectionFileCheckStage6(array &$tableNames, &$where) + private function collectionFileCheckStage6(&$where) { + $this->pdo->queryExec( sprintf(" UPDATE %s c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM %s b WHERE b.collection_id = c.id) WHERE c.dateadded < NOW() - INTERVAL '%d' HOUR AND c.filecheck IN (%d, %d, 10) %s", - $tableNames['cname'], + $this->tables['cname'], self::COLLFC_COMPPART, - $tableNames['bname'], + $this->tables['bname'], $this->collectionDelayTime, self::COLLFC_DEFAULT, self::COLLFC_COMPCOLL, @@ -1762,13 +1790,12 @@ class ProcessReleases /** * If a collection has been stuck for $this->collectionTimeout hours, delete it, it's bad. * - * @param array $tableNames * @param string $where * * @void * @access private */ - private function processStuckCollections(array $tableNames, $where) + private function processStuckCollections($where) { $lastRun = Settings::value('indexer.processing.last_run_time'); @@ -1781,9 +1808,9 @@ class ProcessReleases c.added < DATE_SUB({$this->pdo->escapeString($lastRun)}, INTERVAL %d HOUR) %s", - $tableNames['cname'], - $tableNames['bname'], - $tableNames['pname'], + $this->tables['cname'], + $this->tables['bname'], + $this->tables['pname'], $this->collectionTimeout, $where ) diff --git a/nntmux/processing/ProcessReleasesMultiGroup.php b/nntmux/processing/ProcessReleasesMultiGroup.php index 81d0722fd..d72a6d140 100644 --- a/nntmux/processing/ProcessReleasesMultiGroup.php +++ b/nntmux/processing/ProcessReleasesMultiGroup.php @@ -9,17 +9,6 @@ use nntmux\utility\Utility; class ProcessReleasesMultiGroup extends ProcessReleases { - /** - * @var - */ - protected $mgrFromNames; - - /** - * @var NZBMultiGroup - */ - protected $mgrnzb; - - /** * ProcessReleasesMultiGroup constructor. * @@ -28,7 +17,17 @@ class ProcessReleasesMultiGroup extends ProcessReleases public function __construct(array $options = []) { parent::__construct($options); - $this->mgrnzb = new NZBMultiGroup($this->pdo); + $this->nzb = new NZBMultiGroup($this->pdo); + } + + /** + * Form fromNamesQuery for creating NZBs + * + * @void + */ + protected function formFromNamesQuery() + { + $this->fromNamesQuery = ''; } /** @@ -42,126 +41,20 @@ class ProcessReleasesMultiGroup extends ProcessReleases return (empty($poster) ? false : true); } - - protected function initiateMgrTableNames() + /** + * This method exists to prevent the parent one from over-writing the $this->tables property. + * + * @param int $groupID Unused with mgr + * + * @return void + */ + protected function initiateTableNames($groupID) { - $group = [ + $this->tables = [ 'cname' => 'multigroup_collections', 'bname' => 'multigroup_binaries', 'pname' => 'multigroup_parts' ]; - - return $group; - } - - /** - * Process incomplete MultiGroup Releases - * - * @param $groupID - */ - public function processIncompleteMgrCollections($groupID) - { - $tableNames = $this->initiateMgrTableNames(); - $this->processIncompleteCollectionsMain($groupID, $tableNames); - } - - /** - * Process MultiGroup collection sizes - * - * @param $groupID - */ - public function processMgrCollectionSizes($groupID) - { - $tableNames = $this->initiateMgrTableNames(); - $this->processCollectionSizesMain($groupID, $tableNames); - } - - /** - * Delete unwanted MultiGroup collections - * - * @param $groupID - */ - public function deleteUnwantedMgrCollections($groupID) - { - $tableNames = $this->initiateMgrTableNames(); - $this->deleteUnwantedCollectionsMain($groupID, $tableNames); - } - - public function deleteMgrCollections($groupID) - { - $tableNames = $this->initiateMgrTableNames(); - $this->deleteCollectionsMain($groupID, $tableNames); - } - - /** - * Create releases from complete MultiGroup collections. - * - * @param $groupID - * - * @return array - * @access public - */ - public function createMGRReleases($groupID) - { - $tableNames = $this->initiateMgrTableNames(); - return $this->createReleasesMain($groupID, $tableNames); - } - - /** - * Create NZB files from complete MultiGroup releases. - * - * - * @param $groupID - * - * @return int - * @access public - */ - public function createMGRNZBs($groupID) - { - $list = []; - $posters = MultigroupPosters::find('all', - [ - 'fields' => ['poster'], - 'order' => ['poster' => 'ASC'], - ] - ); - - foreach ($posters as $poster) { - $list[] = $poster->poster; - } - - $this->mgrFromNames = implode("','", $list); - - $releases = $this->pdo->queryDirect( - sprintf(" - SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, - r.name, r.id, r.guid - FROM releases r - INNER JOIN categories c ON r.categories_id = c.id - INNER JOIN categories cp ON cp.id = c.parentid - WHERE %s r.nzbstatus = 0 AND r.fromname IN ('%s')", - (!empty($groupID) ? ' r.groups_id = ' . $groupID . ' AND ' : ' '), - $this->mgrFromNames - ) - ); - - $nzbCount = 0; - - if ($releases && $releases->rowCount()) { - $total = $releases->rowCount(); - // Init vars for writing the NZB's. - $this->mgrnzb->initiateForMgrWrite(); - foreach ($releases as $release) { - - if ($this->mgrnzb->writeNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { - $nzbCount++; - if ($this->echoCLI) { - echo $this->pdo->log->primaryOver("Creating NZBs and deleting MGR Collections:\t" . $nzbCount . '/' . $total . "\r"); - } - } - } - } - - return $nzbCount; } } +