From 951d85f3befa71b35d994a5251be2764701509cb Mon Sep 17 00:00:00 2001 From: Darko Date: Mon, 13 Oct 2014 12:55:25 +0200 Subject: [PATCH] Many updates, including table per group and working threaded release update. --- bin/monitor.php | 7 +- bin/update_releases.php | 95 +- lib/DB/patches/0096~site.sql | 9 + lib/DB/patches/0097~releases.sql | 3 + lib/Enzebe.php | 292 +++--- lib/ReleaseRemover.php | 2 +- .../multiprocessing/.do_not_run/switch.php | 10 +- .../nix_scripts/multiprocessing/Forking.php | 24 +- lib/copy_this/www/lib/SphinxSearch.php | 4 +- lib/copy_this/www/lib/backfill.php | 4 +- lib/copy_this/www/lib/binaries.php | 56 +- lib/copy_this/www/lib/groups.php | 127 ++- lib/copy_this/www/lib/nzb.php | 48 +- lib/copy_this/www/lib/releases.php | 916 +++++++++--------- .../nntmux/views/admin/site-edit.tpl | 58 +- lib/testing/DB/convert_to_tpg.php | 169 ++++ lib/testing/DB/convert_to_tpg_alt.php | 80 ++ lib/testing/DB/reset_truncate.php | 56 ++ lib/testing/_run_once/tpg_fixes.php | 84 ++ python/postprocess_threaded.py | 109 +-- python/releases_threaded.py | 21 +- python/update_threaded.py | 104 ++ start.php | 2 +- 23 files changed, 1485 insertions(+), 795 deletions(-) create mode 100644 lib/DB/patches/0096~site.sql create mode 100644 lib/DB/patches/0097~releases.sql create mode 100644 lib/testing/DB/convert_to_tpg.php create mode 100644 lib/testing/DB/convert_to_tpg_alt.php create mode 100644 lib/testing/DB/reset_truncate.php create mode 100644 lib/testing/_run_once/tpg_fixes.php create mode 100644 python/update_threaded.py diff --git a/bin/monitor.php b/bin/monitor.php index 651fe7529..58145ee7e 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -8,7 +8,7 @@ require_once(WWW_DIR . "/lib/showsleep.php"); require_once(dirname(__FILE__) . "/../lib/functions.php"); -$version = "0.4r2305"; +$version = "0.5r0000"; $pdo = new DB(); $s = new Sites(); @@ -1281,7 +1281,10 @@ while ($i > 0) { } if ($releases_run != 0) { - $run_releases = "$_python ${DIR}/../python/releases_threaded.py"; + $run_releases = ($site->tablepergroup == 0 + ? "cd $_bin && $_php update_releases.php 1 false 2>&1 $log" + : "$_php ${DIR}/../../multiprocessing/releases.php 2>&1 $log" + ); } diff --git a/bin/update_releases.php b/bin/update_releases.php index 680c78563..559a36e29 100644 --- a/bin/update_releases.php +++ b/bin/update_releases.php @@ -1,40 +1,75 @@ processReleases(); -//$sphinx->update(); -function relativeTime($_time) -{ - $d[0] = array(1, "\033[1;33msec"); - $d[1] = array(60, "\033[1;33mmin"); - $d[2] = array(3600, "\033[1;33mhr"); - $d[3] = array(86400, "\033[1;33mday"); - $d[4] = array(31104000, "\033[1;33myr"); - $w = array(); +$s = new \Sites(); +$site = $s->get(); +$pdo = new DB(); - $return = ""; - $now = TIME(); - $diff = ($now - $_time); - $secondsLeft = $diff; - - for ($i = 4; $i > -1; $i--) { - $w[$i] = intval($secondsLeft / $d[$i][0]); - $secondsLeft -= ($w[$i] * $d[$i][0]); - if ($w[$i] != 0) { - //$return.= abs($w[$i]). " " . $d[$i][1] . (($w[$i]>1)?'s':'') ." "; - $return .= $w[$i] . " " . $d[$i][1] . (($w[$i] > 1) ? 's' : '') . " "; - } +if (isset($argv[2]) && $argv[2] === 'true') { + // Create the connection here and pass + $nntp = new \NNTP(['Settings' => $pdo]); + if ($nntp->doConnect() !== true) { + exit($pdo->log->error("Unable to connect to usenet.")); } - - //$return .= ($diff>0)?"ago":"left"; - return $return; +} +if ($site->tablepergroup === 1) { + exit($pdo->log->error("You are using 'tablepergroup', you must use releases_threaded.py")); } -echo "\033[1;33mThis loop completed in: " . relativeTime($time) . "\n"; +$groupName = isset($argv[3]) ? $argv[3] : ''; +if (isset($argv[1]) && isset($argv[2])) { + $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); + $releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]); + if ($argv[1] == 1 && $argv[2] == 'true') { + $releases->processReleases(1, 1, $groupName, $nntp, true); + } else if ($argv[1] == 1 && $argv[2] == 'false') { + $releases->processReleases(1, 2, $groupName, $nntp, true); + } else if ($argv[1] == 2 && $argv[2] == 'true') { + $releases->processReleases(2, 1, $groupName, $nntp, true); + } else if ($argv[1] == 2 && $argv[2] == 'false') { + $releases->processReleases(2, 2, $groupName, $nntp, true); + } else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) { + echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient."); + $releases->resetCategorize(); + } else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) { + echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryID = 8010'); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject."); + } else if ($argv[1] == 6 && $argv[2] == 'true') { + echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('searchname', ''); + $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); + } else if ($argv[1] == 6 && $argv[2] == 'false') { + echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('searchname', 'WHERE categoryID IN (1090, 2020, 3050, 5050, 6050, 8010)'); + $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); + } else { + exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments.")); + } +} else { + exit($pdo->log->error("\nWrong set of arguments.\n" + . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n" + . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n" + . "\nYou must pass a second argument whether to post process or not, true or false\n" + . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n" + . "\nExtra commands::\n" + . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n" + . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n" + . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n" + . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n")); +} \ No newline at end of file diff --git a/lib/DB/patches/0096~site.sql b/lib/DB/patches/0096~site.sql new file mode 100644 index 000000000..f68adb36e --- /dev/null +++ b/lib/DB/patches/0096~site.sql @@ -0,0 +1,9 @@ +INSERT IGNORE INTO `site` (`setting`, `value`) VALUES + ('grabstatus', 1), + ('crossposttime', 2), + ('deletepossiblerelease', 0), + ('grabstatus', 0), + ('nzbsplitlevel', 1), + ('maxsizetoformrelease', 0); + +UPDATE `tmux` set `value` = '96' where `setting` = 'sqlpatch'; diff --git a/lib/DB/patches/0097~releases.sql b/lib/DB/patches/0097~releases.sql new file mode 100644 index 000000000..ffebbecac --- /dev/null +++ b/lib/DB/patches/0097~releases.sql @@ -0,0 +1,3 @@ +ALTER TABLE releases ADD COLUMN nzb_guid VARCHAR(50) NULL; +ALTER TABLE releases ADD INDEX ix_releases_nzb_guid (nzb_guid); +UPDATE `tmux` set `value` = '97' where `setting` = 'sqlpatch'; \ No newline at end of file diff --git a/lib/Enzebe.php b/lib/Enzebe.php index f2a32066d..8ba418cb9 100644 --- a/lib/Enzebe.php +++ b/lib/Enzebe.php @@ -10,19 +10,23 @@ require_once(WWW_DIR."/lib/category.php"); class Enzebe { /** - * Instance of class site. - * @var object + * Determines if the site setting table per group is enabled. + * @var bool * @access private */ - private $s; + private $tablePerGroup; /** - * Site settings. - * @var object - * @access private + * Levels deep to store NZB files. + * @var int */ - public $site; + private $nzbSplitLevel; + /** + * Path to store NZB files. + * @var string + */ + private $siteNzbPath; /** * Group ID when writing NZBs. @@ -33,24 +37,38 @@ class Enzebe /** * Instance of class db. - * - * @var $db - * @access protected + * @var \DB + * @access public */ - protected $db; + public $pdo; /** - * Date when writing NZBs. + * Current newznab version. * @var string * @access protected */ - protected $writeDate; + protected $_newznabVersion; - /* - * Table names for TPG. + /** + * Base query for selecting binary data for writing NZB files. + * @var string + * @access protected */ - protected $bname; - protected $pname; + protected $_binariesQuery; + + /** + * Base query for selecting parts data for writing NZB files. + * @var string + * @access protected + */ + protected $_partsQuery; + + /** + * String used for head in NZB XML file. + * @var string + * @access protected + */ + protected $_nzbHeadString; const NZB_NONE = 0; // Release has no NZB file yet. const NZB_ADDED = 1; // Release had an NZB file created. @@ -58,41 +76,68 @@ class Enzebe /** * Default constructor. * + * @param \DB $pdo + * * @access public */ - public function __construct() + public function __construct(&$pdo = null) { + $this->pdo = ($pdo instanceof \DB ? $pdo : new \DB()); $s = new Sites(); $this->site = $s->get(); + + $this->tablePerGroup = ($this->site->tablepergroup == 0 ? false : true); + $nzbSplitLevel = $this->site->nzbsplitlevel; + $this->nzbSplitLevel = (empty($nzbSplitLevel) ? 1 : $nzbSplitLevel); + $this->siteNzbPath = (string)$this->site->nzbpath; + if (substr($this->siteNzbPath, -1) !== DS) { + $this->siteNzbPath .= DS; + } } /** * Initiate class vars when writing NZB's. * - * @param $db - * @param string $date - * @param int $groupID + * @param int $groupID * * @access public */ - public function initiateForWrite($db, $date, $groupID) + public function initiateForWrite($groupID) { - $this->db = $db; - $this->writeDate = $date; $this->groupID = $groupID; - $this->bname = 'binaries'; - $this->pname = 'parts'; - } + $site = new Sites(); + // Set table names + if ($this->tablePerGroup === true) { + if ($this->groupID == '') { + exit("$this->groupID is missing\n"); + } + $bName = 'binaries_' . $this->groupID; + $pName = 'parts_' . $this->groupID; + } else { + $bName = 'binaries'; + $pName = 'parts'; + } - /** - * Clean up class vars when done writing NZB's. - * - * @access public - */ - public function cleanForWrite() - { - $this->writeDate = null; - $this->groupID = null; + $this->_binariesQuery = sprintf( + 'SELECT %s.*, UNIX_TIMESTAMP(%s.date) AS udate, groups.name AS groupname + FROM %s + INNER JOIN groups ON %s.groupID = groups.ID + WHERE %s.releaseID = ', + $bName, + $bName, + $bName, + $bName, + $bName + ); + $this->_partsQuery = ( + 'SELECT DISTINCT(messageID), size, partnumber FROM ' . $pName . ' WHERE binaryID = %d ORDER BY partnumber' + ); + + $this->_nzbHeadString = ( + "\n\n\n\n\n %s\n %s\n\n\n" + ); } /** @@ -109,97 +154,72 @@ class Enzebe */ public function writeNZBforReleaseId($relID, $relGuid, $name, $cTitle) { - $path = $this->getNZBPath($relGuid, true); - $this->site = new Sites(); - $this->bname = 'binaries'; - $this->pname = 'parts'; - $this->db = new DB(); - $fp = gzopen($path, 'w5'); + $path = ($this->buildNZBPath($relGuid, $this->nzbSplitLevel, true) . $relGuid . '.nzb.gz'); + $fp = gzopen($path, 'wb7'); if ($fp) { $nzb_guid = ''; - gzwrite($fp, - '' . - "\n" . - '' . - "\n" . - '' . - "\n" . - "\n\n " . - htmlspecialchars($cTitle, ENT_QUOTES, 'utf-8') . - "\n " . - htmlspecialchars($name, ENT_QUOTES, 'utf-8') . - "\n\n\n" + gzwrite( + $fp, + sprintf( + $this->_nzbHeadString, + htmlspecialchars($cTitle, ENT_QUOTES, 'utf-8'), + htmlspecialchars($name, ENT_QUOTES, 'utf-8') + ) ); - $result = - $this->db->query( - sprintf(' - SELECT %s.*, %s.date AS udate, groups.name AS groupname - FROM %s - INNER JOIN groups ON %s.groupID = groups.ID - WHERE %s.releaseID = %d', - $this->bname, - $this->bname, - $this->bname, - $this->bname, - $this->bname, - $relID - ) - ); + $binaries = $this->pdo->queryDirect($this->_binariesQuery. $relID . ' ORDER BY name'); - foreach ($result as $binRow) { - $unixDate = strtotime($binRow['udate']); - $poster = htmlspecialchars($binRow['fromname'], ENT_QUOTES, 'utf-8'); - $result2 = $this->db->query( - sprintf( - 'SELECT ID, name, totalparts FROM %s WHERE releaseID = %d ORDER BY name', - $this->bname, - $binRow['ID'] - ) - ); + if ($binaries instanceof \Traversable) { - foreach ($result2 as $binRow2) { - gzwrite($fp, - '\n \n " . $binRow['groupname'] . - "\n \n \n" - ); + foreach ($binaries as $binary) { + $poster = htmlspecialchars($binary['fromname'], ENT_QUOTES, 'utf-8'); - $resParts = $this->db->query( - sprintf(' - SELECT DISTINCT(messageID), size, partnumber FROM %s WHERE binaryID = %d ORDER BY partnumber', - $this->pname, - $binRow2['ID'] - ) - ); + if ($binary instanceof \Traversable) { - foreach ($resParts as $partsRow) { - if ($nzb_guid === '') { - $nzb_guid = $partsRow['messageID']; + foreach ($binary as $bin) { + + // Buffer segment writes, increases performance. + $string = ''; + + $parts = $this->pdo->queryDirect(sprintf($this->_partsQuery, $bin['ID'])); + + if ($parts instanceof \Traversable) { + + foreach ($parts as $part) { + if ($nzb_guid === '') { + $nzb_guid = $part['messageID']; + } + + $string .= ( + ' ' + . htmlspecialchars($part['messageID'], ENT_QUOTES, 'utf-8') + . "\n" + ); + } + } + + gzwrite($fp, + '\n \n " . $bin['groupname'] . + "\n \n \n" . $string . " \n\n" + ); } - - gzwrite($fp, ' ' - . htmlspecialchars($partsRow['messageID'], ENT_QUOTES, 'utf-8') - . "\n" - ); } - gzwrite($fp, " \n\n"); } } + gzwrite($fp, ''); gzclose($fp); if (is_file($path)) { - $this->db->queryExec( + $this->pdo->queryExec( sprintf(' UPDATE releases SET nzbstatus = %d %s WHERE ID = %d', - Enzebe::NZB_ADDED, - ($nzb_guid === '' ? '' : ', nzb_guid = ' . $this->db->escapestring(md5($nzb_guid))), + \Enzebe::NZB_ADDED, + ($nzb_guid === '' ? '' : ', nzb_guid = ' . $this->pdo->escapestring(md5($nzb_guid))), $relID ) ); @@ -218,24 +238,24 @@ class Enzebe * Build a folder path on the hard drive where the NZB file will be stored. * * @param string $releaseGuid The guid of the release. + * @param int $levelsToSplit How many sub-paths the folder will be in. * @param bool $createIfNotExist Create the folder if it doesn't exist. * * @return string $nzbpath The path to store the NZB file. * * @access public */ - private function buildNZBPath($releaseGuid, $createIfNotExist) + private function buildNZBPath($releaseGuid, $levelsToSplit, $createIfNotExist) { - $siteNzbPath = $this->site->nzbpath; - if (substr($siteNzbPath, -1) !== '/') { - $siteNzbPath .= '/'; - } - $nzbPath = ''; - $nzbPath = $siteNzbPath . $nzbPath; + for ($i = 0; $i < $levelsToSplit && $i < 32; $i++) { + $nzbPath .= substr($releaseGuid, $i, 1) . DS; + } - if ($createIfNotExist && !is_dir($nzbPath)) { + $nzbPath = $this->siteNzbPath . $nzbPath; + + if ($createIfNotExist === true && !is_dir($nzbPath)) { mkdir($nzbPath, 0777, true); } @@ -246,15 +266,20 @@ class Enzebe * Retrieve path + filename of the NZB to be stored. * * @param string $releaseGuid The guid of the release. + * @param int $levelsToSplit How many sub-paths the folder will be in. (optional) * @param bool $createIfNotExist Create the folder if it doesn't exist. (optional) * * @return string Path+filename. * * @access public */ - public function getNZBPath($releaseGuid, $createIfNotExist = false) + public function getNZBPath($releaseGuid, $levelsToSplit = 0, $createIfNotExist = false) { - return ($this->buildNZBPath($releaseGuid, $createIfNotExist) . $releaseGuid . '.nzb.gz'); + if ($levelsToSplit === 0) { + $levelsToSplit = $this->nzbSplitLevel; + } + + return ($this->buildNZBPath($releaseGuid, $levelsToSplit, $createIfNotExist) . $releaseGuid . '.nzb.gz'); } /** @@ -270,7 +295,7 @@ class Enzebe public function NZBPath($releaseGuid) { $nzbFile = $this->getNZBPath($releaseGuid); - return !is_file($nzbFile) ? false : $nzbFile; + return (is_file($nzbFile) ? $nzbFile : false); } /** @@ -288,6 +313,10 @@ class Enzebe $num_pars = $i = 0; $result = array(); + if (!$nzb) { + return $result; + } + $nzb = str_replace("\x0F", '', $nzb); $xml = @simplexml_load_string($nzb); if (!$xml || strtolower($xml->getName()) !== 'nzb') { @@ -296,7 +325,7 @@ class Enzebe foreach ($xml->file as $file) { // Subject. - $title = $file->attributes()->subject; + $title = (string)$file->attributes()->subject; // Amount of pars. if (stripos($title, '.par2')) { @@ -307,14 +336,17 @@ class Enzebe // Extensions. if (preg_match( - '/\.(\d{2,3}|7z|ace|ai7|srr|srt|sub|aiff|asc|avi|audio|bin|bz2|' - . 'c|cfc|cfm|chm|class|conf|cpp|cs|css|csv|cue|deb|divx|doc|dot|' - . 'eml|enc|exe|file|gif|gz|hlp|htm|html|image|iso|jar|java|jpeg|' - . 'jpg|js|lua|m|m3u|mkv|mm|mov|mp3|mp4|mpg|nfo|nzb|odc|odf|odg|odi|odp|' - . 'ods|odt|ogg|par2|parity|pdf|pgp|php|pl|png|ppt|ps|py|r\d{2,3}|' - . 'ram|rar|rb|rm|rpm|rtf|sfv|sig|sql|srs|swf|sxc|sxd|sxi|sxw|tar|' - . 'tex|tgz|txt|vcf|video|vsd|wav|wma|wmv|xls|xml|xpi|xvid|zip7|zip)' - . '[" ](?!(\)|\-))/i', $file->attributes()->subject, $ext)) { + '/\.(\d{2,3}|7z|ace|ai7|srr|srt|sub|aiff|asc|avi|audio|bin|bz2|' + . 'c|cfc|cfm|chm|class|conf|cpp|cs|css|csv|cue|deb|divx|doc|dot|' + . 'eml|enc|exe|file|gif|gz|hlp|htm|html|image|iso|jar|java|jpeg|' + . 'jpg|js|lua|m|m3u|mkv|mm|mov|mp3|mp4|mpg|nfo|nzb|odc|odf|odg|odi|odp|' + . 'ods|odt|ogg|par2|parity|pdf|pgp|php|pl|png|ppt|ps|py|r\d{2,3}|' + . 'ram|rar|rb|rm|rpm|rtf|sfv|sig|sql|srs|swf|sxc|sxd|sxi|sxw|tar|' + . 'tex|tgz|txt|vcf|video|vsd|wav|wma|wmv|xls|xml|xpi|xvid|zip7|zip)' + . '[" ](?!(\)|\-))/i', + $title, $ext + ) + ) { if (preg_match('/\.r\d{2,3}/i', $ext[0])) { $ext[1] = 'rar'; @@ -359,4 +391,4 @@ class Enzebe return $result; } -} +} \ No newline at end of file diff --git a/lib/ReleaseRemover.php b/lib/ReleaseRemover.php index 9b8d6584f..d1671f16c 100644 --- a/lib/ReleaseRemover.php +++ b/lib/ReleaseRemover.php @@ -912,7 +912,7 @@ class ReleaseRemover $groupID = ' AND r.groupID in (' . $groupIDs . ') '; } - $this->method = 'Blacklist ' . $regex['id']; + $this->method = 'Blacklist ' . $regex['ID']; $this->query = sprintf( "SELECT r.guid, r.searchname, r.ID FROM releases r %s %s %s", $regexSQL, $groupID, $this->crapTime diff --git a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php index 2dd9fc794..ce498c92e 100644 --- a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php +++ b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php @@ -76,7 +76,6 @@ switch ($options[1]) { } $binaries = new \Binaries(); $return = $binaries->scan($nntp, $groupMySQL, $options[4], $options[5], ($site->safepartrepair == 1 ? 'update' : 'backfill')); - var_dump($return); if (empty($return)) { exit(); } @@ -210,7 +209,7 @@ switch ($options[1]) { $backFill = new \Backfill(); // Update the group for new binaries. - (new \Binaries())->updateGroup($groupMySQL); + (new \Binaries())->updateGroup($nntp, $groupMySQL); // BackFill the group with 20k articles. $backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal'); @@ -276,8 +275,11 @@ switch ($options[1]) { */ function processReleases($releases, $groupID) { + $releases->applyRegex($groupID); + $releases->processIncompleteBinaries($groupID); $releases->createReleases($groupID); - $releases->createNZBs($groupID); + $releases->deleteBinaries($groupID); + } /** @@ -326,4 +328,4 @@ function &nntp(&$pdo, $alternate = false) } return $nntp; -} \ No newline at end of file +} diff --git a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/Forking.php b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/Forking.php index 1159f0612..0e8d9a3ba 100644 --- a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/Forking.php +++ b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/Forking.php @@ -582,8 +582,8 @@ class Forking extends \fork_daemon if ($groups instanceof \Traversable) { foreach($groups as $group) { - if ($this->pdo->queryOneRow(sprintf('SELECT ID FROM collections_%d LIMIT 1',$group['id'])) !== false) { - $this->work[] = ['id' => $group['id']]; + if ($this->pdo->queryOneRow(sprintf('SELECT ID FROM binaries_%d LIMIT 1',$group['ID'])) !== false) { + $this->work[] = ['ID' => $group['ID']]; } } } @@ -591,7 +591,7 @@ class Forking extends \fork_daemon $this->work = $this->pdo->query('SELECT name FROM groups WHERE (active = 1 OR backfill = 1)'); } - return $this->site->releasesthreads; + return $this->site->releasethreads; } public function releasesChildWorker($groups, $identifier = '') @@ -599,11 +599,11 @@ class Forking extends \fork_daemon foreach ($groups as $group) { if ($this->tablePerGroup === true) { $this->_executeCommand( - $this->dnr_path . 'releases ' . $group['id'] . '"' + $this->dnr_path . 'releases ' . $group['ID'] . '"' ); } else { $this->_executeCommand( - PHP_BINARY . ' ' . NN_UPDATE . 'update_releases.php 1 false ' . $group['name'] + PHP_BINARY . ' ' . NN_TMUX . 'bin' . DS . 'update_releases.php 1 false ' . $group['name'] ); } } @@ -635,7 +635,7 @@ class Forking extends \fork_daemon if ($type !== '') { $this->_executeCommand( - $this->dnr_path . $type . $group['id'] . (isset($group['renamed']) ? (' ' . $group['renamed']) : '') . '"' + $this->dnr_path . $type . $group['ID'] . (isset($group['renamed']) ? (' ' . $group['renamed']) : '') . '"' ); } } @@ -684,7 +684,7 @@ class Forking extends \fork_daemon $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); $this->work = $this->pdo->query( sprintf(' - SELECT LEFT(r.guid, 1) AS id + SELECT LEFT(r.guid, 1) AS ID FROM releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE r.nzbstatus = %d @@ -734,7 +734,7 @@ class Forking extends \fork_daemon $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); $this->work = $this->pdo->query( sprintf(' - SELECT LEFT(r.guid, 1) AS id + SELECT LEFT(r.guid, 1) AS ID FROM releases r WHERE 1=1 %s GROUP BY LEFT(r.guid, 1) @@ -782,7 +782,7 @@ class Forking extends \fork_daemon $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); $this->work = $this->pdo->query( sprintf(' - SELECT LEFT(guid, 1) AS id, %d AS renamed + SELECT LEFT(guid, 1) AS ID, %d AS renamed FROM releases WHERE nzbstatus = %d AND imdbID IS NULL @@ -837,7 +837,7 @@ class Forking extends \fork_daemon $this->register_child_run([0 => $this, 1 => 'postProcessChildWorker']); $this->work = $this->pdo->query( sprintf(' - SELECT LEFT(guid, 1) AS id, %d AS renamed + SELECT LEFT(guid, 1) AS ID, %d AS renamed FROM releases WHERE nzbstatus = %d AND rageID = -1 @@ -933,14 +933,14 @@ class Forking extends \fork_daemon { $this->register_child_run([0 => $this, 1 => 'updatePerGroupChildWorker']); $this->work = $this->pdo->query('SELECT ID FROM groups WHERE (active = 1 OR backfill = 1)'); - return $this->site->releasesthreads; + return $this->site->releasethreads; } public function updatePerGroupChildWorker($groups, $identifier = '') { foreach ($groups as $group) { $this->_executeCommand( - $this->dnr_path . 'update_per_group ' . $group['id'] . '"' + $this->dnr_path . 'update_per_group ' . $group['ID'] . '"' ); } } diff --git a/lib/copy_this/www/lib/SphinxSearch.php b/lib/copy_this/www/lib/SphinxSearch.php index c670679cd..d4b92a0df 100644 --- a/lib/copy_this/www/lib/SphinxSearch.php +++ b/lib/copy_this/www/lib/SphinxSearch.php @@ -42,11 +42,11 @@ class SphinxSearch */ public function insertRelease($parameters) { - if (!is_null($this->sphinxQL) && $parameters['id']) { + if (!is_null($this->sphinxQL) && $parameters['ID']) { $this->sphinxQL->queryExec( sprintf( 'REPLACE INTO releases_rt (id, guid, name, searchname, fromname) VALUES (%s, %s, %s, %s, %s)', - $parameters['id'], + $parameters['ID'], $parameters['guid'], $parameters['name'], $parameters['searchname'], diff --git a/lib/copy_this/www/lib/backfill.php b/lib/copy_this/www/lib/backfill.php index 91d797272..88f3336da 100644 --- a/lib/copy_this/www/lib/backfill.php +++ b/lib/copy_this/www/lib/backfill.php @@ -129,7 +129,7 @@ class Backfill $done = false; //set first and last, moving the window by maxxMssgs $last = $groupArr['first_record'] - 1; - $first = $last - $binaries->messagebuffer + 1; //set initial "chunk" + $first = $last - $binaries->messageBuffer + 1; //set initial "chunk" if($targetpost > $first) //just in case this is the last chunk we needed $first = $targetpost; while($done === false) @@ -154,7 +154,7 @@ class Backfill else { //Keep going: set new last, new first, check for last chunk. $last = $first - 1; - $first = $last - $binaries->messagebuffer + 1; + $first = $last - $binaries->messageBuffer + 1; if($targetpost > $first) $first = $targetpost; } diff --git a/lib/copy_this/www/lib/binaries.php b/lib/copy_this/www/lib/binaries.php index 32959b265..f868f7d12 100644 --- a/lib/copy_this/www/lib/binaries.php +++ b/lib/copy_this/www/lib/binaries.php @@ -167,8 +167,9 @@ class Binaries $first = $data['last'] - $this->NewGroupMsgsToScan; } $first_record_postdate = $this->postdate($first, $data); - if ($first_record_postdate != "") - $db->queryExec(sprintf("update groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . ") WHERE ID = %d", $db->escapeString($first), $groupArr['ID'])); + $last_record_postdate = $this->postdate($last, $data); + if ($first_record_postdate != '') + $db->queryExec(sprintf('update groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(' . $first_record_postdate . ') WHERE ID = %d', $db->escapeString($first), $groupArr['ID'])); } else { if ($data['last'] < $groupArr['last_record']) { echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}" . $n; @@ -179,12 +180,12 @@ class Binaries } // Generate postdates for first and last records, for those that upgraded - if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) - $db->queryExec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $this->postdate($groupArr['first_record'], $data) . "), last_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['last_record'], false) . ") WHERE ID = %d", $groupArr['ID'])); + if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != '0' && $groupArr['first_record'] != '0')) + $db->queryExec(sprintf('update groups SET first_record_postdate = FROM_UNIXTIME('.$backfill->postdate($nntp,$groupArr['first_record'],false).'), last_record_postdate = FROM_UNIXTIME('.$backfill->postdate($nntp,$groupArr['last_record'],false).') WHERE ID = %d', $groupArr['ID'])); // Deactivate empty groups if (($data['last'] - $data['first']) <= 5) - $db->queryExec(sprintf("update groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID'])); + $db->queryExec(sprintf('update groups SET active = %s, last_updated = now() WHERE ID = %d', $db->escapeString('0'), $groupArr['ID'])); // Calculate total number of parts $total = $grouplast - $first + 1; @@ -259,6 +260,8 @@ class Binaries $releaseRegex = new ReleaseRegex; $n = $this->n; $this->startHeaders = microtime(true); + // Check if MySQL tables exist, create if they do not, get their names at the same time. + $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupArr['ID']); if ($this->compressedHeaders) { $nntpn = new Nntp(); @@ -366,7 +369,7 @@ class Binaries case 'partrepair': case 'update': default: - $this->addMissingParts($rangenotreceived, $groupArr['ID']); + $this->addMissingParts($rangenotreceived, $tableNames['prname'], $groupArr['ID']); break; } echo "Server did not return " . count($rangenotreceived) . " article(s).$n"; @@ -384,7 +387,7 @@ class Binaries $partIds = array(); foreach ($data['Parts'] as $partdata) $partIds[] = $partdata['number']; - $db->queryExec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID'])); + $db->queryExec(sprintf("DELETE FROM %s WHERE numberID IN (%s) AND groupID=%d", $tableNames['prname'], implode(',', $partIds), $groupArr['ID'])); } continue; } @@ -393,7 +396,7 @@ class Binaries //Check for existing binary $binaryID = 0; $binaryHash = md5($subject . $data['From'] . $groupArr['ID']); - $res = $db->queryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $db->escapeString($binaryHash))); + $res = $db->queryOneRow(sprintf("SELECT ID FROM %s WHERE binaryhash = %s", $tableNames['bname'], $db->escapeString($binaryHash))); if (!$res) { //Apply Regexes @@ -409,15 +412,15 @@ class Binaries $sql = ''; if (!empty($regexMatches)) { $relparts = explode("/", $regexMatches['parts']); - $sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, procstat, categoryID, regexID, reqID, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name']))); + $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupID, procstat, categoryID, regexID, reqID, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name']))); } elseif ($this->onlyProcessRegexBinaries === false) { - $sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash)); + $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash)); } //onlyProcessRegexBinaries is true, there was no regex match and we are doing part repair so delete them elseif ($type == 'partrepair') { $partIds = array(); foreach ($data['Parts'] as $partdata) $partIds[] = $partdata['number']; - $db->queryExec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID'])); + $db->queryExec(sprintf('DELETE FROM %s WHERE numberID IN (%s) AND groupID=%d', $tableNames['prname'], implode(',', $partIds), $groupArr['ID'])); continue; } if ($sql != '') { @@ -437,12 +440,12 @@ class Binaries foreach ($data['Parts'] AS $partdata) { $partcount++; - $partParams[] = sprintf("(%d, %s, %s, %s, %s)", $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size'])); + $partParams[] = sprintf('(%d, %s, %s, %s, %s)', $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size'])); $partNumbers[] = $partdata['number']; } - $partSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size) VALUES " . implode(', ', $partParams); - $pidata = $db->queryInsert($partSql, false); + $partSql = sprintf('INSERT INTO ' . $tableNames['pname'] . ' (binaryID, messageID, number, partnumber, size) VALUES '.implode(', ', $partParams)); + $pidata = $db->queryInsert($partSql); if (!$pidata) { $msgsnotinserted = array_merge($msgsnotinserted, $partNumbers); } else { @@ -454,7 +457,7 @@ class Binaries //TODO: determine whether to add to missing articles if insert failed if (sizeof($msgsnotinserted) > 0) { echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n; - $this->addMissingParts($msgsnotinserted, $groupArr['ID']); + $this->addMissingParts($msgsnotinserted, $tableNames['prname'], $groupArr['ID']); } if (($count >= 500) || ($updatecount >= 500)) { echo $n; @@ -494,6 +497,7 @@ class Binaries public function partRepair($nntp, $group) { $db = new DB(); + $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $group['ID']); $parts = array(); $chunks = array(); @@ -501,8 +505,8 @@ class Binaries $query = sprintf ( - "SELECT numberID FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 40000", - $group['ID'] + "SELECT numberID FROM %s WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 40000", + $tableNames['prname'], $group['ID'] ); $result = $db->query($query); @@ -556,8 +560,8 @@ class Binaries $query = sprintf ( - "SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", - $group['ID'], implode(',', $chunk) + "SELECT pr.ID, pr.numberID, p.number from %s pr LEFT JOIN %s p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", + $tableNames['prname'], $tableNames['pname'], $group['ID'], implode(',', $chunk) ); $result = $db->query($query); @@ -565,17 +569,17 @@ class Binaries # TODO: rewrite.. stupid if ($item['number'] == $item['numberID']) { #printf("Repair: %s repaired.%s", $item['ID'], $this->n); - $db->queryExec(sprintf("DELETE FROM partrepair WHERE ID=%d LIMIT 1", $item['ID'])); + $db->queryExec(sprintf("DELETE FROM %s WHERE ID=%d LIMIT 1", $tableNames['prname'], $item['ID'])); $repaired++; continue; } else { #printf("Repair: %s has not arrived yet or deleted.%s", $item['numberID'], $this->n); - $db->queryExec(sprintf("update partrepair SET attempts=attempts+1 WHERE ID=%d LIMIT 1", $item['ID'])); + $db->queryExec(sprintf("update %s SET attempts=attempts+1 WHERE ID=%d LIMIT 1", $tableNames['prname'], $item['ID'])); } } } - $delret = $db->queryExec(sprintf('DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d', $group['ID'])); + $delret = $db->queryExec(sprintf('DELETE FROM %s WHERE attempts >= 5 AND groupID = %d', $tableNames['prname'], $group['ID'])); $delcnt = $delret->rowCount(); $db->log->doEcho($db->log->primary(sprintf('Repair: repaired %s', $repaired))); $db->log->doEcho($db->log->primary(sprintf('Repair: cleaned %s parts.', $delcnt))); @@ -589,17 +593,17 @@ class Binaries /** * Insert a missing part to the database. */ - private function addMissingParts($numbers, $groupID) + private function addMissingParts($numbers, $tablename, $groupID) { $db = new DB(); $added = false; - $insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES "; + $insertStr = "INSERT INTO $tablename (numberID, groupID) VALUES "; foreach ($numbers as $number) { if ($number > 0) { - $checksql = sprintf("select numberID from partrepair where numberID = %u and groupID = %d", $number, $groupID); + $checksql = sprintf("select numberID from $tablename where numberID = %u and groupID = %d", $number, $groupID); $chkrow = $db->queryOneRow($checksql); if ($chkrow) { - $updsql = sprintf("update partrepair set attempts = attempts + 1 where numberID = %u and groupID = %d", $number, $groupID); + $updsql = sprintf('update ' . $tablename . ' set attempts = attempts + 1 where numberID = %u and groupID = %d', $number, $groupID); $db->queryExec($updsql); } else { $added = true; diff --git a/lib/copy_this/www/lib/groups.php b/lib/copy_this/www/lib/groups.php index 304da7b74..1a1da3186 100644 --- a/lib/copy_this/www/lib/groups.php +++ b/lib/copy_this/www/lib/groups.php @@ -9,6 +9,26 @@ require_once(WWW_DIR . "/lib/releases.php"); */ class Groups { + + /** + * @var DB + */ + public $pdo; + + /** + * Construct. + * + * @param array $options Class instances. + */ + public function __construct(array $options = []) + { + $defaults = [ + 'Settings' => null + ]; + $options += $defaults; + + $this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB()); + } /** * Get all group rows. */ @@ -35,9 +55,9 @@ class Groups } $ordersort = (isset($orderArr[1]) && preg_match('/^asc|desc$/i', $orderArr[1])) ? $orderArr[1] : 'desc'; $orderby = $orderfield . " " . $ordersort; - $db = new DB(); - return $db->query(sprintf("SELECT groups.*, COALESCE(rel.num, 0) AS num_releases + + return $this->pdo->query(sprintf("SELECT groups.*, COALESCE(rel.num, 0) AS num_releases FROM groups LEFT OUTER JOIN ( SELECT groupID, COUNT(ID) AS num FROM releases group by groupID ) rel ON rel.groupID = groups.ID @@ -51,8 +71,8 @@ class Groups */ public function getGroupsForSelect() { - $db = new DB(); - $categories = $db->query("SELECT * FROM groups WHERE active = 1 ORDER BY name"); + + $categories = $this->pdo->query("SELECT * FROM groups WHERE active = 1 ORDER BY name"); $temp_array = array(); $temp_array[-1] = "--Please Select--"; @@ -68,9 +88,9 @@ class Groups */ public function getByID($id) { - $db = new DB(); - return $db->queryOneRow(sprintf("select * from groups where ID = %d ", $id)); + + return $this->pdo->queryOneRow(sprintf("select * from groups where ID = %d ", $id)); } /** @@ -78,9 +98,8 @@ class Groups */ public function getActive() { - $db = new DB(); - return $db->query("SELECT * FROM groups WHERE active = 1 ORDER BY name"); + return $this->pdo->query("SELECT * FROM groups WHERE active = 1 ORDER BY name"); } /** @@ -88,9 +107,9 @@ class Groups */ public function getByName($grp) { - $db = new DB(); - return $db->queryOneRow(sprintf("SELECT * FROM groups WHERE name = %s", $db->escapeString($grp))); + + return $this->pdo->queryOneRow(sprintf("SELECT * FROM groups WHERE name = %s", $this->pdo->escapeString($grp))); } /** @@ -102,8 +121,8 @@ class Groups */ public function getByNameByID($id) { - $db = new DB(); - $res = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d ", $id)); + + $res = $this->pdo->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d ", $id)); return ($res === false ? '' : $res["name"]); } @@ -116,8 +135,8 @@ class Groups */ public function getIDByName($name) { - $db = new DB(); - $res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s", $db->escapeString($name))); + + $res = $this->pdo->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s", $this->pdo->escapeString($name))); return ($res === false ? '' : $res["ID"]); } @@ -126,16 +145,16 @@ class Groups */ public function getCount($groupname = "", $activeonly = false) { - $db = new DB(); + $grpsql = ''; if ($groupname != "") - $grpsql .= sprintf("and groups.name like %s ", $db->escapeString("%" . $groupname . "%")); + $grpsql .= sprintf("and groups.name like %s ", $this->pdo->escapeString("%" . $groupname . "%")); if ($activeonly == true) $grpsql .= "and active=1 "; - $res = $db->queryOneRow(sprintf("select count(ID) as num from groups where 1=1 %s", $grpsql)); + $res = $this->pdo->queryOneRow(sprintf("select count(ID) as num from groups where 1=1 %s", $grpsql)); return $res["num"]; } @@ -145,7 +164,7 @@ class Groups */ public function getRange($start, $num, $groupname = "", $activeonly = false) { - $db = new DB(); + if ($start === false) $limit = ""; else @@ -153,7 +172,7 @@ class Groups $grpsql = ''; if ($groupname != "") - $grpsql .= sprintf("and groups.name like %s ", $db->escapeString("%" . $groupname . "%")); + $grpsql .= sprintf("and groups.name like %s ", $this->pdo->escapeString("%" . $groupname . "%")); if ($activeonly == true) $grpsql .= "and active=1 "; @@ -165,7 +184,7 @@ class Groups ) rel ON rel.groupID = groups.ID WHERE 1=1 %s ORDER BY groups.name " . $limit, $grpsql ); - return $db->query($sql); + return $this->pdo->query($sql); } /** @@ -173,7 +192,7 @@ class Groups */ public function add($group) { - $db = new DB(); + if ($group["minfilestoformrelease"] == "" || $group["minfilestoformrelease"] == "0") $minfiles = 'null'; @@ -183,7 +202,7 @@ class Groups if ($group["minsizetoformrelease"] == "" || $group["minsizetoformrelease"] == "0") $minsizetoformrelease = 'null'; else - $minsizetoformrelease = $db->escapeString($group["minsizetoformrelease"]); + $minsizetoformrelease = $this->pdo->escapeString($group["minsizetoformrelease"]); if ($group["backfill_target"] == "" || $group["backfill_target"] == "0") $backfill_target = '0'; @@ -198,9 +217,9 @@ class Groups $first = (isset($group["first_record"]) ? $group["first_record"] : "0"); $last = (isset($group["last_record"]) ? $group["last_record"] : "0"); - $sql = sprintf("insert into groups (name, description, first_record, last_record, last_updated, active, backfill, minfilestoformrelease, minsizetoformrelease, backfill_target, regexmatchonly) values (%s, %s, %s, %s, null, %d, %s, %s, %d, %d) ", $db->escapeString($group["name"]), $db->escapeString($group["description"]), $db->escapeString($first), $db->escapeString($last), $group["active"], $group["backfill"], $minfiles, $minsizetoformrelease, $backfill_target, $regexmatchonly); + $sql = sprintf("insert into groups (name, description, first_record, last_record, last_updated, active, backfill, minfilestoformrelease, minsizetoformrelease, backfill_target, regexmatchonly) values (%s, %s, %s, %s, null, %d, %s, %s, %d, %d) ", $this->pdo->escapeString($group["name"]), $this->pdo->escapeString($group["description"]), $this->pdo->escapeString($first), $this->pdo->escapeString($last), $group["active"], $group["backfill"], $minfiles, $minsizetoformrelease, $backfill_target, $regexmatchonly); - return $db->queryInsert($sql); + return $this->pdo->queryInsert($sql); } /** @@ -208,9 +227,8 @@ class Groups */ public function delete($id) { - $db = new DB(); - return $db->queryExec(sprintf("DELETE from groups where ID = %d", $id)); + return $this->pdo->queryExec(sprintf("DELETE from groups where ID = %d", $id)); } /** @@ -218,9 +236,8 @@ class Groups */ public function reset($id) { - $db = new DB(); - return $db->queryExec(sprintf("update groups set backfill_target=0, first_record=0, first_record_postdate=null, last_record=0, last_record_postdate=null, last_updated=null where ID = %d", $id)); + return $this->pdo->queryExec(sprintf("update groups set backfill_target=0, first_record=0, first_record_postdate=null, last_record=0, last_record_postdate=null, last_updated=null where ID = %d", $id)); } /** @@ -230,17 +247,16 @@ class Groups { require_once(WWW_DIR . "/lib/binaries.php"); - $db = new DB(); $releases = new Releases(); $binaries = new Binaries(); $this->reset($id); - $rels = $db->query(sprintf("select ID from releases where groupID = %d", $id)); + $rels = $this->pdo->query(sprintf("select ID from releases where groupID = %d", $id)); foreach ($rels as $rel) $releases->delete($rel["ID"]); - $bins = $db->query(sprintf("select ID from binaries where groupID = %d", $id)); + $bins = $this->pdo->query(sprintf("select ID from binaries where groupID = %d", $id)); foreach ($bins as $bin) $binaries->delete($bin["ID"]); } @@ -250,7 +266,6 @@ class Groups */ public function update($group) { - $db = new DB(); if ($group["minfilestoformrelease"] == "" || $group["minfilestoformrelease"] == "0") $minfiles = 'null'; @@ -260,9 +275,9 @@ class Groups if ($group["minsizetoformrelease"] == "" || $group["minsizetoformrelease"] == "0") $minsizetoformrelease = 'null'; else - $minsizetoformrelease = $db->escapeString($group["minsizetoformrelease"]); + $minsizetoformrelease = $this->pdo->escapeString($group["minsizetoformrelease"]); - return $db->queryExec(sprintf("update groups set name = %s, description = %s, backfill_target = %s , active=%d, backfill = %s, minfilestoformrelease=%s, minsizetoformrelease=%s, regexmatchonly=%d where ID = %d ", $db->escapeString($group["name"]), $db->escapeString($group["description"]), $db->escapeString($group["backfill_target"]), $group["active"], $group["backfill"], $minfiles, $minsizetoformrelease, $group["regexmatchonly"], $group["id"])); + return $this->pdo->queryExec(sprintf("update groups set name = %s, description = %s, backfill_target = %s , active=%d, backfill = %s, minfilestoformrelease=%s, minsizetoformrelease=%s, regexmatchonly=%d where ID = %d ", $this->pdo->escapeString($group["name"]), $this->pdo->escapeString($group["description"]), $this->pdo->escapeString($group["backfill_target"]), $group["active"], $group["backfill"], $minfiles, $minsizetoformrelease, $group["regexmatchonly"], $group["id"])); } /** @@ -278,7 +293,6 @@ class Groups if ($groupList == "") { $ret[] = "No group list provided."; } else { - $db = new DB(); $nntp = new Nntp(['Echo' => false]); if (!$nntp->doConnect()) { $ret[] = "Failed to get NNTP connection"; @@ -292,14 +306,14 @@ class Groups foreach ($groups AS $group) { if (preg_match($regfilter, $group['group']) > 0) { - $res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $db->escapeString($group['group']))); + $res = $this->pdo->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $this->pdo->escapeString($group['group']))); if ($res) { - $db->queryExec(sprintf("update groups SET active = %d where ID = %d", $active, $res["ID"])); + $this->pdo->queryExec(sprintf("update groups SET active = %d where ID = %d", $active, $res["ID"])); $ret[] = array('group' => $group['group'], 'msg' => 'Updated'); } else { $desc = ""; - $db->queryInsert(sprintf("INSERT INTO groups (name, description, active, backfill) VALUES (%s, %s, %d, %s)", $db->escapeString($group['group']), $db->escapeString($desc), $active, $backfill)); + $this->pdo->queryInsert(sprintf("INSERT INTO groups (name, description, active, backfill) VALUES (%s, %s, %d, %s)", $this->pdo->escapeString($group['group']), $this->pdo->escapeString($desc), $active, $backfill)); $ret[] = array('group' => $group['group'], 'msg' => 'Created'); } } @@ -314,8 +328,7 @@ class Groups */ public function updateGroupStatus($id, $status = 0) { - $db = new DB(); - $db->queryExec(sprintf("update groups SET active = %d WHERE ID = %d", $status, $id)); + $this->pdo->queryExec(sprintf("update groups SET active = %d WHERE ID = %d", $status, $id)); $status = ($status == 0) ? 'deactivated' : 'activated'; return "Group $id has been $status."; @@ -329,8 +342,7 @@ class Groups */ public function updateBackfillStatus($id, $status = 0) { - $db = new DB(); - $db->queryExec(sprintf("UPDATE groups SET backfill = %d WHERE ID = %d", $status, $id)); + $this->pdo->queryExec(sprintf("UPDATE groups SET backfill = %d WHERE ID = %d", $status, $id)); return "Group $id has been " . (($status == 0) ? 'deactivated' : 'activated') . '.'; } @@ -339,8 +351,7 @@ class Groups */ public function getActiveBackfill() { - $db = new DB(); - return $db->query("SELECT * FROM groups WHERE backfill = 1 AND last_record != 0 ORDER BY name"); + return $this->pdo->query("SELECT * FROM groups WHERE backfill = 1 AND last_record != 0 ORDER BY name"); } /** @@ -348,8 +359,7 @@ class Groups */ public function getActiveByDateBackfill() { - $db = new DB(); - return $db->query("SELECT * FROM groups WHERE backfill = 1 AND last_record != 0 ORDER BY first_record_postdate DESC"); + return $this->pdo->query("SELECT * FROM groups WHERE backfill = 1 AND last_record != 0 ORDER BY first_record_postdate DESC"); } /** @@ -406,8 +416,7 @@ class Groups */ public function getActiveIDs() { - $db = new DB(); - return $db->query("SELECT ID FROM groups WHERE active = 1 ORDER BY name"); + return $this->pdo->query("SELECT ID FROM groups WHERE active = 1 ORDER BY name"); } /** @@ -417,7 +426,25 @@ class Groups */ public function disableForPost($name) { - $db = new DB(); - $db->queryExec(sprintf("UPDATE groups SET backfill = 0 WHERE name = %s", $db->escapeString($name))); + $this->pdo->queryExec(sprintf("UPDATE groups SET backfill = 0 WHERE name = %s", $this->pdo->escapeString($name))); + } + + /** + * Check if the tables exists for the group_id, make new tables for table per group. + * + * @param int $groupID + * + * @return bool + */ + public function createNewTPGTables($groupID) + { + foreach (['binaries', 'parts', 'partrepair'] as $tableName) { + if ($this->pdo->queryExec(sprintf('SELECT * FROM %s_%s LIMIT 1', $tableName, $groupID), true) === false) { + if ($this->pdo->queryExec(sprintf('CREATE TABLE %s_%s LIKE %s', $tableName, $groupID, $tableName), true) === false) { + return false; + } + } + } + return true; } } \ No newline at end of file diff --git a/lib/copy_this/www/lib/nzb.php b/lib/copy_this/www/lib/nzb.php index 97fb4aae3..9fc1da929 100644 --- a/lib/copy_this/www/lib/nzb.php +++ b/lib/copy_this/www/lib/nzb.php @@ -9,14 +9,48 @@ require_once(WWW_DIR . "../misc/update_scripts/nix_scripts/tmux/lib/Enzebe.php") */ class NZB { + /** + * Default constructor. + * + * @param \DB $pdo + * + * @access public + */ + public function __construct(&$pdo = null) + { + $this->pdo = ($pdo instanceof \DB ? $pdo : new \DB()); + $s = new Sites(); + $this->site = $s->get(); + + $this->tablePerGroup = ($this->site->tablepergroup == 0 ? false : true); + $nzbSplitLevel = $this->site->nzbsplitlevel; + $this->nzbSplitLevel = (empty($nzbSplitLevel) ? 1 : $nzbSplitLevel); + $this->siteNzbPath = (string)$this->site->nzbpath; + if (substr($this->siteNzbPath, -1) !== DS) { + $this->siteNzbPath .= DS; + } + } + /** * Writes out the nzb when processing releases. Performed outside of smarty due to memory issues * of holding all parts in an array. */ - function writeNZBforReleaseId($relid, $name, $catId, $path) + function writeNZBforReleaseId($relid, $name, $catId, $path, $groupID) { $db = new DB(); $cat = new Category(); + $this->groupID = $groupID; + // Set table names + if ($this->tablePerGroup === true) { + if ($this->groupID == '') { + exit("$this->groupID is missing\n"); + } + $bName = 'binaries_' . $this->groupID; + $pName = 'parts_' . $this->groupID; + } else { + $bName = 'binaries'; + $pName = 'parts'; + } $catrow = $cat->getById($catId); $site = new Sites(); @@ -32,7 +66,13 @@ class NZB gzwrite($fp, " " . htmlspecialchars($name, ENT_QUOTES, 'utf-8') . "\n"); gzwrite($fp, "\n\n"); - $result = $db->queryDirect(sprintf("SELECT binaries.*, UNIX_TIMESTAMP(date) AS unixdate, groups.name as groupname FROM binaries inner join groups on binaries.groupID = groups.ID WHERE binaries.releaseID = %d ORDER BY binaries.name", $relid)); + $result = $db->queryDirect(sprintf("SELECT %s.*, UNIX_TIMESTAMP(date) AS unixdate, groups.name as groupname FROM %s inner join groups on %s.groupID = groups.ID WHERE %s.releaseID = %d ORDER BY %s.name", + $bName, + $bName, + $bName, + $bName, + $relid, + $bName)); while ($binrow = $db->getAssocArray($result)) { $groups = array(); $groupsRaw = explode(' ', $binrow['xref']); @@ -50,7 +90,9 @@ class NZB gzwrite($fp, " \n"); gzwrite($fp, " \n"); - $resparts = $db->queryDirect(sprintf("SELECT DISTINCT(messageID), size, partnumber FROM parts WHERE binaryID = %d ORDER BY partnumber", $binrow["ID"])); + $resparts = $db->queryDirect(sprintf("SELECT DISTINCT(messageID), size, partnumber FROM %s WHERE binaryID = %d ORDER BY partnumber", + $pName, + $binrow["ID"])); while ($partsrow = $db->getAssocArray($resparts)) { gzwrite($fp, " " . htmlspecialchars($partsrow["messageID"], ENT_QUOTES, 'utf-8') . "\n"); } diff --git a/lib/copy_this/www/lib/releases.php b/lib/copy_this/www/lib/releases.php index d4254b0fd..65dd91716 100644 --- a/lib/copy_this/www/lib/releases.php +++ b/lib/copy_this/www/lib/releases.php @@ -19,12 +19,13 @@ require_once(WWW_DIR . "/lib/postprocess.php"); require_once(WWW_DIR . "/lib/sphinx.php"); require_once(WWW_DIR . "lib/Categorize.php"); require_once(NN_TMUX . 'lib' . DS . 'ReleaseCleaner.php'); +require_once(NN_TMUX . 'lib' . DS . 'Pprocess.php'); require_once(NN_TMUX . 'lib' . DS . 'Enzebe.php'); -require_once (NN_LIB . 'SphinxSearch.php'); -require_once (NN_LIB . 'ReleaseSearch.php'); -require_once (NN_LIB . 'ConsoleTools.php'); -require_once (NN_LIB . 'RequestIDLocal.php'); -require_once (NN_LIB . 'RequestIDWeb.php'); +require_once(NN_LIB . 'SphinxSearch.php'); +require_once(NN_LIB . 'ReleaseSearch.php'); +require_once(NN_LIB . 'ConsoleTools.php'); +require_once(NN_LIB . 'RequestIDLocal.php'); +require_once(NN_LIB . 'RequestIDWeb.php'); /** * This class handles storage and retrieval of releases rows and the main processing functions @@ -116,11 +117,6 @@ class Releases */ public $tablePerGroup; - /** - * @var int - */ - public $collectionDelayTime; - /** * @var int */ @@ -190,7 +186,7 @@ class Releases $this->site = $s->get(); $this->consoleTools = ($options['ConsoleTools'] instanceof \ConsoleTools ? $options['ConsoleTools'] : new \ConsoleTools(['ColorCLI' => $this->pdo->log])); $this->groups = ($options['Groups'] instanceof \Groups ? $options['Groups'] : new \Groups(['Settings' => $this->pdo])); - $this->nzb = ($options['NZB'] instanceof \NZB ? $options['NZB'] : new \NZB($this->pdo)); + $this->nzb = ($options['NZB'] instanceof \NZB ? $options['NZB'] : new \NZB()); $this->releaseCleaning = ($options['ReleaseCleaning'] instanceof \ReleaseCleaning ? $options['ReleaseCleaning'] : new \ReleaseCleaning($this->pdo)); $this->releaseImage = ($options['ReleaseImage'] instanceof \ReleaseImage ? $options['ReleaseImage'] : new \ReleaseImage($this->pdo)); $this->updategrabs = ($this->site->grabstatus == '0' ? false : true); @@ -200,9 +196,9 @@ class Releases $this->releaseRegex = new \ReleaseRegex(); $this->tablePerGroup = ($this->site->tablepergroup == 0 ? false : true); - $this->crossPostTime = ($this->site->crossposttime!= '' ? (int)$this->site->crossposttime : 2); + $this->crossPostTime = ($this->site->crossposttime != '' ? (int)$this->site->crossposttime : 2); $this->releaseCreationLimit = ($this->site->maxnzbsprocessed != '' ? (int)$this->site->maxnzbsprocessed : 1000); - $this->completion = ($this->site->releasecompletion!= '' ? (int)$this->site->releasecompletion : 0); + $this->completion = ($this->site->completionpercent != '' ? (int)$this->site->completionpercent : 0); $this->processRequestIDs = (int)$this->site->lookup_reqids; if ($this->completion > 100) { $this->completion = 100; @@ -228,9 +224,9 @@ class Releases $sql = sprintf(" SELECT releases.*, CONCAT(cp.title, ' > ', c.title) AS category_name, m.ID AS movie_id, m.title, m.rating, m.cover, m.plot, m.year, m.genre, m.director, m.actors, m.tagline, - mu.ID AS music_id, mu.title as mu_title, mu.cover as mu_cover, mu.year as mu_year, mu.artist as mu_artist, mu.tracks as mu_tracks, mu.review as mu_review, - ep.ID AS ep_id, ep.showtitle as ep_showtitle, ep.airdate as ep_airdate, ep.fullep as ep_fullep, ep.overview as ep_overview, - tvrage.imgdata as rage_imgdata, tvrage.ID as rg_ID + mu.ID AS music_id, mu.title AS mu_title, mu.cover AS mu_cover, mu.year AS mu_year, mu.artist AS mu_artist, mu.tracks AS mu_tracks, mu.review AS mu_review, + ep.ID AS ep_id, ep.showtitle AS ep_showtitle, ep.airdate AS ep_airdate, ep.fullep AS ep_fullep, ep.overview AS ep_overview, + tvrage.imgdata AS rage_imgdata, tvrage.ID AS rg_ID FROM releases LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID @@ -238,7 +234,7 @@ class Releases LEFT OUTER JOIN musicinfo mu ON mu.ID = releases.musicinfoID LEFT OUTER JOIN episodeinfo ep ON ep.ID = releases.episodeinfoID LEFT OUTER JOIN tvrage ON tvrage.rageID = releases.rageID - where %s", $nsql + WHERE %s", $nsql ); return $this->pdo->queryDirect($sql); @@ -250,7 +246,7 @@ class Releases public function getCount() { - $res = $this->pdo->queryOneRow("select count(ID) as num from releases"); + $res = $this->pdo->queryOneRow("SELECT count(ID) AS num FROM releases"); return $res["num"]; } @@ -267,7 +263,7 @@ class Releases else $limit = " LIMIT " . $start . "," . $num; - return $this->pdo->query(" SELECT releases.*, concat(cp.title, ' > ', c.title) as category_name from releases left outer join category c on c.ID = releases.categoryID left outer join category cp on cp.ID = c.parentID order by postdate desc" . $limit); + return $this->pdo->query(" SELECT releases.*, concat(cp.title, ' > ', c.title) AS category_name FROM releases LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID ORDER BY postdate DESC" . $limit); } /** @@ -299,7 +295,7 @@ class Releases $catsrch .= "1=2 )"; } - $sql = sprintf("select count(ID) as num from releases where haspreview = %d %s ", $previewtype, $catsrch); + $sql = sprintf("SELECT count(ID) AS num FROM releases WHERE haspreview = %d %s ", $previewtype, $catsrch); $res = $this->pdo->queryOneRow($sql); return $res["num"]; @@ -339,7 +335,7 @@ class Releases else $limit = " LIMIT " . $start . "," . $num; - $sql = sprintf(" SELECT releases.*, concat(cp.title, ' > ', c.title) as category_name from releases left outer join category c on c.ID = releases.categoryID left outer join category cp on cp.ID = c.parentID where haspreview = %d %s order by postdate desc %s", $previewtype, $catsrch, $limit); + $sql = sprintf(" SELECT releases.*, concat(cp.title, ' > ', c.title) AS category_name FROM releases LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID WHERE haspreview = %d %s ORDER BY postdate DESC %s", $previewtype, $catsrch, $limit); return $this->pdo->query($sql); } @@ -392,7 +388,7 @@ class Releases if (count($excludedcats) > 0) $exccatlist = " and categoryID not in (" . implode(",", $excludedcats) . ")"; - $sql = sprintf("select count(releases.ID) as num from releases left outer join groups on groups.ID = releases.groupID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s", $catsrch, $maxage, $exccatlist, $grpsql); + $sql = sprintf("SELECT count(releases.ID) AS num FROM releases LEFT OUTER JOIN groups ON groups.ID = releases.groupID WHERE releases.passwordstatus <= (SELECT value FROM site WHERE setting='showpasswordedrelease') %s %s %s %s", $catsrch, $maxage, $exccatlist, $grpsql); $res = $this->pdo->queryOneRow($sql, true); return $res["num"]; @@ -440,7 +436,7 @@ class Releases $grpsql = ""; if (count($grp) > 0) { - $grpsql = "select ID from groups where ("; + $grpsql = "SELECT ID FROM groups WHERE ("; foreach ($grp as $grpname) $grpsql .= sprintf(" groups.name = %s or ", $this->pdo->escapeString(str_replace("a.b.", "alt.binaries.", $grpname))); @@ -462,7 +458,7 @@ class Releases $exccatlist = " and releases.categoryID not in (" . implode(",", $excludedcats) . ")"; $order = $this->getBrowseOrder($orderby); - $sql = sprintf(" SELECT releases.*, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, rn.ID as nfoID, re.releaseID as reID, pre.ctime, pre.nuketype, coalesce(movieinfo.ID,0) as movieinfoID from releases %s left outer join groups on groups.ID = releases.groupID left outer join movieinfo on movieinfo.imdbID = releases.imdbID left outer join releaseaudio re on re.releaseID = releases.ID and re.audioID = 1 left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category c on c.ID = releases.categoryID left outer join category cp on cp.ID = c.parentID left outer join predb pre on pre.ID = releases.preID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s order by %s %s" . $limit, $usecatindex, $catsrch, $maxagesql, $exccatlist, $grpsql, $order[0], $order[1]); + $sql = sprintf(" SELECT releases.*, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID, re.releaseID AS reID, pre.ctime, pre.nuketype, coalesce(movieinfo.ID,0) AS movieinfoID FROM releases %s LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN movieinfo ON movieinfo.imdbID = releases.imdbID LEFT OUTER JOIN releaseaudio re ON re.releaseID = releases.ID AND re.audioID = 1 LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID LEFT OUTER JOIN predb pre ON pre.ID = releases.preID WHERE releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') %s %s %s %s ORDER BY %s %s" . $limit, $usecatindex, $catsrch, $maxagesql, $exccatlist, $grpsql, $order[0], $order[1]); return $this->pdo->query($sql, true); } @@ -555,7 +551,7 @@ class Releases else $cat = ""; - return $this->pdo->queryDirect(sprintf("SELECT searchname, guid, CONCAT(cp.title,'_',category.title) as catName FROM releases INNER JOIN category ON releases.categoryID = category.ID LEFT OUTER JOIN category cp ON cp.ID = category.parentID where 1 = 1 %s %s %s %s", $postfrom, $postto, $group, $cat)); + return $this->pdo->queryDirect(sprintf("SELECT searchname, guid, CONCAT(cp.title,'_',category.title) AS catName FROM releases INNER JOIN category ON releases.categoryID = category.ID LEFT OUTER JOIN category cp ON cp.ID = category.parentID WHERE 1 = 1 %s %s %s %s", $postfrom, $postto, $group, $cat)); } /** @@ -564,7 +560,7 @@ class Releases public function getEarliestUsenetPostDate() { - $row = $this->pdo->queryOneRow("SELECT DATE_FORMAT(min(postdate), '%d/%m/%Y') as postdate from releases"); + $row = $this->pdo->queryOneRow("SELECT DATE_FORMAT(min(postdate), '%d/%m/%Y') AS postdate FROM releases"); return $row["postdate"]; } @@ -575,7 +571,7 @@ class Releases public function getLatestUsenetPostDate() { - $row = $this->pdo->queryOneRow("SELECT DATE_FORMAT(max(postdate), '%d/%m/%Y') as postdate from releases"); + $row = $this->pdo->queryOneRow("SELECT DATE_FORMAT(max(postdate), '%d/%m/%Y') AS postdate FROM releases"); return $row["postdate"]; } @@ -586,7 +582,7 @@ class Releases public function getReleasedGroupsForSelect($blnIncludeAll = true) { - $groups = $this->pdo->query("select distinct groups.ID, groups.name from releases inner join groups on groups.ID = releases.groupID"); + $groups = $this->pdo->query("SELECT DISTINCT groups.ID, groups.name FROM releases INNER JOIN groups ON groups.ID = releases.groupID"); $temp_array = array(); if ($blnIncludeAll) @@ -640,7 +636,7 @@ class Releases $anidb = ($anidbid > -1) ? sprintf(" and releases.anidbID = %d ", $anidbid) : ''; $airdate = ($airdate > -1) ? sprintf(" and releases.tvairdate >= DATE_SUB(CURDATE(), INTERVAL %d DAY) ", $airdate) : ''; - $sql = sprintf(" SELECT releases.*, rn.ID as nfoID, m.title as imdbtitle, m.cover, m.imdbID, m.rating, m.plot, m.year, m.genre, m.director, m.actors, g.name as group_name, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, coalesce(cp.ID,0) as parentCategoryID, mu.title as mu_title, mu.url as mu_url, mu.artist as mu_artist, mu.publisher as mu_publisher, mu.releasedate as mu_releasedate, mu.review as mu_review, mu.tracks as mu_tracks, mu.cover as mu_cover, mug.title as mu_genre, co.title as co_title, co.url as co_url, co.publisher as co_publisher, co.releasedate as co_releasedate, co.review as co_review, co.cover as co_cover, cog.title as co_genre, bo.title as bo_title, bo.url as bo_url, bo.publisher as bo_publisher, bo.author as bo_author, bo.publishdate as bo_publishdate, bo.review as bo_review, bo.cover as bo_cover from releases left outer join category c on c.ID = releases.categoryID left outer join category cp on cp.ID = c.parentID left outer join groups g on g.ID = releases.groupID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join movieinfo m on m.imdbID = releases.imdbID and m.title != '' left outer join musicinfo mu on mu.ID = releases.musicinfoID left outer join genres mug on mug.ID = mu.genreID left outer join bookinfo bo on bo.ID = releases.bookinfoID left outer join consoleinfo co on co.ID = releases.consoleinfoID left outer join genres cog on cog.ID = co.genreID %s where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s order by postdate desc %s", $cartsrch, $catsrch, $rage, $anidb, $airdate, $limit); + $sql = sprintf(" SELECT releases.*, rn.ID AS nfoID, m.title AS imdbtitle, m.cover, m.imdbID, m.rating, m.plot, m.year, m.genre, m.director, m.actors, g.name AS group_name, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, coalesce(cp.ID,0) AS parentCategoryID, mu.title AS mu_title, mu.url AS mu_url, mu.artist AS mu_artist, mu.publisher AS mu_publisher, mu.releasedate AS mu_releasedate, mu.review AS mu_review, mu.tracks AS mu_tracks, mu.cover AS mu_cover, mug.title AS mu_genre, co.title AS co_title, co.url AS co_url, co.publisher AS co_publisher, co.releasedate AS co_releasedate, co.review AS co_review, co.cover AS co_cover, cog.title AS co_genre, bo.title AS bo_title, bo.url AS bo_url, bo.publisher AS bo_publisher, bo.author AS bo_author, bo.publishdate AS bo_publishdate, bo.review AS bo_review, bo.cover AS bo_cover FROM releases LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID LEFT OUTER JOIN groups g ON g.ID = releases.groupID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN movieinfo m ON m.imdbID = releases.imdbID AND m.title != '' LEFT OUTER JOIN musicinfo mu ON mu.ID = releases.musicinfoID LEFT OUTER JOIN genres mug ON mug.ID = mu.genreID LEFT OUTER JOIN bookinfo bo ON bo.ID = releases.bookinfoID LEFT OUTER JOIN consoleinfo co ON co.ID = releases.consoleinfoID LEFT OUTER JOIN genres cog ON cog.ID = co.genreID %s WHERE releases.passwordstatus <= (SELECT value FROM site WHERE setting='showpasswordedrelease') %s %s %s %s ORDER BY postdate DESC %s", $cartsrch, $catsrch, $rage, $anidb, $airdate, $limit); return $this->pdo->query($sql, true); } @@ -656,7 +652,7 @@ class Releases if (count($excludedcats) > 0) $exccatlist = " and releases.categoryID not in (" . implode(",", $excludedcats) . ")"; - $usershows = $this->pdo->query(sprintf("select rageID, categoryID from userseries where userID = %d", $uid), true); + $usershows = $this->pdo->query(sprintf("SELECT rageID, categoryID FROM userseries WHERE userID = %d", $uid), true); $usql = '(1=2 '; foreach ($usershows as $ushow) { $usql .= sprintf('or (releases.rageID = %d', $ushow['rageID']); @@ -675,21 +671,21 @@ class Releases $limit = " LIMIT 0," . ($num > 100 ? 100 : $num); - $sql = sprintf(" SELECT releases.*, tvr.rageID, tvr.releasetitle, epinfo.overview, epinfo.director, epinfo.gueststars, epinfo.writer, epinfo.rating, epinfo.fullep, epinfo.showtitle, epinfo.tvdbID as ep_tvdbID, g.name as group_name, concat(cp.title, '-', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, coalesce(cp.ID,0) as parentCategoryID + $sql = sprintf(" SELECT releases.*, tvr.rageID, tvr.releasetitle, epinfo.overview, epinfo.director, epinfo.gueststars, epinfo.writer, epinfo.rating, epinfo.fullep, epinfo.showtitle, epinfo.tvdbID AS ep_tvdbID, g.name AS group_name, concat(cp.title, '-', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, coalesce(cp.ID,0) AS parentCategoryID FROM releases FORCE INDEX (ix_releases_rageID) - left outer join category c on c.ID = releases.categoryID - left outer join category cp on cp.ID = c.parentID - left outer join groups g on g.ID = releases.groupID - left outer join (SELECT ID, releasetitle, rageid FROM tvrage GROUP BY rageid) tvr on tvr.rageID = releases.rageID - left outer join episodeinfo epinfo on epinfo.ID = releases.episodeinfoID - inner join - ( select ID from - ( select id, rageID, categoryID, season, episode from releases where %s order by season desc, episode desc, postdate asc ) releases - group by rageID, season, episode, categoryID - ) z on z.ID = releases.ID - where %s %s %s - and releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') - order by postdate desc %s", $usql, $usql, $exccatlist, $airdate, $limit + LEFT OUTER JOIN category c ON c.ID = releases.categoryID + LEFT OUTER JOIN category cp ON cp.ID = c.parentID + LEFT OUTER JOIN groups g ON g.ID = releases.groupID + LEFT OUTER JOIN (SELECT ID, releasetitle, rageid FROM tvrage GROUP BY rageid) tvr ON tvr.rageID = releases.rageID + LEFT OUTER JOIN episodeinfo epinfo ON epinfo.ID = releases.episodeinfoID + INNER JOIN + ( SELECT ID FROM + ( SELECT id, rageID, categoryID, season, episode FROM releases WHERE %s ORDER BY season DESC, episode DESC, postdate ASC ) releases + GROUP BY rageID, season, episode, categoryID + ) z ON z.ID = releases.ID + WHERE %s %s %s + AND releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') + ORDER BY postdate DESC %s", $usql, $usql, $exccatlist, $airdate, $limit ); return $this->pdo->query($sql); @@ -706,7 +702,7 @@ class Releases if (count($excludedcats) > 0) $exccatlist = " and releases.categoryID not in (" . implode(",", $excludedcats) . ")"; - $usermovies = $this->pdo->query(sprintf("select imdbID, categoryID from usermovies where userID = %d", $uid), true); + $usermovies = $this->pdo->query(sprintf("SELECT imdbID, categoryID FROM usermovies WHERE userID = %d", $uid), true); $usql = '(1=2 '; foreach ($usermovies as $umov) { $usql .= sprintf('or (releases.imdbID = %d', $umov['imdbID']); @@ -723,15 +719,15 @@ class Releases $limit = " LIMIT 0," . ($num > 100 ? 100 : $num); - $sql = sprintf(" SELECT releases.*, mi.title as releasetitle, g.name as group_name, concat(cp.title, '-', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, coalesce(cp.ID,0) as parentCategoryID + $sql = sprintf(" SELECT releases.*, mi.title AS releasetitle, g.name AS group_name, concat(cp.title, '-', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, coalesce(cp.ID,0) AS parentCategoryID FROM releases - left outer join category c on c.ID = releases.categoryID - left outer join category cp on cp.ID = c.parentID - left outer join groups g on g.ID = releases.groupID - left outer join movieinfo mi on mi.imdbID = releases.imdbID - where %s %s - and releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') - order by postdate desc %s", $usql, $exccatlist, $limit + LEFT OUTER JOIN category c ON c.ID = releases.categoryID + LEFT OUTER JOIN category cp ON cp.ID = c.parentID + LEFT OUTER JOIN groups g ON g.ID = releases.groupID + LEFT OUTER JOIN movieinfo mi ON mi.imdbID = releases.imdbID + WHERE %s %s + AND releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') + ORDER BY postdate DESC %s", $usql, $exccatlist, $limit ); return $this->pdo->query($sql); @@ -772,7 +768,7 @@ class Releases $maxagesql = sprintf(" and releases.postdate > now() - interval %d day ", $maxage); $order = $this->getBrowseOrder($orderby); - $sql = sprintf(" SELECT releases.*, concat(cp.title, '-', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, pre.ctime, pre.nuketype, rn.ID as nfoID, re.releaseID as reID from releases left outer join releasevideo re on re.releaseID = releases.ID left outer join groups on groups.ID = releases.groupID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category c on c.ID = releases.categoryID left outer join predb pre on pre.ID = releases.preID left outer join category cp on cp.ID = c.parentID where %s %s and releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s order by %s %s" . $limit, $usql, $exccatlist, $maxagesql, $order[0], $order[1]); + $sql = sprintf(" SELECT releases.*, concat(cp.title, '-', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, pre.ctime, pre.nuketype, rn.ID AS nfoID, re.releaseID AS reID FROM releases LEFT OUTER JOIN releasevideo re ON re.releaseID = releases.ID LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN predb pre ON pre.ID = releases.preID LEFT OUTER JOIN category cp ON cp.ID = c.parentID WHERE %s %s AND releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') %s ORDER BY %s %s" . $limit, $usql, $exccatlist, $maxagesql, $order[0], $order[1]); return $this->pdo->query($sql, true); } @@ -806,7 +802,7 @@ class Releases if ($maxage > 0) $maxagesql = sprintf(" and releases.postdate > now() - interval %d day ", $maxage); - $res = $this->pdo->queryOneRow(sprintf(" SELECT count(releases.ID) as num from releases where %s %s and releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s", $usql, $exccatlist, $maxagesql), true); + $res = $this->pdo->queryOneRow(sprintf(" SELECT count(releases.ID) AS num FROM releases WHERE %s %s AND releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') %s", $usql, $exccatlist, $maxagesql), true); return $res["num"]; } @@ -827,7 +823,7 @@ class Releases public function updateHasPreview($guid, $haspreview) { - $this->pdo->exec(sprintf("update releases set haspreview = %d where guid = %s", $haspreview, $this->pdo->escapeString($guid))); + $this->pdo->queryExec(sprintf("UPDATE releases SET haspreview = %d WHERE guid = %s", $haspreview, $this->pdo->escapeString($guid))); } /** @@ -837,7 +833,7 @@ class Releases { - $this->pdo->exec(sprintf("update releases set name=%s, searchname=%s, fromname=%s, categoryID=%d, totalpart=%d, grabs=%d, size=%s, postdate=%s, adddate=%s, rageID=%d, seriesfull=%s, season=%s, episode=%s, imdbID=%d, anidbID=%d, tvdbID=%d,consoleinfoID=%d where id = %d", + $this->pdo->queryExec(sprintf("UPDATE releases SET name=%s, searchname=%s, fromname=%s, categoryID=%d, totalpart=%d, grabs=%d, size=%s, postdate=%s, adddate=%s, rageID=%d, seriesfull=%s, season=%s, episode=%s, imdbID=%d, anidbID=%d, tvdbID=%d,consoleinfoID=%d WHERE id = %d", $this->pdo->escapeString($name), $this->pdo->escapeString($searchname), $this->pdo->escapeString($fromname), $category, $parts, $grabs, $this->pdo->escapeString($size), $this->pdo->escapeString($posteddate), $this->pdo->escapeString($addeddate), $rageid, $this->pdo->escapeString($seriesfull), $this->pdo->escapeString($season), $this->pdo->escapeString($episode), $imdbid, $anidbid, $tvdbid, $consoleinfoid, $id ) ); @@ -876,18 +872,11 @@ class Releases $updateGuids[] = $this->pdo->escapeString($guid); } - $sql = sprintf('update releases set ' . implode(', ', $updateSql) . ' where guid in (%s)', implode(', ', $updateGuids)); + $sql = sprintf('UPDATE releases SET ' . implode(', ', $updateSql) . ' WHERE guid IN (%s)', implode(', ', $updateGuids)); - return $this->pdo->exec($sql); + return $this->pdo->queryExec($sql); } - /** - * Not yet implemented. - */ - public function searchadv($searchname, $filename, $poster, $group, $cat = array(-1), $sizefrom, $sizeto, $offset = 0, $limit = 1000, $orderby = '', $maxage = -1, $excludedcats = array()) - { - return array(); - } /** * Search for releases by rage id. Used by API/Sickbeard. @@ -906,7 +895,6 @@ class Releases } - if ($rageId != "-1") $rageId = sprintf(" and rageID = %d ", $rageId); else @@ -982,10 +970,10 @@ class Releases else $maxage = ""; - $sql = sprintf("select releases.*, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, rn.ID as nfoID, re.releaseID as reID from releases %s left outer join category c on c.ID = releases.categoryID left outer join groups on groups.ID = releases.groupID left outer join releasevideo re on re.releaseID = releases.ID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category cp on cp.ID = c.parentID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s %s %s order by postdate desc limit %d, %d ", $usecatindex, $rageId, $series, $episode, $searchsql, $catsrch, $maxage, $offset, $limit); + $sql = sprintf("SELECT releases.*, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID, re.releaseID AS reID FROM releases %s LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN releasevideo re ON re.releaseID = releases.ID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID WHERE releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') %s %s %s %s %s %s ORDER BY postdate DESC LIMIT %d, %d ", $usecatindex, $rageId, $series, $episode, $searchsql, $catsrch, $maxage, $offset, $limit); $orderpos = strpos($sql, "order by"); $wherepos = strpos($sql, "where"); - $sqlcount = "select count(releases.ID) as num from releases " . substr($sql, $wherepos, $orderpos - $wherepos); + $sqlcount = "SELECT count(releases.ID) AS num FROM releases " . substr($sql, $wherepos, $orderpos - $wherepos); $countres = $this->pdo->queryOneRow($sqlcount, true); $res = $this->pdo->query($sql, true); @@ -1010,7 +998,6 @@ class Releases } - $anidbID = ($anidbID > -1) ? sprintf(" AND anidbID = %d ", $anidbID) : ''; $epno = is_numeric($epno) ? sprintf(" AND releases.episode LIKE '%s' ", $this->pdo->escapeString('%' . $epno . '%')) : ''; @@ -1045,8 +1032,8 @@ class Releases $sql = sprintf("SELECT releases.*, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID FROM releases LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN groups ON groups.ID = releases.groupID - LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID and rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID - WHERE releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s ORDER BY postdate desc LIMIT %d, %d ", + LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID + WHERE releases.passwordstatus <= (SELECT value FROM site WHERE setting='showpasswordedrelease') %s %s %s %s ORDER BY postdate DESC LIMIT %d, %d ", $anidbID, $epno, $searchsql, $maxage, $offset, $limit ); $orderpos = strpos($sql, "ORDER BY"); @@ -1128,10 +1115,10 @@ class Releases $genresql .= "1=2 )"; } - $sql = sprintf("select releases.*, musicinfo.cover as mi_cover, musicinfo.review as mi_review, musicinfo.tracks as mi_tracks, musicinfo.publisher as mi_publisher, musicinfo.title as mi_title, musicinfo.artist as mi_artist, genres.title as music_genrename, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, rn.ID as nfoID from releases %s left outer join musicinfo on musicinfo.ID = releases.musicinfoID left join genres on genres.ID = musicinfo.genreID left outer join groups on groups.ID = releases.groupID left outer join category c on c.ID = releases.categoryID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category cp on cp.ID = c.parentID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s order by postdate desc limit %d, %d ", $usecatindex, $searchsql, $catsrch, $maxage, $genresql, $offset, $limit); + $sql = sprintf("SELECT releases.*, musicinfo.cover AS mi_cover, musicinfo.review AS mi_review, musicinfo.tracks AS mi_tracks, musicinfo.publisher AS mi_publisher, musicinfo.title AS mi_title, musicinfo.artist AS mi_artist, genres.title AS music_genrename, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID FROM releases %s LEFT OUTER JOIN musicinfo ON musicinfo.ID = releases.musicinfoID LEFT JOIN genres ON genres.ID = musicinfo.genreID LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID WHERE releases.passwordstatus <= (SELECT VALUE FROM site WHERE setting='showpasswordedrelease') %s %s %s %s ORDER BY postdate DESC LIMIT %d, %d ", $usecatindex, $searchsql, $catsrch, $maxage, $genresql, $offset, $limit); $orderpos = strpos($sql, "order by"); $wherepos = strpos($sql, "where"); - $sqlcount = "select count(releases.ID) as num from releases inner join musicinfo on musicinfo.ID = releases.musicinfoID " . substr($sql, $wherepos, $orderpos - $wherepos); + $sqlcount = "SELECT count(releases.ID) AS num FROM releases INNER JOIN musicinfo ON musicinfo.ID = releases.musicinfoID " . substr($sql, $wherepos, $orderpos - $wherepos); $countres = $this->pdo->queryOneRow($sqlcount, true); $res = $this->pdo->query($sql, true); @@ -1168,10 +1155,10 @@ class Releases else $maxage = ""; - $sql = sprintf("select releases.*, bookinfo.cover as bi_cover, bookinfo.review as bi_review, bookinfo.publisher as bi_publisher, bookinfo.pages as bi_pages, bookinfo.publishdate as bi_publishdate, bookinfo.title as bi_title, bookinfo.author as bi_author, genres.title as book_genrename, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, rn.ID as nfoID from releases left outer join bookinfo on bookinfo.ID = releases.bookinfoID left join genres on genres.ID = bookinfo.genreID left outer join groups on groups.ID = releases.groupID left outer join category c on c.ID = releases.categoryID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category cp on cp.ID = c.parentID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s order by postdate desc limit %d, %d ", $searchsql, $maxage, $offset, $limit); + $sql = sprintf("SELECT releases.*, bookinfo.cover AS bi_cover, bookinfo.review AS bi_review, bookinfo.publisher AS bi_publisher, bookinfo.pages AS bi_pages, bookinfo.publishdate AS bi_publishdate, bookinfo.title AS bi_title, bookinfo.author AS bi_author, genres.title AS book_genrename, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID FROM releases LEFT OUTER JOIN bookinfo ON bookinfo.ID = releases.bookinfoID LEFT JOIN genres ON genres.ID = bookinfo.genreID LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID WHERE releases.passwordstatus <= (SELECT value FROM site WHERE setting='showpasswordedrelease') %s %s ORDER BY postdate DESC LIMIT %d, %d ", $searchsql, $maxage, $offset, $limit); $orderpos = strpos($sql, "order by"); $wherepos = strpos($sql, "where"); - $sqlcount = "select count(releases.ID) as num from releases inner join bookinfo on bookinfo.ID = releases.bookinfoID " . substr($sql, $wherepos, $orderpos - $wherepos); + $sqlcount = "SELECT count(releases.ID) AS num FROM releases INNER JOIN bookinfo ON bookinfo.ID = releases.bookinfoID " . substr($sql, $wherepos, $orderpos - $wherepos); $countres = $this->pdo->queryOneRow($sqlcount, true); $res = $this->pdo->query($sql, true); @@ -1196,7 +1183,6 @@ class Releases } - if ($imdbId != "-1" && is_numeric($imdbId)) { //pad id with zeros just in case $imdbId = str_pad($imdbId, 7, "0", STR_PAD_LEFT); @@ -1261,10 +1247,10 @@ class Releases $genre = sprintf(" and movieinfo.genre like %s", $this->pdo->escapeString("%" . $genre . "%")); } - $sql = sprintf("select releases.*, movieinfo.title as moi_title, movieinfo.tagline as moi_tagline, movieinfo.rating as moi_rating, movieinfo.plot as moi_plot, movieinfo.year as moi_year, movieinfo.genre as moi_genre, movieinfo.director as moi_director, movieinfo.actors as moi_actors, movieinfo.cover as moi_cover, movieinfo.backdrop as moi_backdrop, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, rn.ID as nfoID from releases left outer join groups on groups.ID = releases.groupID left outer join category c on c.ID = releases.categoryID left outer join releasenfo rn on rn.releaseID = releases.ID and rn.nfo is not null left outer join category cp on cp.ID = c.parentID left outer join movieinfo on releases.imdbID = movieinfo.imdbID where releases.passwordstatus <= (select value from site where setting='showpasswordedrelease') %s %s %s %s %s order by postdate desc limit %d, %d ", $searchsql, $imdbId, $catsrch, $maxage, $genre, $offset, $limit); + $sql = sprintf("SELECT releases.*, movieinfo.title AS moi_title, movieinfo.tagline AS moi_tagline, movieinfo.rating AS moi_rating, movieinfo.plot AS moi_plot, movieinfo.year AS moi_year, movieinfo.genre AS moi_genre, movieinfo.director AS moi_director, movieinfo.actors AS moi_actors, movieinfo.cover AS moi_cover, movieinfo.backdrop AS moi_backdrop, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, rn.ID AS nfoID FROM releases LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN releasenfo rn ON rn.releaseID = releases.ID AND rn.nfo IS NOT NULL LEFT OUTER JOIN category cp ON cp.ID = c.parentID LEFT OUTER JOIN movieinfo ON releases.imdbID = movieinfo.imdbID WHERE releases.passwordstatus <= (SELECT value FROM site WHERE setting='showpasswordedrelease') %s %s %s %s %s ORDER BY postdate DESC LIMIT %d, %d ", $searchsql, $imdbId, $catsrch, $maxage, $genre, $offset, $limit); $orderpos = strpos($sql, "order by"); $wherepos = strpos($sql, "where"); - $sqlcount = "select count(releases.ID) as num from releases left outer join movieinfo on releases.imdbID = movieinfo.imdbID " . substr($sql, $wherepos, $orderpos - $wherepos); + $sqlcount = "SELECT count(releases.ID) AS num FROM releases LEFT OUTER JOIN movieinfo ON releases.imdbID = movieinfo.imdbID " . substr($sql, $wherepos, $orderpos - $wherepos); $countres = $this->pdo->queryOneRow($sqlcount, true); $res = $this->pdo->query($sql, true); @@ -1343,7 +1329,7 @@ class Releases $sphinxSearch = new SphinxSearch(); $releaseSearch = new ReleaseSearch($this->pdo, $sphinxSearch); - $sizeRange = range(1,11); + $sizeRange = range(1, 11); $groups = new Groups(); if ($orderBy == '') { @@ -1417,6 +1403,7 @@ class Releases if ($releases && count($releases)) { $releases[0]['_totalrows'] = $this->getPagerCount($baseSql); } + return $releases; } @@ -1440,6 +1427,7 @@ class Releases if (isset($count['count']) && is_numeric($count['count'])) { return $count['count']; } + return 0; } @@ -1483,7 +1471,7 @@ class Releases { - return $this->pdo->queryOneRow(sprintf("select releases.*, groups.name as group_name from releases left outer join groups on groups.ID = releases.groupID where releases.ID = %d ", $id)); + return $this->pdo->queryOneRow(sprintf("SELECT releases.*, groups.name AS group_name FROM releases LEFT OUTER JOIN groups ON groups.ID = releases.groupID WHERE releases.ID = %d ", $id)); } /** @@ -1531,7 +1519,7 @@ class Releases } else { $gsql = sprintf('guid = %s', $this->pdo->escapeString($guid)); } - $sql = sprintf("select releases.*, musicinfo.cover as mi_cover, musicinfo.review as mi_review, musicinfo.tracks as mi_tracks, musicinfo.publisher as mi_publisher, musicinfo.title as mi_title, musicinfo.artist as mi_artist, music_genre.title as music_genrename, bookinfo.cover as bi_cover, bookinfo.review as bi_review, bookinfo.publisher as bi_publisher, bookinfo.publishdate as bi_publishdate, bookinfo.title as bi_title, bookinfo.author as bi_author, bookinfo.pages as bi_pages, bookinfo.isbn as bi_isbn, concat(cp.title, ' > ', c.title) as category_name, concat(cp.ID, ',', c.ID) as category_ids, groups.name as group_name, movieinfo.title AS movietitle, movieinfo.year AS movieyear, (SELECT releasetitle FROM tvrage WHERE rageid = releases.rageid AND rageid > 0 LIMIT 1) AS tvreleasetitle from releases left outer join groups on groups.ID = releases.groupID left outer join category c on c.ID = releases.categoryID left outer join category cp on cp.ID = c.parentID left outer join musicinfo on musicinfo.ID = releases.musicinfoID left outer join bookinfo on bookinfo.ID = releases.bookinfoID left outer join movieinfo on movieinfo.imdbID = releases.imdbID left join genres music_genre on music_genre.ID = musicinfo.genreID where %s", $gsql); + $sql = sprintf("SELECT releases.*, musicinfo.cover AS mi_cover, musicinfo.review AS mi_review, musicinfo.tracks AS mi_tracks, musicinfo.publisher AS mi_publisher, musicinfo.title AS mi_title, musicinfo.artist AS mi_artist, music_genre.title AS music_genrename, bookinfo.cover AS bi_cover, bookinfo.review AS bi_review, bookinfo.publisher AS bi_publisher, bookinfo.publishdate AS bi_publishdate, bookinfo.title AS bi_title, bookinfo.author AS bi_author, bookinfo.pages AS bi_pages, bookinfo.isbn AS bi_isbn, concat(cp.title, ' > ', c.title) AS category_name, concat(cp.ID, ',', c.ID) AS category_ids, groups.name AS group_name, movieinfo.title AS movietitle, movieinfo.year AS movieyear, (SELECT releasetitle FROM tvrage WHERE rageid = releases.rageid AND rageid > 0 LIMIT 1) AS tvreleasetitle FROM releases LEFT OUTER JOIN groups ON groups.ID = releases.groupID LEFT OUTER JOIN category c ON c.ID = releases.categoryID LEFT OUTER JOIN category cp ON cp.ID = c.parentID LEFT OUTER JOIN musicinfo ON musicinfo.ID = releases.musicinfoID LEFT OUTER JOIN bookinfo ON bookinfo.ID = releases.bookinfoID LEFT OUTER JOIN movieinfo ON movieinfo.imdbID = releases.imdbID LEFT JOIN genres music_genre ON music_genre.ID = musicinfo.genreID WHERE %s", $gsql); return (is_array($guid)) ? $this->pdo->query($sql) : $this->pdo->queryOneRow($sql); } @@ -1542,9 +1530,9 @@ class Releases public function removeRageIdFromReleases($rageid) { - $res = $this->pdo->queryOneRow(sprintf("select count(ID) as num from releases where rageID = %d", $rageid)); + $res = $this->pdo->queryOneRow(sprintf("SELECT count(ID) AS num FROM releases WHERE rageID = %d", $rageid)); $ret = $res["num"]; - $this->pdo->exec(sprintf("update releases set rageID = -1, seriesfull = null, season = null, episode = null where rageID = %d", $rageid)); + $this->pdo->queryExec(sprintf("UPDATE releases SET rageID = -1, seriesfull = NULL, season = NULL, episode = NULL WHERE rageID = %d", $rageid)); return $ret; } @@ -1557,7 +1545,7 @@ class Releases $res = $this->pdo->queryOneRow(sprintf("SELECT count(ID) AS num FROM releases WHERE tvdbID = %d", $tvdbID)); $ret = $res["num"]; - $res = $this->pdo->exec(sprintf("update releases SET tvdbID = -1 where tvdbID = %d", $tvdbID)); + $res = $this->pdo->queryExec(sprintf("UPDATE releases SET tvdbID = -1 WHERE tvdbID = %d", $tvdbID)); return $ret; } @@ -1567,7 +1555,7 @@ class Releases $res = $this->pdo->queryOneRow(sprintf("SELECT count(ID) AS num FROM releases WHERE anidbID = %d", $anidbID)); $ret = $res["num"]; - $this->pdo->exec(sprintf("update releases SET anidbID = -1, episode = null, tvtitle = null, tvairdate = null where anidbID = %d", $anidbID)); + $this->pdo->queryExec(sprintf("UPDATE releases SET anidbID = -1, episode = NULL, tvtitle = NULL, tvairdate = NULL WHERE anidbID = %d", $anidbID)); return $ret; } @@ -1583,15 +1571,22 @@ class Releases public function updateGrab($guid) { - $this->pdo->exec(sprintf("update releases set grabs = grabs + 1 where guid = %s", $this->pdo->escapeString($guid))); + $this->pdo->queryExec(sprintf("UPDATE releases SET grabs = grabs + 1 WHERE guid = %s", $this->pdo->escapeString($guid))); } - public function processReleases($categorize, $groupName, $echooutput) - { - $currTime_ori = $this->pdo->queryOneRow("SELECT NOW() as now"); + /** + * @param $categorize + * @param $postProcess + * @param $groupName + * @param $nntp + * @param $echooutput + * + * @return int + */ + public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput) + { $retcount = 0; - $miscRetentionDeleted = $miscHashedDeleted = 0; $this->echoCLI = ($echooutput && NN_ECHOCLI); $page = new Page(); $groupID = ''; @@ -1603,73 +1598,156 @@ class Releases $groupID = $groupInfo['ID']; } - echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n"; + $this->pdo->log->doEcho($this->pdo->log->primary('Starting release update process' . date("Y-m-d H:i:s")), true); if (!file_exists($page->site->nzbpath)) { - echo "Bad or missing nzb directory - " . $page->site->nzbpath; + $this->pdo->log->doEcho($this->pdo->log->primary('Bad or missing nzb directory - ' . $page->site->nzbpath), true); return -1; } $this->checkRegexesUptoDate($page->site->latestregexurl, $page->site->latestregexrevision, $page->site->newznabID); - // - // Get all regexes for all groups which are to be applied to new binaries - // in order of how they should be applied - // - $this->releaseRegex->get(); - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 1 : Applying regex to binaries')); - $activeGroups = $this->groups->getActive(false); - foreach ($activeGroups as $groupArr) { - //check if regexes have already been applied during update binaries - if ($groupArr['regexmatchonly'] == 1) - continue; + $this->applyRegex($groupID); + $this->processIncompleteBinaries($groupID); - $groupRegexes = $this->releaseRegex->getForGroup($groupArr['name']); + $DIR = NN_MISC; + $PYTHON = shell_exec('which python3 2>/dev/null'); + $PYTHON = (empty($PYTHON) ? 'python -OOu' : 'python3 -OOu'); + $processRequestIDs = (int)$page->site->lookup_reqids; + $consoleTools = new ConsoleTools(['ColorCLI' => $this->pdo->log]); + $totalReleasesAdded = $loops = 0; + do { + $releasesAdded = $nzbFilesAdded = $this->createReleases($groupID); + $totalReleasesAdded += $releasesAdded; - echo "Stage 1 : Applying " . sizeof($groupRegexes) . " regexes to group " . $groupArr['name'] . "\n"; - - // Get out all binaries of STAGE0 for current group - $newUnmatchedBinaries = array(); - $ressql = sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts, binaries.procstat, binaries.fromname from binaries where groupID = %d and procstat IN (%d,%d) and regexID IS NULL order by binaries.date asc", $groupArr['ID'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED); - $resbin = $this->pdo->queryDirect($ressql); - - $matchedbins = 0; - while ($rowbin = $this->pdo->getAssocArray($resbin)) { - $regexMatches = array(); - foreach ($groupRegexes as $groupRegex) { - $regexCheck = $this->releaseRegex->performMatch($groupRegex, $rowbin['name']); - if ($regexCheck !== false) { - $regexMatches = $regexCheck; - break; + if ($processRequestIDs === 0) { + $this->processRequestIDs($groupID, 5000, true); + } else if ($processRequestIDs === 1) { + $this->processRequestIDs($groupID, 5000, true); + $this->processRequestIDs($groupID, 1000, false); + } else if ($processRequestIDs === 2) { + $requestIDTime = time(); + if ($this->echoCLI) { + $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request ID Threaded lookup.")); } - } - - if (!empty($regexMatches)) { - $matchedbins++; - $relparts = explode("/", $regexMatches['parts']); - $this->pdo->exec(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", - $this->pdo->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $this->pdo->escapeString($regexMatches['reqID']), $rowbin["ID"] + passthru("$PYTHON ${DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py"); + if ($this->echoCLI) { + $this->pdo->log->doEcho( + $this->pdo->log->primary( + "\nReleases updated in " . + $consoleTools->convertTime(time() - $requestIDTime) ) ); - } else { - if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) - $newUnmatchedBinaries[] = $rowbin['ID']; } - } + $this->categorizeReleases($categorize, $groupID); + $this->postProcessReleases($postProcess, $nntp); + $this->deleteBinaries($groupID); + // This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately + } while (($nzbFilesAdded > 0 || $releasesAdded > 0) && $loops++ < 3); - //mark as not matched - if (!empty($newUnmatchedBinaries)) - $this->pdo->exec(sprintf("update binaries set procstat=%d where ID IN (%s)", Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries))); + $this->deletedReleasesByGroup($groupID); + $this->deleteReleases(); + // + // User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically. + // + $users = new Users; + $users->pruneRequestHistory($page->site->userdownloadpurgedays); + + $this->pdo->log->doEcho($this->pdo->log->primary('Done : Added ' . $retcount . ' releases')); + + return $retcount; + } + + public function deleteBinaries($groupID) + { + $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $page = new Page(); + $currTime_ori = $this->pdo->queryOneRow("SELECT NOW() as now"); + // + // aggregate the releasefiles upto the releases. + // + $this->pdo->log->doEcho($this->pdo->log->primary('Aggregating Files')); + $this->pdo->queryExec("UPDATE releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID AND releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num"); + + // Remove the binaries and parts used to form releases, or that are duplicates. + // + if ($page->site->partsdeletechunks > 0) { + $this->pdo->log->doEcho($this->pdo->log->primary('Chunk deleting unused binaries and parts')); + $query = sprintf("SELECT p.ID AS partsID,b.ID AS binariesID FROM %s p + LEFT JOIN %s b ON b.ID = p.binaryID + WHERE b.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", + $group['pname'], + $group['bname'], + $this->pdo->escapeString($currTime_ori["now"]), + ceil($page->site->rawretentiondays * 24), + $page->site->partsdeletechunks + ); + + $cc = 0; + $done = false; + while (!$done) { + $dd = $cc; + $result = $this->pdo->query($query); + if (count($result) > 0) { + $pID = array(); + $bID = array(); + foreach ($result as $row) { + $pID[] = $row['partsID']; + $bID[] = $row['binariesID']; + } + $pID = '(' . implode(',', $pID) . ')'; + $bID = '(' . implode(',', $bID) . ')'; + $fr = $this->pdo->queryExec("DELETE FROM %s WHERE ID IN {$pID}", $group['pname']); + if ($fr > 0) { + $cc += $fr; + $cc += $this->pdo->queryExec("DELETE FROM %s WHERE ID IN {$bID}", $group['bname']); + } + unset($pID); + unset($bID); + if ($cc == $dd) { + $done = true; + } + echo($cc % 10000 ? '.' : ''); + } else { + $done = true; + } + } + $this->pdo->log->doEcho($this->pdo->log->primary('Complete - ' . $cc . ' rows affected')); + } else { + $this->pdo->log->doEcho($this->pdo->log->primary('Deleting unused binaries and parts')); + $this->pdo->queryExec(sprintf("DELETE %s, %s FROM %s JOIN %s ON %s.ID = %s.binaryID + WHERE %s.dateadded < %s - INTERVAL %d HOUR", + $group['pname'], + $group['bname'], + $group['pname'], + $group['bname'], + $group['bname'], + $group['pname'], + $group['bname'], + $this->pdo->escapeString($currTime_ori["now"]), + ceil($page->site->rawretentiondays * 24) + ) + ); } + } + /** + * @param $groupID + */ + public function processIncompleteBinaries($groupID) + { // // Move all binaries from releases which have the correct number of files on to the next stage. // - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 2 : Marking binaries where all parts are available')); - $result = $this->pdo->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) as date, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", Releases::PROCSTAT_TITLEMATCHED)); + $retcount = 0; + $currTime_ori = $this->pdo->queryOneRow("SELECT NOW() as now"); + $page = new Page(); + $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $this->pdo->log->doEcho($this->pdo->log->primary('Marking binaries where all parts are available')); + $result = $this->pdo->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) AS minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) AS date, COUNT(ID) AS num FROM %s WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x LEFT OUTER JOIN groups g ON g.ID = x.groupID INNER JOIN ( SELECT value AS minfilestoformrelease FROM site WHERE setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", $group['bname'], Releases::PROCSTAT_TITLEMATCHED)); while ($row = $this->pdo->getAssocArray($result)) { $retcount++; @@ -1680,7 +1758,7 @@ class Releases // if ($row["num"] < $row["minfilestoformrelease"]) { //echo "Number of files in release ".$row["relname"]." less than site/group setting (".$row['num']."/".$row["minfilestoformrelease"].")\n"; - $this->pdo->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]))); + $this->pdo->queryExec(sprintf("UPDATE %s SET procattempts = procattempts + 1 WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname = %s", $group['bname'], $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]))); } // @@ -1694,7 +1772,27 @@ class Releases $incomplete = true; } else { // Check that the binary is complete - $binlist = $this->pdo->query(sprintf("SELECT binaries.ID, totalParts, date, COUNT(DISTINCT parts.messageID) AS num FROM binaries, parts WHERE binaries.ID=parts.binaryID AND binaries.relname = %s AND binaries.procstat = %d AND binaries.groupID = %d AND binaries.fromname = %s GROUP BY binaries.ID ORDER BY NULL", $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]))); + $binlist = $this->pdo->query(sprintf('SELECT %s.ID, totalParts, date, COUNT(DISTINCT %s.messageID) AS num FROM %s, + %s WHERE %s.ID=%s.binaryID AND %s.relname = %s + AND %s.procstat = %d AND %s.groupID = %d AND %s.fromname = %s + GROUP BY %s.ID ORDER BY NULL', + $group['bname'], + $group['pname'], + $group['bname'], + $group['pname'], + $group['bname'], + $group['pname'], + $group['bname'], + $this->pdo->escapeString($row["relname"]), + $group['bname'], + Releases::PROCSTAT_TITLEMATCHED, + $group['bname'], + $row['groupID'], + $group['bname'], + $this->pdo->escapeString($row["fromname"]), + $group['bname'] + ) + ); foreach ($binlist as $rowbin) { if ($rowbin['num'] < $rowbin['totalParts']) { @@ -1715,8 +1813,8 @@ class Releases // // Try and get the name using the group // - $binGroup = $this->pdo->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"])); - $newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup["name"], $row["reqID"]); + $binGroup = $this->groups->getByNameById($groupID); + $newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup, $row["reqID"]); // // if the feed/group wasnt supported by the scraper, then just use the release name as the title. @@ -1729,16 +1827,16 @@ class Releases // Valid release with right number of files and title now, so move it on // if ($newtitle != "") { - $this->pdo->exec(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", - $this->pdo->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]) + $this->pdo->queryExec(sprintf("UPDATE %s SET relname = %s, procstat=%d WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname=%s", + $group['bname'], $this->pdo->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]) ) ); } else { // // Item not found, if the binary was added to the index yages ago, then give up. // - $maxaddeddate = $this->pdo->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", - $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]) + $maxaddeddate = $this->pdo->queryOneRow(sprintf("SELECT NOW() AS now, MAX(dateadded) AS dateadded FROM %s WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname=%s", + $group['bname'], $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]) ) ); @@ -1746,15 +1844,15 @@ class Releases // If added to the index over 48 hours ago, give up trying to determine the title // if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > (60 * 60 * 48)) { - $this->pdo->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", - Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]) + $this->pdo->queryExec(sprintf("UPDATE %s SET procstat=%d WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname=%s", + $group['bname'], Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]) ) ); } } } else { - $this->pdo->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", - Releases::PROCSTAT_READYTORELEASE, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]) + $this->pdo->queryExec(sprintf("UPDATE %s SET procstat=%d WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname=%s", + $group['bname'], Releases::PROCSTAT_READYTORELEASE, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]) ) ); } @@ -1766,33 +1864,28 @@ class Releases // else { // pointless updating of attempts, as was creating a lot of database writes for people regex groups with posts without part numbering. - //$this->pdo->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname=%s", $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $this->pdo->escapeString($row["fromname"]) )); + //$this->pdo->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname=%s", $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $groupID, $this->pdo->escapeString($row["fromname"]) )); } if ($retcount % 100 == 0) echo "."; } + } - $retcount = 0; - - - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 3 : Creating releases from complete binaries')); + /** + * @param $groupID + */ + public function createReleases($groupID) + { + $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $page = new Page(); + $this->pdo->log->doEcho($this->pdo->log->primary('Creating releases from complete binaries')); // // Get out all distinct relname, group from binaries of STAGE2 // - $result = $this->pdo->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, max(categoryID) as categoryID, max(regexID) as regexID, max(reqID) as reqID, MAX(date) as date, count(binaries.ID) as parts from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE)); + $categorize = new \Categorize(['Settings' => $this->pdo]); + $result = $this->pdo->queryDirect(sprintf("SELECT relname, groupID, g.name AS group_name, fromname, max(categoryID) AS categoryID, max(regexID) AS regexID, max(reqID) AS reqID, MAX(date) AS date, count(%s.ID) AS parts FROM %s INNER JOIN groups g ON g.ID = %s.groupID WHERE procstat = %d AND relname IS NOT NULL GROUP BY relname, g.name, groupID, fromname ORDER BY COUNT(%s.ID) DESC", $group['bname'], $group['bname'], $group['bname'], Releases::PROCSTAT_READYTORELEASE, $group['bname'])); while ($row = $this->pdo->getAssocArray($result)) { $relguid = md5(uniqid()); - - // - // Get categoryID if one has been allocated to this - // - if ($row["categoryID"] != "") { - $catId = $row["categoryID"]; - } - else { - $catId = $this->categorizeReleases($categorize, $groupID); - } - // Clean release name $releaseCleaning = new ReleaseCleaning(); $cleanRelName = $this->cleanReleaseName($row['relname']); @@ -1822,255 +1915,89 @@ class Releases } $relid = $this->insertRelease( [ - 'name' => $this->pdo->escapeString($cleanRelName), - 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), - 'totalpart' => $row["parts"], - 'groupID' => $row["groupID"], - 'guid' => $this->pdo->escapeString($relguid), - 'categoryID' => $catId, - 'regexID' => $row["regexID"], - 'postdate' => $this->pdo->escapeString($row['date']), - 'fromname' => $this->pdo->escapeString($row['fromname']), - 'reqID' => $row["reqID"], + 'name' => $this->pdo->escapeString($cleanRelName), + 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), + 'totalpart' => $row["parts"], + 'groupID' => $row["groupID"], + 'guid' => $this->pdo->escapeString($relguid), + 'categoryID' => $categorize->determineCategory($groupID, $cleanedName), + 'regexID' => $row["regexID"], + 'postdate' => $this->pdo->escapeString($row['date']), + 'fromname' => $this->pdo->escapeString($row['fromname']), + 'reqID' => $row["reqID"], 'passwordstatus' => ($page->site->checkpasswordedrar > 0 ? -1 : 0), - 'nzbstatus' => \Enzebe::NZB_NONE, - 'isrenamed' => ($properName === true ? 1 : 0), - 'reqidstatus' => ($isReqID === true ? 1 : 0), - 'prehashID' => ($prehashID === false ? 0 : $prehashID) + 'nzbstatus' => \Enzebe::NZB_NONE, + 'isrenamed' => ($properName === true ? 1 : 0), + 'reqidstatus' => ($isReqID === true ? 1 : 0), + 'prehashID' => ($prehashID === false ? 0 : $prehashID) ] ); // // Tag every binary for this release with its parent release id // - $this->pdo->exec(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", - Releases::PROCSTAT_RELEASED, $relid, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $this->pdo->escapeString($row["fromname"]) + $this->pdo->queryExec(sprintf("UPDATE %s SET procstat = %d, releaseID = %d WHERE relname = %s AND procstat = %d AND groupID = %d AND fromname=%s", + $group['bname'], Releases::PROCSTAT_RELEASED, $relid, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $groupID, $this->pdo->escapeString($row["fromname"]) ) ); - + $cat = new \Categorize(['Settings' => $this->pdo]); + // + // Write the nzb to disk + // // // Write the nzb to disk // + $catId = $cat->determineCategory($groupID, $cleanRelName); $nzbfile = $this->nzb->getNZBPath($relguid, $page->site->nzbpath, true); - $this->nzb->writeNZBforreleaseID($relid, $cleanRelName, $catId, $nzbfile); + $this->nzb->writeNZBforreleaseID($relid, $cleanRelName, $catId, $nzbfile, $groupID); // // Remove used binaries // - $this->pdo->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE releaseID = %d ", $relid)); + $this->pdo->queryExec(sprintf("DELETE %s, %s FROM %s JOIN %s ON %s.ID = %s.binaryID WHERE releaseID = %d ", + $group['pname'], + $group['bname'], + $group['pname'], + $group['bname'], + $group['bname'], + $group['pname'], + $relid + ) + ); // // If nzb successfully written, then load it and get size completion from it // $nzbInfo = new nzbInfo; if (!$nzbInfo->loadFromFile($nzbfile)) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 3 : Failed to write nzb file (bad perms?) ' . $nzbfile . '')); + $this->pdo->log->doEcho($this->pdo->log->primary('Failed to write nzb file (bad perms?) ' . $nzbfile . '')); //copy($nzbfile, "./ERRORNZB_".$relguid); $this->delete($relid); } else { // Check if gid already exists - $dupes = $this->pdo->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $this->pdo->escapeString($nzbInfo->gid))); + $dupes = $this->pdo->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) AS total", $this->pdo->escapeString($nzbInfo->gid))); if ($dupes['total'] > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 3 : Duplicate - ' . $cleanRelName . '')); + $this->pdo->log->doEcho($this->pdo->log->primary('Duplicate - ' . $cleanRelName . '')); $this->delete($relid); } else { - $this->pdo->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $this->pdo->escapeString($nzbInfo->gid), $relid)); - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 3 : Added release ' . $cleanRelName . '')); + $this->pdo->queryExec(sprintf("UPDATE releases SET totalpart = %d, size = %s, COMPLETION = %d, GID=%s WHERE ID = %d", + $nzbInfo->filecount, + $nzbInfo->filesize, + $nzbInfo->completion, + $this->pdo->escapeString($nzbInfo->gid), + $relid + ) + ); + $this->pdo->log->doEcho($this->pdo->log->primary('Added release ' . $cleanRelName . '')); - //Increment new release count - $retcount++; } - } + } } - - $DIR = NN_MISC; - $PYTHON = shell_exec('which python3 2>/dev/null'); - $PYTHON = (empty($PYTHON) ? 'python -OOu' : 'python3 -OOu'); - $processRequestIDs = (int)$page->site->lookup_reqids; - $consoleTools = new ConsoleTools(['ColorCLI' => $this->pdo->log]); - - if ($processRequestIDs === 0) { - $this->processRequestIDs($groupArr['ID'], 5000, true); - } else if ($processRequestIDs === 1) { - $this->processRequestIDs($groupArr['ID'], 5000, true); - $this->processRequestIDs($groupArr['ID'], 1000, false); - } else if ($processRequestIDs === 2) { - $requestIDTime = time(); - if ($echoCLI) { - $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request ID Threaded lookup.")); - } - passthru("$PYTHON ${DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py"); - if ($echoCLI) { - $this->pdo->log->doEcho( - $this->pdo->log->primary( - "\nReleases updated in " . - $consoleTools->convertTime(time() - $requestIDTime) - ) - ); - } - } - - // - // Delete any releases under the minimum completion percent. - // - if ($page->site->completionpercent != 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 4 : Deleting releases less than ' . $page->site->completionpercent . ' complete')); - $result = $this->pdo->query(sprintf("select ID from releases where completion > 0 and completion < %d", $page->site->completionpercent)); - foreach ($result as $row) - $this->delete($row["ID"]); - } - - // - // Delete releases whos minsize is less than the site or group minimum - // - $result = $this->pdo->query("select releases.ID from releases left outer join (SELECT g.ID, coalesce(g.minsizetoformrelease, s.minsizetoformrelease) as minsizetoformrelease FROM groups g inner join ( select value as minsizetoformrelease from site where setting = 'minsizetoformrelease' ) s ) x on x.ID = releases.groupID where minsizetoformrelease != 0 and releases.size < minsizetoformrelease"); - if (count($result) > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 4 : Deleting ' . count($result) . ' release(s) where size is smaller than minsize for site/group')); - foreach ($result as $row) - $this->delete($row["ID"]); - } - - $result = $this->pdo->query("select releases.ID, name, categoryID, size FROM releases JOIN ( - select - catc.ID, - case when catc.minsizetoformrelease = 0 then catp.minsizetoformrelease else catc.minsizetoformrelease end as minsizetoformrelease, - case when catc.maxsizetoformrelease = 0 then catp.maxsizetoformrelease else catc.maxsizetoformrelease end as maxsizetoformrelease - from category catp join category catc on catc.parentID = catp.ID - where (catc.minsizetoformrelease != 0 or catc.maxsizetoformrelease != 0) or (catp.minsizetoformrelease != 0 or catp.maxsizetoformrelease != 0) - ) x on x.ID = releases.categoryID - where - (size < minsizetoformrelease and minsizetoformrelease != 0) or - (size > maxsizetoformrelease and maxsizetoformrelease != 0)" - ); - - if (count($result) > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 4 : Deleting release(s) not matching category min/max size')); - foreach ($result as $r) { - $this->delete($r['ID']); - } - } - - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 5 : Post processing is done in tmux')); - //$postprocess = new PostProcess(true); - //$postprocess->processAll(); - - // - // aggregate the releasefiles upto the releases. - // - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 6 : Aggregating Files')); - $this->pdo->exec("update releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID and releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num"); - - // Remove the binaries and parts used to form releases, or that are duplicates. - // - if ($page->site->partsdeletechunks > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 7 : Chunk deleting unused binaries and parts')); - $query = sprintf("SELECT parts.ID as partsID,binaries.ID as binariesID FROM parts - LEFT JOIN binaries ON binaries.ID = parts.binaryID - WHERE binaries.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", - $this->pdo->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24), - $page->site->partsdeletechunks - ); - - $cc = 0; - $done = false; - while (!$done) { - $dd = $cc; - $result = $this->pdo->query($query); - if (count($result) > 0) { - $pID = array(); - $bID = array(); - foreach ($result as $row) { - $pID[] = $row['partsID']; - $bID[] = $row['binariesID']; - } - $pID = '(' . implode(',', $pID) . ')'; - $bID = '(' . implode(',', $bID) . ')'; - $fr = $this->pdo->exec("DELETE FROM parts WHERE ID IN {$pID}"); - if ($fr > 0) { - $cc += $fr; - $cc += $this->pdo->exec("DELETE FROM binaries WHERE ID IN {$bID}"); - } - unset($pID); - unset($bID); - if ($cc == $dd) { - $done = true; - } - echo($cc % 10000 ? '.' : ''); - } else { - $done = true; - } - } - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 7 : Complete - ' . $cc . ' rows affected')); - } else { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 7 : Deleting unused binaries and parts')); - $this->pdo->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID - WHERE binaries.dateadded < %s - INTERVAL %d HOUR", $this->pdo->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24) - ) - ); - } - // Misc other. - if ($page->site->miscotherretentionhours > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 7a: Deleting releases from misc->other category')); - $releaseImage = new ReleaseImage(); - $releases = $this->pdo->queryDirect( - sprintf(' - SELECT ID, guid - FROM releases - WHERE categoryID = %d - AND adddate <= NOW() - INTERVAL %d HOUR', - \Category::CAT_MISC_OTHER, - $page->site->miscotherretentionhours - ) - ); - if ($releases instanceof \Traversable) { - foreach ($releases as $release) { - $this->deleteSingle(['g' => $release['guid'], 'i' => $release['ID']], $this->nzb, $releaseImage); - $miscRetentionDeleted++; - } - } - } - - // Misc hashed. - if ($page->site->mischashedretentionhours > 0) { - $this->pdo->log->doEcho($this->pdo->log->primary('Stage 7b: Deleting releases from misc->hashed category')); - $releaseImage = new ReleaseImage(); - $releases = $this->pdo->queryDirect( - sprintf(' - SELECT ID, guid - FROM releases - WHERE categoryID = %d - AND adddate <= NOW() - INTERVAL %d HOUR', - \Category::CAT_MISC_HASHED, - $page->site->mischashedretentionhours - ) - ); - if ($releases instanceof \Traversable) { - foreach ($releases as $release) { - $this->deleteSingle(['g' => $release['guid'], 'i' => $release['ID']], $this->nzb, $releaseImage); - $miscHashedDeleted++; - } - } - } - - $this->pdo->log->doEcho($this->pdo->log->primary( - 'Removed releases: ' . - number_format($miscRetentionDeleted) . - ' from misc->other' . - number_format($miscHashedDeleted) . - ' from misc->hashed' - ) - ); - - // - // User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically. - // - $users = new Users; - $users->pruneRequestHistory($page->site->userdownloadpurgedays); - - $this->pdo->log->doEcho($this->pdo->log->primary('Done : Added ' . $retcount . ' releases')); - - return $retcount; } - + /** + * @param $url + * @param $rev + * @param $nnid + */ public function checkRegexesUptoDate($url, $rev, $nnid) { if ($url != "") { @@ -2089,7 +2016,7 @@ class Releases $queries = array_map("trim", $queries); foreach ($queries as $q) { if ($q) { - $this->pdo->exec($q); + $this->pdo->queryExec($q); } } @@ -2107,6 +2034,99 @@ class Releases } } + /** + * Post-process releases. + * + * @param int $postProcess + * @param NNTP $nntp + * + * @void + * @access public + */ + public function postProcessReleases($postProcess, &$nntp) + { + if ($postProcess == 1) { + (new PProcess(['Echo' => $this->echoCLI, 'Settings' => $this->pdo, 'Groups' => $this->groups]))->processAll($nntp); + } else { + if ($this->echoCLI) { + $this->pdo->log->doEcho( + $this->pdo->log->info( + "\nPost-processing is not running inside the Releases class.\n" . + "If you are using tmux or screen they might have their own scripts running Post-processing." + ) + ); + } + } + } + + /** + * Apply regexes to groups + * + * @param $groupID + */ + public function applyRegex($groupID) + { + // + // Get all regexes for all groups which are to be applied to new binaries + // in order of how they should be applied + // + $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $groups = $this->groups->getByID($groupID); + $activeGroups = $this->groups->getActive(false); + $this->releaseRegex->get(); + $this->pdo->log->doEcho($this->pdo->log->primary('Applying regex to binaries'), true); + foreach($activeGroups as $groupArr) { + //check if regexes have already been applied during update binaries + if ($groupArr['regexmatchonly'] == 1) + continue; + + $groupRegexes = $this->releaseRegex->getForGroup($groupArr['name']); + + $this->pdo->log->doEcho($this->pdo->log->primary('Applying ' . sizeof($groupRegexes) . ' regexes to group ' . $groupArr['name']), true); + + // Get out all binaries of STAGE0 for current group + $newUnmatchedBinaries = array(); + $ressql = sprintf('SELECT ID, name, date, totalParts, procstat, fromname FROM %s b + WHERE groupID = %d AND procstat IN (%d, %d) AND regexID IS NULL ORDER BY b.date ASC', + $group['bname'], + $groupArr['ID'], + Releases::PROCSTAT_NEW, + Releases::PROCSTAT_TITLENOTMATCHED + ); + $resbin = $this->pdo->queryDirect($ressql); + + $matchedbins = 0; + while ($rowbin = $this->pdo->getAssocArray($resbin)) { + $regexMatches = array(); + foreach ($groupRegexes as $groupRegex) { + $regexCheck = $this->releaseRegex->performMatch($groupRegex, $rowbin['name']); + if ($regexCheck !== false) { + $regexMatches = $regexCheck; + break; + } + } + + if (!empty($regexMatches)) { + $matchedbins++; + $relparts = explode("/", $regexMatches['parts']); + $this->pdo->queryExec(sprintf("UPDATE %s SET relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s WHERE ID = %d", + $group['bname'], $this->pdo->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $this->pdo->escapeString($regexMatches['reqID']), $rowbin["ID"] + ) + ); + } else { + if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) + $newUnmatchedBinaries[] = $rowbin['ID']; + } + + } + + //mark as not matched + if (!empty($newUnmatchedBinaries)) + $this->pdo->queryExec(sprintf("UPDATE %s SET procstat=%d WHERE ID IN (%s)", $group['bname'], Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries))); + + } + } + public function getReleaseNameForReqId($url, $nnid, $groupname, $reqid) { if ($reqid == " null " || $reqid == "0" || $reqid == "") @@ -2163,27 +2183,29 @@ class Releases else $parameters['reqID'] = " null "; - $parameters['ID'] = $this->pdo->queryInsert(sprintf("insert into releases (name, searchname, totalpart, groupID, adddate, guid, categoryID, regexID, rageID, postdate, fromname, size, reqID, passwordstatus, completion, haspreview, nfostatus, nzbstatus, + $parameters['ID'] = $this->pdo->queryInsert(sprintf("INSERT INTO releases (name, searchname, totalpart, groupID, adddate, guid, categoryID, regexID, rageID, postdate, fromname, size, reqID, passwordstatus, completion, haspreview, nfostatus, nzbstatus, isrenamed, iscategorized, reqidstatus, prehashID) - values (%s, %s, %d, %d, now(), %s, %d, %s, -1, %s, %s, 0, %s, %d, 100,-1, -1, %d, %d, 1, %d, %d)", - $parameters['name'], - $parameters['searchname'], - $parameters['totalpart'], - $parameters['groupID'], - $parameters['guid'], - $parameters['categoryID'], - $parameters['regexID'], - $parameters['postdate'], - $parameters['fromname'], - $parameters['reqID'], - $parameters['passwordstatus'], - $parameters['nzbstatus'], - $parameters['isrenamed'], - $parameters['reqidstatus'], - $parameters['prehashID'] - )); + VALUES (%s, %s, %d, %d, now(), %s, %d, %s, -1, %s, %s, 0, %s, %d, 100,-1, -1, %d, %d, 1, %d, %d)", + $parameters['name'], + $parameters['searchname'], + $parameters['totalpart'], + $parameters['groupID'], + $parameters['guid'], + $parameters['categoryID'], + $parameters['regexID'], + $parameters['postdate'], + $parameters['fromname'], + $parameters['reqID'], + $parameters['passwordstatus'], + $parameters['nzbstatus'], + $parameters['isrenamed'], + $parameters['reqidstatus'], + $parameters['prehashID'] + ) + ); $this->sphinxSearch->insertRelease($parameters); + return $parameters['ID']; } @@ -2238,7 +2260,7 @@ class Releases $re->delete($rel['ID']); $re->deleteFull($rel['ID']); $ri->delete($rel['guid']); - $this->pdo->exec(sprintf("DELETE from releases where id = %d", $rel['ID'])); + $this->pdo->queryExec(sprintf("DELETE FROM releases WHERE id = %d", $rel['ID'])); } } } @@ -2247,13 +2269,13 @@ class Releases * Deletes a single release by GUID, and all the corresponding files. * * @param array $identifiers ['g' => Release GUID(mandatory), 'id => ReleaseID(optional, pass false)] - * @param NZB $this->nzb + * @param NZB $nzb * @param ReleaseImage $releaseImage */ public function deleteSingle($identifiers, $nzb, $releaseImage) { // Delete NZB from disk. - $nzbPath = $this->nzb->getNZBPath($identifiers['g']); + $nzbPath = $nzb->getNZBPath($identifiers['g']); if ($nzbPath) { @unlink($nzbPath); } @@ -2266,7 +2288,7 @@ class Releases $this->sphinxSearch->deleteRelease($identifiers, $this->pdo); // Delete from DB. - $this->pdo->exec( + $this->pdo->queryExec( sprintf(' DELETE r, rn, rc, uc, rf, ra, rs, rv, re FROM releases r @@ -2289,7 +2311,7 @@ class Releases return $this->pdo->query("SELECT ID, searchname, guid, adddate, grabs FROM releases - where grabs > 0 + WHERE grabs > 0 ORDER BY grabs DESC LIMIT 10" ); @@ -2300,7 +2322,7 @@ class Releases return $this->pdo->query("SELECT ID, guid, searchname, adddate, comments FROM releases - where comments > 0 + WHERE comments > 0 ORDER BY comments DESC LIMIT 10" ); @@ -2310,9 +2332,9 @@ class Releases { - return $this->pdo->query("SELECT concat(cp.title, ' > ', category.title) as title, COUNT(*) AS count + return $this->pdo->query("SELECT concat(cp.title, ' > ', category.title) AS title, COUNT(*) AS count FROM category - left outer join category cp on cp.ID = category.parentID + LEFT OUTER JOIN category cp ON cp.ID = category.parentID INNER JOIN releases ON releases.categoryID = category.ID WHERE releases.adddate > NOW() - INTERVAL 1 WEEK GROUP BY concat(cp.title, ' > ', category.title) @@ -2436,7 +2458,7 @@ class Releases return $this->pdo->query( "SELECT DISTINCT (a.bookinfoID), guid, name, b.title, searchname, size, completion, - postdate, categoryID, comments, grabs, url, c.cover, c.title as booktitle, c.author + postdate, categoryID, comments, grabs, url, c.cover, c.title AS booktitle, c.author FROM releases a, category b, bookinfo c WHERE c.cover > 0 AND (a.categoryID BETWEEN 7000 AND 7999 OR a.categoryID = 3030) @@ -2483,7 +2505,7 @@ class Releases return $this->pdo->queryDirect( "SELECT r.rageID, r.guid, r.name, r.searchname, r.size, r.completion, r.postdate, r.categoryID, r.comments, r.grabs, - tv.ID as tvid, tv.imgdata, tv.releasetitle as tvtitle + tv.ID AS tvid, tv.imgdata, tv.releasetitle AS tvtitle FROM releases r INNER JOIN tvrage tv USING (rageID) WHERE r.categoryID BETWEEN 5000 AND 5999 @@ -2498,9 +2520,9 @@ class Releases /** * Process RequestID's. * - * @param int|string $groupID - * @param int $limit - * @param bool $local + * @param int|string $groupID + * @param int $limit + * @param bool $local * * @access public * @void @@ -2533,15 +2555,17 @@ class Releases if ($local === true) { $foundRequestIDs = ( new \RequestIDLocal( - ['Echo' => $echoCLI, 'ConsoleTools' => $consoleTools, - 'Groups' => $groups, 'Settings' => $this->pdo] + ['Echo' => $echoCLI, 'ConsoleTools' => $consoleTools, + 'Groups' => $groups, 'Settings' => $this->pdo + ] ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } else { $foundRequestIDs = ( new \RequestIDWeb( - ['Echo' => $echoCLI, 'ConsoleTools' => $consoleTools, - 'Groups' => $groups, 'Settings' => $this->pdo] + ['Echo' => $echoCLI, 'ConsoleTools' => $consoleTools, + 'Groups' => $groups, 'Settings' => $this->pdo + ] ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } @@ -2777,7 +2801,7 @@ class Releases // Delete smaller than category minimum sizes. $categories = $this->pdo->queryDirect(' SELECT c.ID AS id, - CASE WHEN c.minsize = 0 THEN cp.minsize ELSE c.minsize END AS minsize + CASE WHEN c.minsizetoformrelease = 0 THEN cp.minsizetoformrelease ELSE c.minsizetoformrelease END AS minsize FROM category c INNER JOIN category cp ON cp.ID = c.parentID WHERE c.parentID IS NOT NULL' @@ -2911,16 +2935,16 @@ class Releases } } - /** - * Categorizes releases. - * - * @param string $type name or searchname | Categorize using the search name or subject. - * @param string $where Optional "where" query parameter. - * - * @return int Quantity of categorized releases. - * @access public - */ - public function categorizeRelease($type, $where = '') + /** + * Categorizes releases. + * + * @param string $type name or searchname | Categorize using the search name or subject. + * @param string $where Optional "where" query parameter. + * + * @return int Quantity of categorized releases. + * @access public + */ + public function categorizeRelease($type, $where = '') { $cat = new \Categorize(['Settings' => $this->pdo]); $categorized = $total = 0; @@ -2943,19 +2967,20 @@ class Releases if ($this->echoCLI !== false && $categorized > 0) { echo PHP_EOL; } + return $categorized; } - /** - * Categorize releases. - * - * @param int $categorize - * @param int|string $groupID (optional) - * - * @void - * @access public - */ - public function categorizeReleases($categorize, $groupID = '') + /** + * Categorize releases. + * + * @param int $categorize + * @param int|string $groupID (optional) + * + * @void + * @access public + */ + public function categorizeReleases($categorize, $groupID = '') { $startTime = time(); if ($this->echoCLI) { @@ -2980,4 +3005,19 @@ class Releases $this->pdo->log->doEcho($this->pdo->log->primary($this->consoleTools->convertTime(time() - $startTime)), true); } } + + /** + * Return all releases to other->misc category. + * + * @param string $where Optional "where" query parameter. + * + * @void + * @access public + */ + public function resetCategorize($where = '') + { + $this->pdo->queryExec( + sprintf('UPDATE releases SET categoryID = %d, iscategorized = 0 %s', \Category::CAT_MISC_OTHER, $where) + ); + } } \ No newline at end of file diff --git a/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl b/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl index e6eefdc8a..d21b96b39 100644 --- a/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl +++ b/lib/copy_this/www/templates/nntmux/views/admin/site-edit.tpl @@ -404,6 +404,15 @@
The directory where nzb files will be stored.
+ + + + +
Levels deep to store the nzb Files. +
If you change this you must run the misc/testing/DB/nzb-reorg.php script! +
+ + : @@ -459,6 +468,15 @@
The minimum total size in bytes to make a release. If set to 0, then ignored.
+ + + + +
The maximum total size in bytes to make a release. If set to 0, then ignored. Only deletes + during release creation. +
+ + : @@ -468,6 +486,21 @@ + + + + {html_radios id="grabstatus" name='grabstatus' values=$yesno_ids output=$yesno_names selected=$site->grabstatus separator='
'} +
Whether to update download counts when someone downloads a release.
+ + + + + + +
The time in hours to check for crossposted releases - this will delete 1 of the releases if the 2 are posted by the same person in the same time period.
+ + + : @@ -733,6 +766,13 @@
Whether to delete releases which are passworded or potentially passworded.
+ + + + {html_radios id="deletepossiblerelease" name='deletepossiblerelease' values=$yesno_ids output=$yesno_names selected=$site->deletepossiblerelease separator='
'} +
Whether to delete releases which are potentially passworded. This applies to your post process additional inner file blacklist.
+ + : @@ -945,6 +985,21 @@
Advanced Settings - For advanced users + + + + diff --git a/lib/testing/DB/convert_to_tpg.php b/lib/testing/DB/convert_to_tpg.php new file mode 100644 index 000000000..250e47b86 --- /dev/null +++ b/lib/testing/DB/convert_to_tpg.php @@ -0,0 +1,169 @@ + $pdo]); +$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); +$s = new Sites(); +$site = $s->get(); +$DoPartRepair = ($site->partrepair == '0') ? false : true; + +if ((!isset($argv[1])) || $argv[1] != 'true') { + exit($pdo->log->error("\nMandatory argument missing\n\n" + . "This script will allow you to move from single collections/binaries/parts tables to TPG without having to run reset_truncate.\n" + . "Please STOP all update scripts before running this script.\n\n" + . "Use the following options to run:\n" + . "php $argv[0] true ...: Convert b/p to tpg leaving current binaries/parts tables in-tact.\n" + . "php $argv[0] true delete ...: Convert b/p to tpg and TRUNCATE current binaries/parts tables.\n" + )); +} + +$blen = $pdo->queryOneRow('SELECT COUNT(*) AS total FROM binaries;'); +$bdone = 0; +$bcount = 1; +$gdone = 1; +$actgroups = $groups->getActive(); +$glen = count($actgroups); +$newtables = $glen * 3; +$begintime = time(); + +echo "Creating new binaries, and parts tables for each active group...\n"; + +foreach ($actgroups as $group) { + if ($groups->createNewTPGTables($group['ID']) === false) { + exit($pdo->log->error("There is a problem creating new parts/files tables for group ${group['name']}.")); + } + $consoletools->overWrite("Tables Created: " . $consoletools->percentString($gdone * 3, $newtables)); + $gdone++; +} +$endtime = time(); +echo "\nTable creation took " . $consoletools->convertTime($endtime - $begintime) . ".\n"; +$starttime = time(); +echo "\nNew tables created, moving data from old tables to new tables.\nThis will take awhile....\n\n"; +while ($bdone < $blen['total']) { + // Only load 1000 binaries per loop to not overload memory. + $binaries = $pdo->queryAssoc('SELECT * FROM binaries LIMIT ' . $bdone . ',1000;'); + + if ($binaries instanceof \Traversable) { + foreach ($binaries as $binary) { + $binary['name'] = $pdo->escapeString($binary['name']); + $binary['fromname'] = $pdo->escapeString($binary['fromname']); + $binary['date'] = $pdo->escapeString($binary['date']); + $binary['binaryhash'] = $pdo->escapeString($binary['binarynhash']); + $binary['dateadded'] = $pdo->escapeString($binary['dateadded']); + $binary['xref'] = $pdo->escapeString($binary['xref']); + $binary['releaseID'] = $pdo->escapeString($binary['releaseID']); + $binary['categoryID'] = $pdo->escapeString($binary['categoryID']); + $binary['totalparts'] = $pdo->escapeString($binary['totalparts']); + $binary['relpart'] = $pdo->escapeString($binary['relpart']); + $binary['reltotalpart'] = $pdo->escapeString($binary['reltotalpart']); + $oldbid = array_shift($binary); + + if ($debug) { + echo "\n\nBinaries insert:\n"; + print_r($binary); + echo sprintf("\nINSERT INTO binaries_%d (name, fromname, date, xref, groupID, dateadded, releaseID, categoryID, totalparts, binaryhash, relpart, reltotalpart) VALUES (%s)\n\n", $binary['groupID'], implode(', ', $binary)); + } + $newbid = array('binaryID' => $pdo->queryInsert(sprintf('INSERT INTO binaries_%d (NAME, fromname, date, xref, groupID, dateadded, releaseID, categoryID, totalparts, binaryhash, relpart, reltotalpart) VALUES (%s);', $binary['groupID'], implode(', ', $binarynew)))); + + //Get parts and split to correct group tables. + $parts = $pdo->queryAssoc('SELECT * FROM parts WHERE binaryID = ' . $oldbid . ';'); + if ($parts instanceof \Traversable) { + $firstpart = true; + $partsnew = ''; + foreach ($parts as $part) { + $oldpid = array_shift($part); + $partnew = array_replace($part, $newbid); + + $partsnew .= '(\'' . implode('\', \'', $partnew) . '\'), '; + } + $partsnew = substr($partsnew, 0, -2); + if ($debug) { + echo "\n\nParts insert:\n"; + echo sprintf("\nINSERT INTO parts_%d (binaryID, messageID, number, partnumber, size) VALUES %s;\n\n", $binary['groupID'], $partsnew); + } + $sql = sprintf('INSERT INTO parts_%d (binaryID, messageID, number, partnumber, size) VALUES %s;', $binary['groupID'], $partsnew); + $pdo->queryExec($sql); + } + + } + $bcount++; + } + $bdone += 1000; +} + +if ($DoPartRepair === true) { + foreach ($actgroups as $group) { + $pcount = 1; + $pdone = 0; + $sql = sprintf('SELECT COUNT(*) AS total FROM partrepair WHERE groupID = %d;', $group['ID']); + $plen = $pdo->queryOneRow($sql); + while ($pdone < $plen['total']) { + // Only load 10000 partrepair records per loop to not overload memory. + $partrepairs = $pdo->queryAssoc(sprintf('SELECT * FROM partrepair WHERE groupID = %d LIMIT %d, 10000;', $group['ID'], $pdone)); + if ($partrepairs instanceof \Traversable) { + foreach ($partrepairs as $partrepair) { + $partrepair['numberID'] = $pdo->escapeString($partrepair['numberID']); + $partrepair['groupID'] = $pdo->escapeString($partrepair['groupID']); + $partrepair['attempts'] = $pdo->escapeString($partrepair['attempts']); + if ($debug) { + echo "\n\nPart Repair insert:\n"; + print_r($partrepair); + echo sprintf("\nINSERT INTO partrepair_%d (numberID, groupID, attempts) VALUES (%s, %s, %s)\n\n", $group['ID'], $partrepair['numberID'], $partrepair['groupID'], $partrepair['attempts']); + } + $pdo->queryExec(sprintf('INSERT INTO partrepair_%d (numberID, groupID, attempts) VALUES (%s, %s, %s);', $group['ID'], $partrepair['numberID'], $partrepair['groupID'], $partrepair['attempts'])); + $consoletools->overWrite('Part Repairs Completed for ' . $group['name'] . ':' . $consoletools->percentString($pcount, $plen['total'])); + $pcount++; + } + } + $pdone += 10000; + } + } +} + +$endtime = time(); +echo "\nTable population took " . $consoletools->convertTimer($endtime - $starttime) . ".\n"; + +//Truncate old tables to save space. +if (isset($argv[2]) && $argv[2] == 'delete') { + echo "Truncating old tables...\n"; + $pdo->queryDirect('TRUNCATE TABLE binaries;'); + $pdo->queryDirect('TRUNCATE TABLE parts'); + $pdo->queryDirect('TRUNCATE TABLE partrepair'); + echo "Complete.\n"; +} +// Update TPG setting in site-edit. +$pdo->queryExec('UPDATE site SET value = 1 WHERE setting = \'tablepergroup\';'); +$pdo->queryExec('UPDATE tmux SET value = 2 WHERE setting = \'releases\';'); +echo "New tables have been created.\nTable Per Group has been set to to \"TRUE\" in site-edit.\nUpdate Releases has been set to Threaded in tmux-edit.\n"; + +function multi_implode($array, $glue) +{ + $ret = ''; + + foreach ($array as $item) { + if (is_array($item)) { + $ret .= '(' . multi_implode($item, $glue) . '), '; + } else { + $ret .= $item . $glue; + } + } + + $ret = substr($ret, 0, 0 - strlen($glue)); + + return $ret; +} diff --git a/lib/testing/DB/convert_to_tpg_alt.php b/lib/testing/DB/convert_to_tpg_alt.php new file mode 100644 index 000000000..7c4f5b82c --- /dev/null +++ b/lib/testing/DB/convert_to_tpg_alt.php @@ -0,0 +1,80 @@ +log->error("\nThis script will move all collections, binaries, parts into tables per group.\n\n" + . "php $argv[0] true ...: To process all parts and leave the parts/binaries/collections tables intact.\n" + . "php $argv[0] true truncate ...: To process all parts and truncate parts/binaries/collections tables after completed.\n")); +} + +$start = time(); +$consoleTools = new \ConsoleTools(['ColorCLI' => $pdo->log]); +$groups = new \Groups(['Settings' => $pdo]); + +$actgroups = $pdo->query("SELECT DISTINCT group_id from collections"); + +echo $pdo->log->info("Creating new collections, binaries, and parts tables for each group that has collections."); + +foreach ($actgroups as $group) { + $pdo->queryExec("DROP TABLE IF EXISTS collections_" . $group['group_id']); + $pdo->queryExec("DROP TABLE IF EXISTS binaries_" . $group['group_id']); + $pdo->queryExec("DROP TABLE IF EXISTS parts_" . $group['group_id']); + if ($groups->createNewTPGTables($group['group_id']) === false) { + exit($pdo->log->error("\nThere is a problem creating new parts/files tables for group ${group['name']}.\n")); + } +} + +$collections_rows = $pdo->queryDirect("SELECT group_id FROM collections GROUP BY group_id"); + +echo $pdo->log->info("Counting parts, this could table a few minutes."); +$parts_count = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM parts"); + +$i = 0; +if ($collections_rows instanceof \Traversable) { + foreach ($collections_rows as $row) { + $groupName = $groups->getByNameByID($row['group_id']); + echo $pdo->log->header("Processing ${groupName}"); + //collection + $pdo->queryExec("INSERT IGNORE INTO collections_" . $row['group_id'] . " (subject, fromname, date, xref, totalfiles, group_id, collectionhash, dateadded, filecheck, filesize, releaseid) " + . "SELECT subject, fromname, date, xref, totalfiles, group_id, collectionhash, dateadded, filecheck, filesize, releaseid FROM collections WHERE group_id = ${row['group_id']}"); + $collections = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM collections where group_id = " . $row['group_id']); + $ncollections = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM collections_" . $row['group_id']); + echo $pdo->log->primary("Group ${groupName}, Collections = ${collections['cnt']} [${ncollections['cnt']}]"); + + //binaries + $pdo->queryExec("INSERT IGNORE INTO binaries_${row['group_id']} (name, filenumber, totalparts, currentparts, binaryhash, partcheck, partsize, collection_id) " + . "SELECT name, filenumber, totalparts, currentparts, binaryhash, partcheck, partsize, n.id FROM binaries b " + . "INNER JOIN collections c ON b.collection_id = c.id " + . "INNER JOIN collections_${row['group_id']} n ON c.collectionhash = n.collectionhash AND c.group_id = ${row['group_id']}"); + $binaries = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM binaries b INNER JOIN collections c ON b.collection_id = c.id where c.group_id = ${row['group_id']}"); + $nbinaries = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM binaries_${row['group_id']}"); + echo $pdo->log->primary("Group ${groupName}, Binaries = ${binaries['cnt']} [${nbinaries['cnt']}]"); + + //parts + $pdo->queryExec("INSERT IGNORE INTO parts_${row['group_id']} (messageid, number, partnumber, size, binaryid, collection_id) " + . "SELECT messageid, number, partnumber, size, n.id, c.id FROM parts p " + . "INNER JOIN binaries b ON p.binaryid = b.id " + . "INNER JOIN binaries_${row['group_id']} n ON b.binaryhash = n.binaryhash " + . "INNER JOIN collections_${row['group_id']} c on c.id = n.collection_id AND c.group_id = ${row['group_id']}"); + $parts = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM parts p INNER JOIN binaries b ON p.binaryid = b.id INNER JOIN collections c ON b.collection_id = c.id WHERE c.group_id = ${row['group_id']}"); + $nparts = $pdo->queryOneRow("SELECT COUNT(*) AS cnt FROM parts_${row['group_id']}"); + echo $pdo->log->primary("Group ${groupName}, Parts = ${parts['cnt']} [${nparts['cnt']}]\n"); + $i++; + } +} + +if (isset($argv[2]) && $argv[2] == 'truncate') { + echo $pdo->log->info("Truncating collections, binaries and parts tables."); + $pdo->queryExec("TRUNCATE TABLE collections"); + $pdo->queryExec("TRUNCATE TABLE binaries"); + $pdo->queryExec("TRUNCATE TABLE parts"); +} + +//set tpg active +$pdo->queryExec("UPDATE settings SET value = 1 WHERE setting = 'tablepergroup'"); + +echo $pdo->log->header("Processed: ${i} groups and " . number_format($parts_count['cnt']) . " parts in " . $consoleTools->convertTimer(TIME() - $start)); diff --git a/lib/testing/DB/reset_truncate.php b/lib/testing/DB/reset_truncate.php new file mode 100644 index 000000000..9bf1a75ab --- /dev/null +++ b/lib/testing/DB/reset_truncate.php @@ -0,0 +1,56 @@ +get(); + +if (isset($argv[1]) && ($argv[1] == "true" || $argv[1] == "drop")) { + $pdo->queryExec("UPDATE groups SET first_record = 0, first_record_postdate = NULL, last_record = 0, last_record_postdate = NULL, last_updated = NULL"); + echo $pdo->log->primary("Reseting all groups completed."); + + $arr = array("parts", "partrepair", "binaries"); + foreach ($arr as &$value) { + $rel = $pdo->queryExec("TRUNCATE TABLE $value"); + if ($rel !== false) { + echo $pdo->log->primary("Truncating ${value} completed."); + } + } + unset($value); + + $tpg = $site->tablepergroup; + $tablepergroup = (!empty($tpg)) ? $tpg : 0; + + if ($tablepergroup == 1) { + $sql = 'SHOW table status'; + + $tables = $pdo->query($sql); + foreach ($tables as $row) { + $tbl = $row['Name']; + if (preg_match('/binaries_\d+/', $tbl) || preg_match('/parts_\d+/', $tbl) || preg_match('/partrepair_\d+/', $tbl) || preg_match('/\d+_binaries/', $tbl) || preg_match('/\d+_parts/', $tbl) || preg_match('/\d+_partrepair_\d+/', $tbl)) { + if ($argv[1] == "drop") { + $rel = $pdo->queryDirect(sprintf('DROP TABLE %s', $tbl)); + if ($rel !== false) { + echo $pdo->log->primary("Dropping ${tbl} completed."); + } + } else { + $rel = $pdo->queryDirect(sprintf('TRUNCATE TABLE %s', $tbl)); + if ($rel !== false) { + echo $pdo->log->primary("Truncating ${tbl} completed."); + } + } + } + } + } + + $delcount = $pdo->queryDirect("DELETE FROM releases WHERE nzbstatus = 0"); + echo $pdo->log->primary($delcount->rowCount() . " releases had no nzb, deleted."); +} else { + exit($pdo->log->error("\nThis script removes releases with no NZBs, resets all groups, truncates or drops(tpg) \n" + . "article tables. All other releases are left alone.\n" + . "php $argv[0] [true, drop] ...: To reset all groups and truncate/drop the tables.\n" + )); +} \ No newline at end of file diff --git a/lib/testing/_run_once/tpg_fixes.php b/lib/testing/_run_once/tpg_fixes.php new file mode 100644 index 000000000..98fa10a54 --- /dev/null +++ b/lib/testing/_run_once/tpg_fixes.php @@ -0,0 +1,84 @@ +. + * @author niel / kevin + * @copyright 2014 nZEDb + */ + +if (!isset($argv[1]) || !in_array($argv[1], ['1'])) { + exit( + 'Options: (enter a number, it\'s not recommended to rerun the same fix)' . PHP_EOL . + '1: 2014-07-28: Add unique key to binaryhash to be able to do multiple updates in 1 statement.' . PHP_EOL + ); +} + +require_once(dirname(__FILE__) . "/../../../bin/config.php"); +require_once(WWW_DIR . "/lib/framework/db.php"); +require_once(WWW_DIR . "/lib/site.php"); + +$pdo = new DB(); +$s = new Sites(); +$site = $s->get(); + +if (!$site->tablepergroup) { + exit("Tables per groups is not enabled, quitting!"); +} + +$groups = $pdo->queryDirect('SELECT ID FROM groups WHERE active = 1 OR backfill = 1'); + +if ($groups === false) { + echo "No active groups. Fix not needed.\n"; +} else { + + $queries = array(); + + switch ($argv[1]) { + case 1: + // Drop this index, as we will recreate it as a unique. + $queries[] = ['t' => 1, 'q' => 'ALTER TABLE binaries_%d DROP INDEX ix_binary_binaryhash']; + // Recreate the index as unique so we can use on duplicate key update, saving select / update query. + $queries[] = ['t' => 1, 'q' => 'ALTER IGNORE TABLE binaries_%d ADD UNIQUE INDEX ix_binary_binaryhash(binaryhash)']; + break; + default: + exit(); + } + + $groupCount = $groups->rowCount(); + if ($groups instanceof \Traversable && count($queries) && $groupCount) { + foreach ($groups as $group) { + echo 'Fixing group ' . $group['ID'] . PHP_EOL; + foreach ($queries as $query) { + switch ($query['t']) { + // Queries needing 1 group ID. + case 1: + $pdo->queryExec(sprintf($query['q'], $group['ID']), true); + break; + // Queries needing 2 group IDs. + case 2: + $pdo->queryExec(sprintf($query['q'], $group['ID'], $group['ID']), true); + break; + // Queries needing 3 group IDs. + case 3: + $pdo->queryExec(sprintf($query['q'], $group['ID'], $group['ID'], $group['ID']), true); + break; + } + } + echo 'Finished fixing group ' . $group['ID'] . ', ' . (--$groupCount) . ' to go!' .PHP_EOL; + } + } + echo 'All done!' . PHP_EOL; +} diff --git a/python/postprocess_threaded.py b/python/postprocess_threaded.py index 4a4036ebd..a0e80c4c5 100644 --- a/python/postprocess_threaded.py +++ b/python/postprocess_threaded.py @@ -82,7 +82,7 @@ start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) > 1 and sys.argv[1] == "additional": - cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'postthreads') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+groupID+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+groupID+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+groupID+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+groupID+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+groupID+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+groupID+") as o") + cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'postthreads') AS a, (SELECT value FROM site WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM site WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM site WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM site WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM site WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+groupID+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+groupID+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+groupID+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+groupID+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+groupID+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+groupID+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -91,7 +91,7 @@ if len(sys.argv) > 1 and sys.argv[1] == "additional": ps5 = format(int(dbgrab[0][13])) ps6 = format(int(dbgrab[0][14])) elif len(sys.argv) > 1 and sys.argv[1] == "nfo": - cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'postthreads') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 AND releasenfoID = 0 "+groupID+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 AND releasenfoID = 0 "+groupID+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 AND releasenfoID = 0 "+groupID+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 AND releasenfoID = 0 "+groupID+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 AND releasenfoID = 0 "+groupID+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 AND releasenfoID = 0 "+groupID+") as o") + cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'nfothreads') AS a, (SELECT value FROM site WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM site WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM site WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM site WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM site WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 "+groupID+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 "+groupID+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 "+groupID+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 "+groupID+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 "+groupID+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 "+groupID+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -100,7 +100,7 @@ elif len(sys.argv) > 1 and sys.argv[1] == "nfo": ps5 = format(int(dbgrab[0][13])) ps6 = format(int(dbgrab[0][14])) elif len(sys.argv) > 1 and (sys.argv[1] == "movie" or sys.argv[1] == "tv"): - cur[0].execute("SELECT(SELECT value FROM tmux WHERE setting = 'postthreadsnon') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i") + cur[0].execute("SELECT(SELECT value FROM site WHERE setting = 'postthreadsnon') AS a, (SELECT value FROM site WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM site WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM site WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM site WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM site WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i") dbgrab = cur[0].fetchall() else: print(bcolors.ERROR + "\nAn argument is required, \npostprocess_threaded.py [additional, nfo, movie, tv]\n" + bcolors.ENDC) @@ -117,8 +117,6 @@ posttorun = int(dbgrab[0][7]) postnon = dbgrab[0][8] maxsize = (int(maxsizeck * 1073741824)) -if sys.argv[1] == "additional" or sys.argv[1] == "nfo": - print(bcolors.HEADER + "Available to process: -6 = {}, -5 = {}, -4 = {}, -3 = {}, -2 = {}, -1 = {}".format(ps6, ps5, ps4, ps3, ps2, ps1) + bcolors.ENDC); if maxsize == 0: maxsize = '' @@ -132,78 +130,23 @@ process_additional = run_threads * ppperrun process_nfo = run_threads * nfoperrun if sys.argv[1] == "additional": - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -1 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, process_additional) + cur[0].execute("SELECT LEFT(r.guid, 1) FROM releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE r.nzbstatus = 1 "+maxsize+" AND (r.haspreview = -1 AND c.disablepreview = 0) AND r.passwordstatus BETWEEN -6 AND -1 GROUP BY LEFT(r.guid, 1) LIMIT 16") datas = cur[0].fetchall() - maxtries = -1 - if len(datas) < process_additional: - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -2 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (process_additional - len(datas))) - datas += cur[0].fetchall() - maxtries = -2 - if len(datas) < process_additional: - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -3 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (process_additional - len(datas))) - datas += cur[0].fetchall() - maxtries = -3 - if len(datas) < process_additional: - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -4 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (process_additional - len(datas))) - datas += cur[0].fetchall() - maxtries = -4 - if len(datas) < process_additional: - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -5 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (process_additional - len(datas))) - datas += cur[0].fetchall() - maxtries = -5 - if len(datas) < process_additional: - run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -6 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (process_additional - len(datas))) - datas += cur[0].fetchall() - maxtries = -6 - elif sys.argv[1] == "nfo": - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -1 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo)) + cur[0].execute("SELECT LEFT(guid, 1) FROM releases WHERE nzbstatus = 1 AND nfostatus BETWEEN -8 AND -1 GROUP BY LEFT(guid, 1) LIMIT 16") datas = cur[0].fetchall() - maxtries = -1 - if len(datas) < process_nfo: - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -2 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas))) - datas += cur[0].fetchall() - maxtries = -2 - if len(datas) < process_nfo: - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -3 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas))) - datas += cur[0].fetchall() - maxtries = -3 - if len(datas) < process_nfo: - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -4 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas))) - datas += cur[0].fetchall() - maxtries = -4 - if len(datas) < process_nfo: - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -5 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas))) - datas += cur[0].fetchall() - maxtries = -5 - if len(datas) < process_nfo: - cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -6 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas))) - datas += cur[0].fetchall() - maxtries = -6 - - elif sys.argv[1] == "movie" and len(sys.argv) == 3 and sys.argv[2] == "clean": - run = "SELECT DISTINCT searchname AS name, ID, categoryID from releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND imdbID IS NULL AND categoryID IN (SELECT ID FROM category WHERE parentID = 2000) ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (run_threads * movieperrun)) - datas = cur[0].fetchall() + cur[0].execute("SELECT LEFT(guid, 1) FROM releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND imdbid IS NULL AND categoryID BETWEEN 2000 AND 2999 GROUP BY LEFT(guid, 1) "+orderBY+" LIMIT 16") + datas = cur[0].fetchall() elif sys.argv[1] == "movie": - run = "SELECT searchname AS name, ID, categoryID from releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND imdbID IS NULL AND categoryID IN (SELECT ID FROM category WHERE parentID = 2000) ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (run_threads * movieperrun)) - datas = cur[0].fetchall() + cur[0].execute("SELECT LEFT(guid, 1) FROM releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND imdbid IS NULL AND categoryID BETWEEN 2000 AND 2999 GROUP BY LEFT(guid, 1) "+orderBY+" LIMIT 16") + datas = cur[0].fetchall() elif sys.argv[1] == "tv" and len(sys.argv) == 3 and sys.argv[2] == "clean": - run = "SELECT searchname, ID from releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND rageID = -1 AND categoryID IN (SELECT ID FROM category WHERE parentID = 5000 ) "+orderBY+" LIMIT %s" - cur[0].execute(run, (run_threads * tvrageperrun)) - datas = cur[0].fetchall() + cur[0].execute("SELECT LEFT(guid, 1) FROM releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND rageid = -1 AND categoryID BETWEEN 5000 AND 5999 GROUP BY LEFT(guid, 1) "+orderBY+" LIMIT 16") + datas = cur[0].fetchall() elif sys.argv[1] == "tv": - run = "SELECT searchname, ID from releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND rageID = -1 AND categoryID IN (SELECT ID FROM category WHERE parentID = 5000 ) "+orderBY+" LIMIT %s" - cur[0].execute(run, (run_threads * tvrageperrun)) - datas = cur[0].fetchall() + cur[0].execute("SELECT LEFT(guid, 1) FROM releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND rageid = -1 AND categoryID BETWEEN 5000 AND 5999 GROUP BY LEFT(guid, 1) "+orderBY+" LIMIT 16") + datas = cur[0].fetchall() #close connection to mysql info.disconnect(cur[0], cur[1]) @@ -225,14 +168,14 @@ class queue_runner(threading.Thread): while True: try: - my_ID = self.my_queue.get(True, 1) + my_id = self.my_queue.get(True, 1) except: if time.time() - time_of_last_run > 3: return else: - if my_ID: + if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/postprocess_threaded.php", ""+my_ID]) + subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) time.sleep(.02) self.my_queue.task_done() @@ -247,12 +190,8 @@ def main(args): global time_of_last_run time_of_last_run = time.time() - if sys.argv[1] == "additional": - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. passwordstatus range {} to -1".format(run_threads, "{:,}".format(len(datas)), sys.argv[1], maxtries) + bcolors.ENDC) - elif sys.argv[1] == "nfo": - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. nfostatus range {} to -1".format(run_threads, "{:,}".format(len(datas)), sys.argv[1], maxtries) + bcolors.ENDC) - else: - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases.".format(run_threads, "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {}.".format(run_threads, "{:,}".format(len(datas))) + bcolors.ENDC) + time.sleep(2) def signal_handler(signal, frame): @@ -267,29 +206,29 @@ def main(args): p.setDaemon(False) p.start() - #now load some arbitrary jobs into the queue + #now load some arbitrary jobs into the queue if sys.argv[1] == "additional": for release in datas: time.sleep(.02) - my_queue.put(u("%s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s") % (release[0], release[1], release[2], release[3], release[4], release[5], release[6], release[7], release[8], release[9])) + my_queue.put(u("%s %s") % (sys.argv[1], release[0])) elif sys.argv[1] == "nfo": for release in datas: time.sleep(.02) - my_queue.put(u("%s =+= %s =+= %s =+= %s") % (release[0], release[1], release[2], release[3])) + my_queue.put(u("%s %s") % (sys.argv[1], release[0])) elif sys.argv[1] == "movie": for release in datas: time.sleep(.02) - my_queue.put(u("%s =+= %s =+= %s") % (release[0], release[1], release[2])) + my_queue.put(u("%s %s") % (sys.argv[1], release[0])) elif sys.argv[1] == "tv": for release in datas: time.sleep(.02) - my_queue.put(u("%s =+= %s") % (release[0], release[1])) + my_queue.put(u("%s %s") % (sys.argv[1], release[0])) my_queue.join() if sys.argv[1] == "nfo": cur = info.connect() - cur[0].execute("SELECT ID from releases WHERE nfostatus <= -6") + cur[0].execute("SELECT id from releases WHERE nfostatus <= -6") final = cur[0].fetchall() if len(datas) > 0: for item in final: diff --git a/python/releases_threaded.py b/python/releases_threaded.py index 51b3e4ef7..ea6e1e0c3 100644 --- a/python/releases_threaded.py +++ b/python/releases_threaded.py @@ -21,11 +21,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nUpdate Releases Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT value FROM tmux WHERE setting = 'releasethreads'") -dbgrab = cur[0].fetchone() -threads = int(dbgrab[0][0]) +cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'tablepergroup') AS a, (SELECT value FROM site WHERE setting = 'releasethreads') AS b") +dbgrab = cur[0].fetchall() +allowed = int(dbgrab[0][0]) +threads = int(dbgrab[0][1]) +if allowed == 0: + print(bcolors.ERROR + "Table per group not enabled" + bcolors.ENDC) + info.disconnect(cur[0], cur[1]) + sys.exit() -cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'binaries'") +cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'binaries_%'") datas = cur[0].fetchall() #close connection to mysql @@ -55,7 +60,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/update_releases_new.php", ""+my_id]) + subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "releases "+my_id]) self.my_queue.task_done() def main(): @@ -83,13 +88,15 @@ def main(): if count >= threads: count = 0 count += 1 - my_queue.put("%s %s" % (release[0].replace('binaries', ''), count)) + my_queue.put("%s %s" % (release[0].replace('binaries_', ''), count)) my_queue.join() + #stage7b + subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/python/update_threaded.py b/python/update_threaded.py new file mode 100644 index 000000000..5002253c5 --- /dev/null +++ b/python/update_threaded.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, time +import threading +try: + import queue +except ImportError: + import Queue as queue +import subprocess +import string +import signal +import datetime + +import lib.info as info +from lib.info import bcolors +conf = info.readConfig() +cur = info.connect() +start_time = time.time() +pathname = os.path.abspath(os.path.dirname(sys.argv[0])) +cur[0].execute("SELECT value FROM site WHERE setting = 'releasethreads'") +threads = cur[0].fetchone() +threads = int(threads[0]) + +print(bcolors.HEADER + "\nUpdate Per Group Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + +cur[0].execute("SELECT value FROM site WHERE setting = 'tablepergroup'") +allowed = cur[0].fetchone() +if int(allowed[0]) == 0: + print(bcolors.ERROR + "Table per group not enabled" + bcolors.ENDC) + info.disconnect(cur[0], cur[1]) + sys.exit() + +cur[0].execute("SELECT ID FROM groups WHERE active = 1 ORDER by cast(last_record as signed) - cast(first_record as signed) DESC") +datas = cur[0].fetchall() + +#close connection to mysql +info.disconnect(cur[0], cur[1]) + +if not datas: + print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC) + sys.exit() + +my_queue = queue.Queue() +time_of_last_run = time.time() + +class queue_runner(threading.Thread): + def __init__(self, my_queue): + threading.Thread.__init__(self) + self.my_queue = my_queue + + def run(self): + global time_of_last_run + + while True: + try: + my_id = self.my_queue.get(True, 1) + except: + if time.time() - time_of_last_run > 3: + return + else: + if my_id: + time_of_last_run = time.time() + subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) + self.my_queue.task_done() + +def main(): + global time_of_last_run + time_of_last_run = time.time() + + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC) + time.sleep(2) + + def signal_handler(signal, frame): + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + if True: + #spawn a pool of place worker threads + for i in range(threads): + p = queue_runner(my_queue) + p.setDaemon(False) + p.start() + + #now load some arbitrary jobs into the queue + count = 0 + for release in datas: + if count >= threads: + count = 0 + count += 1 + my_queue.put("%s %s" % (str(release[0]), count)) + + my_queue.join() + + #stage7b + final = "final" + subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + + print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/start.php b/start.php index a2879e743..58f0603e4 100644 --- a/start.php +++ b/start.php @@ -28,7 +28,7 @@ if (count($nntpkill) === 0) { } // Check database patch version -if ($patch < 95) { +if ($patch < 97) { exit($c->error("\nYour database is not up to date. Please update.\nphp ${DIR}/lib/DB/patchDB.php\n")); } $tmux_session = (isset($tmux->tmux_session)) ? $tmux->tmux_session : 0;
+ {html_radios id="tablepergroup" name='tablepergroup' values=$yesno_ids output=$yesno_names selected=$site->tablepergroup separator='
'} +
This uses separate tables for binaries and parts for each group.
This + requires you to run convert_to_tpg.php or reset_truncate.php.
This requires that you also run + releases_threaded.py. +
Run: show variables like '%open%files%'; results should be higher than 10k, twice that if you + are using TokuDB.; +
You may need to increase table_open_cache, open_files_limit and max_allowed_packet in + my.cnf. Also, you may need to add the following to /etc/security/limits.conf
mysql soft + nofile 24000
mysql hard nofile 32000 +
+
@@ -1350,8 +1405,7 @@ -
The number of threads for update_releases. This actualy makes no difference, newznab - update_releases is not threaded. +
The number of threads for releases update scripts.