diff --git a/misc/testing/DB/reset_truncate.php b/misc/testing/DB/reset_truncate.php index eddff5b69..1a74cdb21 100644 --- a/misc/testing/DB/reset_truncate.php +++ b/misc/testing/DB/reset_truncate.php @@ -4,8 +4,6 @@ require_once dirname(__FILE__) . '/../../../www/config.php'; use newznab\db\Settings; $pdo = new Settings(); -$s = new Sites(); -$site = $s->get(); if (isset($argv[1]) && ($argv[1] == "true" || $argv[1] == "drop")) { $pdo->queryExec("UPDATE groups SET first_record = 0, first_record_postdate = NULL, last_record = 0, last_record_postdate = NULL, last_updated = NULL"); @@ -20,7 +18,7 @@ if (isset($argv[1]) && ($argv[1] == "true" || $argv[1] == "drop")) { } unset($value); - $tpg = $site->tablepergroup; + $tpg = $pdo->getSetting('tablepergroup'); $tablepergroup = (!empty($tpg)) ? $tpg : 0; if ($tablepergroup == 1) { diff --git a/misc/testing/DB/resetdb.php b/misc/testing/DB/resetdb.php index 54ba0ce30..550770f4e 100644 --- a/misc/testing/DB/resetdb.php +++ b/misc/testing/DB/resetdb.php @@ -57,12 +57,10 @@ foreach ($tables as $row) { (new \SphinxSearch())->truncateRTIndex('releases_rt'); $pdo->optimise(false, 'full'); -$s = new Sites(); -$site = $s->get(); echo $pdo->log->header("Deleting nzbfiles subfolders."); try { - $files = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($site->nzbpath, \RecursiveDirectoryIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST); + $files = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($pdo->getSetting('nzbpath'), \RecursiveDirectoryIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST); foreach ($files as $file) { if (basename($file) != '.gitignore' && basename($file) != 'tmpunrar') { $todo = ($file->isDir() ? 'rmdir' : 'unlink'); diff --git a/misc/testing/_run_once/tpg_add_collections_column_2015-02-20.php b/misc/testing/_run_once/tpg_add_collections_column_2015-02-20.php index 036b4087f..1739d76b7 100644 --- a/misc/testing/_run_once/tpg_add_collections_column_2015-02-20.php +++ b/misc/testing/_run_once/tpg_add_collections_column_2015-02-20.php @@ -24,10 +24,8 @@ use newznab\db\Settings; $pdo = new Settings(); -$s = new Sites(); -$site = $s->get(); -if (!$site->tablepergroup) { +if (!$pdo->getSetting('tablepergroup')) { exit("Tables per groups is not enabled, quitting!"); } diff --git a/misc/testing/_run_once/tpg_add_collections_column_2015-08-04.php b/misc/testing/_run_once/tpg_add_collections_column_2015-08-04.php new file mode 100644 index 000000000..5321fe83c --- /dev/null +++ b/misc/testing/_run_once/tpg_add_collections_column_2015-08-04.php @@ -0,0 +1,44 @@ +. + * @author niel + * @copyright 2014 nZEDb + */ +require_once dirname(__FILE__) . '/../../../www/config.php'; +use newznab\db\Settings; +$pdo = new Settings(); +if (!$pdo->getSetting('tablepergroup')) { + exit("Tables per groups is not enabled, quitting!"); +} +// Doing it this way in case there are tables existing not related to the active/backfill list (i.e. I don't have a clue when these tables get deleted so I'm doing any that are there). +$tables1 = $pdo->queryDirect("SELECT SUBSTR(TABLE_NAME, 12) AS suffix FROM information_schema.TABLES WHERE TABLE_SCHEMA = (SELECT DATABASE()) AND TABLE_NAME LIKE 'collections%' ORDER BY TABLE_NAME"); +$tables2 = $pdo->queryDirect("SELECT SUBSTR(TABLE_NAME, 9) AS suffix FROM information_schema.TABLES WHERE TABLE_SCHEMA = (SELECT DATABASE()) AND TABLE_NAME LIKE 'binaries%' ORDER BY TABLE_NAME"); +$query1 = "ALTER TABLE collections%s ADD COLUMN date_initial TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP AFTER fromname"; +$query2 = "ALTER TABLE collections%s ADD INDEX ix_collections_date_initial (date_initial)"; +$query3 = "ALTER TABLE binaries%s MODIFY COLUMN partcheck TINYINT(1) NOT NULL DEFAULT '0'"; +if ($tables1 instanceof \Traversable && $tables2 instanceof \Traversable) { + foreach ($tables1 as $table1) { + echo "Updating table collections{$table1['suffix']}" . PHP_EOL; + $pdo->queryExec(sprintf($query1, $table1['suffix']), true); + $pdo->queryExec(sprintf($query2, $table1['suffix']), true); + } + foreach ($tables2 as $table2) { + echo "Updating table binaries{$table2['suffix']}" . PHP_EOL; + $pdo->queryExec(sprintf($query3, $table2['suffix']), true); + } + echo 'All done!' . PHP_EOL; +} \ No newline at end of file diff --git a/misc/testing/_run_once/tpg_fixes.php b/misc/testing/_run_once/tpg_fixes.php index 377168f8e..64c21fe93 100644 --- a/misc/testing/_run_once/tpg_fixes.php +++ b/misc/testing/_run_once/tpg_fixes.php @@ -31,10 +31,8 @@ require_once dirname(__FILE__) . '/../../../www/config.php'; use newznab\db\Settings; $pdo = new Settings(); -$s = new Sites(); -$site = $s->get(); -if (!$site->tablepergroup) { +if (!$pdo->getSetting('tablepergroup')) { exit("Tables per groups is not enabled, quitting!"); } diff --git a/misc/testing/_run_once/tpg_update_triggers_2014-09-25.php b/misc/testing/_run_once/tpg_update_triggers_2014-09-25.php new file mode 100644 index 000000000..e0420f4ae --- /dev/null +++ b/misc/testing/_run_once/tpg_update_triggers_2014-09-25.php @@ -0,0 +1,53 @@ +. + * @author niel + * @copyright 2014 nZEDb + */ +require_once dirname(__FILE__) . '/../../../www/config.php'; + +use newznab\db\Settings; + +$pdo = new Settings(); + +if (!$pdo->getSetting('tablepergroup')) { + exit("Tables per groups is not enabled, quitting!"); +} + +// Doing it this way in case there are tables existing not related to the active/backfill list (i.e. I don't have a clue when these tables get deleted so I'm doing any that are there). +$tables = $pdo->queryDirect("SELECT SUBSTR(TABLE_NAME, 9) AS suffix FROM information_schema.TABLES WHERE TABLE_SCHEMA = (SELECT DATABASE()) AND TABLE_NAME LIKE 'binaries%' ORDER BY TABLE_NAME"); + +$query1 = "ALTER TABLE binaries%s DROP INDEX ix_binary_collection"; +$query2 = "DROP TRIGGER IF EXISTS delete_collections%s"; +$query3 = "CREATE TRIGGER delete_collections%s BEFORE DELETE ON collections%s FOR EACH ROW BEGIN DELETE FROM binaries%s WHERE collection_id = OLD.id; DELETE FROM parts%s WHERE collection_id = OLD.id; END"; +$query4 = "ALTER TABLE binaries%s ADD INDEX ix_parts_collection_id(collection_id)"; + +if ($tables instanceof \Traversable) { + foreach ($tables as $table) { + echo "Updating table binaries{$table['suffix']}" . PHP_EOL; + $pdo->queryExec(sprintf($query1, $table['suffix']), true); + $pdo->queryExec(sprintf($query2, $table['suffix']), true); + $pdo->queryExec(sprintf($query3, + $table['suffix'], + $table['suffix'], + $table['suffix'], + $table['suffix']), + true); + $pdo->queryExec(sprintf($query4, $table['suffix']), true); + } + echo 'All done!' . PHP_EOL; +} \ No newline at end of file diff --git a/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php b/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php index c3812ff45..30b8b2acc 100644 --- a/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php +++ b/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php @@ -62,8 +62,6 @@ switch ($options[1]) { $pdo = new Settings(); $nntp = nntp($pdo); $groups = new \Groups(); - $s = new Sites(); - $site = $s->get(); $groupMySQL = $groups->getByName($options[3]); if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) { if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) { @@ -71,7 +69,7 @@ switch ($options[1]) { } } $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]); - $return = $binaries->scan($groupMySQL, $options[4], $options[5], ($site->safepartrepair == 1 ? 'update' : 'backfill')); + $return = $binaries->scan($groupMySQL, $options[4], $options[5], ($pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill')); if (empty($return)) { exit(); } @@ -144,8 +142,6 @@ switch ($options[1]) { // $options[2] => (string)groupCount, number of groups terminated by _ | (int)groupid, group to work on case 'releases': $pdo = new Settings(); - $s = new Sites(); - $site = $s->get(); $releases = new ProcessReleases(['Settings' => $pdo]); //Runs function that are per group @@ -155,7 +151,7 @@ switch ($options[1]) { collectionCheck($pdo, $options[2]); } - processReleases($site, $releases, $options[2]); + processReleases($pdo, $releases, $options[2]); } else { @@ -273,15 +269,13 @@ switch ($options[1]) { /** * Create / process releases for a groupID. * - * @param Sites $site + * @param Settings $pdo * @param ProcessReleases $releases * @param int $groupID */ -function processReleases($site, $releases, $groupID) +function processReleases($pdo, $releases, $groupID) { - $s = new Sites(); - $site = $s->get(); - $releaseCreationLimit = ($site->maxnzbsprocessed != '' ? (int)$site->maxnzbsprocessed : 1000); + $releaseCreationLimit = ($pdo->getSetting('maxnzbsprocessed') != '' ? (int)$pdo->getSetting('maxnzbsprocessed') : 1000); $releases->processIncompleteCollections($groupID); $releases->processCollectionSizes($groupID); $releases->deleteUnwantedCollections($groupID); @@ -334,9 +328,7 @@ function collectionCheck(&$pdo, $groupID) function &nntp(&$pdo, $alternate = false) { $nntp = new \NNTP(['Settings' => $pdo]); - $s = new Sites(); - $site = $s->get(); - if (($alternate && $site->alternate_nntp == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { + if (($alternate && $pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { exit("ERROR: Unable to connect to usenet." . PHP_EOL); } diff --git a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php index f528de858..8ef91fa40 100644 --- a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php +++ b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php @@ -5,8 +5,6 @@ use newznab\db\Settings; use newznab\processing\PProcess; $pdo = new Settings(); -$s = new Sites(); -$site = $s->get(); if (!isset($argv[1])) { exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py.")); @@ -130,7 +128,7 @@ if (!isset($argv[1])) { if ($releases instanceof Traversable) { $nntp = new NNTP(['Settings' => $pdo]); - if (($site->alternate_nntp == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { + if (($pdo->getSetting('alternate_nntp') == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { exit($pdo->log->error("Unable to connect to usenet.")); } diff --git a/misc/update_scripts/nix_scripts/tmux/bin/postprocess.php b/misc/update_scripts/nix_scripts/tmux/bin/postprocess.php index b4838e23b..16cef7bca 100644 --- a/misc/update_scripts/nix_scripts/tmux/bin/postprocess.php +++ b/misc/update_scripts/nix_scripts/tmux/bin/postprocess.php @@ -6,14 +6,12 @@ use newznab\db\Settings; use newznab\processing\PProcess; $pdo = new Settings(); -$s = new Sites(); -$site = $s->get(); /** Array with possible arguments for run and whether or not those methods of operation require NNTP **/ -$args = array( +$args = [ 'additional' => true, 'all' => true, 'allinf' => true, @@ -31,7 +29,7 @@ $args = array( 'tv' => false, 'tvdb' => false, 'xxx' => false, -); +]; $bool = array( 'true', @@ -67,7 +65,7 @@ if (!isset($argv[1]) || !in_array($argv[1], $args) || !isset($argv[2]) || !in_ar $nntp = null; if ($args[$argv[1]] === true) { $nntp = new NNTP(['Settings' => $pdo]); - if (($site->alternate_nntp == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { + if (($pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { exit($pdo->log->error("Unable to connect to usenet." . PHP_EOL)); } } diff --git a/misc/update_scripts/nix_scripts/tmux/lib/fixReleaseNames.php b/misc/update_scripts/nix_scripts/tmux/lib/fixReleaseNames.php index 08e444a29..b50bd3f83 100644 --- a/misc/update_scripts/nix_scripts/tmux/lib/fixReleaseNames.php +++ b/misc/update_scripts/nix_scripts/tmux/lib/fixReleaseNames.php @@ -16,8 +16,6 @@ $n = "\n"; $pdo = new Settings(); $namefixer = new \NameFixer(['Settings' => $pdo]); $predb = new \PreHash(['Echo' => true, 'Settings' => $pdo]); -$s = new Sites(); -$site = $s->get(); if (isset($argv[1]) && isset($argv[2]) && isset($argv[3]) && isset($argv[4])) { $update = ($argv[2] == "true") ? 1 : 2; @@ -37,7 +35,7 @@ if (isset($argv[1]) && isset($argv[2]) && isset($argv[3]) && isset($argv[4])) { $nntp = null; if ($argv[1] == 7 || $argv[1] == 8) { $nntp = new \NNTP(['Settings' => $pdo]); - if (($site->alternate_nntp == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { + if (($pdo->getSetting('alternate_nntp') == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { echo $pdo->log->error("Unable to connect to usenet.\n"); return; } diff --git a/newznab/build/newznab.xml b/newznab/build/newznab.xml index 7cfb4b21b..38f1a125b 100644 --- a/newznab/build/newznab.xml +++ b/newznab/build/newznab.xml @@ -2,8 +2,8 @@ - 164 - 164 + 166 + 166 0.4.1 diff --git a/newznab/controllers/Backfill.php b/newznab/controllers/Backfill.php index 445b2afd6..fbfcb6f3e 100644 --- a/newznab/controllers/Backfill.php +++ b/newznab/controllers/Backfill.php @@ -223,6 +223,10 @@ class Backfill { // Start time for this group. $startGroup = microtime(true); + + // Log the date we last downloaded headers. + $this->pdo->queryExec("UPDATE settings SET value = NOW() WHERE setting = 'last_run_time'"); + $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']); // If our local oldest article 0, it means we never ran update_binaries on the group. diff --git a/newznab/controllers/Binaries.php b/newznab/controllers/Binaries.php index 31add2da4..70887cf49 100644 --- a/newznab/controllers/Binaries.php +++ b/newznab/controllers/Binaries.php @@ -254,6 +254,9 @@ class Binaries { $startGroup = microtime(true); + // Log the date we last downloaded headers. + $this->_pdo->queryExec("UPDATE settings SET value = NOW() WHERE setting = 'last_run_time'"); + // Select the group on the NNTP server, gets the latest info on it. $groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']); if ($this->_nntp->isError($groupNNTP)) { diff --git a/newznab/processing/ProcessReleases.php b/newznab/processing/ProcessReleases.php index 168464be6..f9c6eb490 100644 --- a/newznab/processing/ProcessReleases.php +++ b/newznab/processing/ProcessReleases.php @@ -87,6 +87,11 @@ class ProcessReleases */ public $releaseImage; + /** + * @var int Time (hours) to wait before creating a stuck collection into a release. + */ + private $collectionTimeout; + /** * @param array $options Class instances / Echo to cli ? */ @@ -113,8 +118,6 @@ class ProcessReleases $this->releaseCleaning = ($options['ReleaseCleaning'] instanceof \ReleaseCleaning ? $options['ReleaseCleaning'] : new \ReleaseCleaning($this->pdo)); $this->releases = ($options['Releases'] instanceof \Releases ? $options['Releases'] : new \Releases(['Settings' => $this->pdo, 'Groups' => $this->groups])); $this->releaseImage = ($options['ReleaseImage'] instanceof \ReleaseImage ? $options['ReleaseImage'] : new \ReleaseImage($this->pdo)); - $s = new \Sites(); - $this->site = $s ->get(); $this->tablePerGroup = ($this->pdo->getSetting('tablepergroup') == 0 ? false : true); $this->collectionDelayTime = ($this->pdo->getSetting('delaytime') != '' ? (int)$this->pdo->getSetting('delaytime') : 2); @@ -126,6 +129,7 @@ class ProcessReleases $this->completion = 100; echo $this->pdo->log->error(PHP_EOL . 'You have an invalid setting for completion. It must be lower than 100.'); } + $this->collectionTimeout = intval($this->pdo->getSetting('collection_timeout')); } /** @@ -296,6 +300,7 @@ class ProcessReleases $where = (!empty($groupID) ? ' AND c.group_id = ' . $groupID . ' ' : ' '); + $this->processStuckCollections($group, $where); $this->collectionFileCheckStage1($group, $where); $this->collectionFileCheckStage2($group, $where); $this->collectionFileCheckStage3($group, $where); @@ -1596,4 +1601,31 @@ class ProcessReleases ) ); } + + /** + * If a collection has been stuck for $this->collectionTimeout hours, force it to become a release. + * + * @param array $group + * @param string $where + * + * @void + * @access private + */ + private function processStuckCollections(array $group, $where) + { + $this->pdo->queryExec( + sprintf(" + UPDATE %s c + SET c.filecheck = %d + WHERE + c.date_initial < + DATE_SUB((SELECT value FROM settings WHERE setting = 'last_run_time'), INTERVAL %d HOUR) + %s", + $group['cname'], + self::COLLFC_COMPCOLL, + $this->collectionTimeout, + $where + ) + ); + } } \ No newline at end of file diff --git a/resources/db/patches/0165~settings.sql b/resources/db/patches/0165~settings.sql new file mode 100644 index 000000000..db23e2d22 --- /dev/null +++ b/resources/db/patches/0165~settings.sql @@ -0,0 +1,16 @@ +INSERT IGNORE INTO settings (section, subsection, name, value, hint, setting) +VALUES ( + 'indexer', + 'processing', + 'collection_timeout', + '48', + 'How many hours to wait before converting a collection into a release that is considered "stuck".', + 'collection_timeout' +), ( + 'indexer', + 'processing', + 'last_run_time', + '3015-08-04 15:58:23', + 'Last date the indexer (update_binaries or backfill) was run.', + 'last_run_time' +); \ No newline at end of file diff --git a/resources/db/patches/0166~collections.sql b/resources/db/patches/0166~collections.sql new file mode 100644 index 000000000..3107f5847 --- /dev/null +++ b/resources/db/patches/0166~collections.sql @@ -0,0 +1,2 @@ +ALTER TABLE collections ADD column date_initial DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP; +ALTER TABLE binaries MODIFY partcheck TINYINT(1) UNSIGNED NOT NULL DEFAULT 0; diff --git a/www/pages/dlbrowse.php b/www/pages/dlbrowse.php index c91e04e42..e3312b4b1 100644 --- a/www/pages/dlbrowse.php +++ b/www/pages/dlbrowse.php @@ -7,22 +7,14 @@ if (!$page->users->isLoggedIn()) $page->show403(); $r = new Releases(); -$s = new Sites(); -$site = $s->get(); -$db = new Settings(); +$pdo = new Settings(); -if ($site->sabvdir == "" || $site->sabcompletedir == "" || !file_exists($site->sabcompletedir)) +if ($pdo->getSetting('sabvdir') == "" || $pdo->getSetting('sabcompletedir') == "" || !file_exists($pdo->getSetting('sabcompletedir'))) $page->show404(); -if (!Utility::endsWith($site->sabcompletedir, "/")) - $site->sabcompletedir.="/"; -if (!Utility::endsWith($site->sabvdir, "/")) - $site->sabvdir.="/"; - - -$basepath = $site->sabcompletedir; -$webpath = $site->sabvdir; +$basepath = $pdo->getSetting('sabcompletedir'); +$webpath = $pdo->getSetting('sabvdir'); $subpath = ""; if (isset($_REQUEST["sp"])) @@ -66,7 +58,7 @@ foreach($files as $f) if (!$listmode) { $relres = $r->getByNames($dirs); - while ($rel = $db->getAssocArray($relres)) + while ($rel = $pdo->getAssocArray($relres)) { if (isset($items[$rel["searchname"]])) $items[$rel["searchname"]]["release"] = $rel; diff --git a/www/templates_shared/views/admin/site-edit.tpl b/www/templates_shared/views/admin/site-edit.tpl index 4d5f6c7d7..79ba43212 100644 --- a/www/templates_shared/views/admin/site-edit.tpl +++ b/www/templates_shared/views/admin/site-edit.tpl @@ -1498,6 +1498,14 @@ + + + + +
How many hours to wait before converting a collection into a release that is considered "stuck".
Default value is 48 hours. +
+ +