From 3da0cfcf5003705acb4b1f8d8b51659d9d8fc0b3 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Tue, 19 Apr 2016 00:39:14 +0200 Subject: [PATCH] Update git, versions, updater for complete update of newznab-tmux --- Changelog | 4 + app/config/bootstrap/connections.php | 4 +- app/config/bootstrap/libraries.php | 4 +- app/extensions/command/Update.php | 90 +++++++++++--- app/extensions/command/Version.php | 104 ++++------------ app/extensions/util/Git.php | 40 +++++-- app/extensions/util/Versions.php | 119 +++++++++++++++++++ cli/update_db.php | 50 -------- composer.json | 3 + misc/update/nix/tmux/monitor.php | 2 +- misc/update/python/backfill_safe_threaded.py | 4 +- misc/update/python/backfill_threaded.py | 4 +- misc/update/python/lib/info.py | 4 +- newznab/Install.php | 2 +- newznab/config/config.example.php | 2 +- newznab/db/DB.php | 2 +- 16 files changed, 268 insertions(+), 170 deletions(-) create mode 100644 app/extensions/util/Versions.php delete mode 100644 cli/update_db.php diff --git a/Changelog b/Changelog index c314f446e..9e0751344 100755 --- a/Changelog +++ b/Changelog @@ -1,4 +1,8 @@ 2016-04-18 DariusIII + * Upd: Update git, versions, updater for complete update of newznab-tmux + NOTE: You need to run ./nntmux update nntmux to completely update nntmux. + Please remove the post-merge git hook, as it will interfere with + new updater. * Upd: Update themes to use new files * Upd: Update composer 2016-04-15 DariusIII diff --git a/app/config/bootstrap/connections.php b/app/config/bootstrap/connections.php index efb6af376..49f4be83a 100644 --- a/app/config/bootstrap/connections.php +++ b/app/config/bootstrap/connections.php @@ -74,12 +74,12 @@ use lithium\data\Connections; // )); $config1 = LITHIUM_APP_PATH . DS . 'config' . DS . 'db-config.php'; -$config2 = NN_ROOT . 'nzedb' . DS . 'config' . DS . 'config.php'; +$config2 = NN_ROOT . 'newznab' . DS . 'config' . DS . 'config.php'; $config = file_exists($config1) ? $config1 : $config2; if (file_exists($config)) { require_once $config; - switch (DB_TYPE) { + switch (DB_SYSTEM) { case 'mysql': $adapter = 'MySql'; break; diff --git a/app/config/bootstrap/libraries.php b/app/config/bootstrap/libraries.php index 0874404bb..ce0a2c810 100644 --- a/app/config/bootstrap/libraries.php +++ b/app/config/bootstrap/libraries.php @@ -128,6 +128,9 @@ Libraries::add('app', array('default' => true)); // Libraries::add(basename($path), array('path' => $path)); // } +require_once NN_ROOT . 'constants.php'; + +require_once LITHIUM_APP_PATH . DS . 'libraries' . DS . 'autoload.php'; /** * Add some plugins: */ @@ -139,5 +142,4 @@ if (is_dir(LITHIUM_LIBRARY_PATH . DS . 'li3_quality')) { Libraries::add('li3_quality'); } -require_once LITHIUM_APP_PATH . DS . 'libraries' . DS . 'autoload.php'; ?> diff --git a/app/extensions/command/Update.php b/app/extensions/command/Update.php index b9ab809bc..181529070 100644 --- a/app/extensions/command/Update.php +++ b/app/extensions/command/Update.php @@ -19,17 +19,32 @@ namespace app\extensions\command; use \Exception; +use \Smarty; +use \app\extensions\command\Version; +use \app\extensions\util\Git; +use \app\extensions\util\Versions; use \lithium\console\command\Help; -use \newznab\utility\Git; +use \newznab\db\DbUpdate; /** * Update various aspects of your indexer. - * - * @package app\extensions\command + + * Actions: + * * all|nntmux Fetches current git repo, composer dependencies, and update latest Db patches. + * * db Update the Db with any patches not yet applied. + * * git Performs git pull. + +* +*@package app\extensions\command */ class Update extends \app\extensions\console\Command { + /** + * @var \app\extensions\util\Git object. + */ + protected $git; + private $gitBranch; private $gitTag; @@ -42,39 +57,66 @@ class Update extends \app\extensions\console\Command public function __construct(array $config = []) { $defaults = [ - 'request' => null, - 'response' => [], - 'classes' => $this->_classes + 'classes' => $this->_classes, + 'git' => null, + 'request' => null, + 'response' => [], ]; parent::__construct($config + $defaults); } + public function all() + { + $this->nntmux(); + } + public function db() { // TODO Add check to determine if the indexer or other scripts are running. Hopefully // also prevent web access. - $this->primary("Checking Schema versions..."); + $this->primary("Checking database version..."); + + $versions = new Versions(['git' => ($this->git instanceof Git) ? $this->git : null]); + + $currentDb = $versions->getSQLPatchFromDB(); + $currentXML = $versions->getSQLPatchFromFile(); + if ($currentDb < $currentXML) { + $db = new DbUpdate(); + $db->processPatches(); + } else { + $this->out("Up to date."); + } + } public function git() { - /* - $git = new Git(); - $this->gitBranch = $git->getBranch(); - $this->gitTag = $git->tagLatest(); - */ - $this->error("Not implemented yet!!"); + $this->initialiseGit(); + if (!in_array($this->git->getBranch(), $this->git->getBranchesMain())) { + $this->error("Not on the stable or dev branch! Refusing to update repository ;-)"); + return; + } + $this->git->run('pull'); } public function nntmux() { try { + $this->git(); $this->composer(); $this->db(); - } catch (Exception $e) { + + $smarty = new Smarty(); + $cleared = $smarty->clearCompiledTemplate(); + if ($cleared) { + $this->primary('The Smarty compiled template cache has been cleaned for you'); + } else { + $this->primary('You should clear your Smarty compiled template cache at: ' . + NN_RES . "smarty" . DS . 'templates_c'); + } + } catch (\Exception $e) { $this->error($e->getMessage()); } - } public function run($command = null) @@ -98,7 +140,19 @@ class Update extends \app\extensions\console\Command protected function composer() { - passthru('composer install --no-dev'); + $this->initialiseGit(); + $command = 'composer install'; + if (in_array($this->gitBranch, $this->git->getBranchesStable())) { + $command .= ' --no-dev'; + } + passthru($command); + } + + protected function initialiseGit() + { + if (!($this->git instanceof Git)) { + $this->git = new Git(); + } } /** @@ -128,5 +182,9 @@ class Update extends \app\extensions\console\Command protected function _init() { parent::_init(); + + if ($this->_config['git'] instanceof Git) { + $this->git =& $this->_config['git']; + } } } diff --git a/app/extensions/command/Version.php b/app/extensions/command/Version.php index ed9db01b8..d97a23504 100644 --- a/app/extensions/command/Version.php +++ b/app/extensions/command/Version.php @@ -18,29 +18,27 @@ */ namespace app\extensions\command; -use \Exception; -use \app\models\Settings; -use \app\extensions\util\Git; +use \app\extensions\util\Versions; use \lithium\console\command\Help; -use newznab\utility\Utility; /** * Returns the current version (or branch) of the indexer. * * Actions: - * * all Show all of following info. - * * git Show git tag for current version. - * * sql Show SQL patch level + * * all Show all of following info. + * * branch Show git branch name. + * * git Show git tag for current version. + * * sql Show SQL patch level * * @package app\extensions\command */ class Version extends \app\extensions\console\Command { /** - * @var object simpleXMLElement + * @var \app\extensions\util\Versions; */ - protected $xml = null; + protected $versions = null; /** * Constructor. @@ -50,9 +48,9 @@ class Version extends \app\extensions\console\Command public function __construct(array $config = []) { $defaults = [ - 'request' => null, - 'response' => [], - 'classes' => $this->_classes + 'classes' => $this->_classes, + 'request' => null, + 'response' => [], ]; parent::__construct($config + $defaults); } @@ -72,28 +70,9 @@ class Version extends \app\extensions\console\Command $this->sql(); } - protected function getGitTagFromFile() + protected function branch() { - $this->_loadVersionsFile(); - return ($this->xml === null) ? null : $this->_vers->git->tag->__toString(); - } - - protected function getGitTagFromRepo() - { - $git = new Git(); - return $git->tagLatest(); - } - - protected function getSQLPatchFromDB() - { - return Settings::find('setting', ['conditions' => '..sqlpatch']); - - } - - protected function getSQLPatchFromFile() - { - $this->_loadVersionsFile(); - return ($this->xml === null) ? null : $this->_vers->sql->file->__toString(); + $this->primary('Git branch: ' . $this->versions->getGitBranch()); } /** @@ -103,12 +82,11 @@ class Version extends \app\extensions\console\Command */ protected function git() { - $current = $this->getGitTagFromFile(); - $latest = $this->getGitTagFromRepo(); - - $this->primary('Looking up Git tag version(s)'); - $this->out("XML version: $current"); - $this->out("Git version: $latest"); + if (!$this->plain) { + $this->primary('Looking up Git tag version(s)'); + } + $this->out("XML version: " . $this->versions->getGitTagFromFile()); + $this->out("Git version: " . $this->versions->getGitTagFromRepo()); } /** @@ -118,16 +96,17 @@ class Version extends \app\extensions\console\Command */ protected function sql() { - $this->request->params['args'] += ['sqlcheck' => 'all']; - $this->primary('Looking up SQL patch version(s)'); + if (!$this->plain) { + $this->primary('Looking up SQL patch version(s)'); + } if (in_array($this->request->params['args']['sqlcheck'], ['xml', 'both', 'all'])) { - $latest = $this->getSQLPatchFromFile(); + $latest = $this->versions->getSQLPatchFromFile(); $this->out("XML version: $latest"); } if (in_array($this->request->params['args']['sqlcheck'], ['db', 'both', 'all'])) { - $dbpatch = self::getSQLPatchFromDB(); + $dbpatch = $this->versions->getSQLPatchFromDB(); if ($dbpatch->count()) { $dbVersion = $dbpatch->data()[0]['value']; @@ -137,41 +116,7 @@ class Version extends \app\extensions\console\Command $this->out(" DB version: " . $dbVersion); } } else { - $this->error("Unable to fetch Databse SQL level "); - } - } - } - - protected function _loadVersionsFile($versions = null) - { - if ($this->xml === null) { - if ($versions == '') { - $versions = NN_VERSIONS; - } - - $temp = libxml_use_internal_errors(true); - $this->xml = simplexml_load_file($versions); - libxml_use_internal_errors($temp); - - if ($this->xml === false) { - if (Utility::isCLI()) { - $this->error("Your versions XML file ($versions) is broken, try updating from git."); - } - throw new \Exception("Failed to open versions XML file '$versions'"); - } - - if ($this->xml->count() > 0) { - $vers = $this->xml->xpath('/nzedb/versions'); - - if ($vers[0]->count() == 0) { - $this->error("Your versions XML file ($versions) does not contain version info, try updating from git."); - throw new \Exception("Failed to find versions node in XML file '$versions'"); - } else { - //$this->primary("Your versions XML file ($versions) looks okay, continuing."); - $this->_vers = &$this->xml->versions; - } - } else { - throw new \RuntimeException("No elements in file!\n"); + $this->error("Unable to fetch Databse SQL level!"); } } } @@ -203,5 +148,8 @@ class Version extends \app\extensions\console\Command protected function _init() { parent::_init(); + + $this->request->params['args'] += ['sqlcheck' => 'all']; // Default to all versions/ + $this->versions = new Versions(); } } diff --git a/app/extensions/util/Git.php b/app/extensions/util/Git.php index f22c36eac..fc9ee800d 100644 --- a/app/extensions/util/Git.php +++ b/app/extensions/util/Git.php @@ -32,14 +32,16 @@ class Git extends \lithium\core\Object public function __construct(array $config = []) { $defaults = [ - 'branches' => ['0.x', 'dev', 'latest-testing', 'dev-test'], + 'branches' => [ + 'stable' => ['0.x', 'latest-testing'], + 'development' => ['dev', 'dev-test'] + ], 'create' => false, 'initialise' => false, 'filepath' => NN_ROOT, ]; - $config += $defaults; - parent::__construct($config); + parent::__construct($config += $defaults); } /** @@ -64,6 +66,28 @@ class Git extends \lithium\core\Object return $this->branch; } + public function getBranchesDevelop() + { + return $this->_config['branches']['development']; + } + + /** + * Fetches the array of branch names that are considered to be core. + * + * @return array + */ + public function getBranchesMain() + { + $main = array_merge($this->getBranchesStable(), $this->getBranchesDevelop()); + + return $main; + } + + public function getBranchesStable() + { + return $this->_config['branches']['stable']; + } + /** * Determine if the supplied object is commited to the repository or not. * @@ -102,16 +126,6 @@ class Git extends \lithium\core\Object return $this->run("log $options"); } - /** - * Fetches the array of branch names that are considered to be core. - * - * @return array - */ - public function mainBranches() - { - return $this->_config->branches; - } - /** * Run a git command in the git repository * Accepts a git command to run diff --git a/app/extensions/util/Versions.php b/app/extensions/util/Versions.php new file mode 100644 index 000000000..074986c88 --- /dev/null +++ b/app/extensions/util/Versions.php @@ -0,0 +1,119 @@ +. + * @author niel + * @copyright 2016 nZEDb + */ +namespace app\extensions\util; + +use \app\models\Settings; + +class Versions extends \lithium\core\Object +{ + /** + * @var \app\extensions\util\Git object. + */ + protected $git; + + /** + * @var \simpleXMLElement object. + */ + protected $xml = null; + + public function __construct(array $config = []) + { + $defaults = [ + 'git' => null, + 'path' => NN_VERSIONS, + ]; + + parent::__construct($config += $defaults); + } + + public function getGitTagFromFile() + { + $this->loadXMLFile(); + return ($this->xml === null) ? null : $this->_vers->git->tag->__toString(); + } + + public function getGitBranch() + { + $this->initialiseGit(); + return $this->git->getBranch(); + } + + public function getGitTagFromRepo() + { + $this->initialiseGit(); + return $this->git->tagLatest(); + } + + public function getSQLPatchFromDB() + { + return Settings::find('setting', ['conditions' => '..sqlpatch']); + } + + public function getSQLPatchFromFile() + { + $this->loadXMLFile(); + return ($this->xml === null) ? null : $this->_vers->sql->file->__toString(); + } + + protected function initialiseGit() + { + if (!($this->git instanceof Git)) { + $this->git = new Git(); + } + } + + protected function loadXMLFile() + { + if ($this->xml === null) { + $versions = $this->_config['path']; + + $temp = libxml_use_internal_errors(true); + $this->xml = simplexml_load_file($versions); + libxml_use_internal_errors($temp); + + if ($this->xml === false) { + $this->error("Your versions XML file ($versions) is broken, try updating from git."); + throw new \Exception("Failed to open versions XML file '$versions'"); + } + + if ($this->xml->count() > 0) { + $vers = $this->xml->xpath('/newznab/versions'); + + if ($vers[0]->count() == 0) { + $this->error("Your versions XML file ($versions) does not contain version info, try updating from git."); + throw new \Exception("Failed to find versions node in XML file '$versions'"); + } else { + //$this->primary("Your versions XML file ($versions) looks okay, continuing."); + $this->_vers = &$this->xml->versions; + } + } else { + throw new \RuntimeException("No elements in file!\n"); + } + } + } + + protected function _init() + { + parent::_init(); + + if ($this->_config['git'] instanceof Git) { + $this->git =& $this->_config['git']; + } + } +} diff --git a/cli/update_db.php b/cli/update_db.php deleted file mode 100644 index 47d2c3db5..000000000 --- a/cli/update_db.php +++ /dev/null @@ -1,50 +0,0 @@ -. - * @author niel - * @copyright 2014 nZEDb - */ -require_once realpath(dirname(__DIR__) . DIRECTORY_SEPARATOR . 'indexer.php'); - -use newznab\db\DbUpdate; -use newznab\utility\Utility; - -if (!Utility::isCLI()) { - exit; -} - -if (isset($argc) && $argc > 1 && isset($argv[1]) && $argv[1] == true) { - $backup = (isset($argv[2]) && $argv[2] == 'safe') ? true : false; - $updater = new DbUpdate(['backup' => $backup]); - echo $updater->log->header("Db updater starting ..."); - $patched = $updater->processPatches(['safe' => $backup]); - - if ($patched > 0) { - echo $updater->log->info("$patched patch(es) applied."); - - $smarty = new Smarty(); - $cleared = $smarty->clearCompiledTemplate(); - if ($cleared) { - $msg = "The smarty template cache has been cleaned for you\n"; - } else { - $msg = "You should clear your smarty template cache at: " . SMARTY_DIR . "templates_c\n"; - } - $updater->log->info($msg); - } -} else { - echo "Usage: php update_db.php true"; -} \ No newline at end of file diff --git a/composer.json b/composer.json index 262158b3b..6800f0fa7 100755 --- a/composer.json +++ b/composer.json @@ -47,6 +47,9 @@ }, "license": "GPL-3.0", "name": "dariusiii/newznab-tmux", + "non-feature-branches": [ + "latest-.*" + ], "require": { "php": ">=5.6.0", "barracudanetworks/forkdaemon-php": "~1.0", diff --git a/misc/update/nix/tmux/monitor.php b/misc/update/nix/tmux/monitor.php index e7d12edff..fe75b990d 100644 --- a/misc/update/nix/tmux/monitor.php +++ b/misc/update/nix/tmux/monitor.php @@ -17,7 +17,7 @@ $runVar['paths']['lib'] = NN_LIB; $runVar['paths']['cli'] = NN_ROOT . 'cli/'; $runVar['paths']['scraper'] = NN_MISC . 'IRCScraper' . DS . 'scrape.php'; $db_name = DB_NAME; -$dbtype = DB_TYPE; +$dbtype = DB_SYSTEM; $tmux = $tRun->get('niceness'); $tmux_niceness = (isset($tmux->niceness) ? $tmux->niceness : 2); diff --git a/misc/update/python/backfill_safe_threaded.py b/misc/update/python/backfill_safe_threaded.py index 9c20e1752..40bd82732 100644 --- a/misc/update/python/backfill_safe_threaded.py +++ b/misc/update/python/backfill_safe_threaded.py @@ -78,9 +78,9 @@ while count < 10000: #query to grab backfill groups if len(sys.argv) == 1: - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) datas = cur[0].fetchone() else: diff --git a/misc/update/python/backfill_threaded.py b/misc/update/python/backfill_threaded.py index 1d030845c..b57b143da 100644 --- a/misc/update/python/backfill_threaded.py +++ b/misc/update/python/backfill_threaded.py @@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all": # removing the % before the variables at the end of the query adds quotes/escapes strings cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group)) else: - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) datas = cur[0].fetchall() diff --git a/misc/update/python/lib/info.py b/misc/update/python/lib/info.py index 2c534a933..21b93eccb 100644 --- a/misc/update/python/lib/info.py +++ b/misc/update/python/lib/info.py @@ -18,7 +18,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) def connect(): conf = readConfig() con = None - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": try: import cymysql as mdb if conf['DB_PORT'] != '': @@ -28,7 +28,7 @@ def connect(): except ImportError: print(bcolors.ERROR + "\nPlease install cymysql for python 3, \ninformation can be found in threaded_scripts_readme.txt\n" + bcolors.ENDC) sys.exit() - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": try: import psycopg2 as mdb con = mdb.connect(host=conf['DB_HOST'], user=conf['DB_USER'], password=conf['DB_PASSWORD'], dbname=conf['DB_NAME'], port=int(conf['DB_PORT'])) diff --git a/newznab/Install.php b/newznab/Install.php index 5c7615ea3..be5a13bdd 100644 --- a/newznab/Install.php +++ b/newznab/Install.php @@ -6,7 +6,7 @@ namespace newznab; */ class Install { - public $DB_TYPE; + public $DB_SYSTEM; public $DB_HOST = "localhost"; public $DB_USER; public $DB_PORT = 3306; diff --git a/newznab/config/config.example.php b/newznab/config/config.example.php index 2e5001ece..e86088d2c 100644 --- a/newznab/config/config.example.php +++ b/newznab/config/config.example.php @@ -3,7 +3,7 @@ //========================= // Config you must change - updated by installer. //========================= -define('DB_TYPE', 'mysql'); +define('DB_SYSTEM', 'mysql'); define('DB_HOST', 'localhost'); define('DB_PORT', '3306'); define('DB_SOCKET', ''); diff --git a/newznab/db/DB.php b/newznab/db/DB.php index 1c70f658f..5b7cbff76 100644 --- a/newznab/db/DB.php +++ b/newznab/db/DB.php @@ -106,7 +106,7 @@ class DB extends \PDO 'dbpass' => defined('DB_PASSWORD') ? DB_PASSWORD : '', 'dbport' => defined('DB_PORT') ? DB_PORT : '', 'dbsock' => defined('DB_SOCKET') ? DB_SOCKET : '', - 'dbtype' => defined('DB_TYPE') ? DB_TYPE : '', + 'dbtype' => defined('DB_SYSTEM') ? DB_SYSTEM : '', 'dbuser' => defined('DB_USER') ? DB_USER : '', 'log' => new ColorCLI(), 'persist' => false,