diff --git a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php index ea29720d7..61bb97763 100644 --- a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php +++ b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php @@ -7,7 +7,7 @@ use newznab\processing\PostProcess; $pdo = new Settings(); if (!isset($argv[1])) { - exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py.")); + exit($pdo->log->error("This script is not intended to be run manually, it is called from Forking.php.")); } else if (isset($argv[1])) { $namefixer = new \NameFixer(['Settings' => $pdo]); $pieces = explode(' ', $argv[1]); diff --git a/misc/update_scripts/nix_scripts/tmux/monitor.php b/misc/update_scripts/nix_scripts/tmux/monitor.php index 409969be4..1b7186b38 100644 --- a/misc/update_scripts/nix_scripts/tmux/monitor.php +++ b/misc/update_scripts/nix_scripts/tmux/monitor.php @@ -107,7 +107,7 @@ while ($runVar['counts']['iterations'] > 0) { $runVar['scripts']['backfill'] = "{$runVar['commands']['_php']} {$runVar['paths']['misc']}update_scripts/nix_scripts/multiprocessing/backfill.php"; break; case 2: - $runVar['scripts']['backfill'] = "{$runVar['commands']['_python']} {$runVar['paths']['misc']}update_scripts/nix_scripts/tmux/python/backfill_threaded.py group"; + $runVar['scripts']['backfill'] = "{$runVar['commands']['_python']} {$runVar['paths']['misc']}update_scripts/python/backfill_threaded.py group"; break; case 4: $runVar['scripts']['backfill'] = "{$runVar['commands']['_php']} {$runVar['paths']['misc']}update_scripts/nix_scripts/multiprocessing/safe.php backfill"; diff --git a/misc/update_scripts/nix_scripts/tmux/run.php b/misc/update_scripts/nix_scripts/tmux/run.php index 963d8b21f..b67f87b74 100644 --- a/misc/update_scripts/nix_scripts/tmux/run.php +++ b/misc/update_scripts/nix_scripts/tmux/run.php @@ -199,9 +199,9 @@ function window_proxy($tmux_session, $window) $nntpproxy = $pdo->getSetting('nntpproxy'); if ($nntpproxy === '1') { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $nntpproxypy = $DIR . "update_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/python/lib/nntpproxy.conf"; exec("tmux new-window -t $tmux_session -n nntpproxy 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); } } diff --git a/misc/update_scripts/nntpproxy.php b/misc/update_scripts/nntpproxy.php index 917abae5b..740f64871 100644 --- a/misc/update_scripts/nntpproxy.php +++ b/misc/update_scripts/nntpproxy.php @@ -42,27 +42,27 @@ function window_proxy($tmux_session, $powerline) if ($powerline === '1') { $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf"; } else { - $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf"; + $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/conf/tmux.conf"; } $nntpproxy = $site->nntpproxy; if ($nntpproxy === '1') { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/_tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux//python/lib/nntpproxy.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/python/lib/nntpproxy.conf"; shell_exec("cd ${DIR}/update_scripts/nix_scripts/tmux; tmux -f $tmuxconfig attach-session -t $tmux_session || tmux -f $tmuxconfig new-session -d -s $tmux_session -n NNTPProxy 'printf \"\033]2;\"NNTPProxy\"\033\" && python $nntpproxypy $nntpproxyconf'"); } } if ($nntpproxy == '1' && ($site->alternate_nntp == '1')) { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/python/lib/nntpproxy_a.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/python/lib/nntpproxy_a.conf"; shell_exec("tmux selectp -t 0; tmux splitw -t $tmux_session:0 -h -p 50 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); } } } -window_proxy($tmux_session, $powerline); +window_proxy($tmux_session, $powerline); \ No newline at end of file diff --git a/misc/update_scripts/python/.gitignore b/misc/update_scripts/python/.gitignore new file mode 100644 index 000000000..1af04760d --- /dev/null +++ b/misc/update_scripts/python/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.pyo +/.idea \ No newline at end of file diff --git a/misc/update_scripts/python/README.txt b/misc/update_scripts/python/README.txt new file mode 100644 index 000000000..3a2edc3b3 --- /dev/null +++ b/misc/update_scripts/python/README.txt @@ -0,0 +1,7 @@ +Requires Python 2.7, with the cymysql module. + +Run like this : + +python -OOu binaries_threaded.py +-or- +python3 -OOu binaries_threaded.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py b/misc/update_scripts/python/backfill_safe_threaded.py similarity index 77% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py rename to misc/update_scripts/python/backfill_safe_threaded.py index 0f3d1bc34..93486cfdb 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py +++ b/misc/update_scripts/python/backfill_safe_threaded.py @@ -22,23 +22,23 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1") +cur[0].execute("SELECT g.name FROM groups g LEFT JOIN short_groups ON short_groups.name = g.name WHERE short_groups.name IS NULL AND backfill = 1") dorun = cur[0].fetchone() #close connection to mysql info.disconnect(cur[0], cur[1]) if dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) else: cur = info.connect() - cur[0].execute("SELECT name FROM shortgroups") + cur[0].execute("SELECT name FROM short_groups") dorun = cur[0].fetchone() info.disconnect(cur[0], cur[1]) if len(sys.argv) > 1 and sys.argv[1] not in dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) count = 0 previous = "'alt.binaries.crap'" @@ -78,13 +78,13 @@ while count < 10000: #query to grab backfill groups if len(sys.argv) == 1: - if conf['DB_TYPE'] == "mysql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) - elif conf['DB_TYPE'] == "pgsql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) + if conf['DB_SYSTEM'] == "mysql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) + elif conf['DB_SYSTEM'] == "pgsql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) datas = cur[0].fetchone() else: - run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" + run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" cur[0].execute(run, (sys.argv[1])) datas = cur[0].fetchone() if not datas or datas[0] is None: @@ -119,7 +119,7 @@ while count < 10000: print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC) print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC) group = ("{} {}".format(datas[0], count)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) #calculate the number of items for queue if (count > (backfill_qty * run_threads)): @@ -147,7 +147,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -177,7 +177,7 @@ def main(args): my_queue.join() group = ("{} {}".format(datas[0], 1000)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) if run_threads <= geteach: print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC) else: @@ -188,4 +188,4 @@ def main(args): if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py b/misc/update_scripts/python/backfill_threaded.py similarity index 92% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py rename to misc/update_scripts/python/backfill_threaded.py index 70a5816f5..9da0f5506 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py +++ b/misc/update_scripts/python/backfill_threaded.py @@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all": # removing the % before the variables at the end of the query adds quotes/escapes strings cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group)) else: - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) datas = cur[0].fetchall() @@ -99,9 +99,9 @@ class queue_runner(threading.Thread): if my_id: time_of_last_run = time.time() if len(sys.argv) > 1 and sys.argv[1] == "all": - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) else: - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -135,4 +135,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py b/misc/update_scripts/python/binaries_safe_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py rename to misc/update_scripts/python/binaries_safe_threaded.py index d6b034f8b..2e2d1729a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py +++ b/misc/update_scripts/python/binaries_safe_threaded.py @@ -30,12 +30,12 @@ dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) maxmssgs = int(dbgrab[0][1]) -#before we get the groups, lets update shortgroups -subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) +#before we get the groups, lets update short_groups +subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) #query to grab all active groups cur = info.connect() -cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN shortgroups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") +cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") datas = cur[0].fetchall() #close connection to mysql @@ -64,7 +64,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -121,4 +121,4 @@ def main(): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py b/misc/update_scripts/python/binaries_threaded.py similarity index 90% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py rename to misc/update_scripts/python/binaries_threaded.py index 879be3be4..09fe97796 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py +++ b/misc/update_scripts/python/binaries_threaded.py @@ -24,15 +24,14 @@ if len(sys.argv) == 1: print(bcolors.HEADER + "\nThis script will run update_binaries per group." "\nThis script can run on 1 group, an array of groups or all groups.\n" "\nEach group is processed in a single thread, for all groups. For example, 10 groups, 10 threads, upto max threads.\n" - "\npython " + sys.argv[0] + " 155 ...: To run against groupid 155." - "\npython " + sys.argv[0] + " '(155, 52)' ...: To run against groupid 155 and 52." + "\npython " + sys.argv[0] + " 155 ...: To run against group_id 155." + "\npython " + sys.argv[0] + " '(155, 52)' ...: To run against group_id 155 and 52." "\npython " + sys.argv[0] + " alt.binaries.tv ...: To run against group alt.binaries.teevee." "\npython " + sys.argv[0] + " ...: To run against all active groups." + bcolors.ENDC) print(bcolors.HEADER + "\nBinaries Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) - #get active groups if len(sys.argv) == 2: try: @@ -51,18 +50,17 @@ else: datas = cur[0].fetchall() if len(datas) == 0: - print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) - info.disconnect(cur[0], cur[1]) - sys.exit + print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) + info.disconnect(cur[0], cur[1]) + sys.exit -cur[0].execute("SELECT value FROM tmux WHERE setting = 'binarythreads'") -dbgrab = cur[0].fetchone() +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'binarythreads') AS a") +dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) #close connection to mysql info.disconnect(cur[0], cur[1]) - my_queue = queue.Queue() time_of_last_run = time.time() @@ -83,7 +81,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/update_binaries.php", ""+my_id]) + subprocess.call(["php", pathname+"/../update_binaries.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py b/misc/update_scripts/python/fixreleasenames_threaded.py similarity index 61% rename from misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py rename to misc/update_scripts/python/fixreleasenames_threaded.py index c27d86e8d..12f77ca65 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py +++ b/misc/update_scripts/python/fixreleasenames_threaded.py @@ -22,16 +22,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) == 1: print(bcolors.ERROR + "\nAn argument is required\n\n" - + "python " + sys.argv[0] + " [md5, nfo, filename, par2] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2].\n" + + "python " + sys.argv[0] + " [md5, nfo, filename, par2, miscsorter] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2, miscsorter].\n" + "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashid, using [nfo, filename, par2].\n" + "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n" + "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n" - + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + bcolors.ENDC) sys.exit() -if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft": - print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC) +if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "miscsorter" and sys.argv[1] != "predbft": + print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, miscsorter, predbft]\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": @@ -39,47 +39,48 @@ if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "clean": clean = " isrenamed = 0 AND proc_files = 1 AND proc_nfo = 1 " elif len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "filename" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " else: clean = " isrenamed = 0 " print(bcolors.HEADER + "\nfixReleasesNames {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") perrun = cur[0].fetchone() datas = [] -maxtries = 0 if len(sys.argv) > 1 and sys.argv[1] == "nfo": - run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() +elif len(sys.argv) > 1 and sys.argv[1] == "miscsorter": + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_sorter = 0 AND isrenamed = 0 ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "filename"): - run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN releasefiles relfiles ON (relfiles.releaseid = rel.id) WHERE categoryid IN (8010, 8020) AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN release_files relfiles ON (relfiles.releaseid = rel.id) WHERE nzbstatus = 1 AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "md5"): - while len(datas) == 0 and maxtries >= -5: - run = "SELECT DISTINCT rel.id FROM releases rel INNER JOIN releasefiles rf ON rel.id = rf.releaseid WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashid = 0 ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0]))) - datas = cur[0].fetchall() - maxtries = maxtries - 1 + run = "SELECT DISTINCT rel.id FROM releases rel LEFT OUTER JOIN release_files rf ON rel.id = rf.releaseid AND rf.ishashed = 1 WHERE nzbstatus = 1 AND rel.dehashstatus BETWEEN -6 AND 0 AND rel.ishashed = 1 AND prehashid = 0 ORDER BY dehashstatus DESC, postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0])*int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "par2"): #This one does from oldest posts to newest posts, since nfo pp does same thing but newest to oldest - run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE categoryid IN (8010, 8020) AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT id AS releaseid, guid, group_id FROM releases WHERE nzbstatus = 1 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"): #This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes - run = "SELECT id AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0]))) + run = "SELECT id AS prehashid FROM prehash WHERE LENGTH(title) >= 15 AND title NOT REGEXP '[\"\<\> ]' AND searched = 0 AND DATEDIFF(NOW(), predate) > 1 ORDER BY predate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) datas = cur[0].fetchall() #close connection to mysql @@ -109,7 +110,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/fixreleasenames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/fixreleasenames.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -117,11 +118,7 @@ def main(): global time_of_last_run time_of_last_run = time.time() - if sys.argv[1] == 'md5': - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. dehashstatus range {} to 0".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1], maxtries + 1) + bcolors.ENDC) - else: - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) - time.sleep(2) + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) def signal_handler(signal, frame): sys.exit(0) @@ -152,6 +149,10 @@ def main(): for release in datas: time.sleep(.03) my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2])) + elif sys.argv[1] == "miscsorter": + for release in datas: + time.sleep(.03) + my_queue.put("%s %s" % ("miscsorter", release[0])) elif sys.argv[1] == "predbft": for release in datas: time.sleep(.03) diff --git a/misc/update_scripts/python/grabheaders_threaded.py b/misc/update_scripts/python/grabheaders_threaded.py new file mode 100644 index 000000000..38eeb3a16 --- /dev/null +++ b/misc/update_scripts/python/grabheaders_threaded.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, time +import threading +try: + import queue +except ImportError: + import Queue as queue +import subprocess +import string +import signal +import datetime + +import lib.info as info +from lib.info import bcolors +conf = info.readConfig() +start_time = time.time() +pathname = os.path.abspath(os.path.dirname(sys.argv[0])) +threads = 10 + +print(bcolors.HEADER + "\nGrab Headers Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + +if len(sys.argv) > 1: + datas = [] + datas.append(sys.argv[1]) +else: + datas = ("alt.binaries.teevee", "alt.binaries.tv", "alt.binaries.audiobooks", "alt.binaries.moovee", "alt.binaries.e-book", "alt.binaries.e-book.technical", "alt.binaries.ebook", "alt.binaries.e-book.magazines") +if not datas: + print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC) + sys.exit() + +my_queue = queue.Queue() +time_of_last_run = time.time() + +class queue_runner(threading.Thread): + def __init__(self, my_queue): + threading.Thread.__init__(self) + self.my_queue = my_queue + + def run(self): + global time_of_last_run + + while True: + try: + my_id = self.my_queue.get(True, 1) + except: + if time.time() - time_of_last_run > 3: + return + else: + if my_id: + time_of_last_run = time.time() + subprocess.call(["php", pathname+"/../../testing/Regex/grabheaders.php", ""+my_id]) + self.my_queue.task_done() + +def main(): + global time_of_last_run + time_of_last_run = time.time() + + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC) + time.sleep(2) + def signal_handler(signal, frame): + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + if True: + #spawn a pool of place worker threads + for i in range(threads): + p = queue_runner(my_queue) + p.setDaemon(False) + p.start() + + #now load some arbitrary jobs into the queue + count = 0 + for group in datas: + my_queue.put(group) + + my_queue.join() + + print(bcolors.HEADER + "\nGrab Headers Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) + +if __name__ == '__main__': + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py b/misc/update_scripts/python/groupfixrelnames_threaded.py similarity index 81% rename from misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py rename to misc/update_scripts/python/groupfixrelnames_threaded.py index f89cdecaf..067bc69f8 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py +++ b/misc/update_scripts/python/groupfixrelnames_threaded.py @@ -37,13 +37,15 @@ limit = 0 join = "" where = "" datelimit = "AND DATEDIFF(NOW(), r.adddate) <= 7" -groupby = "GROUP BY LEFT(r.guid, 1)" -orderby = "ORDER BY LEFT(r.guid, 1) ASC" -rowlimit = "LIMIT 100" +groupby = "GROUP BY guidchar" +orderby = "ORDER BY guidchar ASC" +rowlimit = "LIMIT 16" +extrawhere = "AND r.prehashid = 0 AND r.nzbstatus = 1" +select = "DISTINCT LEFT(r.guid, 1) AS guidchar, COUNT(*) AS count" -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") run_perrun = cur[0].fetchone() threads = int(run_threads[0]) @@ -52,22 +54,23 @@ if threads > 16: maxperrun = int(run_perrun[0]) if sys.argv[1] == "md5": - join = "LEFT OUTER JOIN releasefiles rf ON r.id = rf.releaseid AND rf.ishashed = 1" + join = "LEFT OUTER JOIN release_files rf ON r.id = rf.releaseid AND rf.ishashed = 1" where = "r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0" elif sys.argv[1] == "nfo": where = "r.proc_nfo = 0 AND r.nfostatus = 1" elif sys.argv[1] == "filename": - join = "INNER JOIN releasefiles rf ON r.id = rf.releaseid" + join = "INNER JOIN release_files rf ON r.id = rf.releaseid" where = "r.proc_files = 0" elif sys.argv[1] == "par2": where = "r.proc_par2 = 0" elif sys.argv[1] == "miscsorter": - where = "r.nfostatus = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" + where = "r.nfostatus = 1 AND r.proc_nfo = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" elif sys.argv[1] == "predbft": + extrawhere = "" where = "1=1" rowlimit = "LIMIT %s" % (threads) -cur[0].execute("SELECT DISTINCT LEFT(r.guid, 1), COUNT(*) AS count FROM releases r %s WHERE %s AND r.prehashid = 0 AND r.nzbstatus = 1 %s %s %s %s" % (join, where, datelimit, groupby, orderby, rowlimit)) +cur[0].execute("SELECT %s FROM releases r %s WHERE %s %s %s %s %s" % (select, join, where, extrawhere, groupby, orderby, rowlimit)) datas = cur[0].fetchall() guids = int(len(datas)) @@ -99,7 +102,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/groupfixrelnames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/groupfixrelnames.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py b/misc/update_scripts/python/import_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/import_threaded.py rename to misc/update_scripts/python/import_threaded.py index f62d709e5..b1aaa953a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py +++ b/misc/update_scripts/python/import_threaded.py @@ -24,7 +24,7 @@ print(bcolors.HEADER + "\nNZB Import Threaded Started at {}".format(datetime.dat #get values from db cur[0].execute("SELECT value FROM tmux WHERE setting = 'import'") use_true = cur[0].fetchone() -cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) nzbs = dbgrab[0][1] @@ -57,7 +57,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/nzb-import-new.php", ""+my_id]) + subprocess.call(["php", pathname+"/lib/nzb-import.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -105,6 +105,8 @@ def main(args): my_queue.join() + final = "true" + subprocess.call(["php", pathname+"/../../testing/DB/populate_nzb_guid.php", ""+final]) print(bcolors.HEADER + "\nNZB Import Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php b/misc/update_scripts/python/lib/.gitignore.php similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php rename to misc/update_scripts/python/lib/.gitignore.php diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py b/misc/update_scripts/python/lib/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py rename to misc/update_scripts/python/lib/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/info.py b/misc/update_scripts/python/lib/info.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/info.py rename to misc/update_scripts/python/lib/info.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample b/misc/update_scripts/python/lib/nntpproxy.conf.sample similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample rename to misc/update_scripts/python/lib/nntpproxy.conf.sample diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore b/misc/update_scripts/python/lib/pynntp/.gitignore similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore rename to misc/update_scripts/python/lib/pynntp/.gitignore diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING b/misc/update_scripts/python/lib/pynntp/COPYING similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING rename to misc/update_scripts/python/lib/pynntp/COPYING diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE b/misc/update_scripts/python/lib/pynntp/NOTE similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE rename to misc/update_scripts/python/lib/pynntp/NOTE diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md b/misc/update_scripts/python/lib/pynntp/README.md similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md rename to misc/update_scripts/python/lib/pynntp/README.md diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py b/misc/update_scripts/python/lib/pynntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py b/misc/update_scripts/python/lib/pynntp/nntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/nntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py b/misc/update_scripts/python/lib/pynntp/nntp/date.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py rename to misc/update_scripts/python/lib/pynntp/nntp/date.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py b/misc/update_scripts/python/lib/pynntp/nntp/fifo.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py rename to misc/update_scripts/python/lib/pynntp/nntp/fifo.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py rename to misc/update_scripts/python/lib/pynntp/nntp/iodict.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py b/misc/update_scripts/python/lib/pynntp/nntp/nntp.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py rename to misc/update_scripts/python/lib/pynntp/nntp/nntp.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py b/misc/update_scripts/python/lib/pynntp/nntp/utils.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py rename to misc/update_scripts/python/lib/pynntp/nntp/utils.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py b/misc/update_scripts/python/lib/pynntp/nntp/yenc.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py rename to misc/update_scripts/python/lib/pynntp/nntp/yenc.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py b/misc/update_scripts/python/lib/pynntp/setup.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py rename to misc/update_scripts/python/lib/pynntp/setup.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py b/misc/update_scripts/python/nntpproxy.py similarity index 98% rename from misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py rename to misc/update_scripts/python/nntpproxy.py index 4dac64eba..104f8f189 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py +++ b/misc/update_scripts/python/nntpproxy.py @@ -27,7 +27,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): raise ValueError("Bad backend") nntp.NNTPClient.__init__(self, self.host, self.port, username, password, timeout=timeout, use_ssl=use_ssl) self.id = self.socket.getsockname()[1] - print(bcolors.PRIMARY + "New NNTP connection to %s established with id #%5d" % + print(bcolors.PRIMARY + "New NNTP connection to %s established with ID #%5d" % (self.host, self.id) + bcolors.ENDC) self._connected = True self.xfeature_compress_gzip() @@ -54,7 +54,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): return self._life def invalidate(self): - print(bcolors.PRIMARY + "Disconnecting from NNTP connection id #%5d after %d seconds." % + print(bcolors.PRIMARY + "Disconnecting from NNTP connection ID #%5d after %d seconds." % (self.id, (time.time() - self._start_time)) + bcolors.ENDC) self.close() self._connected = False @@ -68,7 +68,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): self._pool = None -# NNTP proxy request handler for newznab +# NNTP proxy request handler for nZEDb class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): def handle(self): @@ -196,7 +196,7 @@ class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): self.wfile.write("500 What?\r\n") -# NNTP proxy server for newznab +# NNTP proxy server for nZEDb class NNTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): allow_reuse_address = True diff --git a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py b/misc/update_scripts/python/postprocess_threaded.py similarity index 82% rename from misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py rename to misc/update_scripts/python/postprocess_threaded.py index 80768408c..dcf264553 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py +++ b/misc/update_scripts/python/postprocess_threaded.py @@ -19,7 +19,7 @@ conf = info.readConfig() cur = info.connect() if len(sys.argv) == 1: - print(bcolors.ERROR + "\nWrong set of arguments.\nThe first argument [additional, nfo, movie, clean] determines the postprocessing to do.\nThe optional second argument for [additional, nfo] [groupid, categoryid] allows to process only that group or category.\nThe optional second argument for [movies, tv] [clean] allows processing only properly renamed releases.\n\npython postprocess_threaded.py [additional, nfo] (optional [groupid, categoryid])\npython postprocess_threaded.py [movie, tv] (optional [clean])\n" + bcolors.ENDC) + print(bcolors.ERROR + "\nWrong set of arguments.\nThe first argument [additional, nfo, movie, clean] determines the postprocessing to do.\nThe optional second argument for [additional, nfo] [group_id, categoryid] allows to process only that group or category.\nThe optional second argument for [movies, tv] [clean] allows processing only properly renamed releases.\n\npython postprocess_threaded.py [additional, nfo] (optional [group_id, categoryid])\npython postprocess_threaded.py [movie, tv] (optional [clean])\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[2] == "clean": print(bcolors.HEADER + "\nPostProcess {} Clean Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) @@ -33,40 +33,40 @@ if sys.argv[1] == "additional": elif sys.argv[1] == "nfo": print(bcolors.HEADER + "* = hidden NFO, + = NFO, - = no NFO, f = download failed." + bcolors.ENDC) -# You can limit postprocessing for additional and nfo by groupid or categoryid +# You can limit postprocessing for additional and nfo by group_id or categoryid if len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) < 4: - groupid = 'AND groupid = '+sys.argv[2] - print(bcolors.HEADER + "Using groupid "+sys.argv[2] + bcolors.ENDC) + group_id = 'AND group_id = '+sys.argv[2] + print(bcolors.HEADER + "Using group_id "+sys.argv[2] + bcolors.ENDC) elif len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) == 4: if sys.argv[2] == '1000': - groupid = 'AND categoryid BETWEEN 1000 AND 1999' + group_id = 'AND categoryid BETWEEN 1000 AND 1999' print(bcolors.HEADER + "Using categoryids 1000-1999" + bcolors.ENDC) elif sys.argv[2] == '2000': - groupid = 'AND categoryid BETWEEN 2000 AND 2999' + group_id = 'AND categoryid BETWEEN 2000 AND 2999' print(bcolors.HEADER + "Using categoryids 2000-2999" + bcolors.ENDC) elif sys.argv[2] == '3000': - groupid = 'AND categoryid BETWEEN 3000 AND 3999' + group_id = 'AND categoryid BETWEEN 3000 AND 3999' print(bcolors.HEADER + "Using categoryids 3000-3999" + bcolors.ENDC) elif sys.argv[2] == '4000': - groupid = 'AND categoryid BETWEEN 4000 AND 4999' + group_id = 'AND categoryid BETWEEN 4000 AND 4999' print(bcolors.HEADER + "Using categoryids 4000-4999" + bcolors.ENDC) elif sys.argv[2] == '5000': - groupid = 'AND categoryid BETWEEN 5000 AND 5999' + group_id = 'AND categoryid BETWEEN 5000 AND 5999' print(bcolors.HEADER + "Using categoryids 5000-5999" + bcolors.ENDC) elif sys.argv[2] == '6000': - groupid = 'AND categoryid BETWEEN 6000 AND 6999' + group_id = 'AND categoryid BETWEEN 6000 AND 6999' print(bcolors.HEADER + "Using categoryids 6000-6999" + bcolors.ENDC) elif sys.argv[2] == '7000': - groupid = 'AND categoryid BETWEEN 7000 AND 7999' + group_id = 'AND categoryid BETWEEN 7000 AND 7999' print(bcolors.HEADER + "Using categoryids 7000-7999" + bcolors.ENDC) elif sys.argv[2] == '8000': - groupid = 'AND categoryid BETWEEN 8000 AND 8999' + group_id = 'AND categoryid BETWEEN 8000 AND 8999' print(bcolors.HEADER + "Using categoryids 8000-8999" + bcolors.ENDC) else: - groupid = 'AND categoryid = '+sys.argv[2] + group_id = 'AND categoryid = '+sys.argv[2] print(bcolors.HEADER + "Using categoryid "+sys.argv[2] + bcolors.ENDC) else: - groupid = '' + group_id = '' #you can sort tv releases by searchname if len(sys.argv) == 3 and (sys.argv[2] == "asc" or sys.argv[2] == "desc"): @@ -82,7 +82,7 @@ start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) > 1 and sys.argv[1] == "additional": - cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'postthreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+groupid+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+groupid+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+groupid+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+groupid+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+groupid+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+groupid+") as o") + cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'postthreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+group_id+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+group_id+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+group_id+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+group_id+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+group_id+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+group_id+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -91,7 +91,7 @@ if len(sys.argv) > 1 and sys.argv[1] == "additional": ps5 = format(int(dbgrab[0][13])) ps6 = format(int(dbgrab[0][14])) elif len(sys.argv) > 1 and sys.argv[1] == "nfo": - cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nfothreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 "+groupid+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 "+groupid+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 "+groupid+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 "+groupid+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 "+groupid+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 "+groupid+") as o") + cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nfothreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 "+group_id+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 "+group_id+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 "+group_id+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 "+group_id+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 "+group_id+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 "+group_id+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -175,7 +175,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) time.sleep(.02) self.my_queue.task_done() @@ -232,7 +232,7 @@ def main(args): final = cur[0].fetchall() if len(datas) > 0: for item in final: - run = "DELETE FROM releasenfo WHERE nfo IS NULL AND releaseid = %s" + run = "DELETE FROM release_nfos WHERE nfo IS NULL AND releaseid = %s" cur[0].execute(run, (item[0])) final = cur[0].fetchall() @@ -243,4 +243,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py b/misc/update_scripts/python/releases_threaded.py similarity index 86% rename from misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py rename to misc/update_scripts/python/releases_threaded.py index 01112c8df..930500d73 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py +++ b/misc/update_scripts/python/releases_threaded.py @@ -21,7 +21,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nUpdate Releases Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasethreads') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasesthreads') AS b") dbgrab = cur[0].fetchall() allowed = int(dbgrab[0][0]) threads = int(dbgrab[0][1]) @@ -30,7 +30,7 @@ if allowed == 0: info.disconnect(cur[0], cur[1]) sys.exit() -cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'binaries_%'") +cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'collections_%'") datas = cur[0].fetchall() #close connection to mysql @@ -60,7 +60,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "releases "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+my_id]) self.my_queue.task_done() def main(): @@ -88,15 +88,15 @@ def main(): if count >= threads: count = 0 count += 1 - my_queue.put("%s %s" % (release[0].replace('binaries_', ''), count)) + my_queue.put("%s %s" % (release[0].replace('collections_', ''), count)) my_queue.join() #stage7b - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py b/misc/update_scripts/python/requestid_threaded.py similarity index 87% rename from misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py rename to misc/update_scripts/python/requestid_threaded.py index 840096a62..0d6b4ec01 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py +++ b/misc/update_scripts/python/requestid_threaded.py @@ -35,7 +35,7 @@ print(bcolors.HEADER + "\n\nRequestID Threaded Started at {}".format(datetime.da cur[0].execute("SELECT value FROM settings WHERE setting = 'request_hours'") dbgrab = cur[0].fetchone() request_hours = str(dbgrab[0]) -cur[0].execute("SELECT DISTINCT(g.id) FROM releases r INNER JOIN groups g ON r.groupid = g.id WHERE r.nzbstatus = 1 AND r.prehashid = 0 AND r.isrequestid = 1 AND r.reqidstatus in (0, -1) OR (r.reqidstatus = -3 AND r.adddate > NOW() - INTERVAL " + request_hours + " HOUR)") +cur[0].execute("SELECT DISTINCT(g.id) FROM releases r INNER JOIN groups g ON r.group_id = g.id WHERE r.nzbstatus = 1 AND r.prehashid = 0 AND r.isrequestid = 1 AND r.reqidstatus in (0, -1) OR (r.reqidstatus = -3 AND r.adddate > NOW() - INTERVAL " + request_hours + " HOUR)") datas = cur[0].fetchall() #close connection to mysql @@ -65,7 +65,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -98,4 +98,4 @@ def main(): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py b/misc/update_scripts/python/update_threaded.py similarity index 90% rename from misc/update_scripts/nix_scripts/tmux/python/update_threaded.py rename to misc/update_scripts/python/update_threaded.py index c88be1fb4..595cbf041 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py +++ b/misc/update_scripts/python/update_threaded.py @@ -18,7 +18,7 @@ conf = info.readConfig() cur = info.connect() start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) -cur[0].execute("SELECT value FROM settings WHERE setting = 'releasethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'releasesthreads'") threads = cur[0].fetchone() threads = int(threads[0]) @@ -61,7 +61,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) self.my_queue.task_done() def main(): @@ -95,10 +95,10 @@ def main(): #stage7b final = "final" - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/newznab/controllers/Tmux.php b/newznab/controllers/Tmux.php index ddb8b4e14..e0a1265b4 100644 --- a/newznab/controllers/Tmux.php +++ b/newznab/controllers/Tmux.php @@ -70,7 +70,7 @@ class Tmux $runVar['connections']['ip_a'] = gethostbyname($runVar['connections']['host_a']); } } else { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); @@ -83,7 +83,7 @@ class Tmux } } if ($constants['alternate_nntp']) { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy_a.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); diff --git a/newznab/processing/ProcessReleases.php b/newznab/processing/ProcessReleases.php index f8eea1f7a..1e7211237 100644 --- a/newznab/processing/ProcessReleases.php +++ b/newznab/processing/ProcessReleases.php @@ -190,7 +190,7 @@ class ProcessReleases if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request id Threaded lookup.")); } - passthru("$PYTHON ${DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py"); + passthru("$PYTHON ${DIR}update_scripts/nix_scripts/requestid_threaded.py"); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( diff --git a/www/templates/omicron/views/frontend/viewnzb.tpl b/www/templates/omicron/views/frontend/viewnzb.tpl index 7614d261c..e4f18bf34 100644 --- a/www/templates/omicron/views/frontend/viewnzb.tpl +++ b/www/templates/omicron/views/frontend/viewnzb.tpl @@ -477,8 +477,7 @@ - RAR - Contains + RAR Contains Files:
diff --git a/www/templates_shared/views/admin/site-edit.tpl b/www/templates_shared/views/admin/site-edit.tpl index 1d74d72f1..deaef6763 100644 --- a/www/templates_shared/views/admin/site-edit.tpl +++ b/www/templates_shared/views/admin/site-edit.tpl @@ -1496,7 +1496,7 @@ reduces time for connection setup/teardown. The proxy also takes care of compressed headers for you. To use this featrure you will need to install socketpool (sudo pip install socketpool or sudo easy_install socketpool) (ensure python2 is default) and edit the configuration file (nntpproxy.conf and nntpproxy_a.conf) in the - update_scripts/nix_scripts/tmux/python/lib (copy sample) directory and finally edit your www/config.php file + update_scripts/python/lib (copy sample) directory and finally edit your www/config.php file to use the proxy (username and password are ignored by the proxy - make then anything you like - the proxy doesn't use ssl either). Make sure you turn off the use compressed headers option here in site preferences (the proxy uses compressed headers by default and passes on decompressed data).