diff --git a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php index ea29720d7..61bb97763 100644 --- a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php +++ b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php @@ -7,7 +7,7 @@ use newznab\processing\PostProcess; $pdo = new Settings(); if (!isset($argv[1])) { - exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py.")); + exit($pdo->log->error("This script is not intended to be run manually, it is called from Forking.php.")); } else if (isset($argv[1])) { $namefixer = new \NameFixer(['Settings' => $pdo]); $pieces = explode(' ', $argv[1]); diff --git a/misc/update_scripts/nix_scripts/tmux/monitor.php b/misc/update_scripts/nix_scripts/tmux/monitor.php index 409969be4..1b7186b38 100644 --- a/misc/update_scripts/nix_scripts/tmux/monitor.php +++ b/misc/update_scripts/nix_scripts/tmux/monitor.php @@ -107,7 +107,7 @@ while ($runVar['counts']['iterations'] > 0) { $runVar['scripts']['backfill'] = "{$runVar['commands']['_php']} {$runVar['paths']['misc']}update_scripts/nix_scripts/multiprocessing/backfill.php"; break; case 2: - $runVar['scripts']['backfill'] = "{$runVar['commands']['_python']} {$runVar['paths']['misc']}update_scripts/nix_scripts/tmux/python/backfill_threaded.py group"; + $runVar['scripts']['backfill'] = "{$runVar['commands']['_python']} {$runVar['paths']['misc']}update_scripts/python/backfill_threaded.py group"; break; case 4: $runVar['scripts']['backfill'] = "{$runVar['commands']['_php']} {$runVar['paths']['misc']}update_scripts/nix_scripts/multiprocessing/safe.php backfill"; diff --git a/misc/update_scripts/nix_scripts/tmux/run.php b/misc/update_scripts/nix_scripts/tmux/run.php index 963d8b21f..b67f87b74 100644 --- a/misc/update_scripts/nix_scripts/tmux/run.php +++ b/misc/update_scripts/nix_scripts/tmux/run.php @@ -199,9 +199,9 @@ function window_proxy($tmux_session, $window) $nntpproxy = $pdo->getSetting('nntpproxy'); if ($nntpproxy === '1') { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $nntpproxypy = $DIR . "update_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/python/lib/nntpproxy.conf"; exec("tmux new-window -t $tmux_session -n nntpproxy 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); } } diff --git a/misc/update_scripts/nntpproxy.php b/misc/update_scripts/nntpproxy.php index 917abae5b..740f64871 100644 --- a/misc/update_scripts/nntpproxy.php +++ b/misc/update_scripts/nntpproxy.php @@ -42,27 +42,27 @@ function window_proxy($tmux_session, $powerline) if ($powerline === '1') { $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf"; } else { - $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf"; + $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/conf/tmux.conf"; } $nntpproxy = $site->nntpproxy; if ($nntpproxy === '1') { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/_tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux//python/lib/nntpproxy.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/python/lib/nntpproxy.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/python/lib/nntpproxy.conf"; shell_exec("cd ${DIR}/update_scripts/nix_scripts/tmux; tmux -f $tmuxconfig attach-session -t $tmux_session || tmux -f $tmuxconfig new-session -d -s $tmux_session -n NNTPProxy 'printf \"\033]2;\"NNTPProxy\"\033\" && python $nntpproxypy $nntpproxyconf'"); } } if ($nntpproxy == '1' && ($site->alternate_nntp == '1')) { $DIR = NN_MISC; - $nntpproxypy = $DIR . "update_scripts/nix_scripts/tmux/python/nntpproxy.py"; - if (file_exists($DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf")) { - $nntpproxyconf = $DIR . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + $nntpproxypy = $DIR . "update_scripts/nix_scripts/python/nntpproxy.py"; + if (file_exists($DIR . "update_scripts/nix_scripts/python/lib/nntpproxy_a.conf")) { + $nntpproxyconf = $DIR . "update_scripts/nix_scripts/python/lib/nntpproxy_a.conf"; shell_exec("tmux selectp -t 0; tmux splitw -t $tmux_session:0 -h -p 50 'printf \"\033]2;NNTPProxy\033\" && python $nntpproxypy $nntpproxyconf'"); } } } -window_proxy($tmux_session, $powerline); +window_proxy($tmux_session, $powerline); \ No newline at end of file diff --git a/misc/update_scripts/python/.gitignore b/misc/update_scripts/python/.gitignore new file mode 100644 index 000000000..1af04760d --- /dev/null +++ b/misc/update_scripts/python/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.pyo +/.idea \ No newline at end of file diff --git a/misc/update_scripts/python/README.txt b/misc/update_scripts/python/README.txt new file mode 100644 index 000000000..3a2edc3b3 --- /dev/null +++ b/misc/update_scripts/python/README.txt @@ -0,0 +1,7 @@ +Requires Python 2.7, with the cymysql module. + +Run like this : + +python -OOu binaries_threaded.py +-or- +python3 -OOu binaries_threaded.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py b/misc/update_scripts/python/backfill_safe_threaded.py similarity index 77% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py rename to misc/update_scripts/python/backfill_safe_threaded.py index 0f3d1bc34..93486cfdb 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py +++ b/misc/update_scripts/python/backfill_safe_threaded.py @@ -22,23 +22,23 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1") +cur[0].execute("SELECT g.name FROM groups g LEFT JOIN short_groups ON short_groups.name = g.name WHERE short_groups.name IS NULL AND backfill = 1") dorun = cur[0].fetchone() #close connection to mysql info.disconnect(cur[0], cur[1]) if dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) else: cur = info.connect() - cur[0].execute("SELECT name FROM shortgroups") + cur[0].execute("SELECT name FROM short_groups") dorun = cur[0].fetchone() info.disconnect(cur[0], cur[1]) if len(sys.argv) > 1 and sys.argv[1] not in dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) count = 0 previous = "'alt.binaries.crap'" @@ -78,13 +78,13 @@ while count < 10000: #query to grab backfill groups if len(sys.argv) == 1: - if conf['DB_TYPE'] == "mysql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) - elif conf['DB_TYPE'] == "pgsql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) + if conf['DB_SYSTEM'] == "mysql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) + elif conf['DB_SYSTEM'] == "pgsql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) datas = cur[0].fetchone() else: - run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" + run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" cur[0].execute(run, (sys.argv[1])) datas = cur[0].fetchone() if not datas or datas[0] is None: @@ -119,7 +119,7 @@ while count < 10000: print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC) print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC) group = ("{} {}".format(datas[0], count)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) #calculate the number of items for queue if (count > (backfill_qty * run_threads)): @@ -147,7 +147,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -177,7 +177,7 @@ def main(args): my_queue.join() group = ("{} {}".format(datas[0], 1000)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) if run_threads <= geteach: print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC) else: @@ -188,4 +188,4 @@ def main(args): if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py b/misc/update_scripts/python/backfill_threaded.py similarity index 92% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py rename to misc/update_scripts/python/backfill_threaded.py index 70a5816f5..9da0f5506 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py +++ b/misc/update_scripts/python/backfill_threaded.py @@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all": # removing the % before the variables at the end of the query adds quotes/escapes strings cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group)) else: - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) datas = cur[0].fetchall() @@ -99,9 +99,9 @@ class queue_runner(threading.Thread): if my_id: time_of_last_run = time.time() if len(sys.argv) > 1 and sys.argv[1] == "all": - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) else: - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -135,4 +135,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py b/misc/update_scripts/python/binaries_safe_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py rename to misc/update_scripts/python/binaries_safe_threaded.py index d6b034f8b..2e2d1729a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py +++ b/misc/update_scripts/python/binaries_safe_threaded.py @@ -30,12 +30,12 @@ dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) maxmssgs = int(dbgrab[0][1]) -#before we get the groups, lets update shortgroups -subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) +#before we get the groups, lets update short_groups +subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) #query to grab all active groups cur = info.connect() -cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN shortgroups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") +cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") datas = cur[0].fetchall() #close connection to mysql @@ -64,7 +64,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -121,4 +121,4 @@ def main(): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py b/misc/update_scripts/python/binaries_threaded.py similarity index 90% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py rename to misc/update_scripts/python/binaries_threaded.py index 879be3be4..09fe97796 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py +++ b/misc/update_scripts/python/binaries_threaded.py @@ -24,15 +24,14 @@ if len(sys.argv) == 1: print(bcolors.HEADER + "\nThis script will run update_binaries per group." "\nThis script can run on 1 group, an array of groups or all groups.\n" "\nEach group is processed in a single thread, for all groups. For example, 10 groups, 10 threads, upto max threads.\n" - "\npython " + sys.argv[0] + " 155 ...: To run against groupid 155." - "\npython " + sys.argv[0] + " '(155, 52)' ...: To run against groupid 155 and 52." + "\npython " + sys.argv[0] + " 155 ...: To run against group_id 155." + "\npython " + sys.argv[0] + " '(155, 52)' ...: To run against group_id 155 and 52." "\npython " + sys.argv[0] + " alt.binaries.tv ...: To run against group alt.binaries.teevee." "\npython " + sys.argv[0] + " ...: To run against all active groups." + bcolors.ENDC) print(bcolors.HEADER + "\nBinaries Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) - #get active groups if len(sys.argv) == 2: try: @@ -51,18 +50,17 @@ else: datas = cur[0].fetchall() if len(datas) == 0: - print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) - info.disconnect(cur[0], cur[1]) - sys.exit + print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) + info.disconnect(cur[0], cur[1]) + sys.exit -cur[0].execute("SELECT value FROM tmux WHERE setting = 'binarythreads'") -dbgrab = cur[0].fetchone() +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'binarythreads') AS a") +dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) #close connection to mysql info.disconnect(cur[0], cur[1]) - my_queue = queue.Queue() time_of_last_run = time.time() @@ -83,7 +81,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/update_binaries.php", ""+my_id]) + subprocess.call(["php", pathname+"/../update_binaries.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py b/misc/update_scripts/python/fixreleasenames_threaded.py similarity index 61% rename from misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py rename to misc/update_scripts/python/fixreleasenames_threaded.py index c27d86e8d..12f77ca65 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py +++ b/misc/update_scripts/python/fixreleasenames_threaded.py @@ -22,16 +22,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) == 1: print(bcolors.ERROR + "\nAn argument is required\n\n" - + "python " + sys.argv[0] + " [md5, nfo, filename, par2] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2].\n" + + "python " + sys.argv[0] + " [md5, nfo, filename, par2, miscsorter] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2, miscsorter].\n" + "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashid, using [nfo, filename, par2].\n" + "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n" + "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n" - + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + bcolors.ENDC) sys.exit() -if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft": - print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC) +if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "miscsorter" and sys.argv[1] != "predbft": + print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, miscsorter, predbft]\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": @@ -39,47 +39,48 @@ if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "clean": clean = " isrenamed = 0 AND proc_files = 1 AND proc_nfo = 1 " elif len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "filename" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " else: clean = " isrenamed = 0 " print(bcolors.HEADER + "\nfixReleasesNames {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") perrun = cur[0].fetchone() datas = [] -maxtries = 0 if len(sys.argv) > 1 and sys.argv[1] == "nfo": - run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() +elif len(sys.argv) > 1 and sys.argv[1] == "miscsorter": + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_sorter = 0 AND isrenamed = 0 ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "filename"): - run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN releasefiles relfiles ON (relfiles.releaseid = rel.id) WHERE categoryid IN (8010, 8020) AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN release_files relfiles ON (relfiles.releaseid = rel.id) WHERE nzbstatus = 1 AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "md5"): - while len(datas) == 0 and maxtries >= -5: - run = "SELECT DISTINCT rel.id FROM releases rel INNER JOIN releasefiles rf ON rel.id = rf.releaseid WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashid = 0 ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0]))) - datas = cur[0].fetchall() - maxtries = maxtries - 1 + run = "SELECT DISTINCT rel.id FROM releases rel LEFT OUTER JOIN release_files rf ON rel.id = rf.releaseid AND rf.ishashed = 1 WHERE nzbstatus = 1 AND rel.dehashstatus BETWEEN -6 AND 0 AND rel.ishashed = 1 AND prehashid = 0 ORDER BY dehashstatus DESC, postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0])*int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "par2"): #This one does from oldest posts to newest posts, since nfo pp does same thing but newest to oldest - run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE categoryid IN (8010, 8020) AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT id AS releaseid, guid, group_id FROM releases WHERE nzbstatus = 1 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"): #This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes - run = "SELECT id AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0]))) + run = "SELECT id AS prehashid FROM prehash WHERE LENGTH(title) >= 15 AND title NOT REGEXP '[\"\<\> ]' AND searched = 0 AND DATEDIFF(NOW(), predate) > 1 ORDER BY predate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) datas = cur[0].fetchall() #close connection to mysql @@ -109,7 +110,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/fixreleasenames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/fixreleasenames.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -117,11 +118,7 @@ def main(): global time_of_last_run time_of_last_run = time.time() - if sys.argv[1] == 'md5': - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. dehashstatus range {} to 0".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1], maxtries + 1) + bcolors.ENDC) - else: - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) - time.sleep(2) + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) def signal_handler(signal, frame): sys.exit(0) @@ -152,6 +149,10 @@ def main(): for release in datas: time.sleep(.03) my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2])) + elif sys.argv[1] == "miscsorter": + for release in datas: + time.sleep(.03) + my_queue.put("%s %s" % ("miscsorter", release[0])) elif sys.argv[1] == "predbft": for release in datas: time.sleep(.03) diff --git a/misc/update_scripts/python/grabheaders_threaded.py b/misc/update_scripts/python/grabheaders_threaded.py new file mode 100644 index 000000000..38eeb3a16 --- /dev/null +++ b/misc/update_scripts/python/grabheaders_threaded.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, time +import threading +try: + import queue +except ImportError: + import Queue as queue +import subprocess +import string +import signal +import datetime + +import lib.info as info +from lib.info import bcolors +conf = info.readConfig() +start_time = time.time() +pathname = os.path.abspath(os.path.dirname(sys.argv[0])) +threads = 10 + +print(bcolors.HEADER + "\nGrab Headers Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + +if len(sys.argv) > 1: + datas = [] + datas.append(sys.argv[1]) +else: + datas = ("alt.binaries.teevee", "alt.binaries.tv", "alt.binaries.audiobooks", "alt.binaries.moovee", "alt.binaries.e-book", "alt.binaries.e-book.technical", "alt.binaries.ebook", "alt.binaries.e-book.magazines") +if not datas: + print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC) + sys.exit() + +my_queue = queue.Queue() +time_of_last_run = time.time() + +class queue_runner(threading.Thread): + def __init__(self, my_queue): + threading.Thread.__init__(self) + self.my_queue = my_queue + + def run(self): + global time_of_last_run + + while True: + try: + my_id = self.my_queue.get(True, 1) + except: + if time.time() - time_of_last_run > 3: + return + else: + if my_id: + time_of_last_run = time.time() + subprocess.call(["php", pathname+"/../../testing/Regex/grabheaders.php", ""+my_id]) + self.my_queue.task_done() + +def main(): + global time_of_last_run + time_of_last_run = time.time() + + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC) + time.sleep(2) + def signal_handler(signal, frame): + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + if True: + #spawn a pool of place worker threads + for i in range(threads): + p = queue_runner(my_queue) + p.setDaemon(False) + p.start() + + #now load some arbitrary jobs into the queue + count = 0 + for group in datas: + my_queue.put(group) + + my_queue.join() + + print(bcolors.HEADER + "\nGrab Headers Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) + +if __name__ == '__main__': + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py b/misc/update_scripts/python/groupfixrelnames_threaded.py similarity index 81% rename from misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py rename to misc/update_scripts/python/groupfixrelnames_threaded.py index f89cdecaf..067bc69f8 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py +++ b/misc/update_scripts/python/groupfixrelnames_threaded.py @@ -37,13 +37,15 @@ limit = 0 join = "" where = "" datelimit = "AND DATEDIFF(NOW(), r.adddate) <= 7" -groupby = "GROUP BY LEFT(r.guid, 1)" -orderby = "ORDER BY LEFT(r.guid, 1) ASC" -rowlimit = "LIMIT 100" +groupby = "GROUP BY guidchar" +orderby = "ORDER BY guidchar ASC" +rowlimit = "LIMIT 16" +extrawhere = "AND r.prehashid = 0 AND r.nzbstatus = 1" +select = "DISTINCT LEFT(r.guid, 1) AS guidchar, COUNT(*) AS count" -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") run_perrun = cur[0].fetchone() threads = int(run_threads[0]) @@ -52,22 +54,23 @@ if threads > 16: maxperrun = int(run_perrun[0]) if sys.argv[1] == "md5": - join = "LEFT OUTER JOIN releasefiles rf ON r.id = rf.releaseid AND rf.ishashed = 1" + join = "LEFT OUTER JOIN release_files rf ON r.id = rf.releaseid AND rf.ishashed = 1" where = "r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0" elif sys.argv[1] == "nfo": where = "r.proc_nfo = 0 AND r.nfostatus = 1" elif sys.argv[1] == "filename": - join = "INNER JOIN releasefiles rf ON r.id = rf.releaseid" + join = "INNER JOIN release_files rf ON r.id = rf.releaseid" where = "r.proc_files = 0" elif sys.argv[1] == "par2": where = "r.proc_par2 = 0" elif sys.argv[1] == "miscsorter": - where = "r.nfostatus = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" + where = "r.nfostatus = 1 AND r.proc_nfo = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" elif sys.argv[1] == "predbft": + extrawhere = "" where = "1=1" rowlimit = "LIMIT %s" % (threads) -cur[0].execute("SELECT DISTINCT LEFT(r.guid, 1), COUNT(*) AS count FROM releases r %s WHERE %s AND r.prehashid = 0 AND r.nzbstatus = 1 %s %s %s %s" % (join, where, datelimit, groupby, orderby, rowlimit)) +cur[0].execute("SELECT %s FROM releases r %s WHERE %s %s %s %s %s" % (select, join, where, extrawhere, groupby, orderby, rowlimit)) datas = cur[0].fetchall() guids = int(len(datas)) @@ -99,7 +102,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/groupfixrelnames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/groupfixrelnames.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py b/misc/update_scripts/python/import_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/import_threaded.py rename to misc/update_scripts/python/import_threaded.py index f62d709e5..b1aaa953a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py +++ b/misc/update_scripts/python/import_threaded.py @@ -24,7 +24,7 @@ print(bcolors.HEADER + "\nNZB Import Threaded Started at {}".format(datetime.dat #get values from db cur[0].execute("SELECT value FROM tmux WHERE setting = 'import'") use_true = cur[0].fetchone() -cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) nzbs = dbgrab[0][1] @@ -57,7 +57,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/nzb-import-new.php", ""+my_id]) + subprocess.call(["php", pathname+"/lib/nzb-import.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -105,6 +105,8 @@ def main(args): my_queue.join() + final = "true" + subprocess.call(["php", pathname+"/../../testing/DB/populate_nzb_guid.php", ""+final]) print(bcolors.HEADER + "\nNZB Import Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php b/misc/update_scripts/python/lib/.gitignore.php similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php rename to misc/update_scripts/python/lib/.gitignore.php diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py b/misc/update_scripts/python/lib/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py rename to misc/update_scripts/python/lib/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/info.py b/misc/update_scripts/python/lib/info.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/info.py rename to misc/update_scripts/python/lib/info.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample b/misc/update_scripts/python/lib/nntpproxy.conf.sample similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample rename to misc/update_scripts/python/lib/nntpproxy.conf.sample diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore b/misc/update_scripts/python/lib/pynntp/.gitignore similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore rename to misc/update_scripts/python/lib/pynntp/.gitignore diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING b/misc/update_scripts/python/lib/pynntp/COPYING similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING rename to misc/update_scripts/python/lib/pynntp/COPYING diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE b/misc/update_scripts/python/lib/pynntp/NOTE similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE rename to misc/update_scripts/python/lib/pynntp/NOTE diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md b/misc/update_scripts/python/lib/pynntp/README.md similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md rename to misc/update_scripts/python/lib/pynntp/README.md diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py b/misc/update_scripts/python/lib/pynntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py b/misc/update_scripts/python/lib/pynntp/nntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/nntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py b/misc/update_scripts/python/lib/pynntp/nntp/date.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py rename to misc/update_scripts/python/lib/pynntp/nntp/date.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py b/misc/update_scripts/python/lib/pynntp/nntp/fifo.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py rename to misc/update_scripts/python/lib/pynntp/nntp/fifo.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py rename to misc/update_scripts/python/lib/pynntp/nntp/iodict.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py b/misc/update_scripts/python/lib/pynntp/nntp/nntp.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py rename to misc/update_scripts/python/lib/pynntp/nntp/nntp.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py b/misc/update_scripts/python/lib/pynntp/nntp/utils.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py rename to misc/update_scripts/python/lib/pynntp/nntp/utils.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py b/misc/update_scripts/python/lib/pynntp/nntp/yenc.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py rename to misc/update_scripts/python/lib/pynntp/nntp/yenc.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py b/misc/update_scripts/python/lib/pynntp/setup.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py rename to misc/update_scripts/python/lib/pynntp/setup.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py b/misc/update_scripts/python/nntpproxy.py similarity index 98% rename from misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py rename to misc/update_scripts/python/nntpproxy.py index 4dac64eba..104f8f189 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py +++ b/misc/update_scripts/python/nntpproxy.py @@ -27,7 +27,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): raise ValueError("Bad backend") nntp.NNTPClient.__init__(self, self.host, self.port, username, password, timeout=timeout, use_ssl=use_ssl) self.id = self.socket.getsockname()[1] - print(bcolors.PRIMARY + "New NNTP connection to %s established with id #%5d" % + print(bcolors.PRIMARY + "New NNTP connection to %s established with ID #%5d" % (self.host, self.id) + bcolors.ENDC) self._connected = True self.xfeature_compress_gzip() @@ -54,7 +54,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): return self._life def invalidate(self): - print(bcolors.PRIMARY + "Disconnecting from NNTP connection id #%5d after %d seconds." % + print(bcolors.PRIMARY + "Disconnecting from NNTP connection ID #%5d after %d seconds." % (self.id, (time.time() - self._start_time)) + bcolors.ENDC) self.close() self._connected = False @@ -68,7 +68,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): self._pool = None -# NNTP proxy request handler for newznab +# NNTP proxy request handler for nZEDb class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): def handle(self): @@ -196,7 +196,7 @@ class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): self.wfile.write("500 What?\r\n") -# NNTP proxy server for newznab +# NNTP proxy server for nZEDb class NNTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): allow_reuse_address = True diff --git a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py b/misc/update_scripts/python/postprocess_threaded.py similarity index 82% rename from misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py rename to misc/update_scripts/python/postprocess_threaded.py index 80768408c..dcf264553 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py +++ b/misc/update_scripts/python/postprocess_threaded.py @@ -19,7 +19,7 @@ conf = info.readConfig() cur = info.connect() if len(sys.argv) == 1: - print(bcolors.ERROR + "\nWrong set of arguments.\nThe first argument [additional, nfo, movie, clean] determines the postprocessing to do.\nThe optional second argument for [additional, nfo] [groupid, categoryid] allows to process only that group or category.\nThe optional second argument for [movies, tv] [clean] allows processing only properly renamed releases.\n\npython postprocess_threaded.py [additional, nfo] (optional [groupid, categoryid])\npython postprocess_threaded.py [movie, tv] (optional [clean])\n" + bcolors.ENDC) + print(bcolors.ERROR + "\nWrong set of arguments.\nThe first argument [additional, nfo, movie, clean] determines the postprocessing to do.\nThe optional second argument for [additional, nfo] [group_id, categoryid] allows to process only that group or category.\nThe optional second argument for [movies, tv] [clean] allows processing only properly renamed releases.\n\npython postprocess_threaded.py [additional, nfo] (optional [group_id, categoryid])\npython postprocess_threaded.py [movie, tv] (optional [clean])\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[2] == "clean": print(bcolors.HEADER + "\nPostProcess {} Clean Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) @@ -33,40 +33,40 @@ if sys.argv[1] == "additional": elif sys.argv[1] == "nfo": print(bcolors.HEADER + "* = hidden NFO, + = NFO, - = no NFO, f = download failed." + bcolors.ENDC) -# You can limit postprocessing for additional and nfo by groupid or categoryid +# You can limit postprocessing for additional and nfo by group_id or categoryid if len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) < 4: - groupid = 'AND groupid = '+sys.argv[2] - print(bcolors.HEADER + "Using groupid "+sys.argv[2] + bcolors.ENDC) + group_id = 'AND group_id = '+sys.argv[2] + print(bcolors.HEADER + "Using group_id "+sys.argv[2] + bcolors.ENDC) elif len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) == 4: if sys.argv[2] == '1000': - groupid = 'AND categoryid BETWEEN 1000 AND 1999' + group_id = 'AND categoryid BETWEEN 1000 AND 1999' print(bcolors.HEADER + "Using categoryids 1000-1999" + bcolors.ENDC) elif sys.argv[2] == '2000': - groupid = 'AND categoryid BETWEEN 2000 AND 2999' + group_id = 'AND categoryid BETWEEN 2000 AND 2999' print(bcolors.HEADER + "Using categoryids 2000-2999" + bcolors.ENDC) elif sys.argv[2] == '3000': - groupid = 'AND categoryid BETWEEN 3000 AND 3999' + group_id = 'AND categoryid BETWEEN 3000 AND 3999' print(bcolors.HEADER + "Using categoryids 3000-3999" + bcolors.ENDC) elif sys.argv[2] == '4000': - groupid = 'AND categoryid BETWEEN 4000 AND 4999' + group_id = 'AND categoryid BETWEEN 4000 AND 4999' print(bcolors.HEADER + "Using categoryids 4000-4999" + bcolors.ENDC) elif sys.argv[2] == '5000': - groupid = 'AND categoryid BETWEEN 5000 AND 5999' + group_id = 'AND categoryid BETWEEN 5000 AND 5999' print(bcolors.HEADER + "Using categoryids 5000-5999" + bcolors.ENDC) elif sys.argv[2] == '6000': - groupid = 'AND categoryid BETWEEN 6000 AND 6999' + group_id = 'AND categoryid BETWEEN 6000 AND 6999' print(bcolors.HEADER + "Using categoryids 6000-6999" + bcolors.ENDC) elif sys.argv[2] == '7000': - groupid = 'AND categoryid BETWEEN 7000 AND 7999' + group_id = 'AND categoryid BETWEEN 7000 AND 7999' print(bcolors.HEADER + "Using categoryids 7000-7999" + bcolors.ENDC) elif sys.argv[2] == '8000': - groupid = 'AND categoryid BETWEEN 8000 AND 8999' + group_id = 'AND categoryid BETWEEN 8000 AND 8999' print(bcolors.HEADER + "Using categoryids 8000-8999" + bcolors.ENDC) else: - groupid = 'AND categoryid = '+sys.argv[2] + group_id = 'AND categoryid = '+sys.argv[2] print(bcolors.HEADER + "Using categoryid "+sys.argv[2] + bcolors.ENDC) else: - groupid = '' + group_id = '' #you can sort tv releases by searchname if len(sys.argv) == 3 and (sys.argv[2] == "asc" or sys.argv[2] == "desc"): @@ -82,7 +82,7 @@ start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) > 1 and sys.argv[1] == "additional": - cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'postthreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+groupid+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+groupid+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+groupid+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+groupid+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+groupid+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+groupid+") as o") + cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'postthreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+group_id+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+group_id+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+group_id+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+group_id+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+group_id+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+group_id+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -91,7 +91,7 @@ if len(sys.argv) > 1 and sys.argv[1] == "additional": ps5 = format(int(dbgrab[0][13])) ps6 = format(int(dbgrab[0][14])) elif len(sys.argv) > 1 and sys.argv[1] == "nfo": - cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nfothreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 "+groupid+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 "+groupid+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 "+groupid+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 "+groupid+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 "+groupid+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 "+groupid+") as o") + cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nfothreads') AS a, (SELECT value FROM settings WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM settings WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM settings WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM settings WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM settings WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM settings WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 "+group_id+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 "+group_id+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 "+group_id+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 "+group_id+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 "+group_id+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 "+group_id+") as o") dbgrab = cur[0].fetchall() ps1 = format(int(dbgrab[0][9])) ps2 = format(int(dbgrab[0][10])) @@ -175,7 +175,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) time.sleep(.02) self.my_queue.task_done() @@ -232,7 +232,7 @@ def main(args): final = cur[0].fetchall() if len(datas) > 0: for item in final: - run = "DELETE FROM releasenfo WHERE nfo IS NULL AND releaseid = %s" + run = "DELETE FROM release_nfos WHERE nfo IS NULL AND releaseid = %s" cur[0].execute(run, (item[0])) final = cur[0].fetchall() @@ -243,4 +243,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py b/misc/update_scripts/python/releases_threaded.py similarity index 86% rename from misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py rename to misc/update_scripts/python/releases_threaded.py index 01112c8df..930500d73 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py +++ b/misc/update_scripts/python/releases_threaded.py @@ -21,7 +21,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nUpdate Releases Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasethreads') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasesthreads') AS b") dbgrab = cur[0].fetchall() allowed = int(dbgrab[0][0]) threads = int(dbgrab[0][1]) @@ -30,7 +30,7 @@ if allowed == 0: info.disconnect(cur[0], cur[1]) sys.exit() -cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'binaries_%'") +cur[0].execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '"+conf['DB_NAME']+"' AND table_rows > 0 AND table_name LIKE 'collections_%'") datas = cur[0].fetchall() #close connection to mysql @@ -60,7 +60,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "releases "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+my_id]) self.my_queue.task_done() def main(): @@ -88,15 +88,15 @@ def main(): if count >= threads: count = 0 count += 1 - my_queue.put("%s %s" % (release[0].replace('binaries_', ''), count)) + my_queue.put("%s %s" % (release[0].replace('collections_', ''), count)) my_queue.join() #stage7b - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py b/misc/update_scripts/python/requestid_threaded.py similarity index 87% rename from misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py rename to misc/update_scripts/python/requestid_threaded.py index 840096a62..0d6b4ec01 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py +++ b/misc/update_scripts/python/requestid_threaded.py @@ -35,7 +35,7 @@ print(bcolors.HEADER + "\n\nRequestID Threaded Started at {}".format(datetime.da cur[0].execute("SELECT value FROM settings WHERE setting = 'request_hours'") dbgrab = cur[0].fetchone() request_hours = str(dbgrab[0]) -cur[0].execute("SELECT DISTINCT(g.id) FROM releases r INNER JOIN groups g ON r.groupid = g.id WHERE r.nzbstatus = 1 AND r.prehashid = 0 AND r.isrequestid = 1 AND r.reqidstatus in (0, -1) OR (r.reqidstatus = -3 AND r.adddate > NOW() - INTERVAL " + request_hours + " HOUR)") +cur[0].execute("SELECT DISTINCT(g.id) FROM releases r INNER JOIN groups g ON r.group_id = g.id WHERE r.nzbstatus = 1 AND r.prehashid = 0 AND r.isrequestid = 1 AND r.reqidstatus in (0, -1) OR (r.reqidstatus = -3 AND r.adddate > NOW() - INTERVAL " + request_hours + " HOUR)") datas = cur[0].fetchall() #close connection to mysql @@ -65,7 +65,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -98,4 +98,4 @@ def main(): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py b/misc/update_scripts/python/update_threaded.py similarity index 90% rename from misc/update_scripts/nix_scripts/tmux/python/update_threaded.py rename to misc/update_scripts/python/update_threaded.py index c88be1fb4..595cbf041 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py +++ b/misc/update_scripts/python/update_threaded.py @@ -18,7 +18,7 @@ conf = info.readConfig() cur = info.connect() start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) -cur[0].execute("SELECT value FROM settings WHERE setting = 'releasethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'releasesthreads'") threads = cur[0].fetchone() threads = int(threads[0]) @@ -61,7 +61,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) self.my_queue.task_done() def main(): @@ -95,10 +95,10 @@ def main(): #stage7b final = "final" - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/newznab/controllers/Tmux.php b/newznab/controllers/Tmux.php index ddb8b4e14..e0a1265b4 100644 --- a/newznab/controllers/Tmux.php +++ b/newznab/controllers/Tmux.php @@ -70,7 +70,7 @@ class Tmux $runVar['connections']['ip_a'] = gethostbyname($runVar['connections']['host_a']); } } else { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); @@ -83,7 +83,7 @@ class Tmux } } if ($constants['alternate_nntp']) { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy_a.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); diff --git a/newznab/processing/ProcessReleases.php b/newznab/processing/ProcessReleases.php index f8eea1f7a..1e7211237 100644 --- a/newznab/processing/ProcessReleases.php +++ b/newznab/processing/ProcessReleases.php @@ -190,7 +190,7 @@ class ProcessReleases if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request id Threaded lookup.")); } - passthru("$PYTHON ${DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py"); + passthru("$PYTHON ${DIR}update_scripts/nix_scripts/requestid_threaded.py"); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( diff --git a/www/templates/omicron/views/frontend/viewnzb.tpl b/www/templates/omicron/views/frontend/viewnzb.tpl index 7614d261c..e4f18bf34 100644 --- a/www/templates/omicron/views/frontend/viewnzb.tpl +++ b/www/templates/omicron/views/frontend/viewnzb.tpl @@ -477,8 +477,7 @@