diff --git a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php index ea29720d7..61bb97763 100644 --- a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php +++ b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php @@ -7,7 +7,7 @@ use newznab\processing\PostProcess; $pdo = new Settings(); if (!isset($argv[1])) { - exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py.")); + exit($pdo->log->error("This script is not intended to be run manually, it is called from Forking.php.")); } else if (isset($argv[1])) { $namefixer = new \NameFixer(['Settings' => $pdo]); $pieces = explode(' ', $argv[1]); diff --git a/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php b/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php deleted file mode 100644 index 82559ab2d..000000000 --- a/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php +++ /dev/null @@ -1,68 +0,0 @@ - $pdo]); - if ($nntp->doConnect() !== true) { - exit($pdo->log->error("Unable to connect to usenet.")); - } -} -if ($pdo->getSetting('tablepergroup') === 1) { - exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update_scripts/nix_scripts/multiprocessing/releases.php")); -} - -$groupName = isset($argv[3]) ? $argv[3] : ''; -if (isset($argv[1]) && isset($argv[2])) { - $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); - $releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]); - if ($argv[1] == 1 && $argv[2] == 'true') { - $releases->processReleases(1, 1, $groupName, $nntp, true); - } else if ($argv[1] == 1 && $argv[2] == 'false') { - $releases->processReleases(1, 2, $groupName, $nntp, true); - } else if ($argv[1] == 2 && $argv[2] == 'true') { - $releases->processReleases(2, 1, $groupName, $nntp, true); - } else if ($argv[1] == 2 && $argv[2] == 'false') { - $releases->processReleases(2, 2, $groupName, $nntp, true); - } else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) { - echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient."); - $releases->resetCategorize(); - } else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) { - echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient."); - $timestart = TIME(); - $relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryid = 8010'); - $time = $consoletools->convertTime(TIME() - $timestart); - echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject."); - } else if ($argv[1] == 6 && $argv[2] == 'true') { - echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient."); - $timestart = TIME(); - $relcount = $releases->categorizeRelease('searchname', ''); - $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); - $time = $consoletools->convertTime(TIME() - $timestart); - echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); - } else if ($argv[1] == 6 && $argv[2] == 'false') { - echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient."); - $timestart = TIME(); - $relcount = $releases->categorizeRelease('searchname', 'WHERE categoryid IN (1090, 2020, 3050, 5050, 6050, 8010)'); - $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]); - $time = $consoletools->convertTime(TIME() - $timestart); - echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); - } else { - exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments.")); - } -} else { - exit($pdo->log->error("\nWrong set of arguments.\n" - . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n" - . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n" - . "\nYou must pass a second argument whether to post process or not, true or false\n" - . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n" - . "\nExtra commands::\n" - . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n" - . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n" - . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n" - . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n")); -} \ No newline at end of file diff --git a/misc/update_scripts/nntpproxy.php b/misc/update_scripts/nntpproxy.php index fee92abef..7cebf6ca8 100644 --- a/misc/update_scripts/nntpproxy.php +++ b/misc/update_scripts/nntpproxy.php @@ -39,7 +39,7 @@ function window_proxy($tmux_session, $powerline) if ($powerline === '1') { $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf"; } else { - $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf"; + $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/conf/tmux.conf"; } $nntpproxy = $pdo->getSetting('nntpproxy'); diff --git a/misc/update_scripts/python/.gitignore b/misc/update_scripts/python/.gitignore new file mode 100644 index 000000000..1af04760d --- /dev/null +++ b/misc/update_scripts/python/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.pyo +/.idea \ No newline at end of file diff --git a/misc/update_scripts/python/README.txt b/misc/update_scripts/python/README.txt new file mode 100644 index 000000000..3a2edc3b3 --- /dev/null +++ b/misc/update_scripts/python/README.txt @@ -0,0 +1,7 @@ +Requires Python 2.7, with the cymysql module. + +Run like this : + +python -OOu binaries_threaded.py +-or- +python3 -OOu binaries_threaded.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py b/misc/update_scripts/python/backfill_safe_threaded.py similarity index 77% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py rename to misc/update_scripts/python/backfill_safe_threaded.py index 0f3d1bc34..93486cfdb 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py +++ b/misc/update_scripts/python/backfill_safe_threaded.py @@ -22,23 +22,23 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1") +cur[0].execute("SELECT g.name FROM groups g LEFT JOIN short_groups ON short_groups.name = g.name WHERE short_groups.name IS NULL AND backfill = 1") dorun = cur[0].fetchone() #close connection to mysql info.disconnect(cur[0], cur[1]) if dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) else: cur = info.connect() - cur[0].execute("SELECT name FROM shortgroups") + cur[0].execute("SELECT name FROM short_groups") dorun = cur[0].fetchone() info.disconnect(cur[0], cur[1]) if len(sys.argv) > 1 and sys.argv[1] not in dorun: - #before we get the groups, lets update shortgroups - subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) + #before we get the groups, lets update short_groups + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) count = 0 previous = "'alt.binaries.crap'" @@ -78,13 +78,13 @@ while count < 10000: #query to grab backfill groups if len(sys.argv) == 1: - if conf['DB_TYPE'] == "mysql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) - elif conf['DB_TYPE'] == "pgsql": - cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) + if conf['DB_SYSTEM'] == "mysql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group)) + elif conf['DB_SYSTEM'] == "pgsql": + cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups)) datas = cur[0].fetchone() else: - run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" + run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1" cur[0].execute(run, (sys.argv[1])) datas = cur[0].fetchone() if not datas or datas[0] is None: @@ -119,7 +119,7 @@ while count < 10000: print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC) print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC) group = ("{} {}".format(datas[0], count)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) #calculate the number of items for queue if (count > (backfill_qty * run_threads)): @@ -147,7 +147,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -177,7 +177,7 @@ def main(args): my_queue.join() group = ("{} {}".format(datas[0], 1000)) - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)]) if run_threads <= geteach: print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC) else: @@ -188,4 +188,4 @@ def main(args): if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py b/misc/update_scripts/python/backfill_threaded.py similarity index 92% rename from misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py rename to misc/update_scripts/python/backfill_threaded.py index 70a5816f5..9da0f5506 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py +++ b/misc/update_scripts/python/backfill_threaded.py @@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all": # removing the % before the variables at the end of the query adds quotes/escapes strings cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group)) else: - if conf['DB_TYPE'] == "mysql": + if conf['DB_SYSTEM'] == "mysql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) - elif conf['DB_TYPE'] == "pgsql": + elif conf['DB_SYSTEM'] == "pgsql": cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups)) datas = cur[0].fetchall() @@ -99,9 +99,9 @@ class queue_runner(threading.Thread): if my_id: time_of_last_run = time.time() if len(sys.argv) > 1 and sys.argv[1] == "all": - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id]) else: - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -135,4 +135,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py b/misc/update_scripts/python/binaries_safe_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py rename to misc/update_scripts/python/binaries_safe_threaded.py index d6b034f8b..2e2d1729a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py +++ b/misc/update_scripts/python/binaries_safe_threaded.py @@ -30,12 +30,12 @@ dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) maxmssgs = int(dbgrab[0][1]) -#before we get the groups, lets update shortgroups -subprocess.call(["php", pathname+"/../bin/update_groups.php", ""]) +#before we get the groups, lets update short_groups +subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""]) #query to grab all active groups cur = info.connect() -cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN shortgroups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") +cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC") datas = cur[0].fetchall() #close connection to mysql @@ -64,7 +64,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -121,4 +121,4 @@ def main(): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py b/misc/update_scripts/python/binaries_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py rename to misc/update_scripts/python/binaries_threaded.py index 879be3be4..954d581e4 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py +++ b/misc/update_scripts/python/binaries_threaded.py @@ -32,7 +32,6 @@ if len(sys.argv) == 1: print(bcolors.HEADER + "\nBinaries Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) - #get active groups if len(sys.argv) == 2: try: @@ -51,18 +50,17 @@ else: datas = cur[0].fetchall() if len(datas) == 0: - print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) - info.disconnect(cur[0], cur[1]) - sys.exit + print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC) + info.disconnect(cur[0], cur[1]) + sys.exit -cur[0].execute("SELECT value FROM tmux WHERE setting = 'binarythreads'") -dbgrab = cur[0].fetchone() +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'binarythreads') AS a") +dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) #close connection to mysql info.disconnect(cur[0], cur[1]) - my_queue = queue.Queue() time_of_last_run = time.time() @@ -83,7 +81,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/update_binaries.php", ""+my_id]) + subprocess.call(["php", pathname+"/../update_binaries.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py b/misc/update_scripts/python/fixreleasenames_threaded.py similarity index 61% rename from misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py rename to misc/update_scripts/python/fixreleasenames_threaded.py index c27d86e8d..09c0f9731 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py +++ b/misc/update_scripts/python/fixreleasenames_threaded.py @@ -22,16 +22,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) if len(sys.argv) == 1: print(bcolors.ERROR + "\nAn argument is required\n\n" - + "python " + sys.argv[0] + " [md5, nfo, filename, par2] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2].\n" + + "python " + sys.argv[0] + " [md5, nfo, filename, par2, miscsorter] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2, miscsorter].\n" + "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashid, using [nfo, filename, par2].\n" + "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n" + "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n" - + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + bcolors.ENDC) sys.exit() -if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft": - print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC) +if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "miscsorter" and sys.argv[1] != "predbft": + print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, miscsorter, predbft]\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": @@ -39,47 +39,48 @@ if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "clean": clean = " isrenamed = 0 AND proc_files = 1 AND proc_nfo = 1 " elif len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " elif len(sys.argv) == 3 and sys.argv[1] == "filename" and sys.argv[2] == "prehashid": - clean = " prehashid = IS NULL " + clean = " prehashid = 0 " else: clean = " isrenamed = 0 " print(bcolors.HEADER + "\nfixReleasesNames {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") perrun = cur[0].fetchone() datas = [] -maxtries = 0 if len(sys.argv) > 1 and sys.argv[1] == "nfo": - run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() +elif len(sys.argv) > 1 and sys.argv[1] == "miscsorter": + run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_sorter = 0 AND isrenamed = 0 ORDER BY postdate DESC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "filename"): - run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN releasefiles relfiles ON (relfiles.releaseid = rel.id) WHERE categoryid IN (8010, 8020) AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN release_files relfiles ON (relfiles.releaseid = rel.id) WHERE nzbstatus = 1 AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "md5"): - while len(datas) == 0 and maxtries >= -5: - run = "SELECT DISTINCT rel.id FROM releases rel INNER JOIN releasefiles rf ON rel.id = rf.releaseid WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashid = 0 ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0]))) - datas = cur[0].fetchall() - maxtries = maxtries - 1 + run = "SELECT DISTINCT rel.id FROM releases rel LEFT OUTER JOIN release_files rf ON rel.id = rf.releaseid AND rf.ishashed = 1 WHERE nzbstatus = 1 AND rel.dehashstatus BETWEEN -6 AND 0 AND rel.ishashed = 1 AND prehashid = 0 ORDER BY dehashstatus DESC, postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0])*int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "par2"): #This one does from oldest posts to newest posts, since nfo pp does same thing but newest to oldest - run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE categoryid IN (8010, 8020) AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) - datas = cur[0].fetchall() + run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE nzbstatus = 1 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) + datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"): #This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes - run = "SELECT id AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s" - cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0]))) + run = "SELECT id AS prehashid FROM prehash WHERE LENGTH(title) >= 15 AND title NOT REGEXP '[\"\<\> ]' AND searched = 0 AND DATEDIFF(NOW(), predate) > 1 ORDER BY predate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) datas = cur[0].fetchall() #close connection to mysql @@ -109,7 +110,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/fixreleasenames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/fixreleasenames.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -117,11 +118,7 @@ def main(): global time_of_last_run time_of_last_run = time.time() - if sys.argv[1] == 'md5': - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. dehashstatus range {} to 0".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1], maxtries + 1) + bcolors.ENDC) - else: - print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) - time.sleep(2) + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC) def signal_handler(signal, frame): sys.exit(0) @@ -152,6 +149,10 @@ def main(): for release in datas: time.sleep(.03) my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2])) + elif sys.argv[1] == "miscsorter": + for release in datas: + time.sleep(.03) + my_queue.put("%s %s" % ("miscsorter", release[0])) elif sys.argv[1] == "predbft": for release in datas: time.sleep(.03) diff --git a/misc/update_scripts/python/grabheaders_threaded.py b/misc/update_scripts/python/grabheaders_threaded.py new file mode 100644 index 000000000..38eeb3a16 --- /dev/null +++ b/misc/update_scripts/python/grabheaders_threaded.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, time +import threading +try: + import queue +except ImportError: + import Queue as queue +import subprocess +import string +import signal +import datetime + +import lib.info as info +from lib.info import bcolors +conf = info.readConfig() +start_time = time.time() +pathname = os.path.abspath(os.path.dirname(sys.argv[0])) +threads = 10 + +print(bcolors.HEADER + "\nGrab Headers Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + +if len(sys.argv) > 1: + datas = [] + datas.append(sys.argv[1]) +else: + datas = ("alt.binaries.teevee", "alt.binaries.tv", "alt.binaries.audiobooks", "alt.binaries.moovee", "alt.binaries.e-book", "alt.binaries.e-book.technical", "alt.binaries.ebook", "alt.binaries.e-book.magazines") +if not datas: + print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC) + sys.exit() + +my_queue = queue.Queue() +time_of_last_run = time.time() + +class queue_runner(threading.Thread): + def __init__(self, my_queue): + threading.Thread.__init__(self) + self.my_queue = my_queue + + def run(self): + global time_of_last_run + + while True: + try: + my_id = self.my_queue.get(True, 1) + except: + if time.time() - time_of_last_run > 3: + return + else: + if my_id: + time_of_last_run = time.time() + subprocess.call(["php", pathname+"/../../testing/Regex/grabheaders.php", ""+my_id]) + self.my_queue.task_done() + +def main(): + global time_of_last_run + time_of_last_run = time.time() + + print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC) + time.sleep(2) + def signal_handler(signal, frame): + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + if True: + #spawn a pool of place worker threads + for i in range(threads): + p = queue_runner(my_queue) + p.setDaemon(False) + p.start() + + #now load some arbitrary jobs into the queue + count = 0 + for group in datas: + my_queue.put(group) + + my_queue.join() + + print(bcolors.HEADER + "\nGrab Headers Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) + print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) + +if __name__ == '__main__': + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py b/misc/update_scripts/python/groupfixrelnames_threaded.py similarity index 81% rename from misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py rename to misc/update_scripts/python/groupfixrelnames_threaded.py index f89cdecaf..067bc69f8 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py +++ b/misc/update_scripts/python/groupfixrelnames_threaded.py @@ -37,13 +37,15 @@ limit = 0 join = "" where = "" datelimit = "AND DATEDIFF(NOW(), r.adddate) <= 7" -groupby = "GROUP BY LEFT(r.guid, 1)" -orderby = "ORDER BY LEFT(r.guid, 1) ASC" -rowlimit = "LIMIT 100" +groupby = "GROUP BY guidchar" +orderby = "ORDER BY guidchar ASC" +rowlimit = "LIMIT 16" +extrawhere = "AND r.prehashid = 0 AND r.nzbstatus = 1" +select = "DISTINCT LEFT(r.guid, 1) AS guidchar, COUNT(*) AS count" -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'") run_threads = cur[0].fetchone() -cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'") run_perrun = cur[0].fetchone() threads = int(run_threads[0]) @@ -52,22 +54,23 @@ if threads > 16: maxperrun = int(run_perrun[0]) if sys.argv[1] == "md5": - join = "LEFT OUTER JOIN releasefiles rf ON r.id = rf.releaseid AND rf.ishashed = 1" + join = "LEFT OUTER JOIN release_files rf ON r.id = rf.releaseid AND rf.ishashed = 1" where = "r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0" elif sys.argv[1] == "nfo": where = "r.proc_nfo = 0 AND r.nfostatus = 1" elif sys.argv[1] == "filename": - join = "INNER JOIN releasefiles rf ON r.id = rf.releaseid" + join = "INNER JOIN release_files rf ON r.id = rf.releaseid" where = "r.proc_files = 0" elif sys.argv[1] == "par2": where = "r.proc_par2 = 0" elif sys.argv[1] == "miscsorter": - where = "r.nfostatus = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" + where = "r.nfostatus = 1 AND r.proc_nfo = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0" elif sys.argv[1] == "predbft": + extrawhere = "" where = "1=1" rowlimit = "LIMIT %s" % (threads) -cur[0].execute("SELECT DISTINCT LEFT(r.guid, 1), COUNT(*) AS count FROM releases r %s WHERE %s AND r.prehashid = 0 AND r.nzbstatus = 1 %s %s %s %s" % (join, where, datelimit, groupby, orderby, rowlimit)) +cur[0].execute("SELECT %s FROM releases r %s WHERE %s %s %s %s %s" % (select, join, where, extrawhere, groupby, orderby, rowlimit)) datas = cur[0].fetchall() guids = int(len(datas)) @@ -99,7 +102,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/groupfixrelnames.php", ""+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/groupfixrelnames.php", ""+my_id]) self.my_queue.task_done() def main(): diff --git a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py b/misc/update_scripts/python/import_threaded.py similarity index 91% rename from misc/update_scripts/nix_scripts/tmux/python/import_threaded.py rename to misc/update_scripts/python/import_threaded.py index f62d709e5..b1aaa953a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py +++ b/misc/update_scripts/python/import_threaded.py @@ -24,7 +24,7 @@ print(bcolors.HEADER + "\nNZB Import Threaded Started at {}".format(datetime.dat #get values from db cur[0].execute("SELECT value FROM tmux WHERE setting = 'import'") use_true = cur[0].fetchone() -cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b") dbgrab = cur[0].fetchall() run_threads = int(dbgrab[0][0]) nzbs = dbgrab[0][1] @@ -57,7 +57,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../bin/nzb-import-new.php", ""+my_id]) + subprocess.call(["php", pathname+"/lib/nzb-import.php", ""+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -105,6 +105,8 @@ def main(args): my_queue.join() + final = "true" + subprocess.call(["php", pathname+"/../../testing/DB/populate_nzb_guid.php", ""+final]) print(bcolors.HEADER + "\nNZB Import Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php b/misc/update_scripts/python/lib/.gitignore.php similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php rename to misc/update_scripts/python/lib/.gitignore.php diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py b/misc/update_scripts/python/lib/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py rename to misc/update_scripts/python/lib/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/info.py b/misc/update_scripts/python/lib/info.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/info.py rename to misc/update_scripts/python/lib/info.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample b/misc/update_scripts/python/lib/nntpproxy.conf.sample similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample rename to misc/update_scripts/python/lib/nntpproxy.conf.sample diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore b/misc/update_scripts/python/lib/pynntp/.gitignore similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore rename to misc/update_scripts/python/lib/pynntp/.gitignore diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING b/misc/update_scripts/python/lib/pynntp/COPYING similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING rename to misc/update_scripts/python/lib/pynntp/COPYING diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE b/misc/update_scripts/python/lib/pynntp/NOTE similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE rename to misc/update_scripts/python/lib/pynntp/NOTE diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md b/misc/update_scripts/python/lib/pynntp/README.md similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md rename to misc/update_scripts/python/lib/pynntp/README.md diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py b/misc/update_scripts/python/lib/pynntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py b/misc/update_scripts/python/lib/pynntp/nntp/__init__.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py rename to misc/update_scripts/python/lib/pynntp/nntp/__init__.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py b/misc/update_scripts/python/lib/pynntp/nntp/date.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py rename to misc/update_scripts/python/lib/pynntp/nntp/date.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py b/misc/update_scripts/python/lib/pynntp/nntp/fifo.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py rename to misc/update_scripts/python/lib/pynntp/nntp/fifo.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py similarity index 98% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py rename to misc/update_scripts/python/lib/pynntp/nntp/iodict.py index f8b3a08b9..55e3030c7 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py +++ b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py @@ -19,7 +19,7 @@ along with this program. If not, see . #NOTE: docstrings beed to be added -from collections import OrderedDict +from binaries import OrderedDict def _lower(v): """assumes that classes that inherit list, tuple or dict have a constructor diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py b/misc/update_scripts/python/lib/pynntp/nntp/nntp.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/nntp.py rename to misc/update_scripts/python/lib/pynntp/nntp/nntp.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py b/misc/update_scripts/python/lib/pynntp/nntp/utils.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/utils.py rename to misc/update_scripts/python/lib/pynntp/nntp/utils.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py b/misc/update_scripts/python/lib/pynntp/nntp/yenc.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/yenc.py rename to misc/update_scripts/python/lib/pynntp/nntp/yenc.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py b/misc/update_scripts/python/lib/pynntp/setup.py similarity index 100% rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/setup.py rename to misc/update_scripts/python/lib/pynntp/setup.py diff --git a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py b/misc/update_scripts/python/nntpproxy.py similarity index 98% rename from misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py rename to misc/update_scripts/python/nntpproxy.py index 4dac64eba..104f8f189 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/nntpproxy.py +++ b/misc/update_scripts/python/nntpproxy.py @@ -27,7 +27,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): raise ValueError("Bad backend") nntp.NNTPClient.__init__(self, self.host, self.port, username, password, timeout=timeout, use_ssl=use_ssl) self.id = self.socket.getsockname()[1] - print(bcolors.PRIMARY + "New NNTP connection to %s established with id #%5d" % + print(bcolors.PRIMARY + "New NNTP connection to %s established with ID #%5d" % (self.host, self.id) + bcolors.ENDC) self._connected = True self.xfeature_compress_gzip() @@ -54,7 +54,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): return self._life def invalidate(self): - print(bcolors.PRIMARY + "Disconnecting from NNTP connection id #%5d after %d seconds." % + print(bcolors.PRIMARY + "Disconnecting from NNTP connection ID #%5d after %d seconds." % (self.id, (time.time() - self._start_time)) + bcolors.ENDC) self.close() self._connected = False @@ -68,7 +68,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient): self._pool = None -# NNTP proxy request handler for newznab +# NNTP proxy request handler for nZEDb class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): def handle(self): @@ -196,7 +196,7 @@ class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler): self.wfile.write("500 What?\r\n") -# NNTP proxy server for newznab +# NNTP proxy server for nZEDb class NNTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): allow_reuse_address = True diff --git a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py b/misc/update_scripts/python/postprocess_threaded.py similarity index 98% rename from misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py rename to misc/update_scripts/python/postprocess_threaded.py index 80768408c..d76071814 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/postprocess_threaded.py +++ b/misc/update_scripts/python/postprocess_threaded.py @@ -175,7 +175,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python pp_"+my_id]) time.sleep(.02) self.my_queue.task_done() @@ -232,7 +232,7 @@ def main(args): final = cur[0].fetchall() if len(datas) > 0: for item in final: - run = "DELETE FROM releasenfo WHERE nfo IS NULL AND releaseid = %s" + run = "DELETE FROM release_nfos WHERE nfo IS NULL AND releaseid = %s" cur[0].execute(run, (item[0])) final = cur[0].fetchall() @@ -243,4 +243,4 @@ def main(args): print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py b/misc/update_scripts/python/releases_threaded.py similarity index 89% rename from misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py rename to misc/update_scripts/python/releases_threaded.py index 01112c8df..d12f3bd9a 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/releases_threaded.py +++ b/misc/update_scripts/python/releases_threaded.py @@ -21,7 +21,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0])) print(bcolors.HEADER + "\nUpdate Releases Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) -cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasethreads') AS b") +cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasesthreads') AS b") dbgrab = cur[0].fetchall() allowed = int(dbgrab[0][0]) threads = int(dbgrab[0][1]) @@ -60,7 +60,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "releases "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+my_id]) self.my_queue.task_done() def main(): @@ -93,10 +93,10 @@ def main(): my_queue.join() #stage7b - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py b/misc/update_scripts/python/requestid_threaded.py similarity index 80% rename from misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py rename to misc/update_scripts/python/requestid_threaded.py index 14dc810ef..c418701f8 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/requestid_threaded.py +++ b/misc/update_scripts/python/requestid_threaded.py @@ -15,9 +15,9 @@ except ImportError: import Queue as queue try: - import urllib2 + import urllib2 except ImportError: - import urllib.request as urllib2 + import urllib.request as urllib2 import lib.info as info from lib.info import bcolors @@ -25,22 +25,10 @@ conf = info.readConfig() cur = info.connect() start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) -threads = 5 -web = None +cur[0].execute("SELECT value FROM settings WHERE setting = 'reqidthreads'") +threads = cur[0].fetchone() +threads = int(threads[0]) -try: - import urllib -except ImportError: - import urllib.request as httplib - -try: - r1 = urllib.urlopen("http://reqid.nzedb.com").getcode() - if r1 == 200: - web = True - else: - web = False -except: - web = False print(bcolors.HEADER + "\n\nRequestID Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) @@ -77,7 +65,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id]) time.sleep(.03) self.my_queue.task_done() @@ -102,8 +90,7 @@ def main(): #now load some arbitrary jobs into the queue for release in datas: - time.sleep(.03) - my_queue.put("%s %s %s %s" % (release[0], release[1], release[2], web)) + my_queue.put("%s" % (release[0])) my_queue.join() diff --git a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py b/misc/update_scripts/python/update_threaded.py similarity index 90% rename from misc/update_scripts/nix_scripts/tmux/python/update_threaded.py rename to misc/update_scripts/python/update_threaded.py index c88be1fb4..595cbf041 100644 --- a/misc/update_scripts/nix_scripts/tmux/python/update_threaded.py +++ b/misc/update_scripts/python/update_threaded.py @@ -18,7 +18,7 @@ conf = info.readConfig() cur = info.connect() start_time = time.time() pathname = os.path.abspath(os.path.dirname(sys.argv[0])) -cur[0].execute("SELECT value FROM settings WHERE setting = 'releasethreads'") +cur[0].execute("SELECT value FROM settings WHERE setting = 'releasesthreads'") threads = cur[0].fetchone() threads = int(threads[0]) @@ -61,7 +61,7 @@ class queue_runner(threading.Thread): else: if my_id: time_of_last_run = time.time() - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id]) self.my_queue.task_done() def main(): @@ -95,10 +95,10 @@ def main(): #stage7b final = "final" - subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) + subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"]) print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC) print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/misc/update_scripts/update_releases.php b/misc/update_scripts/update_releases.php index debc19630..6ebce1518 100644 --- a/misc/update_scripts/update_releases.php +++ b/misc/update_scripts/update_releases.php @@ -1,6 +1,68 @@ processReleases(1, 1, $groupName, $nntp, true); +$pdo = new Settings(); + +if (isset($argv[2]) && $argv[2] === 'true') { + // Create the connection here and pass + $nntp = new NNTP(['Settings' => $pdo]); + if ($nntp->doConnect() !== true) { + exit($pdo->log->error("Unable to connect to usenet.")); + } +} +if ($pdo->getSetting('tablepergroup') === 1) { + exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update_scripts/nix_scripts/multiprocessing/releases.php")); +} + +$groupName = isset($argv[3]) ? $argv[3] : ''; +if (isset($argv[1]) && isset($argv[2])) { + $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); + $releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]); + if ($argv[1] == 1 && $argv[2] == 'true') { + $releases->processReleases(1, 1, $groupName, $nntp, true); + } else if ($argv[1] == 1 && $argv[2] == 'false') { + $releases->processReleases(1, 2, $groupName, $nntp, true); + } else if ($argv[1] == 2 && $argv[2] == 'true') { + $releases->processReleases(2, 1, $groupName, $nntp, true); + } else if ($argv[1] == 2 && $argv[2] == 'false') { + $releases->processReleases(2, 2, $groupName, $nntp, true); + } else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) { + echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient."); + $releases->resetCategorize(); + } else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) { + echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryid = 8010'); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject."); + } else if ($argv[1] == 6 && $argv[2] == 'true') { + echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('searchname', ''); + $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); + } else if ($argv[1] == 6 && $argv[2] == 'false') { + echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient."); + $timestart = TIME(); + $relcount = $releases->categorizeRelease('searchname', 'WHERE categoryid IN (1090, 2020, 3050, 5050, 6050, 8010)'); + $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); + $time = $consoletools->convertTime(TIME() - $timestart); + echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name."); + } else { + exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments.")); + } +} else { + exit($pdo->log->error("\nWrong set of arguments.\n" + . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n" + . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n" + . "\nYou must pass a second argument whether to post process or not, true or false\n" + . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n" + . "\nExtra commands::\n" + . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n" + . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n" + . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n" + . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n")); +} \ No newline at end of file diff --git a/newznab/controllers/Tmux.php b/newznab/controllers/Tmux.php index 73291e348..dfb3164bd 100644 --- a/newznab/controllers/Tmux.php +++ b/newznab/controllers/Tmux.php @@ -76,7 +76,7 @@ class Tmux $runVar['connections']['ip_a'] = gethostbyname($runVar['connections']['host_a']); } } else { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); @@ -89,7 +89,7 @@ class Tmux } } if ($constants['alternate_nntp']) { - $filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf"; + $filename = NN_MISC . "update_scripts/python/lib/nntpproxy_a.conf"; $fp = fopen($filename, "r") || die("Couldn't open $filename"); while (!feof($fp)) { $line = fgets($fp); diff --git a/www/templates/omicron/views/frontend/viewnzb.tpl b/www/templates/omicron/views/frontend/viewnzb.tpl index 7614d261c..e4f18bf34 100644 --- a/www/templates/omicron/views/frontend/viewnzb.tpl +++ b/www/templates/omicron/views/frontend/viewnzb.tpl @@ -477,8 +477,7 @@ - RAR - Contains + RAR Contains Files:
diff --git a/www/templates_shared/views/admin/site-edit.tpl b/www/templates_shared/views/admin/site-edit.tpl index c09051974..892db2a15 100644 --- a/www/templates_shared/views/admin/site-edit.tpl +++ b/www/templates_shared/views/admin/site-edit.tpl @@ -1440,7 +1440,7 @@ reduces time for connection setup/teardown. The proxy also takes care of compressed headers for you. To use this featrure you will need to install socketpool (sudo pip install socketpool or sudo easy_install socketpool) (ensure python2 is default) and edit the configuration file (nntpproxy.conf and nntpproxy_a.conf) in the - update_scripts/nix_scripts/tmux/python/lib (copy sample) directory and finally edit your www/config.php file + update_scripts/python/lib (copy sample) directory and finally edit your www/config.php file to use the proxy (username and password are ignored by the proxy - make then anything you like - the proxy doesn't use ssl either). Make sure you turn off the use compressed headers option here in site preferences (the proxy uses compressed headers by default and passes on decompressed data).