diff --git a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php
index ea29720d7..61bb97763 100644
--- a/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php
+++ b/misc/update_scripts/nix_scripts/tmux/bin/groupfixrelnames.php
@@ -7,7 +7,7 @@ use newznab\processing\PostProcess;
$pdo = new Settings();
if (!isset($argv[1])) {
- exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py."));
+ exit($pdo->log->error("This script is not intended to be run manually, it is called from Forking.php."));
} else if (isset($argv[1])) {
$namefixer = new \NameFixer(['Settings' => $pdo]);
$pieces = explode(' ', $argv[1]);
diff --git a/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php b/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php
deleted file mode 100644
index 82559ab2d..000000000
--- a/misc/update_scripts/nix_scripts/tmux/bin/update_releases.php
+++ /dev/null
@@ -1,68 +0,0 @@
- $pdo]);
- if ($nntp->doConnect() !== true) {
- exit($pdo->log->error("Unable to connect to usenet."));
- }
-}
-if ($pdo->getSetting('tablepergroup') === 1) {
- exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update_scripts/nix_scripts/multiprocessing/releases.php"));
-}
-
-$groupName = isset($argv[3]) ? $argv[3] : '';
-if (isset($argv[1]) && isset($argv[2])) {
- $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
- $releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]);
- if ($argv[1] == 1 && $argv[2] == 'true') {
- $releases->processReleases(1, 1, $groupName, $nntp, true);
- } else if ($argv[1] == 1 && $argv[2] == 'false') {
- $releases->processReleases(1, 2, $groupName, $nntp, true);
- } else if ($argv[1] == 2 && $argv[2] == 'true') {
- $releases->processReleases(2, 1, $groupName, $nntp, true);
- } else if ($argv[1] == 2 && $argv[2] == 'false') {
- $releases->processReleases(2, 2, $groupName, $nntp, true);
- } else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) {
- echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient.");
- $releases->resetCategorize();
- } else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) {
- echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient.");
- $timestart = TIME();
- $relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryid = 8010');
- $time = $consoletools->convertTime(TIME() - $timestart);
- echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject.");
- } else if ($argv[1] == 6 && $argv[2] == 'true') {
- echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient.");
- $timestart = TIME();
- $relcount = $releases->categorizeRelease('searchname', '');
- $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
- $time = $consoletools->convertTime(TIME() - $timestart);
- echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
- } else if ($argv[1] == 6 && $argv[2] == 'false') {
- echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient.");
- $timestart = TIME();
- $relcount = $releases->categorizeRelease('searchname', 'WHERE categoryid IN (1090, 2020, 3050, 5050, 6050, 8010)');
- $consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
- $time = $consoletools->convertTime(TIME() - $timestart);
- echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
- } else {
- exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments."));
- }
-} else {
- exit($pdo->log->error("\nWrong set of arguments.\n"
- . "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n"
- . "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n"
- . "\nYou must pass a second argument whether to post process or not, true or false\n"
- . "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n"
- . "\nExtra commands::\n"
- . "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n"
- . "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n"
- . "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n"
- . "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n"));
-}
\ No newline at end of file
diff --git a/misc/update_scripts/nntpproxy.php b/misc/update_scripts/nntpproxy.php
index fee92abef..7cebf6ca8 100644
--- a/misc/update_scripts/nntpproxy.php
+++ b/misc/update_scripts/nntpproxy.php
@@ -39,7 +39,7 @@ function window_proxy($tmux_session, $powerline)
if ($powerline === '1') {
$tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf";
} else {
- $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf";
+ $tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/conf/tmux.conf";
}
$nntpproxy = $pdo->getSetting('nntpproxy');
diff --git a/misc/update_scripts/python/.gitignore b/misc/update_scripts/python/.gitignore
new file mode 100644
index 000000000..1af04760d
--- /dev/null
+++ b/misc/update_scripts/python/.gitignore
@@ -0,0 +1,3 @@
+*.pyc
+*.pyo
+/.idea
\ No newline at end of file
diff --git a/misc/update_scripts/python/README.txt b/misc/update_scripts/python/README.txt
new file mode 100644
index 000000000..3a2edc3b3
--- /dev/null
+++ b/misc/update_scripts/python/README.txt
@@ -0,0 +1,7 @@
+Requires Python 2.7, with the cymysql module.
+
+Run like this :
+
+python -OOu binaries_threaded.py
+-or-
+python3 -OOu binaries_threaded.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py b/misc/update_scripts/python/backfill_safe_threaded.py
similarity index 77%
rename from misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py
rename to misc/update_scripts/python/backfill_safe_threaded.py
index 0f3d1bc34..93486cfdb 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/backfill_safe_threaded.py
+++ b/misc/update_scripts/python/backfill_safe_threaded.py
@@ -22,23 +22,23 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
-cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1")
+cur[0].execute("SELECT g.name FROM groups g LEFT JOIN short_groups ON short_groups.name = g.name WHERE short_groups.name IS NULL AND backfill = 1")
dorun = cur[0].fetchone()
#close connection to mysql
info.disconnect(cur[0], cur[1])
if dorun:
- #before we get the groups, lets update shortgroups
- subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
+ #before we get the groups, lets update short_groups
+ subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
else:
cur = info.connect()
- cur[0].execute("SELECT name FROM shortgroups")
+ cur[0].execute("SELECT name FROM short_groups")
dorun = cur[0].fetchone()
info.disconnect(cur[0], cur[1])
if len(sys.argv) > 1 and sys.argv[1] not in dorun:
- #before we get the groups, lets update shortgroups
- subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
+ #before we get the groups, lets update short_groups
+ subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
count = 0
previous = "'alt.binaries.crap'"
@@ -78,13 +78,13 @@ while count < 10000:
#query to grab backfill groups
if len(sys.argv) == 1:
- if conf['DB_TYPE'] == "mysql":
- cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
- elif conf['DB_TYPE'] == "pgsql":
- cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
+ if conf['DB_SYSTEM'] == "mysql":
+ cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
+ elif conf['DB_SYSTEM'] == "pgsql":
+ cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
datas = cur[0].fetchone()
else:
- run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
+ run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
cur[0].execute(run, (sys.argv[1]))
datas = cur[0].fetchone()
if not datas or datas[0] is None:
@@ -119,7 +119,7 @@ while count < 10000:
print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC)
print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC)
group = ("{} {}".format(datas[0], count))
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
#calculate the number of items for queue
if (count > (backfill_qty * run_threads)):
@@ -147,7 +147,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -177,7 +177,7 @@ def main(args):
my_queue.join()
group = ("{} {}".format(datas[0], 1000))
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
if run_threads <= geteach:
print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC)
else:
@@ -188,4 +188,4 @@ def main(args):
if __name__ == '__main__':
- main(sys.argv[1:])
\ No newline at end of file
+ main(sys.argv[1:])
diff --git a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py b/misc/update_scripts/python/backfill_threaded.py
similarity index 92%
rename from misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py
rename to misc/update_scripts/python/backfill_threaded.py
index 70a5816f5..9da0f5506 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/backfill_threaded.py
+++ b/misc/update_scripts/python/backfill_threaded.py
@@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all":
# removing the % before the variables at the end of the query adds quotes/escapes strings
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group))
else:
- if conf['DB_TYPE'] == "mysql":
+ if conf['DB_SYSTEM'] == "mysql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
- elif conf['DB_TYPE'] == "pgsql":
+ elif conf['DB_SYSTEM'] == "pgsql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
datas = cur[0].fetchall()
@@ -99,9 +99,9 @@ class queue_runner(threading.Thread):
if my_id:
time_of_last_run = time.time()
if len(sys.argv) > 1 and sys.argv[1] == "all":
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
else:
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -135,4 +135,4 @@ def main(args):
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
- main(sys.argv[1:])
\ No newline at end of file
+ main(sys.argv[1:])
diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py b/misc/update_scripts/python/binaries_safe_threaded.py
similarity index 91%
rename from misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py
rename to misc/update_scripts/python/binaries_safe_threaded.py
index d6b034f8b..2e2d1729a 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/binaries_safe_threaded.py
+++ b/misc/update_scripts/python/binaries_safe_threaded.py
@@ -30,12 +30,12 @@ dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
maxmssgs = int(dbgrab[0][1])
-#before we get the groups, lets update shortgroups
-subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
+#before we get the groups, lets update short_groups
+subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
#query to grab all active groups
cur = info.connect()
-cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN shortgroups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC")
+cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC")
datas = cur[0].fetchall()
#close connection to mysql
@@ -64,7 +64,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -121,4 +121,4 @@ def main():
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
- main()
\ No newline at end of file
+ main()
diff --git a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py b/misc/update_scripts/python/binaries_threaded.py
similarity index 91%
rename from misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py
rename to misc/update_scripts/python/binaries_threaded.py
index 879be3be4..954d581e4 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/binaries_threaded.py
+++ b/misc/update_scripts/python/binaries_threaded.py
@@ -32,7 +32,6 @@ if len(sys.argv) == 1:
print(bcolors.HEADER + "\nBinaries Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
-
#get active groups
if len(sys.argv) == 2:
try:
@@ -51,18 +50,17 @@ else:
datas = cur[0].fetchall()
if len(datas) == 0:
- print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC)
- info.disconnect(cur[0], cur[1])
- sys.exit
+ print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC)
+ info.disconnect(cur[0], cur[1])
+ sys.exit
-cur[0].execute("SELECT value FROM tmux WHERE setting = 'binarythreads'")
-dbgrab = cur[0].fetchone()
+cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'binarythreads') AS a")
+dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
#close connection to mysql
info.disconnect(cur[0], cur[1])
-
my_queue = queue.Queue()
time_of_last_run = time.time()
@@ -83,7 +81,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../bin/update_binaries.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../update_binaries.php", ""+my_id])
self.my_queue.task_done()
def main():
diff --git a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py b/misc/update_scripts/python/fixreleasenames_threaded.py
similarity index 61%
rename from misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py
rename to misc/update_scripts/python/fixreleasenames_threaded.py
index c27d86e8d..09c0f9731 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/fixreleasenames_threaded.py
+++ b/misc/update_scripts/python/fixreleasenames_threaded.py
@@ -22,16 +22,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
if len(sys.argv) == 1:
print(bcolors.ERROR + "\nAn argument is required\n\n"
- + "python " + sys.argv[0] + " [md5, nfo, filename, par2] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2].\n"
+ + "python " + sys.argv[0] + " [md5, nfo, filename, par2, miscsorter] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2, miscsorter].\n"
+ "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashid, using [nfo, filename, par2].\n"
+ "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n"
+ "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n"
- + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n"
+ + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n"
+ bcolors.ENDC)
sys.exit()
-if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft":
- print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC)
+if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "miscsorter" and sys.argv[1] != "predbft":
+ print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, miscsorter, predbft]\n" + bcolors.ENDC)
sys.exit()
if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean":
@@ -39,47 +39,48 @@ if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean":
elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "clean":
clean = " isrenamed = 0 AND proc_files = 1 AND proc_nfo = 1 "
elif len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "prehashid":
- clean = " prehashid = IS NULL "
+ clean = " prehashid = 0 "
elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "prehashid":
- clean = " prehashid = IS NULL "
+ clean = " prehashid = 0 "
elif len(sys.argv) == 3 and sys.argv[1] == "filename" and sys.argv[2] == "prehashid":
- clean = " prehashid = IS NULL "
+ clean = " prehashid = 0 "
else:
clean = " isrenamed = 0 "
print(bcolors.HEADER + "\nfixReleasesNames {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
-cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'")
+cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'")
run_threads = cur[0].fetchone()
-cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'")
+cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'")
perrun = cur[0].fetchone()
datas = []
-maxtries = 0
if len(sys.argv) > 1 and sys.argv[1] == "nfo":
- run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s"
- cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
- datas = cur[0].fetchall()
+ run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
+ datas = cur[0].fetchall()
+elif len(sys.argv) > 1 and sys.argv[1] == "miscsorter":
+ run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_sorter = 0 AND isrenamed = 0 ORDER BY postdate DESC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
+ datas = cur[0].fetchall()
elif len(sys.argv) > 1 and (sys.argv[1] == "filename"):
- run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN releasefiles relfiles ON (relfiles.releaseid = rel.id) WHERE categoryid IN (8010, 8020) AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
- cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
- datas = cur[0].fetchall()
+ run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN release_files relfiles ON (relfiles.releaseid = rel.id) WHERE nzbstatus = 1 AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
+ datas = cur[0].fetchall()
elif len(sys.argv) > 1 and (sys.argv[1] == "md5"):
- while len(datas) == 0 and maxtries >= -5:
- run = "SELECT DISTINCT rel.id FROM releases rel INNER JOIN releasefiles rf ON rel.id = rf.releaseid WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashid = 0 ORDER BY postdate ASC LIMIT %s"
- cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0])))
- datas = cur[0].fetchall()
- maxtries = maxtries - 1
+ run = "SELECT DISTINCT rel.id FROM releases rel LEFT OUTER JOIN release_files rf ON rel.id = rf.releaseid AND rf.ishashed = 1 WHERE nzbstatus = 1 AND rel.dehashstatus BETWEEN -6 AND 0 AND rel.ishashed = 1 AND prehashid = 0 ORDER BY dehashstatus DESC, postdate ASC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0])*int(run_threads[0])))
+ datas = cur[0].fetchall()
elif len(sys.argv) > 1 and (sys.argv[1] == "par2"):
#This one does from oldest posts to newest posts, since nfo pp does same thing but newest to oldest
- run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE categoryid IN (8010, 8020) AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
- cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
- datas = cur[0].fetchall()
+ run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE nzbstatus = 1 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
+ datas = cur[0].fetchall()
elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"):
#This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes
- run = "SELECT id AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s"
- cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0])))
+ run = "SELECT id AS prehashid FROM prehash WHERE LENGTH(title) >= 15 AND title NOT REGEXP '[\"\<\> ]' AND searched = 0 AND DATEDIFF(NOW(), predate) > 1 ORDER BY predate ASC LIMIT %s"
+ cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
datas = cur[0].fetchall()
#close connection to mysql
@@ -109,7 +110,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../bin/fixreleasenames.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/fixreleasenames.php", ""+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -117,11 +118,7 @@ def main():
global time_of_last_run
time_of_last_run = time.time()
- if sys.argv[1] == 'md5':
- print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. dehashstatus range {} to 0".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1], maxtries + 1) + bcolors.ENDC)
- else:
- print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC)
- time.sleep(2)
+ print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC)
def signal_handler(signal, frame):
sys.exit(0)
@@ -152,6 +149,10 @@ def main():
for release in datas:
time.sleep(.03)
my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2]))
+ elif sys.argv[1] == "miscsorter":
+ for release in datas:
+ time.sleep(.03)
+ my_queue.put("%s %s" % ("miscsorter", release[0]))
elif sys.argv[1] == "predbft":
for release in datas:
time.sleep(.03)
diff --git a/misc/update_scripts/python/grabheaders_threaded.py b/misc/update_scripts/python/grabheaders_threaded.py
new file mode 100644
index 000000000..38eeb3a16
--- /dev/null
+++ b/misc/update_scripts/python/grabheaders_threaded.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys, os, time
+import threading
+try:
+ import queue
+except ImportError:
+ import Queue as queue
+import subprocess
+import string
+import signal
+import datetime
+
+import lib.info as info
+from lib.info import bcolors
+conf = info.readConfig()
+start_time = time.time()
+pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
+threads = 10
+
+print(bcolors.HEADER + "\nGrab Headers Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
+
+if len(sys.argv) > 1:
+ datas = []
+ datas.append(sys.argv[1])
+else:
+ datas = ("alt.binaries.teevee", "alt.binaries.tv", "alt.binaries.audiobooks", "alt.binaries.moovee", "alt.binaries.e-book", "alt.binaries.e-book.technical", "alt.binaries.ebook", "alt.binaries.e-book.magazines")
+if not datas:
+ print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC)
+ sys.exit()
+
+my_queue = queue.Queue()
+time_of_last_run = time.time()
+
+class queue_runner(threading.Thread):
+ def __init__(self, my_queue):
+ threading.Thread.__init__(self)
+ self.my_queue = my_queue
+
+ def run(self):
+ global time_of_last_run
+
+ while True:
+ try:
+ my_id = self.my_queue.get(True, 1)
+ except:
+ if time.time() - time_of_last_run > 3:
+ return
+ else:
+ if my_id:
+ time_of_last_run = time.time()
+ subprocess.call(["php", pathname+"/../../testing/Regex/grabheaders.php", ""+my_id])
+ self.my_queue.task_done()
+
+def main():
+ global time_of_last_run
+ time_of_last_run = time.time()
+
+ print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC)
+ time.sleep(2)
+ def signal_handler(signal, frame):
+ sys.exit(0)
+
+ signal.signal(signal.SIGINT, signal_handler)
+
+ if True:
+ #spawn a pool of place worker threads
+ for i in range(threads):
+ p = queue_runner(my_queue)
+ p.setDaemon(False)
+ p.start()
+
+ #now load some arbitrary jobs into the queue
+ count = 0
+ for group in datas:
+ my_queue.put(group)
+
+ my_queue.join()
+
+ print(bcolors.HEADER + "\nGrab Headers Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
+ print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
+
+if __name__ == '__main__':
+ main()
diff --git a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py b/misc/update_scripts/python/groupfixrelnames_threaded.py
similarity index 81%
rename from misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py
rename to misc/update_scripts/python/groupfixrelnames_threaded.py
index f89cdecaf..067bc69f8 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/groupfixrelnames_threaded.py
+++ b/misc/update_scripts/python/groupfixrelnames_threaded.py
@@ -37,13 +37,15 @@ limit = 0
join = ""
where = ""
datelimit = "AND DATEDIFF(NOW(), r.adddate) <= 7"
-groupby = "GROUP BY LEFT(r.guid, 1)"
-orderby = "ORDER BY LEFT(r.guid, 1) ASC"
-rowlimit = "LIMIT 100"
+groupby = "GROUP BY guidchar"
+orderby = "ORDER BY guidchar ASC"
+rowlimit = "LIMIT 16"
+extrawhere = "AND r.prehashid = 0 AND r.nzbstatus = 1"
+select = "DISTINCT LEFT(r.guid, 1) AS guidchar, COUNT(*) AS count"
-cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'")
+cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'")
run_threads = cur[0].fetchone()
-cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'")
+cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'")
run_perrun = cur[0].fetchone()
threads = int(run_threads[0])
@@ -52,22 +54,23 @@ if threads > 16:
maxperrun = int(run_perrun[0])
if sys.argv[1] == "md5":
- join = "LEFT OUTER JOIN releasefiles rf ON r.id = rf.releaseid AND rf.ishashed = 1"
+ join = "LEFT OUTER JOIN release_files rf ON r.id = rf.releaseid AND rf.ishashed = 1"
where = "r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0"
elif sys.argv[1] == "nfo":
where = "r.proc_nfo = 0 AND r.nfostatus = 1"
elif sys.argv[1] == "filename":
- join = "INNER JOIN releasefiles rf ON r.id = rf.releaseid"
+ join = "INNER JOIN release_files rf ON r.id = rf.releaseid"
where = "r.proc_files = 0"
elif sys.argv[1] == "par2":
where = "r.proc_par2 = 0"
elif sys.argv[1] == "miscsorter":
- where = "r.nfostatus = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0"
+ where = "r.nfostatus = 1 AND r.proc_nfo = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0"
elif sys.argv[1] == "predbft":
+ extrawhere = ""
where = "1=1"
rowlimit = "LIMIT %s" % (threads)
-cur[0].execute("SELECT DISTINCT LEFT(r.guid, 1), COUNT(*) AS count FROM releases r %s WHERE %s AND r.prehashid = 0 AND r.nzbstatus = 1 %s %s %s %s" % (join, where, datelimit, groupby, orderby, rowlimit))
+cur[0].execute("SELECT %s FROM releases r %s WHERE %s %s %s %s %s" % (select, join, where, extrawhere, groupby, orderby, rowlimit))
datas = cur[0].fetchall()
guids = int(len(datas))
@@ -99,7 +102,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../bin/groupfixrelnames.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/groupfixrelnames.php", ""+my_id])
self.my_queue.task_done()
def main():
diff --git a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py b/misc/update_scripts/python/import_threaded.py
similarity index 91%
rename from misc/update_scripts/nix_scripts/tmux/python/import_threaded.py
rename to misc/update_scripts/python/import_threaded.py
index f62d709e5..b1aaa953a 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/import_threaded.py
+++ b/misc/update_scripts/python/import_threaded.py
@@ -24,7 +24,7 @@ print(bcolors.HEADER + "\nNZB Import Threaded Started at {}".format(datetime.dat
#get values from db
cur[0].execute("SELECT value FROM tmux WHERE setting = 'import'")
use_true = cur[0].fetchone()
-cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b")
+cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
nzbs = dbgrab[0][1]
@@ -57,7 +57,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../bin/nzb-import-new.php", ""+my_id])
+ subprocess.call(["php", pathname+"/lib/nzb-import.php", ""+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -105,6 +105,8 @@ def main(args):
my_queue.join()
+ final = "true"
+ subprocess.call(["php", pathname+"/../../testing/DB/populate_nzb_guid.php", ""+final])
print(bcolors.HEADER + "\nNZB Import Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php b/misc/update_scripts/python/lib/.gitignore.php
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/.gitignore.php
rename to misc/update_scripts/python/lib/.gitignore.php
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py b/misc/update_scripts/python/lib/__init__.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/__init__.py
rename to misc/update_scripts/python/lib/__init__.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/info.py b/misc/update_scripts/python/lib/info.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/info.py
rename to misc/update_scripts/python/lib/info.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample b/misc/update_scripts/python/lib/nntpproxy.conf.sample
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf.sample
rename to misc/update_scripts/python/lib/nntpproxy.conf.sample
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore b/misc/update_scripts/python/lib/pynntp/.gitignore
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/.gitignore
rename to misc/update_scripts/python/lib/pynntp/.gitignore
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING b/misc/update_scripts/python/lib/pynntp/COPYING
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/COPYING
rename to misc/update_scripts/python/lib/pynntp/COPYING
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE b/misc/update_scripts/python/lib/pynntp/NOTE
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/NOTE
rename to misc/update_scripts/python/lib/pynntp/NOTE
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md b/misc/update_scripts/python/lib/pynntp/README.md
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/README.md
rename to misc/update_scripts/python/lib/pynntp/README.md
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py b/misc/update_scripts/python/lib/pynntp/__init__.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/__init__.py
rename to misc/update_scripts/python/lib/pynntp/__init__.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py b/misc/update_scripts/python/lib/pynntp/nntp/__init__.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/__init__.py
rename to misc/update_scripts/python/lib/pynntp/nntp/__init__.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py b/misc/update_scripts/python/lib/pynntp/nntp/date.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/date.py
rename to misc/update_scripts/python/lib/pynntp/nntp/date.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py b/misc/update_scripts/python/lib/pynntp/nntp/fifo.py
similarity index 100%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/fifo.py
rename to misc/update_scripts/python/lib/pynntp/nntp/fifo.py
diff --git a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py
similarity index 98%
rename from misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py
rename to misc/update_scripts/python/lib/pynntp/nntp/iodict.py
index f8b3a08b9..55e3030c7 100644
--- a/misc/update_scripts/nix_scripts/tmux/python/lib/pynntp/nntp/iodict.py
+++ b/misc/update_scripts/python/lib/pynntp/nntp/iodict.py
@@ -19,7 +19,7 @@ along with this program. If not, see