mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-09-01 18:58:56 +00:00
Move python scripts out of tmux folder, adjust other scripts and classes for this change.
This commit is contained in:
Darko
@@ -7,7 +7,7 @@ use newznab\processing\PostProcess;
|
||||
$pdo = new Settings();
|
||||
|
||||
if (!isset($argv[1])) {
|
||||
exit($pdo->log->error("This script is not intended to be run manually, it is called from groupfixrelnames_threaded.py."));
|
||||
exit($pdo->log->error("This script is not intended to be run manually, it is called from Forking.php."));
|
||||
} else if (isset($argv[1])) {
|
||||
$namefixer = new \NameFixer(['Settings' => $pdo]);
|
||||
$pieces = explode(' ', $argv[1]);
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
<?php
|
||||
require_once("config.php");
|
||||
|
||||
use newznab\db\Settings;
|
||||
|
||||
$pdo = new Settings();
|
||||
|
||||
if (isset($argv[2]) && $argv[2] === 'true') {
|
||||
// Create the connection here and pass
|
||||
$nntp = new \NNTP(['Settings' => $pdo]);
|
||||
if ($nntp->doConnect() !== true) {
|
||||
exit($pdo->log->error("Unable to connect to usenet."));
|
||||
}
|
||||
}
|
||||
if ($pdo->getSetting('tablepergroup') === 1) {
|
||||
exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update_scripts/nix_scripts/multiprocessing/releases.php"));
|
||||
}
|
||||
|
||||
$groupName = isset($argv[3]) ? $argv[3] : '';
|
||||
if (isset($argv[1]) && isset($argv[2])) {
|
||||
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]);
|
||||
if ($argv[1] == 1 && $argv[2] == 'true') {
|
||||
$releases->processReleases(1, 1, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 1 && $argv[2] == 'false') {
|
||||
$releases->processReleases(1, 2, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 2 && $argv[2] == 'true') {
|
||||
$releases->processReleases(2, 1, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 2 && $argv[2] == 'false') {
|
||||
$releases->processReleases(2, 2, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) {
|
||||
echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient.");
|
||||
$releases->resetCategorize();
|
||||
} else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) {
|
||||
echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryid = 8010');
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject.");
|
||||
} else if ($argv[1] == 6 && $argv[2] == 'true') {
|
||||
echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('searchname', '');
|
||||
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
|
||||
} else if ($argv[1] == 6 && $argv[2] == 'false') {
|
||||
echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('searchname', 'WHERE categoryid IN (1090, 2020, 3050, 5050, 6050, 8010)');
|
||||
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
|
||||
} else {
|
||||
exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments."));
|
||||
}
|
||||
} else {
|
||||
exit($pdo->log->error("\nWrong set of arguments.\n"
|
||||
. "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n"
|
||||
. "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n"
|
||||
. "\nYou must pass a second argument whether to post process or not, true or false\n"
|
||||
. "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n"
|
||||
. "\nExtra commands::\n"
|
||||
. "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n"
|
||||
. "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n"
|
||||
. "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n"
|
||||
. "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n"));
|
||||
}
|
||||
@@ -39,7 +39,7 @@ function window_proxy($tmux_session, $powerline)
|
||||
if ($powerline === '1') {
|
||||
$tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/powerline/tmux.conf";
|
||||
} else {
|
||||
$tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/tmux.conf";
|
||||
$tmuxconfig = $DIR . "update_scripts/nix_scripts/tmux/conf/tmux.conf";
|
||||
}
|
||||
|
||||
$nntpproxy = $pdo->getSetting('nntpproxy');
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
*.pyc
|
||||
*.pyo
|
||||
/.idea
|
||||
@@ -0,0 +1,7 @@
|
||||
Requires Python 2.7, with the cymysql module.
|
||||
|
||||
Run like this :
|
||||
|
||||
python -OOu binaries_threaded.py
|
||||
-or-
|
||||
python3 -OOu binaries_threaded.py
|
||||
+15
-15
@@ -22,23 +22,23 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
|
||||
print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1")
|
||||
cur[0].execute("SELECT g.name FROM groups g LEFT JOIN short_groups ON short_groups.name = g.name WHERE short_groups.name IS NULL AND backfill = 1")
|
||||
dorun = cur[0].fetchone()
|
||||
|
||||
#close connection to mysql
|
||||
info.disconnect(cur[0], cur[1])
|
||||
|
||||
if dorun:
|
||||
#before we get the groups, lets update shortgroups
|
||||
subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
|
||||
#before we get the groups, lets update short_groups
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
|
||||
else:
|
||||
cur = info.connect()
|
||||
cur[0].execute("SELECT name FROM shortgroups")
|
||||
cur[0].execute("SELECT name FROM short_groups")
|
||||
dorun = cur[0].fetchone()
|
||||
info.disconnect(cur[0], cur[1])
|
||||
if len(sys.argv) > 1 and sys.argv[1] not in dorun:
|
||||
#before we get the groups, lets update shortgroups
|
||||
subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
|
||||
#before we get the groups, lets update short_groups
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
|
||||
|
||||
count = 0
|
||||
previous = "'alt.binaries.crap'"
|
||||
@@ -78,13 +78,13 @@ while count < 10000:
|
||||
|
||||
#query to grab backfill groups
|
||||
if len(sys.argv) == 1:
|
||||
if conf['DB_TYPE'] == "mysql":
|
||||
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
|
||||
elif conf['DB_TYPE'] == "pgsql":
|
||||
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
|
||||
if conf['DB_SYSTEM'] == "mysql":
|
||||
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
|
||||
elif conf['DB_SYSTEM'] == "pgsql":
|
||||
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
|
||||
datas = cur[0].fetchone()
|
||||
else:
|
||||
run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
|
||||
run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN short_groups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
|
||||
cur[0].execute(run, (sys.argv[1]))
|
||||
datas = cur[0].fetchone()
|
||||
if not datas or datas[0] is None:
|
||||
@@ -119,7 +119,7 @@ while count < 10000:
|
||||
print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC)
|
||||
print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC)
|
||||
group = ("{} {}".format(datas[0], count))
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
|
||||
|
||||
#calculate the number of items for queue
|
||||
if (count > (backfill_qty * run_threads)):
|
||||
@@ -147,7 +147,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -177,7 +177,7 @@ def main(args):
|
||||
my_queue.join()
|
||||
|
||||
group = ("{} {}".format(datas[0], 1000))
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
|
||||
if run_threads <= geteach:
|
||||
print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC)
|
||||
else:
|
||||
@@ -188,4 +188,4 @@ def main(args):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
main(sys.argv[1:])
|
||||
+5
-5
@@ -63,9 +63,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "all":
|
||||
# removing the % before the variables at the end of the query adds quotes/escapes strings
|
||||
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group))
|
||||
else:
|
||||
if conf['DB_TYPE'] == "mysql":
|
||||
if conf['DB_SYSTEM'] == "mysql":
|
||||
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
|
||||
elif conf['DB_TYPE'] == "pgsql":
|
||||
elif conf['DB_SYSTEM'] == "pgsql":
|
||||
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
|
||||
|
||||
datas = cur[0].fetchall()
|
||||
@@ -99,9 +99,9 @@ class queue_runner(threading.Thread):
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "all":
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
|
||||
else:
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -135,4 +135,4 @@ def main(args):
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
main(sys.argv[1:])
|
||||
+5
-5
@@ -30,12 +30,12 @@ dbgrab = cur[0].fetchall()
|
||||
run_threads = int(dbgrab[0][0])
|
||||
maxmssgs = int(dbgrab[0][1])
|
||||
|
||||
#before we get the groups, lets update shortgroups
|
||||
subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
|
||||
#before we get the groups, lets update short_groups
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/update_groups.php", ""])
|
||||
|
||||
#query to grab all active groups
|
||||
cur = info.connect()
|
||||
cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN shortgroups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC")
|
||||
cur[0].execute("SELECT g.name AS groupname, g.last_record AS our_last, a.last_record AS their_last FROM groups g INNER JOIN short_groups a ON g.active = 1 AND g.name = a.name ORDER BY a.last_record DESC")
|
||||
datas = cur[0].fetchall()
|
||||
|
||||
#close connection to mysql
|
||||
@@ -64,7 +64,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python "+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -121,4 +121,4 @@ def main():
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
+6
-8
@@ -32,7 +32,6 @@ if len(sys.argv) == 1:
|
||||
print(bcolors.HEADER + "\nBinaries Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
|
||||
|
||||
#get active groups
|
||||
if len(sys.argv) == 2:
|
||||
try:
|
||||
@@ -51,18 +50,17 @@ else:
|
||||
datas = cur[0].fetchall()
|
||||
|
||||
if len(datas) == 0:
|
||||
print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC)
|
||||
info.disconnect(cur[0], cur[1])
|
||||
sys.exit
|
||||
print(bcolors.ERROR + "No Active Groups" + bcolors.ENDC)
|
||||
info.disconnect(cur[0], cur[1])
|
||||
sys.exit
|
||||
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'binarythreads'")
|
||||
dbgrab = cur[0].fetchone()
|
||||
cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'binarythreads') AS a")
|
||||
dbgrab = cur[0].fetchall()
|
||||
run_threads = int(dbgrab[0][0])
|
||||
|
||||
#close connection to mysql
|
||||
info.disconnect(cur[0], cur[1])
|
||||
|
||||
|
||||
my_queue = queue.Queue()
|
||||
time_of_last_run = time.time()
|
||||
|
||||
@@ -83,7 +81,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../bin/update_binaries.php", ""+my_id])
|
||||
subprocess.call(["php", pathname+"/../update_binaries.php", ""+my_id])
|
||||
self.my_queue.task_done()
|
||||
|
||||
def main():
|
||||
+33
-32
@@ -22,16 +22,16 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
print(bcolors.ERROR + "\nAn argument is required\n\n"
|
||||
+ "python " + sys.argv[0] + " [md5, nfo, filename, par2] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2].\n"
|
||||
+ "python " + sys.argv[0] + " [md5, nfo, filename, par2, miscsorter] ...: To process all previously unprocessed releases, using [md5, nfo, filename, par2, miscsorter].\n"
|
||||
+ "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashid, using [nfo, filename, par2].\n"
|
||||
+ "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n"
|
||||
+ "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n"
|
||||
+ "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n"
|
||||
+ "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n"
|
||||
+ bcolors.ENDC)
|
||||
sys.exit()
|
||||
|
||||
if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft":
|
||||
print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC)
|
||||
if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "miscsorter" and sys.argv[1] != "predbft":
|
||||
print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, miscsorter, predbft]\n" + bcolors.ENDC)
|
||||
sys.exit()
|
||||
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean":
|
||||
@@ -39,47 +39,48 @@ if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean":
|
||||
elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "clean":
|
||||
clean = " isrenamed = 0 AND proc_files = 1 AND proc_nfo = 1 "
|
||||
elif len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "prehashid":
|
||||
clean = " prehashid = IS NULL "
|
||||
clean = " prehashid = 0 "
|
||||
elif len(sys.argv) == 3 and sys.argv[1] == "par2" and sys.argv[2] == "prehashid":
|
||||
clean = " prehashid = IS NULL "
|
||||
clean = " prehashid = 0 "
|
||||
elif len(sys.argv) == 3 and sys.argv[1] == "filename" and sys.argv[2] == "prehashid":
|
||||
clean = " prehashid = IS NULL "
|
||||
clean = " prehashid = 0 "
|
||||
else:
|
||||
clean = " isrenamed = 0 "
|
||||
|
||||
print(bcolors.HEADER + "\nfixReleasesNames {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'")
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'")
|
||||
run_threads = cur[0].fetchone()
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'")
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'")
|
||||
perrun = cur[0].fetchone()
|
||||
|
||||
datas = []
|
||||
maxtries = 0
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "nfo":
|
||||
run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_nfo = 0 AND" + clean + "ORDER BY postdate DESC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
elif len(sys.argv) > 1 and sys.argv[1] == "miscsorter":
|
||||
run = "SELECT DISTINCT id AS releaseid FROM releases WHERE nzbstatus = 1 AND nfostatus = 1 AND proc_sorter = 0 AND isrenamed = 0 ORDER BY postdate DESC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
elif len(sys.argv) > 1 and (sys.argv[1] == "filename"):
|
||||
run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN releasefiles relfiles ON (relfiles.releaseid = rel.id) WHERE categoryid IN (8010, 8020) AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
run = "SELECT DISTINCT rel.id AS releaseid FROM releases rel INNER JOIN release_files relfiles ON (relfiles.releaseid = rel.id) WHERE nzbstatus = 1 AND proc_files = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
elif len(sys.argv) > 1 and (sys.argv[1] == "md5"):
|
||||
while len(datas) == 0 and maxtries >= -5:
|
||||
run = "SELECT DISTINCT rel.id FROM releases rel INNER JOIN releasefiles rf ON rel.id = rf.releaseid WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashid = 0 ORDER BY postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
maxtries = maxtries - 1
|
||||
run = "SELECT DISTINCT rel.id FROM releases rel LEFT OUTER JOIN release_files rf ON rel.id = rf.releaseid AND rf.ishashed = 1 WHERE nzbstatus = 1 AND rel.dehashstatus BETWEEN -6 AND 0 AND rel.ishashed = 1 AND prehashid = 0 ORDER BY dehashstatus DESC, postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0])*int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
elif len(sys.argv) > 1 and (sys.argv[1] == "par2"):
|
||||
#This one does from oldest posts to newest posts, since nfo pp does same thing but newest to oldest
|
||||
run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE categoryid IN (8010, 8020) AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
run = "SELECT id AS releaseid, guid, groupid FROM releases WHERE nzbstatus = 1 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"):
|
||||
#This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes
|
||||
run = "SELECT id AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0])))
|
||||
run = "SELECT id AS prehashid FROM prehash WHERE LENGTH(title) >= 15 AND title NOT REGEXP '[\"\<\> ]' AND searched = 0 AND DATEDIFF(NOW(), predate) > 1 ORDER BY predate ASC LIMIT %s"
|
||||
cur[0].execute(run, (int(perrun[0]) * int(run_threads[0])))
|
||||
datas = cur[0].fetchall()
|
||||
|
||||
#close connection to mysql
|
||||
@@ -109,7 +110,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../bin/fixreleasenames.php", ""+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/fixreleasenames.php", ""+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -117,11 +118,7 @@ def main():
|
||||
global time_of_last_run
|
||||
time_of_last_run = time.time()
|
||||
|
||||
if sys.argv[1] == 'md5':
|
||||
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. dehashstatus range {} to 0".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1], maxtries + 1) + bcolors.ENDC)
|
||||
else:
|
||||
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC)
|
||||
time.sleep(2)
|
||||
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} releases using {}".format(run_threads[0], "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC)
|
||||
|
||||
def signal_handler(signal, frame):
|
||||
sys.exit(0)
|
||||
@@ -152,6 +149,10 @@ def main():
|
||||
for release in datas:
|
||||
time.sleep(.03)
|
||||
my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2]))
|
||||
elif sys.argv[1] == "miscsorter":
|
||||
for release in datas:
|
||||
time.sleep(.03)
|
||||
my_queue.put("%s %s" % ("miscsorter", release[0]))
|
||||
elif sys.argv[1] == "predbft":
|
||||
for release in datas:
|
||||
time.sleep(.03)
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys, os, time
|
||||
import threading
|
||||
try:
|
||||
import queue
|
||||
except ImportError:
|
||||
import Queue as queue
|
||||
import subprocess
|
||||
import string
|
||||
import signal
|
||||
import datetime
|
||||
|
||||
import lib.info as info
|
||||
from lib.info import bcolors
|
||||
conf = info.readConfig()
|
||||
start_time = time.time()
|
||||
pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
threads = 10
|
||||
|
||||
print(bcolors.HEADER + "\nGrab Headers Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
datas = []
|
||||
datas.append(sys.argv[1])
|
||||
else:
|
||||
datas = ("alt.binaries.teevee", "alt.binaries.tv", "alt.binaries.audiobooks", "alt.binaries.moovee", "alt.binaries.e-book", "alt.binaries.e-book.technical", "alt.binaries.ebook", "alt.binaries.e-book.magazines")
|
||||
if not datas:
|
||||
print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC)
|
||||
sys.exit()
|
||||
|
||||
my_queue = queue.Queue()
|
||||
time_of_last_run = time.time()
|
||||
|
||||
class queue_runner(threading.Thread):
|
||||
def __init__(self, my_queue):
|
||||
threading.Thread.__init__(self)
|
||||
self.my_queue = my_queue
|
||||
|
||||
def run(self):
|
||||
global time_of_last_run
|
||||
|
||||
while True:
|
||||
try:
|
||||
my_id = self.my_queue.get(True, 1)
|
||||
except:
|
||||
if time.time() - time_of_last_run > 3:
|
||||
return
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../testing/Regex/grabheaders.php", ""+my_id])
|
||||
self.my_queue.task_done()
|
||||
|
||||
def main():
|
||||
global time_of_last_run
|
||||
time_of_last_run = time.time()
|
||||
|
||||
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} groups".format(threads, "{:,}".format(len(datas))) + bcolors.ENDC)
|
||||
time.sleep(2)
|
||||
def signal_handler(signal, frame):
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
if True:
|
||||
#spawn a pool of place worker threads
|
||||
for i in range(threads):
|
||||
p = queue_runner(my_queue)
|
||||
p.setDaemon(False)
|
||||
p.start()
|
||||
|
||||
#now load some arbitrary jobs into the queue
|
||||
count = 0
|
||||
for group in datas:
|
||||
my_queue.put(group)
|
||||
|
||||
my_queue.join()
|
||||
|
||||
print(bcolors.HEADER + "\nGrab Headers Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+13
-10
@@ -37,13 +37,15 @@ limit = 0
|
||||
join = ""
|
||||
where = ""
|
||||
datelimit = "AND DATEDIFF(NOW(), r.adddate) <= 7"
|
||||
groupby = "GROUP BY LEFT(r.guid, 1)"
|
||||
orderby = "ORDER BY LEFT(r.guid, 1) ASC"
|
||||
rowlimit = "LIMIT 100"
|
||||
groupby = "GROUP BY guidchar"
|
||||
orderby = "ORDER BY guidchar ASC"
|
||||
rowlimit = "LIMIT 16"
|
||||
extrawhere = "AND r.prehashid = 0 AND r.nzbstatus = 1"
|
||||
select = "DISTINCT LEFT(r.guid, 1) AS guidchar, COUNT(*) AS count"
|
||||
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamethreads'")
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamethreads'")
|
||||
run_threads = cur[0].fetchone()
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'fixnamesperrun'")
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'fixnamesperrun'")
|
||||
run_perrun = cur[0].fetchone()
|
||||
|
||||
threads = int(run_threads[0])
|
||||
@@ -52,22 +54,23 @@ if threads > 16:
|
||||
maxperrun = int(run_perrun[0])
|
||||
|
||||
if sys.argv[1] == "md5":
|
||||
join = "LEFT OUTER JOIN releasefiles rf ON r.id = rf.releaseid AND rf.ishashed = 1"
|
||||
join = "LEFT OUTER JOIN release_files rf ON r.id = rf.releaseid AND rf.ishashed = 1"
|
||||
where = "r.ishashed = 1 AND r.dehashstatus BETWEEN -6 AND 0"
|
||||
elif sys.argv[1] == "nfo":
|
||||
where = "r.proc_nfo = 0 AND r.nfostatus = 1"
|
||||
elif sys.argv[1] == "filename":
|
||||
join = "INNER JOIN releasefiles rf ON r.id = rf.releaseid"
|
||||
join = "INNER JOIN release_files rf ON r.id = rf.releaseid"
|
||||
where = "r.proc_files = 0"
|
||||
elif sys.argv[1] == "par2":
|
||||
where = "r.proc_par2 = 0"
|
||||
elif sys.argv[1] == "miscsorter":
|
||||
where = "r.nfostatus = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0"
|
||||
where = "r.nfostatus = 1 AND r.proc_nfo = 1 AND r.proc_sorter = 0 AND r.isrenamed = 0"
|
||||
elif sys.argv[1] == "predbft":
|
||||
extrawhere = ""
|
||||
where = "1=1"
|
||||
rowlimit = "LIMIT %s" % (threads)
|
||||
|
||||
cur[0].execute("SELECT DISTINCT LEFT(r.guid, 1), COUNT(*) AS count FROM releases r %s WHERE %s AND r.prehashid = 0 AND r.nzbstatus = 1 %s %s %s %s" % (join, where, datelimit, groupby, orderby, rowlimit))
|
||||
cur[0].execute("SELECT %s FROM releases r %s WHERE %s %s %s %s %s" % (select, join, where, extrawhere, groupby, orderby, rowlimit))
|
||||
datas = cur[0].fetchall()
|
||||
|
||||
guids = int(len(datas))
|
||||
@@ -99,7 +102,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../bin/groupfixrelnames.php", ""+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/tmux/bin/groupfixrelnames.php", ""+my_id])
|
||||
self.my_queue.task_done()
|
||||
|
||||
def main():
|
||||
+4
-2
@@ -24,7 +24,7 @@ print(bcolors.HEADER + "\nNZB Import Threaded Started at {}".format(datetime.dat
|
||||
#get values from db
|
||||
cur[0].execute("SELECT value FROM tmux WHERE setting = 'import'")
|
||||
use_true = cur[0].fetchone()
|
||||
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b")
|
||||
cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'nzbthreads') AS a, (SELECT value FROM tmux WHERE setting = 'nzbs') AS b")
|
||||
dbgrab = cur[0].fetchall()
|
||||
run_threads = int(dbgrab[0][0])
|
||||
nzbs = dbgrab[0][1]
|
||||
@@ -57,7 +57,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../bin/nzb-import-new.php", ""+my_id])
|
||||
subprocess.call(["php", pathname+"/lib/nzb-import.php", ""+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -105,6 +105,8 @@ def main(args):
|
||||
|
||||
my_queue.join()
|
||||
|
||||
final = "true"
|
||||
subprocess.call(["php", pathname+"/../../testing/DB/populate_nzb_guid.php", ""+final])
|
||||
print(bcolors.HEADER + "\nNZB Import Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#NOTE: docstrings beed to be added
|
||||
|
||||
from collections import OrderedDict
|
||||
from binaries import OrderedDict
|
||||
|
||||
def _lower(v):
|
||||
"""assumes that classes that inherit list, tuple or dict have a constructor
|
||||
+4
-4
@@ -27,7 +27,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient):
|
||||
raise ValueError("Bad backend")
|
||||
nntp.NNTPClient.__init__(self, self.host, self.port, username, password, timeout=timeout, use_ssl=use_ssl)
|
||||
self.id = self.socket.getsockname()[1]
|
||||
print(bcolors.PRIMARY + "New NNTP connection to %s established with id #%5d" %
|
||||
print(bcolors.PRIMARY + "New NNTP connection to %s established with ID #%5d" %
|
||||
(self.host, self.id) + bcolors.ENDC)
|
||||
self._connected = True
|
||||
self.xfeature_compress_gzip()
|
||||
@@ -54,7 +54,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient):
|
||||
return self._life
|
||||
|
||||
def invalidate(self):
|
||||
print(bcolors.PRIMARY + "Disconnecting from NNTP connection id #%5d after %d seconds." %
|
||||
print(bcolors.PRIMARY + "Disconnecting from NNTP connection ID #%5d after %d seconds." %
|
||||
(self.id, (time.time() - self._start_time)) + bcolors.ENDC)
|
||||
self.close()
|
||||
self._connected = False
|
||||
@@ -68,7 +68,7 @@ class NNTPClientConnector(socketpool.Connector, nntp.NNTPClient):
|
||||
self._pool = None
|
||||
|
||||
|
||||
# NNTP proxy request handler for newznab
|
||||
# NNTP proxy request handler for nZEDb
|
||||
class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler):
|
||||
|
||||
def handle(self):
|
||||
@@ -196,7 +196,7 @@ class NNTPProxyRequestHandler(SocketServer.StreamRequestHandler):
|
||||
self.wfile.write("500 What?\r\n")
|
||||
|
||||
|
||||
# NNTP proxy server for newznab
|
||||
# NNTP proxy server for nZEDb
|
||||
class NNTPProxyServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
|
||||
|
||||
allow_reuse_address = True
|
||||
+3
-3
@@ -175,7 +175,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python pp_"+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python pp_"+my_id])
|
||||
time.sleep(.02)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -232,7 +232,7 @@ def main(args):
|
||||
final = cur[0].fetchall()
|
||||
if len(datas) > 0:
|
||||
for item in final:
|
||||
run = "DELETE FROM releasenfo WHERE nfo IS NULL AND releaseid = %s"
|
||||
run = "DELETE FROM release_nfos WHERE nfo IS NULL AND releaseid = %s"
|
||||
cur[0].execute(run, (item[0]))
|
||||
final = cur[0].fetchall()
|
||||
|
||||
@@ -243,4 +243,4 @@ def main(args):
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1:])
|
||||
main(sys.argv[1:])
|
||||
+4
-4
@@ -21,7 +21,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
|
||||
print(bcolors.HEADER + "\nUpdate Releases Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasethreads') AS b")
|
||||
cur[0].execute("SELECT (SELECT value FROM settings WHERE setting = 'tablepergroup') AS a, (SELECT value FROM settings WHERE setting = 'releasesthreads') AS b")
|
||||
dbgrab = cur[0].fetchall()
|
||||
allowed = int(dbgrab[0][0])
|
||||
threads = int(dbgrab[0][1])
|
||||
@@ -60,7 +60,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "releases "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+my_id])
|
||||
self.my_queue.task_done()
|
||||
|
||||
def main():
|
||||
@@ -93,10 +93,10 @@ def main():
|
||||
my_queue.join()
|
||||
|
||||
#stage7b
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"])
|
||||
|
||||
print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
+7
-20
@@ -15,9 +15,9 @@ except ImportError:
|
||||
import Queue as queue
|
||||
|
||||
try:
|
||||
import urllib2
|
||||
import urllib2
|
||||
except ImportError:
|
||||
import urllib.request as urllib2
|
||||
import urllib.request as urllib2
|
||||
|
||||
import lib.info as info
|
||||
from lib.info import bcolors
|
||||
@@ -25,22 +25,10 @@ conf = info.readConfig()
|
||||
cur = info.connect()
|
||||
start_time = time.time()
|
||||
pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
threads = 5
|
||||
web = None
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'reqidthreads'")
|
||||
threads = cur[0].fetchone()
|
||||
threads = int(threads[0])
|
||||
|
||||
try:
|
||||
import urllib
|
||||
except ImportError:
|
||||
import urllib.request as httplib
|
||||
|
||||
try:
|
||||
r1 = urllib.urlopen("http://reqid.nzedb.com").getcode()
|
||||
if r1 == 200:
|
||||
web = True
|
||||
else:
|
||||
web = False
|
||||
except:
|
||||
web = False
|
||||
|
||||
print(bcolors.HEADER + "\n\nRequestID Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
|
||||
@@ -77,7 +65,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python requestid "+my_id])
|
||||
time.sleep(.03)
|
||||
self.my_queue.task_done()
|
||||
|
||||
@@ -102,8 +90,7 @@ def main():
|
||||
|
||||
#now load some arbitrary jobs into the queue
|
||||
for release in datas:
|
||||
time.sleep(.03)
|
||||
my_queue.put("%s %s %s %s" % (release[0], release[1], release[2], web))
|
||||
my_queue.put("%s" % (release[0]))
|
||||
|
||||
my_queue.join()
|
||||
|
||||
+4
-4
@@ -18,7 +18,7 @@ conf = info.readConfig()
|
||||
cur = info.connect()
|
||||
start_time = time.time()
|
||||
pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'releasethreads'")
|
||||
cur[0].execute("SELECT value FROM settings WHERE setting = 'releasesthreads'")
|
||||
threads = cur[0].fetchone()
|
||||
threads = int(threads[0])
|
||||
|
||||
@@ -61,7 +61,7 @@ class queue_runner(threading.Thread):
|
||||
else:
|
||||
if my_id:
|
||||
time_of_last_run = time.time()
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python update_per_group "+my_id])
|
||||
self.my_queue.task_done()
|
||||
|
||||
def main():
|
||||
@@ -95,10 +95,10 @@ def main():
|
||||
|
||||
#stage7b
|
||||
final = "final"
|
||||
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"])
|
||||
subprocess.call(["php", pathname+"/../nix_scripts/multiprocessing/.do_not_run/switch.php", "python releases "+str(count)+"_"])
|
||||
|
||||
print(bcolors.HEADER + "\nUpdate Releases Threaded Completed at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
|
||||
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
@@ -1,6 +1,68 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__) . "/config.php");
|
||||
|
||||
require_once("config.php");
|
||||
use newznab\db\Settings;
|
||||
|
||||
$releases = new Releases();
|
||||
$releases->processReleases(1, 1, $groupName, $nntp, true);
|
||||
$pdo = new Settings();
|
||||
|
||||
if (isset($argv[2]) && $argv[2] === 'true') {
|
||||
// Create the connection here and pass
|
||||
$nntp = new NNTP(['Settings' => $pdo]);
|
||||
if ($nntp->doConnect() !== true) {
|
||||
exit($pdo->log->error("Unable to connect to usenet."));
|
||||
}
|
||||
}
|
||||
if ($pdo->getSetting('tablepergroup') === 1) {
|
||||
exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update_scripts/nix_scripts/multiprocessing/releases.php"));
|
||||
}
|
||||
|
||||
$groupName = isset($argv[3]) ? $argv[3] : '';
|
||||
if (isset($argv[1]) && isset($argv[2])) {
|
||||
$consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$releases = new Releases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]);
|
||||
if ($argv[1] == 1 && $argv[2] == 'true') {
|
||||
$releases->processReleases(1, 1, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 1 && $argv[2] == 'false') {
|
||||
$releases->processReleases(1, 2, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 2 && $argv[2] == 'true') {
|
||||
$releases->processReleases(2, 1, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 2 && $argv[2] == 'false') {
|
||||
$releases->processReleases(2, 2, $groupName, $nntp, true);
|
||||
} else if ($argv[1] == 4 && ($argv[2] == 'true' || $argv[2] == 'false')) {
|
||||
echo $pdo->log->header("Moving all releases to other -> misc, this can take a while, be patient.");
|
||||
$releases->resetCategorize();
|
||||
} else if ($argv[1] == 5 && ($argv[2] == 'true' || $argv[2] == 'false')) {
|
||||
echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using usenet subject. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('name', 'WHERE iscategorized = 0 AND categoryid = 8010');
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the usenet subject.");
|
||||
} else if ($argv[1] == 6 && $argv[2] == 'true') {
|
||||
echo $pdo->log->header("Categorizing releases in all sections using the searchname. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('searchname', '');
|
||||
$consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
|
||||
} else if ($argv[1] == 6 && $argv[2] == 'false') {
|
||||
echo $pdo->log->header("Categorizing releases in misc sections using the searchname. This can take a while, be patient.");
|
||||
$timestart = TIME();
|
||||
$relcount = $releases->categorizeRelease('searchname', 'WHERE categoryid IN (1090, 2020, 3050, 5050, 6050, 8010)');
|
||||
$consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]);
|
||||
$time = $consoletools->convertTime(TIME() - $timestart);
|
||||
echo $pdo->log->primary("\n" . 'Finished categorizing ' . $relcount . ' releases in ' . $time . " seconds, using the search name.");
|
||||
} else {
|
||||
exit($pdo->log->error("Wrong argument, type php update_releases.php to see a list of valid arguments."));
|
||||
}
|
||||
} else {
|
||||
exit($pdo->log->error("\nWrong set of arguments.\n"
|
||||
. "php update_releases.php 1 true ...: Creates releases and attempts to categorize new releases\n"
|
||||
. "php update_releases.php 2 true ...: Creates releases and leaves new releases in other -> misc\n"
|
||||
. "\nYou must pass a second argument whether to post process or not, true or false\n"
|
||||
. "You can pass a third optional argument, a group name (ex.: alt.binaries.multimedia).\n"
|
||||
. "\nExtra commands::\n"
|
||||
. "php update_releases.php 4 true ...: Puts all releases in other-> misc (also resets to look like they have never been categorized)\n"
|
||||
. "php update_releases.php 5 true ...: Categorizes all releases in other-> misc (which have not been categorized already)\n"
|
||||
. "php update_releases.php 6 false ...: Categorizes releases in misc sections using the search name\n"
|
||||
. "php update_releases.php 6 true ...: Categorizes releases in all sections using the search name\n"));
|
||||
}
|
||||
@@ -76,7 +76,7 @@ class Tmux
|
||||
$runVar['connections']['ip_a'] = gethostbyname($runVar['connections']['host_a']);
|
||||
}
|
||||
} else {
|
||||
$filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy.conf";
|
||||
$filename = NN_MISC . "update_scripts/python/lib/nntpproxy.conf";
|
||||
$fp = fopen($filename, "r") || die("Couldn't open $filename");
|
||||
while (!feof($fp)) {
|
||||
$line = fgets($fp);
|
||||
@@ -89,7 +89,7 @@ class Tmux
|
||||
}
|
||||
}
|
||||
if ($constants['alternate_nntp']) {
|
||||
$filename = NN_MISC . "update_scripts/nix_scripts/tmux/python/lib/nntpproxy_a.conf";
|
||||
$filename = NN_MISC . "update_scripts/python/lib/nntpproxy_a.conf";
|
||||
$fp = fopen($filename, "r") || die("Couldn't open $filename");
|
||||
while (!feof($fp)) {
|
||||
$line = fgets($fp);
|
||||
|
||||
@@ -477,8 +477,7 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th width="140">RAR
|
||||
Contains
|
||||
<th width="140">RAR Contains
|
||||
</th>
|
||||
<td>
|
||||
<strong>Files:</strong><br/>
|
||||
|
||||
@@ -1440,7 +1440,7 @@
|
||||
reduces time for connection setup/teardown. The proxy also takes care of compressed headers for you.
|
||||
To use this featrure you will need to install socketpool (sudo pip install socketpool or sudo easy_install socketpool)
|
||||
(ensure python2 is default) and edit the configuration file (nntpproxy.conf and nntpproxy_a.conf) in the
|
||||
update_scripts/nix_scripts/tmux/python/lib (copy sample) directory and finally edit your www/config.php file
|
||||
update_scripts/python/lib (copy sample) directory and finally edit your www/config.php file
|
||||
to use the proxy (username and password are ignored by the proxy - make then anything you like - the
|
||||
proxy doesn't use ssl either). Make sure you turn off the use compressed headers option here in site
|
||||
preferences (the proxy uses compressed headers by default and passes on decompressed data).
|
||||
|
||||
Reference in New Issue
Block a user