Files
newznab-tmux/python/postprocess_threaded.py
T
Darko 1ef48e640d Revert "change cases"
This reverts commit 3945429cd8.
2014-06-17 10:22:12 +02:00

308 lines
18 KiB
Python

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
import sys, os, time
import threading
try:
import queue
except ImportError:
import Queue as queue
import subprocess
import string
import signal
import datetime
import lib.info as info
from lib.info import bcolors
conf = info.readConfig()
cur = info.connect()
if len(sys.argv) == 1:
print(bcolors.ERROR + "\nWrong set of arguments.\nThe first argument [additional, nfo, movie, clean] determines the postprocessing to do.\nThe optional second argument for [additional, nfo] [groupID, categoryID] allows to process only that group or category.\nThe optional second argument for [movies, tv] [clean] allows processing only properly renamed releases.\n\npython postprocess_threaded.py [additional, nfo] (optional [groupID, categoryID])\npython postprocess_threaded.py [movie, tv] (optional [clean])\n" + bcolors.ENDC)
sys.exit()
if len(sys.argv) == 3 and sys.argv[2] == "clean":
print(bcolors.HEADER + "\nPostProcess {} Clean Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
else:
print(bcolors.HEADER + "\nPostProcess {} Threaded Started at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
if sys.argv[1] == "additional":
print(bcolors.HEADER + "Downloaded: b = yEnc article, f= failed ;Processing: z = zip file, r = rar file" + bcolors.ENDC);
print(bcolors.HEADER + "Added: s = sample image, j = jpeg image, A = audio sample, a = audio mediainfo, v = video sample" + bcolors.ENDC);
print(bcolors.HEADER + "Added: m = video mediainfo, n = nfo, ^ = file details from inside the rar/zip" + bcolors.ENDC);
elif sys.argv[1] == "nfo":
print(bcolors.HEADER + "* = hidden NFO, + = NFO, - = no NFO, f = download failed." + bcolors.ENDC)
# You can limit postprocessing for additional and nfo by groupID or categoryID
if len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) < 4:
groupID = 'AND groupID = '+sys.argv[2]
print(bcolors.HEADER + "Using groupID "+sys.argv[2] + bcolors.ENDC)
elif len(sys.argv) == 3 and sys.argv[2].isdigit() and len(sys.argv[2]) == 4:
if sys.argv[2] == '1000':
groupID = 'AND categoryID BETWEEN 1000 AND 1999'
print(bcolors.HEADER + "Using categoryids 1000-1999" + bcolors.ENDC)
elif sys.argv[2] == '2000':
groupID = 'AND categoryID BETWEEN 2000 AND 2999'
print(bcolors.HEADER + "Using categoryids 2000-2999" + bcolors.ENDC)
elif sys.argv[2] == '3000':
groupID = 'AND categoryID BETWEEN 3000 AND 3999'
print(bcolors.HEADER + "Using categoryids 3000-3999" + bcolors.ENDC)
elif sys.argv[2] == '4000':
groupID = 'AND categoryID BETWEEN 4000 AND 4999'
print(bcolors.HEADER + "Using categoryids 4000-4999" + bcolors.ENDC)
elif sys.argv[2] == '5000':
groupID = 'AND categoryID BETWEEN 5000 AND 5999'
print(bcolors.HEADER + "Using categoryids 5000-5999" + bcolors.ENDC)
elif sys.argv[2] == '6000':
groupID = 'AND categoryID BETWEEN 6000 AND 6999'
print(bcolors.HEADER + "Using categoryids 6000-6999" + bcolors.ENDC)
elif sys.argv[2] == '7000':
groupID = 'AND categoryID BETWEEN 7000 AND 7999'
print(bcolors.HEADER + "Using categoryids 7000-7999" + bcolors.ENDC)
elif sys.argv[2] == '8000':
groupID = 'AND categoryID BETWEEN 8000 AND 8999'
print(bcolors.HEADER + "Using categoryids 8000-8999" + bcolors.ENDC)
else:
groupID = 'AND categoryID = '+sys.argv[2]
print(bcolors.HEADER + "Using categoryID "+sys.argv[2] + bcolors.ENDC)
else:
groupID = ''
#you can sort tv releases by searchname
if len(sys.argv) == 3 and (sys.argv[2] == "asc" or sys.argv[2] == "desc"):
orderBY = 'ORDER BY searchname '+sys.argv[2]
print(bcolors.HEADER + "Using ORDER BY searchname "+sys.argv[2] + bcolors.ENDC)
if len(sys.argv) == 4 and (sys.argv[3] == "asc" or sys.argv[3] == "desc"):
orderBY = 'ORDER BY searchname '+sys.argv[3]
print(bcolors.HEADER + "Using CLEAN - ORDER BY searchname "+sys.argv[3] + bcolors.ENDC)
else:
orderBY = 'ORDER BY postdate DESC'
start_time = time.time()
pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
if len(sys.argv) > 1 and sys.argv[1] == "additional":
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'postthreads') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -1 "+groupID+") as j, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -2 "+groupID+") as k, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -3 "+groupID+") as l, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -4 "+groupID+") as m, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -5 "+groupID+") as n, (SELECT count(*) FROM releases WHERE haspreview = -1 and passwordstatus = -6 "+groupID+") as o")
dbgrab = cur[0].fetchall()
ps1 = format(int(dbgrab[0][9]))
ps2 = format(int(dbgrab[0][10]))
ps3 = format(int(dbgrab[0][11]))
ps4 = format(int(dbgrab[0][12]))
ps5 = format(int(dbgrab[0][13]))
ps6 = format(int(dbgrab[0][14]))
elif len(sys.argv) > 1 and sys.argv[1] == "nfo":
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'postthreads') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i, (SELECT count(*) FROM releases WHERE nfostatus = -1 AND releasenfoID = 0 "+groupID+") as j, (SELECT count(*) FROM releases WHERE nfostatus = -2 AND releasenfoID = 0 "+groupID+") as k, (SELECT count(*) FROM releases WHERE nfostatus = -3 AND releasenfoID = 0 "+groupID+") as l, (SELECT count(*) FROM releases WHERE nfostatus = -4 AND releasenfoID = 0 "+groupID+") as m, (SELECT count(*) FROM releases WHERE nfostatus = -5 AND releasenfoID = 0 "+groupID+") as n, (SELECT count(*) FROM releases WHERE nfostatus = -6 AND releasenfoID = 0 "+groupID+") as o")
dbgrab = cur[0].fetchall()
ps1 = format(int(dbgrab[0][9]))
ps2 = format(int(dbgrab[0][10]))
ps3 = format(int(dbgrab[0][11]))
ps4 = format(int(dbgrab[0][12]))
ps5 = format(int(dbgrab[0][13]))
ps6 = format(int(dbgrab[0][14]))
elif len(sys.argv) > 1 and (sys.argv[1] == "movie" or sys.argv[1] == "tv"):
cur[0].execute("SELECT(SELECT value FROM tmux WHERE setting = 'postthreadsnon') AS a, (SELECT value FROM tmux WHERE setting = 'maxaddprocessed') AS b, (SELECT value FROM tmux WHERE setting = 'maxnfoprocessed') AS c, (SELECT value FROM tmux WHERE setting = 'maximdbprocessed') AS d, (SELECT value FROM tmux WHERE setting = 'maxrageprocessed') AS e, (SELECT value FROM tmux WHERE setting = 'maxsizetopostprocess') AS f, (SELECT value FROM site WHERE setting = 'tmpunrarpath') AS g, (SELECT value FROM tmux WHERE setting = 'post') AS h, (SELECT value FROM tmux WHERE setting = 'post_non') AS i")
dbgrab = cur[0].fetchall()
else:
print(bcolors.ERROR + "\nAn argument is required, \npostprocess_threaded.py [additional, nfo, movie, tv]\n" + bcolors.ENDC)
sys.exit()
run_threads = int(dbgrab[0][0])
ppperrun = int(dbgrab[0][1])
nfoperrun = int(dbgrab[0][2])
movieperrun = int(dbgrab[0][3])
tvrageperrun = int(dbgrab[0][4])
maxsizeck = int(dbgrab[0][5])
tmppath = dbgrab[0][6]
posttorun = int(dbgrab[0][7])
postnon = dbgrab[0][8]
maxsize = (int(maxsizeck * 1073741824))
if sys.argv[1] == "additional" or sys.argv[1] == "nfo":
print(bcolors.HEADER + "Available to process: -6 = {}, -5 = {}, -4 = {}, -3 = {}, -2 = {}, -1 = {}".format(ps6, ps5, ps4, ps3, ps2, ps1) + bcolors.ENDC);
if maxsize == 0:
maxsize = ''
else:
maxsize = 'AND r.size < '+str(maxsizeck * 1073741824)
datas = []
maxtries = -1
process_additional = run_threads * ppperrun
process_nfo = run_threads * nfoperrun
if sys.argv[1] == "additional":
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -1 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, process_additional)
datas = cur[0].fetchall()
maxtries = -1
if len(datas) < process_additional:
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -2 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (process_additional - len(datas)))
datas += cur[0].fetchall()
maxtries = -2
if len(datas) < process_additional:
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -3 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (process_additional - len(datas)))
datas += cur[0].fetchall()
maxtries = -3
if len(datas) < process_additional:
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -4 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (process_additional - len(datas)))
datas += cur[0].fetchall()
maxtries = -4
if len(datas) < process_additional:
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -5 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (process_additional - len(datas)))
datas += cur[0].fetchall()
maxtries = -5
if len(datas) < process_additional:
run = "SELECT r.ID, r.guid, r.name, c.disablepreview, r.size, r.groupID, r.nfostatus, r.releasenfoID, r.categoryID, r.searchname from releases r LEFT JOIN category c ON c.ID = r.categoryID WHERE nzbstatus = 1 "+maxsize+" AND r.passwordstatus = -6 AND r.haspreview = -1 AND c.disablepreview = 0 "+groupID+" ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (process_additional - len(datas)))
datas += cur[0].fetchall()
maxtries = -6
elif sys.argv[1] == "nfo":
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -1 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo))
datas = cur[0].fetchall()
maxtries = -1
if len(datas) < process_nfo:
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -2 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas)))
datas += cur[0].fetchall()
maxtries = -2
if len(datas) < process_nfo:
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -3 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas)))
datas += cur[0].fetchall()
maxtries = -3
if len(datas) < process_nfo:
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -4 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas)))
datas += cur[0].fetchall()
maxtries = -4
if len(datas) < process_nfo:
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -5 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas)))
datas += cur[0].fetchall()
maxtries = -5
if len(datas) < process_nfo:
cur[0].execute("SELECT ID, guid, groupID, name from releases WHERE nzbstatus = 1 AND releasenfoID = 0 AND nfostatus = -6 "+groupID+" ORDER BY postdate DESC LIMIT "+str(process_nfo - len(datas)))
datas += cur[0].fetchall()
maxtries = -6
elif sys.argv[1] == "movie" and len(sys.argv) == 3 and sys.argv[2] == "clean":
run = "SELECT DISTINCT searchname AS name, ID, categoryID from releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND imdbID IS NULL AND categoryID IN (SELECT ID FROM category WHERE parentID = 2000) ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (run_threads * movieperrun))
datas = cur[0].fetchall()
elif sys.argv[1] == "movie":
run = "SELECT searchname AS name, ID, categoryID from releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND imdbID IS NULL AND categoryID IN (SELECT ID FROM category WHERE parentID = 2000) ORDER BY postdate DESC LIMIT %s"
cur[0].execute(run, (run_threads * movieperrun))
datas = cur[0].fetchall()
elif sys.argv[1] == "tv" and len(sys.argv) == 3 and sys.argv[2] == "clean":
run = "SELECT searchname, ID from releases WHERE nzbstatus = 1 AND isrenamed = 1 AND searchname IS NOT NULL AND rageID = -1 AND categoryID IN (SELECT ID FROM category WHERE parentID = 5000 ) "+orderBY+" LIMIT %s"
cur[0].execute(run, (run_threads * tvrageperrun))
datas = cur[0].fetchall()
elif sys.argv[1] == "tv":
run = "SELECT searchname, ID from releases WHERE nzbstatus = 1 AND searchname IS NOT NULL AND rageID = -1 AND categoryID IN (SELECT ID FROM category WHERE parentID = 5000 ) "+orderBY+" LIMIT %s"
cur[0].execute(run, (run_threads * tvrageperrun))
datas = cur[0].fetchall()
#close connection to mysql
info.disconnect(cur[0], cur[1])
if not datas:
print(bcolors.HEADER + "No Work to Process" + bcolors.ENDC)
sys.exit()
my_queue = queue.Queue()
time_of_last_run = time.time()
class queue_runner(threading.Thread):
def __init__(self, my_queue):
threading.Thread.__init__(self)
self.my_queue = my_queue
def run(self):
global time_of_last_run
while True:
try:
my_ID = self.my_queue.get(True, 1)
except:
if time.time() - time_of_last_run > 3:
return
else:
if my_ID:
time_of_last_run = time.time()
subprocess.call(["php", pathname+"/../bin/postprocess_alpha.php", ""+my_ID])
time.sleep(.02)
self.my_queue.task_done()
def u(x):
if sys.version_info[0] < 3:
import codecs
return codecs.unicode_escape_decode(x)[0]
else:
return x
def main(args):
global time_of_last_run
time_of_last_run = time.time()
if sys.argv[1] == "additional":
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. passwordstatus range {} to -1".format(run_threads, "{:,}".format(len(datas)), sys.argv[1], maxtries) + bcolors.ENDC)
elif sys.argv[1] == "nfo":
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases. nfostatus range {} to -1".format(run_threads, "{:,}".format(len(datas)), sys.argv[1], maxtries) + bcolors.ENDC)
else:
print(bcolors.HEADER + "We will be using a max of {} threads, a queue of {} {} releases.".format(run_threads, "{:,}".format(len(datas)), sys.argv[1]) + bcolors.ENDC)
time.sleep(2)
def signal_handler(signal, frame):
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
if True:
#spawn a pool of place worker threads
for i in range(run_threads):
p = queue_runner(my_queue)
p.setDaemon(False)
p.start()
#now load some arbitrary jobs into the queue
if sys.argv[1] == "additional":
for release in datas:
time.sleep(.02)
my_queue.put(u("%s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s =+= %s") % (release[0], release[1], release[2], release[3], release[4], release[5], release[6], release[7], release[8], release[9]))
elif sys.argv[1] == "nfo":
for release in datas:
time.sleep(.02)
my_queue.put(u("%s =+= %s =+= %s =+= %s") % (release[0], release[1], release[2], release[3]))
elif sys.argv[1] == "movie":
for release in datas:
time.sleep(.02)
my_queue.put(u("%s =+= %s =+= %s") % (release[0], release[1], release[2]))
elif sys.argv[1] == "tv":
for release in datas:
time.sleep(.02)
my_queue.put(u("%s =+= %s") % (release[0], release[1]))
my_queue.join()
if sys.argv[1] == "nfo":
cur = info.connect()
cur[0].execute("SELECT ID from releases WHERE nfostatus <= -6")
final = cur[0].fetchall()
if len(datas) > 0:
for item in final:
run = "DELETE FROM releasenfo WHERE nfo IS NULL AND releaseID = %s"
cur[0].execute(run, (item[0]))
final = cur[0].fetchall()
#close connection to mysql
info.disconnect(cur[0], cur[1])
print(bcolors.HEADER + "\nPostProcess {} Threaded Completed at {}".format(sys.argv[1],datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
main(sys.argv[1:])