starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-09-01 10:48:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add functioning backfill.

Browse Source
This commit is contained in:
Darko
2014-09-29 13:20:18 +02:00
parent e0f073f87a
commit 1b5cc06fab
12 changed files with 800 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/backfill_threaded.py
+9 -9
View File
@@ -23,7 +23,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
#get values from db
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
type = int(dbgrab[0][1])
@@ -49,7 +49,7 @@ else:
if intbackfilltype == 1:
backfilldays = "backfill_target"
elif intbackfilltype == 2:
backfilldays = "datediff(curdate(),(SELECT value FROM tmux WHERE setting = 'safebackfilldate'))"
backfilldays = "datediff(curdate(),(SELECT value FROM site WHERE setting = 'safebackfilldate'))"
#exit is set to safe backfill
if len(sys.argv) == 1 and type == 4:
@@ -61,12 +61,12 @@ if len(sys.argv) == 1 and type == 4:
if len(sys.argv) > 1 and sys.argv[1] == "all":
# Using string formatting is not the correct way to do this, but using +group is even worse
# removing the % before the variables at the end of the query adds quotes/escapes strings
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 %s" % (group))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group))
else:
if conf['DB_TYPE'] == "mysql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
elif conf['DB_TYPE'] == "pgsql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
datas = cur[0].fetchall()
@@ -99,9 +99,9 @@ class queue_runner(threading.Thread):
if my_id:
time_of_last_run = time.time()
if len(sys.argv) > 1 and sys.argv[1] == "all":
subprocess.call(["php", pathname+"/../bin/backfill_all_quick.php", ""+my_id])
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
else:
subprocess.call(["php", pathname+"/../bin/backfill_interval.php", ""+my_id])
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -127,7 +127,7 @@ def main(args):
#now load some arbitrary jobs into the queue
for gnames in datas:
time.sleep(.03)
my_queue.put("%s %s" % (gnames[0], type))
my_queue.put("%s %s" % (gnames[0], type))
my_queue.join()
@@ -135,4 +135,4 @@ def main(args):
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
main(sys.argv[1:])
main(sys.argv[1:])