starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-09-01 18:58:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add functioning backfill.

Browse Source
This commit is contained in:
Darko
2014-09-29 13:20:18 +02:00
parent e0f073f87a
commit 1b5cc06fab
12 changed files with 800 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/backfill_safe_threaded.py
+24 -19
View File
@@ -22,7 +22,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL")
cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1")
dorun = cur[0].fetchone()
#close connection to mysql
@@ -38,7 +38,7 @@ else:
info.disconnect(cur[0], cur[1])
if len(sys.argv) > 1 and sys.argv[1] not in dorun:
#before we get the groups, lets update shortgroups
subprocess.call(["php", pathname+"/../bin//update_groups.php", ""])
subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
count = 0
previous = "'alt.binaries.crap'"
@@ -47,7 +47,7 @@ previous = "'alt.binaries.crap'"
while count < 10000:
#get values from db
cur = info.connect()
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'backfillthreads') AS a, (SELECT value FROM tmux WHERE setting = 'backfill_qty') AS b, (SELECT value FROM tmux WHERE setting = 'backfill') AS c, (SELECT value FROM tmux WHERE setting = 'backfill_order') AS e, (SELECT value FROM tmux WHERE setting = 'backfill_days') AS f, (SELECT value FROM site WHERE setting = 'maxmssgs') AS g")
cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'backfillthreads') AS a, (SELECT value FROM tmux WHERE setting = 'backfill_qty') AS b, (SELECT value FROM tmux WHERE setting = 'backfill') AS c, (SELECT value FROM tmux WHERE setting = 'backfill_order') AS e, (SELECT value FROM tmux WHERE setting = 'backfill_days') AS f, (SELECT value FROM site WHERE setting = 'maxmssgs') AS g")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
backfill_qty = int(dbgrab[0][1])
@@ -74,17 +74,17 @@ while count < 10000:
if intbackfilltype == 1:
backfilldays = "backfill_target"
elif intbackfilltype == 2:
backfilldays = "datediff(curdate(),(SELECT value FROM tmux WHERE setting = 'safebackfilldate'))"
backfilldays = "datediff(curdate(),(SELECT value FROM site WHERE setting = 'safebackfilldate'))"
#query to grab backfill groups
if len(sys.argv) == 1:
if conf['DB_TYPE'] == "mysql":
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
elif conf['DB_TYPE'] == "pgsql":
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
datas = cur[0].fetchone()
else:
run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL LIMIT 1"
run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
cur[0].execute(run, (sys.argv[1]))
datas = cur[0].fetchone()
if not datas or datas[0] is None:
@@ -103,14 +103,23 @@ while count < 10000:
sys.exit()
if count == 0:
print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, skipping it".format(datas[0]) + bcolors.ENDC)
if len(sys.argv) == 2:
print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, disabling it".format(datas[0]) + bcolors.ENDC)
remove = "UPDATE groups SET backfill = 0 WHERE name = %s"
cur = info.connect()
cur[0].execute(remove, (sys.argv[1]))
cur[1].autocommit(True)
info.disconnect(cur[0], cur[1])
sys.exit()
else:
print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, skipping it".format(datas[0]) + bcolors.ENDC)
if count < 10000 and count > 0:
print(bcolors.PRIMARY + "Group {} has {} articles, in the range {} to {}".format(datas[0], "{:,}".format(count), "{:,}".format(datas[2]), "{:,}".format(datas[3])) + bcolors.ENDC)
print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC)
print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC)
group = ("{} {} BackfillAll".format(datas[0], count))
subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(group)])
group = ("{} {}".format(datas[0], count))
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
#calculate the number of items for queue
if (count > (backfill_qty * run_threads)):
@@ -138,7 +147,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+my_id])
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -163,16 +172,12 @@ def main(args):
#now load some arbitrary jobs into the queue
for i in range(0, int(geteach)):
time.sleep(.03)
my_queue.put("%s %s %s %s" % (datas[0], datas[1] - i * maxmssgs - 1, datas[1] - i * maxmssgs - maxmssgs, i+1))
my_queue.put("get_range backfill %s %s %s %s" % (datas[0], datas[1] - i * maxmssgs - maxmssgs, datas[1] - i * maxmssgs - 1, i+1))
my_queue.join()
#get postdate
final = ("{} {} Backfill".format(datas[0], int(datas[1] - (maxmssgs * geteach))))
subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(final)])
group = ("{} {}".format(datas[0], 1000))
subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(group)])
group = ("{} {}".format(datas[0], 1000))
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
if run_threads <= geteach:
print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC)
else:
@@ -183,4 +188,4 @@ def main(args):
if __name__ == '__main__':
main(sys.argv[1:])
main(sys.argv[1:])
python/backfill_threaded.py
+9 -9
View File
@@ -23,7 +23,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
#get values from db
cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
type = int(dbgrab[0][1])
@@ -49,7 +49,7 @@ else:
if intbackfilltype == 1:
backfilldays = "backfill_target"
elif intbackfilltype == 2:
backfilldays = "datediff(curdate(),(SELECT value FROM tmux WHERE setting = 'safebackfilldate'))"
backfilldays = "datediff(curdate(),(SELECT value FROM site WHERE setting = 'safebackfilldate'))"
#exit is set to safe backfill
if len(sys.argv) == 1 and type == 4:
@@ -61,12 +61,12 @@ if len(sys.argv) == 1 and type == 4:
if len(sys.argv) > 1 and sys.argv[1] == "all":
# Using string formatting is not the correct way to do this, but using +group is even worse
# removing the % before the variables at the end of the query adds quotes/escapes strings
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 %s" % (group))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group))
else:
if conf['DB_TYPE'] == "mysql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
elif conf['DB_TYPE'] == "pgsql":
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
datas = cur[0].fetchall()
@@ -99,9 +99,9 @@ class queue_runner(threading.Thread):
if my_id:
time_of_last_run = time.time()
if len(sys.argv) > 1 and sys.argv[1] == "all":
subprocess.call(["php", pathname+"/../bin/backfill_all_quick.php", ""+my_id])
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
else:
subprocess.call(["php", pathname+"/../bin/backfill_interval.php", ""+my_id])
subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -127,7 +127,7 @@ def main(args):
#now load some arbitrary jobs into the queue
for gnames in datas:
time.sleep(.03)
my_queue.put("%s %s" % (gnames[0], type))
my_queue.put("%s %s" % (gnames[0], type))
my_queue.join()
@@ -135,4 +135,4 @@ def main(args):
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
main(sys.argv[1:])
main(sys.argv[1:])