Whether to put unreceived parts into partrepair table when running binaries(safe) or backfill scripts.
+
+
+
+
+
+
+
The maximum amount of articles to attempt to repair at a time. If you notice that you
+ are getting a lot of parts into the partrepair table, it is possible that you USP is not keeping up
+ with the requests. Try to reduce the threads to safe scripts, stop using safe scripts or stop using
+ nntpproxy until improves. Ar least until the cause can be determined.
+
+
+
+
+
+
+
+
Maximum amount of times to try part repair.
+
+
diff --git a/python/backfill_safe_threaded.py b/python/backfill_safe_threaded.py
index 381fdddc1..6bb6d0aa9 100644
--- a/python/backfill_safe_threaded.py
+++ b/python/backfill_safe_threaded.py
@@ -22,7 +22,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Safe Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
-cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL")
+cur[0].execute("SELECT g.name FROM groups g LEFT JOIN shortgroups ON shortgroups.name = g.name WHERE shortgroups.name IS NULL AND backfill = 1")
dorun = cur[0].fetchone()
#close connection to mysql
@@ -38,7 +38,7 @@ else:
info.disconnect(cur[0], cur[1])
if len(sys.argv) > 1 and sys.argv[1] not in dorun:
#before we get the groups, lets update shortgroups
- subprocess.call(["php", pathname+"/../bin//update_groups.php", ""])
+ subprocess.call(["php", pathname+"/../bin/update_groups.php", ""])
count = 0
previous = "'alt.binaries.crap'"
@@ -47,7 +47,7 @@ previous = "'alt.binaries.crap'"
while count < 10000:
#get values from db
cur = info.connect()
- cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'backfillthreads') AS a, (SELECT value FROM tmux WHERE setting = 'backfill_qty') AS b, (SELECT value FROM tmux WHERE setting = 'backfill') AS c, (SELECT value FROM tmux WHERE setting = 'backfill_order') AS e, (SELECT value FROM tmux WHERE setting = 'backfill_days') AS f, (SELECT value FROM site WHERE setting = 'maxmssgs') AS g")
+ cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'backfillthreads') AS a, (SELECT value FROM tmux WHERE setting = 'backfill_qty') AS b, (SELECT value FROM tmux WHERE setting = 'backfill') AS c, (SELECT value FROM tmux WHERE setting = 'backfill_order') AS e, (SELECT value FROM tmux WHERE setting = 'backfill_days') AS f, (SELECT value FROM site WHERE setting = 'maxmssgs') AS g")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
backfill_qty = int(dbgrab[0][1])
@@ -74,17 +74,17 @@ while count < 10000:
if intbackfilltype == 1:
backfilldays = "backfill_target"
elif intbackfilltype == 2:
- backfilldays = "datediff(curdate(),(SELECT value FROM tmux WHERE setting = 'safebackfilldate'))"
+ backfilldays = "datediff(curdate(),(SELECT value FROM site WHERE setting = 'safebackfilldate'))"
#query to grab backfill groups
if len(sys.argv) == 1:
if conf['DB_TYPE'] == "mysql":
- cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
+ cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL %s DAY) < g.first_record_postdate AND g.name NOT IN (%s) GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, previous, group))
elif conf['DB_TYPE'] == "pgsql":
- cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
+ cur[0].execute("SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 AND (NOW() - INTERVAL '%s DAYS') < g.first_record_postdate GROUP BY a.name, a.last_record, g.name, g.first_record %s LIMIT 1" % (backfilldays, group, groups))
datas = cur[0].fetchone()
else:
- run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL LIMIT 1"
+ run = "SELECT g.name, g.first_record AS our_first, MAX(a.first_record) AS thier_first, MAX(a.last_record) AS their_last FROM groups g INNER JOIN shortgroups a ON g.name = a.name WHERE g.name = %s AND g.first_record IS NOT NULL AND g.first_record_postdate IS NOT NULL AND g.backfill = 1 LIMIT 1"
cur[0].execute(run, (sys.argv[1]))
datas = cur[0].fetchone()
if not datas or datas[0] is None:
@@ -103,14 +103,23 @@ while count < 10000:
sys.exit()
if count == 0:
- print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, skipping it".format(datas[0]) + bcolors.ENDC)
+ if len(sys.argv) == 2:
+ print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, disabling it".format(datas[0]) + bcolors.ENDC)
+ remove = "UPDATE groups SET backfill = 0 WHERE name = %s"
+ cur = info.connect()
+ cur[0].execute(remove, (sys.argv[1]))
+ cur[1].autocommit(True)
+ info.disconnect(cur[0], cur[1])
+ sys.exit()
+ else:
+ print(bcolors.ERROR + "We have hit the maximum we can backfill for {}, skipping it".format(datas[0]) + bcolors.ENDC)
if count < 10000 and count > 0:
print(bcolors.PRIMARY + "Group {} has {} articles, in the range {} to {}".format(datas[0], "{:,}".format(count), "{:,}".format(datas[2]), "{:,}".format(datas[3])) + bcolors.ENDC)
print(bcolors.PRIMARY + "Our oldest post is: {}".format("{:,}".format(datas[1])) + bcolors.ENDC)
print(bcolors.PRIMARY + "Available Posts: {}".format("{:,}".format(count)) + bcolors.ENDC)
- group = ("{} {} BackfillAll".format(datas[0], count))
- subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(group)])
+ group = ("{} {}".format(datas[0], count))
+ subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
#calculate the number of items for queue
if (count > (backfill_qty * run_threads)):
@@ -138,7 +147,7 @@ class queue_runner(threading.Thread):
else:
if my_id:
time_of_last_run = time.time()
- subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -163,16 +172,12 @@ def main(args):
#now load some arbitrary jobs into the queue
for i in range(0, int(geteach)):
time.sleep(.03)
- my_queue.put("%s %s %s %s" % (datas[0], datas[1] - i * maxmssgs - 1, datas[1] - i * maxmssgs - maxmssgs, i+1))
+ my_queue.put("get_range backfill %s %s %s %s" % (datas[0], datas[1] - i * maxmssgs - maxmssgs, datas[1] - i * maxmssgs - 1, i+1))
my_queue.join()
- #get postdate
- final = ("{} {} Backfill".format(datas[0], int(datas[1] - (maxmssgs * geteach))))
- subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(final)])
-
- group = ("{} {}".format(datas[0], 1000))
- subprocess.call(["php", pathname+"/../bin/safe_pull.php", ""+str(group)])
+ group = ("{} {}".format(datas[0], 1000))
+ subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quantity "+str(group)])
if run_threads <= geteach:
print(bcolors.HEADER + "\nWe used {} threads, a queue of {} and grabbed {} headers".format(run_threads, "{:,}".format(geteach), "{:,}".format(geteach * maxmssgs)) + bcolors.ENDC)
else:
@@ -183,4 +188,4 @@ def main(args):
if __name__ == '__main__':
- main(sys.argv[1:])
+ main(sys.argv[1:])
\ No newline at end of file
diff --git a/python/backfill_threaded.py b/python/backfill_threaded.py
index 2e3956d22..702de8481 100644
--- a/python/backfill_threaded.py
+++ b/python/backfill_threaded.py
@@ -23,7 +23,7 @@ pathname = os.path.abspath(os.path.dirname(sys.argv[0]))
print(bcolors.HEADER + "\nBackfill Threaded Started at {}".format(datetime.datetime.now().strftime("%H:%M:%S")) + bcolors.ENDC)
#get values from db
-cur[0].execute("SELECT (SELECT value FROM tmux WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
+cur[0].execute("SELECT (SELECT value FROM site WHERE setting = 'backfillthreads') as a, (SELECT value FROM tmux WHERE setting = 'backfill') as c, (SELECT value FROM tmux WHERE setting = 'backfill_groups') as d, (SELECT value FROM tmux WHERE setting = 'backfill_order') as e, (SELECT value FROM tmux WHERE setting = 'backfill_days') as f")
dbgrab = cur[0].fetchall()
run_threads = int(dbgrab[0][0])
type = int(dbgrab[0][1])
@@ -49,7 +49,7 @@ else:
if intbackfilltype == 1:
backfilldays = "backfill_target"
elif intbackfilltype == 2:
- backfilldays = "datediff(curdate(),(SELECT value FROM tmux WHERE setting = 'safebackfilldate'))"
+ backfilldays = "datediff(curdate(),(SELECT value FROM site WHERE setting = 'safebackfilldate'))"
#exit is set to safe backfill
if len(sys.argv) == 1 and type == 4:
@@ -61,12 +61,12 @@ if len(sys.argv) == 1 and type == 4:
if len(sys.argv) > 1 and sys.argv[1] == "all":
# Using string formatting is not the correct way to do this, but using +group is even worse
# removing the % before the variables at the end of the query adds quotes/escapes strings
- cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 %s" % (group))
+ cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND backfill = 1 %s" % (group))
else:
if conf['DB_TYPE'] == "mysql":
- cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
+ cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval %s DAY) < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
elif conf['DB_TYPE'] == "pgsql":
- cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
+ cur[0].execute("SELECT name, first_record FROM groups WHERE first_record != 0 AND first_record_postdate IS NOT NULL AND backfill = 1 AND (NOW() - interval '%s DAYS') < first_record_postdate %s LIMIT %s" % (backfilldays, group, groups))
datas = cur[0].fetchall()
@@ -99,9 +99,9 @@ class queue_runner(threading.Thread):
if my_id:
time_of_last_run = time.time()
if len(sys.argv) > 1 and sys.argv[1] == "all":
- subprocess.call(["php", pathname+"/../bin/backfill_all_quick.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill_all_quick "+my_id])
else:
- subprocess.call(["php", pathname+"/../bin/backfill_interval.php", ""+my_id])
+ subprocess.call(["php", pathname+"/../../multiprocessing/.do_not_run/switch.php", "python backfill "+my_id])
time.sleep(.03)
self.my_queue.task_done()
@@ -127,7 +127,7 @@ def main(args):
#now load some arbitrary jobs into the queue
for gnames in datas:
time.sleep(.03)
- my_queue.put("%s %s" % (gnames[0], type))
+ my_queue.put("%s %s" % (gnames[0], type))
my_queue.join()
@@ -135,4 +135,4 @@ def main(args):
print(bcolors.HEADER + "Running time: {}\n\n".format(str(datetime.timedelta(seconds=time.time() - start_time))) + bcolors.ENDC)
if __name__ == '__main__':
- main(sys.argv[1:])
+ main(sys.argv[1:])
\ No newline at end of file
diff --git a/start.php b/start.php
index df462df99..be94fc9aa 100644
--- a/start.php
+++ b/start.php
@@ -15,7 +15,7 @@ $tmux = $t->get();
$patch = (isset($tmux->sqlpatch)) ? $tmux->sqlpatch : 0;
// Check database patch version
-if ($patch < 86) {
+if ($patch < 87) {
exit($c->error("\nYour database is not up to date. Please update.\nphp ${DIR}/lib/DB/patchDB.php\n"));
}
$tmux_session = (isset($tmux->tmux_session)) ? $tmux->tmux_session : 0;