From 1f171f44ee9e4c8676781605686d01ccca7e2cd7 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Mon, 19 May 2014 00:02:25 +0200 Subject: [PATCH] Updated fixreleasenames_threaded. --- bin/fixreleasenames.php | 13 +++++++++++++ bin/monitor.php | 7 +++---- python/fixreleasenames_threaded.py | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/bin/fixreleasenames.php b/bin/fixreleasenames.php index 185e7abd1..29c4ae908 100644 --- a/bin/fixreleasenames.php +++ b/bin/fixreleasenames.php @@ -75,5 +75,18 @@ if (!isset($argv[1])) { $nntp->doQuit(); + } else if (isset($pieces[1]) && $pieces[0] == 'predbft') { + $pre = $pieces[1]; + if ($res = $db->queryOneRow(sprintf('SELECT ID AS preid, title, source, searched FROM prehash ' + . 'WHERE ID = %d', $pre + ) + ) + ) { + $namefixer->done = $namefixer->matched = false; + if ($namefixer->matchPredbFT($res, 1, 1, true, 1) !== true) { + echo '.'; + } + $namefixer->checked++; + } } } diff --git a/bin/monitor.php b/bin/monitor.php index 361451273..6c0e4dfe8 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -9,7 +9,7 @@ require_once(dirname(__FILE__) . "/../lib/showsleep.php"); require_once(dirname(__FILE__) . "/../lib/functions.php"); -$version = "0.3r1165"; +$version = "0.3r1166"; $db = new DB(); $functions = new Functions(); @@ -1750,9 +1750,8 @@ while ($i > 0) { cd $_py && $_python ${DIR}/../python/fixreleasenames_threaded.py nfo 2>&1 $log; \ $_python ${DIR}/../python/fixreleasenames_threaded.py filename 2>&1 $log; \ $_python ${DIR}/../python/fixreleasenames_threaded.py par2 2>&1 $log; \ - $_php ${DIR}/../lib/fixReleaseNames.php 4 true other yes show $log; \ - $_php ${DIR}/../lib/prehashftmatch.php 100 show $log; - $_sleep $fix_timer' 2>&1 1> /dev/null" + $_python ${DIR}/../python/fixreleasenames_threaded.py predbft 2>&1 $log; \ + $_php ${DIR}/../lib/fixReleaseNames.php 4 true other yes show $log; $_sleep $fix_timer' 2>&1 1> /dev/null" ); $time27 = TIME(); } elseif (($maxload >= get_load()) && ($fix_names == 1)) { diff --git a/python/fixreleasenames_threaded.py b/python/fixreleasenames_threaded.py index 0176dbb64..ebf19fdab 100644 --- a/python/fixreleasenames_threaded.py +++ b/python/fixreleasenames_threaded.py @@ -26,11 +26,12 @@ if len(sys.argv) == 1: + "python " + sys.argv[0] + " [nfo, filename, par2] prehashid ...: To process all releases not matched to prehashID, using [nfo, filename, par2].\n" + "python " + sys.argv[0] + " nfo clean ...: To process all releases processed by filename, using nfo.\n" + "python " + sys.argv[0] + " par2 clean ...: To process all releases processed by filename and nfo, using par2.\n" + + "python " + sys.argv[0] + " predbft clean ...: To process all releases using reverse match by PreDB title.\n" + bcolors.ENDC) sys.exit() -if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2": - print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2]\n" + bcolors.ENDC) +if sys.argv[1] != "nfo" and sys.argv[1] != "filename" and sys.argv[1] != "md5" and sys.argv[1] != "par2" and sys.argv[1] != "predbft": + print(bcolors.ERROR + "\n\An invalid argument was supplied\npostprocess_threaded.py [md5, nfo, filename, par2, predbft]\n" + bcolors.ENDC) sys.exit() if len(sys.argv) == 3 and sys.argv[1] == "nfo" and sys.argv[2] == "clean": @@ -66,7 +67,7 @@ elif len(sys.argv) > 1 and (sys.argv[1] == "filename"): datas = cur[0].fetchall() elif len(sys.argv) > 1 and (sys.argv[1] == "md5"): while len(datas) == 0 and maxtries >= -5: - run = "SELECT DISTINCT rel.ID FROM releases rel INNER JOIN releasefiles rf ON rel.ID = rf.releaseID WHERE rel.dehashstatus BETWEEN %s AND 0 AND rel.passwordstatus >= -1 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashID = 0 ORDER BY postdate ASC LIMIT %s" + run = "SELECT DISTINCT rel.ID FROM releases rel INNER JOIN releasefiles rf ON rel.ID = rf.releaseID WHERE rel.dehashstatus BETWEEN %s AND 0 AND (rel.ishashed = 1 OR rf.ishashed = 1) AND prehashID = 0 ORDER BY postdate ASC LIMIT %s" cur[0].execute(run, (maxtries, int(perrun[0])*int(run_threads[0]))) datas = cur[0].fetchall() maxtries = maxtries - 1 @@ -75,6 +76,11 @@ elif len(sys.argv) > 1 and (sys.argv[1] == "par2"): run = "SELECT ID AS releaseID, guid, groupID FROM releases WHERE categoryID = 8010 AND proc_par2 = 0 AND" + clean + "ORDER BY postdate ASC LIMIT %s" cur[0].execute(run, (int(perrun[0]) * int(run_threads[0]))) datas = cur[0].fetchall() +elif len(sys.argv) > 1 and (sys.argv[1] == "predbft"): + #This one does from oldest posts to newest posts since there are many other more efficient PreDB matching schemes + run = "SELECT ID AS preid FROM prehash WHERE LENGTH(title) >= 15 AND searched = 0 AND title NOT REGEXP '[\"\<\> ]' ORDER BY predate ASC LIMIT %s" + cur[0].execute(run, (int(perrun[0])/10 * int(run_threads[0]))) + datas = cur[0].fetchall() #close connection to mysql info.disconnect(cur[0], cur[1]) @@ -146,6 +152,10 @@ def main(): for release in datas: time.sleep(.03) my_queue.put("%s %s %s %s" % ("par2", release[0], release[1], release[2])) + elif sys.argv[1] == "predbft": + for release in datas: + time.sleep(.03) + my_queue.put("%s %s" % ("predbft", release[0])) my_queue.join()