From a8b413a2c479ca3e5416fc3046febb1f620a343f Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 15 May 2014 21:49:59 +0200 Subject: [PATCH] Added one more IRC chanel to scrape, updated match_prefiles script. --- bin/monitor.php | 2 +- lib/DB/patches/0040_tmux.sql | 3 ++ lib/IRCScraper.php | 3 +- lib/IRCScraper/settings_example.php | 3 +- lib/match_prefiles.php | 51 +++++++++++++++++++---------- lib/namefixer.php | 2 +- start.php | 2 +- 7 files changed, 43 insertions(+), 23 deletions(-) create mode 100644 lib/DB/patches/0040_tmux.sql diff --git a/bin/monitor.php b/bin/monitor.php index 494e63c7a..5df856c87 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -9,7 +9,7 @@ require_once(dirname(__FILE__) . "/../lib/showsleep.php"); require_once(dirname(__FILE__) . "/../lib/functions.php"); -$version = "0.3r1160"; +$version = "0.3r1161"; $db = new DB(); $functions = new Functions(); diff --git a/lib/DB/patches/0040_tmux.sql b/lib/DB/patches/0040_tmux.sql new file mode 100644 index 000000000..612bcc906 --- /dev/null +++ b/lib/DB/patches/0040_tmux.sql @@ -0,0 +1,3 @@ +INSERT IGNORE INTO tmux (setting, value) VALUE ('lastpretime', '0'); + +UPDATE `tmux` SET value = '40' WHERE `setting` = 'sqlpatch'; \ No newline at end of file diff --git a/lib/IRCScraper.php b/lib/IRCScraper.php index cb87be5f7..a60f1402b 100644 --- a/lib/IRCScraper.php +++ b/lib/IRCScraper.php @@ -103,7 +103,8 @@ class IRCScraper extends IRCClient '#a.b.games.xbox360' => false, '#pre@corrupt' => false, '#scnzb' => false, - '#tvnzb' => false + '#tvnzb' => false, + 'u4all.eu' => false ); } diff --git a/lib/IRCScraper/settings_example.php b/lib/IRCScraper/settings_example.php index fe5194a39..c8e18516d 100644 --- a/lib/IRCScraper/settings_example.php +++ b/lib/IRCScraper/settings_example.php @@ -39,7 +39,8 @@ serialize( '#a.b.games.xbox360' => false, '#pre@corrupt' => false, '#scnzb' => false, - '#tvnzb' => false + '#tvnzb' => false, + 'u4all.eu' => false ) ) ); \ No newline at end of file diff --git a/lib/match_prefiles.php b/lib/match_prefiles.php index 46ef17c85..dfea6e41f 100644 --- a/lib/match_prefiles.php +++ b/lib/match_prefiles.php @@ -7,48 +7,63 @@ require_once("consoletools.php"); $c = new ColorCLI(); -if (!isset($argv[1]) || ($argv[1] != "console")) { +if (!isset($argv[1]) && (($argv[1] !== "console") || ($argv[1] !== "expanded") OR ($argv[1] !== "misc") OR ($argv[1] !== "xxx"))) { exit($c->error( "\nThis script tries to match release filenames to PreDB filenames.\n" . "To display the changes, use 'show' as the second argument.\n\n" - . "php match_prefiles.php 1000 ...: to limit to 1000 releases in all categories not previously renamed sorted by newest postdate.\n" - . "php match_prefiles.php full ...: to run on full database.\n" - . "php match_prefiles.php console ...: to run on all console releases (recommended).\n" - . "php match_prefiles.php all ...: to run on all releases with filenames (including previously renamed).\n" - . "\nCurrently the only available method is console.\n" + . "php match_prefiles.php full ...: to run on full database.\n" + . "php match_prefiles.php expanded ...: to run on console, xxx, and misc releases.\n" + . "php match_prefiles.php console ...: to run on all console releases.\n" + . "php match_prefiles.php misc ...: to run on all misc releases.\n" + . "php match_prefiles.php xxx ...: to run on all xxx releases.\n" + . "php match_prefiles.php all ...: to run on all releases with filenames (including previously renamed).\n" + . "\nCurrently the only available methods are: console, misc, and xxx. You can also use the expanded keyword to run against all of these choices.\n" )); } echo $c->header("\nMatch PreFiles (${argv[1]}) Started at " . date('g:i:s')); echo $c->primary("Matching prehash filename to SUBSTRING_INDEX releasefiles.name.\n"); -preName($argv); +preFileName($argv); -function preName($argv) +function preFileName($argv) { $db = new DB(); $timestart = TIME(); $namefixer = new Namefixer(); $c = new ColorCLI(); - $catrange = $renamed = $orderby = $limit = ''; + $qrycat = $renamed = $orderby = $limit = ''; + $regfilter = "AND rf.name NOT REGEXP '\\\\\\\\' AND rf.name REGEXP BINARY '^[a-z0-9]{1,20}-[a-z0-9]{1,20}\..{3}$'"; + $rfname = "SUBSTRING_INDEX(rf.name, '.', 1)"; + if (isset($argv[1]) && $argv[1] === "full") { - $renamed = "AND isrenamed = 0"; + $rfname = "rf.name"; + } else if (isset($argv[1]) && $argv[1] === "expanded") { + $qrycat = "AND (r.categoryID BETWEEN 1000 AND 1999 OR r.categoryID BETWEEN 6000 AND 6999 OR r.categoryID BETWEEN 8000 AND 8999) "; + } else if (isset($argv[1]) && $argv[1] === "console") { - $catrange = "AND categoryID BETWEEN 1000 AND 1999"; + $qrycat = "AND r.categoryID BETWEEN 1000 AND 1999 "; + } else if (isset($argv[1]) && $argv[1] === "xxx") { + $qrycat = "AND r.categoryID BETWEEN 6000 AND 6999 "; + } else if (isset($argv[1]) && $argv[1] === "misc") { + $qrycat = "AND r.categoryID BETWEEN 8000 AND 8999 "; + } else if (isset($argv[1]) && is_numeric($argv[1])) { - $renamed = "AND isrenamed = 0"; $limit = "LIMIT " . $argv[1]; $orderby = "ORDER BY postdate DESC"; } - $query = $db->queryDirect(sprintf("SELECT DISTINCT r.ID AS releaseID, r.name, r.searchname, r.groupID, r.categoryID, SUBSTRING_INDEX(rf.name, '.', 1) AS filename - FROM releases r INNER JOIN releasefiles rf ON r.ID = rf.releaseID - WHERE r.prehashID = 0 %s AND rf.name REGEXP BINARY '[a-z0-9]{1,20}-[a-z0-9]{1,20}\..{3}' - %s %s GROUP BY r.ID %s", $catrange, $renamed, $orderby, $limit - ) - ); + echo $c->headerOver(sprintf("SELECT DISTINCT r.ID AS releaseID, r.name, r.searchname, r.groupID, r.categoryID, %s AS filename " . + "FROM releases r INNER JOIN releasefiles rf ON r.ID = rf.releaseID " . + "WHERE r.prehashID = 0 %s %s %s %s" . + "GROUP BY r.ID %s", $rfname, $qrycat, $regfilter, $renamed, $orderby, $limit)) . "\n\n"; + $query = $db->queryDirect(sprintf("SELECT DISTINCT r.ID AS releaseID, r.name, r.searchname, r.groupID, r.categoryID, %s AS filename + FROM releases r INNER JOIN releasefiles rf ON r.ID = rf.releaseID + WHERE r.prehashID = 0 %s %s %s %s + GROUP BY r.ID %s", $rfname, $qrycat, $regfilter, $renamed, $orderby, $limit)); + $total = $query->rowCount(); $counter = $counted = 0; diff --git a/lib/namefixer.php b/lib/namefixer.php index 7f7ac4d3e..317bd8daa 100644 --- a/lib/namefixer.php +++ b/lib/namefixer.php @@ -469,7 +469,7 @@ class Namefixer } if ($echooutput && $show === 1) { - $this->updateRelease($release, $pre['title'], $method = "filename match source: " . $pre['source'], $echo, "Prehash file match, ", $namestatus, $show); + $this->updateRelease($release, $pre['title'], $method = "file " . $release['filename'] . " matched source: " . $pre['source'], $echo, "PreHash file match, ", $namestatus, $show); } $matching++; } diff --git a/start.php b/start.php index f8a78753e..88693eef2 100644 --- a/start.php +++ b/start.php @@ -15,7 +15,7 @@ $tmux = $t->get(); $patch = (isset($tmux->sqlpatch)) ? $tmux->sqlpatch : 0; // Check database patch version -if ($patch < 39) { +if ($patch < 40) { exit($c->error("\nYour database is not up to date. Please update.\nphp ${DIR}/lib/DB/patchDB.php\n")); } $tmux_session = (isset($tmux->tmux_session)) ? $tmux->tmux_session : 0;