diff --git a/bin/monitor.php b/bin/monitor.php index 7e21b5457..37abfc69f 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r067"; +$version="0.3r077"; $db = new DB(); $s = new Sites(); @@ -1379,7 +1379,7 @@ if ($array ['FIXRELEASES'] = "true") { if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time27 ) >= $array['FIXRELEASES_TIMER'] ) && ( $array['FIXRELEASES'] == "true" )) { $color = get_color(); $log = writelog($panes4[0]); - shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:4.0 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes4[0] $ds2 && cd $_test && $_php postprocess_pre.php 2>&1 $log && $_php fixReleaseNames.php 3 true other yes 2>&1 $log && $_php fixReleaseNames.php 5 true other yes 2>&1 $log && $_php fixReleaseNames.php 7 true other yes 2>&1 $log && $_php hash_decrypt.php true 2>&1 $log && $_php jb_fix_names.php 6 2>&1 $log && $_php removeCrapReleases.php true full 2>&1 $log echo \" \033[1;0;33m\" && $ds1 $panes4[0] $ds3' 2>&1 1> /dev/null"); + shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:4.0 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes4[0] $ds2 && cd $_test && $_php postprocess_pre.php 2>&1 $log && $_php fixReleaseNames.php 3 true other yes 2>&1 $log && $_php fixReleaseNames.php 5 true other yes 2>&1 $log && $_php fixReleaseNames.php 7 true other yes 2>&1 $log && $_php hash_decrypt.php true 2>&1 $log && $_php jb_fix_names.php 6 2>&1 $log && $_php removeCrapReleases.php true 2 2>&1 $log echo \" \033[1;0;33m\" && $ds1 $panes4[0] $ds3' 2>&1 1> /dev/null"); $time27 = TIME();} elseif ( $array['FIXRELEASES'] != "true" ) { $color = get_color(); diff --git a/test/db_updates/db.sql b/test/db_updates/db.sql index 0b9be4880..167d4c41d 100644 --- a/test/db_updates/db.sql +++ b/test/db_updates/db.sql @@ -3,8 +3,7 @@ ALTER TABLE `predb` ADD INDEX ( `hash` ( 32 ) ); ALTER TABLE `releases` ADD `dehashstatus` TINYINT( 1 ) NOT NULL DEFAULT '0' AFTER `haspreview`; ALTER TABLE `releases` ADD `nfostatus` TINYINT NOT NULL DEFAULT 0 after `dehashstatus`; ALTER TABLE `releases` ADD `relnamestatus` TINYINT NOT NULL DEFAULT 1 after `nfostatus`; -ALTER TABLE `releases` ADD `relstatus` TINYINT(4) NOT NULL DEFAULT 0 after `relnamestatus`; -ALTER TABLE `releases` ADD `hashed` BOOL DEFAULT FALSE after `relstatus`; +ALTER TABLE `releases` ADD `hashed` BOOL DEFAULT FALSE after `relnamestatus`; ALTER TABLE `releases` ADD `nzbstatus` TINYINT NOT NULL DEFAULT 0 after `hashed`; CREATE INDEX ix_releases_hashed on releases(hashed); CREATE INDEX ix_releases_mergedreleases on releases(dehashstatus, relnamestatus, passwordstatus); diff --git a/test/db_updates/db_update_04_12_2013.sql b/test/db_updates/db_update_04_12_2013.sql new file mode 100644 index 000000000..1d72709dc --- /dev/null +++ b/test/db_updates/db_update_04_12_2013.sql @@ -0,0 +1 @@ +ALTER TABLE releases DROP relstatus; diff --git a/test/db_updates/db_update_08_10_2013.sql b/test/db_updates/db_update_08_10_2013.sql index 6a7c7d335..cf1460cb9 100644 --- a/test/db_updates/db_update_08_10_2013.sql +++ b/test/db_updates/db_update_08_10_2013.sql @@ -1,4 +1,4 @@ -ALTER TABLE `releases` ADD `hashed` BOOL DEFAULT FALSE after `relstatus`; +ALTER TABLE `releases` ADD `hashed` BOOL DEFAULT FALSE after `relnamestatus`; CREATE INDEX ix_releases_hashed on releases(hashed); CREATE INDEX ix_releases_mergedreleases on releases(dehashstatus, relnamestatus, passwordstatus); UPDATE releases SET hashed = true WHERE searchname REGEXP '[a-fA-F0-9]{32}' OR name REGEXP '[a-fA-F0-9]{32}'; diff --git a/test/functions.php b/test/functions.php index bbfc935f1..854f0fb57 100755 --- a/test/functions.php +++ b/test/functions.php @@ -22,13 +22,6 @@ require_once("ColorCLI.php"); class Functions { - // - // the element relstatus of table releases is used to hold the status of the release - // The variable is a bitwise AND of status - // List of processed constants - used in releases table. Constants need to be powers of 2: 1, 2, 4, 8, 16 etc... - const NFO_PROCESSED_NAMEFIXER = 1; // We have processed the release against its .nfo file in the namefixer - const PREHASH_PROCESSED_NAMEFIXER = 2; // We have processed the release against a predb name - // database function public function fetchArray($result) { diff --git a/test/namefixer.php b/test/namefixer.php index 02601b3b0..78ba8168c 100644 --- a/test/namefixer.php +++ b/test/namefixer.php @@ -62,7 +62,7 @@ class Namefixer $functions = new Functions (); $type = "NFO, "; // Only select releases we haven't checked here before - $query = "SELECT nfo.releaseID as nfoID, rel.groupID, rel.categoryID, rel.searchname, uncompress(nfo) as textstring, rel.ID as releaseID from releases rel inner join releasenfo nfo on (nfo.releaseID = rel.ID) where relnamestatus = 1 and relstatus & " . Functions::NFO_PROCESSED_NAMEFIXER . " = 0"; + $query = "SELECT nfo.releaseID as nfoID, rel.groupID, rel.categoryID, rel.searchname, uncompress(nfo) as textstring, rel.ID as releaseID from releases rel inner join releasenfo nfo on (nfo.releaseID = rel.ID) where relnamestatus = 1"; //24 hours, other cats if ($time == 1 && $cats == 1) @@ -445,6 +445,22 @@ class Namefixer $newresult = str_replace("-ALiAS"," PC GAME ALiAS",$result['0']); $this->updateRelease($release, $newresult, $method="gameCheck: PC Games -ALiAS", $echo, $type, $namestatus); } + elseif ($this->done === false && $this->relid !== $release["releaseID"] && preg_match('/\w[\w.\',;-].+\-RELOADED/i', $release["textstring"], $result)) + { + $newresult = str_replace("-RELOADED"," PC GAME RELOADED",$result['0']); + $this->updateRelease($release, $newresult, $method="gameCheck: PC Games -RELOADED", $echo, $type, $namestatus); + } + elseif ($this->done === false && $this->relid !== $release["releaseID"] && preg_match('/\w[\w.\',;-].+\-SKIDROW/i', $release["textstring"], $result)) + { + $newresult = str_replace("-SKIDROW"," PC GAME SKIDROW",$result['0']); + $this->updateRelease($release, $newresult, $method="gameCheck: PC Games -SKIDROW", $echo, $type, $namestatus); + } + elseif ($this->done === false && $this->relid !== $release["releaseID"] && preg_match('/\w[\w.\',;-].+\-FLT/i', $release["textstring"], $result)) + { + $newresult = str_replace("-FLT"," PC GAME FLT",$result['0']); + $this->updateRelease($release, $newresult, $method="gameCheck: PC Games -FAIRLIGHT", $echo, $type, $namestatus); + } + } // diff --git a/test/prehash.php b/test/prehash.php index 8127f88b2..a0f17aacf 100755 --- a/test/prehash.php +++ b/test/prehash.php @@ -118,7 +118,7 @@ Class Predb else $nfo = $db->escapeString("http://nzb.isasecret.com/".$matches2["nfo"]); - $db->query(sprintf("INSERT IGNORE INTO prehash (title, nfo, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $nfo, $size, $db->escapeString($matches2["category"]), $db->escapeString("womble"), $db->escapeString(md5($matches2["title"])))); + $db->query(sprintf("INSERT INTO prehash (title, nfo, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $nfo, $size, $db->escapeString($matches2["category"]), $db->escapeString("womble"), $db->escapeString(md5($matches2["title"])))); $newnames++; } } @@ -164,7 +164,7 @@ Class Predb { $size = $db->escapeString(round($matches2["size1"]).$matches2["size2"]); $title = preg_replace("/ - omgwtfnzbs.org/", "", $matches2["title"]); - $db->query(sprintf("INSERT IGNORE INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($title), $size, $db->escapeString($matches2["category"]), $db->escapeString("omgwtfnzbs"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($title), $size, $db->escapeString($matches2["category"]), $db->escapeString("omgwtfnzbs"), $db->escapeString($md5))); $newnames++; } } @@ -176,49 +176,49 @@ Class Predb } public function retrieveZenet() - { - $db = new DB(); - $newnames = $updated = 0; + { + $db = new DB(); + $newnames = $updated = 0; - $buffer = getUrl("http://pre.zenet.org/live.php"); - if ($buffer !== false && strlen($buffer)) - { - if (preg_match_all('/(?P.+?)<\/td.+?(?P.+?)<\/a.+?|">(?PNUKE)+?)?<\/td.+?(?P.+?-)<a.+?<b>(?P<title2>.+?)<\/b>.+?<\/td.+?<td.+<td.+?(">(?P<size1>[\d.]+)<b>(?P<size2>.+?)<\/b>.+)?<\/tr>/s', $m, $matches2)) - { + $buffer = getUrl("http://pre.zenet.org/live.php"); + if ($buffer !== false && strlen($buffer)) + { + if (preg_match_all('/<tr bgcolor=".+?<\/tr>/s', $buffer, $matches)) + { + foreach ($matches as $match) + { + foreach ($match as $m) + { + if (preg_match('/<tr bgcolor=".+?<td.+?">(?P<date>.+?)<\/td.+?<td.+?(<font.+?">(?P<category>.+?)<\/a.+?|">(?P<category1>NUKE)+?)?<\/td.+?<td.+?">(?P<title>.+?-)<a.+?<b>(?P<title2>.+?)<\/b>.+?<\/td.+?<td.+<td.+?(">(?P<size1>[\d.]+)<b>(?P<size2>.+?)<\/b>.+)?<\/tr>/s', $m, $matches2)) + { $md5 = md5($matches2["title"].$matches2["title2"]); - $oldname = $db->queryOneRow(sprintf("SELECT title FROM prehash WHERE title = %s", $db->escapeString($md5))); - if ($oldname !== false && $oldname["md5"] == $md5) - continue; - else - { - if (!isset($matches2["size1"]) && empty($matches2["size1"])) - $size = "NULL"; - else - $size = $db->escapeString(round($matches2["size1"]).$matches2["size2"]); + $oldname = $db->queryOneRow(sprintf("SELECT hash FROM prehash WHERE hash = %s", $db->escapeString($md5))); + if ($oldname !== false && $oldname["md5"] == $md5) + continue; + else + { + if (!isset($matches2["size1"]) && empty($matches2["size1"])) + $size = "NULL"; + else + $size = $db->escapeString(round($matches2["size1"]).$matches2["size2"]); - if (isset($matches2["category"]) && !empty($matches2["category"])) - $category = $db->escapeString($matches2["category"]); - else if (isset($matches2["category1"]) && !empty($matches2["category1"])) - $category = $db->escapeString($matches2["category1"]); - else - $category = "NULL"; + if (isset($matches2["category"]) && !empty($matches2["category"])) + $category = $db->escapeString($matches2["category"]); + else if (isset($matches2["category1"]) && !empty($matches2["category1"])) + $category = $db->escapeString($matches2["category1"]); + else + $category = "NULL"; - $db->query(sprintf("INSERT IGNORE INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"].$matches2["title2"]), $size, $category, $db->escapeString("zenet"), $db->escapeString($md5))); - $newnames++; - } - } - } - } - } - } - return $newnames; - } + $db->query(sprintf("INSERT INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"].$matches2["title2"]), $size, $category, $db->escapeString("zenet"), $db->escapeString($md5))); + $newnames++; + } + } + } + } + } + } + return $newnames; + } public function retrievePrelist() { @@ -247,7 +247,7 @@ Class Predb else $size = $db->escapeString(round($matches2["size"])); - $db->query(sprintf("INSERT IGNORE INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $size, $db->escapeString($matches2["category"]), $db->escapeString("prelist"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $size, $db->escapeString($matches2["category"]), $db->escapeString("prelist"), $db->escapeString($md5))); $newnames++; } } @@ -261,7 +261,7 @@ Class Predb { $category = $db->escapeString($matches2["category"].", ".$matches2["category1"]); - $db->query(sprintf("INSERT IGNORE INTO prehash (title, category, predate, adddate, source, hash) VALUES (%s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $category, $db->escapeString("prelist"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, category, predate, adddate, source, hash) VALUES (%s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $category, $db->escapeString("prelist"), $db->escapeString($md5))); $newnames++; } } @@ -301,7 +301,7 @@ Class Predb else $size = $db->escapeString($matches2["size"]); - $db->query(sprintf("INSERT IGNORE INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $size, $db->escapeString($matches2["category"]), $db->escapeString("orlydb"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, size, category, predate, adddate, source, hash) VALUES (%s, %s, %s, FROM_UNIXTIME(".strtotime($matches2["date"])."), now(), %s, %s)", $db->escapeString($matches2["title"]), $size, $db->escapeString($matches2["category"]), $db->escapeString("orlydb"), $db->escapeString($md5))); $newnames++; } } @@ -328,7 +328,7 @@ Class Predb continue; else { - $db->query(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash) VALUES (%s, FROM_UNIXTIME(".strtotime($release->pubDate)."), now(), %s, %s)", $db->escapeString($release->title), $db->escapeString("srrdb"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, predate, adddate, source, hash) VALUES (%s, FROM_UNIXTIME(".strtotime($release->pubDate)."), now(), %s, %s)", $db->escapeString($release->title), $db->escapeString("srrdb"), $db->escapeString($md5))); $newnames++; } } @@ -354,7 +354,7 @@ Class Predb continue; else { - $db->query(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash) VALUES (%s, now(), now(), %s, %s)", $db->escapeString($release->title), $db->escapeString("predbme"), $db->escapeString($md5))); + $db->query(sprintf("INSERT INTO prehash (title, predate, adddate, source, hash) VALUES (%s, now(), now(), %s, %s)", $db->escapeString($release->title), $db->escapeString("predbme"), $db->escapeString($md5))); $newnames++; } } diff --git a/test/removeCrapReleases.php b/test/removeCrapReleases.php index 670d399bd..fb3e21760 100755 --- a/test/removeCrapReleases.php +++ b/test/removeCrapReleases.php @@ -82,7 +82,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Gibberish"; $db = new DB(); - $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '^[a-zA-Z0-9]{15,}$' AND nfostatus = 0 AND relnamestatus > 1 AND rarinnerfilecount = 0".$and); + $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '^[a-zA-Z0-9]{15,}$' AND nfostatus = 0 AND relnamestatus >= 20 AND rarinnerfilecount >= 0".$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -92,7 +92,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Hashed"; $db = new DB(); - $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '[a-zA-Z0-9]{25,}' AND nfostatus = 0 AND relnamestatus > 1 AND rarinnerfilecount = 0".$and); + $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '[a-zA-Z0-9]{25,}' AND nfostatus = 0 AND relnamestatus >= 20 AND rarinnerfilecount >= 0".$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -102,7 +102,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Short"; $db = new DB(); - $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '^[a-zA-Z0-9]{0,5}$' AND nfostatus = 0 AND relnamestatus > 1 AND rarinnerfilecount = 0".$and); + $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE searchname REGEXP '^[a-zA-Z0-9]{0,5}$' AND nfostatus = 0 AND relnamestatus >= 20 AND rarinnerfilecount >= 0".$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -112,11 +112,11 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Executable"; $db = new DB(); - $sql = $db->query('SELECT r.ID, r.guid, r.searchname FROM releases r INNER JOIN releasefiles rf ON rf.releaseID = r.ID WHERE rf.name LIKE "%.exe%" AND r.categoryID NOT IN (4000, 4010, 4020, 4050, 8010)'.$and); + $sql = $db->query('SELECT r.ID, r.guid, r.searchname FROM releases r INNER JOIN releasefiles rf ON rf.releaseID = r.ID WHERE rf.name LIKE "%.exe%" AND r.categoryID NOT IN (4000, 4010, 4020, 4050)'.$and); $delcount = deleteReleases($sql, $type); return $delcount; } - + // Anything with an install.bin file. function deleteInstallBin($and) { @@ -142,7 +142,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Passworded"; $db = new DB(); - $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE ( searchname LIKE '%passworded%' OR searchname LIKE '%password protect%' OR searchname LIKE '%password%' OR searchname LIKE '%passwort%' ) and searchname NOT LIKE '%no password%' and searchname NOT LIKE '%not passworded%' and searchname NOT LIKE '%unlocker%' and searchname NOT LIKE '%reset%' and searchname NOT LIKE '%recovery%' and searchname NOT LIKE '%keygen%' and searchname NOT LIKE '%advanced%' and nzbstatus in (1, 2) and categoryID not in (4000, 4010, 4020, 4030, 4040, 4050, 4060, 4070)".$and); + $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE ( searchname LIKE '%passworded%' OR searchname LIKE '%password protect%' OR searchname LIKE '%password%' OR searchname LIKE '%passwort%' OR searchname LIKE '%[pw]%' ) AND searchname NOT LIKE '%no password%' AND searchname NOT LIKE '%not passworded%' AND searchname NOT LIKE '%unlocker%' AND searchname NOT LIKE '%reset%' AND searchname NOT LIKE '%recovery%' AND searchname NOT LIKE '%keygen%' AND searchname NOT LIKE '%advanced%' AND nzbstatus in (1, 2) AND categoryID not in (4000, 4010, 4020, 4030, 4040, 4050, 4060, 4070)".$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -152,7 +152,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Size"; $db = new DB(); - $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE totalPart = 1 and size < 1000000 and categoryID not in (7000, 7010, 7020, 7030, 3010)".$and); + $sql = $db->query("SELECT ID, guid, searchname FROM releases WHERE totalPart = 1 AND size < 1000000 AND categoryID not in (7000, 7010, 7020, 7030, 3010)".$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -162,7 +162,7 @@ if (isset($argv[1]) && $argv[1] == "true") { $type = "Sample"; $db = new DB(); - $sql = $db->query('SELECT ID, guid, searchname FROM releases WHERE totalPart > 1 and name LIKE "%sample%" and size < 40000000 and categoryID in (5020, 5030, 5040, 5050, 5060, 5070, 5080, 2020, 2030, 2040, 2050, 2060)'.$and); + $sql = $db->query('SELECT ID, guid, searchname FROM releases WHERE totalPart > 1 AND name LIKE "%sample%" AND size < 40000000 AND categoryID IN (5020, 5030, 5040, 5050, 5060, 5070, 5080, 2020, 2030, 2040, 2050, 2060)'.$and); $delcount = deleteReleases($sql, $type); return $delcount; } @@ -255,7 +255,7 @@ if (isset($argv[1]) && $argv[1] == "true") if($PURLDeleted > 0) echo "PURL : ".$PURLDeleted."\n"; if($PassDeleted > 0) - echo "Passwordeded : ".$PassDeleted."\n"; + echo "Passworded : ".$PassDeleted."\n"; if($sizeDeleted > 0) echo "Size : ".$sizeDeleted."\n"; if($sampleDeleted > 0)