From d4be0036502397e013d08506df63690d56a986e2 Mon Sep 17 00:00:00 2001 From: Darko Date: Thu, 26 Dec 2013 10:21:26 +0100 Subject: [PATCH 1/5] Fixes in namecleaner and requestID --- bin/monitor.php | 2 +- test/namecleaner.php | 80 ++++++++++++++++++++++++++++++++++++++++++++ test/requestID.php | 16 +++++---- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/bin/monitor.php b/bin/monitor.php index a387d4027..f509433c4 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r398"; +$version="0.3r400"; $db = new DB(); $s = new Sites(); diff --git a/test/namecleaner.php b/test/namecleaner.php index 4c6aa3e3c..05d844cfc 100755 --- a/test/namecleaner.php +++ b/test/namecleaner.php @@ -20,6 +20,86 @@ class nameCleaning $this->subject = $this->groupName = ''; $this->nofiles = false; } + + public function generic() + { + $subject = $this->subject; + $groupName = $this->groupName; + $nofiles = $this->nofiles; + + // For non music groups. + if (!preg_match('/\.(flac|lossless|mp3|music|sounds)/', $groupName)) + { + // File/part count. + $cleansubject = preg_replace('/((( \(\d\d\) -|(\d\d)? - \d\d\.|\d{4} \d\d -) | - \d\d-| \d\d\. [a-z]).+| \d\d of \d\d| \dof\d)\.mp3"?|(\)|\(|\[|\s)\d{1,5}(\/|(\s|_)of(\s|_)|-)\d{1,5}(\)|\]|\s|$|:)|\(\d{1,3}\|\d{1,3}\)|[^\d]{4}-\d{1,3}-\d{1,3}\.|\s\d{1,3}\sof\s\d{1,3}\.|\s\d{1,3}\/\d{1,3}|\d{1,3}of\d{1,3}\.|^\d{1,3}\/\d{1,3}\s|\d{1,3} - of \d{1,3}/i', ' ', $subject); + // File extensions. + $cleansubject = preg_replace('/' . $this->e0 . '/i', ' ', $cleansubject); + // File extensions - If it was not in quotes. + $cleansubject = preg_replace('/(-? [a-z0-9]+-?|\(?\d{4}\)?(_|-)[a-z0-9]+)\.jpg"?| [a-z0-9]+\.mu3"?|((\d{1,3})?\.part(\d{1,5})?|\d{1,5} ?|sample|- Partie \d+)?\.(7z|\d{3}(?=(\s|"))|avi|diz|docx?|epub|idx|iso|jpg|m3u|m4a|mds|mkv|mobi|mp4|nfo|nzb|par(\s?2|")|pdf|rar|rev|rtf|r\d\d|sfv|srs|srr|sub|txt|vol.+(par2)|xls|zip|z{2,3})"?|(\s|(\d{2,3})?-)\d{2,3}\.mp3|\d{2,3}\.pdf|\.part\d{1,4}\./i', ' ', $cleansubject); + // File Sizes - Non unique ones. + $cleansubject = preg_replace('/\d{1,3}(,|\.|\/)\d{1,3}\s(k|m|g)b|(\])?\s\d+KB\s(yENC)?|"?\s\d+\sbytes?|[- ]?\d+(\.|,)?\d+\s(g|k|m)?B\s-?(\s?yenc)?|\s\(d{1,3},\d{1,3}\s{K,M,G}B\)\s|yEnc \d+k$|{\d+ yEnc bytes}|yEnc \d+ |\(\d+ ?(k|m|g)?b(ytes)?\) yEnc$/i', ' ', $cleansubject); + // Random stuff. + $cleansubject = preg_replace('/AutoRarPar\d{1,5}|\(\d+\)( | )yEnc|\d+(Amateur|Classic)| \d{4,}[a-z]{4,} |part\d+/i', ' ', $cleansubject); + // Multi spaces. + return utf8_encode(trim(preg_replace('/\s\s+/', ' ', $cleansubject))); + } // Music groups. + else + { + // Try some music group regexes. + $musicsubject = $this->musicSubject($subject); + if ($musicsubject !== false) + return $musicsubject; + else + { + // Parts/files + $cleansubject = preg_replace('/((( \(\d\d\) -|(\d\d)? - \d\d\.|\d{4} \d\d -) | - \d\d-| \d\d\. [a-z]).+| \d\d of \d\d| \dof\d)\.mp3"?|(\(|\[|\s)\d{1,4}(\/|(\s|_)of(\s|_)|-)\d{1,4}(\)|\]|\s|$|:)|\(\d{1,3}\|\d{1,3}\)|-\d{1,3}-\d{1,3}\.|\s\d{1,3}\sof\s\d{1,3}\.|\s\d{1,3}\/\d{1,3}|\d{1,3}of\d{1,3}\.|^\d{1,3}\/\d{1,3}\s|\d{1,3} - of \d{1,3}/i', ' ', $subject); + // Anything between the quotes. Too much variance within the quotes, so remove it completely. + $cleansubject = preg_replace('/".+"/i', ' ', $cleansubject); + // File extensions - If it was not in quotes. + $cleansubject = preg_replace('/(-? [a-z0-9]+-?|\(?\d{4}\)?(_|-)[a-z0-9]+)\.jpg"?| [a-z0-9]+\.mu3"?|((\d{1,3})?\.part(\d{1,5})?|\d{1,5} ?|sample|- Partie \d+)?\.(7z|\d{3}(?=(\s|"))|avi|diz|docx?|epub|idx|iso|jpg|m3u|m4a|mds|mkv|mobi|mp4|nfo|nzb|par(\s?2|")|pdf|rar|rev|rtf|r\d\d|sfv|srs|srr|sub|txt|vol.+(par2)|xls|zip|z{2,3})"?|(\s|(\d{2,3})?-)\d{2,3}\.mp3|\d{2,3}\.pdf|\.part\d{1,4}\./i', ' ', $cleansubject); + // File Sizes - Non unique ones. + $cleansubject = preg_replace('/\d{1,3}(,|\.|\/)\d{1,3}\s(k|m|g)b|(\])?\s\d+KB\s(yENC)?|"?\s\d+\sbytes?|[- ]?\d+[.,]?\d+\s(g|k|m)?B\s-?(\s?yenc)?|\s\(d{1,3},\d{1,3}\s{K,M,G}B\)\s|yEnc \d+k$|{\d+ yEnc bytes}|yEnc \d+ |\(\d+ ?(k|m|g)?b(ytes)?\) yEnc$/i', ' ', $cleansubject); + // Random stuff. + $cleansubject = preg_replace('/AutoRarPar\d{1,5}|\(\d+\)( | )yEnc|\d+(Amateur|Classic)| \d{4,}[a-z]{4,} |part\d+/i', ' ', $cleansubject); + // Multi spaces. + $cleansubject = utf8_encode(trim(preg_replace('/\s\s+/i', ' ', $cleansubject))); + + // If the subject is too similar to another because it is so short, try to extract info from the subject. + if (strlen($cleansubject) <= 10 || preg_match('/^[-a-z0-9$ ]{1,7}yEnc$/i', $cleansubject)) + { + $x = ''; + if (preg_match('/.*("[A-Z0-9]+).*?"/i', $subject, $match)) + $x = $match[1]; + if (preg_match_all('/[^A-Z0-9]/i', $subject, $match1)) + { + $start = 0; + foreach ($match1[0] as $add) + { + if ($start > 2) + break; + $x .= $add; + $start++; + } + } + $newname = preg_replace('/".+?"/', '', $subject); + $newname = preg_replace('/[a-z0-9]|' . $this->e0 . '/i', '', $newname); + return $cleansubject . $newname . $x; + } + else + return $cleansubject; + } + } + } + + // Generic regexes for music groups. + public function musicSubject($subject) + { + //Broderick_Smith-Unknown_Country-2009-404 "00-broderick_smith-unknown_country-2009.sfv" yEnc + if (preg_match('/^(\w{10,}-[a-zA-Z0-9]+ ")\d\d-.+?" yEnc$/', $subject, $match)) + return $match[1]; + else + return false; + } /* Cleans a usenet subject before inserting, used for searchname. Also used for imports. diff --git a/test/requestID.php b/test/requestID.php index e82eb57f4..36b086e5d 100644 --- a/test/requestID.php +++ b/test/requestID.php @@ -98,13 +98,14 @@ else if (isset($argv[1]) && is_numeric($argv[1])) echo '.'; } } - if ($total > 0) - echo $c->header("\nRenamed ".$counter." releases in ".$consoletools->convertTime(TIME() - $timestart)."."); - else - echo $c->info("\nNothing to do."); + if ($total > 0) { + echo $c->header("\nRenamed " . number_format($counter) . " releases in " . $consoletools->convertTime(TIME() - $timestart) . "."); + } else { + echo $c->info("\nNothing to do."); } - else - echo $c->info("No work to process\n"); +} else { + echo $c->info("No work to process\n"); +} function localLookup($requestID, $groupName, $oldname) @@ -114,6 +115,9 @@ else if (isset($argv[1]) && is_numeric($argv[1])) $functions = new Functions(); $groupID = $functions->getIDByName($groupName); $run = $db->queryOneRow(sprintf("SELECT title FROM prehash WHERE requestID = %d AND groupID = %d", $requestID, $groupID)); + if (isset($run['title']) && preg_match('/s\d+/i', $run['title']) && !preg_match('/s\d+e\d+/i', $run['title'])) { + return false; + } if (isset($run['title'])) return $run['title']; if (preg_match('/\[#?a\.b\.teevee\]/', $oldname)) From e978dbc7735de6c5714b7db1c21c388ed81f9177 Mon Sep 17 00:00:00 2001 From: Darko Date: Thu, 26 Dec 2013 10:34:47 +0100 Subject: [PATCH 2/5] bugfix --- bin/monitor.php | 2 +- test/jb_fix_names.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/monitor.php b/bin/monitor.php index f509433c4..d2b8bbd7d 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r400"; +$version="0.3r401"; $db = new DB(); $s = new Sites(); diff --git a/test/jb_fix_names.php b/test/jb_fix_names.php index dde9b986b..92d967a71 100644 --- a/test/jb_fix_names.php +++ b/test/jb_fix_names.php @@ -183,7 +183,7 @@ function releaseCleaner($subject, $groupID, $ID, $groupname) else if (!empty($cleanerName) && !is_array($cleanerName)) return array("cleansubject" => $cleanerName, "properlynamed" => true, "increment" => false); - if ($groupName == "alt.binaries.classic.tv.shows") + if ($groupname == "alt.binaries.classic.tv.shows") { if (preg_match('/^(?P.+\d+x\d+.+)[ _-]{0,3}\[\d+\/\d+\][ _-]{0,3}("|#34;).+("|#34;)[ _-]{0,3}(yEnc|rar|par2)$/i', $subject, $match)) { From d9d96a24f0b469548b1a0276a7ccfe4f177d4f2e Mon Sep 17 00:00:00 2001 From: Darko <dkrisan@gmail.com> Date: Thu, 26 Dec 2013 13:39:47 +0100 Subject: [PATCH 3/5] Bugfix --- bin/monitor.php | 2 +- test/prehash.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/monitor.php b/bin/monitor.php index d2b8bbd7d..caa350c19 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r401"; +$version="0.3r402"; $db = new DB(); $s = new Sites(); diff --git a/test/prehash.php b/test/prehash.php index a9d2cff55..8494cfdf7 100755 --- a/test/prehash.php +++ b/test/prehash.php @@ -492,7 +492,7 @@ Class Predb $md5 = $db->escapeString(md5($matches2["title"])); $predate = $db->escapeString($matches2["predate"]); $source = $db->escapeString('allfilled'); - $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $predate, $requestID, $groupID)); + $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $requestID, $groupID)); $newnames++; } } @@ -529,7 +529,7 @@ Class Predb $md5 = $db->escapeString(md5($matches2["title"])); $predate = $db->escapeString($matches2["predate"]); $source = $db->escapeString('allfilled'); - $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $predate, $requestID, $groupID)); + $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $requestID, $groupID)); $newnames++; } } @@ -566,7 +566,7 @@ Class Predb $md5 = $db->escapeString(md5($matches2["title"])); $predate = $db->escapeString($matches2["predate"]); $source = $db->escapeString('allfilled'); - $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $predate, $requestID, $groupID)); + $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $requestID, $groupID)); $newnames++; } } @@ -603,7 +603,7 @@ Class Predb $md5 = $db->escapeString(md5($matches2["title"])); $predate = $db->escapeString($matches2["predate"]); $source = $db->escapeString('allfilled'); - $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $predate, $requestID, $groupID)); + $run = $db->exec(sprintf("INSERT IGNORE INTO prehash (title, predate, adddate, source, hash, requestID, groupID) VALUES (%s, %s, now(), %s, %s, %s, %d) ON DUPLICATE KEY UPDATE requestID = %d, groupID = %d", $title, $predate, $source, $md5, $requestID, $groupID, $requestID, $groupID)); $newnames++; } } From 14525eab600bf40b8aef2f67963819b11b3205b0 Mon Sep 17 00:00:00 2001 From: Darko <dkrisan@gmail.com> Date: Thu, 26 Dec 2013 15:56:23 +0100 Subject: [PATCH 4/5] Small changes. --- bin/monitor.php | 2 +- test/requestID.php | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/monitor.php b/bin/monitor.php index caa350c19..59ccf39b5 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r402"; +$version="0.3r403"; $db = new DB(); $s = new Sites(); diff --git a/test/requestID.php b/test/requestID.php index 36b086e5d..4335a6481 100644 --- a/test/requestID.php +++ b/test/requestID.php @@ -35,16 +35,16 @@ else if (isset($argv[1]) && is_numeric($argv[1])) $total = $res->rowCount(); if ($total > 0) { - $precount = $db->queryOneRow('SELECT COUNT(*) AS count FROM prehash WHERE requestID > 0'); + $precount = $db->queryOneRow("SELECT COUNT(*) AS count FROM prehash WHERE requestID > 0"); echo $c->header("\nComparing ".number_format($total).' releases against '.number_format($precount['count'])." Local requestID's."); sleep(2); foreach ($res as $row) { - if (!preg_match('/^\[\d+\]/', $row['name']) && !preg_match('/^\[ \d+ \]/', $row['name'])) + if (!preg_match('/^\[\d+\]/', $row["name"]) && !preg_match('/^\[ \d+ \]/', $row["name"])) { - $db->exec('UPDATE releases SET reqidstatus = -2 WHERE ID = ' . $row['ID']); - exit('-'); + $db->exec("UPDATE releases SET reqidstatus = -2 WHERE ID = " . $row["ID"]); + continue; } $requestIDtmp = explode(']', substr($row["name"], 1)); @@ -68,7 +68,7 @@ else if (isset($argv[1]) && is_numeric($argv[1])) { $groupname = $functions->getByNameByID($row["groupname"]); $determinedcat = $category->determineCategory($groupname, $newTitle); - $run = $db->prepare(sprintf('UPDATE releases SET reqidstatus = 1, relnamestatus = 12, searchname = %s, categoryID = %d where ID = %d', $db->escapeString($newTitle), $determinedcat, $row["ID"])); + $run = $db->prepare(sprintf("UPDATE releases SET reqidstatus = 1, relnamestatus = 12, searchname = %s, categoryID = %d where ID = %d", $db->escapeString($newTitle), $determinedcat, $row["ID"])); $run->execute(); $counter++; if (isset($argv[2]) && $argv[2] === 'true') @@ -77,12 +77,12 @@ else if (isset($argv[1]) && is_numeric($argv[1])) $oldcatname = $functions->getNameByID($row['categoryID']); echo $c->headerOver($n.$n.'New name: ').$c->primary($newTitle). - $c->headerOver('Old name: ').$c->primary($row['name']). + $c->headerOver('Old name: ').$c->primary($row["name"]). $c->headerOver('New cat: ').$c->primary($newcatname). $c->headerOver('Old cat: ').$c->primary($oldcatname). - $c->headerOver('Group: ').$c->primary($row['groupname']). + $c->headerOver('Group: ').$c->primary($row["groupname"]). $c->headerOver('Method: ').$c->primary('requestID local'). - $c->headerOver('ReleaseID: ').$c->primary($row['ID']); + $c->headerOver('ReleaseID: ').$c->primary($row["ID"]); } else { @@ -94,7 +94,7 @@ else if (isset($argv[1]) && is_numeric($argv[1])) } else { - $db->exec('UPDATE releases SET reqidstatus = -3 WHERE ID = ' . $row["ID"]); + $db->exec("UPDATE releases SET reqidstatus = -3 WHERE ID = " . $row["ID"]); echo '.'; } } @@ -115,11 +115,11 @@ else if (isset($argv[1]) && is_numeric($argv[1])) $functions = new Functions(); $groupID = $functions->getIDByName($groupName); $run = $db->queryOneRow(sprintf("SELECT title FROM prehash WHERE requestID = %d AND groupID = %d", $requestID, $groupID)); - if (isset($run['title']) && preg_match('/s\d+/i', $run['title']) && !preg_match('/s\d+e\d+/i', $run['title'])) { + if (isset($run["title"]) && preg_match('/s\d+/i', $run["title"]) && !preg_match('/s\d+e\d+/i', $run["title"])) { return false; } - if (isset($run['title'])) - return $run['title']; + if (isset($run["title"])) + return $run["title"]; if (preg_match('/\[#?a\.b\.teevee\]/', $oldname)) $groupID = $functions->getIDByName('alt.binaries.teevee'); else if (preg_match('/\[#?a\.b\.moovee\]/', $oldname)) From 18464f9b077ff3c6dbc3469804f3f74379ec02a3 Mon Sep 17 00:00:00 2001 From: DariusIII <dkrisan@gmail.com> Date: Thu, 26 Dec 2013 18:17:24 +0100 Subject: [PATCH 5/5] Change in requestID lookup --- bin/monitor.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/monitor.php b/bin/monitor.php index 59ccf39b5..66300af6b 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -4,7 +4,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); require_once (WWW_DIR.'/lib/site.php'); -$version="0.3r403"; +$version="0.3r404"; $db = new DB(); $s = new Sites(); @@ -1380,7 +1380,7 @@ if ($array ['FIXRELEASES'] = "true") { if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time27 ) >= $array['FIXRELEASES_TIMER'] ) && ( $array['FIXRELEASES'] == "true" )) { $color = get_color(); $log = writelog($panes4[0]); - shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:4.0 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes4[0] $ds2 && cd $_test && $_php postprocess_pre.php 2>&1 $log && $_php fixReleaseNames.php 3 true other yes 2>&1 $log && $_php fixReleaseNames.php 5 true other yes 2>&1 $log && $_php fixReleaseNames.php 7 true other yes 2>&1 $log && $_php hash_decrypt.php true 2>&1 $log && $_php jb_fix_names.php 6 2>&1 $log && $_php requestID.php full true 2>&1 $log && $_php removeCrapReleases.php true 2 2>&1 $log echo \" \033[1;0;33m\" && $ds1 $panes4[0] $ds3' 2>&1 1> /dev/null"); + shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:4.0 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes4[0] $ds2 && cd $_test && $_php postprocess_pre.php 2>&1 $log && $_php fixReleaseNames.php 3 true other yes 2>&1 $log && $_php fixReleaseNames.php 5 true other yes 2>&1 $log && $_php fixReleaseNames.php 7 true other yes 2>&1 $log && $_php hash_decrypt.php true 2>&1 $log && $_php jb_fix_names.php 6 2>&1 $log && $_php requestID.php 1000 true 2>&1 $log && $_php removeCrapReleases.php true 2 2>&1 $log echo \" \033[1;0;33m\" && $ds1 $panes4[0] $ds3' 2>&1 1> /dev/null"); $time27 = TIME();} elseif ( $array['FIXRELEASES'] != "true" ) { $color = get_color();