Move files around.

This commit is contained in:
Darko
2015-05-28 12:36:00 +02:00
parent a5ea73deec
commit ae0434ac11
1403 changed files with 13 additions and 18 deletions
@@ -0,0 +1,78 @@
<?php
use newznab\db\DB;
use newznab\utility\Utility;
$pdo = new DB();
$s = new Sites();
$site = $s->get();
if (isset($argv[1]) && ($argv[1] === "true" || $argv[1] === "delete")) {
$releases = new \Releases(['Settings' => $pdo]);
$nzb = new \NZB($pdo);
$releaseImage = new \ReleaseImage($pdo);
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$timestart = time();
$checked = $deleted = 0;
$couldbe = $argv[1] === "true" ? $couldbe = "could be " : "were ";
echo $pdo->log->header('Getting List of nzbs to check against db.');
$dirItr = new \RecursiveDirectoryIterator($site->nzbpath);
$itr = new \RecursiveIteratorIterator($dirItr, \RecursiveIteratorIterator::LEAVES_ONLY);
foreach ($itr as $filePath) {
if (is_file($filePath) && preg_match('/([a-f-0-9]+)\.nzb\.gz/', $filePath, $guid)) {
$nzbfile = Utility::unzipGzipFile($filePath);
if ($nzbfile) {
$nzbfile = @simplexml_load_string($nzbfile);
}
if ($nzbfile) {
$res = $pdo->queryOneRow(sprintf("SELECT id, guid FROM releases WHERE guid = %s", $pdo->escapeString(stristr($filePath->getFilename(), '.nzb.gz', true))));
if ($res === false) {
if ($argv[1] === "delete") {
@copy($filePath, NN_ROOT . "pooped/" . $guid[1] . ".nzb.gz");
$releases->deleteSingle(['g' => $guid[1], 'i' => false], $nzb, $releaseImage);
$deleted++;
}
} else if (isset($res)) {
$pdo->queryExec(sprintf("UPDATE releases SET nzbstatus = 1 WHERE id = %s", $res['id']));
}
} else {
if ($argv[1] === "delete") {
@copy($filePath, NN_ROOT . "pooped/" . $guid[1] . ".nzb.gz");
unlink($filePath);
$deleted++;
}
}
$time = $consoletools->convertTime(time() - $timestart);
$consoletools->overWritePrimary('Checking NZBs: ' . $deleted . ' nzbs of ' . ++$checked . ' releases checked ' . $couldbe . 'deleted from disk, Running time: ' . $time);
}
}
echo $pdo->log->header("\n" . number_format($checked) . ' nzbs checked, ' . number_format($deleted) . ' nzbs ' . $couldbe . 'deleted.');
$timestart = time();
$checked = $deleted = 0;
echo $pdo->log->header("Getting List of releases to check against nzbs.");
$res = $pdo->queryDirect('SELECT id, guid FROM releases');
if ($res instanceof \Traversable) {
foreach ($res as $row) {
$nzbpath = $nzb->getNZBPath($row["guid"]);
if (!file_exists($nzbpath)) {
if ($argv[1] === "delete") {
@copy($nzbpath, NN_ROOT . "pooped/" . $row["guid"] . ".nzb.gz");
$releases->deleteSingle(['g' => $row['guid'], 'i' => $row['id']], $nzb, $releaseImage);
}
$deleted++;
} else if (file_exists($nzbpath) && isset($row)) {
$pdo->queryExec(sprintf("UPDATE releases SET nzbstatus = 1 WHERE id = %s", $row['id']));
}
$time = $consoletools->convertTime(TIME() - $timestart);
$consoletools->overWritePrimary('Checking Releases: ' . $deleted . " releases have no nzb of " . ++$checked . " and " . $couldbe . "deleted from db, Running time: " . $time);
}
}
echo $pdo->log->header("\n" . number_format($checked) . " releases checked, " . number_format($deleted) . " releases " . $couldbe . "deleted.");
} else {
exit($pdo->log->error("\nThis script can remove all nzbs not found in the db and all releases with no nzbs found. It can also delete invalid nzbs.\n\n"
. "php $argv[0] true ...: For a dry run, to see how many would be deleted.\n"
. "php $argv[0] delete ...: To delete all affected.\n"));
}
@@ -0,0 +1,302 @@
<?php
// TODO: bunch of if/elses need converting to switches
use newznab\db\DB;
/*
*
* This was added because I starting writing this before
* all of the regexes were converted to by group in ReleaseCleaning.php
* and I do not want to convert these regexes to run per group.
* ReleaseCleaning.php is where the regexes should go
* so that all new releases can be effected by them
* instead of having to run this script to rename after the
* release has been created
*
*/
$pdo = new DB();
if (!(isset($argv[1]) && ($argv[1] == "all" || $argv[1] == "full" || $argv[1] == "preid" || is_numeric($argv[1])))) {
exit($pdo->log->error(
"\nThis script will attempt to rename releases using regexes first from ReleaseCleaning.php and then from this file.\n"
. "An optional last argument, show, will display the release name changes.\n\n"
. "php $argv[0] full ...: To process all releases not previously renamed.\n"
. "php $argv[0] 2 ...: To process all releases added in the previous 2 hours not previously renamed.\n"
. "php $argv[0] all ...: To process all releases.\n"
. "php $argv[0] full 155 ...: To process all releases in groupid 155 not previously renamed.\n"
. "php $argv[0] all 155 ...: To process all releases in groupid 155.\n"
. "php $argv[0] all '(155, 140)' ...: To process all releases in group_ids 155 and 140.\n"
. "php $argv[0] preid ...: To process all releases where not matched to predb.\n"
));
}
preName($argv, $argc);
function preName($argv, $argc)
{
global $pdo;
$groups = new \Groups(['Settings' => $pdo]);
$category = new \Categorize(['Settings' => $pdo]);
$internal = $external = $pre = 0;
$show = 2;
if ($argv[$argc - 1] === 'show') {
$show = 1;
} else if ($argv[$argc - 1] === 'bad') {
$show = 3;
}
$counter = 0;
$pdo->log = new \ColorCLI();
$full = $all = $usepre = false;
$what = $where = '';
if ($argv[1] === 'full') {
$full = true;
} else if ($argv[1] === 'all') {
$all = true;
} else if ($argv[1] === 'preid') {
$usepre = true;
} else if (is_numeric($argv[1])) {
$what = ' AND adddate > NOW() - INTERVAL ' . $argv[1] . ' HOUR';
}
if ($usepre === true) {
$where = '';
$why = ' WHERE prehashid = 0 AND nzbstatus = 1';
} else if (isset($argv[1]) && is_numeric($argv[1])) {
$where = '';
$why = ' WHERE nzbstatus = 1 AND isrenamed = 0';
} else if (isset($argv[2]) && is_numeric($argv[2]) && $full === true) {
$where = ' AND groupid = ' . $argv[2];
$why = ' WHERE nzbstatus = 1 AND isrenamed = 0';
} else if (isset($argv[2]) && preg_match('/\([\d, ]+\)/', $argv[2]) && $full === true) {
$where = ' AND groupid IN ' . $argv[2];
$why = ' WHERE nzbstatus = 1 AND isrenamed = 0';
} else if (isset($argv[2]) && preg_match('/\([\d, ]+\)/', $argv[2]) && $all === true) {
$where = ' AND groupid IN ' . $argv[2];
$why = ' WHERE nzbstatus = 1';
} else if (isset($argv[2]) && is_numeric($argv[2]) && $all === true) {
$where = ' AND groupid = ' . $argv[2];
$why = ' WHERE nzbstatus = 1 and prehashid = 0';
} else if (isset($argv[2]) && is_numeric($argv[2])) {
$where = ' AND groupid = ' . $argv[2];
$why = ' WHERE nzbstatus = 1 AND isrenamed = 0';
} else if ($full === true) {
$why = ' WHERE nzbstatus = 1 AND (isrenamed = 0 OR categoryid between 8000 AND 8999)';
} else if ($all === true) {
$why = ' WHERE nzbstatus = 1';
} else {
$why = ' WHERE 1=1';
}
resetSearchnames();
echo $pdo->log->header(
"SELECT id, name, searchname, fromname, size, groupid, categoryid FROM releases" . $why . $what .
$where . ";\n"
);
$res = $pdo->queryDirect("SELECT id, name, searchname, fromname, size, groupid, categoryid FROM releases" . $why . $what . $where);
$total = $res->rowCount();
if ($total > 0) {
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
foreach ($res as $row) {
$groupname = $groups->getByNameByID($row['groupid']);
$cleanerName = releaseCleaner($row['name'], $row['fromname'], $row['size'], $groupname, $usepre);
$preid = 0;
$predb = $predbfile = $increment = false;
if (!is_array($cleanerName)) {
$cleanName = trim((string)$cleanerName);
$propername = $increment = true;
if ($cleanName != '' && $cleanerName != false) {
$run = $pdo->queryOneRow("SELECT id FROM prehash WHERE title = " . $pdo->escapeString($cleanName));
if (isset($run['id'])) {
$preid = $run['id'];
$predb = true;
}
}
} else {
$cleanName = trim($cleanerName["cleansubject"]);
$propername = $cleanerName["properlynamed"];
if (isset($cleanerName["increment"])) {
$increment = $cleanerName["increment"];
}
if (isset($cleanerName["predb"])) {
$preid = $cleanerName["predb"];
$predb = true;
}
}
if ($cleanName != '') {
if (preg_match('/alt\.binaries\.e\-?book(\.[a-z]+)?/', $groupname)) {
if (preg_match('/^[0-9]{1,6}-[0-9]{1,6}-[0-9]{1,6}$/', $cleanName, $match)) {
$rf = new \ReleaseFiles($pdo);
$files = $rf->get($row['id']);
foreach ($files as $f) {
if (preg_match(
'/^(?P<title>.+?)(\\[\w\[\]\(\). -]+)?\.(pdf|htm(l)?|epub|mobi|azw|tif|doc(x)?|lit|txt|rtf|opf|fb2|prc|djvu|cb[rz])/', $f["name"],
$match
)
) {
$cleanName = $match['title'];
break;
}
}
}
}
//try to match clean name against predb filename
$prefile = $pdo->queryOneRow("SELECT id, title FROM prehash WHERE filename = " . $pdo->escapeString($cleanName));
if (isset($prefile['id'])) {
$preid = $prefile['id'];
$cleanName = $prefile['title'];
$predbfile = true;
$propername = true;
}
if ($cleanName != $row['name'] && $cleanName != $row['searchname']) {
if (strlen(utf8_decode($cleanName)) <= 3) {
} else {
$determinedcat = $category->determineCategory($row["groupid"], $cleanName);
if ($propername == true) {
$pdo->queryExec(
sprintf(
"UPDATE releases SET rageid = -1, seriesfull = NULL, season = NULL, episode = NULL, tvtitle = NULL, tvairdate = NULL, imdbid = NULL, musicinfoid = NULL, consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, "
. "iscategorized = 1, isrenamed = 1, searchname = %s, categoryid = %d, prehashid = " . $preid . " WHERE id = %d", $pdo->escapeString($cleanName), $determinedcat, $row['id']
)
);
} else {
$pdo->queryExec(
sprintf(
"UPDATE releases SET rageid = -1, seriesfull = NULL, season = NULL, episode = NULL, tvtitle = NULL, tvairdate = NULL, imdbid = NULL, musicinfoid = NULL, consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, "
. "iscategorized = 1, searchname = %s, categoryid = %d, prehashid = " . $preid . " WHERE id = %d", $pdo->escapeString($cleanName), $determinedcat, $row['id']
)
);
}
if ($increment === true) {
$internal++;
} else if ($predb === true) {
$pre++;
} else if ($predbfile === true) {
$pre++;
} else if ($propername === true) {
$external++;
}
if ($show === 1) {
$oldcatname = $category->getNameByID($row["categoryid"]);
$newcatname = $category->getNameByID($determinedcat);
\NameFixer::echoChangedReleaseName(array(
'new_name' => $cleanName,
'old_name' => $row["searchname"],
'new_category' => $newcatname,
'old_category' => $oldcatname,
'group' => $groupname,
'release_id' => $row["id"],
'method' => 'lib/testing/Dev/renametopre.php'
)
);
}
}
} else if ($show === 3 && preg_match('/^\[?\d*\].+?yEnc/i', $row['name'])) {
echo $pdo->log->primary($row['name']);
}
}
if ($cleanName == $row['name']) {
$pdo->queryExec(sprintf("UPDATE releases SET isrenamed = 1, iscategorized = 1 WHERE id = %d", $row['id']));
}
if ($show === 2 && $usepre === false) {
$consoletools->overWritePrimary("Renamed Releases: [Internal=" . number_format($internal) . "][External=" . number_format($external) . "][Predb=" . number_format($pre) . "] " . $consoletools->percentString(++$counter, $total));
} else if ($show === 2 && $usepre === true) {
$consoletools->overWritePrimary("Renamed Releases: [" . number_format($pre) . "] " . $consoletools->percentString(++$counter, $total));
}
}
}
echo $pdo->log->header("\n" . number_format($pre) . " renamed using preDB Match\n" . number_format($external) . " renamed using ReleaseCleaning.php\n" . number_format($internal) . " using renametopre.php\nout of " . number_format($total) . " releases.\n");
if (isset($argv[1]) && is_numeric($argv[1]) && !isset($argv[2])) {
echo $pdo->log->header("Categorizing all releases using searchname from the last ${argv[1]} hours. This can take a while, be patient.");
} else if (isset($argv[1]) && $argv[1] !== "all" && isset($argv[2]) && !is_numeric($argv[2]) && !preg_match('/\([\d, ]+\)/', $argv[2])) {
echo $pdo->log->header("Categorizing all non-categorized releases in other->misc using searchname. This can take a while, be patient.");
} else if (isset($argv[1]) && isset($argv[2]) && (is_numeric($argv[2]) || preg_match('/\([\d, ]+\)/', $argv[2]))) {
echo $pdo->log->header("Categorizing all non-categorized releases in ${argv[2]} using searchname. This can take a while, be patient.");
} else {
echo $pdo->log->header("Categorizing all releases using searchname. This can take a while, be patient.");
}
$timestart = TIME();
if (isset($argv[1]) && is_numeric($argv[1])) {
$relcount = catRelease("searchname", "WHERE (iscategorized = 0 OR categoryid = 8010) AND adddate > NOW() - INTERVAL " . $argv[1] . " HOUR", true);
} else if (isset($argv[2]) && preg_match('/\([\d, ]+\)/', $argv[2]) && $full === true) {
$relcount = catRelease("searchname", str_replace(" AND", "WHERE", $where) . " AND iscategorized = 0 ", true);
} else if (isset($argv[2]) && preg_match('/\([\d, ]+\)/', $argv[2]) && $all === true) {
$relcount = catRelease("searchname", str_replace(" AND", "WHERE", $where), true);
} else if (isset($argv[2]) && is_numeric($argv[2]) && $argv[1] == "full") {
$relcount = catRelease("searchname", str_replace(" AND", "WHERE", $where) . " AND iscategorized = 0 ", true);
} else if (isset($argv[2]) && is_numeric($argv[2]) && $argv[1] == "all") {
$relcount = catRelease("searchname", str_replace(" AND", "WHERE", $where), true);
} else if (isset($argv[1]) && $argv[1] == "full") {
$relcount = catRelease("searchname", "WHERE categoryid = 8010 OR iscategorized = 0", true);
} else if (isset($argv[1]) && $argv[1] == "all") {
$relcount = catRelease("searchname", "", true);
} else if (isset($argv[1]) && $argv[1] == "preid") {
$relcount = catRelease("searchname", "WHERE prehashid = 0 AND nzbstatus = 1", true);
} else {
$relcount = catRelease("searchname", "WHERE (iscategorized = 0 OR categoryid = 8010) AND adddate > NOW() - INTERVAL " . $argv[1] . " HOUR", true);
}
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$time = $consoletools->convertTime(TIME() - $timestart);
echo $pdo->log->header("Finished categorizing " . number_format($relcount) . " releases in " . $time . " seconds, using the usenet subject.\n");
resetSearchnames();
}
function resetSearchnames()
{
global $pdo;
echo $pdo->log->header("Resetting blank searchnames.");
$bad = $pdo->queryDirect(
"UPDATE releases SET rageid = -1, seriesfull = NULL, season = NULL, episode = NULL, tvtitle = NULL, tvairdate = NULL, imdbid = NULL, musicinfoid = NULL, consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, "
. "prehashid = 0, searchname = name, isrenamed = 0, iscategorized = 0 WHERE searchname = ''"
);
$tot = $bad->rowCount();
if ($tot > 0) {
echo $pdo->log->primary(number_format($tot) . " Releases had no searchname.");
}
echo $pdo->log->header("Resetting searchnames that are 8 characters or less.");
$run = $pdo->queryDirect(
"UPDATE releases SET rageid = -1, seriesfull = NULL, season = NULL, episode = NULL, tvtitle = NULL, tvairdate = NULL, imdbid = NULL, musicinfoid = NULL, consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, "
. "prehashid = 0, searchname = name, isrenamed = 0, iscategorized = 0 WHERE LENGTH(searchname) <= 8 AND LENGTH(name) > 8"
);
$total = $run->rowCount();
if ($total > 0) {
echo $pdo->log->primary(number_format($total) . " Releases had searchnames that were 8 characters or less.");
}
}
// Categorizes releases.
// $type = name or searchname
// Returns the quantity of categorized releases.
function catRelease($type, $where, $echooutput = false)
{
global $pdo;
$cat = new \Categorize(['Settings' => $pdo]);
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$relcount = 0;
echo $pdo->log->primary("SELECT id, " . $type . ", groupid FROM releases " . $where);
$resrel = $pdo->queryDirect("SELECT id, " . $type . ", groupid FROM releases " . $where);
$total = $resrel->rowCount();
if ($total > 0) {
foreach ($resrel as $rowrel) {
$catId = $cat->determineCategory($rowrel['groupid'], $rowrel[$type]);
$pdo->queryExec(sprintf("UPDATE releases SET iscategorized = 1, categoryid = %d WHERE id = %d", $catId, $rowrel['id']));
$relcount++;
if ($echooutput) {
$consoletools->overWritePrimary("Categorizing: " . $consoletools->percentString($relcount, $total));
}
}
}
if ($echooutput !== false && $relcount > 0) {
echo "\n";
}
return $relcount;
}
function releaseCleaner($subject, $fromName, $size, $groupname, $usepre)
{
$groups = new \Groups();
$releaseCleaning = new \ReleaseCleaning($groups->pdo);
$cleanerName = $releaseCleaning->releaseCleaner($subject, $fromName, $size, $groupname, $usepre);
if (!is_array($cleanerName) && $cleanerName != false) {
return array("cleansubject" => $cleanerName, "properlynamed" => true, "increment" => false);
} else {
return $cleanerName;
}
}
@@ -0,0 +1,191 @@
<?php
if (!isset($argv[1]) || !isset($argv[2])) {
exit (
'Argument 1 is a input string. ie PRE name.' . PHP_EOL .
'Argument 2 is a expected hash or encoding. ie MD5 string. Passing true on Argument 3 ignores this.' . PHP_EOL .
'Argument 3 (optional) False, exit on first match. True, write all matches to text file in current path.' . PHP_EOL .
'ie: php test_hash_algorithms.php Dog.with.a.Blog.S02E16.Love.Loss.and.a.Beanbag.Toss.HDTV.x264-QCF 11506192c6d92e0c9c795b9997d9396226dbdf62' . PHP_EOL .
'ie: php test_hash_algorithms.php Anchorman.2.The.Legend.Continues.2013.UNRATED.WEBRip.x264-FLS false true' . PHP_EOL
);
}
/**
* Test various hashing/encoding/etc on a string.
* Class hash_algorithms
*/
class HashAlgorithms
{
/**
* The input string.
* @var string
*/
protected $_inputString;
/**
* The string we are expecting to get.
* @var array
*/
protected $_expectedString;
/**
* Write results to file?
* @var bool
*/
protected $_writeToFile;
/**
* @param string $inputString
* @param string $expectedString
* @param bool $writeToFile
*
* @access public
*/
public function __construct($inputString, $expectedString, $writeToFile)
{
$this->_inputString = $inputString;
$this->_expectedString = array(
$expectedString,
strtolower($expectedString),
strtoupper($expectedString),
strrev($expectedString)
);
$this->_writeToFile = $writeToFile;
$this->_testStrings();
}
/**
* Test various hash algorithms on strings.
*
* @access protected
* @void
*/
protected function _testStrings()
{
if ($this->_writeToFile) {
file_put_contents('hash_matches.txt', '');
}
$firstArray = $this->_hashesToArray($this->_inputString);
$secondArray = array();
foreach ($firstArray as $key => $value) {
if (!$this->_writeToFile) {
if (in_array($value, $this->_expectedString)) {
exit(
'[' .
$this->_inputString .
']=>[' .
$key .
']=>' .
$value .
']' .
PHP_EOL
);
}
} else {
file_put_contents('hash_matches.txt', $key . "\t\t" . $value . PHP_EOL, FILE_APPEND);
}
$secondArray[$key] = $this->_hashesToArray($value);
}
$thirdArray = array();
foreach ($secondArray as $key => $value) {
foreach ($value as $key2 => $value2) {
if (!$this->_writeToFile) {
if (in_array($value2, $this->_expectedString)) {
exit(
'[' .
$this->_inputString .
']=>[' .
$key .
']=>[' .
$firstArray[$key] .
']=>[' .
$key2 .
']=>[' .
$value2 .
']' .
PHP_EOL
);
}
} else {
file_put_contents('hash_matches.txt', $key . ' => ' . $key2 . "\t\t" . $value2 . PHP_EOL, FILE_APPEND);
}
$thirdArray[$key][$key2] = $this->_hashesToArray($value2);
}
}
foreach ($thirdArray as $key => $value) {
foreach ($value as $key2 => $value2) {
foreach ($value2 as $key3 => $value3) {
if (!$this->_writeToFile) {
if (in_array($value3, $this->_expectedString)) {
exit(
'[' .
$this->_inputString .
']=>[' .
$key .
']=>[' .
$firstArray[$key] .
']=>[' .
$key2 .
']=>[' .
$value2 .
']=>[' .
$key3 .
']=>[' .
$value3 .
']' .
PHP_EOL
);
}
} else {
file_put_contents('hash_matches.txt',
$key . ' => ' . $key2 . ' => ' . $key3 . "\t\t" . $value3 . PHP_EOL, FILE_APPEND
);
}
}
}
}
}
/**
* Return various versions of a input string to hash.
*
* @param string $string
*
* @return array
*/
protected function _hashesToArray($string)
{
$strings = array(
'input' => $string,
'lower' => strtolower($string),
'lower_reverse' => strtolower(strrev($string)),
'upper_reverse' => strtoupper(strrev($string)),
'upper' => strtoupper($string),
'reverse' => strrev($string),
'reverse_upper' => strrev(strtoupper($string)),
'reverse_lower' => strrev(strtolower($string)),
);
$hashTypes = array('md5', 'md4', 'sha1', 'sha256', 'sha512');
$tmpArray = array();
foreach ($hashTypes as $hash) {
foreach ($strings as $key => $value) {
$tmpArray[$hash . '_' . $key] = hash($hash, $value, false);
}
}
foreach ($strings as $key => $value) {
$tmpArray['input_' . $key] = $value;
$tmpArray['base64_' . $key] = base64_encode($value);
$tmpArray['crc32_' . $key] = crc32($value);
}
return $tmpArray;
}
}
new HashAlgorithms($argv[1], $argv[2], ((isset($argv[3]) && strtolower($argv[3]) === 'true') ? true : false));