starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-09-01 02:38:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move files around.

Browse Source
This commit is contained in:
Darko
2015-05-28 12:36:00 +02:00
parent a5ea73deec
commit ae0434ac11
1403 changed files with 13 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/change_USP_provider.php
+160
View File
@@ -0,0 +1,160 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
/* This script will update the groups table to get the new article numbers for each group you have activated.
It will also truncate the parts, binaries, collections, and partrepair tables.
*/
// TODO: Make this threaded so it goes faster.
$pdo = new DB();
if (!isset($argv[1]) || $argv[1] != 'true') {
printf($pdo->log->setColor('Yellow') . "This script is used when you have switched UseNet Providers(USP) so you can pickup where you left off, rather than resetting all the groups.\nOnly use this script after you have updated your config.php file with your new USP info!!\nMake sure you " . $pdo->log->setColor('Red', 'Bold') . "DO NOT" . $pdo->log->setcolor('Yellow') . " have any update or postprocess scripts running when running this script!\n\n" . $pdo->log->setColor('Cyan') . "Usage: php change_USP_provider true\n");
exit();
}
$groups = $pdo->query("SELECT id, name, first_record_postdate, last_record_postdate FROM groups WHERE active = 1");
$numofgroups = count($groups);
$guesstime = $numofgroups * 2;
$totalstart = microtime(true);
echo "You have $numofgroups active, it takes about 2 minutes on average to processes each group.\n";
foreach ($groups as $group) {
$starttime = microtime(true);
$nntp = new \NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
return;
}
//printf("Updating group ".$group['name']."..\n");
$bfdays = daysOldstr($group['first_record_postdate']);
$currdays = daysOldstr($group['last_record_postdate']);
$bfartnum = daytopost($nntp, $group['name'], $bfdays, true, true);
echo "Our Current backfill postdate was: " . $pdo->log->setColor('Yellow') . date('r', strtotime($group['first_record_postdate'])) . $pdo->log->rsetcolor() . "\n";
$currartnum = daytopost($nntp, $group['name'], $currdays, true, false);
echo "Our Current current postdate was: " . $pdo->log->setColor('Yellow') . date('r', strtotime($group['last_record_postdate'])) . $pdo->log->rsetcolor() . "\n";
$pdo->queryExec(sprintf("UPDATE groups SET first_record = %s, last_record = %s WHERE id = %d", $pdo->escapeString($bfartnum), $pdo->escapeString($currartnum), $group['id']));
$endtime = microtime(true);
echo $pdo->log->setColor('Gray', 'Dim') . "This group took " . gmdate("H:i:s", $endtime - $starttime) . " to process.\n";
$numofgroups--;
echo "There are " . $numofgroups . " left to process.\n\n" . $pdo->log->rsetcolor() . "";
}
$totalend = microtime(true);
echo $pdo->log->header('Total time to update all groups ' . gmdate("H:i:s", $totalend - $totalstart));
// Truncate tables to complete the change to the new USP.
$arr = array("parts", "partrepair", "binaries");
foreach ($arr as &$value) {
$rel = $pdo->queryExec("TRUNCATE TABLE $value");
if ($rel !== false) {
echo $pdo->log->header("Truncating $value completed.");
}
}
unset($value);
function daysOldstr($timestamp)
{
return round((time() - strtotime($timestamp)) / 86400, 5);
}
function daysOld($timestamp)
{
return round((time() - $timestamp) / 86400, 5);
}
// This function taken from lib/backfill.php, and modified to fit our needs.
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
global $pdo;
$st = false;
if ($debug && $bfcheck) {
echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
}
if (!isset($nntp)) {
$nntp = new \NNTP(['Settings' => $pdo]);
if ($nntp->doConnect(false) !== true) {
return;
}
$st = true;
}
$binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$data = $nntp->dataError($nntp, $group, false);
if ($data === false) {
return;
}
}
// Goal timestamp.
$goaldate = date('U') - (86400 * $days);
$totalnumberofarticles = $data['last'] - $data['first'];
$upperbound = $data['last'];
$lowerbound = $data['first'];
if ($debug && $bfcheck) {
echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
}
if ($data['last'] == PHP_INT_MAX) {
exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
}
$firstDate = $binaries->postdate($data['first'], $data);
$lastDate = $binaries->postdate($data['last'], $data);
if ($goaldate < $firstDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->warning("The oldest post indexed from $days day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', $firstDate) or daysOld($firstDate) days).");
return $data['first'];
} else if ($goaldate > $lastDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->error("ERROR: The oldest post indexed from $days day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld($lastDate)+1) days (date('r', $lastDate-86400).");
return '';
}
if ($debug && $bfcheck) {
echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . ((is_int($firstDate)) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
}
$interval = floor(($upperbound - $lowerbound) * 0.5);
$templowered = '';
$dateofnextone = $lastDate;
// Match on days not timestamp to speed things up.
while (daysOld($dateofnextone) < $days) {
while (($tmpDate = $binaries->postdate(($upperbound - $interval), $data)) > $goaldate) {
$upperbound = $upperbound - $interval;
}
if (!$templowered) {
$interval = ceil(($interval / 2));
}
$dateofnextone = $binaries->postdate(($upperbound - 1), $data);
while (!$dateofnextone) {
$dateofnextone = $binaries->postdate(($upperbound - 1), $data);
}
}
if ($st === true) {
$nntp->doQuit();
}
if ($bfcheck) {
echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
} // which is '.daysOld($dateofnextone)." days old.\n";
else {
echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
} // which is '.daysOld($dateofnextone)." days old.\n";
return $upperbound;
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/convert_mysql_tables.php
+190
View File
@@ -0,0 +1,190 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
$cli = new \ColorCLI();
$pdo = new DB(['checkVersion' => true]);
$ftinnodb = $pdo->isDbVersionAtLeast('5.6');
if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") {
$tbl = $argv[1];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=FIXED");
} else if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "dmyisam") {
$tbl = $argv[1];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=DYNAMIC");
} else if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "cinnodb") {
$tbl = $argv[1];
if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'predbhash')) {
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED");
} else {
printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes"));
}
} else if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "dinnodb") {
$tbl = $argv[1];
if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'predbhash')) {
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC");
} else {
printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes"));
}
} else if (isset($argv[1]) && $argv[1] == "fmyisam") {
$sql = 'SHOW TABLE STATUS WHERE (Engine != "MyIsam" OR Row_format != "FIXED") AND Engine != "SPHINX"';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=FIXED");
}
} else if (isset($argv[1]) && $argv[1] == "dmyisam") {
$sql = 'SHOW TABLE STATUS WHERE (Engine != "MyIsam" OR Row_format != "Dynamic") AND Engine != "SPHINX"';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=DYNAMIC");
}
} else if (isset($argv[1]) && $argv[1] == "dinnodb") {
$sql = 'SHOW TABLE STATUS WHERE (Engine != "InnoDB" OR Row_format != "Dynamic") AND Engine != "SPHINX"';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
if ($tbl !== 'releasesearch' && $tbl !== 'predbhash') {
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC");
}
}
if ($ftinnodb) {
$sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Dynamic")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC");
}
} else {
printf($cli->header("Not converting releasesearch as your INNODB version does not support fulltext indexes"));
}
} else if (isset($argv[1]) && $argv[1] == "cinnodb") {
$sql = 'SHOW TABLE STATUS WHERE (Engine != "InnoDB" OR Row_format != "Compressed") AND Engine != "SPHINX"';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
if ($tbl !== 'releasenfo' && $tbl !== 'releasesearch') {
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED");
}
}
$sql = 'SHOW TABLE STATUS WHERE Name = "releasenfo" AND (Engine != "InnoDB" || Row_format != "Dynamic")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC");
}
if ($ftinnodb) {
$sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Compressed")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED");
}
} else {
printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes"));
}
} else if (isset($argv[1]) && $argv[1] == "cinnodb-noparts") {
$sql = 'SHOW TABLE STATUS WHERE (Engine != "InnoDB" OR Row_format != "Compressed") AND Engine != "SPHINX"';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
if ($tbl !== 'releasenfo' && $tbl !== 'releasesearch' && !preg_match('/parts/', $tbl)) {
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED");
}
}
$sql = 'SHOW TABLE STATUS WHERE Name = "releasenfo" AND (Engine != "InnoDB" || Row_format != "Dynamic")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC");
}
$sql = 'SHOW TABLE STATUS WHERE Name LIKE "parts%" AND (Engine != "MyISAM" || Row_format != "Dynamic")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MyISAM ROW_FORMAT=DYNAMIC");
}
if ($ftinnodb) {
$sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Compressed")';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED");
}
} else {
printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes"));
}
} else if (isset($argv[1]) && $argv[1] == "binaries") {
$arr = array("parts", "binaries");
foreach ($arr as $row) {
$tbl = $row;
printf($cli->header("Converting $tbl"));
$pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=FIXED");
}
} else if (isset($argv[1]) && $argv[1] == "mariadb-tokudb") {
$tables = $pdo->query('SHOW TABLE STATUS WHERE (Engine != "TokuDB" OR Create_options != "`COMPRESSION`=tokudb_lzma") AND Engine != "SPHINX"');
foreach ($tables as $row) {
$tbl = $row['name'];
if ($tbl !== 'releasesearch') {
printf($cli->header("Converting $tbl"));
$sql = "ALTER TABLE $tbl ENGINE=TokuDB Compression=tokudb_lzma";
$pdo->queryExec($sql);
$pdo->queryExec("OPTIMIZE TABLE $tbl");
}
}
} else if (isset($argv[1]) && $argv[1] == "tokudb") {
$tables = $pdo->query('SHOW TABLE STATUS WHERE (Engine != "TokuDB" OR ROW_FORMAT="tokudb_lzma" OR Create_options != "`COMPRESSION`=tokudb_lzma") AND Engine != "SPHINX"');
foreach ($tables as $row) {
$tbl = $row['name'];
if ($tbl !== 'releasesearch') {
printf($cli->header("Converting $tbl"));
$sql = "ALTER TABLE $tbl ENGINE=TokuDB row_format=tokudb_lzma";
$pdo->queryExec($sql);
$pdo->queryExec("OPTIMIZE TABLE $tbl");
}
}
} else {
exit($cli->error(
"\nThis script will convert your tables to a new engine/format. Only tables not meeting the new engine/format will be converted.\n"
. "A comparison of these, excluding TokuDB, https://github.com/nZEDb/nZEDb/wiki/MySQL-Storage-Engine-Comparison\n\n"
. "php convert_mysql_tables.php dmyisam ...: Converts all the tables to Myisam Dynamic. This is the default and is recommended where ram is limited.\n"
. "php convert_mysql_tables.php fmyisam ...: Converts all the tables to Myisam Fixed. This can be faster, but to fully convert all tables requires changing varchar columns to char.\n"
. " This will use much more space than dynamic.\n"
. "php convert_mysql_tables.php dinnodb ...: Converts all the tables to InnoDB Dynamic. This is recommended when the total data and indexes can fit into the innodb_buffer_pool.\n"
. " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n"
. "php convert_mysql_tables.php cinnodb ...: Converts all the tables to InnoDB Compressed. All tables except releasenfo will be converted to Compressed row format.\n"
. " This is recommended when the total data and indexes can not fit into the innodb_buffer_pool using DYNAMIC row format.\n"
. " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n"
. "php convert_mysql_tables.php cinnodb-noparts ...: Converts all the tables to InnoDB Compressed. All tables except parts and releasenfo will be converted to Compressed row format.\n"
. " Alls parts* will be converted to MyISAM Dynamic. This is recommended when using Table Per Group.\n"
. " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n"
. "php convert_mysql_tables.php binaries ...: Converts binaries, parts to MyIsam.\n"
. "php convert_mysql_tables.php mariadb-tokudb ...: Converts all the tables to MariaDB Tokutek DB. Use this is you installed mariadb-tokudb-engine. \n"
. " The TokuDB engine needs to be activated first.\n"
. " https://mariadb.com/kb/en/how-to-enable-tokudb-in-mariadb/\n"
. " NB releasesearch will not be converted as tokudb does not support fulltext indexes.\n"
. "php convert_mysql_tables.php tokudb ...: Converts all the tables to Tokutek DB. Use this if you downloaded and installed the TokuDB binaries.\n"
. " http://www.tokutek.com/resources/support/gadownloads/\n"
. " NB releasesearch will not be converted as tokudb does not support fulltext indexes.\n"
. "php convert_mysql_tables.php table [ fmyisam, dmyisam, dinnodb, cinnodb ] ...: Converts 1 table to Engine, row_format specified.\n"
. " NB if converting to innodb and your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n"
));
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/convert_to_tpg.php
+166
View File
@@ -0,0 +1,166 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
/* This script will allow you to move from single binaries/parts tables to TPG without having to run reset_truncate.
Please STOP all update scripts before running this script.
Use the following options to run:
php convert_to_tpg.php true Convert c/b/p to tpg leaving current binaries/parts tables in-tact.
php convert_to_tgp.php true delete Convert c/b/p to tpg and TRUNCATE current binaries/parts tables.
*/
$debug = false;
$pdo = new DB();
$groups = new \Groups(['Settings' => $pdo]);
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$s = new Sites();
$site = $s->get();
$DoPartRepair = ($site->partrepair == '0') ? false : true;
if ((!isset($argv[1])) || $argv[1] != 'true') {
exit($pdo->log->error("\nMandatory argument missing\n\n"
. "This script will allow you to move from single binaries/parts tables to TPG without having to run reset_truncate.\n"
. "Please STOP all update scripts before running this script.\n\n"
. "Use the following options to run:\n"
. "php $argv[0] true ...: Convert b/p to tpg leaving current binaries/parts tables in-tact.\n"
. "php $argv[0] true delete ...: Convert b/p to tpg and TRUNCATE current binaries/parts tables.\n"
));
}
$blen = $pdo->queryOneRow('SELECT COUNT(*) AS total FROM binaries;');
$bdone = 0;
$bcount = 1;
$gdone = 1;
$actgroups = $groups->getActive();
$glen = count($actgroups);
$newtables = $glen * 3;
$begintime = time();
echo "Creating new binaries, and parts tables for each active group...\n";
foreach ($actgroups as $group) {
if ($groups->createNewTPGTables($group['id']) === false) {
exit($pdo->log->error("There is a problem creating new parts/files tables for group ${group['name']}."));
}
$consoletools->overWrite("Tables Created: " . $consoletools->percentString($gdone * 3, $newtables));
$gdone++;
}
$endtime = time();
echo "\nTable creation took " . $consoletools->convertTime($endtime - $begintime) . ".\n";
$starttime = time();
echo "\nNew tables created, moving data from old tables to new tables.\nThis will take awhile....\n\n";
while ($bdone < $blen['total']) {
// Only load 1000 binaries per loop to not overload memory.
$binaries = $pdo->queryAssoc('SELECT * FROM binaries LIMIT ' . $bdone . ',1000;');
if ($binaries instanceof \Traversable) {
foreach ($binaries as $binary) {
$binary['name'] = $pdo->escapeString($binary['name']);
$binary['fromname'] = $pdo->escapeString($binary['fromname']);
$binary['date'] = $pdo->escapeString($binary['date']);
$binary['binaryhash'] = $pdo->escapeString($binary['binarynhash']);
$binary['dateadded'] = $pdo->escapeString($binary['dateadded']);
$binary['xref'] = $pdo->escapeString($binary['xref']);
$binary['releaseid'] = $pdo->escapeString($binary['releaseid']);
$binary['categoryid'] = $pdo->escapeString($binary['categoryid']);
$binary['totalparts'] = $pdo->escapeString($binary['totalparts']);
$binary['relpart'] = $pdo->escapeString($binary['relpart']);
$binary['reltotalpart'] = $pdo->escapeString($binary['reltotalpart']);
$oldbid = array_shift($binary);
if ($debug) {
echo "\n\nBinaries insert:\n";
print_r($binary);
echo sprintf("\nINSERT INTO binaries_%d (name, fromname, date, xref, groupid, dateadded, releaseid, categoryid, totalparts, binaryhash, relpart, reltotalpart) VALUES (%s)\n\n", $binary['groupid'], implode(', ', $binary));
}
$newbid = array('binaryid' => $pdo->queryInsert(sprintf('INSERT INTO binaries_%d (NAME, fromname, date, xref, groupid, dateadded, releaseid, categoryid, totalparts, binaryhash, relpart, reltotalpart) VALUES (%s);', $binary['groupid'], implode(', ', $binarynew))));
//Get parts and split to correct group tables.
$parts = $pdo->queryAssoc('SELECT * FROM parts WHERE binaryid = ' . $oldbid . ';');
if ($parts instanceof \Traversable) {
$firstpart = true;
$partsnew = '';
foreach ($parts as $part) {
$oldpid = array_shift($part);
$partnew = array_replace($part, $newbid);
$partsnew .= '(\'' . implode('\', \'', $partnew) . '\'), ';
}
$partsnew = substr($partsnew, 0, -2);
if ($debug) {
echo "\n\nParts insert:\n";
echo sprintf("\nINSERT INTO parts_%d (binaryid, messageid, number, partnumber, size) VALUES %s;\n\n", $binary['groupid'], $partsnew);
}
$sql = sprintf('INSERT INTO parts_%d (binaryid, messageid, number, partnumber, size) VALUES %s;', $binary['groupid'], $partsnew);
$pdo->queryExec($sql);
}
}
$bcount++;
}
$bdone += 1000;
}
if ($DoPartRepair === true) {
foreach ($actgroups as $group) {
$pcount = 1;
$pdone = 0;
$sql = sprintf('SELECT COUNT(*) AS total FROM partrepair WHERE groupid = %d;', $group['id']);
$plen = $pdo->queryOneRow($sql);
while ($pdone < $plen['total']) {
// Only load 10000 partrepair records per loop to not overload memory.
$partrepairs = $pdo->queryAssoc(sprintf('SELECT * FROM partrepair WHERE groupid = %d LIMIT %d, 10000;', $group['id'], $pdone));
if ($partrepairs instanceof \Traversable) {
foreach ($partrepairs as $partrepair) {
$partrepair['numberid'] = $pdo->escapeString($partrepair['numberid']);
$partrepair['groupid'] = $pdo->escapeString($partrepair['groupid']);
$partrepair['attempts'] = $pdo->escapeString($partrepair['attempts']);
if ($debug) {
echo "\n\nPart Repair insert:\n";
print_r($partrepair);
echo sprintf("\nINSERT INTO partrepair_%d (numberid, groupid, attempts) VALUES (%s, %s, %s)\n\n", $group['id'], $partrepair['numberid'], $partrepair['groupid'], $partrepair['attempts']);
}
$pdo->queryExec(sprintf('INSERT INTO partrepair_%d (numberid, groupid, attempts) VALUES (%s, %s, %s);', $group['id'], $partrepair['numberid'], $partrepair['groupid'], $partrepair['attempts']));
$consoletools->overWrite('Part Repairs Completed for ' . $group['name'] . ':' . $consoletools->percentString($pcount, $plen['total']));
$pcount++;
}
}
$pdone += 10000;
}
}
}
$endtime = time();
echo "\nTable population took " . $consoletools->convertTimer($endtime - $starttime) . ".\n";
//Truncate old tables to save space.
if (isset($argv[2]) && $argv[2] == 'delete') {
echo "Truncating old tables...\n";
$pdo->queryDirect('TRUNCATE TABLE binaries;');
$pdo->queryDirect('TRUNCATE TABLE parts');
$pdo->queryDirect('TRUNCATE TABLE partrepair');
echo "Complete.\n";
}
// Update TPG setting in site-edit.
$pdo->queryExec('UPDATE site SET value = 1 WHERE setting = \'tablepergroup\';');
$pdo->queryExec('UPDATE tmux SET value = 2 WHERE setting = \'releases\';');
echo "New tables have been created.\nTable Per Group has been set to to \"TRUE\" in site-edit.\nUpdate Releases has been set to Threaded in tmux-edit.\n";
function multi_implode($array, $glue)
{
$ret = '';
foreach ($array as $item) {
if (is_array($item)) {
$ret .= '(' . multi_implode($item, $glue) . '), ';
} else {
$ret .= $item . $glue;
}
}
$ret = substr($ret, 0, 0 - strlen($glue));
return $ret;
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/mysqldump_tables.php
+182
View File
@@ -0,0 +1,182 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
// This script can dump all tables or just collections/binaries/parts/partrepair/groups.
$pdo = new DB();
$exportopts = "";
//determine mysql platform Percona or Other
$mysqlplatform = exec('mysqladmin version | grep "Percona"');
if (strlen($mysqlplatform) > 0) {
//Percona only has --innodb-optimize-keys
$exportopts = "--opt --innodb-optimize-keys --complete-insert --skip-quick";
} else {
//generic (or unknown) instance of MySQL
$exportopts = "--opt --complete-insert --skip-quick";
}
function newname($filename)
{
rename($filename, dirname($filename)."/".basename($filename,".gz")."_".date("Y_m_d_His", filemtime($filename)).".gz");
}
function builddefaultsfile()
{
//generate file contents
$filetext = "[mysqldump]"
."\n"
."user = " . DB_USER
."\n"
."password = " . DB_PASSWORD
."\n[mysql]"
."\n"
."user = " . DB_USER
."\n"
."password = " . DB_PASSWORD;
$filehandle = fopen("mysql-defaults.txt", "w+");
if(!$filehandle) {
exit("Unable to write mysql defaults file! Exiting");
} else {
fwrite($filehandle, $filetext);
fclose($filehandle);
chmod("mysql-defaults.txt", 0600);
}
}
$dbhost = DB_HOST;
$dbport = DB_PORT;
$dbsocket = DB_SOCKET;
$dbuser = DB_USER;
$dbpass = DB_PASSWORD;
$dbname = DB_NAME;
if (DB_SOCKET != '') {
$use = "-S $dbsocket";
} else {
$use = "-P$dbport";
}
//generate defaults file used to store database login information so it is not in cleartext in ps command for mysqldump
builddefaultsfile();
if((isset($argv[1]) && $argv[1] == "db") && (isset($argv[2]) && $argv[2] == "dump") && (isset($argv[3]) && file_exists($argv[3]))) {
$filename = $argv[3]."/".$dbname.".gz";
echo $pdo->log->header("Dumping $dbname.");
if (file_exists($filename)) {
newname($filename);
}
$command = "mysqldump --defaults-file=mysql-defaults.txt $exportopts -h$dbhost $use "."$dbname | gzip -9 > $filename";
system($command);
} else if((isset($argv[1]) && $argv[1] == "db") && (isset($argv[2]) && $argv[2] == "restore") && (isset($argv[3]) && file_exists($argv[3]))) {
$filename = $argv[3]."/".$dbname.".gz";
if (file_exists($filename)) {
echo $pdo->log->header("Restoring $dbname.");
$command = "zcat < $filename | mysql --defaults-file=mysql-defaults.txt -h$dbhost $use $dbname";
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=0");
system($command);
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=1");
}
} else if((isset($argv[1]) && $argv[1] == "all") && (isset($argv[2]) && $argv[2] == "dump") && (isset($argv[3]) && file_exists($argv[3]))) {
$sql = "SHOW tables";
$tables = $pdo->query($sql);
foreach($tables as $row) {
$tbl = $row['Tables_in_'.DB_NAME];
$filename = $argv[3]."/".$tbl.".gz";
echo $pdo->log->header("Dumping $tbl.");
if (file_exists($filename)) {
newname($filename);
}
$command = "mysqldump --defaults-file=mysql-defaults.txt $exportopts -h$dbhost $use "."$dbname $tbl | gzip -9 > $filename";
system($command);
}
} else if((isset($argv[1]) && $argv[1] == "all") && (isset($argv[2]) && $argv[2] == "restore") && (isset($argv[3]) && file_exists($argv[3]))) {
$sql = "SHOW tables";
$tables = $pdo->query($sql);
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=0");
foreach($tables as $row) {
$tbl = $row['Tables_in_'.DB_NAME];
$filename = $argv[3]."/".$tbl.".gz";
if (file_exists($filename)) {
echo $pdo->log->header("Restoring $tbl.");
$command = "zcat < $filename | mysql --defaults-file=mysql-defaults.txt -h$dbhost $use $dbname";
system($command);
}
}
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=1");
} else if((isset($argv[1]) && $argv[1] == "test") && (isset($argv[2]) && $argv[2] == "dump") && (isset($argv[3]) && file_exists($argv[3]))) {
$arr = array("parts", "binaries", "partrepair", "groups");
foreach ($arr as &$tbl) {
$filename = $argv[3]."/".$tbl.".gz";
echo $pdo->log->header("Dumping $tbl..");
if (file_exists($filename)) {
newname($filename);
}
$command = "mysqldump --defaults-file=mysql-defaults.txt $exportopts -h$dbhost $use "."$dbname $tbl | gzip -9 > $filename";
system($command);
}
} else if((isset($argv[1]) && $argv[1] == "test") && (isset($argv[2]) && $argv[2] == "restore") && (isset($argv[3]) && file_exists($argv[3]))) {
$arr = array("parts", "binaries", "partrepair", "groups");
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=0");
foreach ($arr as &$tbl) {
$filename = $argv[3]."/".$tbl.".gz";
if (file_exists($filename)) {
echo $pdo->log->header("Restoring $tbl.");
$command = "zcat < $filename | mysql --defaults-file=mysql-defaults.txt -h$dbhost $use $dbname";
system($command);
}
}
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=1");
} else if((isset($argv[1]) && $argv[1] == "all") && (isset($argv[2]) && $argv[2] == "outfile") && (isset($argv[3]) && file_exists($argv[3]))) {
$sql = "SHOW tables";
$tables = $pdo->query($sql);
foreach($tables as $row) {
$tbl = $row['Tables_in_'.DB_NAME];
$filename = $argv[3].$tbl.".csv";
echo $pdo->log->header("Dumping $tbl.");
if (file_exists($filename)) {
newname($filename);
}
$pdo->queryDirect(sprintf("SELECT * INTO OUTFILE %s FROM %s", $pdo->escapeString($filename), $tbl));
}
} else if((isset($argv[1]) && $argv[1] == "all") && (isset($argv[2]) && $argv[2] == "infile") && (isset($argv[3]) && is_dir($argv[3]))) {
$sql = "SHOW tables";
$tables = $pdo->query($sql);
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=0");
foreach($tables as $row) {
$tbl = $row['Tables_in_'.DB_NAME];
$filename = $argv[3].$tbl.".csv";
if (file_exists($filename)) {
echo $pdo->log->header("Restoring $tbl.");
$pdo->queryExec(sprintf("LOAD DATA INFILE %s INTO TABLE %s", $pdo->escapeString($filename), $tbl));
}
}
$pdo->queryExec("SET FOREIGN_KEY_CHECKS=1");
} else {
passthru("clear");
echo $pdo->log->error("\nThis script can dump/restore all tables, compressed or OUTFILE/INFILE, or just collections/binaries/parts.\n\n"
. "**Single File\n"
. "php $argv[0] db dump /path/to/save/to ...: To dump the database.\n"
. "php $argv[0] db restore /path/to/restore/from ...: To restore the database.\n\n"
. "**Individual Table Files\n"
. "php $argv[0] all dump /path/to/save/to ...: To dump all tables.\n"
. "php $argv[0] all restore /path/to/restore/from ...: To restore all tables.\n\n"
. "**Three Tables (collections, binaries, parts)\n"
. "php $argv[0] test dump /path/to/save/to ...: To dump binaries and parts tables.\n"
. "php $argv[0] test restore /path/to/restore/from ...: To restore binaries and parts tables.\n\n"
. "**Individal Files - OUTFILE/INFILE - No schema\n"
. "**MySQL MUST have write permissions to this path\n"
. "php $argv[0] all outfile /path/to/save/to ...: To dump all tables, using OUTFILE.\n"
. "php $argv[0] all infile /path/to/restore/from ...: To restore all tables, using INFILE.\n\n");
}
if(file_exists("mysql-defaults.txt")) {
@unlink("mysql-defaults.txt");
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/populate_nzb_guid.php
+101
View File
@@ -0,0 +1,101 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
use newznab\utility\Utility;
$cli = new \ColorCLI();
if (isset($argv[1])) {
$del = false;
if (isset($argv[2])) {
$del = $argv[2];
}
create_guids($argv[1], $del);
} else {
exit($cli->error("\nThis script updates all releases with the guid (md5 hash of the first message-id) from the nzb file.\n\n"
. "php $argv[0] true ...: To create missing nzb_guids.\n"
. "php $argv[0] true delete ...: To create missing nzb_guids and delete invalid nzbs and releases.\n"));
}
function create_guids($live, $delete = false)
{
$pdo = new DB();
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$timestart = TIME();
$relcount = $deleted = $total = 0;
$relrecs = false;
if ($live == "true") {
$relrecs = $pdo->queryDirect(sprintf("SELECT id, guid FROM releases WHERE nzbstatus = 1 AND nzb_guid IS NULL ORDER BY id DESC"));
} else if ($live == "limited") {
$relrecs = $pdo->queryDirect(sprintf("SELECT id, guid FROM releases WHERE nzbstatus = 1 AND nzb_guid IS NULL ORDER BY id DESC LIMIT 10000"));
}
if ($relrecs) {
$total = $relrecs->rowCount();
}
if ($total > 0) {
echo $pdo->log->header("Creating nzb_guids for " . number_format($total) . " releases.");
$releases = new \Releases(['Settings' => $pdo]);
$nzb = new \NZB($pdo);
$releaseImage = new \ReleaseImage($pdo);
$reccnt = 0;
if ($relrecs instanceof \Traversable) {
foreach ($relrecs as $relrec) {
$reccnt++;
$nzbpath = $nzb->getNZBPath($relrec['guid']);
if ($nzbpath !== false) {
$nzbfile = Utility::unzipGzipFile($nzbpath);
if ($nzbfile) {
$nzbfile = @simplexml_load_string($nzbfile);
}
if (!$nzbfile) {
if (isset($delete) && $delete == 'delete') {
//echo "\n".$nzb->NZBPath($relrec['guid'])." is not a valid xml, deleting release.\n";
$releases->deleteSingle(['g' => $relrec['guid'], 'i' => $relrec['id']], $nzb, $releaseImage);
$deleted++;
}
continue;
}
$binary_names = array();
foreach ($nzbfile->file as $file) {
$binary_names[] = $file["subject"];
}
if (count($binary_names) == 0) {
if (isset($delete) && $delete == 'delete') {
//echo "\n".$nzb->NZBPath($relrec['guid'])." has no binaries, deleting release.\n";
$releases->deleteSingle(['g' => $relrec['guid'], 'i' => $relrec['id']], $nzb, $releaseImage);
$deleted++;
}
continue;
}
asort($binary_names);
foreach ($nzbfile->file as $file) {
if ($file["subject"] == $binary_names[0]) {
$segment = $file->segments->segment;
$nzb_guid = md5($segment);
$pdo->queryExec("UPDATE releases set nzb_guid = " . $pdo->escapestring($nzb_guid) . " WHERE id = " . $relrec["id"]);
$relcount++;
$consoletools->overWritePrimary("Created: [" . $deleted . "] " . $consoletools->percentString($reccnt, $total) . " Time:" . $consoletools->convertTimer(TIME() - $timestart));
break;
}
}
} else {
if (isset($delete) && $delete == 'delete') {
//echo $pdo->log->primary($nzb->NZBPath($relrec['guid']) . " does not have an nzb, deleting.");
$releases->deleteSingle(['g' => $relrec['guid'], 'i' => $relrec['id']], $nzb, $releaseImage);
}
}
}
}
if ($relcount > 0) {
echo "\n";
}
echo $pdo->log->header("Updated " . $relcount . " release(s). This script ran for " . $consoletools->convertTime(TIME() - $timestart));
} else {
echo $pdo->log->info('Query time: ' . $consoletools->convertTime(TIME() - $timestart));
exit($pdo->log->info("No releases are missing the guid."));
}
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/rename_to_lower.php
+64
View File
@@ -0,0 +1,64 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
passthru('clear');
$pdo = new newznab\db\DB();
if (!isset($argv[1]) || (isset($argv[1]) && $argv[1] !== 'true')) {
exit($pdo->log->error("\nThis script renames all table columns to lowercase, it can be dangerous. Please BACKUP your database before running this script.\n"
. "php rename_to_lower.php true ...: To rename all table columns to lowercase.\n"));
}
echo $pdo->log->warning("This script renames all table colums to lowercase.");
echo $pdo->log->header("Have you backed up your database? Type 'BACKEDUP' to continue: \n");
echo $pdo->log->warningOver("\n");
$line = fgets(STDIN);
if (trim($line) != 'BACKEDUP') {
exit($pdo->log->error("This script is dangerous you must type BACKEDUP for it function."));
}
echo "\n";
echo $pdo->log->header("Thank you, continuing...\n\n");
if ($argc == 1 || $argv[1] != 'true') {
exit($pdo->log->error("\nThis script will rename every table column to lowercase that is not already lowercase.\nTo run:\nphp $argv[0] true\n"));
}
$database = DB_NAME;
$count = 0;
$list = $pdo->query("SELECT TABLE_NAME, COLUMN_NAME, UPPER(COLUMN_TYPE), EXTRA FROM information_schema.columns WHERE table_schema = '" . $database . "'");
if (count($list) == 0) {
echo $pdo->log->info("No table columns to rename");
} else {
foreach ($list as $column) {
if ($column['column_name'] !== strtolower($column['column_name'])) {
echo $pdo->log->header("Renaming Table " . $column['table_name'] . " Column " . $column['column_name']);
if (isset($column['extra'])) {
$extra = strtoupper($column['extra']);
} else {
$extra = '';
}
$pdo->queryDirect("ALTER TABLE " . $column['table_name'] . " CHANGE " . $column['column_name'] . " " . strtolower($column['column_name']) . " " . $column['upper(column_type)'] . " " . $extra);
$count++;
}
if (strtolower($column['column_name']) === 'id' && strtolower($column['extra']) !== 'auto_increment') {
echo $pdo->log->header("Renaming Table " . $column['table_name'] . " Column " . $column['column_name']);
$extra = 'AUTO_INCREMENT';
if ($column['table_name'] != "releases_se") {
$placeholder = $pdo->queryDirect("SELECT MAX(id) FROM " . $column['table_name']);
$pdo->queryDirect("ALTER IGNORE TABLE " . $column['table_name'] . " CHANGE " . $column['column_name'] . " " . strtolower($column['column_name']) . " " . $column['upper(column_type)'] . " " . $extra);
$pdo->queryDirect("ALTER IGNORE TABLE " . $column['table_name'] . " AUTO_INCREMENT = " . $placeholder + 1);
$count++;
}
}
}
}
if ($count == 0) {
echo $pdo->log->info("All table column names are already lowercase");
} else {
echo $pdo->log->header($count . " colums renamed");
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/reset_postprocessing.php
+307
View File
@@ -0,0 +1,307 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
$pdo = new DB();
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$ran = false;
if (isset($argv[1]) && $argv[1] === "all") {
if (isset($argv[2]) && $argv[2] === "true") {
$ran = true;
$where = '';
if (isset($argv[3]) && $argv[3] === "truncate") {
echo "Truncating tables\n";
$pdo->queryExec("TRUNCATE TABLE consoleinfo");
$pdo->queryExec("TRUNCATE TABLE gamesinfo");
$pdo->queryExec("TRUNCATE TABLE movieinfo");
$pdo->queryExec("TRUNCATE TABLE releasevideo");
$pdo->queryExec("TRUNCATE TABLE musicinfo");
$pdo->queryExec("TRUNCATE TABLE bookinfo");
$pdo->queryExec("TRUNCATE TABLE releasenfo");
$pdo->queryExec("TRUNCATE TABLE releaseextrafull");
$pdo->queryExec("TRUNCATE TABLE xxxinfo");
}
echo $pdo->log->header("Resetting all postprocessing");
$qry = $pdo->queryDirect("SELECT id FROM releases");
$affected = 0;
if ($qry instanceof Traversable) {
$total = $qry->rowCount();
foreach ($qry as $releases) {
$pdo->queryExec(
sprintf("
UPDATE releases
SET consoleinfoid = NULL, gamesinfo_id = 0, imdbid = NULL, musicinfoid = NULL,
bookinfoid = NULL, rageid = -1, xxxinfo_id = 0, passwordstatus = -1, haspreview = -1,
jpgstatus = 0, videostatus = 0, audiostatus = 0, nfostatus = -1
WHERE id = %d",
$releases['id']
)
);
$consoletools->overWritePrimary("Resetting Releases: " . $consoletools->percentString(++$affected, $total));
}
}
}
}
if (isset($argv[1]) && ($argv[1] === "consoles" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE consoleinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Console postprocessing");
$where = ' WHERE consoleinfoid IS NOT NULL';
} else {
echo $pdo->log->header("Resetting all failed Console postprocessing");
$where = " WHERE consoleinfoid IN (-2, 0) AND categoryid BETWEEN 1000 AND 1999";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
if ($qry !== false) {
$total = $qry->rowCount();
} else {
$total = 0;
}
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET consoleinfoid = NULL WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Console Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " consoleinfoid's reset.");
}
if (isset($argv[1]) && ($argv[1] === "games" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE gamesinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Games postprocessing");
$where = ' WHERE gamesinfo_id != 0';
} else {
echo $pdo->log->header("Resetting all failed Games postprocessing");
$where = " WHERE gamesinfo_id IN (-2, 0) AND categoryid = 4050";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
if ($qry !== false) {
$total = $qry->rowCount();
} else {
$total = 0;
}
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET gamesinfo_id = 0 WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Games Releases: " . $consoletools->percentString(++$concount, $total));
}
echo $pdo->log->header("\n" . number_format($concount) . " gameinfo_ID's reset.");
}
}
if (isset($argv[1]) && ($argv[1] === "movies" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE movieinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Movie postprocessing");
$where = ' WHERE imdbid IS NOT NULL';
} else {
echo $pdo->log->header("Resetting all failed Movie postprocessing");
$where = " WHERE imdbid IN (-2, 0) AND categoryid BETWEEN 2000 AND 2999";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
if ($qry !== false) {
$total = $qry->rowCount();
} else {
$total = 0;
}
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET imdbid = NULL WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Movie Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " imdbid's reset.");
}
if (isset($argv[1]) && ($argv[1] === "music" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE musicinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Music postprocessing");
$where = ' WHERE musicinfoid IS NOT NULL';
} else {
echo $pdo->log->header("Resetting all failed Music postprocessing");
$where = " WHERE musicinfoid IN (-2, 0) AND categoryid BETWEEN 3000 AND 3999";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
$total = $qry->rowCount();
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET musicinfoid = NULL WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Music Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " musicinfoid's reset.");
}
if (isset($argv[1]) && ($argv[1] === "misc" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Additional postprocessing");
$where = ' WHERE (haspreview != -1 AND haspreview != 0) OR (passwordstatus != -1 AND passwordstatus != 0) OR jpgstatus != 0 OR videostatus != 0 OR audiostatus != 0';
} else {
echo $pdo->log->header("Resetting all failed Additional postprocessing");
$where = " WHERE haspreview < -1 OR haspreview = 0 OR passwordstatus < -1 OR passwordstatus = 0 OR jpgstatus < 0 OR videostatus < 0 OR audiostatus < 0";
}
echo $pdo->log->primary("SELECT id FROM releases" . $where);
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
if ($qry !== false) {
$total = $qry->rowCount();
} else {
$total = 0;
}
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET passwordstatus = -1, haspreview = -1, jpgstatus = 0, videostatus = 0, audiostatus = 0 WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " Release's reset.");
}
if (isset($argv[1]) && ($argv[1] === "tv" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE tvrage");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all TV postprocessing");
$where = ' WHERE rageid != -1';
} else {
echo $pdo->log->header("Resetting all failed TV postprocessing");
$where = " WHERE rageid IN (-2, 0) OR rageid IS NULL AND categoryid BETWEEN 5000 AND 5999";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
if ($qry !== false) {
$total = $qry->rowCount();
} else {
$total = 0;
}
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET rageid = -1 WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting TV Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " rageid's reset.");
}
if (isset($argv[1]) && ($argv[1] === "books" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE bookinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all Book postprocessing");
$where = ' WHERE bookinfoid IS NOT NULL';
} else {
echo $pdo->log->header("Resetting all failed Book postprocessing");
$where = " WHERE bookinfoid IN (-2, 0) AND categoryid BETWEEN 7000 AND 7899";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
$total = $qry->rowCount();
$concount = 0;
if ($qry instanceof Traversable) {
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET bookinfoid = NULL WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting Book Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " bookinfoid's reset.");
}
if (isset($argv[1]) && ($argv[1] === "xxx" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE xxxinfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all XXX postprocessing");
$where = ' WHERE xxxinfo_id != 0';
} else {
echo $pdo->log->header("Resetting all failed XXX postprocessing");
$where = " WHERE xxxinfo_id IN (-2, 0) AND categoryid BETWEEN 6000 AND 6040";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
$concount = 0;
if ($qry instanceof Traversable) {
$total = $qry->rowCount();
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET xxxinfo_id = 0 WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting XXX Releases: " . $consoletools->percentString(++$concount,
$total
)
);
}
}
echo $pdo->log->header("\n" . number_format($concount) . " xxxinfo_ID's reset.");
}
if (isset($argv[1]) && ($argv[1] === "nfos" || $argv[1] === "all")) {
$ran = true;
if (isset($argv[3]) && $argv[3] === "truncate") {
$pdo->queryExec("TRUNCATE TABLE releasenfo");
}
if (isset($argv[2]) && $argv[2] === "true") {
echo $pdo->log->header("Resetting all NFO postprocessing");
$where = ' WHERE nfostatus != -1';
} else {
echo $pdo->log->header("Resetting all failed NFO postprocessing");
$where = " WHERE nfostatus < -1";
}
$qry = $pdo->queryDirect("SELECT id FROM releases" . $where);
$concount = 0;
if ($qry instanceof Traversable) {
$total = $qry->rowCount();
foreach ($qry as $releases) {
$pdo->queryExec("UPDATE releases SET nfostatus = -1 WHERE id = " . $releases['id']);
$consoletools->overWritePrimary("Resetting NFO Releases: " . $consoletools->percentString(++$concount, $total));
}
}
echo $pdo->log->header("\n" . number_format($concount) . " NFO's reset.");
}
if ($ran === false) {
exit(
$pdo->log->error(
"\nThis script will reset postprocessing per category. It can also truncate the associated tables."
. "\nTo reset only those that have previously failed, those without covers, samples, previews, etc. use the "
. "second argument false.\n"
. "To reset even those previously post processed, use the second argument true.\n"
. "To truncate the associated table, use the third argument truncate.\n\n"
. "php reset_postprocessing.php consoles true ...: To reset all consoles.\n"
. "php reset_postprocessing.php games true ...: To reset all games.\n"
. "php reset_postprocessing.php movies true ...: To reset all movies.\n"
. "php reset_postprocessing.php music true ...: To reset all music.\n"
. "php reset_postprocessing.php misc true ...: To reset all misc.\n"
. "php reset_postprocessing.php tv true ...: To reset all tv.\n"
. "php reset_postprocessing.php books true ...: To reset all books.\n"
. "php reset_postprocessing.php xxx true ...: To reset all xxx.\n"
. "php reset_postprocessing.php nfos true ...: To reset all nfos.\n"
. "php reset_postprocessing.php all true ...: To reset everything.\n"
)
);
} else {
echo "\n";
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/reset_truncate.php
+55
View File
@@ -0,0 +1,55 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
$pdo = new DB();
$s = new Sites();
$site = $s->get();
if (isset($argv[1]) && ($argv[1] == "true" || $argv[1] == "drop")) {
$pdo->queryExec("UPDATE groups SET first_record = 0, first_record_postdate = NULL, last_record = 0, last_record_postdate = NULL, last_updated = NULL");
echo $pdo->log->primary("Reseting all groups completed.");
$arr = array("parts", "partrepair", "binaries", "collections");
foreach ($arr as &$value) {
$rel = $pdo->queryExec("TRUNCATE TABLE $value");
if ($rel !== false) {
echo $pdo->log->primary("Truncating ${value} completed.");
}
}
unset($value);
$tpg = $site->tablepergroup;
$tablepergroup = (!empty($tpg)) ? $tpg : 0;
if ($tablepergroup == 1) {
$sql = 'SHOW table status';
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
if (preg_match('/collections_\d+/', $tbl) || preg_match('/binaries_\d+/', $tbl) || preg_match('/parts_\d+/', $tbl) || preg_match('/partrepair_\d+/', $tbl) || preg_match('/\d+_collections/', $tbl) || preg_match('/\d+_binaries/', $tbl) || preg_match('/\d+_parts/', $tbl) || preg_match('/\d+_partrepair_\d+/', $tbl)) {
if ($argv[1] == "drop") {
$rel = $pdo->queryDirect(sprintf('DROP TABLE %s', $tbl));
if ($rel !== false) {
echo $pdo->log->primary("Dropping ${tbl} completed.");
}
} else {
$rel = $pdo->queryDirect(sprintf('TRUNCATE TABLE %s', $tbl));
if ($rel !== false) {
echo $pdo->log->primary("Truncating ${tbl} completed.");
}
}
}
}
}
$delcount = $pdo->queryDirect("DELETE FROM releases WHERE nzbstatus = 0");
echo $pdo->log->primary($delcount->rowCount() . " releases had no nzb, deleted.");
} else {
exit($pdo->log->error("\nThis script removes releases with no NZBs, resets all groups, truncates or drops(tpg) \n"
. "article tables. All other releases are left alone.\n"
. "php $argv[0] [true, drop] ...: To reset all groups and truncate/drop the tables.\n"
));
}
misc/update_scripts/nix_scripts/tmux/lib/testing/DB/resetdb.php
+100
View File
@@ -0,0 +1,100 @@
<?php
require_once(dirname(__FILE__) . "/../../../bin/config.php");
use newznab\db\DB;
use newznab\utility\Utility;
Utility::clearScreen();
$pdo = new DB();
if (!isset($argv[1]) || (isset($argv[1]) && $argv[1] !== 'true'))
exit($pdo->log->error("\nThis script removes all releases and release related files. To run:\nphp resetdb.php true\n"));
echo $pdo->log->warning("This script removes all releases, nzb files, samples, previews , nfos, truncates all article tables and resets all groups.");
echo $pdo->log->header("Are you sure you want reset the DB? Type 'DESTROY' to continue: \n");
echo $pdo->log->warningOver("\n");
$line = fgets(STDIN);
if (trim($line) != 'DESTROY')
exit($pdo->log->error("This script is dangerous you must type DESTROY for it function."));
echo "\n";
echo $pdo->log->header("Thank you, continuing...\n\n");
$timestart = time();
$relcount = 0;
$ri = new \ReleaseImage($pdo);
$nzb = new \NZB($pdo);
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$pdo->queryExec("UPDATE groups SET first_record = 0, first_record_postdate = NULL, last_record = 0, last_record_postdate = NULL, last_updated = NULL");
echo $pdo->log->primary("Reseting all groups completed.");
$arr = [
"tvrage", "releasenfo", "releasecomment", 'sharing', 'sharing_sites',
"usercart", "usermovies", "userseries", "movieinfo", "musicinfo", "releasefiles",
"releaseaudio", "releasesubs", "releasevideo", "releaseextrafull", "parts",
"partrepair", "binaries", "collections", "releases", "spotnabsources"
];
foreach ($arr as &$value) {
$rel = $pdo->queryExec("TRUNCATE TABLE $value");
if ($rel !== false)
echo $pdo->log->primary("Truncating ${value} completed.");
}
unset($value);
$sql = "SHOW table status";
$tables = $pdo->query($sql);
foreach ($tables as $row) {
$tbl = $row['name'];
if (preg_match('/binaries_\d+/', $tbl) || preg_match('/parts_\d+/', $tbl) || preg_match('/collections_\d+/', $tbl) || preg_match('/partrepair_\d+/', $tbl) || preg_match('/\d+_binaries/', $tbl) || preg_match('/\d+_collections/', $tbl) || preg_match('/\d+_parts/', $tbl) || preg_match('/\d+_partrepair_\d+/', $tbl)) {
$rel = $pdo->queryDirect(sprintf('DROP TABLE %s', $tbl));
if ($rel !== false)
echo $pdo->log->primary("Dropping ${tbl} completed.");
}
}
(new \SphinxSearch())->truncateRTIndex('releases_rt');
$pdo->optimise(false, 'full');
$s = new Sites();
$site = $s->get();
echo $pdo->log->header("Deleting nzbfiles subfolders.");
try {
$files = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($site->nzbpath, \RecursiveDirectoryIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST);
foreach ($files as $file) {
if (basename($file) != '.gitignore' && basename($file) != 'tmpunrar') {
$todo = ($file->isDir() ? 'rmdir' : 'unlink');
@$todo($file);
}
}
} catch (UnexpectedValueException $e) {
echo $pdo->log->error($e->getMessage());
}
echo $pdo->log->header("Deleting all images, previews and samples that still remain.");
try {
$dirItr = new \RecursiveDirectoryIterator(NN_COVERS);
$itr = new \RecursiveIteratorIterator($dirItr, \RecursiveIteratorIterator::LEAVES_ONLY);
foreach ($itr as $filePath) {
if (basename($filePath) != '.gitignore' && basename($filePath) != 'no-cover.jpg' && basename($filePath) != 'no-backdrop.jpg') {
@unlink($filePath);
}
}
} catch (UnexpectedValueException $e) {
echo $pdo->log->error($e->getMessage());
}
echo $pdo->log->header("Getting Updated List of TV Shows from TVRage.");
$tvshows = @simplexml_load_file('http://services.tvrage.com/feeds/show_list.php');
if ($tvshows !== false) {
foreach ($tvshows->show as $rage) {
if (isset($rage->id) && isset($rage->name) && !empty($rage->id) && !empty($rage->name))
$pdo->queryInsert(sprintf('INSERT INTO tvrage (rageid, releasetitle, country) VALUES (%s, %s, %s)', $pdo->escapeString($rage->id), $pdo->escapeString($rage->name), $pdo->escapeString($rage->country)));
}
} else {
echo $pdo->log->error("TVRage site has a hard limit of 400 concurrent api requests. At the moment, they have reached that limit. Please wait before retrying again.");
}
echo $pdo->log->header("Deleted all releases, images, previews and samples. This script ran for " . $consoletools->convertTime(TIME() - $timestart));