From 3feb3bef658a6cf7599a708ecf15337af6b2d9d9 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 3 Dec 2015 12:14:27 +0100 Subject: [PATCH] Remove traktid columns from releases and movieinfo tables, add nzb_reorg script and script to check missing unique indexes --- Changelog | 3 + misc/testing/DB/check_unique_indexes.php | 121 +++++++++++++++ misc/testing/NZB/nzb-reorg.php | 74 +++++++++ misc/testing/PostProc/getTrakt.php | 30 ---- misc/testing/PostProc/populate_tvrage.php | 44 ------ misc/testing/PostProc/updateTvRage.php | 141 ------------------ newznab/build/nntmux.xml | 6 +- resources/db/patches/mysql/+1~movieinfo.sql | 3 + resources/db/patches/mysql/+2~releases.sql | 3 + resources/db/patches/mysql/0227~movieinfo.sql | 3 + resources/db/patches/mysql/0228~releases.sql | 3 + resources/db/schema/nntmux_fi_schema.sql | 2 - 12 files changed, 213 insertions(+), 220 deletions(-) create mode 100755 misc/testing/DB/check_unique_indexes.php create mode 100755 misc/testing/NZB/nzb-reorg.php delete mode 100644 misc/testing/PostProc/getTrakt.php delete mode 100644 misc/testing/PostProc/populate_tvrage.php delete mode 100644 misc/testing/PostProc/updateTvRage.php create mode 100644 resources/db/patches/mysql/+1~movieinfo.sql create mode 100644 resources/db/patches/mysql/+2~releases.sql create mode 100644 resources/db/patches/mysql/0227~movieinfo.sql create mode 100644 resources/db/patches/mysql/0228~releases.sql diff --git a/Changelog b/Changelog index 353199e79..29e6fbe3f 100755 --- a/Changelog +++ b/Changelog @@ -1,3 +1,6 @@ +2015-12-03 DariusIII + Chg: remove traktid columns from movieinfo and releases table, add nzb_reorg script, add script to check + missing unique indexes based on nntmux_fi_schema.sql 2015-12-01 DariusIII Chg: Replace Cart with Download Basket Fix: Add posterwall.css diff --git a/misc/testing/DB/check_unique_indexes.php b/misc/testing/DB/check_unique_indexes.php new file mode 100755 index 000000000..11800e9d8 --- /dev/null +++ b/misc/testing/DB/check_unique_indexes.php @@ -0,0 +1,121 @@ +log->error("\nThis script will scan nntmux_fi_schema.sql for all UNIQUE INDEXES.\n" + . "It will verify that you have them. If you do not, you can choose to run manually or allow the script to run them.\n\n" + . "php $argv[0] test ...: To verify all unique indexes.\n" + . "php $argv[0] alter ...: To add missing unique indexes.\n")); + } +} + +// Set for Session +if (isset($argv[1]) && $argv[1] === 'alter') { + $pdo->queryExec("SET SESSION old_alter_table = 1"); +} + +function run_query($query, $test) +{ + global $pdo; + + if ($test === 'alter') { + try { + $qry = $pdo->prepare($query); + $qry->execute(); + echo $pdo->log->alternateOver('SUCCESS: ') . $pdo->log->primary($query); + } catch (\PDOException $e) { + if ($e->errorInfo[1] == 1061) { + // Duplicate key exists + echo $pdo->log->alternateOver('SKIPPED Index name exists: ') . $pdo->log->primary($query); + } else { + echo $pdo->log->alternateOver('FAILED: ') . $pdo->log->primary($query); + } + } + } else { + echo $pdo->log->header($query); + } +} + +$path = nZEDb_RES . 'db' . DS . 'schema' . DS . 'nntmux_fi_schema.sql'; +$handle = fopen($path, "r"); +if ($handle) { + while (($line = fgets($handle)) !== false) { + if (preg_match('/(?PCREATE UNIQUE INDEX)\s+(?P[\w-]+)\s+ON\s+(?P[\w-]+)\s*\((?P[\w-]+(?:\s*\((?P\d+)\))?)\);/i', $line, $match)) { + $columns = explode(',', $match['column']); + foreach ($columns as $column) { + $check = $pdo->checkColumnIndex($match['table'], $column); + if (!isset($check['key_name'])) { + if (trim($match['table']) === 'collections') { + $tables = $pdo->query("SHOW TABLES"); + foreach ($tables as $row) { + $tbl = $row['tables_in_' . DB_NAME]; + if (preg_match('/collections_\d+/', $tbl)) { + $check = $pdo->checkColumnIndex($tbl, $column); + if (!isset($check_collections['key_name'])) { + $qry = "ALTER IGNORE TABLE ${tbl} ADD CONSTRAINT {$match['index']} UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } + } + } + $qry = "ALTER IGNORE TABLE " . trim($match['table']) . " ADD CONSTRAINT " . trim($match['index']) . " UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } else if (trim($match['table']) === 'binaries') { + $tables = $pdo->query("SHOW TABLES"); + foreach ($tables as $row) { + $tbl = $row['tables_in_' . DB_NAME]; + if (preg_match('/binaries_\d+/', $tbl)) { + $checkBinaries = $pdo->checkColumnIndex($tbl, $column); + if (!isset($checkBinaries['key_name'])) { + $qry = "ALTER IGNORE TABLE ${tbl} ADD CONSTRAINT {$match['index']} UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } + } + } + $qry = "ALTER IGNORE TABLE " . trim($match['table']) . " ADD CONSTRAINT " . trim($match['index']) . " UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } else if (trim($match['table']) === 'parts') { + $tables = $pdo->query("SHOW TABLES"); + foreach ($tables as $row) { + $tbl = $row['tables_in_' . DB_NAME]; + if (preg_match('/parts_\d+/', $tbl)) { + $checkParts = $pdo->checkColumnIndex($tbl, $column); + if (!isset($checkParts['key_name'])) { + $qry = "ALTER IGNORE TABLE ${tbl} ADD CONSTRAINT {$match['index']} UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } + } + } + $qry = "ALTER IGNORE TABLE " . trim($match['table']) . " ADD CONSTRAINT " . trim($match['index']) . " UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } else if (trim($match['table']) === 'partrepair') { + $tables = $pdo->query("SHOW TABLES"); + foreach ($tables as $row) { + $tbl = $row['tables_in_' . DB_NAME]; + if (preg_match('/partrepair_\d+/', $tbl)) { + $checkPartRepair = $pdo->checkColumnIndex($tbl, $column); + if (!isset($checkPartRepair['key_name'])) { + $qry = "ALTER IGNORE TABLE ${tbl} ADD CONSTRAINT {$match['index']} UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } + } + } + $qry = "ALTER IGNORE TABLE " . trim($match['table']) . " ADD CONSTRAINT " . trim($match['index']) . " UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } else { + $qry = "ALTER IGNORE TABLE " . trim($match['table']) . " ADD CONSTRAINT " . trim($match['index']) . " UNIQUE (${match['column']})"; + run_query($qry, $argv[1]); + } + } else { + echo $pdo->log->primary("A Unique Index exists for " . trim($match['table']) . " on " . trim($match['column'])); + } + } + } + } +} else { + echo $pdo->log->error("\nCan not open nntmux_fi_schema.sql."); +} diff --git a/misc/testing/NZB/nzb-reorg.php b/misc/testing/NZB/nzb-reorg.php new file mode 100755 index 000000000..bb58e3a70 --- /dev/null +++ b/misc/testing/NZB/nzb-reorg.php @@ -0,0 +1,74 @@ + $nzbFile) { + if ($nzbFile->getExtension() != "gz") { + continue; + } + + $newFileName = $nzb->getNZBPath(str_replace(".nzb.gz", "", $nzbFile->getBasename()), + $newLevel, + true); + if ($newFileName != $nzbFile) { + rename($nzbFile, $newFileName); + chmod($newFileName, 0777); + } + $iFilesProcessed++; + if ($iFilesProcessed % 100 == 0) { + $consoleTools->overWrite("Reorganized $iFilesProcessed"); + } +} + +$pdo->ping(true); +$pdo->queryExec(sprintf("UPDATE settings SET value = %s WHERE setting = 'nzbsplitlevel'", $argv[1])); +$consoleTools->overWrite("Processed $iFilesProcessed nzbs in " . relativeTime($time) . "\n"); + +function relativeTime($_time) +{ + $d = array(); + $d[0] = array(1, "sec"); + $d[1] = array(60, "min"); + $d[2] = array(3600, "hr"); + $d[3] = array(86400, "day"); + $d[4] = array(31104000, "yr"); + + $w = array(); + + $return = ""; + $now = TIME(); + $diff = ($now - $_time); + $secondsLeft = $diff; + + for ($i = 4; $i > -1; $i--) { + $w[$i] = intval($secondsLeft / $d[$i][0]); + $secondsLeft -= ($w[$i] * $d[$i][0]); + if ($w[$i] != 0) { + $return .= $w[$i] . " " . $d[$i][1] . (($w[$i] > 1) ? 's' : '') . " "; + } + } + return $return; +} + +?> diff --git a/misc/testing/PostProc/getTrakt.php b/misc/testing/PostProc/getTrakt.php deleted file mode 100644 index 43bd887e8..000000000 --- a/misc/testing/PostProc/getTrakt.php +++ /dev/null @@ -1,30 +0,0 @@ - true, 'Settings' => $pdo]); - - -$movies = $pdo->queryDirect("SELECT imdbid FROM movieinfo WHERE traktid = 0 ORDER BY id ASC"); -if ($movies instanceof \Traversable) { - echo $pdo->log->header("Updating Trakt ID info for " . number_format($movies->rowCount()) . " movies."); - - foreach ($movies as $mov) { - $starttime = microtime(true); - $mov = $movie->updateMovieInfo($mov['imdbid']); - - // tmdb limits are 30 per 10 sec, not certain for imdb - $diff = floor((microtime(true) - $starttime) * 1000000); - if (333333 - $diff > 0) { - echo "sleeping\n"; - usleep(333333 - $diff); - } - } - echo "\n"; -} diff --git a/misc/testing/PostProc/populate_tvrage.php b/misc/testing/PostProc/populate_tvrage.php deleted file mode 100644 index 0dc801194..000000000 --- a/misc/testing/PostProc/populate_tvrage.php +++ /dev/null @@ -1,44 +0,0 @@ -log->error("\nThis script will download all tvrage shows and insert into the db.\n\n" - . "php $argv[0] true ...: To run.\n")); -} - -$newnames = $updated = 0; - -echo "Attempting to fetch data file from TVRage...\n"; -$tvshows = @simplexml_load_file('http://services.tvrage.com/feeds/show_list.php'); -if ($tvshows !== false) { - echo "Starting to process file entries...\n"; - foreach ($tvshows->show as $rage) { - echo "RageID: " . $rage->id . ", name: " . $rage->name . " - "; - $dupecheck = $pdo->queryOneRow(sprintf('SELECT COUNT(id) AS count FROM tvrage_titles WHERE id = %s', $pdo->escapeString($rage->id))); - if (isset($rage->id) && isset($rage->name) && !empty($rage->id) && !empty($rage->name) && - $dupecheck !== false && $dupecheck['count'] == 0) { - $pdo->queryInsert(sprintf('INSERT INTO tvrage_titles (rageid, releasetitle, country) VALUES (%s, %s, %s)', $pdo->escapeString($rage->id), $pdo->escapeString($rage->name), $pdo->escapeString($rage->country))); - $updated++; - echo "added\n"; - } elseif (isset($rage->id) && isset($rage->name) && !empty($rage->id) && !empty($rage->name) && - $dupecheck !== false && $dupecheck['count'] > 0) { - echo "Up to date\n"; - } else { - echo "FAILED\n"; - } - } -} else { - exit($pdo->log->info("TVRage site has a hard limit of 400 concurrent API requests. At the moment, they have reached that limit. Please wait before retrying\n")); -} -if ($updated != 0) { - echo $pdo->log->info("Inserted " . $updated . " new shows into the TvRage table. To fill out the newly populated TvRage table\n" - . "php misc/update_scripts/nix_scripts/tmux/lib/testing/PostProc/updateTvRage.php\n"); -} else { - echo "\n"; - echo $pdo->log->info("TvRage database is already up to date!\n"); -} \ No newline at end of file diff --git a/misc/testing/PostProc/updateTvRage.php b/misc/testing/PostProc/updateTvRage.php deleted file mode 100644 index c4d6a26bf..000000000 --- a/misc/testing/PostProc/updateTvRage.php +++ /dev/null @@ -1,141 +0,0 @@ -log->error("You must provide an argument. Use update to try and grab new images and check to reconcile the image directory against the database.") . PHP_EOL; - exit; -} - -$tvrage = new TvRage(['Settings' => $pdo, 'Echo' => true]); - -switch((string)$argv[1]) { - case 'update': - updateTvImages($pdo, $tvrage); - break; - case 'check': - checkTvImages($pdo, $tvrage); - break; - default: - exit; -} - -// This function checks for TV shows in tvrage missing covers and tries to grab them -function updateTvImages($pdo, $tvrage) -{ - $shows = $pdo->queryDirect("SELECT rageid, releasetitle FROM tvrage_titles WHERE hascover = 0 ORDER BY rageid DESC LIMIT 2000"); - - if ($shows->rowCount() > 0) { - echo "\n"; - echo $pdo->log->header("Updating " . number_format($shows->rowCount()) . " tv shows.\n"); - } else { - echo "\n"; - echo $pdo->log->info("All shows in TvRage database have been updated.\n"); - usleep(5000000); - } - - if ($shows instanceof \Traversable) { - foreach ($shows as $show) { - $starttime = microtime(true); - $rageid = $show['rageid']; - $tvrShow = $tvrage->getRageInfoFromService($rageid); - $genre = ''; - if (isset($tvrShow['genres']) && is_array($tvrShow['genres']) && !empty($tvrShow['genres'])) { - if (is_array($tvrShow['genres']['genre'])) { - $genre = @implode('|', $tvrShow['genres']['genre']); - } else { - $genre = $tvrShow['genres']['genre']; - } - } - $country = ''; - if (isset($tvrShow['country']) && !empty($tvrShow['country'])) { - $country = $tvrage->countryCode($tvrShow['country']); - } - - $rInfo = $tvrage->getRageInfoFromPage($rageid); - $desc = ''; - if (isset($rInfo['desc']) && !empty($rInfo['desc'])) { - $desc = $rInfo['desc']; - } - - $hasCover = 0; - - if (isset($rInfo['imgurl']) && !empty($rInfo['imgurl'])) { - $hasCover = (new ReleaseImage($pdo))->saveImage($rageid, $rInfo['imgurl'], $tvrage->imgSavePath, '', ''); - } - - $pdo->queryDirect( - sprintf(" - UPDATE tvrage_titles - SET description = %s, genre = %s, country = %s, hascover = %d - WHERE rageid = %d", - $pdo->escapeString(substr($desc, 0, 10000)), - $pdo->escapeString(substr($genre, 0, 64)), - $pdo->escapeString($country), - $hasCover, - $rageid - ) - ); - - echo $pdo->log->primary("Updated: " . $show['releasetitle']); - $diff = floor((microtime(true) - $starttime) * 1000000); - if (1000000 - $diff > 0) { - echo $pdo->log->alternate("Sleeping"); - usleep(1000000 - $diff); - } - } - } -} - -// This function checks for covers in the tvrage folder that aren't tied to a tvrage_title row (hascover = 1) -function checkTvImages($pdo, $tvrage) -{ - $text = new Utility(); - $images = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($tvrage->imgSavePath), RecursiveIteratorIterator::SELF_FIRST); - - if ($images instanceof \Traversable) { - $imgCount = iterator_count($images); - echo PHP_EOL . $pdo->log->header("Matching " . number_format($imgCount) . " files to their RageIDs and setting hascover.") . PHP_EOL; - $checkCnt = $notReqdCnt = $missingRageCnt = $badRageCnt = 0; - - foreach($images as $file) { - - $rightCut = $text->cutStringUsingLast('/', $file, "right", false); - $leftCut = $text->cutStringUsingLast('.', $rightCut, "left", false); - $check = $pdo->queryOneRow( - sprintf( - "SELECT id, hascover - FROM tvrage - WHERE rageid = %d", - $leftCut - ) - ); - - if ($check) { - if ($check['hascover'] == 0) { - $pdo->queryExec( - sprintf( - "UPDATE tvrage_titles - SET hascover = 1 - WHERE id = %d", - $check['id'] - ) - ); - $checkCnt++; - } else { - $notReqdCnt++; - } - } else { - $missingRageCnt++; - } - } - } - echo PHP_EOL . $pdo->log->primary("Updated " . number_format($checkCnt) . " RageIDs. " . number_format($notReqdCnt) . " were already set and " . number_format($missingRageCnt) . " images did not have matching RageIDs in the database.") . PHP_EOL; -} diff --git a/newznab/build/nntmux.xml b/newznab/build/nntmux.xml index 2546a2d01..580693c2b 100755 --- a/newznab/build/nntmux.xml +++ b/newznab/build/nntmux.xml @@ -2,8 +2,8 @@ - 226 - 226 + 228 + 228 0.4.2 @@ -12,6 +12,6 @@ 9 - 4532 + 4533 diff --git a/resources/db/patches/mysql/+1~movieinfo.sql b/resources/db/patches/mysql/+1~movieinfo.sql new file mode 100644 index 000000000..10bdb8f7f --- /dev/null +++ b/resources/db/patches/mysql/+1~movieinfo.sql @@ -0,0 +1,3 @@ +#Drop the traktid COLUMN from movieinfo table + +ALTER TABLE movieinfo drop traktid; diff --git a/resources/db/patches/mysql/+2~releases.sql b/resources/db/patches/mysql/+2~releases.sql new file mode 100644 index 000000000..b6190105d --- /dev/null +++ b/resources/db/patches/mysql/+2~releases.sql @@ -0,0 +1,3 @@ +#Drop the traktid COLUMN from releases table + +ALTER TABLE releases drop traktid; diff --git a/resources/db/patches/mysql/0227~movieinfo.sql b/resources/db/patches/mysql/0227~movieinfo.sql new file mode 100644 index 000000000..10bdb8f7f --- /dev/null +++ b/resources/db/patches/mysql/0227~movieinfo.sql @@ -0,0 +1,3 @@ +#Drop the traktid COLUMN from movieinfo table + +ALTER TABLE movieinfo drop traktid; diff --git a/resources/db/patches/mysql/0228~releases.sql b/resources/db/patches/mysql/0228~releases.sql new file mode 100644 index 000000000..b6190105d --- /dev/null +++ b/resources/db/patches/mysql/0228~releases.sql @@ -0,0 +1,3 @@ +#Drop the traktid COLUMN from releases table + +ALTER TABLE releases drop traktid; diff --git a/resources/db/schema/nntmux_fi_schema.sql b/resources/db/schema/nntmux_fi_schema.sql index cc8287d0b..0ced4c7c2 100644 --- a/resources/db/schema/nntmux_fi_schema.sql +++ b/resources/db/schema/nntmux_fi_schema.sql @@ -2268,7 +2268,6 @@ CREATE TABLE IF NOT EXISTS movieinfo ( createddate DATETIME NOT NULL, updateddate DATETIME NOT NULL, banner TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', - traktid INT(10) UNSIGNED NOT NULL DEFAULT '0', PRIMARY KEY (id), UNIQUE KEY imdbid (imdbid), KEY ix_movieinfo_title (title) @@ -2574,7 +2573,6 @@ CREATE TABLE IF NOT EXISTS releases ( proc_sorter TINYINT(1) NOT NULL DEFAULT '0', nzbstatus TINYINT(1) NOT NULL DEFAULT '0', nzb_guid BINARY(16) DEFAULT NULL, - traktid INT(10) UNSIGNED NOT NULL DEFAULT '0', PRIMARY KEY (id,categoryid), KEY ix_releases_name (name), KEY ix_releases_group_id (groupid,passwordstatus),