diff --git a/misc/testing/DB/convert_mysql_tables.php b/misc/testing/DB/convert_mysql_tables.php index ef499b993..950272ba4 100644 --- a/misc/testing/DB/convert_mysql_tables.php +++ b/misc/testing/DB/convert_mysql_tables.php @@ -3,7 +3,7 @@ require_once dirname(__FILE__) . '/../../../www/config.php'; use newznab\db\Settings; -$cli = new \ColorCLI(); +$cli = new ColorCLI(); $pdo = new Settings(['checkVersion' => true]); $ftinnodb = $pdo->isDbVersionAtLeast('5.6'); @@ -18,19 +18,19 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=MYISAM ROW_FORMAT=DYNAMIC"); } else if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "cinnodb") { $tbl = $argv[1]; - if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'predbhash')) { + if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'bookinfo' && $tbl !== 'consoleinfo' && $tbl !== 'musicinfo')) { printf($cli->header("Converting $tbl")); $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED"); } else { - printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes")); + printf($cli->header("Not converting bookinfo / consoleinfo / musicinfo / releasesearch as your INNODB version does not support fulltext indexes")); } } else if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "dinnodb") { $tbl = $argv[1]; - if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'predbhash')) { + if ($ftinnodb || (!$ftinnodb && $tbl !== 'releasesearch' && $tbl !== 'bookinfo' && $tbl !== 'consoleinfo' && $tbl !== 'musicinfo')) { printf($cli->header("Converting $tbl")); $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC"); } else { - printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes")); + printf($cli->header("Not converting bookinfo / consoleinfo / musicinfo / releasesearch as your INNODB version does not support fulltext indexes")); } } else if (isset($argv[1]) && $argv[1] == "fmyisam") { $sql = 'SHOW TABLE STATUS WHERE (Engine != "MyIsam" OR Row_format != "FIXED") AND Engine != "SPHINX"'; @@ -53,13 +53,13 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; - if ($tbl !== 'releasesearch' && $tbl !== 'predbhash') { + if ($tbl !== 'releasesearch' && $tbl !== 'bookinfo' && $tbl !== 'consoleinfo' && $tbl !== 'musicinfo') { printf($cli->header("Converting $tbl")); $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC"); } } if ($ftinnodb) { - $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Dynamic")'; + $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "bookinfo", "consoleinfo", "musicinfo") AND (Engine != "InnoDB" || Row_format != "Dynamic")'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; @@ -67,19 +67,19 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC"); } } else { - printf($cli->header("Not converting releasesearch as your INNODB version does not support fulltext indexes")); + printf($cli->header("Not converting bookinfo / consoleinfo / musicinfo / releasesearch as your INNODB version does not support fulltext indexes")); } } else if (isset($argv[1]) && $argv[1] == "cinnodb") { $sql = 'SHOW TABLE STATUS WHERE (Engine != "InnoDB" OR Row_format != "Compressed") AND Engine != "SPHINX"'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; - if ($tbl !== 'releasenfo' && $tbl !== 'releasesearch') { + if ($tbl !== 'release_nfos' && $tbl !== 'releasesearch' && $tbl !== 'bookinfo' && $tbl !== 'consoleinfo' && $tbl !== 'musicinfo') { printf($cli->header("Converting $tbl")); $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED"); } } - $sql = 'SHOW TABLE STATUS WHERE Name = "releasenfo" AND (Engine != "InnoDB" || Row_format != "Dynamic")'; + $sql = 'SHOW TABLE STATUS WHERE Name = "release_nfos" AND (Engine != "InnoDB" || Row_format != "Dynamic")'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; @@ -87,7 +87,7 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=DYNAMIC"); } if ($ftinnodb) { - $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Compressed")'; + $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "bookinfo", "consoleinfo", "musicinfo") AND (Engine != "InnoDB" || Row_format != "Compressed")'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; @@ -95,19 +95,19 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED"); } } else { - printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes")); + printf($cli->header("Not converting bookinfo / consoleinfo / musicinfo / releasesearch as your INNODB version does not support fulltext indexes")); } } else if (isset($argv[1]) && $argv[1] == "cinnodb-noparts") { $sql = 'SHOW TABLE STATUS WHERE (Engine != "InnoDB" OR Row_format != "Compressed") AND Engine != "SPHINX"'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; - if ($tbl !== 'releasenfo' && $tbl !== 'releasesearch' && !preg_match('/parts/', $tbl)) { + if ($tbl !== 'release_nfos' && $tbl !== 'releasesearch' && $tbl !== 'bookinfo' && $tbl !== 'consoleinfo' && $tbl !== 'musicinfo' && !preg_match('/parts/', $tbl)) { printf($cli->header("Converting $tbl")); $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED"); } } - $sql = 'SHOW TABLE STATUS WHERE Name = "releasenfo" AND (Engine != "InnoDB" || Row_format != "Dynamic")'; + $sql = 'SHOW TABLE STATUS WHERE Name = "release_nfos" AND (Engine != "InnoDB" || Row_format != "Dynamic")'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; @@ -122,7 +122,7 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=MyISAM ROW_FORMAT=DYNAMIC"); } if ($ftinnodb) { - $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "predbhash") AND (Engine != "InnoDB" || Row_format != "Compressed")'; + $sql = 'SHOW TABLE STATUS WHERE Name IN ("releasesearch", "bookinfo", "consoleinfo", "musicinfo") AND (Engine != "InnoDB" || Row_format != "Compressed")'; $tables = $pdo->query($sql); foreach ($tables as $row) { $tbl = $row['name']; @@ -130,10 +130,10 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { $pdo->queryExec("ALTER TABLE $tbl ENGINE=INNODB ROW_FORMAT=COMPRESSED"); } } else { - printf($cli->header("Not converting releasesearch / predbhash as your INNODB version does not support fulltext indexes")); + printf($cli->header("Not converting bookinfo / consoleinfo / musicinfo / releasesearch as your INNODB version does not support fulltext indexes")); } -} else if (isset($argv[1]) && $argv[1] == "binaries") { - $arr = array("parts", "binaries"); +} else if (isset($argv[1]) && $argv[1] == "collections") { + $arr = array("parts", "binaries", "collections"); foreach ($arr as $row) { $tbl = $row; printf($cli->header("Converting $tbl")); @@ -169,14 +169,14 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { . "php convert_mysql_tables.php fmyisam ...: Converts all the tables to Myisam Fixed. This can be faster, but to fully convert all tables requires changing varchar columns to char.\n" . " This will use much more space than dynamic.\n" . "php convert_mysql_tables.php dinnodb ...: Converts all the tables to InnoDB Dynamic. This is recommended when the total data and indexes can fit into the innodb_buffer_pool.\n" - . " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n" - . "php convert_mysql_tables.php cinnodb ...: Converts all the tables to InnoDB Compressed. All tables except releasenfo will be converted to Compressed row format.\n" + . " NB if your innodb version < 5.6 bookinfo / consoleinfo / musicinfo / releasesearch will not be converted as fulltext indexes are not supported.\n" + . "php convert_mysql_tables.php cinnodb ...: Converts all the tables to InnoDB Compressed. All tables except release_nfos will be converted to Compressed row format.\n" . " This is recommended when the total data and indexes can not fit into the innodb_buffer_pool using DYNAMIC row format.\n" - . " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n" - . "php convert_mysql_tables.php cinnodb-noparts ...: Converts all the tables to InnoDB Compressed. All tables except parts and releasenfo will be converted to Compressed row format.\n" + . " NB if your innodb version < 5.6 bookinfo / consoleinfo / musicinfo / releasesearch will not be converted as fulltext indexes are not supported.\n" + . "php convert_mysql_tables.php cinnodb-noparts ...: Converts all the tables to InnoDB Compressed. All tables except parts and release_nfos will be converted to Compressed row format.\n" . " Alls parts* will be converted to MyISAM Dynamic. This is recommended when using Table Per Group.\n" - . " NB if your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n" - . "php convert_mysql_tables.php binaries ...: Converts binaries, parts to MyIsam.\n" + . " NB if your innodb version < 5.6 bookinfo / consoleinfo / musicinfo / releasesearch will not be converted as fulltext indexes are not supported.\n" + . "php convert_mysql_tables.php collections ...: Converts collections, binaries, parts to MyIsam.\n" . "php convert_mysql_tables.php mariadb-tokudb ...: Converts all the tables to MariaDB Tokutek DB. Use this is you installed mariadb-tokudb-engine. \n" . " The TokuDB engine needs to be activated first.\n" . " https://mariadb.com/kb/en/how-to-enable-tokudb-in-mariadb/\n" @@ -185,6 +185,6 @@ if (isset($argv[1]) && isset($argv[2]) && $argv[2] == "fmyisam") { . " http://www.tokutek.com/resources/support/gadownloads/\n" . " NB releasesearch will not be converted as tokudb does not support fulltext indexes.\n" . "php convert_mysql_tables.php table [ fmyisam, dmyisam, dinnodb, cinnodb ] ...: Converts 1 table to Engine, row_format specified.\n" - . " NB if converting to innodb and your innodb version < 5.6 releasesearch / predbhash will not be converted as fulltext indexes are not supported.\n" + . " NB if converting to innodb and your innodb version < 5.6 releasesearch will not be converted as fulltext indexes are not supported.\n" )); } \ No newline at end of file diff --git a/misc/testing/PreDB/dump_predb.php b/misc/testing/PreDB/dump_predb.php index 16b184e7a..b21a8a0f8 100644 --- a/misc/testing/PreDB/dump_predb.php +++ b/misc/testing/PreDB/dump_predb.php @@ -32,7 +32,7 @@ if (isset($argv[1]) && $argv[1] == 'export' && isset($argv[2])) { $table = 'prehash'; } echo $pdo->log->header("SELECT title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, g.name FROM " . $table . " p LEFT OUTER JOIN groups g ON p.groupid = g.id INTO OUTFILE '" . $path . "' FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n';n"); - $pdo->queryDirect("SELECT title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, g.name FROM " . $table . " p LEFT OUTER JOIN groups g ON p.groupid = g.id INTO OUTFILE '" . $path . "' FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n'"); + $pdo->queryExec("SELECT title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, g.name FROM " . $table . " p LEFT OUTER JOIN groups g ON p.groupid = g.id INTO OUTFILE '" . $path . "' FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n'"); } else if (isset($argv[1]) && ($argv[1] == 'local' || $argv[1] == 'remote') && isset($argv[2]) && is_file($argv[2])) { if (!preg_match('/^\//', $path)) { $path = require_once getcwd() . '/' . $argv[2]; @@ -40,45 +40,46 @@ if (isset($argv[1]) && $argv[1] == 'export' && isset($argv[2])) { if (isset($argv[3])) { $table = $argv[3]; } else { - $table = 'prehash'; + $table = 'predb'; } - // Truncate prehash_imports to clear any old data - $pdo->queryDirect("TRUNCATE TABLE prehash_imports"); + // Truncate predb_imports to clear any old data + $pdo->queryExec("TRUNCATE TABLE predb_imports"); - // Import file into prehash_imports + // Import file into predb_imports if ($argv[1] == 'remote') { - echo $pdo->log->header("LOAD DATA LOCAL INFILE '" . $path . "' IGNORE into table prehash_imports FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);"); - $pdo->queryDirect("LOAD DATA LOCAL INFILE '" . $path . "' IGNORE into table prehash_imports FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname)"); + echo $pdo->log->header("LOAD DATA LOCAL INFILE '" . $path . "' IGNORE into table predb_imports FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);"); + $pdo->queryExec("LOAD DATA LOCAL INFILE '" . $path . "' IGNORE into table predb_imports FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname)"); } else { - echo $pdo->log->header("LOAD DATA INFILE '" . $path . "' IGNORE into table prehash_imports FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);"); - $pdo->queryDirect("LOAD DATA INFILE '" . $path . "' IGNORE into table prehash_imports FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname)"); + echo $pdo->log->header("LOAD DATA INFILE '" . $path . "' IGNORE into table predb_imports FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);"); + $pdo->queryExec("LOAD DATA INFILE '" . $path . "' IGNORE into table predb_imports FIELDS TERMINATED BY '\t\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\r\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname)"); } // Remove any titles where length <=8 echo $pdo->log->info("Deleting any records where title <=8 from Temporary Table"); - $pdo->queryDirect("DELETE FROM prehash_imports WHERE LENGTH(title) <= 8"); + $pdo->queryExec("DELETE FROM predb_imports WHERE LENGTH(title) <= 8"); // Add any groups that do not currently exist $sqlAddGroups = <<queryDirect($sqlAddGroups); + $pdo->queryExec($sqlAddGroups); - // Drop triggers on prehash - echo $pdo->log->info("Dropping predb_hashes triggers"); - $pdo->queryDirect("DROP TRIGGER IF EXISTS insert_hashes"); - $pdo->queryDirect("DROP TRIGGER IF EXISTS update_hashes"); + // Drop triggers on predb + echo $pdo->log->info("Dropping predbhash triggers"); + $pdo->queryExec("DROP TRIGGER IF EXISTS insert_hashes"); + $pdo->queryExec("DROP TRIGGER IF EXISTS update_hashes"); + $pdo->queryExec("DROP TRIGGER IF EXISTS delete_hashes"); // Insert and update table $sqlInsert = <<log->primary($sqlInsert); - $pdo->queryDirect($sqlInsert); + $pdo->queryExec($sqlInsert); - // Add hashes - echo $pdo->log->info("Adding predb_hashes entries"); - $pdo->queryDirect("INSERT IGNORE INTO predb_hashes (pre_id, hashes) (SELECT id, CONCAT_WS(',', MD5(title), MD5(MD5(title)), SHA1(title)) FROM prehash)"); + // Add hashes g + echo $pdo->log->info("Adding predbhash entries"); + echo $pdo->log->info("Stage 1: UNHEX(MD5(TITLE))"); + $pdo->queryExec("INSERT IGNORE INTO predbhash (hash, pre_id) SELECT UNHEX(md5(title)), id from predb;"); + echo $pdo->log->info("Stage 1: UNHEX(MD5(MD5(TITLE)))"); + $pdo->queryExec("INSERT IGNORE INTO predbhash (hash, pre_id) SELECT UNHEX(md5(md5(title))), id from predb;"); + echo $pdo->log->info("Stage 1: UNHEX(SHA1(TITLE))"); + $pdo->queryExec("INSERT IGNORE INTO predbhash (hash, pre_id) SELECT UNHEX(sha1(title)), id from predb;"); - // Re-add triggers on prehash - echo $pdo->log->info("Adding predb_hashes triggers"); - $pdo->exec("CREATE TRIGGER insert_hashes AFTER INSERT ON prehash FOR EACH ROW BEGIN INSERT INTO predb_hashes (pre_id, hashes) VALUES (NEW.id, CONCAT_WS(',', MD5(NEW.title), MD5(MD5(NEW.title)), SHA1(NEW.title))); END;"); - $pdo->exec("CREATE TRIGGER update_hashes AFTER UPDATE ON prehash FOR EACH ROW BEGIN IF NEW.title != OLD.title THEN UPDATE predb_hashes SET hashes = CONCAT_WS(',', MD5(NEW.title), MD5(MD5(NEW.title)), SHA1(NEW.title)) WHERE pre_id = OLD.id; END IF; END;"); + // Re-add triggers on predb + echo $pdo->log->info("Adding predbhash triggers"); + $pdo->queryExec("CREATE TRIGGER insert_hashes AFTER INSERT ON predb FOR EACH ROW BEGIN INSERT INTO predbhash (hash, pre_id) VALUES (UNHEX(MD5(NEW.title)), NEW.id), (UNHEX(MD5(MD5(NEW.title))), NEW.id), ( UNHEX(SHA1(NEW.title)), NEW.id); END;"); + $pdo->queryExec("CREATE TRIGGER update_hashes AFTER UPDATE ON predb FOR EACH ROW BEGIN IF NEW.title != OLD.title THEN DELETE FROM predbhash WHERE hash IN ( UNHEX(md5(OLD.title)), UNHEX(md5(md5(OLD.title))), UNHEX(sha1(OLD.title)) ) AND pre_id = OLD.id; INSERT INTO predbhash (hash, pre_id) VALUES ( UNHEX(MD5(NEW.title)), NEW.id ), ( UNHEX(MD5(MD5(NEW.title))), NEW.id ), ( UNHEX(SHA1(NEW.title)), NEW.id ); END IF; END;"); + $pdo->queryExec("CREATE TRIGGER delete_hashes BEGIN DELETE FROM predbhash WHERE hash IN ( UNHEX(md5(OLD.title)), UNHEX(md5(md5(OLD.title))), UNHEX(sha1(OLD.title)) ) AND pre_id = OLD.id; END;"); - $pdo->queryDirect("TRUNCATE TABLE prehash_imports"); + $pdo->queryExec("TRUNCATE TABLE predb_imports"); } else { - exit($pdo->log->error("\nThis script can export or import a prehash dump file. You may use the full path, or a relative path.\n" - . "For importing, the script insert new rows and update existing matched rows. For databases not on the local system, use remote, else use local.\n" - . "For exporting, the path must be writeable by mysql, any existing file[predb_dump.csv] will be + exit($pdo->log->error("\nThis script can export or import a predb dump file. You may use the full path, or a relative path.\n" + . "For importing, the script insert new rows and update existing matched rows. For databases not on the local system, use remote, else use local.\n" + . "For exporting, the path must be writeable by mysql, any existing file[predb_dump.csv] will be overwritten.\n\n" - . "php dump_predb.php export /path/to/write/to ...: To export.\n" - . "php dump_predb.php [remote | local] /path/to/filename ...: To import.\n")); + . "php dump_predb.php export /path/to/write/to ...: To export.\n" + . "php dump_predb.php [remote | local] /path/to/filename ...: To import.\n")); } diff --git a/newznab/build/newznab.xml b/newznab/build/newznab.xml index e7f2c1374..253a266f4 100644 --- a/newznab/build/newznab.xml +++ b/newznab/build/newznab.xml @@ -2,8 +2,8 @@ - 172 - 172 + 174 + 174 0.4.1 diff --git a/newznab/controllers/NameFixer.php b/newznab/controllers/NameFixer.php index 449fd1805..6be792be6 100644 --- a/newznab/controllers/NameFixer.php +++ b/newznab/controllers/NameFixer.php @@ -852,9 +852,9 @@ class NameFixer sprintf(" SELECT p.id AS prehashid, p.title, p.source FROM prehash p INNER JOIN predbhash h ON h.pre_id = p.id - WHERE MATCH (h.hashes) AGAINST (%s) + WHERE h.hash = UNHEX(%s) LIMIT 1", - $pdo->escapeString(strtolower($hash)) + $pdo->escapeString($hash) ) ); diff --git a/newznab/db/DbUpdate.php b/newznab/db/DbUpdate.php index 8c57fd626..c895fe6f3 100644 --- a/newznab/db/DbUpdate.php +++ b/newznab/db/DbUpdate.php @@ -283,31 +283,44 @@ class DbUpdate if (is_resource($file)) { $query = []; - $oldDelimiter = ''; + $delimiter = $options['delimiter']; while (!feof($file)) { $line = fgets($file); - // Skip comments. - if (preg_match('!^\s*(#|--|//)!', $line)) { - echo $this->pdo->log->info("COMMENT: " . $line); + if ($line === false) { continue; } - // Check for DELIMITER $$, set delimiter to $$ to send query as is to MySQL. - if (preg_match('#^\s*DELIMITER\s+(?P.+)\s*$#i', $line, $matches)) { - $oldDelimiter = $options['delimiter']; - $options['delimiter'] = $matches['delimiter']; + // Skip comments. + if (preg_match('!^\s*(#|--|//)\s*(.+?)\s*$!', $line, $matches)) { + echo $this->pdo->log->info("COMMENT: " . $matches[2]); continue; - } elseif ($oldDelimiter != '' && $options['delimiter'] != $oldDelimiter - && preg_match('#' . preg_quote($options['delimiter']) . '\s*$#', $line)) { - // Replace the last delimiter ($$) with ; - $line = str_replace($options['delimiter'], ';', $line); - // Reset the delimiter back to ; - $options['delimiter'] = $oldDelimiter; } + + // Check for non default delimiters ($$ for example). + if (preg_match('#^\s*DELIMITER\s+(?P.+)\s*$#i', $line, $matches)) { + $delimiter = $matches['delimiter']; + if (NN_DEBUG) { + echo $this->pdo->log->debug("DEBUG: Delimiter switched to $delimiter"); + } + if ($delimiter != $options['delimiter']) { + continue; + } + } + + // Check if the line has delimiter that is non default ($$ for example). + if ($delimiter != $options['delimiter'] && preg_match('#^(.+?)' . preg_quote($delimiter) . '\s*$#', $line, $matches)) { + // Check if the line has also the default delimiter (;), remove it. + if (preg_match('#^(.+?)' . preg_quote($options['delimiter']) . '\s*$#', $matches[1], $matches2)) { + $matches[1] = $matches2[1]; + } + // Change the non default delimiter ($$) to the default one(;). + $line = $matches[1] . $options['delimiter']; + } + $query[] = $line; - if (preg_match('~' . preg_quote($options['delimiter'], '~') . '\s*$~iS', end($query)) == 1) { + if (preg_match('~' . preg_quote($delimiter, '~') . '\s*$~iS', $line) == 1) { $query = trim(implode('', $query)); if ($options['local'] !== null) { $query = str_replace('{:local:}', $options['local'], $query); @@ -319,8 +332,7 @@ class DbUpdate try { $qry = $this->pdo->prepare($query); $qry->execute(); - echo $this->log->alternateOver('SUCCESS: ') . - $this->log->primary($query); + echo $this->log->alternateOver('SUCCESS: ') . $this->log->primary($query); } catch (\PDOException $e) { // Log the problem and the query. file_put_contents( @@ -334,25 +346,25 @@ class DbUpdate if ( in_array($e->errorInfo[1], [1091, 1060, 1061, 1071, 1146]) || - in_array($e->errorInfo[0], - [23505, 42701, 42703, '42P07', '42P16']) + in_array($e->errorInfo[0], [23505, 42701, 42703, '42P07', '42P16']) ) { if ($e->errorInfo[1] == 1060) { echo $this->log->warning( - "$query The column already exists - No need to worry \{" . - $e->errorInfo[1] . "}.\n"); + "$query The column already exists - No need to worry \{" . + $e->errorInfo[1] . "}.\n" + ); } else { echo $this->log->warning( - "$query Skipped - No need to worry \{" . - $e->errorInfo[1] . "}.\n"); + "$query Skipped - No need to worry \{" . + $e->errorInfo[1] . "}.\n" + ); } } else { if (preg_match('/ALTER IGNORE/i', $query)) { $this->pdo->queryExec("SET SESSION old_alter_table = 1"); try { $this->pdo->exec($query); - echo $this->log->alternateOver('SUCCESS: ') . - $this->log->primary($query); + echo $this->log->alternateOver('SUCCESS: ') . $this->log->primary($query); } catch (\PDOException $e) { exit($this->log->error("$query Failed \{" . $e->errorInfo[1] . "}\n\t" . $e->errorInfo[2])); } diff --git a/resources/db/patches/0173~predb_hashes.sql b/resources/db/patches/0173~predb_hashes.sql new file mode 100644 index 000000000..95f1741ee --- /dev/null +++ b/resources/db/patches/0173~predb_hashes.sql @@ -0,0 +1,22 @@ +-- This can be a long patch, you can check in mysql (mysql -p yourDatabaseName) what is going on +-- with this command: show processlist; + +-- Truncating predbhash table. +TRUNCATE TABLE predbhash; + +-- Dropping primary key from predbhash table. +ALTER TABLE predbhash DROP PRIMARY KEY; +-- Dropping the hashes column from the predbhash table. +ALTER TABLE predbhash DROP COLUMN hashes; +-- Adding the hash column to the predbhash table. +ALTER TABLE predbhash ADD hash VARBINARY(20) FIRST; +-- Adding a primary key to the hash column. +ALTER TABLE predbhash ADD PRIMARY KEY(hash); + +-- Creating hashes from predb titles, this can be long. +-- Stage 1: UNHEX(md5(title)) +INSERT IGNORE INTO predbhash SELECT UNHEX(md5(title)), id from predb; +-- Stage 2: UNHEX(md5(md5(title))) +INSERT IGNORE INTO predbhash SELECT UNHEX(md5(md5(title))), id from predb; +-- Stage 3: UNHEX(sha1(title)) +INSERT IGNORE INTO predbhash SELECT UNHEX(sha1(title)), id from predb; diff --git a/resources/db/patches/0174~predb.sql b/resources/db/patches/0174~predb.sql new file mode 100644 index 000000000..4f200e254 --- /dev/null +++ b/resources/db/patches/0174~predb.sql @@ -0,0 +1,28 @@ +-- This should be a quick patch. +-- You can verify them by running this command in MySQL: SHOW TRIGGERS LIKE 'prehash' \g + +DELIMITER $$ + +DROP TRIGGER IF EXISTS insert_hashes $$ +CREATE TRIGGER insert_hashes AFTER INSERT ON prehash FOR EACH ROW + BEGIN + INSERT INTO predbhash (hash, pre_id) VALUES (UNHEX(MD5(NEW.title)), NEW.id), (UNHEX(MD5(MD5(NEW.title))), NEW.id), ( UNHEX(SHA1(NEW.title)), NEW.id); + END; $$ + +DROP TRIGGER IF EXISTS update_hashes $$ +CREATE TRIGGER update_hashes AFTER UPDATE ON prehash FOR EACH ROW + BEGIN + IF NEW.title != OLD.title + THEN + DELETE FROM predbhash WHERE hash IN ( UNHEX(md5(OLD.title)), UNHEX(md5(md5(OLD.title))), UNHEX(sha1(OLD.title)) ) AND pre_id = OLD.id; + INSERT INTO predbhash (hash, pre_id) VALUES ( UNHEX(MD5(NEW.title)), NEW.id ), ( UNHEX(MD5(MD5(NEW.title))), NEW.id ), ( UNHEX(SHA1(NEW.title)), NEW.id ); + END IF; + END; $$ + +DROP TRIGGER IF EXISTS delete_hashes $$ +CREATE TRIGGER delete_hashes AFTER DELETE ON prehash FOR EACH ROW + BEGIN + DELETE FROM predbhash WHERE hash IN ( UNHEX(md5(OLD.title)), UNHEX(md5(md5(OLD.title))), UNHEX(sha1(OLD.title)) ) AND pre_id = OLD.id; + END; $$ + +DELIMITER ;