From 603c22d62ca9c11cbdbc5f38e98dec36229e8700 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Mon, 29 Jan 2018 13:17:35 +0100 Subject: [PATCH] Remove partitioning from releases table, add foreign keys to related tables --- ...018_01_20_195528_create_releases_table.php | 10 ------ ...018_01_22_220858_add_stored_procedures.php | 24 ++----------- misc/update/tmux/monitor.php | 34 +++++++++--------- nntmux/Releases.php | 17 ++++----- nntmux/libraries/Forking.php | 16 ++++----- resources/db/patches/mysql/+1~releases.sql | 3 ++ resources/db/patches/mysql/+2~general.sql | 29 +++++++++++++++ resources/db/patches/mysql/+3~general.sql | 31 ++++++++++++++++ resources/db/schema/mysql-ddl.sql | 35 ++----------------- 9 files changed, 102 insertions(+), 97 deletions(-) create mode 100644 resources/db/patches/mysql/+1~releases.sql create mode 100644 resources/db/patches/mysql/+2~general.sql create mode 100644 resources/db/patches/mysql/+3~general.sql diff --git a/database/migrations/2018_01_20_195528_create_releases_table.php b/database/migrations/2018_01_20_195528_create_releases_table.php index 6baa55d2d..837bbdeae 100644 --- a/database/migrations/2018_01_20_195528_create_releases_table.php +++ b/database/migrations/2018_01_20_195528_create_releases_table.php @@ -75,16 +75,6 @@ processed'); }); DB::unprepared('ALTER TABLE releases DROP PRIMARY KEY , ADD PRIMARY KEY (id, categories_id)'); - DB::unprepared('ALTER TABLE releases PARTITION BY RANGE (categories_id) ( - PARTITION misc VALUES LESS THAN (1000), - PARTITION console VALUES LESS THAN (2000), - PARTITION movies VALUES LESS THAN (3000), - PARTITION audio VALUES LESS THAN (4000), - PARTITION pc VALUES LESS THAN (5000), - PARTITION tv VALUES LESS THAN (6000), - PARTITION xxx VALUES LESS THAN (7000), - PARTITION books VALUES LESS THAN (8000) - );'); DB::unprepared('ALTER TABLE releases ADD COLUMN nzb_guid BINARY(16) NULL'); DB::unprepared('ALTER TABLE releases ADD INDEX ix_releases_nzb_guid (nzb_guid)'); } diff --git a/database/migrations/2018_01_22_220858_add_stored_procedures.php b/database/migrations/2018_01_22_220858_add_stored_procedures.php index fd6b43c1b..986fd606c 100644 --- a/database/migrations/2018_01_22_220858_add_stored_procedures.php +++ b/database/migrations/2018_01_22_220858_add_stored_procedures.php @@ -59,34 +59,14 @@ CREATE PROCEDURE delete_release(IN is_numeric BOOLEAN, IN identifier VARCHAR(40) IF is_numeric IS TRUE THEN - DELETE r, rn, rc, uc, rf, ra, rs, rv, re, df, rg + DELETE r FROM releases r - LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id - LEFT OUTER JOIN release_comments rc ON rc.releases_id = r.id - LEFT OUTER JOIN users_releases uc ON uc.releases_id = r.id - LEFT OUTER JOIN release_files rf ON rf.releases_id = r.id - LEFT OUTER JOIN audio_data ra ON ra.releases_id = r.id - LEFT OUTER JOIN release_subtitles rs ON rs.releases_id = r.id - LEFT OUTER JOIN video_data rv ON rv.releases_id = r.id - LEFT OUTER JOIN releaseextrafull re ON re.releases_id = r.id - LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id - LEFT OUTER JOIN releases_groups rg ON rg.releases_id = r.id WHERE r.id = identifier; ELSEIF is_numeric IS FALSE THEN - DELETE r, rn, rc, uc, rf, ra, rs, rv, re, df, rg + DELETE r FROM releases r - LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id - LEFT OUTER JOIN release_comments rc ON rc.releases_id = r.id - LEFT OUTER JOIN users_releases uc ON uc.releases_id = r.id - LEFT OUTER JOIN release_files rf ON rf.releases_id = r.id - LEFT OUTER JOIN audio_data ra ON ra.releases_id = r.id - LEFT OUTER JOIN release_subtitles rs ON rs.releases_id = r.id - LEFT OUTER JOIN video_data rv ON rv.releases_id = r.id - LEFT OUTER JOIN releaseextrafull re ON re.releases_id = r.id - LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id - LEFT OUTER JOIN releases_groups rg ON rg.releases_id = r.id WHERE r.guid = identifier; ELSE LEAVE main; diff --git a/misc/update/tmux/monitor.php b/misc/update/tmux/monitor.php index 2ab85e8dc..87b409ca9 100644 --- a/misc/update/tmux/monitor.php +++ b/misc/update/tmux/monitor.php @@ -2,6 +2,7 @@ require_once dirname(__DIR__, 3).DIRECTORY_SEPARATOR.'bootstrap/autoload.php'; +use App\Models\Release; use nntmux\Tmux; use nntmux\db\DB; use nntmux\TmuxRun; @@ -150,22 +151,23 @@ while ($runVar['counts']['iterations'] > 0) { $timer03 = time(); - //This is subpartition compatible -- loops through all partitions and adds their total row counts instead of doing a slow query count - $partitions = $pdo->queryDirect( - sprintf( - " - SELECT SUM(TABLE_ROWS) AS count, PARTITION_NAME AS category - FROM information_schema.PARTITIONS - WHERE TABLE_NAME = 'releases' - AND TABLE_SCHEMA = %s - GROUP BY PARTITION_NAME", - $pdo->escapeString($db_name) - ) - ); - foreach ($partitions as $partition) { - $runVar['counts']['now'][$partition['category']] = $partition['count']; - } - unset($partitions); + $tvCount = Release::query()->whereBetween('categories_id', [Category::TV_ROOT, Category::TV_OTHER])->count(['id']); + $movieCount = Release::query()->whereBetween('categories_id', [Category::MOVIE_ROOT, Category::MOVIE_OTHER])->count(['id']); + $audioCount = Release::query()->whereBetween('categories_id', [Category::MUSIC_ROOT, Category::MUSIC_OTHER])->count(['id']); + $bookCount = Release::query()->whereBetween('categories_id', [Category::BOOKS_ROOT, Category::BOOKS_UNKNOWN])->count(['id']); + $consoleCount = Release::query()->whereBetween('categories_id', [Category::GAME_ROOT, Category::GAME_OTHER])->count(['id']); + $pcCount = Release::query()->whereBetween('categories_id', [Category::PC_ROOT, Category::PC_PHONE_ANDROID])->count(['id']); + $xxxCount = Release::query()->whereBetween('categories_id', [Category::XXX_ROOT, Category::XXX_OTHER])->count(['id']); + $miscCount = Release::query()->whereBetween('categories_id', [Category::OTHER_ROOT, Category::OTHER_HASHED])->count(['id']); + + $runVar['counts']['now']['audio'] = $audioCount; + $runVar['counts']['now']['books'] = $bookCount; + $runVar['counts']['now']['console'] = $consoleCount; + $runVar['counts']['now']['misc'] = $miscCount; + $runVar['counts']['now']['movies'] = $movieCount; + $runVar['counts']['now']['pc'] = $pcCount; + $runVar['counts']['now']['tv'] = $tvCount; + $runVar['counts']['now']['xxx'] = $xxxCount; $runVar['timers']['query']['init_time'] = (time() - $timer03); $runVar['timers']['query']['init1_time'] = (time() - $timer01); diff --git a/nntmux/Releases.php b/nntmux/Releases.php index 37628559f..5eb42f32f 100755 --- a/nntmux/Releases.php +++ b/nntmux/Releases.php @@ -425,7 +425,7 @@ class Releases rn.releases_id AS nfoid, re.releases_id AS reid, tve.firstaired, (SELECT df.failed) AS failed - FROM releases PARTITION (tv) r + FROM releases r LEFT OUTER JOIN video_data re ON re.releases_id = r.id LEFT JOIN groups g ON g.id = r.groups_id LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id @@ -433,7 +433,7 @@ class Releases LEFT JOIN categories c ON c.id = r.categories_id LEFT JOIN categories cp ON cp.id = c.parentid LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id - WHERE %s %s + WHERE r.categories_id BETWEEN 5000 AND 5999 %s %s AND r.nzbstatus = %d AND r.passwordstatus %s %s @@ -476,8 +476,8 @@ class Releases return $this->getPagerCount( sprintf( 'SELECT r.id - FROM releases PARTITION (tv) r - WHERE %s %s + FROM releases r + WHERE r.categories_id BETWEEN 5000 AND 5999 %s %s AND r.nzbstatus = %d AND r.passwordstatus %s %s', @@ -504,8 +504,8 @@ class Releases return $this->getPagerCount( sprintf( 'SELECT r.id - FROM releases PARTITION (movies) r - WHERE %s %s + FROM releases r + WHERE r.categories_id BETWEEN 3000 AND 3999 %s %s AND r.nzbstatus = %d AND r.passwordstatus %s %s', @@ -880,7 +880,7 @@ class Releases g.name AS group_name, rn.releases_id AS nfoid, re.releases_id AS reid - FROM releases PARTITION (tv) r + FROM releases r LEFT OUTER JOIN videos v ON r.videos_id = v.id AND v.type = 0 LEFT OUTER JOIN tv_info tvi ON v.id = tvi.videos_id LEFT OUTER JOIN tv_episodes tve ON r.tv_episodes_id = tve.id @@ -1021,11 +1021,12 @@ class Releases %s AS category_ids, g.name AS group_name, rn.releases_id AS nfoid - FROM releases PARTITION (movies) r + FROM releases r LEFT JOIN groups g ON g.id = r.groups_id LEFT JOIN categories c ON c.id = r.categories_id LEFT JOIN categories cp ON cp.id = c.parentid LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id + WHERE r.categories_id BETWEEN 5000 AND 5999 %s", $this->getConcatenatedCategoryIDs(), $whereSql diff --git a/nntmux/libraries/Forking.php b/nntmux/libraries/Forking.php index bfa64fc16..ade99e479 100755 --- a/nntmux/libraries/Forking.php +++ b/nntmux/libraries/Forking.php @@ -860,8 +860,8 @@ class Forking extends \fork_daemon return $this->pdo->queryOneRow(sprintf(' SELECT id FROM releases - PARTITION (movies) - WHERE nzbstatus = %d + WHERE categories_id BETWEEN 5000 AND 5999 + AND nzbstatus = %d AND imdbid IS NULL %s %s LIMIT 1', NZB::NZB_ADDED, ((int) Settings::settingValue('..lookupimdb') === 2 ? 'AND isrenamed = 1' : ''), ($this->ppRenamedOnly ? 'AND isrenamed = 1' : ''))) !== false; @@ -885,8 +885,8 @@ class Forking extends \fork_daemon ' SELECT leftguid AS id, %d AS renamed FROM releases - PARTITION (movies) - WHERE nzbstatus = %d + WHERE categories_id BETWEEN 5000 AND 5999 + AND nzbstatus = %d AND imdbid IS NULL %s %s GROUP BY leftguid @@ -914,8 +914,8 @@ class Forking extends \fork_daemon return $this->pdo->queryOneRow(sprintf(' SELECT id FROM releases - PARTITION (tv) - WHERE nzbstatus = %d + WHERE categories_id BETWEEN 3000 AND 53999 + AND nzbstatus = %d AND size > 1048576 AND tv_episodes_id BETWEEN -2 AND 0 %s %s @@ -940,8 +940,8 @@ class Forking extends \fork_daemon ' SELECT leftguid AS id, %d AS renamed FROM releases - PARTITION (tv) - WHERE nzbstatus = %d + WHERE categories_id BETWEEN 3000 AND 3999 + AND nzbstatus = %d AND tv_episodes_id BETWEEN -2 AND 0 AND size > 1048576 %s %s diff --git a/resources/db/patches/mysql/+1~releases.sql b/resources/db/patches/mysql/+1~releases.sql new file mode 100644 index 000000000..1f76c769c --- /dev/null +++ b/resources/db/patches/mysql/+1~releases.sql @@ -0,0 +1,3 @@ +# Remove partitions from releases table + +ALTER TABLE releases REMOVE PARTITIONING; \ No newline at end of file diff --git a/resources/db/patches/mysql/+2~general.sql b/resources/db/patches/mysql/+2~general.sql new file mode 100644 index 000000000..67ced0c0c --- /dev/null +++ b/resources/db/patches/mysql/+2~general.sql @@ -0,0 +1,29 @@ +# Alter tables related to releases table to add foreign keys to them + +# Alter tables related to releases table to add foreign keys to them + +ALTER TABLE audio_data ADD CONSTRAINT FK_ad_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE par_hashes ADD CONSTRAINT FK_ph_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_comments ADD CONSTRAINT FK_rc_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE releases_groups ADD CONSTRAINT FK_rg_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_unique ADD CONSTRAINT FK_ru_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE releaseextrafull ADD CONSTRAINT FK_ref_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_files ADD CONSTRAINT FK_rf_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_nfos ADD CONSTRAINT FK_rn_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_search_data ADD CONSTRAINT FK_rsd_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE release_subtitles ADD CONSTRAINT FK_rs_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE users_releases ADD CONSTRAINT FK_ur_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE video_data ADD CONSTRAINT FK_vd_releases FOREIGN KEY (releases_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE dnzb_failures ADD CONSTRAINT FK_df_releases FOREIGN KEY (release_id) REFERENCES releases(id) ON DELETE CASCADE ON UPDATE CASCADE; + + + + + + + + + + + + diff --git a/resources/db/patches/mysql/+3~general.sql b/resources/db/patches/mysql/+3~general.sql new file mode 100644 index 000000000..c9fc8daa7 --- /dev/null +++ b/resources/db/patches/mysql/+3~general.sql @@ -0,0 +1,31 @@ +# Recreate delete_release stored procedure + +DELIMITER $$ + +DROP PROCEDURE IF EXISTS delete_release; +CREATE PROCEDURE delete_release(IN is_numeric BOOLEAN, IN identifier VARCHAR(40)) + COMMENT 'Cascade deletes release from child tables when parent row is deleted' + COMMENT 'If is_numeric is true, identifier should be the releases_id, if false the guid' + + main: BEGIN + + DECLARE where_constr VARCHAR(255) DEFAULT ''; + + IF is_numeric IS TRUE + THEN + DELETE r + FROM releases r + WHERE r.id = identifier; + + ELSEIF is_numeric IS FALSE + THEN + DELETE r + FROM releases r + WHERE r.guid = identifier; + + ELSE LEAVE main; + END IF; + + END;$$ + +DELIMITER; \ No newline at end of file diff --git a/resources/db/schema/mysql-ddl.sql b/resources/db/schema/mysql-ddl.sql index 5664e6dc7..82316dd98 100755 --- a/resources/db/schema/mysql-ddl.sql +++ b/resources/db/schema/mysql-ddl.sql @@ -112,17 +112,6 @@ processed', ROW_FORMAT = DYNAMIC AUTO_INCREMENT = 1 - PARTITION BY RANGE (categories_id) ( - PARTITION misc VALUES LESS THAN (1000), - PARTITION console VALUES LESS THAN (2000), - PARTITION movies VALUES LESS THAN (3000), - PARTITION audio VALUES LESS THAN (4000), - PARTITION pc VALUES LESS THAN (5000), - PARTITION tv VALUES LESS THAN (6000), - PARTITION xxx VALUES LESS THAN (7000), - PARTITION books VALUES LESS THAN (8000) - ); - DROP TABLE IF EXISTS users; CREATE TABLE users ( id INT(16) UNSIGNED NOT NULL AUTO_INCREMENT, @@ -1319,34 +1308,14 @@ CREATE PROCEDURE delete_release(IN is_numeric BOOLEAN, IN identifier VARCHAR(40) IF is_numeric IS TRUE THEN - DELETE r, rn, rc, uc, rf, ra, rs, rv, re, df, rg + DELETE r FROM releases r - LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id - LEFT OUTER JOIN release_comments rc ON rc.releases_id = r.id - LEFT OUTER JOIN users_releases uc ON uc.releases_id = r.id - LEFT OUTER JOIN release_files rf ON rf.releases_id = r.id - LEFT OUTER JOIN audio_data ra ON ra.releases_id = r.id - LEFT OUTER JOIN release_subtitles rs ON rs.releases_id = r.id - LEFT OUTER JOIN video_data rv ON rv.releases_id = r.id - LEFT OUTER JOIN releaseextrafull re ON re.releases_id = r.id - LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id - LEFT OUTER JOIN releases_groups rg ON rg.releases_id = r.id WHERE r.id = identifier; ELSEIF is_numeric IS FALSE THEN - DELETE r, rn, rc, uc, rf, ra, rs, rv, re, df, rg + DELETE r FROM releases r - LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id - LEFT OUTER JOIN release_comments rc ON rc.releases_id = r.id - LEFT OUTER JOIN users_releases uc ON uc.releases_id = r.id - LEFT OUTER JOIN release_files rf ON rf.releases_id = r.id - LEFT OUTER JOIN audio_data ra ON ra.releases_id = r.id - LEFT OUTER JOIN release_subtitles rs ON rs.releases_id = r.id - LEFT OUTER JOIN video_data rv ON rv.releases_id = r.id - LEFT OUTER JOIN releaseextrafull re ON re.releases_id = r.id - LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id - LEFT OUTER JOIN releases_groups rg ON rg.releases_id = r.id WHERE r.guid = identifier; ELSE LEAVE main;