From 308cbd85f2d5240f80bb323b54f20ba80bfcc56d Mon Sep 17 00:00:00 2001 From: DariusIII Date: Sun, 29 Jan 2017 22:51:19 +0100 Subject: [PATCH] Refactor mgr support, add collections used tracking --- nntmux/Binaries.php | 52 +++++++++++-------- nntmux/CollectionsCleaning.php | 44 +++++++++++++--- nntmux/Regexes.php | 26 ++++++---- .../processing/ProcessReleasesMultiGroup.php | 7 +-- resources/db/patches/mysql/+8~collections.sql | 34 ++++++++++++ 5 files changed, 122 insertions(+), 41 deletions(-) create mode 100644 resources/db/patches/mysql/+8~collections.sql diff --git a/nntmux/Binaries.php b/nntmux/Binaries.php index 563f8d9a3..a869e52f6 100755 --- a/nntmux/Binaries.php +++ b/nntmux/Binaries.php @@ -80,7 +80,7 @@ class Binaries protected $_partRepair; /** - * @var \nntmux\db\DB + * @var DB */ protected $_pdo; @@ -179,6 +179,7 @@ class Binaries * @var float microseconds time part repair was started */ protected $startPR; + /** * @var array The CBP/MGR tables names */ @@ -757,6 +758,8 @@ class Binaries } } + unset($headers); // Reclaim memory now that headers are split. + if (!empty($this->_binaryBlacklistIdsToUpdate)) { $this->updateBlacklistUsage(); } @@ -765,12 +768,18 @@ class Binaries $this->outputHeaderInitial(); } - if (isset($stdHeaders) && count($stdHeaders) > 0) { - $this->storeHeaders($stdHeaders, false); - } + // MGR headers goes first if (isset($mgrHeaders) && count($mgrHeaders) > 0) { - $this->tableNames = ProcessReleasesMultiGroup::returnTableNames(); + $this->tableNames = ProcessReleasesMultiGroup::tableNames(); $this->storeHeaders($mgrHeaders, true); + unset($mgrHeaders); + } + + // Standard headers go second so we can switch tableNames back and do part repair to standard group tables + if (isset($stdHeaders) && count($stdHeaders) > 0) { + $this->tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $this->groupMySQL['id']); + $this->storeHeaders($stdHeaders, false); + unset($stdHeaders); } // Start of part repair. @@ -782,6 +791,7 @@ class Binaries if ($partRepair && count($headersRepaired) > 0) { $this->removeRepairedParts($headersRepaired, $this->tableNames['prname'], $this->groupMySQL['id']); } + unset($headersRepaired); if ($this->addToPartRepair) { @@ -796,13 +806,13 @@ class Binaries 'warning' ); } + unset($this->headersNotInserted); // Check if we have any missing headers. if (($this->last - $this->first - $this->notYEnc - $this->headersBlackListed + 1) > count($this->headersReceived)) { $rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($this->first, $this->last), $this->headersReceived)); } $notReceivedCount = count($rangeNotReceived); - if ($notReceivedCount > 0) { $this->addMissingParts($rangeNotReceived, $this->tableNames['prname'], $this->groupMySQL['id']); @@ -815,14 +825,15 @@ class Binaries ); } } + unset($rangeNotReceived); } - $this->outputHeaderResults(); + $this->outputHeaderDuration(); return $returnArray; } /** - * Parse and store retrieved headers + * Parse headers into collections/binaries and store header data as parts * * @param array $headers The retrieved headers * @param bool $multiGroup Is this task being run in MGR mode? @@ -861,7 +872,6 @@ class Binaries } } - // Used to group articles together when forming the release. MGR requires this to be group irrespective if ($this->multiGroup) { $ckName = ''; $ckId = ''; @@ -870,16 +880,14 @@ class Binaries $ckId = $this->groupMySQL['id']; } - // Used to group articles together when forming the release. MGR requires this to be group irrespective - $this->header['CollectionKey'] = ( - $this->_collectionsCleaning->collectionsCleaner( - $this->header['matches'][1], - $ckName - ) . - $this->header['From'] . - ($this->multiGroup ? $fileCount[3] : $ckId . $fileCount[3]) + $collMatch = $this->_collectionsCleaning->collectionsCleaner( + $this->header['matches'][1], + $ckName ); + // Used to group articles together when forming the release. MGR requires this to be group irrespective + $this->header['CollectionKey'] = $collMatch['name'] . $this->header['From'] . $ckId . $fileCount[3]; + // If this header's collection key isn't in memory, attempt to insert the collection if (!isset($collectionIDs[$this->header['CollectionKey']])) { @@ -899,8 +907,8 @@ class Binaries $collectionID = $this->_pdo->queryInsert( sprintf(" INSERT INTO %s (subject, fromname, date, xref, groups_id, - totalfiles, collectionhash, dateadded) - VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', NOW()) + totalfiles, collectionhash, collection_regexes_id, dateadded) + VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW()) ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", $this->tableNames['cname'], $this->_pdo->escapeString(substr(utf8_encode($this->header['matches'][1]), 0, 255)), @@ -910,6 +918,7 @@ class Binaries $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), + $collMatch['id'], $xref, bin2hex(openssl_random_pseudo_bytes(16)) ) @@ -964,7 +973,6 @@ class Binaries } else { $binaryID = $articles[$this->header['matches'][1]]['BinaryID']; - //$collectionID = $articles[$matches[1]]['CollectionID']; $binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; $binariesUpdate[$binaryID]['Parts']++; } @@ -1080,9 +1088,9 @@ class Binaries } /** - * Outputs results of the scan function to CLI + * Outputs speed metrics of the scan function to CLI */ - protected function outputHeaderResults() + protected function outputHeaderDuration() { $currentMicroTime = microtime(true); if ($this->_echoCLI) { diff --git a/nntmux/CollectionsCleaning.php b/nntmux/CollectionsCleaning.php index 1f7601116..b0d68ba14 100755 --- a/nntmux/CollectionsCleaning.php +++ b/nntmux/CollectionsCleaning.php @@ -32,6 +32,21 @@ class CollectionsCleaning */ const REGEX_SUBJECT_SIZE = '[-_\s]{0,3}\d+([.,]\d+)? [kKmMgG][bB][-_\s]{0,3}'; + /** + * Collection subject failed to match any regular expression + */ + const REGEX_NO_MATCH = 0; + + /** + * Collection subject matched the Generic regular expression + */ + const REGEX_GENERIC_MATCH = -10; + + /** + * Collection subject matched the Music generic regular expression + */ + const REGEX_MUSIC_MATCH = -20; + /** * @var string */ @@ -58,7 +73,7 @@ class CollectionsCleaning public $subject = ''; /** - * @var \nntmux\db\Settings + * @var DB */ public $pdo; @@ -92,7 +107,7 @@ class CollectionsCleaning * @param string $subject Subject to parse. * @param string $groupName Group to work in. * - * @return string + * @return array The ID of the Regex Matched and the cleaned collection name */ public function collectionsCleaner($subject, $groupName) { @@ -102,7 +117,10 @@ class CollectionsCleaning // Try DB regex first. $potentialString = $this->_regexes->tryRegex($subject, $groupName); if ($potentialString) { - return $potentialString; + return [ + 'id' => $this->_regexes->matchedRegex, + 'name' => $potentialString + ]; } switch ($groupName) { @@ -132,13 +150,19 @@ class CollectionsCleaning // Random stuff. $cleanSubject = preg_replace('/AutoRarPar\d{1,5}|\(\d+\)( | )yEnc|\d+(Amateur|Classic)| \d{4,}[a-z]{4,} |part\d+/i', ' ', $cleanSubject); // Multi spaces. - return utf8_encode(trim(preg_replace('/\s\s+/', ' ', $cleanSubject))); + return [ + 'id' => self::REGEX_GENERIC_MATCH, + 'name' => utf8_encode(trim(preg_replace('/\s\s+/', ' ', $cleanSubject))) + ]; } // Music groups. else { // Try some music group regexes. $musicSubject = $this->musicSubject(); if ($musicSubject !== false) { - return $musicSubject; + return [ + 'id' => self::REGEX_MUSIC_MATCH, + 'name' => $musicSubject + ]; // Parts/files } else { $cleanSubject = preg_replace('/((( \(\d\d\) -|(\d\d)? - \d\d\.|\d{4} \d\d -) | - \d\d-| \d\d\. [a-z]).+| \d\d of \d\d| \dof\d)\.mp3"?|(\(|\[|\s)\d{1,4}(\/|(\s|_)of(\s|_)|-)\d{1,4}(\)|\]|\s|$|:)|\(\d{1,3}\|\d{1,3}\)|-\d{1,3}-\d{1,3}\.|\s\d{1,3}\sof\s\d{1,3}\.|\s\d{1,3}\/\d{1,3}|\d{1,3}of\d{1,3}\.|^\d{1,3}\/\d{1,3}\s|\d{1,3} - of \d{1,3}/i', ' ', $this->subject); @@ -171,9 +195,15 @@ class CollectionsCleaning } $newName = preg_replace('/".+?"/', '', $this->subject); $newName = preg_replace('/[a-z0-9]|' . $this->e0 . '/i', '', $newName); - return $cleanSubject . $newName . $x; + return [ + 'id' => self::REGEX_MUSIC_MATCH, + 'name' => $cleanSubject . $newName . $x + ]; } else { - return $cleanSubject; + return [ + 'id' => self::REGEX_MUSIC_MATCH, + 'name' => $cleanSubject + ]; } } } diff --git a/nntmux/Regexes.php b/nntmux/Regexes.php index c1e9f3cda..2c6860b8e 100755 --- a/nntmux/Regexes.php +++ b/nntmux/Regexes.php @@ -6,10 +6,15 @@ use nntmux\db\DB; class Regexes { /** - * @var \nntmux\db\Settings + * @var DB */ public $pdo; + /** + * @var mixed The ID of the Regex inut string matched or the generic name + */ + public $matchedRegex; + /** * @var string Name of the current table we are working on. */ @@ -23,7 +28,7 @@ class Regexes /** * @var int */ - protected $_categoryID = Category::OTHER_MISC; + protected $_categoriesID = Category::OTHER_MISC; /** * @param array $options @@ -53,13 +58,13 @@ class Regexes sprintf( 'INSERT INTO %s (group_regex, regex, status, description, ordinal%s) VALUES (%s, %s, %d, %s, %d%s)', $this->tableName, - ($this->tableName === 'category_regexes' ? ', category_id' : ''), + ($this->tableName === 'category_regexes' ? ', categories_id' : ''), trim($this->pdo->escapeString($data['group_regex'])), trim($this->pdo->escapeString($data['regex'])), $data['status'], trim($this->pdo->escapeString($data['description'])), $data['ordinal'], - ($this->tableName === 'category_regexes' ? (', ' . $data['category_id']) : '') + ($this->tableName === 'category_regexes' ? (', ' . $data['categories_id']) : '') ) ); } @@ -84,7 +89,7 @@ class Regexes $data['status'], trim($this->pdo->escapeString($data['description'])), $data['ordinal'], - ($this->tableName === 'category_regexes' ? (', category_id = ' . $data['category_id']) : ''), + ($this->tableName === 'category_regexes' ? (', categories_id = ' . $data['categories_id']) : ''), $data['id'] ) ); @@ -281,6 +286,8 @@ class Regexes */ public function tryRegex($subject, $groupName) { + $this->matchedRegex = 0; + $this->_fetchRegex($groupName); $returnString = ''; @@ -289,12 +296,13 @@ class Regexes foreach ($this->_regexCache[$groupName]['regex'] as $regex) { if ($this->tableName === 'category_regexes') { - $this->_categoryID = $regex['category_id']; + $this->_categoriesID = $regex['categories_id']; } $returnString = $this->_matchRegex($regex['regex'], $subject); // If this regex found something, break and return, or else continue trying other regex. if ($returnString) { + $this->matchedRegex = $regex['id']; break; } } @@ -318,7 +326,7 @@ class Regexes // Get all regex from DB which match the current group name. Cache them for 15 minutes. #CACHEDQUERY# $this->_regexCache[$groupName]['regex'] = $this->pdo->query( sprintf( - 'SELECT r.regex%s FROM %s r WHERE %s REGEXP r.group_regex AND r.status = 1 ORDER BY r.ordinal ASC, r.group_regex ASC', + 'SELECT r.id, r.regex%s FROM %s r WHERE %s REGEXP r.group_regex AND r.status = 1 ORDER BY r.ordinal ASC, r.group_regex ASC', ($this->tableName === 'category_regexes' ? ', r.categories_id' : ''), $this->tableName, $this->pdo->escapeString($groupName) @@ -356,7 +364,7 @@ class Regexes $returnString .= $value; // Concatenate the string to return. break; case 'category_regexes': - $returnString = $this->_categoryID; // Regex matched, so return the category ID. + $returnString = $this->_categoriesID; // Regex matched, so return the category ID. break 2; } } @@ -376,4 +384,4 @@ class Regexes { return ($group_regex ? ('WHERE group_regex ' . $this->pdo->likeString($group_regex)) : ''); } -} +} \ No newline at end of file diff --git a/nntmux/processing/ProcessReleasesMultiGroup.php b/nntmux/processing/ProcessReleasesMultiGroup.php index 6bca6409b..e10146434 100644 --- a/nntmux/processing/ProcessReleasesMultiGroup.php +++ b/nntmux/processing/ProcessReleasesMultiGroup.php @@ -50,7 +50,7 @@ class ProcessReleasesMultiGroup extends ProcessReleases */ protected function initiateTableNames($groupID) { - $this->tables = $this->returnTableNames(); + $this->tables = self::tableNames(); } /** @@ -58,12 +58,13 @@ class ProcessReleasesMultiGroup extends ProcessReleases * * @return array */ - public static function returnTableNames() + public static function tableNames() { return [ 'cname' => 'multigroup_collections', 'bname' => 'multigroup_binaries', - 'pname' => 'multigroup_parts' + 'pname' => 'multigroup_parts', + 'prname' => 'multigroup_missed_parts', ]; } } diff --git a/resources/db/patches/mysql/+8~collections.sql b/resources/db/patches/mysql/+8~collections.sql new file mode 100644 index 000000000..4405db4e8 --- /dev/null +++ b/resources/db/patches/mysql/+8~collections.sql @@ -0,0 +1,34 @@ +# Add collection_regexes_id row to collections tables for tracking usage +# +DROP PROCEDURE IF EXISTS change_collections; +DELIMITER $$ +CREATE PROCEDURE change_collections() + BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE _table CHAR(255); + DECLARE _stmt VARCHAR(1000); + DECLARE cur1 CURSOR FOR + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND (TABLE_NAME LIKE "collections\_%" OR TABLE_NAME IN ("collections","multigroup_collections")); + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + OPEN cur1; + myloop: LOOP FETCH cur1 + INTO _table; + IF done + THEN LEAVE myloop; END IF; + SET @sql1 := CONCAT("ALTER TABLE ", _table, + " ADD COLUMN collection_regexes_id INT SIGNED NOT NULL DEFAULT '0'", + " COMMENT 'FK to collection_regexes.id' AFTER collectionhash" + ); + PREPARE _stmt FROM @sql1; + EXECUTE _stmt; + DROP PREPARE _stmt; + END LOOP; + CLOSE cur1; + END $$ +DELIMITER ; + +CALL change_collections(); +DROP PROCEDURE change_collections;