From 149f8ea5c2075cfd29c2d324d66ebf1850364a9f Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 29 Dec 2016 16:01:04 +0100 Subject: [PATCH] Update adding/checking mgr releases --- nntmux/Binaries.php | 18 ++-- nntmux/ReleasesMultiGroup.php | 147 ++------------------------ nntmux/processing/ProcessReleases.php | 2 - 3 files changed, 18 insertions(+), 149 deletions(-) diff --git a/nntmux/Binaries.php b/nntmux/Binaries.php index 72dc3ee86..bcf7eae53 100755 --- a/nntmux/Binaries.php +++ b/nntmux/Binaries.php @@ -516,7 +516,6 @@ class Binaries { // Start time of scan method and of fetching headers. $startLoop = microtime(true); - $multiGroup = false; // Check if MySQL tables exist, create if they do not, get their names at the same time. $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupMySQL['id']); @@ -704,7 +703,7 @@ class Binaries } // Used to group articles together when forming the release/nzb. - if ($multiGroup) { + if ($multiGroup === true) { $header['CollectionKey'] = ( $this->_collectionsCleaning->collectionsCleaner($matches[1], '') . $header['From'] . @@ -736,8 +735,8 @@ class Binaries INSERT INTO %s (subject, fromname, date, xref, group_id, totalfiles, collectionhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', NOW()) - ON DUPLICATE KEY UPDATE dateadded = NOW(), noise = '%s'", - ($multiGroup ? $tableNames['mgrcname'] : $tableNames['cname']), + ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", + ($multiGroup === true ? $tableNames['mgrcname'] : $tableNames['cname']), $this->_pdo->escapeString(substr(utf8_encode($matches[1]), 0, 255)), $this->_pdo->escapeString(utf8_encode($header['From'])), (is_numeric($header['Date']) ? ($header['Date'] > $now ? $now : $header['Date']) : $now), @@ -745,6 +744,7 @@ class Binaries $groupMySQL['id'], $fileCount[3], sha1($header['CollectionKey']), + ($multiGroup === true ? sprintf("xref = CONCAT(xref, \\n%s ),", $this->_pdo->escapeString(substr($header['Xref'], 2, 255))) : ''), bin2hex(openssl_random_pseudo_bytes(16)) ) ); @@ -767,8 +767,8 @@ class Binaries INSERT INTO %s (binaryhash, name, collection_id, totalparts, currentparts, filenumber, partsize) VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", - ($multiGroup ? $tableNames['mgrbname'] : $tableNames['bname']), - ($multiGroup ? md5($matches[1] . $header['From']) : md5($matches[1] . $header['From'] . $groupMySQL['id'])), + ($multiGroup === true ? $tableNames['mgrbname'] : $tableNames['bname']), + ($multiGroup === true ? md5($matches[1] . $header['From']) : md5($matches[1] . $header['From'] . $groupMySQL['id'])), $this->_pdo->escapeString(utf8_encode($matches[1])), $collectionID, $matches[3], @@ -807,7 +807,7 @@ class Binaries '(' . $binaryID . ',' . $header['Number'] . ',' . rtrim($header['Message-ID'], '>') . "'," . $matches[2] . ',' . $header['Bytes'] . '),'; - if ($multiGroup) { + if ($multiGroup === true) { $mgrPartsQuery .= '(' . $binaryID . ',' . $header['Number'] . ',' . rtrim($header['Message-ID'], '>') . "'," . $matches[2] . ',' . $header['Bytes'] . '),'; @@ -823,7 +823,7 @@ class Binaries // End of processing headers. $timeCleaning = number_format($startUpdate - $startCleaning, 2); - $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', ($multiGroup ? $tableNames['mgrbname'] : $tableNames['bname'])); + $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', ($multiGroup === true ? $tableNames['mgrbname'] : $tableNames['bname'])); foreach ($binariesUpdate as $binaryID => $binary) { $binariesQuery .= '(' . $binaryID . ',' . $binary['Size'] . ',' . $binary['Parts'] . '),'; } @@ -849,7 +849,7 @@ class Binaries $this->_pdo->Rollback(); } - if ($multiGroup) { + if ($multiGroup === true) { $this->_pdo->beginTransaction(); if (((strlen($mgrPartsQuery) === strlen($mgrPartsCheck)) ? true : $this->_pdo->queryExec(rtrim($mgrPartsQuery, ',')))) { $this->_pdo->Commit(); diff --git a/nntmux/ReleasesMultiGroup.php b/nntmux/ReleasesMultiGroup.php index e234f39e7..007e02bd5 100644 --- a/nntmux/ReleasesMultiGroup.php +++ b/nntmux/ReleasesMultiGroup.php @@ -4,7 +4,6 @@ namespace nntmux; use app\models\ReleasesGroups; use app\models\Settings; -use nntmux\db\DB; use nntmux\processing\ProcessReleases; @@ -41,6 +40,11 @@ class ReleasesMultiGroup extends ProcessReleases */ protected $mgrFromNames; + /** + * @var NZBMultiGroup + */ + protected $mgrnzb; + /** * ReleasesMultiGroup constructor. @@ -50,7 +54,9 @@ class ReleasesMultiGroup extends ProcessReleases public function __construct(array $options = []) { parent::__construct(); + $this->mgrFromNames = implode(",", self::$mgrPosterNames); + $this->mgrnzb = new NZBMultiGroup(); } /** @@ -292,10 +298,10 @@ class ReleasesMultiGroup extends ProcessReleases if ($releases && $releases->rowCount()) { $total = $releases->rowCount(); // Init vars for writing the NZB's. - $this->nzb->initiateForMgrWrite(); + $this->mgrnzb->initiateForMgrWrite(); foreach ($releases as $release) { - if ($this->nzb->writeMgrNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { + if ($this->mgrnzb->writeMgrNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { $nzbCount++; if ($this->echoCLI) { echo $this->pdo->log->primaryOver("Creating NZBs and deleting Collections:\t" . $nzbCount . '/' . $total . "\r"); @@ -307,141 +313,6 @@ class ReleasesMultiGroup extends ProcessReleases return $nzbCount; } - /** - * Delete unwanted collections based on size/file count using admin settings. - * - * - * @void - * @access public - */ - public function deleteUnwantedMGRCollections() - { - $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, '', true); - - if ($this->echoCLI) { - $this->pdo->log->doEcho( - $this->pdo->log->header( - "Process Releases -> Delete collections smaller/larger than minimum size/file count from group/site setting." - ) - ); - } - - - $groupIDs = $this->groups->getActiveIDs(); - - - $minSizeDeleted = $maxSizeDeleted = $minFilesDeleted = 0; - - $maxSizeSetting = Settings::value('.release.maxsizetoformrelease'); - $minSizeSetting = Settings::value('.release.minsizetoformrelease'); - $minFilesSetting = Settings::value('.release.minfilestoformrelease'); - - foreach ($groupIDs as $groupID) { - - $groupMinSizeSetting = $groupMinFilesSetting = 0; - - $groupMinimums = $this->groups->getByID($groupID['id']); - if ($groupMinimums !== false) { - if (!empty($groupMinimums['minsizetoformrelease']) && $groupMinimums['minsizetoformrelease'] > 0) { - $groupMinSizeSetting = (int)$groupMinimums['minsizetoformrelease']; - } - if (!empty($groupMinimums['minfilestoformrelease']) && $groupMinimums['minfilestoformrelease'] > 0) { - $groupMinFilesSetting = (int)$groupMinimums['minfilestoformrelease']; - } - } - - if ($this->pdo->queryOneRow( - sprintf( - 'SELECT SQL_NO_CACHE id FROM %s c WHERE c.filecheck = %d AND c.filesize > 0 LIMIT 1', - $group['mgrcname'], - ProcessReleases::COLLFC_SIZED - ) - ) !== false - ) { - - $deleteQuery = $this->pdo->queryExec( - sprintf(' - DELETE c, b, p - FROM %s c - LEFT JOIN %s b ON c.id = b.collection_id - LEFT JOIN %s p ON b.id = p.binaryid - WHERE c.filecheck = %d - AND c.filesize > 0 - AND GREATEST(%d, %d) > 0 - AND c.filesize < GREATEST(%d, %d)', - $group['mgrcname'], - $group['mgrbname'], - $group['mgrpname'], - ProcessReleases::COLLFC_SIZED, - $groupMinSizeSetting, - $minSizeSetting, - $groupMinSizeSetting, - $minSizeSetting - ) - ); - if ($deleteQuery !== false) { - $minSizeDeleted += $deleteQuery->rowCount(); - } - - - if ($maxSizeSetting > 0) { - $deleteQuery = $this->pdo->queryExec( - sprintf(' - DELETE c, b, p FROM %s c - LEFT JOIN %s b ON c.id = b.collection_id - LEFT JOIN %s p ON b.id = p.binaryid - WHERE c.filecheck = %d - AND c.filesize > %d', - $group['mgrcname'], - $group['mgrbname'], - $group['mgrpname'], - ProcessReleases::COLLFC_SIZED, - $maxSizeSetting - ) - ); - if ($deleteQuery !== false) { - $maxSizeDeleted += $deleteQuery->rowCount(); - } - } - - $deleteQuery = $this->pdo->queryExec( - sprintf(' - DELETE c, b, p FROM %s c - LEFT JOIN %s b ON (c.id=b.collection_id) - LEFT JOIN %s p ON (b.id=p.binaryid) - WHERE c.filecheck = %d - AND GREATEST(%d, %d) > 0 - AND c.totalfiles < GREATEST(%d, %d)', - $group['mgrcname'], - $group['mgrbname'], - $group['mgrpname'], - ProcessReleases::COLLFC_SIZED, - $groupMinFilesSetting, - $minFilesSetting, - $groupMinFilesSetting, - $minFilesSetting - ) - ); - if ($deleteQuery !== false) { - $minFilesDeleted += $deleteQuery->rowCount(); - } - } - } - - if ($this->echoCLI) { - $this->pdo->log->doEcho( - $this->pdo->log->primary( - 'Deleted ' . ($minSizeDeleted + $maxSizeDeleted + $minFilesDeleted) . ' MGR collections: ' . PHP_EOL . - $minSizeDeleted . ' smaller than, ' . - $maxSizeDeleted . ' bigger than, ' . - $minFilesDeleted . ' with less files than site/group settings in: ' . - $this->consoleTools->convertTime(time() - $startTime) - ), true - ); - } - } - /** * Delete MGR collections (complete/incomplete/old/etc). * diff --git a/nntmux/processing/ProcessReleases.php b/nntmux/processing/ProcessReleases.php index bb15ed692..39a9e0a15 100755 --- a/nntmux/processing/ProcessReleases.php +++ b/nntmux/processing/ProcessReleases.php @@ -427,8 +427,6 @@ class ProcessReleases ); } - (new ReleasesMultiGroup())->deleteUnwantedMGRCollections(); - if ($groupID == '') { $groupIDs = $this->groups->getActiveIDs(); } else {