diff --git a/nntmux/Binaries.php b/nntmux/Binaries.php index 43148b351..ca451b16d 100755 --- a/nntmux/Binaries.php +++ b/nntmux/Binaries.php @@ -516,6 +516,7 @@ class Binaries { // Start time of scan method and of fetching headers. $startLoop = microtime(true); + $multiGroup = false; // Check if MySQL tables exist, create if they do not, get their names at the same time. $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupMySQL['id']); @@ -610,11 +611,11 @@ class Binaries } } - $headersRepaired = $articles = $rangeNotReceived = $collectionIDs = $binariesUpdate = $headersReceived = $headersNotInserted = []; + $headersRepaired = $mgrHeadersRepaired = $articles = $rangeNotReceived = $mgrRangeNotReceived = $collectionIDs = $binariesUpdate = $headersReceived = $headersNotInserted = $mgrHeadersReceived = $mgrHeadersNotInserted = []; $notYEnc = $headersBlackListed = 0; $partsQuery = $partsCheck = sprintf('INSERT IGNORE INTO %s (binaryid, number, messageid, partnumber, size) VALUES ', $tableNames['pname']); - $mgrPartsQuery = $mgrPartsCheck = sprintf('INSERT IGNORE INTO %s (binaryid, number, messageid, partnumber, size) VALUES ', $tableNames['mgrpname']); + $mgrPartsQuery = $mgrPartsCheck = sprintf('INSERT IGNORE INTO mgr_parts (binaryid, number, messageid, partnumber, size) VALUES '); $this->_pdo->beginTransaction(); // Loop articles, figure out files/parts. @@ -624,10 +625,18 @@ class Binaries // Check if we got the article or not. if (isset($header['Number'])) { - $headersReceived[] = $header['Number']; + if($multiGroup === true) { + $mgrHeadersReceived[] = $header['Number']; + } else { + $headersReceived[] = $header['Number']; + } } else { if ($addToPartRepair) { - $rangeNotReceived[] = $header['Number']; + if ($multiGroup === true) { + $mgrRangeNotReceived[] = $header['Number']; + } else { + $rangeNotReceived[] = $header['Number']; + } } continue; } @@ -639,7 +648,11 @@ class Binaries continue; } else { // We got the part this time. Remove article from part repair. - $headersRepaired[] = $header['Number']; + if ($multiGroup === true) { + $mgrHeadersRepaired[] = $header['Number']; + } else { + $headersRepaired[] = $header['Number']; + } } } @@ -737,7 +750,7 @@ class Binaries totalfiles, collectionhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', NOW()) ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", - ($multiGroup === true ? $tableNames['mgrcname'] : $tableNames['cname']), + ($multiGroup === true ? 'mgr_collections' : $tableNames['cname']), $this->_pdo->escapeString(substr(utf8_encode($matches[1]), 0, 255)), $this->_pdo->escapeString(utf8_encode($header['From'])), (is_numeric($header['Date']) ? ($header['Date'] > $now ? $now : $header['Date']) : $now), @@ -768,7 +781,7 @@ class Binaries INSERT INTO %s (binaryhash, name, collection_id, totalparts, currentparts, filenumber, partsize) VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", - ($multiGroup === true ? $tableNames['mgrbname'] : $tableNames['bname']), + ($multiGroup === true ? 'mgr_binaries' : $tableNames['bname']), ($multiGroup === true ? md5($matches[1] . $header['From']) : md5($matches[1] . $header['From'] . $groupMySQL['id'])), $this->_pdo->escapeString(utf8_encode($matches[1])), $collectionID, @@ -824,7 +837,8 @@ class Binaries // End of processing headers. $timeCleaning = number_format($startUpdate - $startCleaning, 2); - $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', ($multiGroup === true ? $tableNames['mgrbname'] : $tableNames['bname'])); + $bTable = ($multiGroup === true ? 'mgr_binaries' : $tableNames['bname']); + $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', $bTable); foreach ($binariesUpdate as $binaryID => $binary) { $binariesQuery .= '(' . $binaryID . ',' . $binary['Size'] . ',' . $binary['Parts'] . '),'; } @@ -841,26 +855,15 @@ class Binaries ); } - if (((strlen($partsQuery) === strlen($partsCheck)) ? true : $this->_pdo->queryExec(rtrim($partsQuery, ',')))) { + if (((strlen($partsQuery) === strlen($partsCheck)) ? true : $this->_pdo->queryExec(rtrim($partsQuery, ','))) || ((strlen($mgrPartsQuery) === strlen($mgrPartsCheck)) ? true : $this->_pdo->queryExec(rtrim($mgrPartsQuery, ',')))) { $this->_pdo->Commit(); } else { if ($addToPartRepair) { $headersNotInserted += $headersReceived; + $mgrHeadersNotInserted += $mgrHeadersReceived; } $this->_pdo->Rollback(); } - - if ($multiGroup === true) { - $this->_pdo->beginTransaction(); - if (((strlen($mgrPartsQuery) === strlen($mgrPartsCheck)) ? true : $this->_pdo->queryExec(rtrim($mgrPartsQuery, ',')))) { - $this->_pdo->Commit(); - } else { - if ($addToPartRepair) { - $headersNotInserted += $headersReceived; - } - $this->_pdo->Rollback(); - } - } } else { if ($addToPartRepair) { $headersNotInserted += $headersReceived; @@ -896,6 +899,9 @@ class Binaries if ($partRepair && count($headersRepaired) > 0) { $this->removeRepairedParts($headersRepaired, $tableNames['prname'], $groupMySQL['id']); } + if ($partRepair && count($mgrHeadersRepaired) > 0) { + $this->removeRepairedMGRParts($mgrHeadersRepaired); + } if ($addToPartRepair) { @@ -911,6 +917,20 @@ class Binaries ); } + if ($multiGroup === true) { + $notInsertedCount = count($mgrHeadersNotInserted); + if ($notInsertedCount > 0) { + $this->addMissingMGRParts($mgrHeadersNotInserted); + + $this->log( + $notInsertedCount . ' articles failed to insert!', + __FUNCTION__, + Logger::LOG_WARNING, + 'warning' + ); + } + } + // Check if we have any missing headers. if (($last - $first - $notYEnc - $headersBlackListed + 1) > count($headersReceived)) { $rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($first, $last), $headersReceived)); @@ -929,6 +949,23 @@ class Binaries ); } } + + if ($multiGroup === true) { + $notReceivedCount = count($mgrRangeNotReceived); + + if ($notReceivedCount > 0) { + $this->addMissingMGRParts($mgrRangeNotReceived); + + if ($this->_echoCLI) { + $this->_colorCLI->doEcho( + $this->_colorCLI->alternate( + 'Server did not return ' . $notReceivedCount . + ' articles from ' . $groupMySQL['name'] . '.' + ), true + ); + } + } + } } $currentMicroTime = microtime(true); @@ -1334,6 +1371,22 @@ class Binaries return $this->_pdo->queryInsert((rtrim($insertStr, ',') . ' ON DUPLICATE KEY UPDATE attempts=attempts+1')); } + /** + * Add article numbers from missing headers to DB. + * + * @param array $numbers The article numbers of the missing headers. + * + * @return bool + */ + private function addMissingMGRParts($numbers) + { + $insertStr = 'INSERT INTO mgr_missed_parts (numberid) VALUES '; + foreach ($numbers as $number) { + $insertStr .= '(' . $number . '),'; + } + return $this->_pdo->queryInsert((rtrim($insertStr, ',') . ' ON DUPLICATE KEY UPDATE attempts=attempts+1')); + } + /** * Clean up part repair table. * @@ -1352,6 +1405,22 @@ class Binaries $this->_pdo->queryExec((rtrim($sql, ',') . ') AND group_id = ' . $groupID)); } + /** + * Clean up part repair table. + * + * @param array $numbers The article numbers. + * + * @return void + */ + private function removeRepairedMGRParts($numbers) + { + $sql = 'DELETE FROM mgr_missed_parts WHERE numberid in ('; + foreach ($numbers as $number) { + $sql .= $number . ','; + } + $this->_pdo->queryExec((rtrim($sql, ',') . ')')); + } + /** * Are white or black lists loaded for a group name? * @var array diff --git a/nntmux/Groups.php b/nntmux/Groups.php index 06a2975c4..bfc505cd6 100755 --- a/nntmux/Groups.php +++ b/nntmux/Groups.php @@ -586,11 +586,9 @@ class Groups * @param bool $tpgSetting false, tpg is off in site setting, true tpg is on in site setting. * @param int $groupID ID of the group. * - * @param bool $multiGroup - * * @return array The table names. */ - public function getCBPTableNames($tpgSetting, $groupID, $multiGroup = false) + public function getCBPTableNames($tpgSetting, $groupID) { $groupKey = ($groupID . '_' . (int)$tpgSetting); @@ -604,17 +602,13 @@ class Groups $tables['bname'] = 'binaries'; $tables['pname'] = 'parts'; $tables['prname'] = 'missed_parts'; - $tables['mgrcname'] = 'mgr_collections'; - $tables['mgrbname'] = 'mgr_binaries'; - $tables['mgrpname'] = 'mgr_parts'; - $tables['mgrprname'] = 'mgr_missed_parts'; if ($tpgSetting === true) { - if ($multiGroup === false && $groupID == '') { + if ($groupID == '') { exit('Error: You must use .../misc/update/nix/multiprocessing/releases.php since you have enabled TPG!'); } - if ($multiGroup === false && $this->createNewTPGTables($groupID) === false && NN_ECHOCLI) { + if ($this->createNewTPGTables($groupID) === false && NN_ECHOCLI) { exit('There is a problem creating new TPG tables for this group ID: ' . $groupID . PHP_EOL); } @@ -623,10 +617,6 @@ class Groups $tables['bname'] .= $groupEnding; $tables['pname'] .= $groupEnding; $tables['prname'] .= $groupEnding; - $tables['mgrcname']; - $tables['mgrbname']; - $tables['mgrpname']; - $tables['mgrprname']; } // Buffer. diff --git a/nntmux/NZBMultiGroup.php b/nntmux/NZBMultiGroup.php index c60c40d94..6622ddb35 100644 --- a/nntmux/NZBMultiGroup.php +++ b/nntmux/NZBMultiGroup.php @@ -61,27 +61,20 @@ class NZBMultiGroup public function initiateForMgrWrite() { - // Set table names - $this->_tableNames = [ - 'mgrcName' => 'mgr_collections', - 'mgrbName' => 'mgr_binaries', - 'mgrpName' => 'mgr_parts' - ]; - $this->_collectionsQuery = " SELECT c.*, UNIX_TIMESTAMP(c.date) AS udate, g.name AS groupname - FROM {$this->_tableNames['mgrcName']} c + FROM mgr_collections c INNER JOIN groups g ON c.group_id = g.id WHERE c.releaseid = "; $this->_binariesQuery = " SELECT b.id, b.name, b.totalparts - FROM {$this->_tableNames['mgrbName']} b + FROM mgr_binaries b WHERE b.collection_id = %d ORDER BY b.name ASC"; $this->_partsQuery = " SELECT DISTINCT(p.messageid), p.size, p.partnumber - FROM {$this->_tableNames['mgrpName']} p + FROM mgr_parts p WHERE p.binaryid = %d ORDER BY p.partnumber ASC"; @@ -204,8 +197,8 @@ class NZBMultiGroup // Delete CBP for release that has its NZB created. $this->pdo->queryExec( sprintf(' - DELETE c, b, p FROM %s c JOIN %s b ON(c.id=b.collection_id) STRAIGHT_JOIN %s p ON(b.id=p.binaryid) WHERE c.releaseid = %d', - $this->_tableNames['mgrcName'], $this->_tableNames['mgrbName'], $this->_tableNames['mgrpName'], $relID + DELETE c, b, p FROM mgr_collections c JOIN mgr_binaries b ON(c.id=b.collection_id) STRAIGHT_JOIN mgr_parts p ON(b.id=p.binaryid) WHERE c.releaseid = %d', + $relID ) ); // Chmod to fix issues some users have with file permissions. diff --git a/nntmux/ReleasesMultiGroup.php b/nntmux/ReleasesMultiGroup.php index a1eefd53f..0f2751bc9 100644 --- a/nntmux/ReleasesMultiGroup.php +++ b/nntmux/ReleasesMultiGroup.php @@ -107,7 +107,6 @@ class ReleasesMultiGroup public function createMGRReleases() { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, '', true); $categorize = new Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; @@ -120,15 +119,11 @@ class ReleasesMultiGroup $collections = $this->pdo->queryDirect( sprintf(' - SELECT SQL_NO_CACHE %s.*, groups.name AS gname - FROM %s - INNER JOIN groups ON %s.group_id = groups.id - WHERE %s.filecheck = %d + SELECT SQL_NO_CACHE mgr_collections.*, groups.name AS gname + FROM mgr_collections + INNER JOIN groups ON mgr_collections.group_id = groups.id + WHERE mgr_collections.filecheck = %d AND filesize > 0 LIMIT %d', - $group['mgrcname'], - $group['mgrcname'], - $group['mgrcname'], - $group['mgrcname'], ProcessReleases::COLLFC_SIZED, $this->releaseCreationLimit ) @@ -217,10 +212,9 @@ class ReleasesMultiGroup // Update collections table to say we inserted the release. $this->pdo->queryExec( sprintf(' - UPDATE %s + UPDATE mgr_collections SET filecheck = %d, releaseid = %d WHERE id = %d', - $group['mgrcname'], ProcessReleases::COLLFC_INSERTED, $releaseID, $collection['id'] @@ -271,11 +265,10 @@ class ReleasesMultiGroup // The release was already in the DB, so delete the collection. $this->pdo->queryExec( sprintf(' - DELETE c, b, p FROM %s c - INNER JOIN %s b ON(c.id=b.collection_id) - STRAIGHT_JOIN %s p ON(b.id=p.binaryid) + DELETE c, b, p FROM mgr_collections c + INNER JOIN mgr_binaries b ON(c.id=b.collection_id) + STRAIGHT_JOIN mgr_missed_parts p ON(b.id=p.binaryid) WHERE c.collectionhash = %s', - $group['mgrcname'], $group['mgrbname'], $group['mgrpname'], $this->pdo->escapeString($collection['collectionhash']) ) ); @@ -332,7 +325,7 @@ class ReleasesMultiGroup if ($this->mgrnzb->writeMgrNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { $nzbCount++; if ($this->echoCLI) { - echo $this->pdo->log->primaryOver("Creating NZBs and deleting Collections:\t" . $nzbCount . '/' . $total . "\r"); + echo $this->pdo->log->primaryOver("Creating NZBs and deleting MGR Collections:\t" . $nzbCount . '/' . $total . "\r"); } } } diff --git a/nntmux/processing/ProcessReleases.php b/nntmux/processing/ProcessReleases.php index ac109e1fd..177d60acc 100755 --- a/nntmux/processing/ProcessReleases.php +++ b/nntmux/processing/ProcessReleases.php @@ -191,9 +191,11 @@ class ProcessReleases $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); - $totalReleasesAdded += $releasesCount['added']; + $mgrReleasesCount = (new ReleasesMultiGroup())->createMGRReleases(); + $totalReleasesAdded += $releasesCount['added'] += $mgrReleasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); + $mgrFilesAdded = (new ReleasesMultiGroup())->createMGRNZBs(); $this->deleteCollections($groupID); if ($this->processRequestIDs === 0) { $this->processRequestIDs($groupID, 5000, true); @@ -220,7 +222,7 @@ class ProcessReleases $this->postProcessReleases($postProcess, $nntp); // This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately - } while (($releasesCount['added'] + $releasesCount['dupes']) >= $this->releaseCreationLimit || $nzbFilesAdded >= $this->releaseCreationLimit); + } while (($releasesCount['added'] + $releasesCount['dupes']) + $mgrReleasesCount['added'] + $mgrReleasesCount['dupes'] >= $this->releaseCreationLimit || $nzbFilesAdded + $mgrFilesAdded >= $this->releaseCreationLimit); @@ -385,13 +387,11 @@ class ProcessReleases // Get the total size in bytes of the mgr collection for mgr collections where filecheck = 2. $mgrChecked = $this->pdo->queryExec( sprintf( - 'UPDATE %s c - SET filesize = (SELECT COALESCE(SUM(b.partsize), 0) FROM %s b WHERE b.collection_id = c.id), + 'UPDATE mgr_collections c + SET filesize = (SELECT COALESCE(SUM(b.partsize), 0) FROM mgr_binaries b WHERE b.collection_id = c.id), filecheck = %d WHERE c.filecheck = %d AND c.filesize = 0', - $group['mgrcname'], - $group['mgrbname'], self::COLLFC_SIZED, self::COLLFC_COMPPART ) @@ -570,8 +570,6 @@ class ProcessReleases $this->pdo->ping(true); - (new ReleasesMultiGroup())->createMGRReleases(); - $collections = $this->pdo->queryDirect( sprintf(' SELECT SQL_NO_CACHE %s.*, groups.name AS gname @@ -771,8 +769,6 @@ class ProcessReleases $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create the NZB, delete collections/binaries/parts.")); } - (new ReleasesMultiGroup())->createMGRNZBs(); - $releases = $this->pdo->queryDirect( sprintf(" SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, @@ -1541,17 +1537,14 @@ class ProcessReleases $this->pdo->queryExec( sprintf(' - UPDATE %s c INNER JOIN - (SELECT c.id FROM %s c - INNER JOIN %s b ON b.collection_id = c.id + UPDATE mgr_collections c INNER JOIN + (SELECT c.id FROM mgr_collections c + INNER JOIN mgr_binaries b ON b.collection_id = c.id WHERE c.totalfiles > 0 AND c.filecheck = %d GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) IN (c.totalfiles, c.totalfiles + 1) ) r ON c.id = r.id SET filecheck = %d', - $group['mgrcname'], - $group['mgrcname'], - $group['mgrbname'], self::COLLFC_DEFAULT, self::COLLFC_COMPCOLL ) @@ -1607,28 +1600,24 @@ class ProcessReleases $this->pdo->queryExec( sprintf(' - UPDATE %s c INNER JOIN - (SELECT c.id FROM %s c - INNER JOIN %s b ON b.collection_id = c.id + UPDATE mgr_collections c INNER JOIN + (SELECT c.id FROM mgr_collections c + INNER JOIN mgr_binaries b ON b.collection_id = c.id WHERE b.filenumber = 0 AND c.totalfiles > 0 AND c.filecheck = %d GROUP BY c.id ) r ON c.id = r.id SET c.filecheck = %d', - $group['mgrcname'], - $group['mgrcname'], - $group['mgrbname'], self::COLLFC_COMPCOLL, self::COLLFC_ZEROPART ) ); $this->pdo->queryExec( sprintf(' - UPDATE %s c + UPDATE mgr_collections c SET filecheck = %d WHERE filecheck = %d', - $group['mgrcname'], self::COLLFC_TEMPCOMP, self::COLLFC_COMPCOLL ) @@ -1686,16 +1675,13 @@ class ProcessReleases $this->pdo->queryExec( sprintf(' - UPDATE %s b INNER JOIN - (SELECT b.id FROM %s b - INNER JOIN %s c ON c.id = b.collection_id + UPDATE mgr_binaries b INNER JOIN + (SELECT b.id FROM mgr_binaries b + INNER JOIN mgr_collections c ON c.id = b.collection_id WHERE c.filecheck = %d AND b.partcheck = %d AND b.currentparts = b.totalparts GROUP BY b.id, b.totalparts) r ON b.id = r.id SET b.partcheck = %d', - $group['mgrbname'], - $group['mgrbname'], - $group['mgrcname'], self::COLLFC_TEMPCOMP, self::FILE_INCOMPLETE, self::FILE_COMPLETE @@ -1703,16 +1689,13 @@ class ProcessReleases ); $this->pdo->queryExec( sprintf(' - UPDATE %s b INNER JOIN - (SELECT b.id FROM %s b - INNER JOIN %s c ON c.id = b.collection_id + UPDATE mgr_binaries b INNER JOIN + (SELECT b.id FROM mgr_binaries b + INNER JOIN mgr_collections c ON c.id = b.collection_id WHERE c.filecheck = %d AND b.partcheck = %d AND b.currentparts >= (b.totalparts + 1) GROUP BY b.id, b.totalparts) r ON b.id = r.id SET b.partcheck = %d', - $group['mgrbname'], - $group['mgrbname'], - $group['mgrcname'], self::COLLFC_ZEROPART, self::FILE_INCOMPLETE, self::FILE_COMPLETE @@ -1753,15 +1736,12 @@ class ProcessReleases $this->pdo->queryExec( sprintf(' - UPDATE %s c INNER JOIN - (SELECT c.id FROM %s c - INNER JOIN %s b ON c.id = b.collection_id + UPDATE mgr_collections c INNER JOIN + (SELECT c.id FROM mgr_collections c + INNER JOIN mgr_binaries b ON c.id = b.collection_id WHERE b.partcheck = 1 AND c.filecheck IN (%d, %d) GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) >= c.totalfiles) r ON c.id = r.id SET filecheck = %d', - $group['mgrcname'], - $group['mgrcname'], - $group['mgrbname'], self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART, self::COLLFC_COMPPART @@ -1796,10 +1776,9 @@ class ProcessReleases $this->pdo->queryExec( sprintf(' - UPDATE %s c + UPDATE mgr_collections c SET filecheck = %d WHERE filecheck IN (%d, %d)', - $group['mgrcname'], self::COLLFC_COMPCOLL, self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART @@ -1836,12 +1815,10 @@ class ProcessReleases $this->pdo->queryExec( sprintf(" - UPDATE %s c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM %s b WHERE b.collection_id = c.id) + UPDATE mgr_collections c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM mgr_binaries b WHERE b.collection_id = c.id) WHERE c.dateadded < NOW() - INTERVAL '%d' HOUR AND c.filecheck IN (%d, %d, 10)", - $group['mgrcname'], self::COLLFC_COMPPART, - $group['mgrbname'], $this->collectionDelayTime, self::COLLFC_DEFAULT, self::COLLFC_COMPCOLL