Improve handling of MultiGroup releases

This commit is contained in:
DariusIII
2017-01-11 10:49:07 +01:00
parent 1145efa574
commit 9bd9f31571
6 changed files with 174 additions and 159 deletions
+112 -138
View File
@@ -12,12 +12,12 @@ use nntmux\ReleaseImage;
use nntmux\NZB;
use nntmux\Categorize;
use nntmux\Category;
use nntmux\processing\ProcessMultiGroupReleases;
use nntmux\RequestIDLocal;
use nntmux\RequestIDWeb;
use nntmux\PreDb;
use nntmux\Genres;
use nntmux\NNTP;
use nntmux\processing\ProcessReleasesMultiGroup;
use nntmux\utility\Utility;
class ProcessReleases
@@ -135,6 +135,7 @@ class ProcessReleases
$this->releaseCleaning = ($options['ReleaseCleaning'] instanceof ReleaseCleaning ? $options['ReleaseCleaning'] : new ReleaseCleaning($this->pdo));
$this->releases = ($options['Releases'] instanceof Releases ? $options['Releases'] : new Releases(['Settings' => $this->pdo, 'Groups' => $this->groups]));
$this->releaseImage = ($options['ReleaseImage'] instanceof ReleaseImage ? $options['ReleaseImage'] : new ReleaseImage($this->pdo));
$this->mgr = new ProcessReleasesMultiGroup(['Settings' => $this->pdo]);
$this->tablePerGroup = (Settings::value('..tablepergroup') == 0 ? false : true);
$this->collectionDelayTime = (Settings::value('..delaytime') != '' ? (int)Settings::value('..delaytime') : 2);
@@ -186,18 +187,21 @@ class ProcessReleases
$this->processIncompleteCollections($groupID);
$this->processCollectionSizes($groupID);
$this->deleteUnwantedCollections($groupID);
$this->deleteUnwantedCollections($groupID, true);
$this->mgr->processIncompleteMgrCollections($groupID);
$this->mgr->processMgrCollectionSizes($groupID);
$this->mgr->deleteUnwantedMgrCollections($groupID);
$DIR = NN_MISC;
$totalReleasesAdded = 0;
do {
$releasesCount = $this->createReleases($groupID);
$mgrReleasesCount = (new ProcessMultiGroupReleases())->createMGRReleases($groupID);
$mgrReleasesCount = $this->mgr->createMGRReleases($groupID);
$totalReleasesAdded += $releasesCount['added'] += $mgrReleasesCount['added'];
$nzbFilesAdded = $this->createNZBs($groupID);
$mgrFilesAdded = (new ProcessMultiGroupReleases())->createMGRNZBs($groupID);
$mgrFilesAdded = $this->mgr->createMGRNZBs($groupID);
$this->deleteCollections($groupID);
if ($this->processRequestIDs === 0) {
$this->processRequestIDs($groupID, 5000, true);
@@ -300,17 +304,24 @@ class ProcessReleases
return $categorized;
}
public function processIncompleteCollections($groupID)
{
$tableNames = $this->initiateTableNames($groupID);
$this->processIncompleteCollectionsMain($groupID, $tableNames);
}
/**
* Find complete collections to be processed by processCollectionSizes.
*
* @param int $groupID
* @param $tableNames
*
* @void
* @access public
*/
public function processIncompleteCollections($groupID)
public function processIncompleteCollectionsMain($groupID, $tableNames)
{
$startTime = time();
$group = $this->initiateTableNames($groupID);
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Attempting to find complete collections."));
@@ -318,27 +329,19 @@ class ProcessReleases
$where = (!empty($groupID) ? ' AND c.group_id = ' . $groupID . ' ' : ' ');
$this->processStuckCollections($group, $where);
$this->collectionFileCheckStage1($group, $where);
$this->collectionFileCheckStage2($group, $where);
$this->collectionFileCheckStage3($group, $where);
$this->collectionFileCheckStage4($group, $where);
$this->collectionFileCheckStage5($group, $where);
$this->collectionFileCheckStage6($group, $where);
$this->processStuckCollections($group, $where, true);
$this->collectionFileCheckStage1($group, $where, true);
$this->collectionFileCheckStage2($group, $where, true);
$this->collectionFileCheckStage3($group, $where, true);
$this->collectionFileCheckStage4($group, $where, true);
$this->collectionFileCheckStage5($group, $where, true);
$this->collectionFileCheckStage6($group, $where, true);
$this->processStuckCollections($tableNames, $where);
$this->collectionFileCheckStage1($tableNames, $where);
$this->collectionFileCheckStage2($tableNames, $where);
$this->collectionFileCheckStage3($tableNames, $where);
$this->collectionFileCheckStage4($tableNames, $where);
$this->collectionFileCheckStage5($tableNames, $where);
$this->collectionFileCheckStage6($tableNames, $where);
if ($this->echoCLI) {
$count = $this->pdo->queryOneRow(
sprintf(
'SELECT COUNT(id) AS complete FROM %s c WHERE filecheck = %d %s',
$group['cname'],
$tableNames['cname'],
self::COLLFC_COMPPART,
$where
)
@@ -352,23 +355,26 @@ class ProcessReleases
}
}
public function processCollectionSizes($groupID)
{
$tableNames = $this->initiateTableNames($groupID);
$this->processCollectionSizesMain($groupID, $tableNames);
}
/**
* Calculate the total size of a complete collection (COLLFC_COMPPART) in bytes.
* Set the collection to (COLLFC_SIZED)
*
* @param string|int $groupID (optional)
*
* @param bool $multiGroup
* @param $tableNames
*
* @void
* @access public
*/
public function processCollectionSizes($groupID, $multiGroup = false)
public function processCollectionSizesMain($groupID, $tableNames)
{
$startTime = time();
$group = $this->initiateTableNames($groupID);
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Calculating collection sizes (in bytes)."));
@@ -382,8 +388,8 @@ class ProcessReleases
filecheck = %d
WHERE c.filecheck = %d
AND c.filesize = 0 %s',
$tablec,
$tableb,
$tableNames['cname'],
$tableNames['bname'],
self::COLLFC_SIZED,
self::COLLFC_COMPPART,
(!empty($groupID) ? ' AND c.group_id = ' . $groupID : '')
@@ -399,23 +405,25 @@ class ProcessReleases
}
}
public function deleteUnwantedCollections($groupID)
{
$tableNames = $this->initiateTableNames($groupID);
$this->deleteUnwantedCollectionsMain($groupID, $tableNames);
}
/**
* Delete unwanted collections based on size/file count using admin settings.
*
* @param int|string $groupID (optional)
*
* @param bool $multiGroup
* @param $tableNames
*
* @void
* @access public
* @access public
*/
public function deleteUnwantedCollections($groupID, $multiGroup = false)
public function deleteUnwantedCollectionsMain($groupID, $tableNames)
{
$startTime = time();
$group = $this->initiateTableNames($groupID);
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$tablep = $multiGroup === true ? 'mgr_parts' : $group['pname'];
if ($this->echoCLI) {
$this->pdo->log->doEcho(
@@ -454,7 +462,7 @@ class ProcessReleases
if ($this->pdo->queryOneRow(
sprintf(
'SELECT SQL_NO_CACHE id FROM %s c WHERE c.filecheck = %d AND c.filesize > 0 %s LIMIT 1',
$tablec,
$tableNames['cname'],
self::COLLFC_SIZED,
$this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : ''
)
@@ -471,9 +479,9 @@ class ProcessReleases
AND c.filesize > 0
AND GREATEST(%d, %d) > 0
AND c.filesize < GREATEST(%d, %d)',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
self::COLLFC_SIZED,
$this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '',
$groupMinSizeSetting,
@@ -495,9 +503,9 @@ class ProcessReleases
LEFT JOIN %s p ON b.id = p.binaryid
WHERE c.filecheck = %d %s
AND c.filesize > %d',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
self::COLLFC_SIZED,
$this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '',
$maxSizeSetting
@@ -516,9 +524,9 @@ class ProcessReleases
WHERE c.filecheck = %d %s
AND GREATEST(%d, %d) > 0
AND c.totalfiles < GREATEST(%d, %d)',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
self::COLLFC_SIZED,
$this->tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID['id']) : '',
$groupMinFilesSetting,
@@ -791,8 +799,7 @@ class ProcessReleases
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create the NZB, delete collections/binaries/parts."));
}
$relMgrp = new ProcessMultiGroupReleases();
$posters = Utility::convertMultiArray($relMgrp->getAllPosters(), "','");
$posters = Utility::convertMultiArray($this->mgr->getAllPosters(), "','");
$releases = $this->pdo->queryDirect(
sprintf("
SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title,
@@ -956,28 +963,28 @@ class ProcessReleases
}
}
public function deleteCollections($groupID)
{
$tableNames = $this->initiateTableNames($groupID);
$this->deleteCollectionsMain($groupID, $tableNames);
}
/**
* Delete collections (complete/incomplete/old/etc).
*
* @param int|string $groupID (optional)
*
* @param bool $multiGroup
* @param $tableNames
*
* @void
* @access public
*/
public function deleteCollections($groupID, $multiGroup = false)
public function deleteCollectionsMain($groupID, $tableNames)
{
$startTime = time();
$group = $this->initiateTableNames($groupID);
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$tablep = $multiGroup === true ? 'mgr_parts' : $group['pname'];
$deletedCount = 0;
//(new ProcessMultiGroupReleases())->deleteMGRCollections();
// CBP older than retention.
if ($this->echoCLI) {
echo (
@@ -996,9 +1003,9 @@ class ProcessReleases
LEFT JOIN %s b ON (c.id=b.collection_id)
LEFT JOIN %s p ON (b.id=p.binaryid)
WHERE (c.dateadded < NOW() - INTERVAL %d HOUR) %s',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
Settings::value('..partretentionhours'),
(!empty($groupID) && $this->tablePerGroup === false ? ' AND c.group_id = ' . $groupID : '')
)
@@ -1036,9 +1043,9 @@ class ProcessReleases
LEFT JOIN %s b ON (c.id=b.collection_id)
LEFT JOIN %s p ON (b.id=p.binaryid)
WHERE (b.id IS NULL OR p.binaryid IS NULL) %s',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
(!empty($groupID) && $this->tablePerGroup === false ? ' AND c.group_id = ' . $groupID : '')
)
);
@@ -1070,7 +1077,7 @@ class ProcessReleases
LEFT JOIN %s p ON(b.id=p.binaryid)
LEFT JOIN %s c ON(b.collection_id=c.id)
WHERE (p.binaryid IS NULL OR c.id IS NULL) AND b.id < %d ',
$tableb, $tablep, $tablec, $this->maxQueryFormulator($tableb, 20000)
$tableNames['bname'], $tableNames['pname'], $tableNames['cname'], $this->maxQueryFormulator($tableNames['bname'], 20000)
)
);
@@ -1097,7 +1104,7 @@ class ProcessReleases
$deleteQuery = $this->pdo->queryExec(
sprintf(
'DELETE p FROM %s p LEFT JOIN %s b ON (p.binaryid=b.id) WHERE b.id IS NULL AND p.binaryid < %d',
$tablep, $tableb, $this->maxQueryFormulator($tableb, 20000)
$tableNames['pname'], $tableNames['bname'], $this->maxQueryFormulator($tableNames['bname'], 20000)
)
);
if ($deleteQuery !== false) {
@@ -1129,7 +1136,7 @@ class ProcessReleases
FROM %s c
INNER JOIN releases r ON r.id = c.releaseid
WHERE r.nzbstatus = 1',
$tablec
$tableNames['cname']
)
);
@@ -1143,9 +1150,9 @@ class ProcessReleases
LEFT JOIN %s b ON(c.id=b.collection_id)
LEFT JOIN %s p ON(b.id=p.binaryid)
WHERE c.id = %d',
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
$collection['id']
)
);
@@ -1539,19 +1546,14 @@ class ProcessReleases
* This means the the binary table has the same count as the file count in the subject, but
* the collection might not be complete yet since we might not have all the articles in the parts table.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
* @access private
*/
private function collectionFileCheckStage1(array &$group, &$where, $multiGroup = false)
private function collectionFileCheckStage1(array &$tableNames, &$where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$this->pdo->queryExec(
sprintf('
UPDATE %s c INNER JOIN
@@ -1562,9 +1564,9 @@ class ProcessReleases
HAVING COUNT(b.id) IN (c.totalfiles, c.totalfiles + 1)
)
r ON c.id = r.id SET filecheck = %d',
$tablec,
$tablec,
$tableb,
$tableNames['cname'],
$tableNames['cname'],
$tableNames['bname'],
self::COLLFC_DEFAULT,
$where,
self::COLLFC_COMPCOLL
@@ -1580,19 +1582,13 @@ class ProcessReleases
* at 0 then you would never get a complete collection if it starts with 1 and if it starts, you can end up creating
* a incomplete collection, since you assumed it was complete.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function collectionFileCheckStage2(array &$group, &$where, $multiGroup = false)
private function collectionFileCheckStage2(array &$tableNames, &$where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$this->pdo->queryExec(
sprintf('
UPDATE %s c INNER JOIN
@@ -1604,9 +1600,9 @@ class ProcessReleases
GROUP BY c.id
)
r ON c.id = r.id SET c.filecheck = %d',
$tablec,
$tablec,
$tableb,
$tableNames['cname'],
$tableNames['cname'],
$tableNames['bname'],
self::COLLFC_COMPCOLL,
$where,
self::COLLFC_ZEROPART
@@ -1617,7 +1613,7 @@ class ProcessReleases
UPDATE %s c
SET filecheck = %d
WHERE filecheck = %d %s',
$tablec,
$tableNames['cname'],
self::COLLFC_TEMPCOMP,
self::COLLFC_COMPCOLL,
$where
@@ -1629,19 +1625,15 @@ class ProcessReleases
* Check if the files (binaries table) in a complete collection has all the parts.
* If we have all the parts, set binaries table partcheck to FILE_COMPLETE.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function collectionFileCheckStage3(array &$group, $where, $multiGroup = false)
private function collectionFileCheckStage3(array &$tableNames, $where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$this->pdo->queryExec(
sprintf('
UPDATE %s b INNER JOIN
@@ -1651,9 +1643,9 @@ class ProcessReleases
AND b.currentparts = b.totalparts
GROUP BY b.id, b.totalparts)
r ON b.id = r.id SET b.partcheck = %d',
$tableb,
$tableb,
$tablec,
$tableNames['bname'],
$tableNames['bname'],
$tableNames['cname'],
self::COLLFC_TEMPCOMP,
self::FILE_INCOMPLETE,
$where,
@@ -1669,9 +1661,9 @@ class ProcessReleases
AND b.currentparts >= (b.totalparts + 1)
GROUP BY b.id, b.totalparts)
r ON b.id = r.id SET b.partcheck = %d',
$tableb,
$tableb,
$tablec,
$tableNames['bname'],
$tableNames['bname'],
$tableNames['cname'],
self::COLLFC_ZEROPART,
self::FILE_INCOMPLETE,
$where,
@@ -1685,18 +1677,14 @@ class ProcessReleases
* Set collections filecheck column to COLLFC_COMPPART.
* This means the collection is complete.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function collectionFileCheckStage4(array &$group, &$where, $multiGroup= false)
private function collectionFileCheckStage4(array &$tableNames, &$where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$this->pdo->queryExec(
sprintf('
@@ -1706,9 +1694,9 @@ class ProcessReleases
WHERE b.partcheck = 1 AND c.filecheck IN (%d, %d) %s
GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) >= c.totalfiles)
r ON c.id = r.id SET filecheck = %d',
$tablec,
$tablec,
$tableb,
$tableNames['cname'],
$tableNames['cname'],
$tableNames['bname'],
self::COLLFC_TEMPCOMP,
self::COLLFC_ZEROPART,
$where,
@@ -1721,24 +1709,20 @@ class ProcessReleases
* If not all files (binaries table) had their parts on the previous stage,
* reset the collection filecheck column to COLLFC_COMPCOLL so we reprocess them next time.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function collectionFileCheckStage5(array &$group, &$where, $multiGroup = false)
private function collectionFileCheckStage5(array &$tableNames, &$where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$this->pdo->queryExec(
sprintf('
UPDATE %s c
SET filecheck = %d
WHERE filecheck IN (%d, %d) %s',
$tablec,
$tableNames['cname'],
self::COLLFC_COMPCOLL,
self::COLLFC_TEMPCOMP,
self::COLLFC_ZEROPART,
@@ -1751,27 +1735,22 @@ class ProcessReleases
* If a collection did not have the file count (ie: [00/12]) or the collection is incomplete after
* $this->collectionDelayTime hours, set the collection to complete to create it into a release/nzb.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function collectionFileCheckStage6(array &$group, &$where, $multiGroup = false)
private function collectionFileCheckStage6(array &$tableNames, &$where)
{
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$this->pdo->queryExec(
sprintf("
UPDATE %s c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM %s b WHERE b.collection_id = c.id)
WHERE c.dateadded < NOW() - INTERVAL '%d' HOUR
AND c.filecheck IN (%d, %d, 10) %s",
$tablec,
$tableNames['cname'],
self::COLLFC_COMPPART,
$tableb,
$tableNames['bname'],
$this->collectionDelayTime,
self::COLLFC_DEFAULT,
self::COLLFC_COMPCOLL,
@@ -1783,20 +1762,15 @@ class ProcessReleases
/**
* If a collection has been stuck for $this->collectionTimeout hours, delete it, it's bad.
*
* @param array $group
* @param array $tableNames
* @param string $where
*
* @param bool $multiGroup
*
* @void
* @access private
*/
private function processStuckCollections(array $group, $where, $multiGroup = false)
private function processStuckCollections(array $tableNames, $where)
{
$lastRun = Settings::value('indexer.processing.last_run_time');
$tablec = $multiGroup === true ? 'mgr_collections' : $group['cname'];
$tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname'];
$tablep = $multiGroup === true ? 'mgr_parts' : $group['pname'];
$obj = $this->pdo->queryExec(
sprintf("
@@ -1807,9 +1781,9 @@ class ProcessReleases
c.added <
DATE_SUB({$this->pdo->escapeString($lastRun)}, INTERVAL %d HOUR)
%s",
$tablec,
$tableb,
$tablep,
$tableNames['cname'],
$tableNames['bname'],
$tableNames['pname'],
$this->collectionTimeout,
$where
)