diff --git a/misc/testing/Dev/test_posters.php b/misc/testing/Dev/test_posters.php deleted file mode 100644 index 12629482d..000000000 --- a/misc/testing/Dev/test_posters.php +++ /dev/null @@ -1,13 +0,0 @@ -getAllPosters(), "','"); - -var_dump($posters); \ No newline at end of file diff --git a/misc/update/nix/multiprocessing/.do_not_run/switch.php b/misc/update/nix/multiprocessing/.do_not_run/switch.php index 262cabd9c..435ba8a04 100644 --- a/misc/update/nix/multiprocessing/.do_not_run/switch.php +++ b/misc/update/nix/multiprocessing/.do_not_run/switch.php @@ -16,7 +16,7 @@ use nntmux\Binaries; use nntmux\Groups; use nntmux\Nfo; use nntmux\NNTP; -use nntmux\ReleasesMultiGroup; +use nntmux\processing\ProcessMultiGroupReleases; use nntmux\RequestIDLocal; // Are we coming from python or php ? $options[0] => (string): python|php @@ -151,7 +151,7 @@ switch ($options[1]) { case 'releases': $pdo = new DB(); $releases = new ProcessReleases(['Settings' => $pdo]); - $mgrreleases = new ReleasesMultiGroup(['Settings' => $pdo]); + $mgrreleases = new ProcessMultiGroupReleases(['Settings' => $pdo]); //Runs function that are per group if (is_numeric($options[2])) { @@ -223,7 +223,7 @@ switch ($options[1]) { $backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal'); // Create releases. - processReleases(new ProcessReleases(['Settings' => $pdo]), new ReleasesMultiGroup(['Settings' => $pdo]), $options[2]); + processReleases(new ProcessReleases(['Settings' => $pdo]), new ProcessMultiGroupReleases(['Settings' => $pdo]), $options[2]); // Post process the releases. (new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start($options[2]); @@ -279,7 +279,7 @@ switch ($options[1]) { * Create / process releases for a groupID. * * @param ProcessReleases $releases - * @param ReleasesMultiGroup $mgrreleases + * @param ProcessMultiGroupReleases $mgrreleases * @param int $groupID */ function processReleases($releases, $mgrreleases, $groupID) diff --git a/nntmux/Binaries.php b/nntmux/Binaries.php index e54e37c7d..734022fd5 100755 --- a/nntmux/Binaries.php +++ b/nntmux/Binaries.php @@ -3,6 +3,7 @@ namespace nntmux; use app\models\Settings; use nntmux\db\DB; +use nntmux\processing\ProcessMultiGroupReleases; /** * Class Binaries @@ -84,7 +85,7 @@ class Binaries protected $_pdo; /** - * @var ReleasesMultiGroup + * @var ProcessMultiGroupReleases */ protected $_multiGrp; @@ -179,7 +180,7 @@ class Binaries $this->_colorCLI = ($options['ColorCLI'] instanceof ColorCLI ? $options['ColorCLI'] : new ColorCLI()); $this->_nntp = ($options['NNTP'] instanceof NNTP ? $options['NNTP'] : new NNTP(['Echo' => $this->_colorCLI, 'Settings' => $this->_pdo, 'ColorCLI' => $this->_colorCLI])); $this->_collectionsCleaning = ($options['CollectionsCleaning'] instanceof CollectionsCleaning ? $options['CollectionsCleaning'] : new CollectionsCleaning(['Settings' => $this->_pdo])); - $this->_multiGrp = new ReleasesMultiGroup(['Settings' => $this->_pdo]); + $this->_multiGrp = new ProcessMultiGroupReleases(['Settings' => $this->_pdo]); $this->_debug = (NN_DEBUG || NN_LOGGING); diff --git a/nntmux/ReleasesMultiGroup.php b/nntmux/ReleasesMultiGroup.php deleted file mode 100644 index e06a8aee9..000000000 --- a/nntmux/ReleasesMultiGroup.php +++ /dev/null @@ -1,348 +0,0 @@ - null, - ]; - $options += $defaults; - $this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB()); - $this->mgrnzb = new NZBMultiGroup($this->pdo); - $this->consoleTools = new ConsoleTools(['ColorCLI' => $this->pdo->log]); - $this->groups = new Groups(['Settings' => $this->pdo]); - $this->releaseCleaning = new ReleaseCleaning($this->pdo); - $this->releases = new Releases(['Settings' => $this->pdo, 'Groups' => $this->groups]); - $this->tablePerGroup = (Settings::value('..tablepergroup') == 0 ? false : true); - $this->echoCLI = NN_ECHOCLI; - $this->releaseCreationLimit = (Settings::value('..maxnzbsprocessed') != '' ? (int)Settings::value('..maxnzbsprocessed') : 1000); - } - - /** - * @param $fromName - * - * @return bool - */ - public function isMultiGroup($fromName) - { - $array = array_column($this->getAllPosters(), 'poster'); - return in_array($fromName, $array); - } - - /** - * Create releases from complete collections. - * - * - * @param $groupID - * - * @return array - * @access public - */ - public function createMGRReleases($groupID) - { - $startTime = time(); - - $categorize = new Categorize(['Settings' => $this->pdo]); - $returnCount = $duplicate = 0; - - if ($this->echoCLI) { - $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create releases from complete MGR collections.")); - } - - $this->pdo->ping(true); - - $collections = $this->pdo->queryDirect( - sprintf(' - SELECT SQL_NO_CACHE mgr_collections.*, groups.name AS gname - FROM mgr_collections - INNER JOIN groups ON mgr_collections.group_id = groups.id - WHERE %s mgr_collections.filecheck = %d - AND filesize > 0 LIMIT %d', - (!empty($groupID) ? ' group_id = ' . $groupID . ' AND ' : ' '), - ProcessReleases::COLLFC_SIZED, - $this->releaseCreationLimit - ) - ); - - if ($this->echoCLI && $collections !== false) { - echo $this->pdo->log->primary($collections->rowCount() . " MGR Collections ready to be converted to releases."); - } - - if ($collections instanceof \Traversable) { - $preDB = new PreDb(['Echo' => $this->echoCLI, 'Settings' => $this->pdo]); - - foreach ($collections as $collection) { - - $cleanRelName = $this->pdo->escapeString( - utf8_encode( - str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection['subject']) - ) - ); - $fromName = $this->pdo->escapeString( - utf8_encode(trim($collection['fromname'], "'")) - ); - - // Look for duplicates, duplicates match on releases.name, releases.fromname and releases.size - // A 1% variance in size is considered the same size when the subject and poster are the same - $dupeCheck = $this->pdo->queryOneRow( - sprintf(" - SELECT SQL_NO_CACHE id - FROM releases - WHERE name = %s - AND fromname = %s - AND size BETWEEN '%s' - AND '%s'", - $cleanRelName, - $fromName, - ($collection['filesize'] * .99), - ($collection['filesize'] * 1.01) - ) - ); - - if ($dupeCheck === false) { - - $cleanedName = $this->releaseCleaning->releaseCleaner( - $collection['subject'], $collection['fromname'], $collection['filesize'], $collection['gname'] - ); - - if (is_array($cleanedName)) { - $properName = $cleanedName['properlynamed']; - $preID = (isset($cleanerName['predb']) ? $cleanerName['predb'] : false); - $isReqID = (isset($cleanerName['requestid']) ? $cleanerName['requestid'] : false); - $cleanedName = $cleanedName['cleansubject']; - } else { - $properName = true; - $isReqID = $preID = false; - } - - if ($preID === false && $cleanedName !== '') { - // try to match the cleaned searchname to predb title or filename here - $preMatch = $preDB->matchPre($cleanedName); - if ($preMatch !== false) { - $cleanedName = $preMatch['title']; - $preID = $preMatch['predb_id']; - $properName = true; - } - } - - $releaseID = $this->releases->insertRelease( - [ - 'name' => $cleanRelName, - 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), - 'totalpart' => $collection['totalfiles'], - 'groups_id' => $collection['group_id'], - 'guid' => $this->pdo->escapeString($this->releases->createGUID()), - 'postdate' => $this->pdo->escapeString($collection['date']), - 'fromname' => $fromName, - 'size' => $collection['filesize'], - 'categories_id' => $categorize->determineCategory($collection['group_id'], $cleanedName, $fromName), - 'isrenamed' => ($properName === true ? 1 : 0), - 'reqidstatus' => ($isReqID === true ? 1 : 0), - 'predb_id' => ($preID === false ? 0 : $preID), - 'nzbstatus' => NZB::NZB_NONE - ] - ); - - if ($releaseID !== false) { - // Update collections table to say we inserted the release. - $this->pdo->queryExec( - sprintf(' - UPDATE mgr_collections - SET filecheck = %d, releaseid = %d - WHERE id = %d', - ProcessReleases::COLLFC_INSERTED, - $releaseID, - $collection['id'] - ) - ); - - if (preg_match_all('#(\S+):\S+#', $collection['xref'], $matches)) { - $matches = array_unique($matches[1]); - foreach ($matches as $grp) { - //check if the group name is in a valid format - $grpTmp = $this->groups->isValidGroup($grp); - if ($grpTmp !== false) { - //check if the group already exists in database - $xrefGrpID = $this->groups->getIDByName($grpTmp); - if ($xrefGrpID === '') { - $xrefGrpID = $this->groups->add( - [ - 'name' => $grpTmp, - 'description' => 'Added by Release processing', - 'backfill_target' => 1, - 'first_record' => 0, - 'last_record' => 0, - 'active' => 0, - 'backfill' => 0, - 'minfilestoformrelease' => '', - 'minsizetoformrelease' => '' - ] - ); - } - - $relGroups = ReleasesGroups::create( - [ - 'releases_id' => $releaseID, - 'groups_id' => $xrefGrpID, - ] - ); - $relGroups->save(); - } - } - } - - $returnCount++; - - if ($this->echoCLI) { - echo "Added $returnCount releases.\r"; - } - } - } else { - // The release was already in the DB, so delete the collection. - $this->pdo->queryExec( - sprintf(' - DELETE c, b, p FROM mgr_collections c - INNER JOIN mgr_binaries b ON(c.id=b.collection_id) - STRAIGHT_JOIN mgr_parts p ON(b.id=p.binaryid) - WHERE c.collectionhash = %s', - $this->pdo->escapeString($collection['collectionhash']) - ) - ); - $duplicate++; - } - } - } - - if ($this->echoCLI) { - $this->pdo->log->doEcho( - $this->pdo->log->primary( - PHP_EOL . - number_format($returnCount) . - ' Releases added and ' . - number_format($duplicate) . - ' duplicate collections deleted in ' . - $this->consoleTools->convertTime(time() - $startTime) - ), true - ); - } - - return ['added' => $returnCount, 'dupes' => $duplicate]; - } - - /** - * Create NZB files from complete releases. - * - * - * @param $groupID - * - * @return int - * @access public - */ - public function createMGRNZBs($groupID) - { - $this->mgrFromNames = Utility::convertMultiArray($this->getAllPosters(), "','"); - - $releases = $this->pdo->queryDirect( - sprintf(" - SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, - r.name, r.id, r.guid - FROM releases r - INNER JOIN categories c ON r.categories_id = c.id - INNER JOIN categories cp ON cp.id = c.parentid - WHERE %s r.nzbstatus = 0 AND r.fromname IN ('%s')", - (!empty($groupID) ? ' r.groups_id = ' . $groupID . ' AND ' : ' '), - $this->mgrFromNames - ) - ); - - $nzbCount = 0; - - if ($releases && $releases->rowCount()) { - $total = $releases->rowCount(); - // Init vars for writing the NZB's. - $this->mgrnzb->initiateForMgrWrite(); - foreach ($releases as $release) { - - if ($this->mgrnzb->writeNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { - $nzbCount++; - if ($this->echoCLI) { - echo $this->pdo->log->primaryOver("Creating NZBs and deleting MGR Collections:\t" . $nzbCount . '/' . $total . "\r"); - } - } - } - } - - return $nzbCount; - } - - /** - * Add multi group posters to database - * - * @param $poster - */ - public function addPoster($poster) - { - $this->pdo->queryInsert(sprintf('INSERT INTO mgr_posters (poster) VALUE (%s)', $this->pdo->escapeString($poster))); - } - - /** - * Update multi group poster - * - * @param $id - * @param $poster - */ - public function updatePoster($id, $poster) - { - $this->pdo->queryExec(sprintf('UPDATE mgr_posters SET poster = %s WHERE id = %d', $this->pdo->escapeString($poster), $id)); - } - - /** - * Delete multi group posters from database - * - * @param $id - */ - public function deletePoster($id) - { - $this->pdo->queryExec(sprintf('DELETE FROM mgr_posters WHERE id = %d', $id)); - } - - /** - * Fetch all multi group posters from database - * - * @return array|bool - */ - public function getAllPosters() - { - $result = $this->pdo->query(sprintf('SELECT poster AS poster FROM mgr_posters')); - if (is_array($result) && !empty($result)) { - return $result; - } - return false; - } -} diff --git a/nntmux/processing/ProcessMultiGroupReleases.php b/nntmux/processing/ProcessMultiGroupReleases.php new file mode 100644 index 000000000..0d6885c14 --- /dev/null +++ b/nntmux/processing/ProcessMultiGroupReleases.php @@ -0,0 +1,164 @@ +mgrnzb = new NZBMultiGroup($this->pdo); + } + + /** + * @param $fromName + * + * @return bool + */ + public function isMultiGroup($fromName) + { + $array = array_column($this->getAllPosters(), 'poster'); + return in_array($fromName, $array); + } + + + protected function initiateMgrTableNames() + { + $group = [ + 'cname' => 'mgr_collections', + 'bname' => 'mgr_binaries', + 'pname' => 'mgr_parts' + ]; + + return $group; + } + + /** + * Create releases from complete collections. + * + * @param $groupID + * + * @return array + * @access public + */ + public function createMGRReleases($groupID) + { + $tableNames = $this->initiateMgrTableNames(); + return $this->createReleasesMain($groupID, $tableNames); + } + + /** + * Create NZB files from complete releases. + * + * + * @param $groupID + * + * @return int + * @access public + */ + public function createMGRNZBs($groupID) + { + $this->mgrFromNames = Utility::convertMultiArray($this->getAllPosters(), "','"); + + $releases = $this->pdo->queryDirect( + sprintf(" + SELECT SQL_NO_CACHE CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, + r.name, r.id, r.guid + FROM releases r + INNER JOIN categories c ON r.categories_id = c.id + INNER JOIN categories cp ON cp.id = c.parentid + WHERE %s r.nzbstatus = 0 AND r.fromname IN ('%s')", + (!empty($groupID) ? ' r.groups_id = ' . $groupID . ' AND ' : ' '), + $this->mgrFromNames + ) + ); + + $nzbCount = 0; + + if ($releases && $releases->rowCount()) { + $total = $releases->rowCount(); + // Init vars for writing the NZB's. + $this->mgrnzb->initiateForMgrWrite(); + foreach ($releases as $release) { + + if ($this->mgrnzb->writeNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { + $nzbCount++; + if ($this->echoCLI) { + echo $this->pdo->log->primaryOver("Creating NZBs and deleting MGR Collections:\t" . $nzbCount . '/' . $total . "\r"); + } + } + } + } + + return $nzbCount; + } + + /** + * Add multi group posters to database + * + * @param $poster + */ + public function addPoster($poster) + { + $this->pdo->queryInsert(sprintf('INSERT INTO mgr_posters (poster) VALUE (%s)', $this->pdo->escapeString($poster))); + } + + /** + * Update multi group poster + * + * @param $id + * @param $poster + */ + public function updatePoster($id, $poster) + { + $this->pdo->queryExec(sprintf('UPDATE mgr_posters SET poster = %s WHERE id = %d', $this->pdo->escapeString($poster), $id)); + } + + /** + * Delete multi group posters from database + * + * @param $id + */ + public function deletePoster($id) + { + $this->pdo->queryExec(sprintf('DELETE FROM mgr_posters WHERE id = %d', $id)); + } + + /** + * Fetch all multi group posters from database + * + * @return array|bool + */ + public function getAllPosters() + { + $result = $this->pdo->query(sprintf('SELECT poster AS poster FROM mgr_posters')); + if (is_array($result) && !empty($result)) { + return $result; + } + return false; + } +} diff --git a/nntmux/processing/ProcessReleases.php b/nntmux/processing/ProcessReleases.php index 8699fc012..332a7d23c 100755 --- a/nntmux/processing/ProcessReleases.php +++ b/nntmux/processing/ProcessReleases.php @@ -12,7 +12,7 @@ use nntmux\ReleaseImage; use nntmux\NZB; use nntmux\Categorize; use nntmux\Category; -use nntmux\ReleasesMultiGroup; +use nntmux\processing\ProcessMultiGroupReleases; use nntmux\RequestIDLocal; use nntmux\RequestIDWeb; use nntmux\PreDb; @@ -193,11 +193,11 @@ class ProcessReleases $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); - $mgrReleasesCount = (new ReleasesMultiGroup())->createMGRReleases($groupID); + $mgrReleasesCount = (new ProcessMultiGroupReleases())->createMGRReleases($groupID); $totalReleasesAdded += $releasesCount['added'] += $mgrReleasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); - $mgrFilesAdded = (new ReleasesMultiGroup())->createMGRNZBs($groupID); + $mgrFilesAdded = (new ProcessMultiGroupReleases())->createMGRNZBs($groupID); $this->deleteCollections($groupID); if ($this->processRequestIDs === 0) { $this->processRequestIDs($groupID, 5000, true); @@ -310,7 +310,7 @@ class ProcessReleases public function processIncompleteCollections($groupID) { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $group = $this->initiateTableNames($groupID); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Attempting to find complete collections.")); @@ -366,7 +366,7 @@ class ProcessReleases public function processCollectionSizes($groupID, $multiGroup = false) { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $group = $this->initiateTableNames($groupID); $tablec = $multiGroup === true ? 'mgr_collections' : $group['cname']; $tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname']; @@ -404,13 +404,15 @@ class ProcessReleases * * @param int|string $groupID (optional) * + * @param bool $multiGroup + * * @void * @access public */ public function deleteUnwantedCollections($groupID, $multiGroup = false) { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $group = $this->initiateTableNames($groupID); $tablec = $multiGroup === true ? 'mgr_collections' : $group['cname']; $tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname']; $tablep = $multiGroup === true ? 'mgr_parts' : $group['pname']; @@ -544,18 +546,32 @@ class ProcessReleases } } + protected function initiateTableNames($groupID) + { + return $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + } + + public function createReleases($groupID) + { + $tableNames = $this->initiateTableNames($groupID); + return $this->createReleasesMain($groupID, $tableNames); + } + + /** * Create releases from complete collections. * * @param int|string $groupID (optional) * + * @param array $tableNames + * * @return array * @access public */ - public function createReleases($groupID) + public function createReleasesMain($groupID, $tableNames) { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $group = $tableNames; $categorize = new Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; @@ -764,7 +780,7 @@ class ProcessReleases if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create the NZB, delete collections/binaries/parts.")); } - $relMgrp = new ReleasesMultiGroup(); + $relMgrp = new ProcessMultiGroupReleases(); $posters = Utility::convertMultiArray($relMgrp->getAllPosters(), "','"); $releases = $this->pdo->queryDirect( sprintf(" @@ -942,14 +958,14 @@ class ProcessReleases public function deleteCollections($groupID, $multiGroup = false) { $startTime = time(); - $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); + $group = $this->initiateTableNames($groupID); $tablec = $multiGroup === true ? 'mgr_collections' : $group['cname']; $tableb = $multiGroup === true ? 'mgr_binaries' : $group['bname']; $tablep = $multiGroup === true ? 'mgr_parts' : $group['pname']; $deletedCount = 0; - //(new ReleasesMultiGroup())->deleteMGRCollections(); + //(new ProcessMultiGroupReleases())->deleteMGRCollections(); // CBP older than retention. if ($this->echoCLI) { diff --git a/www/admin/poster-delete.php b/www/admin/poster-delete.php index abb5aad1e..fd02215a9 100644 --- a/www/admin/poster-delete.php +++ b/www/admin/poster-delete.php @@ -2,13 +2,13 @@ require_once './config.php'; -use nntmux\ReleasesMultiGroup; +use nntmux\processing\ProcessMultiGroupReleases; $page = new AdminPage(); if (isset($_GET['id'])) { - $mgr = new ReleasesMultiGroup(); + $mgr = new ProcessMultiGroupReleases(); $mgr->deletePoster($_GET['id']); } diff --git a/www/admin/posters-edit.php b/www/admin/posters-edit.php index 9538513ec..447d5cf1b 100644 --- a/www/admin/posters-edit.php +++ b/www/admin/posters-edit.php @@ -3,10 +3,10 @@ require_once './config.php'; -use nntmux\ReleasesMultiGroup; +use nntmux\processing\ProcessMultiGroupReleases; $page = new AdminPage(); -$relPosters = new ReleasesMultiGroup(['Settings' => $page->settings]); +$relPosters = new ProcessMultiGroupReleases(['Settings' => $page->settings]); // Set the current action. $action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view';