true, 'ConsoleTools' => null, 'Groups' => null, 'NZB' => null, 'ReleaseCleaning' => null, 'ReleaseImage' => null, 'Releases' => null, 'Settings' => null, ]; $options += $defaults; $this->echoCLI = ($options['Echo'] && NN_ECHOCLI); $this->pdo = ($options['Settings'] instanceof Settings ? $options['Settings'] : new Settings()); $this->consoleTools = ($options['ConsoleTools'] instanceof \ConsoleTools ? $options['ConsoleTools'] : new \ConsoleTools(['ColorCLI' => $this->pdo->log])); $this->groups = ($options['Groups'] instanceof \Groups ? $options['Groups'] : new \Groups(['Settings' => $this->pdo])); $this->nzb = ($options['NZB'] instanceof \NZB ? $options['NZB'] : new \NZB($this->pdo)); $this->releaseCleaning = ($options['ReleaseCleaning'] instanceof \ReleaseCleaning ? $options['ReleaseCleaning'] : new \ReleaseCleaning($this->pdo)); $this->releases = ($options['Releases'] instanceof \Releases ? $options['Releases'] : new \Releases(['Settings' => $this->pdo, 'Groups' => $this->groups])); $this->releaseImage = ($options['ReleaseImage'] instanceof \ReleaseImage ? $options['ReleaseImage'] : new \ReleaseImage($this->pdo)); $this->tablePerGroup = ($this->pdo->getSetting('tablepergroup') == 0 ? false : true); $this->collectionDelayTime = ($this->pdo->getSetting('delaytime') != '' ? (int)$this->pdo->getSetting('delaytime') : 2); $this->crossPostTime = ($this->pdo->getSetting('crossposttime') != '' ? (int)$this->pdo->getSetting('crossposttime') : 2); $this->releaseCreationLimit = ($this->pdo->getSetting('maxnzbsprocessed') != '' ? (int)$this->pdo->getSetting('maxnzbsprocessed') : 1000); $this->completion = ($this->pdo->getSetting('completionpercent') != '' ? (int)$this->pdo->getSetting('completionpercent') : 0); $this->processRequestIDs = (int)$this->pdo->getSetting('lookup_reqids'); if ($this->completion > 100) { $this->completion = 100; echo $this->pdo->log->error(PHP_EOL . 'You have an invalid setting for completion. It must be lower than 100.'); } $this->collectionTimeout = intval($this->pdo->getSetting('collection_timeout')); } /** * Main method for creating releases/NZB files from collections. * * @param int $categorize * @param int $postProcess * @param string $groupName (optional) * @param \NNTP $nntp * @param bool $echooutput * * @return int */ public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput) { $this->echoCLI = ($echooutput && NN_ECHOCLI); $groupID = ''; if (!empty($groupName)) { $groupInfo = $this->groups->getByName($groupName); $groupID = $groupInfo['id']; } $processReleases = microtime(true); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Starting release update process (" . date('Y-m-d H:i:s') . ")"), true); } if (!file_exists($this->pdo->getSetting('nzbpath'))) { if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->error('Bad or missing nzb directory - ' . $this->pdo->getSetting('nzbpath')), true); } return 0; } $this->processIncompleteCollections($groupID); $this->processCollectionSizes($groupID); $this->deleteUnwantedCollections($groupID); $DIR = NN_MISC; $PYTHON = shell_exec('which python3 2>/dev/null'); $PYTHON = (empty($PYTHON) ? 'python -OOu' : 'python3 -OOu'); $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); $totalReleasesAdded += $releasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); if ($this->processRequestIDs === 0) { $this->processRequestIDs($groupID, 5000, true); } else if ($this->processRequestIDs === 1) { $this->processRequestIDs($groupID, 5000, true); $this->processRequestIDs($groupID, 1000, false); } else if ($this->processRequestIDs === 2) { $requestIDTime = time(); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request id Threaded lookup.")); } passthru("$PYTHON ${DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py"); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( "\nReleases updated in " . $this->consoleTools->convertTime(time() - $requestIDTime) ) ); } } $this->categorizeReleases($categorize, $groupID); $this->postProcessReleases($postProcess, $nntp); $this->deleteCollections($groupID); // This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately } while (($releasesCount['added'] + $releasesCount['dupes']) >= $this->releaseCreationLimit || $nzbFilesAdded >= $this->releaseCreationLimit); $this->deletedReleasesByGroup($groupID); $this->deleteReleases(); //Print amount of added releases and time it took. if ($this->echoCLI && $this->tablePerGroup === false) { $countID = $this->pdo->queryOneRow('SELECT COUNT(id) AS count FROM collections ' . (!empty($groupID) ? ' WHERE group_id = ' . $groupID : '')); $this->pdo->log->doEcho( $this->pdo->log->primary( 'Completed adding ' . number_format($totalReleasesAdded) . ' releases in ' . $this->consoleTools->convertTime(number_format(microtime(true) - $processReleases, 2)) . '. ' . number_format(($countID === false ? 0 : $countID['count'])) . ' collections waiting to be created (still incomplete or in queue for creation)' ), true ); } return $totalReleasesAdded; } /** * Return all releases to other->misc category. * * @param string $where Optional "where" query parameter. * * @void * @access public */ public function resetCategorize($where = '') { $this->pdo->queryExec( sprintf('UPDATE releases SET categoryid = %d, iscategorized = 0 %s', \Category::CAT_MISC_OTHER, $where) ); } /** * Categorizes releases. * * @param string $type name or searchname | Categorize using the search name or subject. * @param string $where Optional "where" query parameter. * * @return int Quantity of categorized releases. * @access public */ public function categorizeRelease($type, $where = '') { $cat = new \Categorize(['Settings' => $this->pdo]); $categorized = $total = 0; $releases = $this->pdo->queryDirect(sprintf('SELECT id, %s, groupid FROM releases %s', $type, $where)); if ($releases && $releases->rowCount()) { $total = $releases->rowCount(); foreach ($releases as $release) { $catId = $cat->determineCategory($release['groupid'], $release[$type]); $this->pdo->queryExec( sprintf('UPDATE releases SET categoryid = %d, iscategorized = 1 WHERE id = %d', $catId, $release['id']) ); $categorized++; if ($this->echoCLI) { $this->consoleTools->overWritePrimary( 'Categorizing: ' . $this->consoleTools->percentString($categorized, $total) ); } } } if ($this->echoCLI !== false && $categorized > 0) { echo PHP_EOL; } return $categorized; } /** * Find complete collections to be processed by processCollectionSizes. * * @param int $groupID * @void * @access public */ public function processIncompleteCollections($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Attempting to find complete collections.")); } $where = (!empty($groupID) ? ' AND c.group_id = ' . $groupID . ' ' : ' '); $this->processStuckCollections($group, $where); $this->collectionFileCheckStage1($group, $where); $this->collectionFileCheckStage2($group, $where); $this->collectionFileCheckStage3($group, $where); $this->collectionFileCheckStage4($group, $where); $this->collectionFileCheckStage5($group, $where); $this->collectionFileCheckStage6($group, $where); if ($this->echoCLI) { $count = $this->pdo->queryOneRow( sprintf( 'SELECT COUNT(*) AS complete FROM %s c WHERE filecheck = %d %s', $group['cname'], self::COLLFC_COMPPART, $where ) ); $this->pdo->log->doEcho( $this->pdo->log->primary( ($count === false ? 0 : $count['complete']) . ' collections were found to be complete. Time: ' . $this->consoleTools->convertTime(time() - $startTime) ), true ); } } /** * Calculate the total size of a complete collection (COLLFC_COMPPART) in bytes. * Set the collection to (COLLFC_SIZED) * * @param string|int $groupID (optional) * * @void * @access public */ public function processCollectionSizes($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Calculating collection sizes (in bytes).")); } // Get the total size in bytes of the collection for collections where filecheck = 2. $checked = $this->pdo->queryExec( sprintf( 'UPDATE %s c SET filesize = (SELECT COALESCE(SUM(b.partsize), 0) FROM %s b WHERE b.collection_id = c.id), filecheck = %d WHERE c.filecheck = %d AND c.filesize = 0 %s', $group['cname'], $group['bname'], self::COLLFC_SIZED, self::COLLFC_COMPPART, (!empty($groupID) ? ' AND c.group_id = ' . $groupID : ' ') ) ); if ($checked !== false && $this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( $checked->rowCount() . " collections set to filecheck = 3(size calculated)" ) ); $this->pdo->log->doEcho($this->pdo->log->primary($this->consoleTools->convertTime(time() - $startTime)), true); } } /** * Delete unwanted collections based on size/file count using admin settings. * * @param int|string $groupID (optional) * * @void * @access public */ public function deleteUnwantedCollections($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->header( "Process Releases -> Delete collections smaller/larger than minimum size/file count from group/site setting." ) ); } if ($groupID == '') { $groupIDs = $this->groups->getActiveIDs(); } else { $groupIDs = [['id' => $groupID]]; } $minSizeDeleted = $maxSizeDeleted = $minFilesDeleted = 0; $maxSizeSetting = $this->pdo->getSetting('maxsizetoformrelease'); $minSizeSetting = $this->pdo->getSetting('minsizetoformrelease'); $minFilesSetting = $this->pdo->getSetting('minfilestoformrelease'); foreach ($groupIDs as $groupID) { if ($this->pdo->queryOneRow( sprintf( 'SELECT id FROM %s WHERE filecheck = %d AND filesize > 0 AND group_id = %d LIMIT 1', $group['cname'], self::COLLFC_SIZED, $groupID['id'] ) ) !== false ) { $deleteQuery = $this->pdo->queryExec( sprintf(' DELETE c FROM %s c INNER JOIN groups g ON g.id = c.group_id WHERE c.group_id = %d AND c.filecheck = %d AND c.filesize > 0 AND greatest(IFNULL(g.minsizetoformrelease, 0), %d) > 0 AND c.filesize < greatest(IFNULL(g.minsizetoformrelease, 0), %d)', $group['cname'], $groupID['id'], self::COLLFC_SIZED, $minSizeSetting, $minSizeSetting ) ); if ($deleteQuery !== false) { $minSizeDeleted += $deleteQuery->rowCount(); } if ($maxSizeSetting > 0) { $deleteQuery = $this->pdo->queryExec( sprintf(' DELETE FROM %s WHERE filecheck = %d AND group_id = %d AND filesize > %d', $group['cname'], self::COLLFC_SIZED, $groupID['id'], $maxSizeSetting ) ); if ($deleteQuery !== false) { $maxSizeDeleted += $deleteQuery->rowCount(); } } $deleteQuery = $this->pdo->queryExec( sprintf(' DELETE c FROM %s c INNER JOIN groups g ON g.id = c.group_id WHERE c.group_id = %d AND c.filecheck = %d AND greatest(IFNULL(g.minfilestoformrelease, 0), %d) > 0 AND c.totalfiles < greatest(IFNULL(g.minfilestoformrelease, 0), %d)', $group['cname'], $groupID['id'], self::COLLFC_SIZED, $minFilesSetting, $minFilesSetting ) ); if ($deleteQuery !== false) { $minFilesDeleted += $deleteQuery->rowCount(); } } } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( 'Deleted ' . ($minSizeDeleted + $maxSizeDeleted + $minFilesDeleted) . ' collections: ' . PHP_EOL . $minSizeDeleted . ' smaller than, ' . $maxSizeDeleted . ' bigger than, ' . $minFilesDeleted . ' with less files than site/group settings in: ' . $this->consoleTools->convertTime(time() - $startTime) ), true ); } } /** * Create releases from complete collections. * * @param int|string $groupID (optional) * * @return int * @access public */ public function createReleases($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); $categorize = new \Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create releases from complete collections.")); } $this->pdo->ping(true); $collections = $this->pdo->queryDirect( sprintf(' SELECT %s.*, groups.name AS gname FROM %s INNER JOIN groups ON %s.group_id = groups.id WHERE %s %s.filecheck = %d AND filesize > 0 LIMIT %d', $group['cname'], $group['cname'], $group['cname'], (!empty($groupID) ? ' group_id = ' . $groupID . ' AND ' : ' '), $group['cname'], self::COLLFC_SIZED, $this->releaseCreationLimit ) ); if ($this->echoCLI && $collections !== false) { echo $this->pdo->log->primary($collections->rowCount() . " Collections ready to be converted to releases."); } if ($collections instanceof \Traversable) { $preDB = new \PreHash(['Echo' => $this->echoCLI, 'Settings' => $this->pdo]); foreach ($collections as $collection) { $cleanRelName = $this->pdo->escapeString( utf8_encode( str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection['subject']) ) ); $fromName = $this->pdo->escapeString( utf8_encode(trim($collection['fromname'], "'")) ); // Look for duplicates, duplicates match on releases.name, releases.fromname and releases.size // A 1% variance in size is considered the same size when the subject and poster are the same $dupeCheck = $this->pdo->queryOneRow( sprintf(" SELECT id FROM releases WHERE name = %s AND fromname = %s AND size BETWEEN '%s' AND '%s'", $cleanRelName, $fromName, ($collection['filesize'] * .99), ($collection['filesize'] * 1.01) ) ); if ($dupeCheck === false) { $cleanedName = $this->releaseCleaning->releaseCleaner( $collection['subject'], $collection['fromname'], $collection['filesize'], $collection['gname'] ); if (is_array($cleanedName)) { $properName = $cleanedName['properlynamed']; $preID = (isset($cleanerName['predb']) ? $cleanerName['predb'] : false); $isReqID = (isset($cleanerName['requestid']) ? $cleanerName['requestid'] : false); $cleanedName = $cleanedName['cleansubject']; } else { $properName = true; $isReqID = $preID = false; } if ($preID === false && $cleanedName !== '') { // try to match the cleaned searchname to predb title or filename here $preMatch = $preDB->matchPre($cleanedName); if ($preMatch !== false) { $cleanedName = $preMatch['title']; $preID = $preMatch['prehashid']; $properName = true; } } $releaseID = $this->releases->insertRelease( [ 'name' => $cleanRelName, 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), 'totalpart' => $collection['totalfiles'], 'groupid' => $collection['group_id'], 'guid' => $this->pdo->escapeString($this->releases->createGUID($cleanRelName)), 'postdate' => $this->pdo->escapeString($collection['date']), 'fromname' => $fromName, 'size' => $collection['filesize'], 'categoryid' => $categorize->determineCategory($collection['group_id'], $cleanedName), 'isrenamed' => ($properName === true ? 1 : 0), 'reqidstatus' => ($isReqID === true ? 1 : 0), 'prehashid' => ($preID === false ? 0 : $preID), 'nzbstatus' => \NZB::NZB_NONE ] ); if ($releaseID !== false) { // Update collections table to say we inserted the release. $this->pdo->queryExec( sprintf(' UPDATE %s SET filecheck = %d, releaseid = %d WHERE id = %d', $group['cname'], self::COLLFC_INSERTED, $releaseID, $collection['id'] ) ); $returnCount++; if ($this->echoCLI) { echo "Added $returnCount releases.\r"; } } } else { // The release was already in the DB, so delete the collection. $this->pdo->queryExec( sprintf(' DELETE FROM %s WHERE collectionhash = %s', $group['cname'], $this->pdo->escapeString($collection['collectionhash']) ) ); $duplicate++; } } } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( PHP_EOL . number_format($returnCount) . ' Releases added and ' . number_format($duplicate) . ' duplicate collections deleted in ' . $this->consoleTools->convertTime(time() - $startTime) ), true ); } return ['added' => $returnCount, 'dupes' => $duplicate]; } /** * Create NZB files from complete releases. * * @param int|string $groupID (optional) * * @return int * @access public */ public function createNZBs($groupID) { $startTime = time(); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create the NZB, delete collections/binaries/parts.")); } $releases = $this->pdo->queryDirect( sprintf(" SELECT CONCAT(COALESCE(cp.title,'') , CASE WHEN cp.title IS NULL THEN '' ELSE ' > ' END , c.title) AS title, r.name, r.id, r.guid FROM releases r INNER JOIN category c ON r.categoryid = c.id INNER JOIN category cp ON cp.id = c.parentid WHERE %s nzbstatus = 0", (!empty($groupID) ? ' r.groupid = ' . $groupID . ' AND ' : ' ') ) ); $nzbCount = 0; if ($releases && $releases->rowCount()) { $total = $releases->rowCount(); // Init vars for writing the NZB's. $this->nzb->initiateForWrite($groupID); foreach ($releases as $release) { if ($this->nzb->writeNZBforReleaseId($release['id'], $release['guid'], $release['name'], $release['title']) === true) { $nzbCount++; if ($this->echoCLI) { echo $this->pdo->log->primaryOver("Creating NZBs and deleting Collections:\t" . $nzbCount . '/' . $total . "\r"); } } } } $totalTime = (time() - $startTime); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( number_format($nzbCount) . ' NZBs created/Collections deleted in ' . $totalTime . ' seconds.' . PHP_EOL . 'Total time: ' . $this->pdo->log->primary($this->consoleTools->convertTime($totalTime)) ) ); } return $nzbCount; } /** * Process RequestID's. * * @param int|string $groupID * @param int $limit * @param bool $local * * @access public * @void */ public function processRequestIDs($groupID = '', $limit = 5000, $local = true) { if ($local === false && $this->pdo->getSetting('lookup_reqids') == 0) { return; } $startTime = time(); if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->header( sprintf( "Process Releases -> Request id %s lookup -- limit %s", ($local === true ? 'local' : 'web'), $limit ) ) ); } if ($local === true) { $foundRequestIDs = ( new \RequestIDLocal( ['Echo' => $this->echoCLI, 'ConsoleTools' => $this->consoleTools, 'Groups' => $this->groups, 'Settings' => $this->pdo] ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } else { $foundRequestIDs = ( new \RequestIDWeb( ['Echo' => $this->echoCLI, 'ConsoleTools' => $this->consoleTools, 'Groups' => $this->groups, 'Settings' => $this->pdo] ) )->lookupRequestIDs(['GroupID' => $groupID, 'limit' => $limit, 'time' => 168]); } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( number_format($foundRequestIDs) . ' releases updated in ' . $this->consoleTools->convertTime(time() - $startTime) ), true ); } } /** * Categorize releases. * * @param int $categorize * @param int|string $groupID (optional) * * @void * @access public */ public function categorizeReleases($categorize, $groupID = '') { $startTime = time(); if ($this->echoCLI) { echo $this->pdo->log->header("Process Releases -> Categorize releases."); } switch ((int)$categorize) { case 2: $type = 'searchname'; break; case 1: default: $type = 'name'; break; } $this->categorizeRelease( $type, (!empty($groupID) ? 'WHERE iscategorized = 0 AND groupid = ' . $groupID : 'WHERE iscategorized = 0') ); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary($this->consoleTools->convertTime(time() - $startTime)), true); } } /** * Post-process releases. * * @param int $postProcess * @param \NNTP $nntp * * @void * @access public */ public function postProcessReleases($postProcess, &$nntp) { if ($postProcess == 1) { (new PProcess(['Echo' => $this->echoCLI, 'Settings' => $this->pdo, 'Groups' => $this->groups]))->processAll($nntp); } else { if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->info( "\nPost-processing is not running inside the Process Releases class.\n" . "If you are using tmux or screen they might have their own scripts running Post-processing." ) ); } } } /** * Delete collections (complete/incomplete/old/etc). * * @param int|string $groupID (optional) * * @void * @access public */ public function deleteCollections($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); $deletedCount = 0; if ($this->echoCLI) { echo ( $this->pdo->log->header("Process Releases -> Delete finished collections." . PHP_EOL) . $this->pdo->log->primary('Deleting old collections/binaries/parts.') ); } $deleted = 0; // CBP older than retention. $deleteQuery = $this->pdo->queryExec( sprintf( 'DELETE FROM %s WHERE dateadded < (NOW() - INTERVAL %d HOUR) %s', $group['cname'], $this->pdo->getSetting('partretentionhours'), (!empty($groupID) && $this->tablePerGroup === false ? ' AND group_id = ' . $groupID : '') ) ); if ($deleteQuery !== false) { $deleted = $deleteQuery->rowCount(); $deletedCount += $deleted; } $firstQuery = time(); if ($this->echoCLI) { echo $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' old collections/binaries/parts in ' . ($firstQuery - $startTime) . ' seconds.' . PHP_EOL . 'Deleting binaries/parts with no collections.' ); } $deleted = 0; // Binaries/parts that somehow have no collection. $deleteQuery = $this->pdo->queryExec( sprintf( 'DELETE %s, %s FROM %s, %s WHERE %s.collection_id = 0 AND %s.id = %s.binaryid', $group['bname'], $group['pname'], $group['bname'], $group['pname'], $group['bname'], $group['bname'], $group['pname'] ) ); if ($deleteQuery !== false) { $deleted = $deleteQuery->rowCount(); $deletedCount += $deleted; } $secondQuery = time(); if ($this->echoCLI) { echo $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' binaries/parts with no collections in ' . ($secondQuery - $firstQuery) . ' seconds.' . PHP_EOL . 'Deleting parts with no binaries.' ); } $deleted = 0; // Parts that somehow have no binaries. Don't delete parts currently inserting, by checking the max id. if (mt_rand(0, 100) <= 5) { $deleteQuery = $this->pdo->queryExec( sprintf( 'DELETE FROM %s WHERE binaryid NOT IN (SELECT id FROM %s) %s', $group['pname'], $group['bname'], $this->minMaxQueryFormulator($group['pname'], 40000) ) ); if ($deleteQuery !== false) { $deleted = $deleteQuery->rowCount(); $deletedCount += $deleted; } } $thirdQuery = time(); if ($this->echoCLI) { echo $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' parts with no binaries in ' . ($thirdQuery - $secondQuery) . ' seconds.' . PHP_EOL . 'Deleting binaries with no collections.' ); } $deleted = 0; // Binaries that somehow have no collection. Don't delete currently inserting binaries by checking the max id. $deleteQuery = $this->pdo->queryExec( sprintf( 'DELETE FROM %s WHERE collection_id NOT IN (SELECT id FROM %s) %s', $group['bname'], $group['cname'], $this->minMaxQueryFormulator($group['bname'], 20000) ) ); if ($deleteQuery !== false) { $deleted = $deleteQuery->rowCount(); $deletedCount += $deleted; } $fourthQuery = time(); if ($this->echoCLI) { echo $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' binaries with no collections in ' . ($fourthQuery - $thirdQuery) . ' seconds.' . PHP_EOL . 'Deleting collections with no binaries.' ); } $deleted = 0; // Collections that somehow have no binaries. $collectionIDs = $this->pdo->queryDirect( sprintf( 'SELECT id FROM %s WHERE id NOT IN (SELECT collection_id FROM %s) %s', $group['cname'], $group['bname'], $this->minMaxQueryFormulator($group['cname'], 10000) ) ); if ($collectionIDs instanceof \Traversable) { foreach ($collectionIDs as $collectionID) { $deleted++; $this->pdo->queryExec(sprintf('DELETE FROM %s WHERE id = %d', $group['cname'], $collectionID['id'])); } $deletedCount += $deleted; } $fifthQuery = time(); if ($this->echoCLI) { echo $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' collections with no binaries in ' . ($fifthQuery - $fourthQuery) . ' seconds.' . PHP_EOL . 'Deleting collections that were missed after NZB creation.' ); } $deleted = 0; // Collections that were missing on NZB creation. $collections = $this->pdo->queryDirect( sprintf(' SELECT c.id FROM %s c INNER JOIN releases r ON r.id = c.releaseid WHERE r.nzbstatus = 1', $group['cname'] ) ); if ($collections instanceof \Traversable) { foreach($collections as $collection) { $deleted++; $this->pdo->queryExec( sprintf(' DELETE FROM %s WHERE id = %d', $group['cname'], $collection['id'] ) ); } $deletedCount += $deleted; } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( 'Finished deleting ' . $deleted . ' collections missed after NZB creation in ' . (time() - $fifthQuery) . ' seconds.' . PHP_EOL . 'Removed ' . number_format($deletedCount) . ' parts/binaries/collection rows in ' . $this->consoleTools->convertTime(($fifthQuery - $startTime)) . PHP_EOL ) ); } } /** * Delete unwanted releases based on admin settings. * This deletes releases based on group. * * @param int|string $groupID (optional) * * @void * @access public */ public function deletedReleasesByGroup($groupID = '') { $startTime = time(); $minSizeDeleted = $maxSizeDeleted = $minFilesDeleted = 0; if ($this->echoCLI) { echo $this->pdo->log->header("Process Releases -> Delete releases smaller/larger than minimum size/file count from group/site setting."); } if ($groupID == '') { $groupIDs = $this->groups->getActiveIDs(); } else { $groupIDs = [['id' => $groupID]]; } $maxSizeSetting = $this->pdo->getSetting('maxsizetoformrelease'); $minSizeSetting = $this->pdo->getSetting('minsizetoformrelease'); $minFilesSetting = $this->pdo->getSetting('minfilestoformrelease'); foreach ($groupIDs as $groupID) { $releases = $this->pdo->queryDirect( sprintf(" SELECT r.guid, r.id FROM releases r INNER JOIN groups g ON g.id = r.groupid WHERE r.groupid = %d AND greatest(IFNULL(g.minsizetoformrelease, 0), %d) > 0 AND r.size < greatest(IFNULL(g.minsizetoformrelease, 0), %d)", $groupID['id'], $minSizeSetting, $minSizeSetting ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $minSizeDeleted++; } } if ($maxSizeSetting > 0) { $releases = $this->pdo->queryDirect( sprintf(' SELECT id, guid FROM releases WHERE groupid = %d AND size > %d', $groupID['id'], $maxSizeSetting ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $maxSizeDeleted++; } } } $releases = $this->pdo->queryDirect( sprintf(" SELECT r.id, r.guid FROM releases r INNER JOIN groups g ON g.id = r.groupid WHERE r.groupid = %d AND greatest(IFNULL(g.minfilestoformrelease, 0), %d) > 0 AND r.totalpart < greatest(IFNULL(g.minfilestoformrelease, 0), %d)", $groupID['id'], $minFilesSetting, $minFilesSetting ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $minFilesDeleted++; } } } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( 'Deleted ' . ($minSizeDeleted + $maxSizeDeleted + $minFilesDeleted) . ' releases: ' . PHP_EOL . $minSizeDeleted . ' smaller than, ' . $maxSizeDeleted . ' bigger than, ' . $minFilesDeleted . ' with less files than site/groups setting in: ' . $this->consoleTools->convertTime(time() - $startTime) ), true ); } } /** * Delete releases using admin settings. * This deletes releases, regardless of group. * * @void * @access public */ public function deleteReleases() { $startTime = time(); $category = new \Category(['Settings' => $this->pdo]); $genres = new \Genres(['Settings' => $this->pdo]); $passwordDeleted = $duplicateDeleted = $retentionDeleted = $completionDeleted = $disabledCategoryDeleted = 0; $disabledGenreDeleted = $miscRetentionDeleted = $miscHashedDeleted = $categoryMinSizeDeleted = 0; // Delete old releases and finished collections. if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Delete old releases and passworded releases.")); } // Releases past retention. if ($this->pdo->getSetting('releaseretentiondays') != 0) { $releases = $this->pdo->queryDirect( sprintf( 'SELECT id, guid FROM releases WHERE postdate < (NOW() - INTERVAL %d DAY)', $this->pdo->getSetting('releaseretentiondays') ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $retentionDeleted++; } } } // Passworded releases. if ($this->pdo->getSetting('deletepasswordedrelease') == 1) { $releases = $this->pdo->queryDirect( sprintf( 'SELECT id, guid FROM releases WHERE passwordstatus = %d', \Releases::PASSWD_RAR ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $passwordDeleted++; } } } // Possibly passworded releases. if ($this->pdo->getSetting('deletepossiblerelease') == 1) { $releases = $this->pdo->queryDirect( sprintf( 'SELECT id, guid FROM releases WHERE passwordstatus = %d', \Releases::PASSWD_POTENTIAL ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $passwordDeleted++; } } } if ($this->crossPostTime != 0) { // Crossposted releases. do { $releases = $this->pdo->queryDirect( sprintf( 'SELECT id, guid FROM releases WHERE adddate > (NOW() - INTERVAL %d HOUR) GROUP BY name HAVING COUNT(name) > 1', $this->crossPostTime ) ); $total = 0; if ($releases && $releases->rowCount()) { $total = $releases->rowCount(); foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $duplicateDeleted++; } } } while ($total > 0); } if ($this->completion > 0) { $releases = $this->pdo->queryDirect( sprintf('SELECT id, guid FROM releases WHERE completion < %d AND completion > 0', $this->completion) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $completionDeleted++; } } } // Disabled categories. $disabledCategories = $category->getDisabledIDs(); if (count($disabledCategories) > 0) { foreach ($disabledCategories as $disabledCategory) { $releases = $this->pdo->queryDirect( sprintf('SELECT id, guid FROM releases WHERE categoryid = %d', $disabledCategory['id']) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $disabledCategoryDeleted++; $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); } } } } // Delete smaller than category minimum sizes. $categories = $this->pdo->queryDirect(' SELECT c.id AS id, CASE WHEN c.minsizetoformrelease = 0 THEN cp.minsizetoformrelease ELSE c.minsizetoformrelease END AS minsize FROM category c INNER JOIN category cp ON cp.id = c.parentid WHERE c.parentid IS NOT NULL' ); if ($categories instanceof \Traversable) { foreach ($categories as $category) { if ($category['minsize'] > 0) { $releases = $this->pdo->queryDirect( sprintf(' SELECT r.id, r.guid FROM releases r WHERE r.categoryid = %d AND r.size < %d', $category['id'], $category['minsize'] ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $categoryMinSizeDeleted++; } } } } } // Disabled music genres. $genrelist = $genres->getDisabledIDs(); if (count($genrelist) > 0) { foreach ($genrelist as $genre) { $releases = $this->pdo->queryDirect( sprintf(' SELECT id, guid FROM releases INNER JOIN (SELECT id AS mid FROM musicinfo WHERE musicinfo.genreID = %d) mi ON musicinfoid = mid', $genre['id'] ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $disabledGenreDeleted++; $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); } } } } // Misc other. if ($this->pdo->getSetting('miscotherretentionhours') > 0) { $releases = $this->pdo->queryDirect( sprintf(' SELECT id, guid FROM releases WHERE categoryid = %d AND adddate <= NOW() - INTERVAL %d HOUR', \Category::CAT_MISC_OTHER, $this->pdo->getSetting('miscotherretentionhours') ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $miscRetentionDeleted++; } } } // Misc hashed. if ($this->pdo->getSetting('mischashedretentionhours') > 0) { $releases = $this->pdo->queryDirect( sprintf(' SELECT id, guid FROM releases WHERE categoryid = %d AND adddate <= NOW() - INTERVAL %d HOUR', \Category::CAT_MISC_HASHED, $this->pdo->getSetting('mischashedretentionhours') ) ); if ($releases instanceof \Traversable) { foreach ($releases as $release) { $this->releases->deleteSingle(['g' => $release['guid'], 'i' => $release['id']], $this->nzb, $this->releaseImage); $miscHashedDeleted++; } } } if ($this->echoCLI) { $this->pdo->log->doEcho( $this->pdo->log->primary( 'Removed releases: ' . number_format($retentionDeleted) . ' past retention, ' . number_format($passwordDeleted) . ' passworded, ' . number_format($duplicateDeleted) . ' crossposted, ' . number_format($disabledCategoryDeleted) . ' from disabled categories, ' . number_format($categoryMinSizeDeleted) . ' smaller than category settings, ' . number_format($disabledGenreDeleted) . ' from disabled music genres, ' . number_format($miscRetentionDeleted) . ' from misc->other' . number_format($miscHashedDeleted) . ' from misc->hashed' . ($this->completion > 0 ? ', ' . number_format($completionDeleted) . ' under ' . $this->completion . '% completion.' : '.' ) ) ); $totalDeleted = ( $retentionDeleted + $passwordDeleted + $duplicateDeleted + $disabledCategoryDeleted + $disabledGenreDeleted + $miscRetentionDeleted + $miscHashedDeleted + $completionDeleted + $categoryMinSizeDeleted ); if ($totalDeleted > 0) { $this->pdo->log->doEcho( $this->pdo->log->primary( "Removed " . number_format($totalDeleted) . ' releases in ' . $this->consoleTools->convertTime(time() - $startTime) ) ); } } } /** * Formulate part of a query to prevent deletion of currently inserting parts / binaries / collections. * * @param string $groupName * @param int $difference * * @return string * @access private */ private function minMaxQueryFormulator($groupName, $difference) { $minMaxId = $this->pdo->queryOneRow(sprintf('SELECT MIN(id) AS min, MAX(id) AS max FROM %s', $groupName)); if ($minMaxId === false) { $minMaxId = ''; } else { $minMaxId = ' AND id < ' . ((($minMaxId['max'] - $minMaxId['min']) >= $difference) ? ($minMaxId['max'] - $difference) : 1); } return $minMaxId; } /** * Look if we have all the files in a collection (which have the file count in the subject). * Set file check to complete. * This means the the binary table has the same count as the file count in the subject, but * the collection might not be complete yet since we might not have all the articles in the parts table. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage1(array &$group, &$where) { $this->pdo->queryExec( sprintf(' UPDATE %s c INNER JOIN (SELECT c.id FROM %s c INNER JOIN %s b ON b.collection_id = c.id WHERE c.totalfiles > 0 AND c.filecheck = %d %s GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) IN (c.totalfiles, c.totalfiles + 1) ) r ON c.id = r.id SET filecheck = %d', $group['cname'], $group['cname'], $group['bname'], self::COLLFC_DEFAULT, $where, self::COLLFC_COMPCOLL ) ); } /** * The first query sets filecheck to COLLFC_ZEROPART if there's a file that starts with 0 (ex. [00/100]). * The second query sets filecheck to COLLFC_TEMPCOMP on everything left over, so anything that starts with 1 (ex. [01/100]). * * This is done because some collections start at 0 and some at 1, so if you were to assume the collection is complete * at 0 then you would never get a complete collection if it starts with 1 and if it starts, you can end up creating * a incomplete collection, since you assumed it was complete. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage2(array &$group, &$where) { $this->pdo->queryExec( sprintf(' UPDATE %s c INNER JOIN (SELECT c.id FROM %s c INNER JOIN %s b ON b.collection_id = c.id WHERE b.filenumber = 0 AND c.totalfiles > 0 AND c.filecheck = %d %s GROUP BY c.id ) r ON c.id = r.id SET c.filecheck = %d', $group['cname'], $group['cname'], $group['bname'], self::COLLFC_COMPCOLL, $where, self::COLLFC_ZEROPART ) ); $this->pdo->queryExec( sprintf(' UPDATE %s c SET filecheck = %d WHERE filecheck = %d %s', $group['cname'], self::COLLFC_TEMPCOMP, self::COLLFC_COMPCOLL, $where ) ); } /** * Check if the files (binaries table) in a complete collection has all the parts. * If we have all the parts, set binaries table partcheck to FILE_COMPLETE. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage3(array &$group, $where) { $this->pdo->queryExec( sprintf(' UPDATE %s b INNER JOIN (SELECT b.id FROM %s b INNER JOIN %s c ON c.id = b.collection_id WHERE c.filecheck = %d AND b.partcheck = %d %s AND b.currentparts = b.totalparts GROUP BY b.id, b.totalparts) r ON b.id = r.id SET b.partcheck = %d', $group['bname'], $group['bname'], $group['cname'], self::COLLFC_TEMPCOMP, self::FILE_INCOMPLETE, $where, self::FILE_COMPLETE ) ); $this->pdo->queryExec( sprintf(' UPDATE %s b INNER JOIN (SELECT b.id FROM %s b INNER JOIN %s c ON c.id = b.collection_id WHERE c.filecheck = %d AND b.partcheck = %d %s AND b.currentparts >= (b.totalparts + 1) GROUP BY b.id, b.totalparts) r ON b.id = r.id SET b.partcheck = %d', $group['bname'], $group['bname'], $group['cname'], self::COLLFC_ZEROPART, self::FILE_INCOMPLETE, $where, self::FILE_COMPLETE ) ); } /** * Check if all files (binaries table) for a collection are complete (if they all have the "parts"). * Set collections filecheck column to COLLFC_COMPPART. * This means the collection is complete. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage4(array &$group, &$where) { $this->pdo->queryExec( sprintf(' UPDATE %s c INNER JOIN (SELECT c.id FROM %s c INNER JOIN %s b ON c.id = b.collection_id WHERE b.partcheck = 1 AND c.filecheck IN (%d, %d) %s GROUP BY b.collection_id, c.totalfiles, c.id HAVING COUNT(b.id) >= c.totalfiles) r ON c.id = r.id SET filecheck = %d', $group['cname'], $group['cname'], $group['bname'], self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART, $where, self::COLLFC_COMPPART ) ); } /** * If not all files (binaries table) had their parts on the previous stage, * reset the collection filecheck column to COLLFC_COMPCOLL so we reprocess them next time. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage5(array &$group, &$where) { $this->pdo->queryExec( sprintf(' UPDATE %s c SET filecheck = %d WHERE filecheck IN (%d, %d) %s', $group['cname'], self::COLLFC_COMPCOLL, self::COLLFC_TEMPCOMP, self::COLLFC_ZEROPART, $where ) ); } /** * If a collection did not have the file count (ie: [00/12]) or the collection is incomplete after * $this->collectionDelayTime hours, set the collection to complete to create it into a release/nzb. * * @param array $group * @param string $where * * @void * @access private */ private function collectionFileCheckStage6(array &$group, &$where) { $this->pdo->queryExec( sprintf(" UPDATE %s c SET filecheck = %d, totalfiles = (SELECT COUNT(b.id) FROM %s b WHERE b.collection_id = c.id) WHERE c.dateadded < NOW() - INTERVAL '%d' HOUR AND c.filecheck IN (%d, %d, 10) %s", $group['cname'], self::COLLFC_COMPPART, $group['bname'], $this->collectionDelayTime, self::COLLFC_DEFAULT, self::COLLFC_COMPCOLL, $where ) ); } /** * If a collection has been stuck for $this->collectionTimeout hours, force it to become a release. * * @param array $group * @param string $where * * @void * @access private */ private function processStuckCollections(array $group, $where) { $this->pdo->queryExec( sprintf(" UPDATE %s c SET c.filecheck = %d WHERE c.added < DATE_SUB((SELECT value FROM settings WHERE setting = 'last_run_time'), INTERVAL %d HOUR) %s", $group['cname'], self::COLLFC_COMPCOLL, $this->collectionTimeout, $where ) ); } }