From 9bb992c62c92024df52776e8d7ad1d719d2fd91b Mon Sep 17 00:00:00 2001 From: Darko Date: Fri, 19 Jun 2015 08:59:30 +0200 Subject: [PATCH] Add last rar/zip file checking support. --- newznab/build/newznab.xml | 4 +- newznab/controllers/NZB.php | 41 +++++++++++++--- newznab/db/DbUpdate.php | 2 +- newznab/processing/post/ProcessAdditional.php | 49 ++++++++++++++++--- resources/db/patches/0159~settings.sql | 9 ++++ .../nntmux/views/admin/site-edit.tpl | 8 +++ 6 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 resources/db/patches/0159~settings.sql diff --git a/newznab/build/newznab.xml b/newznab/build/newznab.xml index f7a4bce8b..882d95875 100644 --- a/newznab/build/newznab.xml +++ b/newznab/build/newznab.xml @@ -2,8 +2,8 @@ - 158 - 158 + 159 + 159 0.4.1 diff --git a/newznab/controllers/NZB.php b/newznab/controllers/NZB.php index ffe2b0d56..a49a30874 100644 --- a/newznab/controllers/NZB.php +++ b/newznab/controllers/NZB.php @@ -324,16 +324,25 @@ class NZB * Retrieve various information on a NZB file (the subject, # of pars, * file extensions, file sizes, file completion, group names, # of parts). * - * @param string $nzb The NZB contents in a string. + * @param string $nzb The NZB contents in a string. + * @param array $options + * 'no-file-key' => True - use numeric array key; False - Use filename as array key. + * 'strip-count' => True - Strip file/part count from file name to make the array key; False - Leave file name as is. * * @return array $result Empty if not an NZB or the contents of the NZB. * * @access public */ - public function nzbFileList($nzb) + public function nzbFileList($nzb, array $options = null) { + $defaults = [ + 'no-file-key' => true, + 'strip-count' => false, + ]; + $options += $defaults; + $num_pars = $i = 0; - $result = array(); + $result = []; if (!$nzb) { return $result; @@ -353,6 +362,20 @@ class NZB $num_pars++; } + if ($options['no-file-key'] == false) { + $i = $title; + if ($options['strip-count']) { + // Strip file / part count to get proper sorting. + $i = preg_replace('#\d+[- ._]?(/|\||[o0]f)[- ._]?\d+?(?![- ._]\d)#i', '', $i); + // Change .rar and .par2 to be sorted before .part0x.rar and .volxxx+xxx.par2 + if (strpos($i, '.par2') !== false && !preg_match('#\.vol\d+\+\d+\.par2#i', $i)) { + $i = str_replace('.par2', '.vol0.par2', $i); + } else if (preg_match('#\.rar[^a-z0-9]#i', $i) && !preg_match('#\.part\d+\.rar#i', $i)) { + $i = preg_replace('#\.rar(?:[^a-z0-9])#i', '.part0.rar', $i); + } + } + } + $result[$i]['title'] = $title; // Extensions. @@ -381,12 +404,12 @@ class NZB // Parts. if (!isset($result[$i]['segments'])) { - $result[$i]['segments'] = array(); + $result[$i]['segments'] = []; } // File size. foreach ($file->segments->segment as $segment) { - array_push($result[$i]['segments'], (string) $segment); + array_push($result[$i]['segments'], (string)$segment); $fileSize += $segment->attributes()->bytes; $numSegments++; } @@ -400,14 +423,16 @@ class NZB // Groups. if (!isset($result[$i]['groups'])) { - $result[$i]['groups'] = array(); + $result[$i]['groups'] = []; } foreach ($file->groups->group as $g) { - array_push($result[$i]['groups'], (string) $g); + array_push($result[$i]['groups'], (string)$g); } unset($result[$i]['segments']['@attributes']); - $i++; + if ($options['no-file-key']) { + $i++; + } } return $result; } diff --git a/newznab/db/DbUpdate.php b/newznab/db/DbUpdate.php index c7e51a0ad..249cbc8a5 100644 --- a/newznab/db/DbUpdate.php +++ b/newznab/db/DbUpdate.php @@ -180,7 +180,7 @@ class DbUpdate rename($matches[0], $newName); $this->git->add($newName); if ($this->git->isCommited($this->git->getBranch() . ':' . $matches[0])) { - $this->git->rm("{$matches[0]}"); // remove old filename from the index. + $this->git->add(" -u {$matches[0]}"); // remove old filename from the index. } } } diff --git a/newznab/processing/post/ProcessAdditional.php b/newznab/processing/post/ProcessAdditional.php index 912715ce7..64859acb9 100644 --- a/newznab/processing/post/ProcessAdditional.php +++ b/newznab/processing/post/ProcessAdditional.php @@ -608,6 +608,11 @@ class ProcessAdditional // Download the RARs/ZIPs, extract the files inside them and insert the file info into the DB. $this->_processNZBCompressedFiles(); + // Download rar/zip in reverse order, to get the last rar or zip file. + if ($this->pdo->getSetting('fetchlastcompressedfiles') == 1) { + $this->_processNZBCompressedFiles(true); + } + if ($this->_releaseHasPassword === false) { // Process the extracted files to get video/audio samples/etc. $this->_processExtractedFiles(); @@ -708,8 +713,8 @@ class ProcessAdditional return $this->_decrementPasswordStatus(); } - // Sort the files inside the NZB. - usort($this->_nzbContents, ['\newznab\processing\post\ProcessAdditional', '_sortNZB']); + // Sort keys. + ksort($this->_nzbContents, SORT_NATURAL); return true; } @@ -763,7 +768,7 @@ class ProcessAdditional // Check if it's a rar/zip. if ($this->_NZBHasCompressedFile === false && preg_match( - '/\.(part0*1|part0+|r0+|r0*1|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i', + '/\.(part\d+|r\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i', $this->_currentNZBFile['title'] ) ) { @@ -844,13 +849,30 @@ class ProcessAdditional } /** - * Process the NZB contents, find RAR/ZIP files, download them and extract them. + * List of message-id's we have tried for rar/zip files. + * @var array */ - protected function _processNZBCompressedFiles() + protected $_triedCompressedMids = []; + + /** + * Process the NZB contents, find RAR/ZIP files, download them and extract them. + * + * @param bool $reverse Reverse sort $this->_nzbContents ? - To find the largest rar / zip file first. + */ + protected function _processNZBCompressedFiles($reverse = false) { + if ($reverse) { + if (!krsort($this->_nzbContents)) { + return; + } + } else { + $this->_triedCompressedMids = []; + } + $failed = $downloaded = 0; // Loop through the files, attempt to find if password-ed and files. Starting with what not to process. foreach ($this->_nzbContents as $nzbFile) { + // TODO change this to max calculated size, as segments vary in size greatly. if ($downloaded >= $this->_maximumRarSegments) { break; } else if ($failed >= $this->_maximumRarPasswordChecks) { @@ -864,7 +886,7 @@ class ProcessAdditional // Probably not a rar/zip. if (!preg_match( - '/\.(part0*1|part0+|r0+|r0*1|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i', + '/\.(part\d+|r\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i', $nzbFile['title'] ) ) { @@ -873,12 +895,23 @@ class ProcessAdditional // Get message-id's for the rar file. $segCount = (count($nzbFile['segments']) - 1); - $mID = array(); + $mID = []; for ($i = 0; $i < $this->_maximumRarSegments; $i++) { if ($i > $segCount) { break; } - $mID[] = (string)$nzbFile['segments'][$i]; + $segment = (string)$nzbFile['segments'][$i]; + if (!$reverse) { + $this->_triedCompressedMids[] = $segment; + } else if (in_array($segment, $this->_triedCompressedMids)) { + // We already downloaded this file. + continue 2; + } + $mID[] = $segment; + } + // Nothing to download. + if (empty($mID)) { + continue; } // Download the article(s) from usenet. diff --git a/resources/db/patches/0159~settings.sql b/resources/db/patches/0159~settings.sql new file mode 100644 index 000000000..f3942f8f5 --- /dev/null +++ b/resources/db/patches/0159~settings.sql @@ -0,0 +1,9 @@ +INSERT IGNORE INTO settings (section, subsection, name, value, hint, setting) +VALUES ( +'archive', +'fetch', +'end', +0, +'Try to download the last rar or zip file? (This is good if most of the files are at the end.) Note: The first rar/zip is still downloaded.', +'fetchlastcompressedfiles' +); \ No newline at end of file diff --git a/www/templates/nntmux/views/admin/site-edit.tpl b/www/templates/nntmux/views/admin/site-edit.tpl index 0c31c3ac0..dd5fdb46c 100644 --- a/www/templates/nntmux/views/admin/site-edit.tpl +++ b/www/templates/nntmux/views/admin/site-edit.tpl @@ -829,6 +829,14 @@
Whether to attempt to peek into every release, to see if rar files are password protected.
+ + + + {html_radios id="fetchlastcompressedfiles" name='fetchlastcompressedfiles' values=$yesno_ids output=$yesno_names selected=$fsite->fetchlastcompressedfiles separator='
'} +
Try to download the last rar or zip file? (This is good if most of the files are at the end.) Note: The first rar/zip is still downloaded. +
+ + :