Add last rar/zip file checking support.

newznab/build/newznab.xml

@@ -2,8 +2,8 @@
 <newznab>
 	<versions>
 		<sql>
-			<db>158</db>
-			<file>158</file>
+			<db>159</db>
+			<file>159</file>
 		</sql>
 		<git>
 			<tag>0.4.1</tag>

newznab/controllers/NZB.php

@@ -324,16 +324,25 @@ class NZB
 	 * Retrieve various information on a NZB file (the subject, # of pars,
 	 * file extensions, file sizes, file completion, group names, # of parts).
 	 *
-	 * @param string $nzb The NZB contents in a string.
+	 * @param string	$nzb		The NZB contents in a string.
+	 * @param array		$options
+	 * 					'no-file-key'	=> True - use numeric array key; False - Use filename as array key.
+	 * 					'strip-count'	=> True - Strip file/part count from file name to make the array key; False - Leave file name as is.
 	 *
 	 * @return array $result Empty if not an NZB or the contents of the NZB.
 	 *
 	 * @access public
 	 */
-	public function nzbFileList($nzb)
+	public function nzbFileList($nzb, array $options = null)
 	{
+		$defaults = [
+			'no-file-key' => true,
+			'strip-count' => false,
+		];
+		$options += $defaults;
+
 		$num_pars = $i = 0;
-		$result = array();
+		$result = [];
 
 		if (!$nzb) {
 			return $result;
@@ -353,6 +362,20 @@ class NZB
 				$num_pars++;
 			}
 
+			if ($options['no-file-key'] == false) {
+				$i = $title;
+				if ($options['strip-count']) {
+					// Strip file / part count to get proper sorting.
+					$i = preg_replace('#\d+[- ._]?(/|\||[o0]f)[- ._]?\d+?(?![- ._]\d)#i', '', $i);
+					// Change .rar and .par2 to be sorted before .part0x.rar and .volxxx+xxx.par2
+					if (strpos($i, '.par2') !== false && !preg_match('#\.vol\d+\+\d+\.par2#i', $i)) {
+						$i = str_replace('.par2', '.vol0.par2', $i);
+					} else if (preg_match('#\.rar[^a-z0-9]#i', $i) && !preg_match('#\.part\d+\.rar#i', $i)) {
+						$i = preg_replace('#\.rar(?:[^a-z0-9])#i', '.part0.rar', $i);
+					}
+				}
+			}
+
 			$result[$i]['title'] = $title;
 
 			// Extensions.
@@ -381,12 +404,12 @@ class NZB
 
 			// Parts.
 			if (!isset($result[$i]['segments'])) {
-				$result[$i]['segments'] = array();
+				$result[$i]['segments'] = [];
 			}
 
 			// File size.
 			foreach ($file->segments->segment as $segment) {
-				array_push($result[$i]['segments'], (string) $segment);
+				array_push($result[$i]['segments'], (string)$segment);
 				$fileSize += $segment->attributes()->bytes;
 				$numSegments++;
 			}
@@ -400,14 +423,16 @@ class NZB
 
 			// Groups.
 			if (!isset($result[$i]['groups'])) {
-				$result[$i]['groups'] = array();
+				$result[$i]['groups'] = [];
 			}
 			foreach ($file->groups->group as $g) {
-				array_push($result[$i]['groups'], (string) $g);
+				array_push($result[$i]['groups'], (string)$g);
 			}
 
 			unset($result[$i]['segments']['@attributes']);
-			$i++;
+			if ($options['no-file-key']) {
+				$i++;
+			}
 		}
 		return $result;
 	}

newznab/db/DbUpdate.php

@@ -180,7 +180,7 @@ class DbUpdate
 					rename($matches[0], $newName);
 					$this->git->add($newName);
 					if ($this->git->isCommited($this->git->getBranch() . ':' . $matches[0])) {
-						$this->git->rm("{$matches[0]}"); // remove old filename from the index.
+						$this->git->add(" -u {$matches[0]}"); // remove old filename from the index.
 					}
 				}
 			}

newznab/processing/post/ProcessAdditional.php

@@ -608,6 +608,11 @@ class ProcessAdditional
 					// Download the RARs/ZIPs, extract the files inside them and insert the file info into the DB.
 					$this->_processNZBCompressedFiles();
 
+					// Download rar/zip in reverse order, to get the last rar or zip file.
+					if ($this->pdo->getSetting('fetchlastcompressedfiles') == 1) {
+						$this->_processNZBCompressedFiles(true);
+					}
+
 					if ($this->_releaseHasPassword === false) {
 						// Process the extracted files to get video/audio samples/etc.
 						$this->_processExtractedFiles();
@@ -708,8 +713,8 @@ class ProcessAdditional
 			return $this->_decrementPasswordStatus();
 		}
 
-		// Sort the files inside the NZB.
-		usort($this->_nzbContents, ['\newznab\processing\post\ProcessAdditional', '_sortNZB']);
+		// Sort keys.
+		ksort($this->_nzbContents, SORT_NATURAL);
 
 		return true;
 	}
@@ -763,7 +768,7 @@ class ProcessAdditional
 			// Check if it's a rar/zip.
 			if ($this->_NZBHasCompressedFile === false &&
 				preg_match(
-					'/\.(part0*1|part0+|r0+|r0*1|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
+					'/\.(part\d+|r\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
 					$this->_currentNZBFile['title']
 				)
 			) {
@@ -844,13 +849,30 @@ class ProcessAdditional
 	}
 
 	/**
-	 * Process the NZB contents, find RAR/ZIP files, download them and extract them.
+	 * List of message-id's we have tried for rar/zip files.
+	 * @var array
 	 */
-	protected function _processNZBCompressedFiles()
+	protected $_triedCompressedMids = [];
+
+	/**
+	 * Process the NZB contents, find RAR/ZIP files, download them and extract them.
+	 *
+	 * @param bool $reverse Reverse sort $this->_nzbContents ? - To find the largest rar / zip file first.
+	 */
+	protected function _processNZBCompressedFiles($reverse = false)
 	{
+		if ($reverse) {
+			if (!krsort($this->_nzbContents)) {
+				return;
+			}
+		} else {
+			$this->_triedCompressedMids = [];
+		}
+
 		$failed = $downloaded = 0;
 		// Loop through the files, attempt to find if password-ed and files. Starting with what not to process.
 		foreach ($this->_nzbContents as $nzbFile) {
+			// TODO change this to max calculated size, as segments vary in size greatly.
 			if ($downloaded >= $this->_maximumRarSegments) {
 				break;
 			} else if ($failed >= $this->_maximumRarPasswordChecks) {
@@ -864,7 +886,7 @@ class ProcessAdditional
 
 			// Probably not a rar/zip.
 			if (!preg_match(
-				'/\.(part0*1|part0+|r0+|r0*1|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
+				'/\.(part\d+|r\d+|rar|0+|0*10?|zipr\d{2,3}|zipx?)(\s*\.rar)*($|[ ")\]-])|"[a-f0-9]{32}\.[1-9]\d{1,2}".*\(\d+\/\d{2,}\)$/i',
 				$nzbFile['title']
 			)
 			) {
@@ -873,12 +895,23 @@ class ProcessAdditional
 
 			// Get message-id's for the rar file.
 			$segCount = (count($nzbFile['segments']) - 1);
-			$mID = array();
+			$mID = [];
 			for ($i = 0; $i < $this->_maximumRarSegments; $i++) {
 				if ($i > $segCount) {
 					break;
 				}
-				$mID[] = (string)$nzbFile['segments'][$i];
+				$segment = (string)$nzbFile['segments'][$i];
+				if (!$reverse) {
+					$this->_triedCompressedMids[] = $segment;
+				} else if (in_array($segment, $this->_triedCompressedMids)) {
+					// We already downloaded this file.
+					continue 2;
+				}
+				$mID[] = $segment;
+			}
+			// Nothing to download.
+			if (empty($mID)) {
+				continue;
 			}
 
 			// Download the article(s) from usenet.

resources/db/patches/0159~settings.sql

@@ -0,0 +1,9 @@
+INSERT IGNORE INTO settings (section, subsection, name, value, hint, setting)
+VALUES (
+'archive',
+'fetch',
+'end',
+0,
+'Try to download the last rar or zip file? (This is good if most of the files are at the end.) Note: The first rar/zip is still downloaded.',
+'fetchlastcompressedfiles'
+);

www/templates/nntmux/views/admin/site-edit.tpl

@@ -829,6 +829,14 @@
 		<div class="hint">Whether to attempt to peek into every release, to see if rar files are password protected.<br/></div>
 	</td>
 </tr>
+	<tr>
+		<td style="width:180px;"><label for="fetchlastcompressedfiles">Download last compressed file:</label></td>
+		<td>
+			{html_radios id="fetchlastcompressedfiles" name='fetchlastcompressedfiles' values=$yesno_ids output=$yesno_names selected=$fsite->fetchlastcompressedfiles separator='<br />'}
+			<div class="hint">Try to download the last rar or zip file? (This is good if most of the files are at the end.) Note: The first rar/zip is still downloaded.
+			</div>
+		</td>
+	</tr>
 
 <tr>
 	<td><label for="deletepasswordedrelease">Delete Passworded Releases</label>:</td>