diff --git a/bin/monitor.php b/bin/monitor.php
index 7346bd396..e51b068a0 100644
--- a/bin/monitor.php
+++ b/bin/monitor.php
@@ -8,7 +8,7 @@ require_once(WWW_DIR . "/lib/showsleep.php");
require_once(dirname(__FILE__) . "/../lib/functions.php");
-$version = "0.5r0006";
+$version = "0.5r0010";
$pdo = new DB();
$s = new Sites();
@@ -33,6 +33,7 @@ $seq = (isset($tmux->sequential)) ? $tmux->sequential : 0;
$powerline = (isset($tmux->powerline)) ? $tmux->powerline : 0;
$tpatch = $tmux->sqlpatch;
$run_ircscraper = $tmux->scrape;
+$tmuxImport = $tmux->import;
@@ -1394,11 +1395,21 @@ while ($i > 0) {
//run update_binaries, backfill and import using seq in pane 0.2
$dead = `tmux list-panes -t${tmux_session}:0 | grep 2: | grep dead`;
if (($seq == 1) && (strlen($dead) > "4")) {
+ switch ($tmuxImport) {
+ case 1:
+ $useFilenames = 'false';
+ break;
+ case 2:
+ $useFilenames = 'true';
+ break;
+ default:
+ $useFilenames = 'false';
+ }
//run nzb-import
if (($import != 0) && ($kill_pp == "false")) {
$log = writelog($panes0[2]);
shell_exec("tmux respawnp -t${tmux_session}:0.2 ' \
- $_python ${DIR}/../python/import_threaded.py $log; date +\"%D %T\"; $_sleep $import_timer' 2>&1 1> /dev/null"
+ cd $_multi && $_php import.php $site->nzbs $site->nzbthreads true $useFilename false $log; date +\"%D %T\"; $_sleep $import_timer' 2>&1 1> /dev/null"
);
} else {
$color = get_color($colors_start, $colors_end, $colors_exc);
@@ -1501,9 +1512,19 @@ while ($i > 0) {
//runs nzb-import in 0.4
if (($import != 0) && ($kill_pp == "false")) {
+ switch ($tmuxImport) {
+ case 1:
+ $useFilenames = 'false';
+ break;
+ case 2:
+ $useFilenames = 'true';
+ break;
+ default:
+ $useFilenames = 'false';
+ }
$log = writelog($panes0[4]);
shell_exec("tmux respawnp -t${tmux_session}:0.4 ' \
- $_python ${DIR}/../python/import_threaded.py $log; date +\"%D %T\"; $_sleep $import_timer' 2>&1 1> /dev/null"
+ cd $_multi && $_php import.php $site->nzbs $site->nzbthreads true $useFilenames false $log; date +\"%D %T\"; $_sleep $import_timer' 2>&1 1> /dev/null"
);
} else if (($import == 1) && ($maxload <= get_load())) {
$color = get_color($colors_start, $colors_end, $colors_exc);
diff --git a/lib/ReleaseRemover.php b/lib/ReleaseRemover.php
index d1671f16c..83f495e50 100644
--- a/lib/ReleaseRemover.php
+++ b/lib/ReleaseRemover.php
@@ -833,7 +833,11 @@ class ReleaseRemover
)
);
- $join = (NN_RELEASE_SEARCH_TYPE == \ReleaseSearch::SPHINX ? "INNER JOIN releases_se rse ON rse.id = r.ID" : "INNER JOIN releasesearch rs ON rs.releaseID = r.ID");
+ if ($opTypeName == 'Subject') {
+ $join = (NN_RELEASE_SEARCH_TYPE == \ReleaseSearch::SPHINX ? 'INNER JOIN releases_se rse ON rse.id = r.ID' : 'INNER JOIN releasesearch rs ON rs.releaseid = r.ID');
+ } else {
+ $join = '';
+ }
$this->query = sprintf("
SELECT r.guid, r.searchname, r.ID
@@ -1047,6 +1051,55 @@ class ReleaseRemover
$args[1] = $this->cleanSpaces($args[1]);
$args[2] = $this->cleanSpaces($args[2]);
switch ($args[0]) {
+ case 'categoryid':
+ switch ($args[1]) {
+ case 'equals':
+ return ' AND categoryID = ' . $args[2];
+ default:
+ break;
+ }
+ break;
+ case 'imdbid':
+ switch ($args[1]) {
+ case 'equals':
+ if ($args[2] === 'NULL') {
+ return ' AND imdbID IS NULL ';
+ }
+ else {
+ return ' AND imdbID = ' . $args[2];
+ }
+ default:
+ break;
+ }
+ break;
+ case 'nzbstatus':
+ switch ($args[1]) {
+ case 'equals':
+ return ' AND nzbstatus = ' . $args[2];
+ default:
+ break;
+ }
+ break;
+ case 'rageid':
+ switch ($args[1]) {
+ case 'equals':
+ return ' AND rageID = ' . $args[2];
+ default:
+ break;
+ }
+ break;
+ case 'totalpart':
+ switch ($args[1]) {
+ case 'equals':
+ return ' AND totalpart = ' . $args[2];
+ case 'bigger':
+ return ' AND totalpart > ' . $args[2];
+ case 'smaller':
+ return ' AND totalpart < ' . $args[2];
+ default:
+ break;
+ }
+ break;
case 'fromname':
switch ($args[1]) {
case 'equals':
diff --git a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php
index ce498c92e..9ab5a5cd1 100644
--- a/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php
+++ b/lib/copy_this/misc/update_scripts/nix_scripts/multiprocessing/.do_not_run/switch.php
@@ -82,7 +82,7 @@ switch ($options[1]) {
$columns = [];
switch ($options[2]) {
case 'binaries':
- if ($return['last_record'] <= $groupMySQL['last_record']){
+ if ($return['last'] <= $groupMySQL['last_record']){
exit();
}
$columns[1] = sprintf(
@@ -91,23 +91,23 @@ switch ($options[1]) {
(is_numeric($return['last_record_postdate']) ? $return['last_record_postdate'] : strtotime($return['last_record_postdate']))
)
);
- $columns[2] = sprintf('last_record = %s', $return['last_record']);
+ $columns[2] = sprintf('last_record = %s', $return['last']);
$query = sprintf(
'UPDATE groups SET %s, %s, last_updated = NOW() WHERE ID = %d AND last_record < %s',
$columns[1],
$columns[2],
$groupMySQL['ID'],
- $return['last_record']
+ $return['last']
);
break;
case 'backfill':
- if ($return['first_record'] >= $groupMySQL['first_record']){
+ if ($return['first'] >= $groupMySQL['first_record']){
exit();
}
$columns[1] = sprintf(
'first_record_postdate = %s',
$pdo->from_unixtime(
- (is_numeric($return['first_record_postadate']) ? $return['first_record_postdate'] : strtotime($return['first_record_postdate']))
+ (is_numeric($return['first_record_postdate']) ? $return['first_record_postdate'] : strtotime($return['first_record_postdate']))
)
);
$columns[2] = sprintf('first_record = %s', $return['first_record']);
diff --git a/lib/copy_this/www/lib/NZBImport.php b/lib/copy_this/www/lib/NZBImport.php
new file mode 100644
index 000000000..0d1b57316
--- /dev/null
+++ b/lib/copy_this/www/lib/NZBImport.php
@@ -0,0 +1,462 @@
+ false, // Was this started from the browser?
+ 'Echo' => true, // Echo to CLI?
+ 'Binaries' => null,
+ 'Categorize' => null,
+ 'NZB' => null,
+ 'ReleaseCleaning' => null,
+ 'Releases' => null,
+ 'Settings' => null,
+ ];
+ $options += $defaults;
+
+ $this->echoCLI = (!$this->browser && NN_ECHOCLI && $options['Echo']);
+ $this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB());
+ $this->binaries = ($options['Binaries'] instanceof \Binaries ? $options['Binaries'] : new \Binaries(['Settings' => $this->pdo, 'Echo' => $this->echoCLI]));
+ $this->category = ($options['Categorize'] instanceof \Categorize ? $options['Categorize'] : new \Categorize(['Settings' => $this->pdo]));
+ $this->nzb = ($options['NZB'] instanceof \NZB ? $options['NZB'] : new \NZB($this->pdo));
+ $this->releaseCleaner = ($options['ReleaseCleaning'] instanceof \ReleaseCleaning ? $options['ReleaseCleaning'] : new \ReleaseCleaning($this->pdo));
+ $this->releases = ($options['Releases'] instanceof \Releases ? $options['Releases'] : new \Releases(['settings' => $this->pdo]));
+ $s = new Sites();
+ $this->site = $s->get();
+
+ $this->crossPostt = ($this->site->crossposttime != '') ? $this->site->crossposttime : 2;
+ $this->browser = $options['Browser'];
+ $this->retVal = '';
+ }
+
+ /**
+ * @param array $filesToProcess List of NZB files to import.
+ * @param bool|string $useNzbName Use the NZB file name as release name?
+ * @param bool $delete Delete the NZB when done?
+ * @param bool $deleteFailed Delete the NZB if failed importing?
+ *
+ * @return string|bool
+ *
+ * @access public
+ */
+ public function beginImport($filesToProcess, $useNzbName = false, $delete = true, $deleteFailed = true)
+ {
+ // Get all the groups in the DB.
+ if (!$this->getAllGroups()) {
+ if ($this->browser) {
+ return $this->retVal;
+ } else {
+ return false;
+ }
+ }
+
+ $start = date('Y-m-d H:i:s');
+ $nzbsImported = $nzbsSkipped = 0;
+
+ // Loop over the file names.
+ foreach ($filesToProcess as $nzbFile) {
+
+ // Check if the file is really there.
+ if (is_file($nzbFile)) {
+
+ // Get the contents of the NZB file as a string.
+ if (strtolower(substr($nzbFile, -7)) === '.nzb.gz') {
+ $nzbString = Utility::unzipGzipFile($nzbFile);
+ } else {
+ $nzbString = file_get_contents($nzbFile);
+ }
+
+ if ($nzbString === false) {
+ $this->echoOut('ERROR: Unable to read: ' . $nzbFile);
+
+ if ($deleteFailed) {
+ @unlink($nzbFile);
+ }
+ $nzbsSkipped++;
+ continue;
+ }
+
+ // Load it as a XML object.
+ $nzbXML = @simplexml_load_string($nzbString);
+ if ($nzbXML === false || strtolower($nzbXML->getName()) != 'nzb') {
+ $this->echoOut('ERROR: Unable to load NZB XML data: ' . $nzbFile);
+
+ if ($deleteFailed) {
+ @unlink($nzbFile);
+ }
+ $nzbsSkipped++;
+ continue;
+ }
+
+ // Try to insert the NZB details into the DB.
+ $inserted = $this->scanNZBFile($nzbXML, ($useNzbName ? str_ireplace('.nzb', '', basename($nzbFile)) : false));
+
+ if ($inserted) {
+
+ // Try to copy the NZB to the NZB folder.
+ $path = $this->nzb->getNZBPath($this->relGuid, 0, true);
+
+ // Try to compress the NZB file in the NZB folder.
+ $fp = gzopen ($path, 'w5');
+ gzwrite ($fp, $nzbString);
+ gzclose($fp);
+
+ if (!is_file($path)) {
+ $this->echoOut('ERROR: Problem compressing NZB file to: ' . $path);
+
+ // Remove the release.
+ $this->pdo->queryExec(
+ sprintf("DELETE FROM releases WHERE guid = %s", $this->pdo->escapeString($this->relGuid))
+ );
+
+ if ($deleteFailed) {
+ @unlink($nzbFile);
+ }
+ $nzbsSkipped++;
+ continue;
+
+ } else {
+
+ if ($delete) {
+ // Remove the nzb file.
+ @unlink($nzbFile);
+ }
+
+ $nzbsImported++;
+ continue;
+ }
+
+ } else {
+ if ($deleteFailed) {
+ @unlink($nzbFile);
+ }
+ $nzbsSkipped++;
+ continue;
+ }
+
+ } else {
+ $this->echoOut('ERROR: Unable to fetch: ' . $nzbFile);
+ $nzbsSkipped++;
+ continue;
+ }
+ }
+ $this->echoOut(
+ 'Proccessed ' .
+ $nzbsImported .
+ ' NZBs in ' .
+ (strtotime(date('Y-m-d H:i:s')) - strtotime($start)) .
+ ' seconds, ' .
+ $nzbsSkipped .
+ ' NZBs were skipped.'
+ );
+
+ if ($this->browser) {
+ return $this->retVal;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * @param object $nzbXML Reference of simpleXmlObject with NZB contents.
+ * @param bool|string $useNzbName Use the NZB file name as release name?
+ * @return bool
+ *
+ * @access protected
+ */
+ protected function scanNZBFile(&$nzbXML, $useNzbName = false)
+ {
+ $totalFiles = $totalSize = $groupID = 0;
+ $isBlackListed = $groupName = $firstName = $posterName = $postDate = false;
+
+ // Go through the NZB, get the details, look if it's blacklisted, look if we have the groups.
+ foreach ($nzbXML->file as $file) {
+
+ $totalFiles++;
+ $groupID = -1;
+
+ // Get the nzb info.
+ if ($firstName === false ) {
+ $firstName =(string) $file->attributes()->subject;
+ }
+ if ($posterName === false) {
+ $posterName = (string) $file->attributes()->poster;
+ }
+ if ($postDate === false) {
+ $postDate = date("Y-m-d H:i:s", (string) $file->attributes()->date);
+ }
+
+ // Make a fake message array to use to check the blacklist.
+ $msg = array("Subject" => (string) $file->attributes()->subject, "From" => (string) $file->attributes()->poster, "Message-ID" => "");
+
+ // Get the group names, groupID, check if it's blacklisted.
+ $groupArr = array();
+ foreach ($file->groups->group as $group) {
+ $group = (string) $group;
+
+ // If groupID is -1 try to get a groupID.
+ if ($groupID === -1) {
+ if (array_key_exists($group, $this->allGroups)) {
+ $groupID = $this->allGroups[$group];
+ if (!$groupName) {
+ $groupName = $group;
+ }
+ }
+ }
+ // Add all the found groups to an array.
+ $groupArr[] = $group;
+
+ // Check if this NZB is blacklisted.
+ if ($this->binaries->isBlacklisted($msg, $group)) {
+ $isBlackListed = true;
+ break;
+ }
+ }
+
+ // If we found a group and it's not blacklisted.
+ if ($groupID !== -1 && !$isBlackListed) {
+
+ // Get the size of the release.
+ if (count($file->segments->segment) > 0) {
+ foreach ($file->segments->segment as $segment) {
+ $totalSize += (int)$segment->attributes()->bytes;
+ }
+ }
+
+ } else {
+ if ($isBlackListed) {
+ $errorMessage = "Subject is blacklisted: " . utf8_encode(trim($firstName));
+ } else {
+ $errorMessage = "No group found for " . $firstName . " (one of " . implode(', ', $groupArr) . " are missing";
+ }
+ $this->echoOut($errorMessage);
+
+ return false;
+ }
+ }
+
+ // Try to insert the NZB details into the DB.
+ return $this->insertNZB(
+ array(
+ 'subject' => $firstName,
+ 'useFName' => $useNzbName,
+ 'postDate' => (empty($postDate) ? date("Y-m-d H:i:s") : $postDate),
+ 'from' => (empty($posterName) ? '' : $posterName),
+ 'groupID' => $groupID,
+ 'groupName' => $groupName,
+ 'totalFiles' => $totalFiles,
+ 'totalSize' => $totalSize
+ )
+ );
+ }
+
+ /**
+ * Insert the NZB details into the database.
+ *
+ * @param $nzbDetails
+ *
+ * @return bool
+ *
+ * @access protected
+ */
+ protected function insertNZB($nzbDetails)
+ {
+ // Make up a GUID for the release.
+ $this->relGuid = $this->releases->createGUID();
+
+ // Remove part count from subject.
+ $partLess = preg_replace('/(\(\d+\/\d+\))*$/', 'yEnc', $nzbDetails['subject']);
+ // Remove added yEnc from above and anything after.
+ $subject = utf8_encode(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess)));
+
+ $renamed = 0;
+ if ($nzbDetails['useFName']) {
+ // If the user wants to use the file name.. use it.
+ $cleanName = $nzbDetails['useFName'];
+ $renamed = 1;
+ } else {
+ // Pass the subject through release cleaner to get a nicer name.
+ $cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['totalSize'], $nzbDetails['groupName']);
+ if (isset($cleanName['properlynamed'])) {
+ $cleanName = $cleanName['cleansubject'];
+ $renamed = (isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0);
+ }
+ }
+
+ $escapedSubject = $this->pdo->escapeString($subject);
+ $escapedFromName = $this->pdo->escapeString($nzbDetails['from']);
+
+ // Look for a duplicate on name, poster and size.
+ $dupeCheck = $this->pdo->queryOneRow(
+ sprintf(
+ 'SELECT ID FROM releases WHERE name = %s AND fromname = %s AND size BETWEEN %s AND %s',
+ $escapedSubject,
+ $escapedFromName,
+ $this->pdo->escapeString($nzbDetails['totalSize'] * 0.99),
+ $this->pdo->escapeString($nzbDetails['totalSize'] * 1.01)
+ )
+ );
+
+ if ($dupeCheck === false) {
+ $escapedSearchName = $this->pdo->escapeString($cleanName);
+ // Insert the release into the DB.
+ $relID = $this->releases->insertRelease(
+ [
+ 'name' => $escapedSubject,
+ 'searchname' => $escapedSearchName,
+ 'totalpart' => $nzbDetails['totalFiles'],
+ 'groupID' => $nzbDetails['groupID'],
+ 'guid' => $this->pdo->escapeString($this->relGuid),
+ 'postdate' => $this->pdo->escapeString($nzbDetails['postDate']),
+ 'fromname' => $escapedFromName,
+ 'size' => $this->pdo->escapeString($nzbDetails['totalSize']),
+ 'categoryID' => $this->category->determineCategory($cleanName, $nzbDetails['groupID']),
+ 'isrenamed' => $renamed,
+ 'reqidstatus' => 0,
+ 'prehashID' => 0,
+ 'nzbstatus' => \Enzebe::NZB_ADDED
+ ]
+ );
+ } else {
+ //$this->echoOut('This release is already in our DB so skipping: ' . $subject);
+ return false;
+ }
+
+ if (isset($relID) && $relID === false) {
+ $this->echoOut('ERROR: Problem inserting: ' . $subject);
+ return false;
+ }
+ return true;
+ }
+ /**
+ * Get all groups in the DB.
+ * @return bool
+ *
+ * @access protected
+ */
+ protected function getAllGroups()
+ {
+ $this->allGroups = [];
+ $groups = $this->pdo->query("SELECT ID, name FROM groups");
+ foreach ($groups as $group) {
+ $this->allGroups[$group["name"]] = $group["ID"];
+ }
+
+ if (count($this->allGroups) === 0) {
+ $this->echoOut('You have no groups in your database!');
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Echo message to browser or CLI.
+ * @param $message
+ *
+ * @access protected
+ */
+ protected function echoOut($message)
+ {
+ if ($this->browser) {
+ $this->retVal .= $message . "
";
+ } elseif ($this->echoCLI) {
+ echo $message . PHP_EOL;
+ }
+ }
+}
\ No newline at end of file
diff --git a/lib/copy_this/www/lib/releases.php b/lib/copy_this/www/lib/releases.php
index 65dd91716..6e551c2b9 100644
--- a/lib/copy_this/www/lib/releases.php
+++ b/lib/copy_this/www/lib/releases.php
@@ -3020,4 +3020,13 @@ class Releases
sprintf('UPDATE releases SET categoryID = %d, iscategorized = 0 %s', \Category::CAT_MISC_OTHER, $where)
);
}
+
+ /**
+ * Create a GUID for a release.
+ * @return string
+ */
+ public function createGUID()
+ {
+ return sha1(uniqid('', true) . mt_rand());
+ }
}
\ No newline at end of file
diff --git a/lib/copy_this/www/templates/nntmux/views/admin/nzb-import.tpl b/lib/copy_this/www/templates/nntmux/views/admin/nzb-import.tpl
index 2e50a53a9..e15c29352 100644
--- a/lib/copy_this/www/templates/nntmux/views/admin/nzb-import.tpl
+++ b/lib/copy_this/www/templates/nntmux/views/admin/nzb-import.tpl
@@ -1,53 +1,73 @@
-
-Import nzbs via the command line or browser into the system.
+ Import NZB's from a folder or via the browser into the system. Specify the full file path to a folder containing NZB's.
+
+ Importing will add the release to your database, compress the NZB and store it in the nzbfiles/ folder.