diff --git a/bin/mem_usage.sh b/bin/mem_usage.sh deleted file mode 100755 index a301bdb5f..000000000 --- a/bin/mem_usage.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -free -m|awk '/^Mem:/{print "\033[1;31mRam Used:\033[0m " $3 "MB, \033[1;31mRam Free:\033[0m " $4 "MB"}'; -free -m|awk '/^Swap:/{print "\033[1;31mSwap Used:\033[0m " $3 "MB, \033[1;31mSwap Free:\033[0m " $4 "MB"}' -uptime | cut -d ',' -f 2- diff --git a/bin/monitor.php b/bin/monitor.php index 9a03f7686..fec223f1a 100755 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -54,7 +54,8 @@ $releases_start = $releases_start[0]['cnt']; //environment $_DB_USER = getenv('DB_USER'); $_DB_HOST = getenv('DB_HOST'); -$_DB_PASSWORD = getenv('DB_PASSWORD'); +//$_DB_PASSWORD = getenv('DB_PASSWORD'); +$_DB_PASSWORD = escapeshellarg(getenv('DB_PASSWORD')); $_DB_NAME = getenv('DB_NAME'); $_backfill = getenv('BACKFILL'); $_binaries = getenv('BINARIES'); @@ -62,6 +63,7 @@ $_import = getenv('IMPORT'); $_cleanup = getenv('CLEANUP'); $_parsing = getenv('PARSING'); $_optimise = getenv('OPTIMISE'); +$_nzbmulti = getenv('NZBMULTI'); //$_inno_test = shell_exec('svn info /var/www/newznab | grep inno'); $_admin_path = getenv('ADMIN_PATH'); @@ -212,6 +214,10 @@ while($i>0) $parts_rows = number_format("$parts_rows"); printf("\n \033[0mThe parts table has \033[1;31m$parts_rows\033[0m rows and is \033[1;31m$parts_size_gb\n"); + if ($i==1) { + sleep(10); + } + //run update_predb.php in 1.0 ever 15 minutes if ((TIME() - $time2) >= 900 ) { shell_exec("$_tmux respawnp -t $_tmux_session:1.0 'echo \"\033[1;31m\" && cd $_newznab_path && $_php update_predb.php true && date && echo \"$_string\"' 2>&1 1> /dev/null"); @@ -299,6 +305,11 @@ while($i>0) $_backfill_cmd = 'backfill.php'; $_update_cmd = 'update_binaries.php'; } + if ( $_nzbmulti == "true" ){ + $nzb_cmd = "$_php nzb-import-sub.php \"$_nzbs\""; + } else { + $nzb_cmd = "$_php nzb-import.php \"$_nzbs\" true"; + } if ( $_binaries == "true" ) { if (( $total_work_now < $_max_releases ) || ( $_max_releases == 0 )) { @@ -320,13 +331,13 @@ while($i>0) if ( $_import == "true" ) { if (( $total_work_now < $_import_max_releases ) || ( $_import_max_releases == 0 )) { - shell_exec("$_tmux respawnp -t $_tmux_session:0.11 'echo \"\033[1;36m\" && cd bin && $_php nzb-import.php \"$_nzbs\" true && echo \" \" && date && echo \"$_sleep_string $_import_sleep seconds...\" && sleep $_import_sleep && echo \"$_string\"' 2>&1 1> /dev/null"); + shell_exec("$_tmux respawnp -t $_tmux_session:0.11 'echo \"\033[1;36m\" && cd bin && $nzb_cmd && echo \" \" && date && echo \"$_sleep_string $_import_sleep seconds...\" && sleep $_import_sleep && echo \"$_string\"' 2>&1 1> /dev/null"); } else { shell_exec("$_tmux respawnp -t $_tmux_session:0.11 'echo \"\n$_string2\"' 2>&1 1> /dev/null"); } } if (( $_optimise == "true" ) && ( ($i % 5) == 0 )) { - shell_exec("$_tmux respawnp -t $_tmux_session:0.12 'echo \"\033[1;37m\" && cd $_newznab_path && $_php update_releases.php && cd $_current_path && $_php optimize_myisam.php && ./mem_usage.sh && date && echo \"$_sleep_string $_rel_sleep seconds...\" && sleep $_rel_sleep && echo \"$_string\"' 2>&1 1> /dev/null"); + shell_exec("$_tmux respawnp -t $_tmux_session:0.12 'echo \"\033[1;37m\" && cd $_newznab_path && $_php update_releases.php && cd $_current_path && $_php optimize_myisam.php && date && echo \"$_sleep_string $_rel_sleep seconds...\" && sleep $_rel_sleep && echo \"$_string\"' 2>&1 1> /dev/null"); } else { shell_exec("$_tmux respawnp -t $_tmux_session:0.12 'echo \"\033[1;37m\" && cd $_newznab_path && $_php update_releases.php && cd $_current_path && ./mem_usage.sh && date && echo \"$_sleep_string $_rel_sleep seconds...\" && sleep $_rel_sleep && echo \"$_string\"' 2>&1 1> /dev/null"); } diff --git a/bin/nzb-import-sub.php b/bin/nzb-import-sub.php new file mode 100755 index 000000000..c7d5d3ebc --- /dev/null +++ b/bin/nzb-import-sub.php @@ -0,0 +1,66 @@ + 0){ + echo "Directory {$subdir} contains {$filecount} nzb's.\n"; + $subdir_count++; + } +} + +if($subdir_count == 0){ + echo "Nothing to import, sub-directories are empty or there are no sub-directories to process.\n"; + die(); +} + +$ps = new PowerProcess; +$ps->RegisterCallback('psUpdateComplete'); +$ps->maxThreads = 20; +$ps->threadTimeLimit = 0; +$selected = $subdir_count-1; + +echo "Starting threaded import process\n"; + +while ($ps->RunControlCode()) +{ + if ($selected >= 0) + { + if ($ps->SpawnReady()) + { + $ps->threadData = $subdirs[$selected]."/"; + $ps->spawnThread(); + } + } + else + { + $ps->Shutdown(); + } +$selected--; +} + +if ($ps->RunThreadCode()) +{ + $subdir = $ps->threadData; + $thread = sprintf("%05d",$ps->GetPID()); + + $dir = dirname(__FILE__); + $file = 'nzb-importonenzb.php'; + $output = shell_exec("php {$dir}/{$file} {$subdir} true"); + + echo "[Thread-{$thread}] Completed importing from {$subdir}\n"; +} +exit(0); + + +function psUpdateComplete() +{ + echo "[Thread-MASTER] The threaded import process has completed.\n"; +} + +?> + diff --git a/bin/nzb-importonenzb.php b/bin/nzb-importonenzb.php new file mode 100755 index 000000000..ba135ab9c --- /dev/null +++ b/bin/nzb-importonenzb.php @@ -0,0 +1,213 @@ +isPostBack() ) +{ + $retval = ""; + + // + // Via browser, build an array of all the nzb files uploaded into php /tmp location + // + if (isset($_FILES["uploadedfiles"])) + { + foreach ($_FILES["uploadedfiles"]["error"] as $key => $error) + { + if ($error == UPLOAD_ERR_OK) + { + $tmp_name = $_FILES["uploadedfiles"]["tmp_name"][$key]; + $name = $_FILES["uploadedfiles"]["name"][$key]; + $filestoprocess[] = $tmp_name; + $browserpostednames[$tmp_name] = $name; + } + } + } + + if ($using_cli) + { + $strTerminator = "\n"; + $path = $argv[1]; + $usenzbname = (isset($argv[2]) && $argv[2] == 'true') ? true : false; + } + else + { + $strTerminator = "
"; + $path = (isset($_POST["folder"]) ? $_POST["folder"] : ""); + $usenzbname = (isset($_POST['usefilename']) && $_POST["usefilename"] == 'on') ? true : false; + } + + if (substr($path, strlen($path) - 1) != '/') + $path = $path."/"; + + $groups = $db->query("SELECT ID, name FROM groups"); + foreach ($groups as $group) + $siteGroups[$group["name"]] = $group["ID"]; + + if (!isset($groups) || count($groups) == 0) + { + if ($using_cli) + echo "no groups available in the database, add first.\n"; + else + $retval.= "no groups available in the database, add first.".$strTerminator; + } + else + { + $nzbCount = 0; + + // + // read from the path, if no files submitted via the browser + // + if (count($filestoprocess) == 0) + $filestoprocess = glob($path."*.nzb"); + $start=date('Y-m-d H:i:s'); + + foreach($filestoprocess as $nzbFile) + { + $importfailed = false; + $nzb = file_get_contents($nzbFile); + + $xml = @simplexml_load_string($nzb); + if (!$xml || strtolower($xml->getName()) != 'nzb') + { + continue; + } + + $i=0; + foreach($xml->file as $file) + { + //file info + $groupID = -1; + $name = (string)$file->attributes()->subject; + $fromname = (string)$file->attributes()->poster; + $unixdate = (string)$file->attributes()->date; + $date = date("Y-m-d H:i:s", (string)$file->attributes()->date); + + //groups + $groupArr = array(); + foreach($file->groups->group as $group) + { + $group = (string)$group; + if (array_key_exists($group, $siteGroups)) + { + $groupID = $siteGroups[$group]; + } + $groupArr[] = $group; + } + + if ($groupID != -1) + { + $xref = implode(': ', $groupArr).':'; + + $totalParts = sizeof($file->segments->segment); + + //insert binary + $binaryHash = md5($name.$fromname.$groupID); + $binarySql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded, importname) values (%s, %s, %s, %s, %s, %s, %s, NOW(), %s)", + $db->escapeString($name), $db->escapeString($fromname), $db->escapeString($date), + $db->escapeString($xref), $db->escapeString($totalParts), $db->escapeString($groupID), $db->escapeString($binaryHash), $db->escapeString($nzbFile) ); + + $binaryId = $db->queryInsert($binarySql); + + if ($usenzbname) + { + $usename = str_replace('.nzb', '', (!$using_cli ? $browserpostednames[$nzbFile] : basename($nzbFile))); + + $db->query(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", + $db->escapeString($usename), 1, 1, 5, "null", "null", "null", $binaryId)); + } + + //segments (i.e. parts) + if (count($file->segments->segment) > 0) + { + $partsSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) values "; + foreach($file->segments->segment as $segment) + { + $messageId = (string)$segment; + $partnumber = $segment->attributes()->number; + $size = $segment->attributes()->bytes; + + $partsSql .= sprintf("(%s, %s, 0, %s, %s, NOW()),", + $db->escapeString($binaryId), $db->escapeString($messageId), $db->escapeString($partnumber), + $db->escapeString($size)); + } + $partsSql = substr($partsSql, 0, -1); + $partsQuery = $db->queryInsert($partsSql); + } + } + else + { + $importfailed = true; + if ($using_cli) + { + echo ("no group found for ".$name." (one of ".implode(', ', $groupArr)." are missing)".$strTerminator); + flush(); + } + else + { + $retval.= "no group found for ".$name." (one of ".implode(', ', $groupArr)." are missing)".$strTerminator; + } + break; + } + } + + if (!$importfailed) + { + $nzbCount++; + @unlink($nzbFile); + + if ($using_cli) + { + echo ("imported ".$nzbFile.$strTerminator); + flush(); + } + else + { + $retval.= "imported ".$nzbFile.$strTerminator; + } + } + if ($nzbCount == 10) + { + break; + } + } + } + $seconds = strtotime(date('Y-m-d H:i:s')) - strtotime($start); + $retval.= 'Processed '.$nzbCount.' nzbs in '.$seconds.' second(s)'; + + if ($using_cli) + { + echo $retval; + die(); + } + + $page->smarty->assign('output', $retval); +} + +$page->title = "Import Nzbs"; +$page->content = $page->smarty->fetch('nzb-import.tpl'); +$page->render(); + diff --git a/conf/tmux.conf b/conf/tmux.conf index 4126a4016..8bafc9590 100755 --- a/conf/tmux.conf +++ b/conf/tmux.conf @@ -28,12 +28,15 @@ setw -g monitor-activity on set -g visual-activity on # # Refresh the status bar every 30 seconds. Try to keep the nzb folder to a reasonable number -set-option -g status-interval 30 +set-option -g status-interval 1 # Example of using a shell command in the status line #set -g status-right "#[fg=yellow]#(uptime | cut -d ',' -f 2-)" #set -g status-right "#[fg=red]#(ls -1 changeme | wc -l) NZB's left to process #[fg=yellow]#(uptime | cut -d ',' -f 2-)" -set -g status-right "#[fg=yellow]#(free -m | grep 'Mem' | awk '{ print \"Ram Used: \"$3\" MB\";}') #[fg=yellow]#(free -m | grep 'Mem' | awk '{ print \"Ram Free: \"$4\" MB\";}') #[fg=yellow]#(free -m | grep 'Swap' | awk '{ print \"Swap Used: \"$3\" MB\";}') #[fg=yellow]#(uptime | cut -d ',' -f 2-)" +#set -g status-right "#[fg=yellow]#(free -m | grep 'Mem' | awk '{ print \"Ram Used: \"$3\" MB\";}') #[fg=yellow]#(free -m | grep 'Mem' | awk '{ print \"Ram Free: \"$4\" MB\";}') \ +#[fg=yellow]#(free -m | grep 'Swap' | awk '{ print \"Swap Used: \"$3\" MB\";}') #[fg=yellow]#(uptime | cut -d ',' -f 2-)" + +set -g status-right "#[fg=yellow]#(free -m | grep 'Mem' | awk '{ print \"Ram Used: \"$3\" MB, Ram Free: \"$4\" MB\";}')#(free -m | grep 'Swap' | awk '{ print \",Swap Used: \"$3\" MB\";}') #[fg=cyan,bold]%m-%d-%Y #(uptime)" set-option -g status-right-length 200 #set -g status-right '#[fg=green][#[fg=blue]%Y-%m-%d #[fg=white]%H:%M#[default] #($HOME/bin/battery)#[fg=green]]' diff --git a/edit_these.sh b/edit_these.sh index 8077659f4..03e7438e2 100755 --- a/edit_these.sh +++ b/edit_these.sh @@ -9,12 +9,12 @@ export NEWZNAB_PATH=$NEWZPATH"/misc/update_scripts" export TESTING_PATH=$NEWZPATH"/misc/testing" export ADMIN_PATH=$NEWZPATH"/www/admin" -#Post Processing Additional is the processing that downloads rar and attempts to get info for your site +#Post Processing Additional is the processing that downloads rar files and attempts to get info for your site #you are able to set the number of process to be run from 1-9 #trial and error for this and do to the sorting method 1 runs always, 2 if more than 200, 3 more than 300 and so on. #At some point, increasing this begins to slow things down. It will need to be adjusted for your system #to get the desired performance -export POST_TO_RUN="1" +export POST_TO_RUN="5" #Enter the session name to be used by tmux export TMUX_SESSION="Newznab-master" @@ -46,15 +46,20 @@ export MAXDAYS="210" #Set the path to the nzb dump you downloaded from torrents, theis is the path to bulk files folder of nzbs #this does not recurse through subfolders -export NZBS="/path/to/nzbs" +export NZBS="/home/jonnyboy/nzbs/test" + +#If you have all of your nzbs in one folder select true +#If, you have all of you nzbs split into separate in with the root at $NZBS then selct false +#and 10 nzbs will be imported from each subfolder per loop. +export NZBMULTI="true" #Choose to run the threaded or non-threaded newznab scripts true/false #such as update_binaries.php or update_binaries_threaded.php -export THREADS="true" +export THREADS="false" #Choose your database engine, comment the one true/false #you should have already converted your database to InnoDB engine, if you select true here -export INNODB="true" +export INNODB="false" #Choose to run update_cleanup.php and removespecial.php true/false #set to false by default, you will need to edit /misc/testing/update_cleanup.php and /misc/testing/removespecial.php @@ -64,7 +69,7 @@ export CLEANUP="false" #Choose to run update_parsing.php true/false #set to false by default, you will need to edit /misc/testing/update_parsing.php #to actually do anything, directions are in the file -export PARSING="false" +export PARSING="true" #Choose to run update_binaries true/false export BINARIES="true" @@ -77,7 +82,7 @@ export IMPORT="true" #Choose to run optimise_db script true/false #set to false by default, you should test the optimse scripts in bin/innodb first -export OPTIMISE="false" +export OPTIMISE="true" #Set the max amount of unprocessed releases and still allow nzb-import to run #set to 0 to disable @@ -106,11 +111,11 @@ export USE_VNSTAT="false" #Each pane may have periods of inactivity, at the time "Pane is Dead" will be displayed. #To disable my notes about why this is ok, change to false -export SHOW_WHY="true" +export SHOW_WHY="false" #By using this script you understand that the programmer is not responsible for any loss of data, users, or sanity. #You also agree that you were smart enough to make a backup of your database and files. Do you agree? yes/no -export AGREED="no" +export AGREED="yes" ##END OF EDITS## diff --git a/images/newznab-tmux.png b/images/newznab-tmux.png index 9499ccd90..70d1d7c44 100644 Binary files a/images/newznab-tmux.png and b/images/newznab-tmux.png differ diff --git a/svn.sh b/svn.sh index eb4d1e517..55cd033ee 100755 --- a/svn.sh +++ b/svn.sh @@ -1,6 +1,6 @@ #EDIT_THESE export NEWZPATH="/var/www/newznab" -export PASSWORD="password" +export PASSWORD="svnplu5" svn export --force --username svnplus --password $PASSWORD svn://svn.newznab.com/nn/branches/nnplus $NEWZPATH/