created dev branch, added backfill parts, test ugo script to create releases without regexes - this is considered a wip

README.md

@@ -90,7 +90,7 @@
 
  * A how-to from [nevermind](http://pastebin.com/ibpi71iE) will help your db run a little faster/easier. It can be damaging if you make a mistake while doing this.
 
- * A couple of autoinstallers are available to completely install everything needed for Ubuntu 12.10 Newznab+ installation. [zombu2](https://github.com/zombu2/nninstall) and [convict](https://github.com/rcconvict/nninstaller) each have simlar installers and can be found in the gits.
+ * A couple of autoinstallers are available to completely install everything needed for Ubuntu 12.10 Newznab+ installation. [zombu2](https://github.com/zombu2/nninstall) and [convict](https://github.com/rcconvict/nninstaller) each have simlar installers and can be found in the gits. If you prefor to do things yourself there is an excellent guide [here](http://oizopower.tweakblogs.net/blog/8524/newznab%2B-installatie-%28ubuntu-nginx-percona%29.html).
   
  * Join in the conversation at irc://irc.synirc.net/newznab-tmux.
 

bin/backfill_parts.php

@@ -0,0 +1,45 @@
+<?php
+require(dirname(__FILE__)."/config.php");
+require_once(WWW_DIR."/lib/backfill.php");
+require_once(WWW_DIR."/lib/framework/db.php");
+
+//get variables from config.sh and defaults.sh
+$path = dirname(__FILE__);
+$varnames = shell_exec("cat ".$path."/../config.sh | grep ^export | cut -d \= -f1 | awk '{print $2;}'");
+$varnames .= shell_exec("cat ".$path."/../defaults.sh | grep ^export | cut -d \= -f1 | awk '{print $2;}'");
+$vardata = shell_exec("cat ".$path."/../config.sh | grep ^export | cut -d \\\" -f2 | awk '{print $1;}'");
+$vardata .= shell_exec("cat ".$path."/../defaults.sh | grep ^export | cut -d \\\" -f2 | awk '{print $1;}'");
+$varnames = explode("\n", $varnames);
+$vardata = explode("\n", $vardata);
+$array = array_combine($varnames, $vardata);
+unset($array['']);
+
+$_date = $array['KEVIN_DATE'];
+$_parts = $array['KEVIN_PARTS'];
+
+echo "Starting kevin123's backfill process\n\n";
+	$groupPost = $_parts;
+	if (isset($argv[1])) 
+	{
+		$groupName = $argv[1];
+		$db = new DB;
+		$query = $db->queryOneRow(sprintf("select name from groups WHERE (first_record_postdate BETWEEN '${_date}' and now()) and (name = '$groupName')")) or die(mysql_error());
+		if (isset($query)) 
+		{
+			$backfill = new Backfill();
+			$backfill->backfillPostAllGroups($groupName, $groupPost);
+			echo "Backfilling completed.\n";
+		} 
+		else 
+		{ 
+			echo "Already have the target post, skipping the group.\n"; 
+		}
+	} 
+	else 
+	{
+		$groupName = '';
+		$backfill = new Backfill();
+		$backfill->backfillPostAllGroups($groupName, $groupPost);
+	}
+
+?>

bin/backfill_parts_threaded.php

@@ -0,0 +1,80 @@
+<?php
+require(dirname(__FILE__)."/config.php");
+require_once(WWW_DIR."/lib/groups.php");
+require_once(WWW_DIR."/lib/binaries.php");
+require_once(WWW_DIR."/lib/powerprocess.php");
+
+$groups = new Groups;
+$groupList = $groups->getActive();
+unset($groups);
+
+$ps = new PowerProcess;
+$ps->RegisterCallback('psUpdateComplete');
+$ps->maxThreads = 5;
+$ps->tickCount = 50000;	// value in usecs. change this to 1000000 (one second) to reduce cpu use
+$ps->threadTimeLimit = 0;	// Disable child timeout
+
+echo "Starting kevin123's threaded backfill process\n\n";
+
+while ($ps->RunControlCode()) 
+{
+	// Start the parent loop
+	if (count($groupList)) 
+	{
+		// We still have groups to process
+		if ($ps->SpawnReady()) 
+		{
+			// Spawn another thread
+			$ps->threadData = array_pop($groupList);
+			echo "[Thread-MASTER] Spawning new thread.  Still have " . count($groupList) ." group(s) to update after this\n";
+			$ps->spawnThread();
+		} 
+		else 
+		{
+			// There are no more slots available to run
+			//$ps->tick();
+			//echo ".\n";
+		}
+	} 
+	else 
+	{
+		// No more groups to process
+		echo "No more groups to process - Initiating shutdown\n";
+		$ps->Shutdown();
+		echo "Shutdown complete\n";
+	}
+}
+
+unset($groupList);
+
+if ($ps->RunThreadCode()) 
+{
+	$group = $ps->threadData;
+
+	$thread = sprintf("%05d",$ps->GetPID());
+
+	echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n";
+
+	$param = $group['name'];
+
+	$dir = dirname(__FILE__);
+	$file = 'backfill_parts.php';
+
+	$output = shell_exec("php {$dir}/{$file} {$param}");
+	//$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}");
+
+	echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
+}
+
+// Exit to call back to parent - Let know that child has completed
+exit(0);
+
+// Create callback function
+function psUpdateComplete() 
+{
+	echo "[Thread-MASTER] Threaded backfill process complete\n";
+}
+
+
+?>
+

bin/lib/additional

@@ -6,7 +6,6 @@ require('lib/postprocess1.php');
 $postprocess = new PostProcess1(true);
 $postprocess->processAdditional1();
 
-
 ?>
 
 

bin/monitor.php

@@ -2,7 +2,7 @@
 
 require(dirname(__FILE__)."/config.php");
 require(WWW_DIR.'/lib/postprocess.php');
-$version="0.1r743";
+$version="0.1r744";
 
 $db = new DB();
 
@@ -10,7 +10,7 @@ $db = new DB();
 $qry = "SELECT COUNT( releases.categoryID ) AS cnt, parentID FROM releases INNER JOIN category ON releases.categoryID = category.ID WHERE parentID IS NOT NULL GROUP BY parentID";
 
 //needs to be processed query
-$proc = "SELECT ( SELECT COUNT( groupID ) AS cnt from releases where consoleinfoID IS NULL and categoryID BETWEEN 1000 AND 1999 ) AS console, ( SELECT COUNT( groupID ) AS cnt from releases where imdbID IS NULL and categoryID BETWEEN 2000 AND 2999 ) AS movies, ( SELECT COUNT( groupID ) AS cnt from releases where musicinfoID IS NULL and categoryID BETWEEN 3000 AND 3999 ) AS audio, ( SELECT COUNT( groupID ) AS cnt from releases r left join category c on c.ID = r.categoryID where (categoryID BETWEEN 4000 AND 4999 and ((r.passwordstatus between -6 and -1) or (r.haspreview = -1 and c.disablepreview = 0)))) AS pc, ( SELECT COUNT( groupID ) AS cnt from releases where rageID = -1 and categoryID BETWEEN 5000 AND 5999 ) AS tv, ( SELECT COUNT( groupID ) AS cnt from releases where bookinfoID IS NULL and categoryID = 7020 ) AS book, ( SELECT COUNT( groupID ) AS cnt from releases r left join category c on c.ID = r.categoryID where (r.passwordstatus between -6 and -1) or (r.haspreview = -1 and c.disablepreview = 0)) AS work, ( SELECT COUNT( groupID ) AS cnt from releases) AS releases, ( SELECT COUNT( groupID ) AS cnt FROM releases r WHERE r.releasenfoID = 0) AS nforemains, ( SELECT COUNT( groupID ) AS cnt FROM releases WHERE releasenfoID not in (0, -1)) AS nfo, ( SELECT table_rows AS cnt FROM information_schema.TABLES where table_name = 'parts' AND TABLE_SCHEMA = '".DB_NAME."' ) AS parts, ( SELECT concat(round((data_length+index_length)/(1024*1024*1024),2),'GB') AS cnt FROM information_schema.tables where table_name = 'parts' AND TABLE_SCHEMA = '".DB_NAME."' ) AS partsize, ( SELECT UNIX_TIMESTAMP(adddate) from releases order by adddate desc limit 1 ) AS newestadd, ( SELECT name from releases order by adddate desc limit 1 ) AS newestaddname";
+$proc = "SELECT ( SELECT COUNT( groupID ) AS cnt from releases where consoleinfoID IS NULL and categoryID BETWEEN 1000 AND 1999 ) AS console, ( SELECT COUNT( groupID ) AS cnt from releases where imdbID IS NULL and categoryID BETWEEN 2000 AND 2999 ) AS movies, ( SELECT COUNT( groupID ) AS cnt from releases where musicinfoID IS NULL and categoryID BETWEEN 3000 AND 3999 ) AS audio, ( SELECT COUNT( groupID ) AS cnt from releases r left join category c on c.ID = r.categoryID where (categoryID BETWEEN 4000 AND 4999 and ((r.passwordstatus between -6 and -1) or (r.haspreview = -1 and c.disablepreview = 0)))) AS pc, ( SELECT COUNT( groupID ) AS cnt from releases where rageID = -1 and categoryID BETWEEN 5000 AND 5999 ) AS tv, ( SELECT COUNT( groupID ) AS cnt from releases where bookinfoID IS NULL and categoryID = 7020 ) AS book, ( SELECT COUNT( groupID ) AS cnt from releases r left join category c on c.ID = r.categoryID where (r.passwordstatus between -6 and -1) or (r.haspreview = -1 and c.disablepreview = 0)) AS work, ( SELECT COUNT( groupID ) AS cnt from releases) AS releases, ( SELECT COUNT( groupID ) AS cnt FROM releases r WHERE r.releasenfoID = 0) AS nforemains, ( SELECT COUNT( groupID ) AS cnt FROM releases WHERE releasenfoID not in (0, -1)) AS nfo, ( SELECT table_rows AS cnt FROM information_schema.TABLES where table_name = 'parts' AND TABLE_SCHEMA = '".DB_NAME."' ) AS parts, ( SELECT table_rows AS cnt FROM information_schema.TABLES where table_name = 'binaries' AND TABLE_SCHEMA = '".DB_NAME."' ) AS binaries, ( SELECT concat(round((data_length+index_length)/(1024*1024*1024),2),'GB') AS cnt FROM information_schema.tables where table_name = 'parts' AND TABLE_SCHEMA = '".DB_NAME."' ) AS partsize, ( SELECT concat(round((data_length+index_length)/(1024*1024*1024),2),'GB') AS cnt FROM information_schema.tables where table_name = 'binaries' AND TABLE_SCHEMA = '".DB_NAME."' ) AS binariessize, ( SELECT UNIX_TIMESTAMP(adddate) from releases order by adddate desc limit 1 ) AS newestadd, ( SELECT name from releases order by adddate desc limit 1 ) AS newestaddname";
 //$proc = "SELECT * FROM procCnt;";
 
 //get first release inserted datetime and oldest posted datetime
@@ -49,6 +49,7 @@ $_alienx = dirname(__FILE__)."/../alienx";
 $_conf = dirname(__FILE__)."/../conf";
 $_powerline = dirname(__FILE__)."/../powerline";
 $_cj = dirname(__FILE__)."/../nnscripts";
+$_ugo = dirname(__FILE__)."/../ugo";
 $NNPATH="{$array['NEWZPATH']}{$array['NEWZNAB_PATH']}";
 $TESTING="{$array['NEWZPATH']}{$array['TESTING_PATH']}";
 $killed="false";
@@ -179,11 +180,14 @@ $nfo_remaining_now = 0;
 $nfo_now = 0;
 $parts_rows = 0;
 $parts_size_gb = 0;
+$binaries_rows = 0;
+$binaries_size_gb = 0;
 $releases_now = 0;
 //$firstdate = TIME();
 $newestname = "Unknown";
 $newestdate = TIME();
 $parts_rows_unformated = 0;
+$binaries_rows_unformated = 0;
 $releases_now_formatted = 0;
 $nfo_percent = 0;
 $console_percent = 0;
@@ -276,6 +280,7 @@ printf($mask, "Binaries", "$binaries_state", "$binaries_reason");
 printf($mask, "Backfill", "$backfill_state", "$backfill_reason");
 printf($mask, "Import", "$import_state", "$import_reason");
 printf($mask, "Parts", "$parts_size_gb", "$parts_rows rows");
+printf($mask, "Binaries", "$binaries_size_gb", "$binaries_rows bins");
 if ( $array['RAMDISK_PATH'] != "" ) {
     printf($mask, "Ramdisk", "$disk_use", "$disk_free");
 }
@@ -329,7 +334,7 @@ while( $i > 0 )
     //allow tmux to create panes before running queries
     if ( $i == 1 ) {
         printf("\n\033[1;31mTmux panes safely created in ");
-        for($a=15;$a>-1;$a--)
+        for($a=10;$a>-1;$a--)
         {
             printf("$a..");
             sleep(1);
@@ -423,6 +428,11 @@ while( $i > 0 )
     if ( @$proc_result[0]['parts'] != NULL ) { $parts_rows_unformated = $proc_result[0]['parts']; }
     if ( @$proc_result[0]['parts'] != NULL ) { $parts_rows = number_format($proc_result[0]['parts']); }
     if ( @$proc_result[0]['partsize'] != NULL ) { $parts_size_gb = $proc_result[0]['partsize']; }
+
+    if ( @$proc_result[0]['binaries'] != NULL ) { $binaries_rows_unformated = $proc_result[0]['binaries']; }
+    if ( @$proc_result[0]['binaries'] != NULL ) { $binaries_rows = number_format($proc_result[0]['binaries']); }
+    if ( @$proc_result[0]['binariessize'] != NULL ) { $binaries_size_gb = $proc_result[0]['binariessize']; }
+
     if ( @$proc_result[0]['releases'] ) { $releases_now = $proc_result[0]['releases']; }
     if ( @$proc_result[0]['releases'] ) { $releases_now_formatted = number_format($proc_result[0]['releases']); }
     if ( @$proc_result[0]['newestaddname'] ) { $newestname = $proc_result[0]['newestaddname']; }
@@ -603,6 +613,7 @@ while( $i > 0 )
     printf($mask, "Backfill", "$backfill_state", "$backfill_reason");
     printf($mask, "Import", "$import_state", "$import_reason");
     printf($mask, "Parts", "$parts_size_gb", "$parts_rows rows");
+    printf($mask, "Binaries", "$binaries_size_gb", "$binaries_rows bins");
     if ( $array['RAMDISK_PATH'] != "" ) {
         printf($mask, "Ramdisk", "$disk_use used", "$disk_free free");
     }
@@ -1179,11 +1190,16 @@ while( $i > 0 )
     }
 
     //set command for running backfill
-    if (( $array['KEVIN_SAFER'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
+    if ( $array['KEVIN_SAFER'] == "true" ) {
         $log = writelog($panes0[3]);
         $_backfill_cmd = "cd $_bin && $_php safer_backfill_parts.php 2>&1 $log";
-    } else
-    if ( $array['BACKFILL_THREADS'] == "true" ) {
+    } elseif ( $array['KEVIN_BACKFILL_PARTS'] == "true" ) {
+        $log = writelog($panes0[3]);
+        $_backfill_cmd = "cd $_bin && $_php backfill_parts.php 2>&1 $log";
+    } elseif ( $array['KEVIN_THREADED'] == "true" ) {
+        $log = writelog($panes0[3]);
+        $_backfill_cmd = "cd $_bin && $_php backfill_parts_threaded.php 2>&1 $log";
+    } elseif ( $array['BACKFILL_THREADS'] == "true" ) {
         $log = writelog($panes0[3]);
         $_backfill_cmd = "cd $_bin && $_php backfill_threaded.php 2>&1 $log && $mysql_command_1 2>&1 $log";
     } else {
@@ -1346,7 +1362,15 @@ while( $i > 0 )
     }
 
     //runs update_release and in 0.5 once if needed and exits
-    if (( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
+    if (( $array['RELEASES'] == "true" ) && ( $array['UGO_THREADED'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
+        $color = get_color();
+        $log = writelog($panes0[5]);
+        shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_ugo && $_php automake.php reset && $_php automake_threaded.php && cd $_bin && $_php update_releases.php 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null");
+    } elseif (( $array['RELEASES'] == "true" ) && ( $array['UGO'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
+        $color = get_color();
+        $log = writelog($panes0[5]);
+        shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_ugo && $_php automake.php reset && $_php automake.php && cd $_bin && $_php update_releases.php 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null");
+    } elseif (( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
         $color = get_color();
         $log = writelog($panes0[5]);
         shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_bin && $_php update_releases.php 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null");

bin/safer_backfill_parts.php

@@ -22,6 +22,7 @@ $_parts = $array['KEVIN_PARTS'];
 
 $db = new DB;
 
+echo "Starting kevin123's safer backfill process\n\n";
 $query = $db->queryOneRow(sprintf("select name from groups WHERE (first_record_postdate BETWEEN '${_date}' and now()) and (active = 1) order by name ASC"));
 
 $groupPost = $_parts;

config.sh

@@ -162,7 +162,13 @@ export MAXDAYS="210"
 #you also need to enable kevin's compression mod, those files are needed and you still need to enable BACKFILL
 export KEVIN_SAFER="false"
 
-#set the date to go back to, must be in the format of YYYY-MM-DD, like 2012-06-24
+#use kevin123's backfill_parts.php instead of normal backfill
+export KEVIN_BACKFILL_PARTS="false"
+
+#use kevin123's backfill_parts_threaded.php instead of normal backfill_threaded
+export KEVIN_THREADED="false"
+
+#set the date to go back to, must be in the format of YYYY-MM-DD, like 2012-06-24, this is the date of the posted nzbs
 export KEVIN_DATE="2012-06-24"
 
 #set the number of articles/headers to download at one time
@@ -213,6 +219,15 @@ export MISC_ONLY="false"
 
 ############################################################
 
+#run ugo's automake.php script to create releases, this does not use regexes and will run in a loop prior
+#to update_releases.php, this can be considerably slower, but may give you release that were being missed
+export UGO="false"
+
+#run automake_threaded.php
+export UGO_THREADED="false"
+
+############################################################
+
 #Create releases, this is really only necessary to turn off when you only want to post process
 export RELEASES="false"
 

start.sh

@@ -38,7 +38,7 @@ if $TMUXCMD -q has-session -t $TMUX_SESSION; then
 else
     printf "The above is just a TMUX notice, it is saying TMUX, that you do not have a TMUX session currently running. It is not an error. It is TMUX"
     printf "\033]0; $TMUX_SESSION\007\003\n"
-    $TMUXCMD -f $TMUX_CONF new-session -d -s $TMUX_SESSION -n Monitor 'printf "\033]2;Monitor\033\\" && cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n$NICENESS $PHP monitor.php'
+    #$TMUXCMD -f $TMUX_CONF new-session -d -s $TMUX_SESSION -n Monitor 'printf "\033]2;Monitor\033\\" && cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n$NICENESS $PHP monitor.php'
 
     if [ ! -f $NEWZPATH/www/lib/postprocess.php.orig ]; then
         cp $NEWZPATH/www/lib/postprocess.php $NEWZPATH/www/lib/postprocess.php.orig
@@ -242,6 +242,7 @@ else
     #start tmux
     #printf "\033]0; $TMUX_SESSION\007\003\n"
     #$TMUXCMD -f $TMUX_CONF attach-session - $TMUX_SESSION || new-session -d -s $TMUX_SESSION -n $TMUX_SESSION 'cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n 19 $PHP monitor.php'
+    $TMUXCMD -f $TMUX_CONF new-session -d -s $TMUX_SESSION -n Monitor 'printf "\033]2;Monitor\033\\" && cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n$NICENESS $PHP monitor.php'
     $TMUXCMD selectp -t 0
     $TMUXCMD splitw -h -p 67 'printf "\033]2;update_binaries\033\\"'
 
@@ -380,3 +381,4 @@ else
 fi
 exit
 
+

ugo/automake_threaded.php

@@ -0,0 +1,105 @@
+<?php
+//define('FS_ROOT', realpath(dirname(__FILE__)));
+//require_once(FS_ROOT."/../bin/config.php");
+require(dirname(__FILE__).'/../../../../../www/config.php');
+require_once(WWW_DIR."/lib/binaries.php");
+require_once(WWW_DIR."/lib/powerprocess.php");
+require_once(WWW_DIR."/lib/framework/db.php");
+
+$time = microtime(true);
+$db = new Db;
+
+if (isset($argv[1]))
+{
+var_dump($argv);
+
+	if ($argv[1] == "reset")
+	{
+		echo "binaries have been reset\n";
+		$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)");
+	}
+}
+
+$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6");
+
+$query = "SELECT count(*), groups.`name`, groups.ID";
+$query = $query." FROM binaries INNER JOIN groups ON binaries.groupID = groups.ID";
+$query = $query." WHERE binaries.procstat = 0";
+$query = $query." GROUP BY groups.ID ORDER BY count(*) asc";
+
+$groupList = $db->query($query);
+unset($db);
+
+$ps = new PowerProcess;
+$ps->RegisterCallback('psUpdateComplete');
+$ps->maxChildren = 8;
+$ps->tickCount = 10000;	// value in usecs. change this to 1000000 (one second) to reduce cpu use
+$ps->threadTimeLimit = 0;	// Disable child timeout
+
+$tim = microtime(true) - $time;
+echo "time = ".$tim."\n";
+
+echo "Starting threaded assembly process\n";
+
+while ($ps->RunControlCode())
+{
+	// Start the parent loop
+	if (count($groupList))
+	{
+		// We still have groups to process
+		if ($ps->SpawnReady())
+		{
+			// Spawn another thread
+			$ps->threadData = array_pop($groupList);
+			echo "[Thread-MASTER] Spawning new thread.  Still have " . count($groupList) ." group(s) to update after this\n";
+			$ps->spawnThread();
+		}
+		else
+		{
+			// There are no more slots available to run
+			//$ps->tick();
+			//echo ".\n";
+		}
+	}
+	else
+	{
+		// No more groups to process
+		echo "No more groups to process - Initiating shutdown\n";
+		$ps->Shutdown();
+		echo "Shutdown complete\n";
+	}
+}
+$tim = microtime(true) - $time;
+echo "time = ".$tim."\n";
+unset($groupList);
+
+if ($ps->RunThreadCode())
+{
+	$group = $ps->threadData;
+
+	$thread = sprintf("%05d",$ps->GetPID());
+
+	echo "[Thread-{$thread}] Begining assembly processing for group {$group['name']}\n";
+
+	$param = $group['ID'];
+
+	$dir = dirname(__FILE__);
+	$file = 'automake.php';
+
+	$output = shell_exec("php {$dir}/{$file} {$param}");
+
+	echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
+	$tim = microtime(true) - $time;
+	echo "time = ".$tim."\n";
+}
+
+// Exit to call back to parent - Let know that child has completed
+exit(0);
+
+// Create callback function
+function psUpdateComplete()
+{
+	echo "[Thread-MASTER] Threaded assembly process complete\n";
+}
+
+?>

ugo/makenzb.php

@@ -0,0 +1,111 @@
+<?php
+//define('FS_ROOT', realpath(dirname(__FILE__)));
+//require_once(FS_ROOT."/config.php");
+require(dirname(__FILE__).'/../../../../../www/config.php');
+require_once(WWW_DIR."/lib/nntp.php");
+require_once(WWW_DIR."/lib/framework/db.php");
+
+function makenzb($id, $name = '')
+{
+	global $db;
+	$ok = true;
+	$path = FS_ROOT."/nzbs/";
+
+	$query = "SELECT *, sum(parts.size) AS size FROM binaries INNER JOIN parts ON binaries.ID = parts.binaryID WHERE binaries.ID = ".$id;
+	$row = $db->queryOneRow($query);
+
+	if (!isset($name))
+	{
+		$name = $row['relname'];
+	}
+
+	if (preg_match('/\.nzb\b/i', $row['name']) && $row['size'] < 10000000)
+	{
+echo "doing nzb\n";
+		$nntp = new Nntp;
+		if ($nntp->doConnect())
+		{
+			$bin =  $nntp->getBinary($id);
+
+			$name = preg_replace('/ /','_',$name);
+
+			if (file_exists($path.$name.".nzb"))
+			{
+				$name = $name." ".$id;
+			}
+
+			if (strlen($bin) == 0)
+				$ok = false;
+			elseif (file_put_contents($path.$name.".nzb", $bin) == 0)
+				$ok = false;
+
+		}
+		$nntp->doQuit();
+
+		unset($nntp);
+		unset($bin);
+	}
+
+	if ($ok)
+	{
+	echo "clearing \n";
+		$query = "UPDATE `binaries` SET `procstat` = 6 WHERE ID = ".$id;
+
+		$db->query($query);
+
+	}
+
+}
+
+
+mb_internal_encoding("UTF-8");
+
+$db = new Db;
+
+if (isset($argv[1]))
+{
+	if (isset($argv[2]))
+	{
+		makenzb($argv[1], $argv[2]);
+	} else {
+		makenzb($argv[1]);
+	}
+} else {
+
+	$db->query("UPDATE `binaries` SET `binaries`.`procstat` = 6 WHERE `binaries`.`procstat` = - 3 AND binaries.ID NOT IN ( SELECT `parts`.`binaryID` FROM `parts` )");
+
+	$rel = $db->query("SELECT *  FROM `binaries` WHERE `procstat` = -3");
+
+	foreach($rel as $bin)
+	{
+
+		$name = $bin['relname'];
+
+		$name = preg_replace('/yenc/iU',' ',$name);
+
+		$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
+
+		$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
+
+		$name = preg_replace('/[\`\'\"\:\/\-\<\>]/iU',' ',$name);
+
+		$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
+
+		$name = preg_replace('/  +?/iU',' ',$name);
+
+		$name = preg_replace('/^ | $/iU','',$name);
+
+		$name = preg_replace('/ /','_',$name);
+
+		$id = $bin['ID'];
+
+	echo "doing $name\n";
+
+		makenzb($id, $name);
+
+	}
+}
+
+
+unset($db);
+?>

ugo/makenzb_threaded.php

@@ -0,0 +1,166 @@
+<?php
+define('FS_ROOT', realpath(dirname(__FILE__)));
+require("config.php");
+require_once(WWW_DIR."/lib/groups.php");
+require_once(WWW_DIR."/lib/binaries.php");
+require_once(WWW_DIR.'/lib/powerprocess.php');
+require_once(WWW_DIR."/lib/framework/db.php");
+
+$ps = new PowerProcess;
+$ps->RegisterCallback('psUpdateComplete');
+$ps->maxChildren = 2;
+$ps->tickCount = 10000;	// value in usecs. change this to 1000000 (one second) to reduce cpu use
+$ps->threadTimeLimit = 0;	// Disable child timeout
+
+$db = new Db;
+
+$rel = $db->query("SELECT *  FROM `binaries` WHERE `procstat` = -3");
+
+unset($db);
+
+echo "Starting threaded nzb exporter process\n";
+
+while ($ps->RunControlCode())
+{
+	// Start the parent loop
+	if (count($rel))
+	{
+		// We still have groups to process
+		if ($ps->SpawnReady())
+		{
+			// Spawn another thread
+			$ps->threadData = array_pop($rel);
+			echo "[Thread-MASTER] Spawning new thread.  Still have " . count($rel) ." nzb(s) to update after this\n";
+			$ps->spawnThread();
+		}
+		else
+		{
+			// There are no more slots available to run
+			//$ps->tick();
+			//echo ".\n";
+		}
+	}
+	else
+	{
+		// No more groups to process
+//		$query = "UPDATE `binaries` SET `procstat` = 6 WHERE `procstat` = -3 OR `procstat` = -2";
+		$db = new Db;
+		//var_dump($db->query($query));
+		echo "No more nzbs to process - Initiating shutdown\n";
+		$ps->Shutdown();
+		echo "Shutdown complete\n";
+
+		$path = FS_ROOT."/nzbs/";
+
+		$filestoprocess = glob($path."*.nzb");
+
+		foreach($filestoprocess as $nzbFile)
+		{
+
+			$xml = file_get_contents( $nzbFile );
+
+			$matches = preg_split('/\A(\<\?xml[^\r\n]+?\?\>)\s*\R+/iU',  $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
+
+			$name = preg_split('/\.nzb$/i',  $nzbFile, 0, PREG_SPLIT_DELIM_CAPTURE);
+
+			$name = $name[0];
+
+			$marker = '';
+
+			foreach ($matches as $m)
+			{
+				if (preg_match('/^(\<\?xml[^\r\n]+?\?\>)/iU', $m))
+				{
+		echo "$m\n";
+					$marker = $m;
+					continue;
+				}
+			}
+
+
+			if (!empty($marker))
+			{
+				$matches = preg_split('/('.preg_quote($marker).'\s*\R+)/iU',  $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
+
+				if (strlen($matches[0]) == 0)
+				{
+					unset($matches[0]);
+					$matches = array_values($matches);
+				}
+
+				echo count($matches)."\n";
+
+				if (count($matches) > 2)
+				{
+				for ($i = 0; $i < count($matches); $i = $i + 2)
+					{
+				echo $i."\n";
+					$outxml = $matches[$i] .  $matches[$i + 1];
+					$fileout = $name . '.' . $i . '.nzb';
+					file_put_contents($fileout, $outxml);
+					echo $fileout."\n";
+					}
+				}
+			}
+		}
+	}
+}
+
+unset($rel);
+unset($db);
+
+if ($ps->RunThreadCode())
+{
+	$bin = $ps->threadData;
+
+	$thread = sprintf("%05d",$ps->GetPID());
+
+	echo "[Thread-{$thread}] Begining processing for nzb {$bin['ID']}\n";
+
+	$name = $bin['relname'];
+
+	$name = preg_replace('/yenc/iU',' ',$name);
+
+	$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
+
+	$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
+
+	$name = preg_replace('/[\`\'\"\:\/\-\<\>]/iU',' ',$name);
+
+	$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
+
+	$name = preg_replace('/  +?/iU',' ',$name);
+
+	$name = preg_replace('/^ | $/iU','',$name);
+
+	$name = preg_replace('/ /','_',$name);
+
+	$id = $bin['ID'];
+
+	if ($name == '')
+	{
+		$name = $id;
+	}
+
+	echo "\nname: ".$name." \n";
+
+	$dir = dirname(__FILE__);
+	$file = 'makenzb.php';
+
+//	echo "php {$dir}/{$file} {$id} {'$name'}";
+
+	$output = shell_exec("php {$dir}/{$file} {$id} {$name}");
+
+ 	echo "[Thread-{$thread}] Completed update for {$bin['name']}\n";
+}
+
+// Exit to call back to parent - Let know that child has completed
+exit(0);
+
+// Create callback function
+function psUpdateComplete()
+{
+	echo "Threaded export process complete\n";
+}
+
+?>