From c2cc24ef71e68d16050bc568a509bbdddbaa820f Mon Sep 17 00:00:00 2001 From: jonnyboy Date: Mon, 25 Mar 2013 00:37:36 -0400 Subject: [PATCH] changed output of threaded to show all threads, have verified they are still running threaded, not sequentially --- bin/backfill_parts.php | 10 +++++----- bin/backfill_parts_threaded.php | 20 ++++++++++---------- bin/backfill_threaded.php | 9 +++++---- bin/monitor.php | 2 +- bin/update_binaries_threaded.php | 9 +++++---- ugo/automake_threaded.php | 23 +++++++++++------------ 6 files changed, 37 insertions(+), 36 deletions(-) diff --git a/bin/backfill_parts.php b/bin/backfill_parts.php index 4806fbe8f..abac32d40 100644 --- a/bin/backfill_parts.php +++ b/bin/backfill_parts.php @@ -18,22 +18,23 @@ unset($array['']); $_date = $array['KEVIN_DATE']; $_parts = $array['KEVIN_PARTS']; -echo "Starting kevin123's backfill process\n\n"; $groupPost = $_parts; if (isset($argv[1])) { $groupName = $argv[1]; + echo "\n\n===>Starting kevin123's backfill process for ".$groupName."<===\n\n"; + $db = new DB; $query = $db->queryOneRow(sprintf("select name from groups WHERE (first_record_postdate BETWEEN '${_date}' and now()) and (name = '$groupName')")) or die(mysql_error()); if (isset($query)) { $backfill = new Backfill(); $backfill->backfillPostAllGroups($groupName, $groupPost); - echo "Backfilling completed.\n"; + echo "\n===>Backfilling completed for ".$groupName."\n"; } else { - echo "Already have the target post, skipping the group.\n"; + echo "\nAlready have the target post, skipping the ".$groupName."\n"; } } else @@ -72,8 +73,7 @@ function relativeTime($_time) { return $return; } -echo "\n\033[1;33mThis loop completed in: " .relativeTime($time). "\n"; -echo "\n\033[1;33mKevin123's backfill parts process completed in: " .relativeTime($time). "\n"; +echo "\nKevin123's backfill parts process completed in: " .relativeTime($time); ?> diff --git a/bin/backfill_parts_threaded.php b/bin/backfill_parts_threaded.php index f2b65afdc..0ee7e2e09 100644 --- a/bin/backfill_parts_threaded.php +++ b/bin/backfill_parts_threaded.php @@ -9,10 +9,10 @@ $groups = new Groups; $groupList = $groups->getActive(); unset($groups); -$ps = new PowerProcess; +$ps = new PowerProcess(); $ps->RegisterCallback('psUpdateComplete'); $ps->maxThreads = 5; -$ps->tickCount = 50000; // value in usecs. change this to 1000000 (one second) to reduce cpu use +$ps->tickCount = 500000; // value in usecs. change this to 1000000 (one second) to reduce cpu use $ps->threadTimeLimit = 0; // Disable child timeout echo "Starting kevin123's threaded backfill process\n\n"; @@ -27,22 +27,22 @@ while ($ps->RunControlCode()) { // Spawn another thread $ps->threadData = array_pop($groupList); - echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; + echo "\n[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; $ps->spawnThread(); } else { // There are no more slots available to run //$ps->tick(); - //echo ".\n"; + //echo "No threads avaialble.\n"; } } else { // No more groups to process - echo "No more groups to process - Initiating shutdown\n"; + echo "\nNo more groups to process - Initiating shutdown\n"; $ps->Shutdown(); - echo "Shutdown complete\n"; + echo "\nShutdown complete\n"; echo "\n\033[1;33m[Thread-MASTER] Kevin123's threaded backfill parts process completed in: " .relativeTime($time). "\n"; } } @@ -55,17 +55,17 @@ if ($ps->RunThreadCode()) $thread = sprintf("%05d",$ps->GetPID()); - echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n"; + echo "\n[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n"; $param = $group['name']; $dir = dirname(__FILE__); $file = 'backfill_parts.php'; - $output = shell_exec("php {$dir}/{$file} {$param}"); + $output = passthru("php {$dir}/{$file} {$param}"); //$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}"); - echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; + echo "\n[Thread-{$thread}] Completed update for group {$group['name']}\n"; } function relativeTime($_time) { @@ -103,7 +103,7 @@ exit(0); // Create callback function function psUpdateComplete() { - echo "\n\033[1;33m[Thread-MASTER] Kevin123's threaded backfill parts process completed in: " .relativeTime($time). "\n"; + echo "\n\033[1;33mKevin123's threaded backfill parts process completed in: " .relativeTime($time). "\n"; } diff --git a/bin/backfill_threaded.php b/bin/backfill_threaded.php index 9659c2f3d..e2df089ba 100644 --- a/bin/backfill_threaded.php +++ b/bin/backfill_threaded.php @@ -12,9 +12,10 @@ $groups = new Groups; $groupList = $groups->getActive(); unset($groups); -$ps = new PowerProcess; +$ps = new PowerProcess(); $ps->RegisterCallback('psUpdateComplete'); -$ps->maxThreads = 8; +$ps->maxThreads = 5; +$ps->tickCount = 500000; $ps->threadTimeLimit = 0; // Disable child timeout echo "Starting threaded backfill process\n"; @@ -79,7 +80,7 @@ if ($ps->RunThreadCode()) } //End Newzdash - $output = shell_exec("cd {$dir}/ && php {$file} {$param}"); + $output = passthru("cd {$dir}/ && php {$file} {$param}"); //Newzdash if ( $ndCommAllowed ) @@ -130,7 +131,7 @@ exit(0); // Create callback function function psUpdateComplete() { - echo "\n\033[1;33m[Thread-MASTER] Threaded backfill process completed in: " .relativeTime($time). "\n"; + echo "\n\033[1;33mThreaded stock backfill process completed in: " .relativeTime($time). "\n"; } ?> diff --git a/bin/monitor.php b/bin/monitor.php index e48a21184..ea60a9a68 100644 --- a/bin/monitor.php +++ b/bin/monitor.php @@ -2,7 +2,7 @@ require(dirname(__FILE__)."/config.php"); require(WWW_DIR.'/lib/postprocess.php'); -$version="0.1r749"; +$version="0.1r750"; $db = new DB(); diff --git a/bin/update_binaries_threaded.php b/bin/update_binaries_threaded.php index 605206182..0b2543aa3 100644 --- a/bin/update_binaries_threaded.php +++ b/bin/update_binaries_threaded.php @@ -12,9 +12,10 @@ $groups = new Groups; $groupList = $groups->getActive(); unset($groups); -$ps = new PowerProcess; +$ps = new PowerProcess(); $ps->RegisterCallback('psUpdateComplete'); -$ps->maxThreads = 8; +$ps->maxThreads = 5; +$ps->tickCount = 500000; $ps->threadTimeLimit = 0; // Disable child timeout echo "Starting threaded binary update process\n"; @@ -82,7 +83,7 @@ if ($ps->RunThreadCode()) $file = 'update_binaries.php'; //$output = shell_exec("php {$dir}/{$file} {$param}"); - $output = shell_exec("cd {$dir}/ && php {$file} {$param}"); + $output = passthru("cd {$dir}/ && php {$file} {$param}"); //$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}"); echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; @@ -134,7 +135,7 @@ exit(0); // Create callback function function psUpdateComplete() { - echo "[Thread-MASTER] Threaded backfill process completed in: " .relativeTime($time). "\n"; + echo "Threaded stock binaries update completed in: " .relativeTime($time). "\n"; } diff --git a/ugo/automake_threaded.php b/ugo/automake_threaded.php index 77baa0538..1abce670a 100644 --- a/ugo/automake_threaded.php +++ b/ugo/automake_threaded.php @@ -21,7 +21,7 @@ if (isset($argv[1])) } $rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6"); -$query = "SELECT ID FROM binaries WHERE `procstat` = 0 GROUP BY binaryhash order by ID"; +$query = "SELECT ID FROM binaries WHERE procstat = 0 GROUP BY binaryhash order by ID"; echo "Getting work to be done\n"; $theList = $db->query($query); @@ -29,11 +29,9 @@ unset($db); $theCount = count($theList); -$ps = new PowerProcess(); +$ps = new PowerProcess(4,0,false); $ps->RegisterCallback('psUpdateComplete'); -$ps->maxChildren = 10; -$ps->tickCount = 20000; // value in usecs. change this to 1000000 (one second) to reduce cpu use -$ps->threadTimeLimit = 0; // Disable child timeout +$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use $tim = microtime(true) - $time; echo "Query time = ".$tim."\n"; @@ -61,7 +59,7 @@ while ($ps->RunControlCode()) { // There are no more slots available to run //$ps->tick(); - //echo ".\n"; + //echo "\nNo more threads available"; } } else @@ -72,7 +70,7 @@ while ($ps->RunControlCode()) echo "Shutdown complete\n"; } } -$tim = microtime(true) - $time; +//$tim = microtime(true) - $time; // echo "time = ".$tim."\n"; unset($theList); @@ -96,17 +94,18 @@ if ($ps->RunThreadCode()) $cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT); if ($count % 200 == 0 && $count > 0) { + //$ps->tickCount = get_ticks(); $tim = microtime(true) - $time; - echo "\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 7, " ", STR_PAD_LEFT); + echo "\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT); $tim = number_format(($listcount * $tim) / 60 / $count, 1); - if ($countpct > .95 || ($count > 1000 && ($count-100) % 1000 == 0)) + if ($countpct > .95 || ($count > 1000 && ($count-200) % 1000 == 0)) echo "\t".$tim."m to go" ; echo "\n".str_pad($count,7, " ", STR_PAD_LEFT)."\t"; - } elseif (($count % 25 == 0) && ($count != 0)) { - echo " ."; + } elseif (($count % 4 == 0) && ($count != 0)) { + echo "."; } // echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n"; - $tim = microtime(true) - $time; +// $tim = microtime(true) - $time; // echo "time = ".$tim."\n"; }