starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 00:01:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated ugo's scripts, removed load check on threads, added misc_sorter, and disables all regexes if using ugo's assembler, added script to reactivate all regexes that were deactivated

Browse Source
This commit is contained in:
jonnyboy
2013-03-30 16:32:36 -04:00
parent 9fb09a0c5f
commit 81e8ab99b8
17 changed files with 7092 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/monitor.php
+33 -6
View File
@@ -2,7 +2,7 @@
require(dirname(__FILE__)."/config.php");
require(WWW_DIR.'/lib/postprocess.php');
$version="0.1r760";
$version="0.1r761";
$db = new DB();
@@ -49,6 +49,7 @@ $_alienx = dirname(__FILE__)."/../alienx";
$_conf = dirname(__FILE__)."/../conf";
$_powerline = dirname(__FILE__)."/../powerline";
$_cj = dirname(__FILE__)."/../nnscripts";
$_ugo = dirname(__FILE__)."/../ugo";
$_user = dirname(__FILE__)."/../user_scripts";
$NNPATH="{$array['NEWZPATH']}{$array['NEWZNAB_PATH']}";
$TESTING="{$array['NEWZPATH']}{$array['TESTING_PATH']}";
@@ -61,6 +62,14 @@ $_backfill_increment = "UPDATE groups set backfill_target=backfill_target+1 wher
$mysql_command_1 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_backfill_increment\"";
$reset_bin = "UPDATE binaries SET procstat=0, procattempts=0, regexID=NULL, relpart=0, reltotalpart=0, relname=NULL;";
$mysql_command_2 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$reset_bin\"";
$_active_regex = "select ID from releaseregex where status=1;";
$mysql_command_3 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_active_regex\" >> ../conf/active_regexes.txt";
$_disable_regex = "update releaseregex set status=0 where status=1;";
$mysql_command_4 = "$_mysql --defaults-file=$_conf/my.cnf -u$_DB_USER -h $_DB_HOST $_DB_NAME -e \"$_disable_regex\"";
if ( $array['UGO_THREADED'] == "true" ) {
shell_exec("$mysql_command_3 && $mysql_command_4");
}
//got microtime
function microtime_float()
@@ -159,6 +168,7 @@ $time17 = TIME();
$time18 = TIME();
$time19 = TIME();
$time20 = TIME();
$time21 = TIME();
if ( $array['INNODB'] == "true" ) {
$time5 = TIME();
@@ -351,7 +361,6 @@ while( $i > 0 )
printf("$a..");
sleep(1);
}
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)");
}
//defrag the query cache every 15 minutes
@@ -543,6 +552,14 @@ while( $i > 0 )
$panes4 = str_replace("\n", '', explode(" ", $panes_win_5));
$panes5 = str_replace("\n", '', explode(" ", $panes_win_6));
//reset binaries every 2 hours
if ((( TIME() - $time21 >= 7200 ) || ( $i == 1 )) && ($array['UGO_THREADED'] == "true" ))
{
$color = get_color();
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)");
shell_exec("$_tmux respawnp -k -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\n$panes0[5]\nKilled in to reset binaries\" && date +\"%D %T\" && echo \"This is color #$color\" && $ds1 $panes0[5] $ds4'");
}
//kill update_binaries.php backfill.php and import-nzb if timer exceeded
$killit=explode(" ", relativeTime("$newestdate"));
$killed="false";
@@ -1057,10 +1074,10 @@ while( $i > 0 )
}
//runs update_release and in 0.5 once if needed and exits
if (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
if (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $array['UGO_THREADED'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes0[5]);
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_bin && $_php update_releases.php && 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null");
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:0.5 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes0[5] $ds2 && cd $_ugo && $_php automake_threaded.php 2>&1 $log && cd $_bin && $_php update_releases.php 2>&1 $log && echo \" \033[1;0;33m\" && echo \"sleeping\033[38;5;\"$color\"m {$array['RELEASES_SLEEP']} seconds...\" && sleep {$array['RELEASES_SLEEP']} && $ds1 $panes0[5] $ds3' 2>&1 1> /dev/null");
} elseif (( $array['MAX_LOAD_RELEASES'] >= get_load()) && ( $array['RELEASES'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes0[5]);
@@ -1092,12 +1109,22 @@ while( $i > 0 )
}
//run $_php update_parsing.php in 1.1
if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
if (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ($array['MISC_SORTER'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes1[1]);
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_ugo && $_php misc_sorter_threaded.php 2>&1 $log && cd $TESTING && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null");
$time3 = TIME();
} elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ($array['PARSING_MOD'] != "true" ) && ($array['PARSING'] == "true" ) && ($array['MISC_SORTER'] != "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes1[1]);
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $TESTING && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null");
$time3 = TIME();
} elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
} elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ($array['MISC_SORTER'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes1[1]);
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_ugo && $_php misc_sorter_threaded.php 2>&1 $log && cd $_bin && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null");
$time3 = TIME();
} elseif (( $array['MAX_LOAD'] >= get_load()) && (( TIME() - $time3 ) >= $array['PARSING_TIMER'] ) && ( $array['PARSING_MOD'] == "true" ) && ( $array['PARSING'] == "true" ) && ($array['MISC_SORTER'] != "true" ) && ( $optimize_safe_to_run != "true" )) {
$color = get_color();
$log = writelog($panes1[1]);
shell_exec("$_tmux respawnp -t {$array['TMUX_SESSION']}:1.1 'echo \"\033[38;5;\"$color\"m\" && $ds1 $panes1[1] $ds2 && cd $_bin && $_php update_parsing.php 2>&1 $log && echo \" \033[1;0;33m\" && $ds1 $panes1[1] $ds3' 2>&1 1> /dev/null");
conf/.gitignore
+1
View File
@@ -1,2 +1,3 @@
tmux_user.conf
my.cnf
active_regexes.txt
conf/default_regexes.txt
+1711
View File
File diff suppressed because it is too large Load Diff
conf/jonnyboys_defaults.sh
+10 -1
View File
@@ -240,6 +240,12 @@ export MISC_ONLY="true"
############################################################
#run ugo's automake.php script to create releases, this does not use regexes and will run in a loop prior
#to update_releases.php
export UGO_THREADED="true"
############################################################
#Create releases, this is really only necessary to turn off when you only want to post process
export RELEASES="true"
@@ -300,6 +306,9 @@ export PARSING="true"
#this also includes kevin123's categorymod.php, you must run either update_svn.sh or fix_files.sh to copy the file into place
export PARSING_MOD="true"
#choose to use ugo's misc_sorter3.php script also, this will run before update_parsing, in the same loop
export MISC_SORTER="true"
#run update_parsing.php against the whole db or just the last 24 hours
export PAST_24_HOURS="false"
@@ -467,7 +476,7 @@ export RAMDISK_PATH="/var/ramdisk"
###########################################################
#logs can be written, per pane, to the logs folder
export WRITE_LOGS="true"
export WRITE_LOGS="truel"
###########################################################
config.sh
+12
View File
@@ -239,6 +239,15 @@ export MISC_ONLY="false"
############################################################
#run ugo's automake.php script to create releases, this does not use regexes and will run in a loop prior
#to update_releases.php, this can be considerably slower, but may give you release that were being missed
#this might just overwhelm your db, so expect things to move slower, but you might get more in return
#this will deactivate all regexes, to reactive the regexes, run scripts/reactivate_regexes.sh
#still a wip
export UGO_THREADED="false"
############################################################
#Create releases, this is really only necessary to turn off when you only want to post process
export RELEASES="false"
@@ -299,6 +308,9 @@ export PARSING="false"
#this also includes kevin123's categorymod.php, you must run either update_svn.sh or fix_files.sh to copy the file into place
export PARSING_MOD="false"
#choose to use ugo's misc_sorter3.php script also, this will run before update_parsing, in the same loop
export MISC_SORTER="false"
#run update_parsing.php against the whole db or just the last 24 hours
export PAST_24_HOURS="true"
scripts/reactivate_regexes.sh
Executable
+16
View File
@@ -0,0 +1,16 @@
SOURCE="${BASH_SOURCE[0]}"
DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
cd $DIR"/../conf"
while read p; do
mysql --defaults-file=my.cnf -uroot newznab -e "update releaseregex set status=1 where status=0 and ID=$p"
done < active_regexes.txt
rm active_regexes.txt
ugo/automake.php
+1566
View File
File diff suppressed because it is too large Load Diff
ugo/automake_threaded.php
+155
View File
@@ -0,0 +1,155 @@
<?php
//define('FS_ROOT', realpath(dirname(__FILE__)));
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/powerprocess.php");
require_once(WWW_DIR."/lib/framework/db.php");
$time = microtime(true);
$db = new DB();
$cont = true;
if (isset($argv[1]))
{
//var_dump($argv);
if ($argv[1] == "reset")
{
echo "resetting binaries\n";
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)");
echo "binaries have been reset\n";
} else if ($argv[1] == "cont") {
$cont = false;
}
}
echo "cleaning spam..... can take a few moments\n";
//$db->disableAutoCommit();
//$db->disableForeignKeyChecks();
$rel = $db->query("SELECT `ID` from `binaries` WHERE `xref` REGEXP '(.+:[0-9]+ ){5,}' AND `procstat` = 0");
foreach($rel as $r)
{
$db->query("UPDATE `binaries` set `procstat` = -2 WHERE `ID` = ".$r['ID']);
if ($db->getLastError() != '')
var_dump($db->getLastError());
}
echo "spam count ".count($rel)."\n";
//$db->commit();
//$db->enableAutoCommit();
//$db->enableForeignKeyChecks();
if ($cont)
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6");
echo "Getting work to be done\n";
$query = "SELECT ID FROM binaries WHERE `procstat` = 0 GROUP BY binaryhash order by RAND()";
$theList = $db->query($query);
unset($db);
$theCount = count($theList);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxChildren = 8;
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 0; // Disable child timeout
$tim = microtime(true) - $time;
echo "time = ".$tim."\n";
echo "Starting threaded assembly process\n";
echo "\nToday will be doing $theCount binaries\n";
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
while ($ps->RunControlCode())
{
// Start the parent loop
$listcount = count($theList);
if ($listcount)
{
// We still have groups to process
if ($ps->SpawnReady())
{
// Spawn another thread
$ps->threadData = array_pop($theList);
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." post(s) to update after this\n";
$ps->spawnThread();
}
}
else
{
// No more groups to process
echo "\nNo more posts to process - Initiating shutdown\n";
$ps->threadTimeLimit = 200;
do {
$ps->tick();
} while ($ps->myThreads);
$ps->Shutdown();
echo "\nShutdown complete\n";
$tim = microtime(true) - $time;
echo "final time = ".$tim."\n";
}
}
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
unset($theList);
if ($ps->RunThreadCode())
{
$post = $ps->threadData;
$thread = sprintf("%05d",$ps->GetPID());
// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n";
$param = $post['ID'];
$dir = dirname(__FILE__);
$file = 'automake.php';
$output = shell_exec("php {$dir}/{$file} {$param}");
$count = $theCount - $listcount;
$countpct = $count / $theCount;
$cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT);
if ($listcount % 250 == 0 && $listcount > 0)
{
$tim = microtime(true) - $time;
echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
$tim = ($listcount * $tim) / $count + time();
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0))
echo "\t".date('M jS Y H:i:s', $tim);
// echo "\t".$tim."m to go" ;
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
} elseif ($listcount % 25 == 0)
echo " .";
// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n";
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
}
// Exit to call back to parent - Let know that child has completed
exit(0);
// Create callback function
function psUpdateComplete()
{
echo "[Thread-MASTER] Threaded assembly process complete\n";
}
?>
ugo/finish_postproc.php
+39
View File
@@ -0,0 +1,39 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/postprocess.php");
require_once(WWW_DIR."/lib/framework/db.php");
$db = new DB;
$postprocess = new PostProcess(true);
while (true) {
$thequery = "SELECT count(r.ID) AS thecount FROM releases r LEFT OUTER JOIN category c ON c.ID = r.categoryID WHERE (r.passwordstatus between -10 and -1) or (r.haspreview = -1 and c.disablepreview = 0) order by rand()";
$result = $db->query($thequery);
if ($result[0]["thecount"] < 10 )
{
$postprocess->processTv();
$postprocess->processAll();
$postprocess->processSpotNab();
echo "done...waiting";
sleep(600);
} else {
for ($i = 0; $i < $result[0]["thecount"]; $i=$i+20)
{
echo "\n";
echo $result[0]["thecount"] - $i. " more to go: \n";
// $postprocess->processAdditional();
$postprocess->processNfos();
$postprocess->processUnwanted();
$postprocess->processMovies();
$postprocess->processMusic();
$postprocess->processBooks();
$postprocess->processGames();
$postprocess->processTv();
$postprocess->processMusicFromMediaInfo();
$postprocess->processOtherMiscCategory();
$postprocess->processUnknownCategory();
}
}
}
ugo/makenzb.php
+120
View File
@@ -0,0 +1,120 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/framework/db.php");
function makenzb($id, $name = '')
{
global $db;
$ok = true;
$path = FS_ROOT."/nzbs/";
// $query = "SELECT * FROM `binaries` WHERE `ID` = ".$id;
$query = "SELECT *, sum(parts.size) AS size FROM binaries INNER JOIN parts ON binaries.ID = parts.binaryID WHERE binaries.ID = ".$id;
$row = $db->queryOneRow($query);
echo $row['name']." is ".($row['size']/1024/1024)." MB\n";
if (!isset($name))
{
$name = $row['relname'];
}
if (preg_match('/\.nzb\b/i', $row['name']) && $row['size'] < 10000000)
{
echo "doing nzb\n";
$nntp = new Nntp;
if ($nntp->doConnect())
{
$bin = $nntp->getBinary($id);
echo strlen($bin)."\n";
$name = preg_replace('/ /','_',$name);
if (file_exists($path.$name.".nzb"))
{
$name = $name." ".$id;
}
echo $name." ".strlen($bin)."\n";
if (strlen($bin) == 0)
$ok = false;
elseif (file_put_contents($path.$name.".nzb", $bin) == 0)
$ok = false;
echo $ok."\n";
}
$nntp->doQuit();
unset($nntp);
unset($bin);
}
if ($ok)
{
echo "clearing \n";
$query = "UPDATE `binaries` SET `procstat` = 6 WHERE ID = ".$id;
echo $query."\n";
$db->query($query);
}
}
mb_internal_encoding("UTF-8");
$db = new Db;
if (isset($argv[1]))
{
if (isset($argv[2]))
{
makenzb($argv[1], $argv[2]);
} else {
makenzb($argv[1]);
}
} else {
$db->query("UPDATE `binaries` SET `binaries`.`procstat` = 6 WHERE `binaries`.`procstat` = - 3 AND binaries.ID NOT IN ( SELECT `parts`.`binaryID` FROM `parts` )");
$rel = $db->query("SELECT * FROM `binaries` WHERE `procstat` = -3");
foreach($rel as $bin)
{
$name = $bin['relname'];
$name = preg_replace('/yenc/iU',' ',$name);
$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
$name = preg_replace('/[\`\'\"\:\/\-\<\>]/iU',' ',$name);
$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
$name = preg_replace('/ +?/iU',' ',$name);
$name = preg_replace('/^ | $/iU','',$name);
$name = preg_replace('/ /','_',$name);
$id = $bin['ID'];
echo "doing $name\n";
makenzb($id, $name);
}
}
unset($db);
?>
ugo/makenzb_threaded.php
+210
View File
@@ -0,0 +1,210 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/groups.php");
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/powerprocess.php");
require_once(WWW_DIR."/lib/framework/db.php");
$time = microtime(true);
$db = new DB();
$query = "SELECT * FROM `binaries` WHERE `procstat` = -3";
$theList = $db->query($query);
unset($db);
$theCount = count($theList);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxThreads = 20;
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 0; // Disable child timeout
$tim = microtime(true) - $time;
echo "time = ".$tim."\n";
echo "Starting threaded nzb exporter process\n";
echo "\nToday will be doing $theCount binaries\n";
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
while ($ps->RunControlCode())
{
// Start the parent loop
$listcount = count($theList);
if ($listcount)
{
// We still have groups to process
if ($ps->SpawnReady())
{
// Spawn another thread
$ps->threadData = array_pop($theList);
//echo "[Thread-MASTER] Spawning new thread. Still have " . count($rel) ." nzb(s) to update after this\n";
$ps->spawnThread();
}
else
{
// There are no more slots available to run
//$ps->tick();
//echo ".\n";
}
}
else
{
// No more groups to process
// $query = "UPDATE `binaries` SET `procstat` = 6 WHERE `procstat` = -3 OR `procstat` = -2";
// $db = new Db();
// var_dump($db->query($query));
echo "\nNo more nzbs to process - Initiating shutdown\n";
$ps->threadTimeLimit = 200;
do {
$ps->tick();
} while ($ps->myThreads);
$ps->Shutdown();
echo "\nShutdown complete\n";
$path = FS_ROOT."/nzbs/";
$filestoprocess = glob($path."*.nzb");
foreach($filestoprocess as $nzbFile)
{
$xml = file_get_contents( $nzbFile );
$matches = preg_split('/\A(\<\?xml[^\r\n]+?\?\>)\s*\R+/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
$name = preg_split('/\.nzb$/i', $nzbFile, 0, PREG_SPLIT_DELIM_CAPTURE);
$name = $name[0];
// $marker = $matches[0];
//
// if (strlen($marker) == 0 || strlen($marker) >100)
// {
// $marker = $matches[1];
// }
$marker = '';
foreach ($matches as $m)
{
if (preg_match('/^(\<\?xml[^\r\n]+?\?\>)/iU', $m))
{
echo "$m\n";
$marker = $m;
continue;
}
}
echo "marker "."/(".preg_quote($marker)."\s*\R+)/iU\n";
if (!empty($marker))
{
$matches = preg_split('/('.preg_quote($marker).'\s*\R+)/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
if (strlen($matches[0]) == 0)
{
unset($matches[0]);
$matches = array_values($matches);
}
echo count($matches)."\n";
if (count($matches) > 2)
{
for ($i = 0; $i < count($matches); $i = $i + 2)
{
echo $i."\n";
$outxml = $matches[$i] . $matches[$i + 1];
$fileout = $name . '.' . $i . '.nzb';
file_put_contents($fileout, $outxml);
echo $fileout."\n";
}
}
}
}
$tim = microtime(true) - $time;
echo "final time = ".$tim."\n";
}
}
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
unset($theList);
unset($db);
if ($ps->RunThreadCode())
{
$bin = $ps->threadData;
$thread = sprintf("%05d",$ps->GetPID());
// echo "[Thread-{$thread}] Begining processing for nzb {$bin['ID']}\n";
$name = $bin['relname'];
$name = preg_replace('/yenc/iU',' ',$name);
$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
$name = preg_replace('/[\`\'\"\:\/\-\<\>\|]/iU',' ',$name);
$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
$name = preg_replace('/ +?/iU',' ',$name);
$name = preg_replace('/^ | $/iU','',$name);
$name = preg_replace('/ /','_',$name);
$id = $bin['ID'];
if ($name == '')
{
$name = $id;
}
// echo "\nname: ".$name." \n";
$dir = dirname(__FILE__);
$file = 'makenzb.php';
// echo "php {$dir}/{$file} {$id} {$name}\n";
$output = shell_exec("php {$dir}/{$file} {$id} {$name}");
$count = $theCount - $listcount;
$countpct = $count / $theCount;
if ($listcount % 100 == 0 && $listcount > 0)
{
$tim = microtime(true) - $time;
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
if ($countpct > 0.1 || $count > 1500)
echo "\t".date('H:i:s', $tim);
// echo "\t".$tim."m to go" ;
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
}
if ($listcount % 10 == 0)
echo " .";
// echo "[Thread-{$thread}] Completed update for {$bin['name']}\n";
}
// Exit to call back to parent - Let know that child has completed
exit(0);
// Create callback function
function psUpdateComplete()
{
echo "Threaded export process complete\n";
}
?>
ugo/misc_sorter3.php
+2335
View File
File diff suppressed because it is too large Load Diff
ugo/misc_sorter_cleanup.php
+98
View File
@@ -0,0 +1,98 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/releases.php");
require_once(WWW_DIR."/lib/movie.php");
require_once(WWW_DIR."/lib/tvrage.php");
require_once(WWW_DIR."/lib/amazon.php");
require_once(WWW_DIR."/lib/anidb.php");
require_once(WWW_DIR."/lib/nzbinfo.php");
require_once(WWW_DIR."/lib/nzb.php");
require_once(WWW_DIR."/lib/book.php");
require_once(WWW_DIR."/lib/music.php");
$movie = new Movie();
$movie->processMovieReleases();
$movie->updateUpcoming();
$music = new Music(true);
$music->processMusicReleases();
$book = new Book(true);
$book->processBookReleases();
$anidb = new AniDB(true);
$anidb->animetitlesUpdate();
$anidb->processAnimeReleases();
$tvrage = new TVRage(true);
$tvrage->processTvReleases(true);
$thetvdb = new TheTVDB(true);
$thetvdb->processReleases();
echo "cleaning 5000\n";
$db = new DB();
$query = "update releases set releases.categoryID= 5060 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE movieinfo.genre REGEXP 'sport' AND NOT movieinfo.genre REGEXP 'comedy' AND NOT movieinfo.genre REGEXP 'romance' AND NOT movieinfo.genre REGEXP 'drama' )";
$db->query($query);
$query = "update releases set releases.categoryID= 5000 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'tv|talk\-show|reality|episode' ) AND categoryID NOT IN ( SELECT ID FROM category WHERE parentID = 5000 )";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 10500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 2500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2020 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000)";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2040 WHERE `categoryID` = 2050 AND size <10000000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=2030 WHERE `categoryID` = 2040 AND size < 2500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=5040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 1500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=5030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 100000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=5050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000)";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=5030 WHERE `categoryID` = 5040 AND size < 1500000000";
$db->query($query);
$query = "UPDATE `releases` SET `categoryID`=5050 WHERE `categoryID` = 5030 AND size < 100000000";
$db->query($query);
$query = "UPDATE releases SET categoryID = 5070 WHERE rageID IN ( SELECT tvrage.rageID FROM tvrage WHERE tvrage.genre REGEXP 'animat' )";
$db->query($query);
$query = "update releases set releases.categoryID= 5080 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'docum' )";
$db->query($query);
?>
ugo/misc_sorter_threaded.php
+140
View File
@@ -0,0 +1,140 @@
<?php
//define('FS_ROOT', realpath(dirname(__FILE__)));
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/powerprocess.php");
require_once(WWW_DIR."/lib/framework/db.php");
$time = microtime(true);
$db = new DB();
$thecategory = 8000;
if (isset($argv[1]))
{
$thecategory = substr($argv[1], 0, 1)."000";
}
echo "Starting sorting process for category $thecategory\n";
$query = "SELECT ID FROM category WHERE parentID = $thecategory"; // OR parentID = 2000 OR parentID = 5000
$res = $db->query($query);
$thecategory = $res[0]['ID'];
unset($res[0]);
foreach($res as $r)
$thecategory = $thecategory.", ".$r['ID'];
$query = "SELECT releases.ID, releases.`name` FROM releases WHERE releases.categoryID IN ( $thecategory )";
$theList = $db->query($query);
unset($db);
$theCount = count($theList);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxChildren = 24;
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 0; // Disable child timeout
$tim = microtime(true) - $time;
echo "time = ".$tim."\n";
echo "\nToday will be going over $theCount releases\n";
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
while ($ps->RunControlCode())
{
// Start the parent loop
$listcount = count($theList);
if ($listcount)
{
// We still have groups to process
if ($ps->SpawnReady())
{
// Spawn another thread
$ps->threadData = array_pop($theList);
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." posts to sort after this\n";
$ps->spawnThread();
}
else
{
// There are no more slots available to run
//$ps->tick();
//echo ".\n";
}
}
else
{
// No more groups to process
echo "\nNo more to process - Initiating shutdown\n";
$ps->threadTimeLimit = 200;
do {
$ps->tick();
} while ($ps->myThreads);
$ps->Shutdown();
echo "\nShutdown complete\n";
$tim = microtime(true) - $time;
echo "final processing time = ".$tim."\n";
}
}
$tim = microtime(true) - $time;
//echo "time = ".$tim."\n";
unset($theList);
if ($ps->RunThreadCode())
{
$post = $ps->threadData;
$thread = sprintf("%05d",$ps->GetPID());
// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n";
$param = $post['ID'];
$dir = dirname(__FILE__);
$file = 'misc_sorter3.php';
$output = shell_exec("php {$dir}/{$file} {$param}");
$count = $theCount - $listcount;
$countpct = $count / $theCount;
$cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT);
if ($listcount % 250 == 0 && $listcount > 0)
{
$tim = microtime(true) - $time;
echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
$tim = ($listcount * $tim) / $count + time();
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0))
echo "\t".date('M jS Y H:i:s', $tim);
// echo "\t".$tim."m to go" ;
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
} elseif ($listcount % 25 == 0)
echo " .";
// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n";
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
}
// Exit to call back to parent - Let know that child has completed
exit(0);
// Create callback function
function psUpdateComplete()
{
echo "[Thread-MASTER] Threaded sorting process complete\n";
}
?>
ugo/postproc.php
+355
View File
@@ -0,0 +1,355 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/postprocess.php");
require_once(WWW_DIR."/lib/site.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/nzbinfo.php");
require_once(WWW_DIR."/lib/releases.php");
require_once(WWW_DIR."/lib/powerprocess.php");
function watchdog ()
{
global $buffer, $pid;
if (isset($buffer))
{
$ps = shm_get_var($buffer, 1);
if (isset($ps->myThreads[$pid]))
if ($ps->myThreads[$pid]['time'] != time())
{
$ps->myThreads[$pid]['time'] = time();
$lastto = time();
shm_put_var($buffer, 1, $ps);
// trigger_error($pid." ".$ps->myThreads[$pid]['time']);
}
} else {
trigger_error("$pid is orphaned");
}
}
$postprocess = new PostProcess(true);
$s = new Sites();
$site = $s->get();
$nntp = new Nntp;
$db = new DB;
$nzb = new Nzb;
$maxattemptstocheckpassworded = 5;
$numtoProcess = 5;
$processVideoSample = ($site->ffmpegpath != '') ? true : false;
$processMediainfo = ($site->mediainfopath != '') ? true : false;
$processPasswords = ($site->unrarpath != '') ? true : false;
$processAudioSample = ($site->saveaudiopreview == 1) ? true : false;
$nntpconnected = false;
$lastto = time()+1000;
var_dump($argv);
if (is_numeric($argv[1]))
{
if (isset($argv[3]))
{
$buffer = shm_attach($argv[2]);
$pid = $argv[3];
}
$rel = $db->query(sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r
left join category c on c.ID = r.categoryID
where r.ID = %d ", $argv[1]));
$rel = $rel[0];
if ($rel !== false)
{
$tmpPath = $site->tmpunrarpath;
if (substr($tmpPath, -strlen( '/' ) ) != '/')
{
$tmpPath = $tmpPath.'/';
}
$tmpPath = $tmpPath.$rel['guid'].'/';
if (!file_exists($tmpPath))
mkdir($tmpPath, 0766, true);
// Per release defaults
$passStatus = array(Releases::PASSWD_NONE);
$blnTookMediainfo = false;
$blnTookSample = ($rel['disablepreview'] == 1) ? true : false; //only attempt sample if not disabled
if ($blnTookSample)
$db->query(sprintf("update releases set haspreview = 0 where id = %d", $rel['ID']));
//
// Go through the binaries for this release looking for a rar, a sample, and a mediafile
//
$nzbInfo = new nzbInfo;
$norar = 0;
// only load nzbs and check for rar files if we are doing something with them.
if ($processVideoSample || $processMediainfo || $processPasswords || $processAudioSample)
{
$nzbfile = $nzb->getNZBPath($rel['guid'], $site ->nzbpath);
if (!$nzbInfo->loadFromFile($nzbfile))
{
goto end;
}
foreach($nzbInfo->nzb as $nzbsubject)
{
if (preg_match("/\w\.r00/i", $nzbsubject['subject']))
$norar= 1;
}
}
// attempt to process video sample file
if(!empty($nzbInfo->samplefiles) && $processVideoSample && $blnTookSample === false)
{
$sampleFile = $nzbInfo->samplefiles[0]; //first detected sample
$sampleMsgids = array_slice($sampleFile['segments'], 0, 1); //get first segment, increase to get more of the sample
$sampleGroup = $sampleFile['groups'][0];
echo "PostPrc : Fetching ".implode($sampleMsgids, ', ')." from {$sampleGroup}\n";
if (!$nntpconnected)
$nntpconnected = $nntp->doConnect();
if ($nntpconnected) {
$sampleBinary = $nntp->getMessages($sampleGroup, $sampleMsgids);
watchdog ();
if ($sampleBinary === false)
echo "\nPostPrc : Couldnt fetch sample\n";
else
{
$samplefile = $tmpPath.'sample.avi';
file_put_contents($samplefile, $sampleBinary);
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
if ($blnTookSample)
$postprocess->updateReleaseHasPreview($rel['guid']);
unlink($samplefile);
}
unset($sampleBinary);
} else {
echo "\nFailed to get NNTP connection\n";
}
}
// attempt to process loose media file
if(!empty($nzbInfo->mediafiles) && (($processVideoSample && $blnTookSample === false) || $processMediainfo))
{
$mediaFile = $nzbInfo->mediafiles[0]; //first detected media file
$mediaMsgids = array_slice($mediaFile['segments'], 0, 2); //get first two segments
$mediaGroup = $mediaFile['groups'][0];
echo "PostPrc : Fetching ".implode($mediaMsgids, ', ')." from {$mediaGroup}\n";
if (!$nntpconnected)
$nntpconnected = $nntp->doConnect();
if ($nntpconnected) {
$mediaBinary = $nntp->getMessages($mediaGroup, $mediaMsgids);
watchdog ();
if ($mediaBinary === false)
echo "\nPostPrc : Couldnt fetch media file\n";
else
{
$mediafile = $tmpPath.'sample.avi';
file_put_contents($mediafile, $mediaBinary);
if ($processVideoSample && $blnTookSample === false)
{
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
if ($blnTookSample)
$postprocess->updateReleaseHasPreview($rel['guid']);
}
if ($processMediainfo)
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
unlink($mediafile);
}
unset($mediaBinary);
} else {
echo "\nFailed to get NNTP connection\n";
}
}
// attempt to process audio sample file
if(!empty($nzbInfo->audiofiles) && $processAudioSample && $blnTookSample === false)
{
$audioFile = $nzbInfo->audiofiles[0]; //first detected audio file
$audioMsgids = array_slice($audioFile['segments'], 0, 1); //get first segment
$audioGroup = $audioFile['groups'][0];
echo "PostPrc : Fetching ".implode($audioMsgids, ', ')." from {$audioGroup}\n";
if (!$nntpconnected)
$nntpconnected = $nntp->doConnect();
if ($nntpconnected) {
$audioBinary = $nntp->getMessages($audioGroup, $audioMsgids);
watchdog ();
if ($audioBinary === false)
echo "\nPostPrc : Couldnt fetch audio sample\n";
else
{
$audiofile = $tmpPath.'sample.mp3';
file_put_contents($audiofile, $audioBinary);
$blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']);
if ($blnTookSample !== false)
$postprocess->updateReleaseHasPreview($rel['guid'], 2);
if ($processMediainfo)
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
if ($site ->lamepath != "")
$postprocess->lameAudioSample($site ->lamepath, $rel['guid']);
unlink($audiofile);
}
unset($audioBinary);
} else {
echo "\nFailed to get NNTP connection\n";
}
}
if (!empty($nzbInfo->rarfiles) && ($site ->checkpasswordedrar > 0 || (($processVideoSample || $processAudioSample) && $blnTookSample === false) || $processMediainfo))
{
$mysqlkeepalive = 0;
foreach($nzbInfo->rarfiles as $rarFile)
{
//dont process any more rars if a passworded rar has been detected and the site is set to automatically delete them
if ($site ->deletepasswordedrelease == 1 && max($passStatus) == Releases::PASSWD_RAR)
{
echo "-Skipping processing of rar {$rarFile['subject']} as this release has already been marked as passworded.\n";
goto end;
}
$rarMsgids = array_slice($rarFile['segments'], 0, 1); //get first segment
$rarGroup = $rarFile['groups'][0];
echo "PostPrc : Fetching ".implode($rarMsgids, ', ')." from {$rarGroup} (".++$mysqlkeepalive.")\n";
if (!$nntpconnected)
$nntpconnected = $nntp->doConnect();
if ($nntpconnected) {
$fetchedBinary = $nntp->getMessages($rarGroup, $rarMsgids);
watchdog ();
if ($fetchedBinary === false)
{
echo "\nPostPrc : Failed fetching rar file\n";
$db->query(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $rel['ID']));
goto end;
}
else
{
$relFiles = $postprocess->processReleaseFiles($fetchedBinary, $rel['ID']);
if ($site ->checkpasswordedrar > 0 && $processPasswords)
{
$passStatus[] = $postprocess->processReleasePasswords($fetchedBinary, $tmpPath, $site ->unrarpath, $site ->checkpasswordedrar);
}
// we need to unrar the fetched binary if checkpasswordedrar wasnt 2
if ($site ->checkpasswordedrar < 2 && $processPasswords)
{
$rarfile = $tmpPath.'rarfile.rar';
file_put_contents($rarfile, $fetchedBinary);
$execstring = '"'.$site ->unrarpath.'" e -ai -ep -c- -id -r -kb -p- -y -inul "'.$rarfile.'" "'.$tmpPath.'"';
$output = runCmd($execstring, false, true);
unlink($rarfile);
}
if ($processVideoSample && $blnTookSample === false)
{
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
if ($blnTookSample)
$postprocess->updateReleaseHasPreview($rel['guid']);
}
$blnTookAudioSample = false;
if ($processAudioSample && $blnTookSample === false)
{
$blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']);
if ($blnTookSample)
{
$blnTookAudioSample = true;
$postprocess->updateReleaseHasPreview($rel['guid'], 2);
}
}
if ($processMediainfo && $blnTookMediainfo === false)
{
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
}
//
// Has to be done after mediainfo
//
if ($blnTookAudioSample && $site ->lamepath != "")
$postprocess->lameAudioSample($site ->lamepath, $rel['guid']);
if ($mysqlkeepalive % 25 == 0)
{
$db->query("select 1");
watchdog ();
}
}
//clean up all files
foreach(glob($tmpPath.'*') as $v)
{
unlink($v);
}
} else {
echo "\nFailed to get NNTP connection\n";
}
} //end foreach msgid
}
elseif(empty($nzbInfo->rarfiles) && $norar == 1)
{
$passStatus[] = Releases::PASSWD_POTENTIAL;
}
$hpsql = '';
if (!$blnTookSample)
$hpsql = ', haspreview = 0';
$sql = sprintf("update releases set passwordstatus = %d %s where ID = %d", max($passStatus), $hpsql, $rel["ID"]);
$db->query($sql);
}
end:
foreach(glob($tmpPath.'*') as $v)
{
unlink($v);
}
foreach(glob($tmpPath.'.*') as $v)
{
@unlink($v);
}
rmdir($tmpPath);
if ($nntpconnected)
{
$nntp->doQuit();
}
unset($postprocess);
unset($s);
unset($site);
unset($nntp);
unset($db);
unset($nzb);
unset($nzbInfo);
}
?>
ugo/postproc_threaded.php
+171
View File
@@ -0,0 +1,171 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/powerprocess.php");
require_once(WWW_DIR."/lib/framework/db.php");
function processchanges ()
{
global $ps, $buffer;
if (count($ps->myThreads) < 1 || !isset($buffer))
return false;
$ps1 = shm_get_var($buffer, 1);
$change = false;
$ps1c = count($ps1->myThreads);
$psc = count($ps->myThreads);
foreach($ps->myThreads as $k => $v)
{
if (isset($ps1->myThreads[$k]))
{
if ($v['time'] != $ps1->myThreads[$k]['time'])
{
$q = (strval($ps1->myThreads[$k]['time']) - strval($ps->myThreads[$k]['time']));
// echo "time changed for $q\n";
$ps->myThreads[$k]['time'] = $ps1->myThreads[$k]['time'];
$change = true;
}
} else {
// echo "ps not current $psc $ps1c\n";
$change = true;
}
}
if ($change)
shm_put_var($buffer, 1, $ps);
// echo "pause ";
// var_dump($ps->myThreads);
}
$time = microtime(true);
$db = new DB();
$maxattemptstocheckpassworded = 10;
$numtoProcess = 1000;
$query = sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r
left join category c on c.ID = r.categoryID
where (r.passwordstatus between %d and -1)
or (r.haspreview = -1 and c.disablepreview = 0) order by RAND() limit %d ", ($maxattemptstocheckpassworded + 1) * -1, $numtoProcess);
//$query = "SELECT releases.ID, releases.rarinnerfilecount, releases.haspreview, releases.passwordstatus, releases.releasenfoID, releases.guid FROM releases WHERE releases.ID NOT IN ( SELECT releasefiles.releaseID FROM releasefiles ) AND releases.rageID < 1 AND releases.imdbID < 1 OR releases.rarinnerfilecount < 0";
$theList = $db->query($query);
unset($db);
$theCount = count($theList);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxThreads = 16;
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 350; // Disable child timeout
$handle = ftok(__FILE__, chr(1));
$buffer = shm_attach($handle, 10240);
$tim = microtime(true) - $time;
echo "time = ".$tim."\n";
echo "Starting threaded rar inspection process\n";
echo "\nToday will be be inspecting $theCount binaries\n";
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
while ($ps->RunControlCode())
{
// Start the parent loop
$listcount = count($theList);
if ($listcount)
{
// We still have groups to process
if ($ps->SpawnReady())
{
// Spawn another thread
$ps->threadData = array_pop($theList);
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n";
$ps->spawnThread();
processchanges();
}
else
{
//There are no more slots available to run
$ps->tick();
//echo ".\n";
processchanges();
}
}
else
{
// No more groups to process
echo "\nNo more release to inspect - Initiating shutdown\n";
do {
$ps->tick();
processchanges();
} while ($ps->myThreads);
$ps->Shutdown();
echo "\nShutdown complete\n";
// var_dump($ps->myThreads);
$tim = microtime(true) - $time;
echo "final time = ".$tim."\n";
}
}
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
unset($theList);
if ($ps->RunThreadCode())
{
$release = $ps->threadData;
$thread = sprintf("%05d",$ps->GetPID());
// echo "[Thread-{$thread}] Begining rar inspection for {$release['name']} {$handle} id= {$param}\n";
$param = $release['ID'];
$dir = dirname(__FILE__);
$file = 'postproc.php';
//echo "start ";
//var_dump($ps->myThreads);
$output = shell_exec("php {$dir}/{$file} {$param} {$handle} {$thread}");
$count = $theCount - $listcount;
$countpct = $count / $theCount;
if ($listcount % 50 == 0 && $count > 0)
{
$tim = microtime(true) - $time;
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
if ($countpct > 0.1 || $count > 1500)
echo "\t".date('H:i:s', $tim);
// echo "\t".$tim."m to go" ;
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
}
if ($listcount % 2 == 0)
echo " .";
// echo "[Thread-{$thread}] Completed update for binary {$release['name']}\n";
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
}
// Exit to call back to parent - Let know that child has completed
exit(0);
// Create callback function
function psUpdateComplete()
{
echo "[Thread-MASTER] Threaded inspection process complete\n";
}
?>
ugo/update_binaries_threaded.php
+120
View File
@@ -0,0 +1,120 @@
<?php
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."/lib/groups.php");
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/powerprocess.php");
//$groups = new Groups;
//$groupList = $groups->getActive();
$query = "SELECT * FROM groups WHERE groups.active = 1 ORDER BY ( groups.last_record - groups.first_record ) ASC";
$time = microtime(true);
$db = new DB();
$theList = $db->query($query);
//unset($groups);
unset($db);
$theCount = count($theList);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxThreads = 15;
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 1500; // Disable child timeout
$tim = microtime(true) - $time;
echo "time = ".$tim."\n";
echo "Starting threaded binary update process\n";
echo "\nToday will be be inspecting $theCount binaries\n";
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
while ($ps->RunControlCode())
{
// Start the parent loop
$listcount = count($theList);
if ($listcount)
{
// We still have groups to process
if ($ps->SpawnReady())
{
// Spawn another thread
$ps->threadData = array_pop($theList);
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n";
$ps->spawnThread();
}
else
{
// There are no more slots available to run
$ps->tick();
//echo "- \n";
}
}
else
{
// No more groups to process
echo "\nNo more groups to process - Initiating shutdown\n";
do {
$ps->tick();
} while ($ps->myThreads);
$ps->Shutdown();
echo "\nShutdown complete\n";
$tim = microtime(true) - $time;
echo "final time = ".$tim."\n";
}
}
$tim = microtime(true) - $time;
// echo "time = ".$tim."\n";
unset($theList);
if ($ps->RunThreadCode())
{
$group = $ps->threadData;
$thread = sprintf("%05d",$ps->GetPID());
// echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n";
$param = $group['name'];
$dir = dirname(__FILE__);
$file = 'update_binaries.php';
$output = shell_exec("php {$dir}/{$file} {$param}");
//$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}");
$count = $theCount - $listcount;
$countpct = $count / $theCount;
if ($listcount % 12 == 0 && $count > 0)
{
$tim = microtime(true) - $time;
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
if ($countpct > 0.1 || $count > 1500)
echo "\t".date('H:i:s', $tim);
// echo "\t".$tim."m to go" ;
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
}
if ($listcount % 1 == 0)
echo " .";
// echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
}
// Exit to call back to parent - Let know that child has completed
exit(0);
// Create callback function
function psUpdateComplete()
{
echo "Threaded update process complete\n";
}
?>