mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-28 17:01:16 +00:00
Revert "Merge remote-tracking branch 'origin/dev'"
This reverts commit0ef4543c44, reversing changes made to7b987a69f3. Changes to be committed: (use "git reset HEAD <file>..." to unstage) modified: conf/.gitignore deleted: conf/default_regexes.txt modified: conf/jonnyboys_my.cnf deleted: scripts/reactivate_regexes.sh deleted: ugo/automake.php deleted: ugo/automake_threaded.php deleted: ugo/finish_postproc.php deleted: ugo/makenzb.php deleted: ugo/makenzb_threaded.php deleted: ugo/misc_sorter3.php deleted: ugo/misc_sorter_cleanup.php deleted: ugo/misc_sorter_threaded.php deleted: ugo/postproc.php deleted: ugo/postproc_threaded.php# deleted: ugo/update_binaries_threaded.php
This commit is contained in:
Darko
@@ -1,3 +1,2 @@
|
||||
tmux_user.conf
|
||||
my.cnf
|
||||
active_regexes.txt
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+3
-37
@@ -88,8 +88,8 @@ log_error = /var/log/mysql/error.log
|
||||
# other settings you may need to change.
|
||||
#server-id = 1
|
||||
#log_bin = /var/log/mysql/mysql-bin.log
|
||||
#expire_logs_days = 10
|
||||
#max_binlog_size = 100M
|
||||
expire_logs_days = 10
|
||||
max_binlog_size = 100M
|
||||
#binlog_do_db = include_database_name
|
||||
#binlog_ignore_db = include_database_name
|
||||
#
|
||||
@@ -110,41 +110,7 @@ log_error = /var/log/mysql/error.log
|
||||
# ssl-key=/etc/mysql/server-key.pem
|
||||
|
||||
|
||||
max_user_connections = 200
|
||||
table_cache=2000
|
||||
|
||||
#InnoDB specifics
|
||||
|
||||
#Engine options
|
||||
innodb = ON
|
||||
innodb_file_format = Barracuda
|
||||
innodb_file_per_table = 1
|
||||
|
||||
#misc uncategorized
|
||||
innodb_table_locks = false
|
||||
innodb_use_sys_malloc = 1
|
||||
|
||||
#threading
|
||||
innodb_thread_concurrency = 8
|
||||
innodb_read_io_threads = 4
|
||||
innodb_write_io_threads = 4
|
||||
innodb_purge_threads = 1
|
||||
|
||||
#buffer pool related
|
||||
innodb_buffer_pool_size = 2G
|
||||
innodb_buffer_pool_instances = 8
|
||||
innodb_change_buffering = all
|
||||
innodb_max_dirty_pages_pct = 90
|
||||
|
||||
#flush related
|
||||
innodb_flush_method = O_DIRECT
|
||||
innodb_flush_log_at_trx_commit = 0
|
||||
|
||||
#transaction logs
|
||||
innodb_log_buffer_size = 2G
|
||||
innodb_log_files_in_group = 2
|
||||
innodb_log_file_size = 2G
|
||||
|
||||
|
||||
[mysqldump]
|
||||
quick
|
||||
quote-names
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
DIR="$( dirname "$SOURCE" )"
|
||||
while [ -h "$SOURCE" ]
|
||||
do
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
source $DIR"/../defaults.sh"
|
||||
|
||||
eval $( $SED -n "/^define/ { s/.*('\([^']*\)', '*\([^']*\)'*);/export \1=\"\2\"/; p }" "$NEWZPATH"/www/config.php )
|
||||
|
||||
cd $DIR"/conf"
|
||||
for fn in `cat active_regexes.txt`; do
|
||||
echo "Resetting regex $fn"
|
||||
mysql --defaults-file=my.cnf -u$DB_USER -h $DB_HOST $DB_NAME -e "update releaseregex set status=1 where ID=$fn"
|
||||
done
|
||||
|
||||
rm active_regexes.txt
|
||||
-1566
File diff suppressed because it is too large
Load Diff
@@ -1,156 +0,0 @@
|
||||
<?php
|
||||
//define('FS_ROOT', realpath(dirname(__FILE__)));
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
$time = microtime(true);
|
||||
$db = new DB();
|
||||
$cont = true;
|
||||
|
||||
if (isset($argv[1]))
|
||||
{
|
||||
//var_dump($argv);
|
||||
|
||||
if ($argv[1] == "reset")
|
||||
{
|
||||
echo "resetting binaries\n";
|
||||
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat not in (4, 6)");
|
||||
echo "binaries have been reset\n";
|
||||
} else if ($argv[1] == "cont") {
|
||||
$cont = false;
|
||||
}
|
||||
}
|
||||
|
||||
echo "cleaning spam..... can take a few moments\n";
|
||||
|
||||
//$db->disableAutoCommit();
|
||||
//$db->disableForeignKeyChecks();
|
||||
|
||||
$rel = $db->query("SELECT `ID` from `binaries` WHERE `xref` REGEXP '(.+:[0-9]+ ){5,}' AND `procstat` = 0");
|
||||
|
||||
foreach($rel as $r)
|
||||
{
|
||||
$db->query("UPDATE `binaries` set `procstat` = -2 WHERE `ID` = ".$r['ID']);
|
||||
if ($db->getLastError() != '')
|
||||
var_dump($db->getLastError());
|
||||
}
|
||||
|
||||
echo "spam count ".count($rel)."\n";
|
||||
|
||||
//$db->commit();
|
||||
|
||||
//$db->enableAutoCommit();
|
||||
//$db->enableForeignKeyChecks();
|
||||
|
||||
|
||||
if ($cont)
|
||||
$rel = $db->query("UPDATE `binaries` SET `procstat`=0,`procattempts`=0,`regexID`=NULL, `relpart`=0,`reltotalpart`=0,`relname`=NULL WHERE procstat = -6");
|
||||
|
||||
echo "Getting work to be done\n";
|
||||
//$query = "SELECT ID FROM binaries WHERE `procstat` = 0 GROUP BY binaryhash order by RAND()";
|
||||
$query = "SELECT ID FROM binaries WHERE `procstat` = 0 LIMIT 5000";
|
||||
|
||||
$theList = $db->query($query);
|
||||
unset($db);
|
||||
|
||||
$theCount = count($theList);
|
||||
|
||||
$ps = new PowerProcess();
|
||||
$ps->RegisterCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 8;
|
||||
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
|
||||
$ps->threadTimeLimit = 0; // Disable child timeout
|
||||
|
||||
$tim = microtime(true) - $time;
|
||||
echo "time = ".$tim."\n";
|
||||
|
||||
echo "Starting threaded assembly process\n";
|
||||
echo "\nToday will be doing $theCount binaries\n";
|
||||
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
while ($ps->RunControlCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
$listcount = count($theList);
|
||||
if ($listcount)
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->SpawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->threadData = array_pop($theList);
|
||||
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." post(s) to update after this\n";
|
||||
$ps->spawnThread();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "\nNo more posts to process - Initiating shutdown\n";
|
||||
$ps->threadTimeLimit = 200;
|
||||
do {
|
||||
$ps->tick();
|
||||
} while ($ps->myThreads);
|
||||
$ps->Shutdown();
|
||||
echo "\nShutdown complete\n";
|
||||
$tim = microtime(true) - $time;
|
||||
echo "final time = ".$tim."\n";
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
unset($theList);
|
||||
|
||||
if ($ps->RunThreadCode())
|
||||
{
|
||||
$post = $ps->threadData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->GetPID());
|
||||
|
||||
// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n";
|
||||
|
||||
$param = $post['ID'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'automake.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
$count = $theCount - $listcount;
|
||||
$countpct = $count / $theCount;
|
||||
$cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT);
|
||||
|
||||
if ($listcount % 250 == 0 && $count > 0)
|
||||
{
|
||||
$tim = microtime(true) - $time;
|
||||
echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
|
||||
$tim = ((1 + 0.2 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
|
||||
|
||||
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
|
||||
if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0))
|
||||
echo "\t".date('M jS Y H:i:s', $tim);
|
||||
// echo "\t".$tim."m to go" ;
|
||||
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
}
|
||||
if ($listcount % 25 == 0)
|
||||
echo " .";
|
||||
|
||||
|
||||
// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n";
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded assembly process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,39 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
|
||||
require_once(WWW_DIR."/lib/postprocess.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
$db = new DB;
|
||||
$postprocess = new PostProcess(true);
|
||||
|
||||
while (true) {
|
||||
$thequery = "SELECT count(r.ID) AS thecount FROM releases r LEFT OUTER JOIN category c ON c.ID = r.categoryID WHERE (r.passwordstatus between -10 and -1) or (r.haspreview = -1 and c.disablepreview = 0) order by rand()";
|
||||
$result = $db->query($thequery);
|
||||
if ($result[0]["thecount"] < 10 )
|
||||
{
|
||||
$postprocess->processTv();
|
||||
$postprocess->processAll();
|
||||
$postprocess->processSpotNab();
|
||||
echo "done...waiting";
|
||||
sleep(600);
|
||||
} else {
|
||||
for ($i = 0; $i < $result[0]["thecount"]; $i=$i+20)
|
||||
{
|
||||
echo "\n";
|
||||
echo $result[0]["thecount"] - $i. " more to go: \n";
|
||||
// $postprocess->processAdditional();
|
||||
$postprocess->processNfos();
|
||||
$postprocess->processUnwanted();
|
||||
$postprocess->processMovies();
|
||||
$postprocess->processMusic();
|
||||
$postprocess->processBooks();
|
||||
$postprocess->processGames();
|
||||
$postprocess->processTv();
|
||||
$postprocess->processMusicFromMediaInfo();
|
||||
$postprocess->processOtherMiscCategory();
|
||||
$postprocess->processUnknownCategory();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
-120
@@ -1,120 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/nntp.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
function makenzb($id, $name = '')
|
||||
{
|
||||
global $db;
|
||||
$ok = true;
|
||||
$path = FS_ROOT."/nzbs/";
|
||||
|
||||
// $query = "SELECT * FROM `binaries` WHERE `ID` = ".$id;
|
||||
$query = "SELECT *, sum(parts.size) AS size FROM binaries INNER JOIN parts ON binaries.ID = parts.binaryID WHERE binaries.ID = ".$id;
|
||||
$row = $db->queryOneRow($query);
|
||||
|
||||
echo $row['name']." is ".($row['size']/1024/1024)." MB\n";
|
||||
|
||||
if (!isset($name))
|
||||
{
|
||||
$name = $row['relname'];
|
||||
}
|
||||
|
||||
if (preg_match('/\.nzb\b/i', $row['name']) && $row['size'] < 10000000)
|
||||
{
|
||||
echo "doing nzb\n";
|
||||
$nntp = new Nntp;
|
||||
if ($nntp->doConnect())
|
||||
{
|
||||
$bin = $nntp->getBinary($id);
|
||||
|
||||
echo strlen($bin)."\n";
|
||||
|
||||
$name = preg_replace('/ /','_',$name);
|
||||
|
||||
if (file_exists($path.$name.".nzb"))
|
||||
{
|
||||
$name = $name." ".$id;
|
||||
}
|
||||
|
||||
echo $name." ".strlen($bin)."\n";
|
||||
|
||||
|
||||
if (strlen($bin) == 0)
|
||||
$ok = false;
|
||||
elseif (file_put_contents($path.$name.".nzb", $bin) == 0)
|
||||
$ok = false;
|
||||
|
||||
|
||||
echo $ok."\n";
|
||||
}
|
||||
$nntp->doQuit();
|
||||
|
||||
unset($nntp);
|
||||
unset($bin);
|
||||
}
|
||||
|
||||
if ($ok)
|
||||
{
|
||||
echo "clearing \n";
|
||||
$query = "UPDATE `binaries` SET `procstat` = 6 WHERE ID = ".$id;
|
||||
|
||||
echo $query."\n";
|
||||
$db->query($query);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
mb_internal_encoding("UTF-8");
|
||||
|
||||
$db = new Db;
|
||||
|
||||
if (isset($argv[1]))
|
||||
{
|
||||
if (isset($argv[2]))
|
||||
{
|
||||
makenzb($argv[1], $argv[2]);
|
||||
} else {
|
||||
makenzb($argv[1]);
|
||||
}
|
||||
} else {
|
||||
|
||||
$db->query("UPDATE `binaries` SET `binaries`.`procstat` = 6 WHERE `binaries`.`procstat` = - 3 AND binaries.ID NOT IN ( SELECT `parts`.`binaryID` FROM `parts` )");
|
||||
|
||||
$rel = $db->query("SELECT * FROM `binaries` WHERE `procstat` = -3");
|
||||
|
||||
foreach($rel as $bin)
|
||||
{
|
||||
|
||||
$name = $bin['relname'];
|
||||
|
||||
$name = preg_replace('/yenc/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/[\`\'\"\:\/\-\<\>]/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/ +?/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/^ | $/iU','',$name);
|
||||
|
||||
$name = preg_replace('/ /','_',$name);
|
||||
|
||||
$id = $bin['ID'];
|
||||
|
||||
echo "doing $name\n";
|
||||
|
||||
makenzb($id, $name);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
unset($db);
|
||||
?>
|
||||
@@ -1,210 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
$time = microtime(true);
|
||||
$db = new DB();
|
||||
|
||||
$query = "SELECT * FROM `binaries` WHERE `procstat` = -3";
|
||||
|
||||
$theList = $db->query($query);
|
||||
unset($db);
|
||||
|
||||
$theCount = count($theList);
|
||||
|
||||
$ps = new PowerProcess();
|
||||
$ps->RegisterCallback('psUpdateComplete');
|
||||
$ps->maxThreads = 20;
|
||||
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
|
||||
$ps->threadTimeLimit = 0; // Disable child timeout
|
||||
|
||||
$tim = microtime(true) - $time;
|
||||
echo "time = ".$tim."\n";
|
||||
|
||||
echo "Starting threaded nzb exporter process\n";
|
||||
echo "\nToday will be doing $theCount binaries\n";
|
||||
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
while ($ps->RunControlCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
$listcount = count($theList);
|
||||
if ($listcount)
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->SpawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->threadData = array_pop($theList);
|
||||
//echo "[Thread-MASTER] Spawning new thread. Still have " . count($rel) ." nzb(s) to update after this\n";
|
||||
$ps->spawnThread();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
//$ps->tick();
|
||||
//echo ".\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
// $query = "UPDATE `binaries` SET `procstat` = 6 WHERE `procstat` = -3 OR `procstat` = -2";
|
||||
// $db = new Db();
|
||||
// var_dump($db->query($query));
|
||||
echo "\nNo more nzbs to process - Initiating shutdown\n";
|
||||
$ps->threadTimeLimit = 200;
|
||||
do {
|
||||
$ps->tick();
|
||||
} while ($ps->myThreads);
|
||||
$ps->Shutdown();
|
||||
echo "\nShutdown complete\n";
|
||||
|
||||
|
||||
$path = FS_ROOT."/nzbs/";
|
||||
|
||||
$filestoprocess = glob($path."*.nzb");
|
||||
|
||||
foreach($filestoprocess as $nzbFile)
|
||||
{
|
||||
|
||||
$xml = file_get_contents( $nzbFile );
|
||||
|
||||
$matches = preg_split('/\A(\<\?xml[^\r\n]+?\?\>)\s*\R+/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
|
||||
|
||||
$name = preg_split('/\.nzb$/i', $nzbFile, 0, PREG_SPLIT_DELIM_CAPTURE);
|
||||
|
||||
$name = $name[0];
|
||||
|
||||
// $marker = $matches[0];
|
||||
//
|
||||
// if (strlen($marker) == 0 || strlen($marker) >100)
|
||||
// {
|
||||
// $marker = $matches[1];
|
||||
// }
|
||||
|
||||
$marker = '';
|
||||
|
||||
foreach ($matches as $m)
|
||||
{
|
||||
if (preg_match('/^(\<\?xml[^\r\n]+?\?\>)/iU', $m))
|
||||
{
|
||||
echo "$m\n";
|
||||
$marker = $m;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
echo "marker "."/(".preg_quote($marker)."\s*\R+)/iU\n";
|
||||
|
||||
if (!empty($marker))
|
||||
{
|
||||
$matches = preg_split('/('.preg_quote($marker).'\s*\R+)/iU', $xml, 0, PREG_SPLIT_DELIM_CAPTURE);
|
||||
|
||||
if (strlen($matches[0]) == 0)
|
||||
{
|
||||
unset($matches[0]);
|
||||
$matches = array_values($matches);
|
||||
}
|
||||
|
||||
echo count($matches)."\n";
|
||||
|
||||
if (count($matches) > 2)
|
||||
{
|
||||
for ($i = 0; $i < count($matches); $i = $i + 2)
|
||||
{
|
||||
echo $i."\n";
|
||||
$outxml = $matches[$i] . $matches[$i + 1];
|
||||
$fileout = $name . '.' . $i . '.nzb';
|
||||
file_put_contents($fileout, $outxml);
|
||||
echo $fileout."\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
echo "final time = ".$tim."\n";
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
unset($theList);
|
||||
unset($db);
|
||||
|
||||
if ($ps->RunThreadCode())
|
||||
{
|
||||
$bin = $ps->threadData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->GetPID());
|
||||
|
||||
// echo "[Thread-{$thread}] Begining processing for nzb {$bin['ID']}\n";
|
||||
|
||||
$name = $bin['relname'];
|
||||
|
||||
$name = preg_replace('/yenc/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/\d+?\/\d+?/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/[\(\)\{\}\[\]]/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/[\`\'\"\:\/\-\<\>\|]/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/\.(?!nzb)/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/ +?/iU',' ',$name);
|
||||
|
||||
$name = preg_replace('/^ | $/iU','',$name);
|
||||
|
||||
$name = preg_replace('/ /','_',$name);
|
||||
|
||||
$id = $bin['ID'];
|
||||
|
||||
if ($name == '')
|
||||
{
|
||||
$name = $id;
|
||||
}
|
||||
|
||||
// echo "\nname: ".$name." \n";
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'makenzb.php';
|
||||
|
||||
// echo "php {$dir}/{$file} {$id} {$name}\n";
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$id} {$name}");
|
||||
|
||||
$count = $theCount - $listcount;
|
||||
$countpct = $count / $theCount;
|
||||
|
||||
if ($listcount % 100 == 0 && $listcount > 0)
|
||||
{
|
||||
$tim = microtime(true) - $time;
|
||||
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
|
||||
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
|
||||
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
|
||||
if ($countpct > 0.1 || $count > 1500)
|
||||
echo "\t".date('H:i:s', $tim);
|
||||
// echo "\t".$tim."m to go" ;
|
||||
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
}
|
||||
if ($listcount % 10 == 0)
|
||||
echo " .";
|
||||
|
||||
// echo "[Thread-{$thread}] Completed update for {$bin['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "Threaded export process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,98 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
require_once(WWW_DIR."/lib/releases.php");
|
||||
require_once(WWW_DIR."/lib/movie.php");
|
||||
require_once(WWW_DIR."/lib/tvrage.php");
|
||||
require_once(WWW_DIR."/lib/amazon.php");
|
||||
require_once(WWW_DIR."/lib/anidb.php");
|
||||
require_once(WWW_DIR."/lib/nzbinfo.php");
|
||||
require_once(WWW_DIR."/lib/nzb.php");
|
||||
require_once(WWW_DIR."/lib/book.php");
|
||||
require_once(WWW_DIR."/lib/music.php");
|
||||
|
||||
$movie = new Movie();
|
||||
$movie->processMovieReleases();
|
||||
$movie->updateUpcoming();
|
||||
|
||||
$music = new Music(true);
|
||||
$music->processMusicReleases();
|
||||
|
||||
$book = new Book(true);
|
||||
$book->processBookReleases();
|
||||
|
||||
$anidb = new AniDB(true);
|
||||
$anidb->animetitlesUpdate();
|
||||
$anidb->processAnimeReleases();
|
||||
|
||||
$tvrage = new TVRage(true);
|
||||
$tvrage->processTvReleases(true);
|
||||
|
||||
$thetvdb = new TheTVDB(true);
|
||||
$thetvdb->processReleases();
|
||||
|
||||
echo "cleaning 5000\n";
|
||||
|
||||
$db = new DB();
|
||||
|
||||
$query = "update releases set releases.categoryID= 5060 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE movieinfo.genre REGEXP 'sport' AND NOT movieinfo.genre REGEXP 'comedy' AND NOT movieinfo.genre REGEXP 'romance' AND NOT movieinfo.genre REGEXP 'drama' )";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "update releases set releases.categoryID= 5000 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'tv|talk\-show|reality|episode' ) AND categoryID NOT IN ( SELECT ID FROM category WHERE parentID = 5000 )";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 10500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 2500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000) AND size > 500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2020 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `imdbID` > 0) OR `categoryID` = 2000)";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2040 WHERE `categoryID` = 2050 AND size <10000000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=2030 WHERE `categoryID` = 2040 AND size < 2500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=5040 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 1500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=5030 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000) AND size > 100000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=5050 WHERE (`categoryID` in (SELECT ID FROM category WHERE parentID = 8000 and `rageID` > 0) OR `categoryID` = 5000)";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=5030 WHERE `categoryID` = 5040 AND size < 1500000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE `releases` SET `categoryID`=5050 WHERE `categoryID` = 5030 AND size < 100000000";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "UPDATE releases SET categoryID = 5070 WHERE rageID IN ( SELECT tvrage.rageID FROM tvrage WHERE tvrage.genre REGEXP 'animat' )";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
$query = "update releases set releases.categoryID= 5080 WHERE imdbID IN ( SELECT movieinfo.imdbID FROM movieinfo WHERE genre REGEXP 'docum' )";
|
||||
|
||||
$db->query($query);
|
||||
|
||||
?>
|
||||
@@ -1,140 +0,0 @@
|
||||
<?php
|
||||
//define('FS_ROOT', realpath(dirname(__FILE__)));
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
$time = microtime(true);
|
||||
$db = new DB();
|
||||
|
||||
$thecategory = 8000;
|
||||
|
||||
if (isset($argv[1]))
|
||||
{
|
||||
$thecategory = substr($argv[1], 0, 1)."000";
|
||||
}
|
||||
|
||||
echo "Starting sorting process for category $thecategory\n";
|
||||
|
||||
$query = "SELECT ID FROM category WHERE parentID = $thecategory"; // OR parentID = 2000 OR parentID = 5000
|
||||
|
||||
$res = $db->query($query);
|
||||
|
||||
$thecategory = $res[0]['ID'];
|
||||
|
||||
unset($res[0]);
|
||||
|
||||
foreach($res as $r)
|
||||
$thecategory = $thecategory.", ".$r['ID'];
|
||||
|
||||
$query = "SELECT releases.ID, releases.`name` FROM releases WHERE releases.categoryID IN ( $thecategory )";
|
||||
|
||||
$theList = $db->query($query);
|
||||
unset($db);
|
||||
|
||||
$theCount = count($theList);
|
||||
|
||||
$ps = new PowerProcess();
|
||||
$ps->RegisterCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 24;
|
||||
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
|
||||
$ps->threadTimeLimit = 0; // Disable child timeout
|
||||
|
||||
$tim = microtime(true) - $time;
|
||||
echo "time = ".$tim."\n";
|
||||
|
||||
|
||||
echo "\nToday will be going over $theCount releases\n";
|
||||
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
while ($ps->RunControlCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
$listcount = count($theList);
|
||||
if ($listcount)
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->SpawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->threadData = array_pop($theList);
|
||||
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." posts to sort after this\n";
|
||||
$ps->spawnThread();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
//$ps->tick();
|
||||
//echo ".\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "\nNo more to process - Initiating shutdown\n";
|
||||
$ps->threadTimeLimit = 200;
|
||||
do {
|
||||
$ps->tick();
|
||||
} while ($ps->myThreads);
|
||||
$ps->Shutdown();
|
||||
echo "\nShutdown complete\n";
|
||||
$tim = microtime(true) - $time;
|
||||
echo "final processing time = ".$tim."\n";
|
||||
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
//echo "time = ".$tim."\n";
|
||||
unset($theList);
|
||||
|
||||
if ($ps->RunThreadCode())
|
||||
{
|
||||
$post = $ps->threadData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->GetPID());
|
||||
|
||||
// echo "[Thread-{$thread}] Begining assembly processing for post {$post['ID']}\n";
|
||||
|
||||
$param = $post['ID'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'misc_sorter3.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
$count = $theCount - $listcount;
|
||||
$countpct = $count / $theCount;
|
||||
$cleanpct = str_pad(number_format(($countpct * 100), 1), 7, " ", STR_PAD_LEFT);
|
||||
|
||||
if ($listcount % 250 == 0 && $listcount > 0)
|
||||
{
|
||||
$tim = microtime(true) - $time;
|
||||
echo "\t\t".$cleanpct."% = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
|
||||
$tim = ($listcount * $tim) / $count + time();
|
||||
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
|
||||
if ($countpct > .95 || ($count > 1000 && ($listcount+250) % 1000 == 0))
|
||||
echo "\t".date('M jS Y H:i:s', $tim);
|
||||
// echo "\t".$tim."m to go" ;
|
||||
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
} elseif ($listcount % 25 == 0)
|
||||
echo " .";
|
||||
|
||||
|
||||
// echo "[Thread-{$thread}] Completed update for post {$post['ID']}\n";
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded sorting process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/postprocess.php");
|
||||
require_once(WWW_DIR."/lib/site.php");
|
||||
require_once(WWW_DIR."/lib/nntp.php");
|
||||
require_once(WWW_DIR."/lib/nzbinfo.php");
|
||||
require_once(WWW_DIR."/lib/releases.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
|
||||
function watchdog ()
|
||||
{
|
||||
global $buffer, $pid;
|
||||
|
||||
if (isset($buffer))
|
||||
{
|
||||
$ps = shm_get_var($buffer, 1);
|
||||
if (isset($ps->myThreads[$pid]))
|
||||
if ($ps->myThreads[$pid]['time'] != time())
|
||||
{
|
||||
$ps->myThreads[$pid]['time'] = time();
|
||||
$lastto = time();
|
||||
shm_put_var($buffer, 1, $ps);
|
||||
// trigger_error($pid." ".$ps->myThreads[$pid]['time']);
|
||||
}
|
||||
} else {
|
||||
trigger_error("$pid is orphaned");
|
||||
}
|
||||
}
|
||||
|
||||
$postprocess = new PostProcess(true);
|
||||
|
||||
$s = new Sites();
|
||||
$site = $s->get();
|
||||
$nntp = new Nntp;
|
||||
|
||||
$db = new DB;
|
||||
$nzb = new Nzb;
|
||||
|
||||
$maxattemptstocheckpassworded = 5;
|
||||
$numtoProcess = 5;
|
||||
$processVideoSample = ($site->ffmpegpath != '') ? true : false;
|
||||
$processMediainfo = ($site->mediainfopath != '') ? true : false;
|
||||
$processPasswords = ($site->unrarpath != '') ? true : false;
|
||||
$processAudioSample = ($site->saveaudiopreview == 1) ? true : false;
|
||||
|
||||
$nntpconnected = false;
|
||||
$lastto = time()+1000;
|
||||
|
||||
var_dump($argv);
|
||||
|
||||
if (is_numeric($argv[1]))
|
||||
{
|
||||
if (isset($argv[3]))
|
||||
{
|
||||
$buffer = shm_attach($argv[2]);
|
||||
$pid = $argv[3];
|
||||
}
|
||||
|
||||
$rel = $db->query(sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r
|
||||
left join category c on c.ID = r.categoryID
|
||||
where r.ID = %d ", $argv[1]));
|
||||
|
||||
$rel = $rel[0];
|
||||
|
||||
if ($rel !== false)
|
||||
{
|
||||
$tmpPath = $site->tmpunrarpath;
|
||||
|
||||
if (substr($tmpPath, -strlen( '/' ) ) != '/')
|
||||
{
|
||||
$tmpPath = $tmpPath.'/';
|
||||
}
|
||||
|
||||
$tmpPath = $tmpPath.$rel['guid'].'/';
|
||||
|
||||
if (!file_exists($tmpPath))
|
||||
mkdir($tmpPath, 0766, true);
|
||||
|
||||
|
||||
// Per release defaults
|
||||
$passStatus = array(Releases::PASSWD_NONE);
|
||||
$blnTookMediainfo = false;
|
||||
$blnTookSample = ($rel['disablepreview'] == 1) ? true : false; //only attempt sample if not disabled
|
||||
|
||||
if ($blnTookSample)
|
||||
$db->query(sprintf("update releases set haspreview = 0 where id = %d", $rel['ID']));
|
||||
|
||||
//
|
||||
// Go through the binaries for this release looking for a rar, a sample, and a mediafile
|
||||
//
|
||||
$nzbInfo = new nzbInfo;
|
||||
$norar = 0;
|
||||
|
||||
// only load nzbs and check for rar files if we are doing something with them.
|
||||
if ($processVideoSample || $processMediainfo || $processPasswords || $processAudioSample)
|
||||
{
|
||||
$nzbfile = $nzb->getNZBPath($rel['guid'], $site ->nzbpath);
|
||||
if (!$nzbInfo->loadFromFile($nzbfile))
|
||||
{
|
||||
goto end;
|
||||
}
|
||||
|
||||
foreach($nzbInfo->nzb as $nzbsubject)
|
||||
{
|
||||
if (preg_match("/\w\.r00/i", $nzbsubject['subject']))
|
||||
$norar= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// attempt to process video sample file
|
||||
if(!empty($nzbInfo->samplefiles) && $processVideoSample && $blnTookSample === false)
|
||||
{
|
||||
$sampleFile = $nzbInfo->samplefiles[0]; //first detected sample
|
||||
$sampleMsgids = array_slice($sampleFile['segments'], 0, 1); //get first segment, increase to get more of the sample
|
||||
$sampleGroup = $sampleFile['groups'][0];
|
||||
|
||||
echo "PostPrc : Fetching ".implode($sampleMsgids, ', ')." from {$sampleGroup}\n";
|
||||
if (!$nntpconnected)
|
||||
$nntpconnected = $nntp->doConnect();
|
||||
|
||||
if ($nntpconnected) {
|
||||
$sampleBinary = $nntp->getMessages($sampleGroup, $sampleMsgids);
|
||||
watchdog ();
|
||||
if ($sampleBinary === false)
|
||||
echo "\nPostPrc : Couldnt fetch sample\n";
|
||||
else
|
||||
{
|
||||
$samplefile = $tmpPath.'sample.avi';
|
||||
|
||||
file_put_contents($samplefile, $sampleBinary);
|
||||
|
||||
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
|
||||
if ($blnTookSample)
|
||||
$postprocess->updateReleaseHasPreview($rel['guid']);
|
||||
|
||||
unlink($samplefile);
|
||||
}
|
||||
unset($sampleBinary);
|
||||
} else {
|
||||
echo "\nFailed to get NNTP connection\n";
|
||||
}
|
||||
}
|
||||
|
||||
// attempt to process loose media file
|
||||
if(!empty($nzbInfo->mediafiles) && (($processVideoSample && $blnTookSample === false) || $processMediainfo))
|
||||
{
|
||||
$mediaFile = $nzbInfo->mediafiles[0]; //first detected media file
|
||||
$mediaMsgids = array_slice($mediaFile['segments'], 0, 2); //get first two segments
|
||||
$mediaGroup = $mediaFile['groups'][0];
|
||||
|
||||
echo "PostPrc : Fetching ".implode($mediaMsgids, ', ')." from {$mediaGroup}\n";
|
||||
if (!$nntpconnected)
|
||||
$nntpconnected = $nntp->doConnect();
|
||||
|
||||
if ($nntpconnected) {
|
||||
$mediaBinary = $nntp->getMessages($mediaGroup, $mediaMsgids);
|
||||
watchdog ();
|
||||
if ($mediaBinary === false)
|
||||
echo "\nPostPrc : Couldnt fetch media file\n";
|
||||
else
|
||||
{
|
||||
$mediafile = $tmpPath.'sample.avi';
|
||||
|
||||
file_put_contents($mediafile, $mediaBinary);
|
||||
|
||||
if ($processVideoSample && $blnTookSample === false)
|
||||
{
|
||||
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
|
||||
if ($blnTookSample)
|
||||
$postprocess->updateReleaseHasPreview($rel['guid']);
|
||||
}
|
||||
|
||||
if ($processMediainfo)
|
||||
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
|
||||
|
||||
unlink($mediafile);
|
||||
}
|
||||
unset($mediaBinary);
|
||||
} else {
|
||||
echo "\nFailed to get NNTP connection\n";
|
||||
}
|
||||
}
|
||||
|
||||
// attempt to process audio sample file
|
||||
if(!empty($nzbInfo->audiofiles) && $processAudioSample && $blnTookSample === false)
|
||||
{
|
||||
$audioFile = $nzbInfo->audiofiles[0]; //first detected audio file
|
||||
$audioMsgids = array_slice($audioFile['segments'], 0, 1); //get first segment
|
||||
$audioGroup = $audioFile['groups'][0];
|
||||
|
||||
echo "PostPrc : Fetching ".implode($audioMsgids, ', ')." from {$audioGroup}\n";
|
||||
if (!$nntpconnected)
|
||||
$nntpconnected = $nntp->doConnect();
|
||||
|
||||
if ($nntpconnected) {
|
||||
$audioBinary = $nntp->getMessages($audioGroup, $audioMsgids);
|
||||
watchdog ();
|
||||
if ($audioBinary === false)
|
||||
echo "\nPostPrc : Couldnt fetch audio sample\n";
|
||||
else
|
||||
{
|
||||
$audiofile = $tmpPath.'sample.mp3';
|
||||
|
||||
file_put_contents($audiofile, $audioBinary);
|
||||
|
||||
$blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']);
|
||||
if ($blnTookSample !== false)
|
||||
$postprocess->updateReleaseHasPreview($rel['guid'], 2);
|
||||
|
||||
if ($processMediainfo)
|
||||
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
|
||||
|
||||
if ($site ->lamepath != "")
|
||||
$postprocess->lameAudioSample($site ->lamepath, $rel['guid']);
|
||||
|
||||
unlink($audiofile);
|
||||
}
|
||||
unset($audioBinary);
|
||||
} else {
|
||||
echo "\nFailed to get NNTP connection\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($nzbInfo->rarfiles) && ($site ->checkpasswordedrar > 0 || (($processVideoSample || $processAudioSample) && $blnTookSample === false) || $processMediainfo))
|
||||
{
|
||||
$mysqlkeepalive = 0;
|
||||
foreach($nzbInfo->rarfiles as $rarFile)
|
||||
{
|
||||
//dont process any more rars if a passworded rar has been detected and the site is set to automatically delete them
|
||||
if ($site ->deletepasswordedrelease == 1 && max($passStatus) == Releases::PASSWD_RAR)
|
||||
{
|
||||
echo "-Skipping processing of rar {$rarFile['subject']} as this release has already been marked as passworded.\n";
|
||||
goto end;
|
||||
}
|
||||
|
||||
$rarMsgids = array_slice($rarFile['segments'], 0, 1); //get first segment
|
||||
$rarGroup = $rarFile['groups'][0];
|
||||
|
||||
echo "PostPrc : Fetching ".implode($rarMsgids, ', ')." from {$rarGroup} (".++$mysqlkeepalive.")\n";
|
||||
if (!$nntpconnected)
|
||||
$nntpconnected = $nntp->doConnect();
|
||||
|
||||
if ($nntpconnected) {
|
||||
$fetchedBinary = $nntp->getMessages($rarGroup, $rarMsgids);
|
||||
watchdog ();
|
||||
if ($fetchedBinary === false)
|
||||
{
|
||||
echo "\nPostPrc : Failed fetching rar file\n";
|
||||
$db->query(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $rel['ID']));
|
||||
goto end;
|
||||
}
|
||||
else
|
||||
{
|
||||
$relFiles = $postprocess->processReleaseFiles($fetchedBinary, $rel['ID']);
|
||||
|
||||
if ($site ->checkpasswordedrar > 0 && $processPasswords)
|
||||
{
|
||||
$passStatus[] = $postprocess->processReleasePasswords($fetchedBinary, $tmpPath, $site ->unrarpath, $site ->checkpasswordedrar);
|
||||
}
|
||||
|
||||
// we need to unrar the fetched binary if checkpasswordedrar wasnt 2
|
||||
if ($site ->checkpasswordedrar < 2 && $processPasswords)
|
||||
{
|
||||
$rarfile = $tmpPath.'rarfile.rar';
|
||||
file_put_contents($rarfile, $fetchedBinary);
|
||||
$execstring = '"'.$site ->unrarpath.'" e -ai -ep -c- -id -r -kb -p- -y -inul "'.$rarfile.'" "'.$tmpPath.'"';
|
||||
$output = runCmd($execstring, false, true);
|
||||
unlink($rarfile);
|
||||
}
|
||||
|
||||
if ($processVideoSample && $blnTookSample === false)
|
||||
{
|
||||
$blnTookSample = $postprocess->getSample($tmpPath, $site ->ffmpegpath, $rel['guid']);
|
||||
if ($blnTookSample)
|
||||
$postprocess->updateReleaseHasPreview($rel['guid']);
|
||||
}
|
||||
|
||||
$blnTookAudioSample = false;
|
||||
if ($processAudioSample && $blnTookSample === false)
|
||||
{
|
||||
$blnTookSample = $postprocess->getAudioSample($tmpPath, $rel['guid']);
|
||||
if ($blnTookSample)
|
||||
{
|
||||
$blnTookAudioSample = true;
|
||||
$postprocess->updateReleaseHasPreview($rel['guid'], 2);
|
||||
}
|
||||
}
|
||||
|
||||
if ($processMediainfo && $blnTookMediainfo === false)
|
||||
{
|
||||
$blnTookMediainfo = $postprocess->getMediainfo($tmpPath, $site ->mediainfopath, $rel['ID']);
|
||||
}
|
||||
|
||||
//
|
||||
// Has to be done after mediainfo
|
||||
//
|
||||
if ($blnTookAudioSample && $site ->lamepath != "")
|
||||
$postprocess->lameAudioSample($site ->lamepath, $rel['guid']);
|
||||
|
||||
if ($mysqlkeepalive % 25 == 0)
|
||||
{
|
||||
$db->query("select 1");
|
||||
watchdog ();
|
||||
}
|
||||
}
|
||||
|
||||
//clean up all files
|
||||
foreach(glob($tmpPath.'*') as $v)
|
||||
{
|
||||
unlink($v);
|
||||
}
|
||||
} else {
|
||||
echo "\nFailed to get NNTP connection\n";
|
||||
}
|
||||
|
||||
} //end foreach msgid
|
||||
}
|
||||
elseif(empty($nzbInfo->rarfiles) && $norar == 1)
|
||||
{
|
||||
$passStatus[] = Releases::PASSWD_POTENTIAL;
|
||||
}
|
||||
|
||||
$hpsql = '';
|
||||
if (!$blnTookSample)
|
||||
$hpsql = ', haspreview = 0';
|
||||
|
||||
$sql = sprintf("update releases set passwordstatus = %d %s where ID = %d", max($passStatus), $hpsql, $rel["ID"]);
|
||||
$db->query($sql);
|
||||
|
||||
}
|
||||
end:
|
||||
foreach(glob($tmpPath.'*') as $v)
|
||||
{
|
||||
unlink($v);
|
||||
}
|
||||
foreach(glob($tmpPath.'.*') as $v)
|
||||
{
|
||||
@unlink($v);
|
||||
}
|
||||
rmdir($tmpPath);
|
||||
|
||||
if ($nntpconnected)
|
||||
{
|
||||
$nntp->doQuit();
|
||||
}
|
||||
unset($postprocess);
|
||||
unset($s);
|
||||
unset($site);
|
||||
unset($nntp);
|
||||
unset($db);
|
||||
unset($nzb);
|
||||
unset($nzbInfo);
|
||||
|
||||
}
|
||||
?>
|
||||
@@ -1,171 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
|
||||
function processchanges ()
|
||||
{
|
||||
global $ps, $buffer;
|
||||
|
||||
if (count($ps->myThreads) < 1 || !isset($buffer))
|
||||
return false;
|
||||
|
||||
$ps1 = shm_get_var($buffer, 1);
|
||||
$change = false;
|
||||
|
||||
$ps1c = count($ps1->myThreads);
|
||||
$psc = count($ps->myThreads);
|
||||
|
||||
foreach($ps->myThreads as $k => $v)
|
||||
{
|
||||
if (isset($ps1->myThreads[$k]))
|
||||
{
|
||||
if ($v['time'] != $ps1->myThreads[$k]['time'])
|
||||
{
|
||||
$q = (strval($ps1->myThreads[$k]['time']) - strval($ps->myThreads[$k]['time']));
|
||||
// echo "time changed for $q\n";
|
||||
$ps->myThreads[$k]['time'] = $ps1->myThreads[$k]['time'];
|
||||
$change = true;
|
||||
}
|
||||
} else {
|
||||
// echo "ps not current $psc $ps1c\n";
|
||||
$change = true;
|
||||
}
|
||||
}
|
||||
|
||||
if ($change)
|
||||
shm_put_var($buffer, 1, $ps);
|
||||
|
||||
// echo "pause ";
|
||||
// var_dump($ps->myThreads);
|
||||
}
|
||||
|
||||
$time = microtime(true);
|
||||
$db = new DB();
|
||||
|
||||
$maxattemptstocheckpassworded = 10;
|
||||
$numtoProcess = 1000;
|
||||
|
||||
$query = sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r
|
||||
left join category c on c.ID = r.categoryID
|
||||
where (r.passwordstatus between %d and -1)
|
||||
or (r.haspreview = -1 and c.disablepreview = 0) order by RAND() limit %d ", ($maxattemptstocheckpassworded + 1) * -1, $numtoProcess);
|
||||
|
||||
//$query = "SELECT releases.ID, releases.rarinnerfilecount, releases.haspreview, releases.passwordstatus, releases.releasenfoID, releases.guid FROM releases WHERE releases.ID NOT IN ( SELECT releasefiles.releaseID FROM releasefiles ) AND releases.rageID < 1 AND releases.imdbID < 1 OR releases.rarinnerfilecount < 0";
|
||||
|
||||
$theList = $db->query($query);
|
||||
unset($db);
|
||||
|
||||
$theCount = count($theList);
|
||||
|
||||
$ps = new PowerProcess();
|
||||
$ps->RegisterCallback('psUpdateComplete');
|
||||
$ps->maxThreads = 16;
|
||||
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
|
||||
$ps->threadTimeLimit = 350; // Disable child timeout
|
||||
|
||||
|
||||
$handle = ftok(__FILE__, chr(1));
|
||||
$buffer = shm_attach($handle, 10240);
|
||||
|
||||
$tim = microtime(true) - $time;
|
||||
echo "time = ".$tim."\n";
|
||||
|
||||
echo "Starting threaded rar inspection process\n";
|
||||
echo "\nToday will be be inspecting $theCount binaries\n";
|
||||
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
while ($ps->RunControlCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
$listcount = count($theList);
|
||||
if ($listcount)
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->SpawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->threadData = array_pop($theList);
|
||||
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n";
|
||||
$ps->spawnThread();
|
||||
processchanges();
|
||||
}
|
||||
else
|
||||
{
|
||||
//There are no more slots available to run
|
||||
$ps->tick();
|
||||
//echo ".\n";
|
||||
processchanges();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "\nNo more release to inspect - Initiating shutdown\n";
|
||||
do {
|
||||
$ps->tick();
|
||||
processchanges();
|
||||
} while ($ps->myThreads);
|
||||
$ps->Shutdown();
|
||||
echo "\nShutdown complete\n";
|
||||
// var_dump($ps->myThreads);
|
||||
$tim = microtime(true) - $time;
|
||||
echo "final time = ".$tim."\n";
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
unset($theList);
|
||||
|
||||
if ($ps->RunThreadCode())
|
||||
{
|
||||
$release = $ps->threadData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->GetPID());
|
||||
|
||||
// echo "[Thread-{$thread}] Begining rar inspection for {$release['name']} {$handle} id= {$param}\n";
|
||||
|
||||
$param = $release['ID'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'postproc.php';
|
||||
|
||||
//echo "start ";
|
||||
//var_dump($ps->myThreads);
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param} {$handle} {$thread}");
|
||||
|
||||
$count = $theCount - $listcount;
|
||||
$countpct = $count / $theCount;
|
||||
|
||||
if ($listcount % 50 == 0 && $count > 0)
|
||||
{
|
||||
$tim = microtime(true) - $time;
|
||||
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
|
||||
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
|
||||
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
|
||||
if ($countpct > 0.1 || $count > 1500)
|
||||
echo "\t".date('H:i:s', $tim);
|
||||
// echo "\t".$tim."m to go" ;
|
||||
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
}
|
||||
if ($listcount % 2 == 0)
|
||||
echo " .";
|
||||
|
||||
// echo "[Thread-{$thread}] Completed update for binary {$release['name']}\n";
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded inspection process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../bin/config.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once(WWW_DIR."/lib/powerprocess.php");
|
||||
|
||||
//$groups = new Groups;
|
||||
//$groupList = $groups->getActive();
|
||||
|
||||
$query = "SELECT * FROM groups WHERE groups.active = 1 ORDER BY ( groups.last_record - groups.first_record ) ASC";
|
||||
|
||||
$time = microtime(true);
|
||||
$db = new DB();
|
||||
$theList = $db->query($query);
|
||||
|
||||
//unset($groups);
|
||||
unset($db);
|
||||
|
||||
$theCount = count($theList);
|
||||
|
||||
$ps = new PowerProcess();
|
||||
$ps->RegisterCallback('psUpdateComplete');
|
||||
$ps->maxThreads = 15;
|
||||
$ps->tickCount = 10000; // value in usecs. change this to 1000000 (one second) to reduce cpu use
|
||||
$ps->threadTimeLimit = 1500; // Disable child timeout
|
||||
|
||||
$tim = microtime(true) - $time;
|
||||
echo "time = ".$tim."\n";
|
||||
|
||||
echo "Starting threaded binary update process\n";
|
||||
echo "\nToday will be be inspecting $theCount binaries\n";
|
||||
echo str_pad($theCount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
while ($ps->RunControlCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
$listcount = count($theList);
|
||||
if ($listcount)
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->SpawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->threadData = array_pop($theList);
|
||||
// echo "[Thread-MASTER] Spawning new thread. Still have " . count($theList) ." releases to rummage after this\n";
|
||||
$ps->spawnThread();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
$ps->tick();
|
||||
//echo "- \n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "\nNo more groups to process - Initiating shutdown\n";
|
||||
do {
|
||||
$ps->tick();
|
||||
} while ($ps->myThreads);
|
||||
$ps->Shutdown();
|
||||
echo "\nShutdown complete\n";
|
||||
$tim = microtime(true) - $time;
|
||||
echo "final time = ".$tim."\n";
|
||||
}
|
||||
}
|
||||
$tim = microtime(true) - $time;
|
||||
// echo "time = ".$tim."\n";
|
||||
unset($theList);
|
||||
|
||||
if ($ps->RunThreadCode())
|
||||
{
|
||||
$group = $ps->threadData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->GetPID());
|
||||
|
||||
// echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n";
|
||||
|
||||
$param = $group['name'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'update_binaries.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
//$output = shell_exec("/usr/bin/php -c /etc/php5/cli/php.ini {$dir}/{$file} {$param}");
|
||||
|
||||
$count = $theCount - $listcount;
|
||||
$countpct = $count / $theCount;
|
||||
|
||||
if ($listcount % 12 == 0 && $count > 0)
|
||||
{
|
||||
$tim = microtime(true) - $time;
|
||||
echo "\t\tt = ".str_pad(number_format($tim, 1), 8, " ", STR_PAD_LEFT);
|
||||
$tim = ((1 + 0.3 * (1 - $countpct)) * ($listcount * $tim) / $count) + time();
|
||||
// $tim = number_format(($listcount * $tim) / 60 / $count, 1);
|
||||
if ($countpct > 0.1 || $count > 1500)
|
||||
echo "\t".date('H:i:s', $tim);
|
||||
// echo "\t".$tim."m to go" ;
|
||||
echo "\n".str_pad($listcount, 7, " ", STR_PAD_LEFT)."\t";
|
||||
|
||||
}
|
||||
if ($listcount % 1 == 0)
|
||||
echo " .";
|
||||
|
||||
|
||||
// echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "Threaded update process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
|
||||
Reference in New Issue
Block a user