mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
split development to it's own branch
This commit is contained in:
jonnyboy
@@ -1,6 +1,6 @@
|
||||
# SETUP
|
||||
|
||||
* These scripts were written and tested on Ubuntu 12.10 where bash is located at /bin/bash. You may need to create a symlink or edit these script accordingly.
|
||||
* These scripts were written and tested on Ubuntu 12.10 where bash is located at /bin/bash. You may need to create a symlink or edit these scripts accordingly.
|
||||
|
||||
* Please backup your database first. Something like this should do it.
|
||||
|
||||
@@ -11,16 +11,16 @@
|
||||
|
||||
* If you decide to convert your database, I recommend using [kevin123's github](https://github.com/kevinlekiller/Newznab-Barracuda.git). I recommend only converting the binaries a parts table, using compressed tables. But, there are many choices. I suggest you read his README and follow his recommendations. Or, simply:
|
||||
|
||||
`cd /var/www/newznab/misc/testing/`
|
||||
`cd /var/www/newznab/misc/testing`
|
||||
|
||||
`git clone https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git innodb`
|
||||
`git clone https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git kev-innodb`
|
||||
|
||||
`cd innodb/lib/innodb`
|
||||
`cd innodb/lib/kev-innodb`
|
||||
|
||||
`php convertToInnoDB.php`
|
||||
|
||||
|
||||
* If, you have already converted your database, and didn't to the steps above. You will need to clone [kevin123's github](https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git) and get the scripts.
|
||||
* Now, you will need to clone [andrewit's github](https://github.com/itandrew/Newznab-InnoDB-Dropin.git) and get his scripts.
|
||||
|
||||
`cd /var/www/newznab/misc/testing/`
|
||||
|
||||
@@ -33,10 +33,6 @@
|
||||
|
||||
`git clone https://github.com/jonnyboy/newznab-tmux.git tmux`
|
||||
|
||||
-or-
|
||||
|
||||
`git clone https://github.com/kevinlekiller/newznab-tmux tmux`
|
||||
|
||||
`cd tmux`
|
||||
|
||||
`nano edit_these.sh`
|
||||
@@ -56,7 +52,7 @@
|
||||
|
||||
* If something looks stalled, it probably isn't. If all 13 panes are still there, it is most likely, as it should be.
|
||||
|
||||
* update_cleanup needs to be uncommented to actually do something, and update_parsing is good for fixing a few releases everytime it runs, not a silver bullet though
|
||||
* **misc/testing/update_cleanup.php** needs to be edited to actually do something, and update_parsing is good for fixing a few releases everytime it runs, not a silver bullet though
|
||||
|
||||
* If you are running this on an OVH or kimsufi server, you may need to run sudo ./start.sh because they built grsecurity into the kernel.
|
||||
|
||||
@@ -64,6 +60,8 @@
|
||||
|
||||
* Join in the converstion at irc://moonlight.se.eu.synirc.net/newznab-tmux.
|
||||
|
||||
* The development branch of this git is still under heavy development.
|
||||
|
||||
* Thanks go to all who offered their assistance and improvement to these scripts.
|
||||
|
||||
<hr>
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
<?php
|
||||
|
||||
require_once("config.php");
|
||||
require_once("lib/backfill.php");
|
||||
|
||||
if (isset($argv[1]))
|
||||
$groupName = $argv[1];
|
||||
else
|
||||
$groupName = '';
|
||||
|
||||
$backfill = new Backfill();
|
||||
$backfill->backfillAllGroups($groupName);
|
||||
|
||||
?>
|
||||
@@ -1,76 +0,0 @@
|
||||
<?php
|
||||
require_once("config.php");
|
||||
require_once(dirname(__FILE__)."/lib/groups.php");
|
||||
require_once(dirname(__FILE__)."/lib/innodb/binaries.php");
|
||||
require_once(dirname(__FILE__)."/lib/powerspawn.php");
|
||||
|
||||
$groups = new Groups;
|
||||
$groupList = $groups->getActive();
|
||||
unset($groups);
|
||||
|
||||
$ps = new PowerSpawn;
|
||||
$ps->setCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 10;
|
||||
$ps->timeLimit = 0; // Disable child timeout
|
||||
|
||||
echo "Starting threaded backfill process\n";
|
||||
|
||||
while ($ps->runParentCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
if (count($groupList))
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->spawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->childData = array_pop($groupList);
|
||||
echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n";
|
||||
$ps->spawnChild();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
$ps->tick();
|
||||
#echo ". \n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "No more groups to process - Initiating shutdown\n";
|
||||
$ps->shutdown();
|
||||
echo "Shutdown complete\n";
|
||||
}
|
||||
}
|
||||
|
||||
unset($groupList);
|
||||
|
||||
if ($ps->runChildCode())
|
||||
{
|
||||
$group = $ps->childData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->myPID());
|
||||
|
||||
echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n";
|
||||
|
||||
$param = $group['name'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'backfill.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded backfill process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,3 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../config.php");
|
||||
?>
|
||||
@@ -1,300 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/innodb/config.php");
|
||||
require_once(dirname(__FILE__)."/framework/db.php");
|
||||
require_once(dirname(__FILE__)."/groups.php");
|
||||
require_once(dirname(__FILE__)."/site.php");
|
||||
require_once(dirname(__FILE__)."/nntp.php");
|
||||
require_once(dirname(__FILE__)."/innodb/binaries.php");
|
||||
|
||||
/**
|
||||
* Retrieves messages from usenet based on provided backfill-to date.
|
||||
*/
|
||||
class Backfill
|
||||
{
|
||||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
function Backfill($db = null)
|
||||
{
|
||||
if($db == null)
|
||||
{
|
||||
$this->db = new DB();
|
||||
}
|
||||
else
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
$this->n = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all active groups categories and descriptions.
|
||||
*/
|
||||
function backfillAllGroups($groupName='', $backfillDate=null)
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = false;
|
||||
if ($groupName != '')
|
||||
{
|
||||
$grp = $groups->getByName($groupName);
|
||||
if ($grp)
|
||||
$res = array($grp);
|
||||
}
|
||||
else
|
||||
{
|
||||
$res = $groups->getActive();
|
||||
}
|
||||
|
||||
if ($res)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$this->backfillGroup($nntp, $groupArr, $backfillDate);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database for updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a group back to a specified date.
|
||||
*/
|
||||
function backfillGroup($nntp, $groupArr, $backfillDate=null)
|
||||
{
|
||||
$this->db->disableAutoCommit(); //Turn off auto committing
|
||||
|
||||
$binaries = new Binaries($this->db);
|
||||
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
if ($backfillDate)
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date
|
||||
else
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target
|
||||
if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0)
|
||||
{
|
||||
echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
|
||||
echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~";
|
||||
echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400));
|
||||
echo " days.".$n."Local first = ".$groupArr['first_record']." (";
|
||||
echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400));
|
||||
echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n";
|
||||
|
||||
if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish
|
||||
{
|
||||
echo "Nothing to do, we already have the target post.$n $n";
|
||||
|
||||
$this->db->commit(); //Not an error so commit and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
//get first and last part numbers from newsgroup
|
||||
if($targetpost < $data['first'])
|
||||
{
|
||||
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n";
|
||||
echo "Skipping Group $n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
//calculate total number of parts
|
||||
$total = $groupArr['first_record'] - $targetpost;
|
||||
$done = false;
|
||||
//set first and last, moving the window by maxxMssgs
|
||||
$last = $groupArr['first_record'] - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1; //set initial "chunk"
|
||||
if($targetpost > $first) //just in case this is the last chunk we needed
|
||||
$first = $targetpost;
|
||||
while($done === false)
|
||||
{
|
||||
$binaries->startLoop = microtime(true);
|
||||
echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n;
|
||||
flush();
|
||||
$success = $binaries->scan($nntp, $groupArr, $first, $last, 'backfill');
|
||||
if (!$success)
|
||||
{
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID']));
|
||||
|
||||
$this->db->commit(); //At this point we are ready to commit a set up updates to the db.
|
||||
|
||||
if($first==$targetpost)
|
||||
$done = true;
|
||||
else
|
||||
{ //Keep going: set new last, new first, check for last chunk.
|
||||
$last = $first - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1;
|
||||
if($targetpost > $first)
|
||||
$first = $targetpost;
|
||||
}
|
||||
}
|
||||
$first_record_postdate = $this->postdate($nntp,$first,false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate
|
||||
|
||||
$this->db->commit(); //Commit and re-enable auto committing we are done with this group.
|
||||
$this->db->enableAutoCommit();
|
||||
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns single timestamp from a local article number.
|
||||
*/
|
||||
function postdate($nntp,$post,$debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$attempts=0;
|
||||
do
|
||||
{
|
||||
$msgs = $nntp->getOverview($post."-".$post,true,false);
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Returning from postdate$n";
|
||||
return "";
|
||||
}
|
||||
|
||||
if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date']))
|
||||
{
|
||||
$success=false;
|
||||
} else {
|
||||
$date = $msgs[0]['Date'];
|
||||
$success=true;
|
||||
}
|
||||
if($debug && $attempts > 0) echo "retried $attempts time(s)".$n;
|
||||
$attempts++;
|
||||
} while($attempts <= 3 && $success == false);
|
||||
|
||||
if (!$success) { return ""; }
|
||||
|
||||
if($debug) echo "DEBUG: postdate for post: $post came back $date (";
|
||||
$date = strtotime($date);
|
||||
if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n;
|
||||
return $date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the post number for a given number of days back in a group.
|
||||
*/
|
||||
function daytopost($nntp, $group, $days, $debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$pddebug = false; //DEBUG every postdate call?!?!
|
||||
if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n;
|
||||
|
||||
$data = $nntp->selectGroup($group);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Error {$data->code}: {$data->message}$n";
|
||||
echo "Returning from daytopost$n";
|
||||
return "";
|
||||
}
|
||||
$goaldate = date('U')-(86400*$days); //goaltimestamp
|
||||
$totalnumberofarticles = $data['last'] - $data['first'];
|
||||
$upperbound = $data['last'];
|
||||
$lowerbound = $data['first'];
|
||||
|
||||
if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n";
|
||||
if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); }
|
||||
|
||||
$firstDate = $this->postdate($nntp, $data['first'], $pddebug);
|
||||
$lastDate = $this->postdate($nntp, $data['last'], $pddebug);
|
||||
if ($goaldate < $firstDate)
|
||||
{
|
||||
echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n";
|
||||
echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n";
|
||||
return $data['first'];
|
||||
}
|
||||
elseif ($goaldate > $lastDate)
|
||||
{
|
||||
echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n";
|
||||
echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n";
|
||||
return "";
|
||||
}
|
||||
if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n";
|
||||
|
||||
$interval = floor(($upperbound - $lowerbound) * 0.5);
|
||||
$dateofnextone = "";
|
||||
$templowered = "";
|
||||
|
||||
if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n";
|
||||
|
||||
$dateofnextone = $lastDate;
|
||||
|
||||
while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up
|
||||
{
|
||||
$nskip = 1;
|
||||
while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate)
|
||||
{
|
||||
$upperbound = $upperbound - $interval - ($nskip - 1);
|
||||
if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n";
|
||||
$nskip = $nskip * 2;
|
||||
}
|
||||
if(!$templowered)
|
||||
{
|
||||
$interval = ceil(($interval/2));
|
||||
if($debug) echo "Set interval to $interval articles. $n";
|
||||
}
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
|
||||
$skip = 1;
|
||||
while(!$dateofnextone)
|
||||
{
|
||||
$upperbound = $upperbound - $skip;
|
||||
$skip = $skip * 2;
|
||||
if($debug) echo "Getting next article date... $upperbound\n";
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
}
|
||||
}
|
||||
echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n";
|
||||
return $upperbound;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a timestamp.
|
||||
*/
|
||||
private function daysOld($timestamp)
|
||||
{
|
||||
return round((time()-$timestamp)/86400, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a date.
|
||||
*/
|
||||
private function dateToDays($backfillDate)
|
||||
{
|
||||
return floor(-($backfillDate - time())/(60*60*24));
|
||||
}
|
||||
}
|
||||
@@ -1,742 +0,0 @@
|
||||
<?php
|
||||
$newzpath = getenv('NEWZPATH');
|
||||
require_once("$newzpath/www/config.php");
|
||||
require_once("framework/db.php");
|
||||
require_once("nntp.php");
|
||||
require_once("groups.php");
|
||||
require_once("site.php");
|
||||
require_once("backfill.php");
|
||||
|
||||
/**
|
||||
* This class manages the downloading of binaries and parts from usenet, and the
|
||||
* managing of data in the binaries and parts tables.
|
||||
*/
|
||||
class Binaries
|
||||
{
|
||||
const BLACKLIST_FIELD_SUBJECT = 1;
|
||||
const BLACKLIST_FIELD_FROM = 2;
|
||||
const BLACKLIST_FIELD_MESSAGEID = 3;
|
||||
|
||||
/**
|
||||
* Default constructor
|
||||
*/
|
||||
function Binaries($db = null)
|
||||
{
|
||||
$this->n = "\n";
|
||||
|
||||
$s = new Sites();
|
||||
$site = $s->get();
|
||||
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
|
||||
$this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000;
|
||||
$this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false;
|
||||
$this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000;
|
||||
$this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3;
|
||||
|
||||
$this->blackList = array(); //cache of our black/white list
|
||||
$this->message = array();
|
||||
|
||||
if($db == null)
|
||||
{
|
||||
$this->db = new DB();
|
||||
}
|
||||
else
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Allows you to set the db that the current object should use
|
||||
*/
|
||||
public function setDB($db)
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for all active groups.
|
||||
*/
|
||||
function updateAllGroups()
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = $groups->getActive();
|
||||
|
||||
$s = new Sites();
|
||||
echo $s->getLicense();
|
||||
|
||||
if ($res)
|
||||
{
|
||||
shuffle($res);
|
||||
$alltime = microtime(true);
|
||||
echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n;
|
||||
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
|
||||
$pos = 0;
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$pos++;
|
||||
echo 'Group '.$pos.' of '.sizeof($res).$n;
|
||||
$this->message = array();
|
||||
$this->updateGroup($nntp, $groupArr);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n;
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for a group.
|
||||
*/
|
||||
function updateGroup($nntp=null, $groupArr)
|
||||
{
|
||||
$this->db->disableAutoCommit();
|
||||
|
||||
$blnDoDisconnect = false;
|
||||
if ($nntp == null)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
$this->message = array();
|
||||
$blnDoDisconnect = true;
|
||||
}
|
||||
|
||||
$backfill = new Backfill();
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
// Connect to server
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if (PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
|
||||
//Attempt to repair any missing parts before grabbing new ones
|
||||
$this->partRepair($nntp, $groupArr);
|
||||
|
||||
//Get first and last part numbers from newsgroup
|
||||
$last = $grouplast = $data['last'];
|
||||
|
||||
// For new newsgroups - determine here how far you want to go back.
|
||||
if ($groupArr['last_record'] == 0)
|
||||
{
|
||||
if ($this->NewGroupScanByDays)
|
||||
{
|
||||
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
|
||||
if ($first == '')
|
||||
{
|
||||
echo "Skipping group: {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan))
|
||||
$first = $data['first'];
|
||||
else
|
||||
$first = $data['last'] - $this->NewGroupMsgsToScan;
|
||||
}
|
||||
$first_record_postdate = $backfill->postdate($nntp, $first, false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID']));
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['last'] < $groupArr['last_record'])
|
||||
{
|
||||
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n;
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
$first = $groupArr['last_record'] + 1;
|
||||
}
|
||||
|
||||
// Generate postdates for first and last records, for those that upgraded
|
||||
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0"))
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID']));
|
||||
|
||||
// Deactivate empty groups
|
||||
if (($data['last'] - $data['first']) <= 5)
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString('0'), $groupArr['ID']));
|
||||
|
||||
// Calculate total number of parts
|
||||
$total = $grouplast - $first + 1;
|
||||
|
||||
// If total is bigger than 0 it means we have new parts in the newsgroup
|
||||
if($total > 0)
|
||||
{
|
||||
echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n;
|
||||
echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n;
|
||||
if ($groupArr['last_record'] == 0)
|
||||
echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n;
|
||||
|
||||
$done = false;
|
||||
|
||||
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
|
||||
while ($done === false)
|
||||
{
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
if ($total > $this->messagebuffer)
|
||||
{
|
||||
if ($first + $this->messagebuffer > $grouplast)
|
||||
$last = $grouplast;
|
||||
else
|
||||
$last = $first + $this->messagebuffer;
|
||||
}
|
||||
|
||||
echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n;
|
||||
flush();
|
||||
|
||||
//get headers from newsgroup
|
||||
$lastId = $this->scan($nntp, $groupArr, $first, $last);
|
||||
if ($lastId === false)
|
||||
{
|
||||
//scan failed - skip group
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($lastId), $groupArr['ID']));
|
||||
|
||||
$this->db->commit(); //At this point we are ready to commit a whole group to the db.
|
||||
|
||||
if ($last == $grouplast)
|
||||
$done = true;
|
||||
else
|
||||
{
|
||||
$last = $lastId;
|
||||
$first = $last + 1;
|
||||
}
|
||||
}
|
||||
|
||||
$last_record_postdate = $backfill->postdate($nntp,$last,false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n $n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n;
|
||||
}
|
||||
|
||||
if ($blnDoDisconnect)
|
||||
{
|
||||
$nntp->doQuit();
|
||||
}
|
||||
|
||||
/*
|
||||
* Got through the updating of this group successfully
|
||||
* so commit and re-enable the auto committing.
|
||||
*/
|
||||
$this->db->commit(); //For anything not yet committed.
|
||||
$this->db->enableAutoCommit();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a range of usenet messages. Store binaries with subjects matching a
|
||||
* specific pattern in the database.
|
||||
*/
|
||||
function scan($nntp, $groupArr, $first, $last, $type='update')
|
||||
{
|
||||
$n = $this->n;
|
||||
$this->startHeaders = microtime(true);
|
||||
|
||||
if ($this->compressedHeaders)
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
|
||||
if (PEAR::isError($msgs) && $msgs->code == 400)
|
||||
{
|
||||
echo "NNTP connection timed out. Reconnecting...$n";
|
||||
$nntp->doConnect();
|
||||
$nntp->selectGroup($groupArr['name']);
|
||||
if ($this->compressedHeaders)
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
|
||||
$rangerequested = range($first, $last);
|
||||
$msgsreceived = array();
|
||||
$msgsblacklisted = array();
|
||||
$msgsignored = array();
|
||||
$msgsinserted = array();
|
||||
$msgsnotinserted = array();
|
||||
|
||||
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
|
||||
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->startUpdate = microtime(true);
|
||||
if (is_array($msgs))
|
||||
{
|
||||
//loop headers, figure out parts
|
||||
foreach($msgs AS $msg)
|
||||
{
|
||||
if (!isset($msg['Number']))
|
||||
continue;
|
||||
|
||||
$msgsreceived[] = $msg['Number'];
|
||||
$msgPart = $msgTotalParts = 0;
|
||||
|
||||
$pattern = '|\((\d+)[\/](\d+)\)|i';
|
||||
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
|
||||
$matchcnt = sizeof($matches[0]);
|
||||
for ($i=0; $i<$matchcnt; $i++)
|
||||
{
|
||||
$msgPart = $matches[1][$i];
|
||||
$msgTotalParts = $matches[2][$i];
|
||||
}
|
||||
|
||||
if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue
|
||||
{
|
||||
$msgsignored[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
//Filter binaries based on black/white list
|
||||
if ($this->isBlackListed($msg, $groupArr['name']))
|
||||
{
|
||||
$msgsblacklisted[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
if((int)$msgPart > 0 && (int)$msgTotalParts > 0)
|
||||
{
|
||||
$subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject'])));
|
||||
|
||||
if(!isset($this->message[$subject]))
|
||||
{
|
||||
$this->message[$subject] = $msg;
|
||||
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
|
||||
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
|
||||
}
|
||||
if((int)$msgPart > 0)
|
||||
{
|
||||
$this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']);
|
||||
}
|
||||
}
|
||||
}
|
||||
unset($msg);
|
||||
unset($msgs);
|
||||
$count = 0;
|
||||
$updatecount = 0;
|
||||
$partcount = 0;
|
||||
$maxnum = $last;
|
||||
|
||||
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
|
||||
|
||||
|
||||
if ($type != 'partrepair')
|
||||
echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n";
|
||||
|
||||
if ($type == 'update' && sizeof($msgsreceived) == 0)
|
||||
{
|
||||
echo "Error: Server did not return any articles.$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (sizeof($rangenotreceived) > 0) {
|
||||
switch($type)
|
||||
{
|
||||
case 'backfill':
|
||||
//don't add missing articles
|
||||
break;
|
||||
case 'partrepair':
|
||||
case 'update':
|
||||
default:
|
||||
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
|
||||
break;
|
||||
}
|
||||
echo "Server did not return ".count($rangenotreceived)." article(s).$n";
|
||||
}
|
||||
|
||||
if(isset($this->message) && count($this->message))
|
||||
{
|
||||
$maxnum = $first;
|
||||
//insert binaries and parts into database. when binary already exists; only insert new parts
|
||||
foreach($this->message AS $subject => $data)
|
||||
{
|
||||
if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '')
|
||||
{
|
||||
$binaryHash = md5($subject.$data['From'].$groupArr['ID']);
|
||||
$res = $this->db->mysqliQueryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $this->db->escapeString($binaryHash)));
|
||||
if(!$res)
|
||||
{
|
||||
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $this->db->escapeString($subject), $this->db->escapeString($data['From']), $this->db->escapeString($data['Date']), $this->db->escapeString($data['Xref']), $this->db->escapeString($data['MaxParts']), $groupArr['ID'], $this->db->escapeString($binaryHash));
|
||||
$binaryID = $this->db->mysqliQueryInsert($sql);
|
||||
$count++;
|
||||
if ($count%500==0) echo "$count bin adds...";
|
||||
}
|
||||
else
|
||||
{
|
||||
$binaryID = $res["ID"];
|
||||
$updatecount++;
|
||||
if ($updatecount%500==0) echo "$updatecount bin updates...";
|
||||
}
|
||||
|
||||
foreach($data['Parts'] AS $partdata)
|
||||
{
|
||||
$maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum;
|
||||
$partcount++;
|
||||
$pidata = $this->db->mysqliQueryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $this->db->escapeString($partdata['Message-ID']), $this->db->escapeString($partdata['number']), $this->db->escapeString(round($partdata['part'])), $this->db->escapeString($partdata['size'])), false);
|
||||
if (!$pidata) {
|
||||
$msgsnotinserted[] = $partdata['number'];
|
||||
} else {
|
||||
$msgsinserted[] = $partdata['number'];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//TODO: determine whether to add to missing articles if insert failed
|
||||
if (sizeof($msgsnotinserted) > 0)
|
||||
{
|
||||
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n;
|
||||
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
|
||||
}
|
||||
if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output
|
||||
}
|
||||
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
|
||||
$timeLoop = number_format(microtime(true)-$this->startLoop, 2);
|
||||
|
||||
if ($type != 'partrepair')
|
||||
{
|
||||
echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.';
|
||||
echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n";
|
||||
}
|
||||
unset($this->message);
|
||||
unset($data);
|
||||
return $maxnum;
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "Error: Can't get parts from server (msgs not array) $n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Go through all rows in partrepair table and see if theyve arrived on usenet yet.
|
||||
*/
|
||||
private function partRepair($nntp, $groupArr)
|
||||
{
|
||||
$n = $this->n;
|
||||
|
||||
//get all parts in partrepair table
|
||||
|
||||
$missingParts = $this->db->mysqliQuery(sprintf("SELECT * FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 30000", $groupArr['ID']));
|
||||
$partsRepaired = $partsFailed = 0;
|
||||
|
||||
if (sizeof($missingParts) > 0)
|
||||
{
|
||||
echo 'Attempting to repair '.sizeof($missingParts).' parts...';
|
||||
|
||||
//loop through each part to group into ranges
|
||||
$ranges = array();
|
||||
$lastnum = $lastpart = 0;
|
||||
foreach($missingParts as $part)
|
||||
{
|
||||
if (($lastnum+1) == $part['numberID']) {
|
||||
$ranges[$lastpart] = $part['numberID'];
|
||||
} else {
|
||||
$lastpart = $part['numberID'];
|
||||
$ranges[$lastpart] = $part['numberID'];
|
||||
}
|
||||
$lastnum = $part['numberID'];
|
||||
}
|
||||
|
||||
//download missing parts in ranges
|
||||
foreach($ranges as $partfrom=>$partto)
|
||||
{
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
echo ".";
|
||||
|
||||
//get article from newsgroup
|
||||
$this->scan($nntp, $groupArr, $partfrom, $partto, 'partrepair');
|
||||
|
||||
//check if the articles were added
|
||||
$articles = implode(',', range($partfrom, $partto));
|
||||
$sql = sprintf("SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", $groupArr['ID'], $articles);
|
||||
|
||||
$result = $this->db->mysqliQueryDirect($sql);
|
||||
while ($r = mysqli_fetch_assoc($result))
|
||||
{
|
||||
if (isset($r['number']) && $r['number'] == $r['numberID'])
|
||||
{
|
||||
$partsRepaired++;
|
||||
|
||||
//article was added, delete from partrepair
|
||||
$this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE ID=%d", $r['ID']));
|
||||
}
|
||||
else
|
||||
{
|
||||
$partsFailed++;
|
||||
|
||||
//article was not added, increment attempts
|
||||
$this->db->mysqliQuery(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d", $r['ID']));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
echo $n.$partsRepaired.' parts repaired.'.$n;
|
||||
}
|
||||
|
||||
//remove articles that we cant fetch after 5 attempts
|
||||
$this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $groupArr['ID']));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a missing part to the database.
|
||||
*/
|
||||
private function addMissingParts($numbers, $groupID)
|
||||
{
|
||||
|
||||
$added = false;
|
||||
$insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES ";
|
||||
foreach($numbers as $number)
|
||||
{
|
||||
if ($number > 0)
|
||||
{
|
||||
$added = true;
|
||||
$insertStr .= sprintf("(%u, %d), ", $number, $groupID);
|
||||
}
|
||||
}
|
||||
if ($added)
|
||||
{
|
||||
$insertStr = substr($insertStr, 0, -2);
|
||||
$insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1";
|
||||
return $this->db->mysqliQueryInsert($insertStr, false);
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return internally cached list of binary blacklist patterns.
|
||||
*/
|
||||
public function retrieveBlackList()
|
||||
{
|
||||
if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; }
|
||||
$blackList = $this->getBlacklist(true);
|
||||
$this->blackList = $blackList;
|
||||
return $blackList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if a message subject is blacklisted.
|
||||
*/
|
||||
public function isBlackListed($msg, $groupName)
|
||||
{
|
||||
$blackList = $this->retrieveBlackList();
|
||||
$field = array();
|
||||
if (isset($msg["Subject"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"];
|
||||
|
||||
if (isset($msg["From"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"];
|
||||
|
||||
if (isset($msg["Message-ID"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"];
|
||||
|
||||
foreach ($blackList as $blist)
|
||||
{
|
||||
if (preg_match('/^'.$blist['groupname'].'$/i', $groupName))
|
||||
{
|
||||
//blacklist
|
||||
if ($blist['optype'] == 1)
|
||||
{
|
||||
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else if ($blist['optype'] == 2)
|
||||
{
|
||||
if (!preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rawsearch. Perform a simple like match on binary subjects matching a pattern.
|
||||
*/
|
||||
public function search($search, $limit=1000, $excludedcats=array())
|
||||
{
|
||||
|
||||
|
||||
//
|
||||
// if the query starts with a ^ it indicates the search is looking for items which start with the term
|
||||
// still do the like match, but mandate that all items returned must start with the provided word
|
||||
//
|
||||
$words = explode(" ", $search);
|
||||
$searchsql = "";
|
||||
$intwordcount = 0;
|
||||
if (count($words) > 0)
|
||||
{
|
||||
foreach ($words as $word)
|
||||
{
|
||||
//
|
||||
// see if the first word had a caret, which indicates search must start with term
|
||||
//
|
||||
if ($intwordcount == 0 && (strpos($word, "^") === 0))
|
||||
$searchsql.= sprintf(" and b.name like %s", $this->db->escapeString(substr($word, 1)."%"));
|
||||
else
|
||||
$searchsql.= sprintf(" and b.name like %s", $this->db->escapeString("%".$word."%"));
|
||||
|
||||
$intwordcount++;
|
||||
}
|
||||
}
|
||||
|
||||
$exccatlist = "";
|
||||
if (count($excludedcats) > 0)
|
||||
$exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") ";
|
||||
|
||||
$res = $this->db->mysqliQuery(sprintf("
|
||||
SELECT b.*,
|
||||
g.name AS group_name,
|
||||
r.guid,
|
||||
(SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum'
|
||||
FROM binaries b
|
||||
INNER JOIN groups g ON g.ID = b.groupID
|
||||
LEFT OUTER JOIN releases r ON r.ID = b.releaseID
|
||||
WHERE 1=1 %s %s order by DATE DESC LIMIT %d ",
|
||||
$searchsql, $exccatlist, $limit));
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all binaries for a release.
|
||||
*/
|
||||
public function getForReleaseId($id)
|
||||
{
|
||||
return $this->db->mysqliQuery(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a binary row.
|
||||
*/
|
||||
public function getById($id)
|
||||
{
|
||||
|
||||
return $this->db->mysqliQueryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of blacklists from database.
|
||||
*/
|
||||
public function getBlacklist($activeonly=true)
|
||||
{
|
||||
|
||||
$where = "";
|
||||
if ($activeonly)
|
||||
$where = " where binaryblacklist.status = 1 ";
|
||||
|
||||
return $this->db->mysqliQuery("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex,
|
||||
groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist
|
||||
left outer JOIN groups ON groups.name = binaryblacklist.groupname
|
||||
".$where."
|
||||
ORDER BY coalesce(groupname,'zzz')");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a blacklist row from database.
|
||||
*/
|
||||
public function getBlacklistByID($id)
|
||||
{
|
||||
return $this->db->mysqliQueryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a blacklist row from database.
|
||||
*/
|
||||
public function deleteBlacklist($id)
|
||||
{
|
||||
|
||||
return $this->db->mysqliQuery(sprintf("delete from binaryblacklist where ID = %d", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a blacklist row.
|
||||
*/
|
||||
public function updateBlacklist($regex)
|
||||
{
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $this->db->escapeString($groupname));
|
||||
}
|
||||
|
||||
$this->db->mysqliQuery(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new blacklist row.
|
||||
*/
|
||||
public function addBlacklist($regex)
|
||||
{
|
||||
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $this->db->escapeString($groupname));
|
||||
}
|
||||
|
||||
return $this->db->mysqliQueryInsert(sprintf("insert into binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ",
|
||||
$groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new binary row and its associated parts.
|
||||
*/
|
||||
public function delete($id)
|
||||
{
|
||||
|
||||
$this->db->mysqliQuery(sprintf("delete from parts where binaryID = %d", $id));
|
||||
$this->db->mysqliQuery(sprintf("delete from binaries where ID = %d", $id));
|
||||
}
|
||||
}
|
||||
@@ -1,175 +0,0 @@
|
||||
<?php
|
||||
|
||||
class DB
|
||||
{
|
||||
private static $initialized = false;
|
||||
|
||||
function DB()
|
||||
{
|
||||
if (DB::$initialized === false)
|
||||
{
|
||||
// initialize db connection
|
||||
mysql_pconnect(DB_HOST, DB_USER, DB_PASSWORD)
|
||||
or die("fatal error: could not connect to database! Check your config.");
|
||||
|
||||
mysql_select_db(DB_NAME)
|
||||
or die("fatal error: could not select database! Check your config.");
|
||||
|
||||
mysql_set_charset('utf8');
|
||||
|
||||
DB::$initialized = true;
|
||||
}
|
||||
|
||||
$this->mysqli = new mysqli(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME);
|
||||
if (mysqli_connect_errno()) {
|
||||
printf("Connect failed: %s\n", mysqli_connect_error());
|
||||
exit();
|
||||
}
|
||||
}
|
||||
|
||||
public function escapeString($str)
|
||||
{
|
||||
return "'".mysql_real_escape_string($str)."'";
|
||||
}
|
||||
|
||||
public function makeLookupTable($rows, $keycol)
|
||||
{
|
||||
$arr = array();
|
||||
foreach($rows as $row)
|
||||
$arr[$row[$keycol]] = $row;
|
||||
return $arr;
|
||||
}
|
||||
|
||||
public function queryInsert($query, $returnlastid=true)
|
||||
{
|
||||
$result = mysql_query($query);
|
||||
return ($returnlastid) ? mysql_insert_id() : $result;
|
||||
}
|
||||
|
||||
public function queryOneRow($query)
|
||||
{
|
||||
$rows = $this->query($query);
|
||||
|
||||
if (!$rows)
|
||||
return false;
|
||||
|
||||
if ($rows)
|
||||
return $rows[0];
|
||||
else
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function query($query)
|
||||
{
|
||||
$result = mysql_query($query);
|
||||
|
||||
if ($result === false || $result === true)
|
||||
return array();
|
||||
|
||||
$rows = array();
|
||||
|
||||
while ($row = mysql_fetch_assoc($result))
|
||||
$rows[] = $row;
|
||||
|
||||
mysql_free_result($result);
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function queryDirect($query)
|
||||
{
|
||||
$result = mysql_query($query);
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
||||
public function optimise()
|
||||
{
|
||||
$ret = array();
|
||||
$alltables = $this->query("SHOW TABLES");
|
||||
|
||||
foreach ($alltables as $tablename)
|
||||
{
|
||||
$ret[] = $tablename['Tables_in_'.DB_NAME];
|
||||
$this->queryDirect("REPAIR TABLE `".$tablename['Tables_in_'.DB_NAME]."`");
|
||||
$this->queryDirect("OPTIMIZE TABLE `".$tablename['Tables_in_'.DB_NAME]."`");
|
||||
}
|
||||
|
||||
return $ret;
|
||||
}
|
||||
|
||||
/* Mysqli Functions */
|
||||
public function multiQueryTransaction($query)
|
||||
{
|
||||
$this->mysqli->autocommit(FALSE);
|
||||
$this->mysqli->multi_query($query);
|
||||
|
||||
while($this->mysqli->next_result()) {
|
||||
$result = $this->mysqli->use_result();
|
||||
if($result instanceof mysqli_result)
|
||||
$result->free();
|
||||
}
|
||||
|
||||
$this->mysqli->commit();
|
||||
}
|
||||
|
||||
public function disableAutoCommit()
|
||||
{
|
||||
$this->mysqli->autocommit(FALSE);
|
||||
}
|
||||
|
||||
public function mysqliQuery($sql)
|
||||
{
|
||||
$result = $this->mysqli->query($sql);
|
||||
if ($result === false || $result === true)
|
||||
return array();
|
||||
|
||||
$rows = array();
|
||||
|
||||
while ($row = mysqli_fetch_assoc($result))
|
||||
$rows[] = $row;
|
||||
|
||||
mysqli_free_result($result);
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function mysqliQueryOneRow($query)
|
||||
{
|
||||
$rows = $this->mysqliQuery($query);
|
||||
|
||||
if (!$rows)
|
||||
return false;
|
||||
|
||||
if ($rows)
|
||||
return $rows[0];
|
||||
else
|
||||
return $rows;
|
||||
}
|
||||
|
||||
public function mysqliQueryInsert($query, $returnlastid=true)
|
||||
{
|
||||
$result = $this->mysqli->query($query);
|
||||
return ($returnlastid) ? $this->mysqli->insert_id : $result;
|
||||
}
|
||||
|
||||
public function commit()
|
||||
{
|
||||
$this->mysqli->commit();
|
||||
}
|
||||
|
||||
public function rollback()
|
||||
{
|
||||
$this->mysqli->rollback();
|
||||
}
|
||||
|
||||
public function mysqliQueryDirect($query)
|
||||
{
|
||||
return $this->mysqli->query($query);
|
||||
}
|
||||
|
||||
|
||||
public function enableAutoCommit()
|
||||
{
|
||||
$this->mysqli->autocommit(TRUE);
|
||||
}
|
||||
}
|
||||
?>
|
||||
@@ -1,27 +0,0 @@
|
||||
<?php
|
||||
require_once("framework/db.php");
|
||||
|
||||
/**
|
||||
* This class handles data access for groups.
|
||||
*/
|
||||
class Groups
|
||||
{
|
||||
/**
|
||||
* Get all active group rows.
|
||||
*/
|
||||
public function getActive()
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->query("SELECT * FROM groups WHERE active = 1 ORDER BY name");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a group row by name.
|
||||
*/
|
||||
public function getByName($grp)
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->queryOneRow(sprintf("select * from groups where name = '%s' ", $grp));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
<?php
|
||||
require_once("config.php");
|
||||
require_once(dirname(__FILE__)."/../framework/db.php");
|
||||
require_once(dirname(__FILE__)."/../groups.php");
|
||||
require_once(dirname(__FILE__)."/../site.php");
|
||||
require_once(dirname(__FILE__)."/../nntp.php");
|
||||
require_once("binaries.php");
|
||||
|
||||
/**
|
||||
* Retrieves messages from usenet based on provided backfill-to date.
|
||||
*/
|
||||
class Backfill
|
||||
{
|
||||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
function Backfill($db = null)
|
||||
{
|
||||
if($db == null)
|
||||
{
|
||||
$this->db = new DB();
|
||||
}
|
||||
else
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
$this->n = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all active groups categories and descriptions.
|
||||
*/
|
||||
function backfillAllGroups($groupName='', $backfillDate=null)
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = false;
|
||||
if ($groupName != '')
|
||||
{
|
||||
$grp = $groups->getByName($groupName);
|
||||
if ($grp)
|
||||
$res = array($grp);
|
||||
}
|
||||
else
|
||||
{
|
||||
$res = $groups->getActive();
|
||||
}
|
||||
|
||||
if ($res)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$this->backfillGroup($nntp, $groupArr, $backfillDate);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database for updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a group back to a specified date.
|
||||
*/
|
||||
function backfillGroup($nntp, $groupArr, $backfillDate=null)
|
||||
{
|
||||
$this->db->disableAutoCommit(); //Turn off auto committing
|
||||
|
||||
$binaries = new Binaries($this->db);
|
||||
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
if ($backfillDate)
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date
|
||||
else
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target
|
||||
if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0)
|
||||
{
|
||||
echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
|
||||
echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~";
|
||||
echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400));
|
||||
echo " days.".$n."Local first = ".$groupArr['first_record']." (";
|
||||
echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400));
|
||||
echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n";
|
||||
|
||||
if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish
|
||||
{
|
||||
echo "Nothing to do, we already have the target post.$n $n";
|
||||
|
||||
$this->db->commit(); //Not an error so commit and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
//get first and last part numbers from newsgroup
|
||||
if($targetpost < $data['first'])
|
||||
{
|
||||
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n";
|
||||
echo "Skipping Group $n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
//calculate total number of parts
|
||||
$total = $groupArr['first_record'] - $targetpost;
|
||||
$done = false;
|
||||
//set first and last, moving the window by maxxMssgs
|
||||
$last = $groupArr['first_record'] - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1; //set initial "chunk"
|
||||
if($targetpost > $first) //just in case this is the last chunk we needed
|
||||
$first = $targetpost;
|
||||
while($done === false)
|
||||
{
|
||||
$binaries->startLoop = microtime(true);
|
||||
echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n;
|
||||
flush();
|
||||
$success = $binaries->scan($nntp, $groupArr, $first, $last, 'backfill');
|
||||
if (!$success)
|
||||
{
|
||||
$this->db->rollback(); //Rollback and re-enable autocommitting
|
||||
$this->db->enableAutoCommit();
|
||||
return "";
|
||||
}
|
||||
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID']));
|
||||
|
||||
$this->db->commit(); //At this point we are ready to commit a set up updates to the db.
|
||||
|
||||
if($first==$targetpost)
|
||||
$done = true;
|
||||
else
|
||||
{ //Keep going: set new last, new first, check for last chunk.
|
||||
$last = $first - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1;
|
||||
if($targetpost > $first)
|
||||
$first = $targetpost;
|
||||
}
|
||||
}
|
||||
$first_record_postdate = $this->postdate($nntp,$first,false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate
|
||||
|
||||
$this->db->commit(); //Commit and re-enable auto committing we are done with this group.
|
||||
$this->db->enableAutoCommit();
|
||||
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns single timestamp from a local article number.
|
||||
*/
|
||||
function postdate($nntp,$post,$debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$attempts=0;
|
||||
do
|
||||
{
|
||||
$msgs = $nntp->getOverview($post."-".$post,true,false);
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Returning from postdate$n";
|
||||
return "";
|
||||
}
|
||||
|
||||
if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date']))
|
||||
{
|
||||
$success=false;
|
||||
} else {
|
||||
$date = $msgs[0]['Date'];
|
||||
$success=true;
|
||||
}
|
||||
if($debug && $attempts > 0) echo "retried $attempts time(s)".$n;
|
||||
$attempts++;
|
||||
} while($attempts <= 3 && $success == false);
|
||||
|
||||
if (!$success) { return ""; }
|
||||
|
||||
if($debug) echo "DEBUG: postdate for post: $post came back $date (";
|
||||
$date = strtotime($date);
|
||||
if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n;
|
||||
return $date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the post number for a given number of days back in a group.
|
||||
*/
|
||||
function daytopost($nntp, $group, $days, $debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$pddebug = false; //DEBUG every postdate call?!?!
|
||||
if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n;
|
||||
|
||||
$data = $nntp->selectGroup($group);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Error {$data->code}: {$data->message}$n";
|
||||
echo "Returning from daytopost$n";
|
||||
return "";
|
||||
}
|
||||
$goaldate = date('U')-(86400*$days); //goaltimestamp
|
||||
$totalnumberofarticles = $data['last'] - $data['first'];
|
||||
$upperbound = $data['last'];
|
||||
$lowerbound = $data['first'];
|
||||
|
||||
if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n";
|
||||
if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); }
|
||||
|
||||
$firstDate = $this->postdate($nntp, $data['first'], $pddebug);
|
||||
$lastDate = $this->postdate($nntp, $data['last'], $pddebug);
|
||||
if ($goaldate < $firstDate)
|
||||
{
|
||||
echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n";
|
||||
echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n";
|
||||
return $data['first'];
|
||||
}
|
||||
elseif ($goaldate > $lastDate)
|
||||
{
|
||||
echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n";
|
||||
echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n";
|
||||
return "";
|
||||
}
|
||||
if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n";
|
||||
|
||||
$interval = floor(($upperbound - $lowerbound) * 0.5);
|
||||
$dateofnextone = "";
|
||||
$templowered = "";
|
||||
|
||||
if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n";
|
||||
|
||||
$dateofnextone = $lastDate;
|
||||
|
||||
while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up
|
||||
{
|
||||
$nskip = 1;
|
||||
while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate)
|
||||
{
|
||||
$upperbound = $upperbound - $interval - ($nskip - 1);
|
||||
if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n";
|
||||
$nskip = $nskip * 2;
|
||||
}
|
||||
if(!$templowered)
|
||||
{
|
||||
$interval = ceil(($interval/2));
|
||||
if($debug) echo "Set interval to $interval articles. $n";
|
||||
}
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
|
||||
$skip = 1;
|
||||
while(!$dateofnextone)
|
||||
{
|
||||
$upperbound = $upperbound - $skip;
|
||||
$skip = $skip * 2;
|
||||
if($debug) echo "Getting next article date... $upperbound\n";
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
}
|
||||
}
|
||||
echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n";
|
||||
return $upperbound;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a timestamp.
|
||||
*/
|
||||
private function daysOld($timestamp)
|
||||
{
|
||||
return round((time()-$timestamp)/86400, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a date.
|
||||
*/
|
||||
private function dateToDays($backfillDate)
|
||||
{
|
||||
return floor(-($backfillDate - time())/(60*60*24));
|
||||
}
|
||||
}
|
||||
@@ -1,741 +0,0 @@
|
||||
<?php
|
||||
require_once("config.php");
|
||||
require_once(dirname(__FILE__)."/../framework/db.php");
|
||||
require_once(dirname(__FILE__)."/../nntp.php");
|
||||
require_once(dirname(__FILE__)."/../groups.php");
|
||||
require_once(dirname(__FILE__)."/../site.php");
|
||||
require_once(dirname(__FILE__)."/../backfill.php");
|
||||
|
||||
/**
|
||||
* This class manages the downloading of binaries and parts from usenet, and the
|
||||
* managing of data in the binaries and parts tables.
|
||||
*/
|
||||
class Binaries
|
||||
{
|
||||
const BLACKLIST_FIELD_SUBJECT = 1;
|
||||
const BLACKLIST_FIELD_FROM = 2;
|
||||
const BLACKLIST_FIELD_MESSAGEID = 3;
|
||||
|
||||
/**
|
||||
* Default constructor
|
||||
*/
|
||||
function Binaries($db = null)
|
||||
{
|
||||
$this->n = "\n";
|
||||
|
||||
$s = new Sites();
|
||||
$site = $s->get();
|
||||
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
|
||||
$this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000;
|
||||
$this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false;
|
||||
$this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000;
|
||||
$this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3;
|
||||
|
||||
$this->blackList = array(); //cache of our black/white list
|
||||
$this->message = array();
|
||||
|
||||
if($db == null)
|
||||
{
|
||||
$this->db = new DB();
|
||||
}
|
||||
else
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Allows you to set the db that the current object should use
|
||||
*/
|
||||
public function setDB($db)
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for all active groups.
|
||||
*/
|
||||
function updateAllGroups()
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = $groups->getActive();
|
||||
|
||||
$s = new Sites();
|
||||
echo $s->getLicense();
|
||||
|
||||
if ($res)
|
||||
{
|
||||
shuffle($res);
|
||||
$alltime = microtime(true);
|
||||
echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n;
|
||||
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
|
||||
$pos = 0;
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$pos++;
|
||||
echo 'Group '.$pos.' of '.sizeof($res).$n;
|
||||
$this->message = array();
|
||||
$this->updateGroup($nntp, $groupArr);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n;
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for a group.
|
||||
*/
|
||||
function updateGroup($nntp=null, $groupArr)
|
||||
{
|
||||
$this->db->disableAutoCommit();
|
||||
|
||||
$blnDoDisconnect = false;
|
||||
if ($nntp == null)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
$nntp->doConnect();
|
||||
$this->message = array();
|
||||
$blnDoDisconnect = true;
|
||||
}
|
||||
|
||||
$backfill = new Backfill();
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
// Connect to server
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if (PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
|
||||
//Attempt to repair any missing parts before grabbing new ones
|
||||
$this->partRepair($nntp, $groupArr);
|
||||
|
||||
//Get first and last part numbers from newsgroup
|
||||
$last = $grouplast = $data['last'];
|
||||
|
||||
// For new newsgroups - determine here how far you want to go back.
|
||||
if ($groupArr['last_record'] == 0)
|
||||
{
|
||||
if ($this->NewGroupScanByDays)
|
||||
{
|
||||
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
|
||||
if ($first == '')
|
||||
{
|
||||
echo "Skipping group: {$groupArr['name']}$n";
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan))
|
||||
$first = $data['first'];
|
||||
else
|
||||
$first = $data['last'] - $this->NewGroupMsgsToScan;
|
||||
}
|
||||
$first_record_postdate = $backfill->postdate($nntp, $first, false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID']));
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['last'] < $groupArr['last_record'])
|
||||
{
|
||||
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n;
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
$first = $groupArr['last_record'] + 1;
|
||||
}
|
||||
|
||||
// Generate postdates for first and last records, for those that upgraded
|
||||
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0"))
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID']));
|
||||
|
||||
// Deactivate empty groups
|
||||
if (($data['last'] - $data['first']) <= 5)
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString('0'), $groupArr['ID']));
|
||||
|
||||
// Calculate total number of parts
|
||||
$total = $grouplast - $first + 1;
|
||||
|
||||
// If total is bigger than 0 it means we have new parts in the newsgroup
|
||||
if($total > 0)
|
||||
{
|
||||
echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n;
|
||||
echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n;
|
||||
if ($groupArr['last_record'] == 0)
|
||||
echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n;
|
||||
|
||||
$done = false;
|
||||
|
||||
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
|
||||
while ($done === false)
|
||||
{
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
if ($total > $this->messagebuffer)
|
||||
{
|
||||
if ($first + $this->messagebuffer > $grouplast)
|
||||
$last = $grouplast;
|
||||
else
|
||||
$last = $first + $this->messagebuffer;
|
||||
}
|
||||
|
||||
echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n;
|
||||
flush();
|
||||
|
||||
//get headers from newsgroup
|
||||
$lastId = $this->scan($nntp, $groupArr, $first, $last);
|
||||
if ($lastId === false)
|
||||
{
|
||||
//scan failed - skip group
|
||||
|
||||
$this->db->rollback(); //Rollback and re-enable auto committing
|
||||
$this->db->enableAutoCommit();
|
||||
return;
|
||||
}
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($lastId), $groupArr['ID']));
|
||||
|
||||
$this->db->commit(); //At this point we are ready to commit a whole group to the db.
|
||||
|
||||
if ($last == $grouplast)
|
||||
$done = true;
|
||||
else
|
||||
{
|
||||
$last = $lastId;
|
||||
$first = $last + 1;
|
||||
}
|
||||
}
|
||||
|
||||
$last_record_postdate = $backfill->postdate($nntp,$last,false);
|
||||
$this->db->mysqliQuery(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n $n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n;
|
||||
}
|
||||
|
||||
if ($blnDoDisconnect)
|
||||
{
|
||||
$nntp->doQuit();
|
||||
}
|
||||
|
||||
/*
|
||||
* Got through the updating of this group successfully
|
||||
* so commit and re-enable the auto committing.
|
||||
*/
|
||||
$this->db->commit(); //For anything not yet committed.
|
||||
$this->db->enableAutoCommit();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a range of usenet messages. Store binaries with subjects matching a
|
||||
* specific pattern in the database.
|
||||
*/
|
||||
function scan($nntp, $groupArr, $first, $last, $type='update')
|
||||
{
|
||||
$n = $this->n;
|
||||
$this->startHeaders = microtime(true);
|
||||
|
||||
if ($this->compressedHeaders)
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
|
||||
if (PEAR::isError($msgs) && $msgs->code == 400)
|
||||
{
|
||||
echo "NNTP connection timed out. Reconnecting...$n";
|
||||
$nntp->doConnect();
|
||||
$nntp->selectGroup($groupArr['name']);
|
||||
if ($this->compressedHeaders)
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
|
||||
$rangerequested = range($first, $last);
|
||||
$msgsreceived = array();
|
||||
$msgsblacklisted = array();
|
||||
$msgsignored = array();
|
||||
$msgsinserted = array();
|
||||
$msgsnotinserted = array();
|
||||
|
||||
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
|
||||
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->startUpdate = microtime(true);
|
||||
if (is_array($msgs))
|
||||
{
|
||||
//loop headers, figure out parts
|
||||
foreach($msgs AS $msg)
|
||||
{
|
||||
if (!isset($msg['Number']))
|
||||
continue;
|
||||
|
||||
$msgsreceived[] = $msg['Number'];
|
||||
$msgPart = $msgTotalParts = 0;
|
||||
|
||||
$pattern = '|\((\d+)[\/](\d+)\)|i';
|
||||
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
|
||||
$matchcnt = sizeof($matches[0]);
|
||||
for ($i=0; $i<$matchcnt; $i++)
|
||||
{
|
||||
$msgPart = $matches[1][$i];
|
||||
$msgTotalParts = $matches[2][$i];
|
||||
}
|
||||
|
||||
if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue
|
||||
{
|
||||
$msgsignored[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
//Filter binaries based on black/white list
|
||||
if ($this->isBlackListed($msg, $groupArr['name']))
|
||||
{
|
||||
$msgsblacklisted[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
if((int)$msgPart > 0 && (int)$msgTotalParts > 0)
|
||||
{
|
||||
$subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject'])));
|
||||
|
||||
if(!isset($this->message[$subject]))
|
||||
{
|
||||
$this->message[$subject] = $msg;
|
||||
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
|
||||
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
|
||||
}
|
||||
if((int)$msgPart > 0)
|
||||
{
|
||||
$this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']);
|
||||
}
|
||||
}
|
||||
}
|
||||
unset($msg);
|
||||
unset($msgs);
|
||||
$count = 0;
|
||||
$updatecount = 0;
|
||||
$partcount = 0;
|
||||
$maxnum = $last;
|
||||
|
||||
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
|
||||
|
||||
|
||||
if ($type != 'partrepair')
|
||||
echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n";
|
||||
|
||||
if ($type == 'update' && sizeof($msgsreceived) == 0)
|
||||
{
|
||||
echo "Error: Server did not return any articles.$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (sizeof($rangenotreceived) > 0) {
|
||||
switch($type)
|
||||
{
|
||||
case 'backfill':
|
||||
//don't add missing articles
|
||||
break;
|
||||
case 'partrepair':
|
||||
case 'update':
|
||||
default:
|
||||
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
|
||||
break;
|
||||
}
|
||||
echo "Server did not return ".count($rangenotreceived)." article(s).$n";
|
||||
}
|
||||
|
||||
if(isset($this->message) && count($this->message))
|
||||
{
|
||||
$maxnum = $first;
|
||||
//insert binaries and parts into database. when binary already exists; only insert new parts
|
||||
foreach($this->message AS $subject => $data)
|
||||
{
|
||||
if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '')
|
||||
{
|
||||
$binaryHash = md5($subject.$data['From'].$groupArr['ID']);
|
||||
$res = $this->db->mysqliQueryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $this->db->escapeString($binaryHash)));
|
||||
if(!$res)
|
||||
{
|
||||
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $this->db->escapeString($subject), $this->db->escapeString($data['From']), $this->db->escapeString($data['Date']), $this->db->escapeString($data['Xref']), $this->db->escapeString($data['MaxParts']), $groupArr['ID'], $this->db->escapeString($binaryHash));
|
||||
$binaryID = $this->db->mysqliQueryInsert($sql);
|
||||
$count++;
|
||||
if ($count%500==0) echo "$count bin adds...";
|
||||
}
|
||||
else
|
||||
{
|
||||
$binaryID = $res["ID"];
|
||||
$updatecount++;
|
||||
if ($updatecount%500==0) echo "$updatecount bin updates...";
|
||||
}
|
||||
|
||||
foreach($data['Parts'] AS $partdata)
|
||||
{
|
||||
$maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum;
|
||||
$partcount++;
|
||||
$pidata = $this->db->mysqliQueryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $this->db->escapeString($partdata['Message-ID']), $this->db->escapeString($partdata['number']), $this->db->escapeString(round($partdata['part'])), $this->db->escapeString($partdata['size'])), false);
|
||||
if (!$pidata) {
|
||||
$msgsnotinserted[] = $partdata['number'];
|
||||
} else {
|
||||
$msgsinserted[] = $partdata['number'];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//TODO: determine whether to add to missing articles if insert failed
|
||||
if (sizeof($msgsnotinserted) > 0)
|
||||
{
|
||||
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n;
|
||||
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
|
||||
}
|
||||
if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output
|
||||
}
|
||||
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
|
||||
$timeLoop = number_format(microtime(true)-$this->startLoop, 2);
|
||||
|
||||
if ($type != 'partrepair')
|
||||
{
|
||||
echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.';
|
||||
echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n";
|
||||
}
|
||||
unset($this->message);
|
||||
unset($data);
|
||||
return $maxnum;
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "Error: Can't get parts from server (msgs not array) $n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Go through all rows in partrepair table and see if theyve arrived on usenet yet.
|
||||
*/
|
||||
private function partRepair($nntp, $groupArr)
|
||||
{
|
||||
$n = $this->n;
|
||||
|
||||
//get all parts in partrepair table
|
||||
|
||||
$missingParts = $this->db->mysqliQuery(sprintf("SELECT * FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 30000", $groupArr['ID']));
|
||||
$partsRepaired = $partsFailed = 0;
|
||||
|
||||
if (sizeof($missingParts) > 0)
|
||||
{
|
||||
echo 'Attempting to repair '.sizeof($missingParts).' parts...';
|
||||
|
||||
//loop through each part to group into ranges
|
||||
$ranges = array();
|
||||
$lastnum = $lastpart = 0;
|
||||
foreach($missingParts as $part)
|
||||
{
|
||||
if (($lastnum+1) == $part['numberID']) {
|
||||
$ranges[$lastpart] = $part['numberID'];
|
||||
} else {
|
||||
$lastpart = $part['numberID'];
|
||||
$ranges[$lastpart] = $part['numberID'];
|
||||
}
|
||||
$lastnum = $part['numberID'];
|
||||
}
|
||||
|
||||
//download missing parts in ranges
|
||||
foreach($ranges as $partfrom=>$partto)
|
||||
{
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
echo ".";
|
||||
|
||||
//get article from newsgroup
|
||||
$this->scan($nntp, $groupArr, $partfrom, $partto, 'partrepair');
|
||||
|
||||
//check if the articles were added
|
||||
$articles = implode(',', range($partfrom, $partto));
|
||||
$sql = sprintf("SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", $groupArr['ID'], $articles);
|
||||
|
||||
$result = $this->db->mysqliQueryDirect($sql);
|
||||
while ($r = mysqli_fetch_assoc($result))
|
||||
{
|
||||
if (isset($r['number']) && $r['number'] == $r['numberID'])
|
||||
{
|
||||
$partsRepaired++;
|
||||
|
||||
//article was added, delete from partrepair
|
||||
$this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE ID=%d", $r['ID']));
|
||||
}
|
||||
else
|
||||
{
|
||||
$partsFailed++;
|
||||
|
||||
//article was not added, increment attempts
|
||||
$this->db->mysqliQuery(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d", $r['ID']));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
echo $n.$partsRepaired.' parts repaired.'.$n;
|
||||
}
|
||||
|
||||
//remove articles that we cant fetch after 5 attempts
|
||||
$this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $groupArr['ID']));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a missing part to the database.
|
||||
*/
|
||||
private function addMissingParts($numbers, $groupID)
|
||||
{
|
||||
|
||||
$added = false;
|
||||
$insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES ";
|
||||
foreach($numbers as $number)
|
||||
{
|
||||
if ($number > 0)
|
||||
{
|
||||
$added = true;
|
||||
$insertStr .= sprintf("(%u, %d), ", $number, $groupID);
|
||||
}
|
||||
}
|
||||
if ($added)
|
||||
{
|
||||
$insertStr = substr($insertStr, 0, -2);
|
||||
$insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1";
|
||||
return $this->db->mysqliQueryInsert($insertStr, false);
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return internally cached list of binary blacklist patterns.
|
||||
*/
|
||||
public function retrieveBlackList()
|
||||
{
|
||||
if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; }
|
||||
$blackList = $this->getBlacklist(true);
|
||||
$this->blackList = $blackList;
|
||||
return $blackList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if a message subject is blacklisted.
|
||||
*/
|
||||
public function isBlackListed($msg, $groupName)
|
||||
{
|
||||
$blackList = $this->retrieveBlackList();
|
||||
$field = array();
|
||||
if (isset($msg["Subject"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"];
|
||||
|
||||
if (isset($msg["From"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"];
|
||||
|
||||
if (isset($msg["Message-ID"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"];
|
||||
|
||||
foreach ($blackList as $blist)
|
||||
{
|
||||
if (preg_match('/^'.$blist['groupname'].'$/i', $groupName))
|
||||
{
|
||||
//blacklist
|
||||
if ($blist['optype'] == 1)
|
||||
{
|
||||
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else if ($blist['optype'] == 2)
|
||||
{
|
||||
if (!preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rawsearch. Perform a simple like match on binary subjects matching a pattern.
|
||||
*/
|
||||
public function search($search, $limit=1000, $excludedcats=array())
|
||||
{
|
||||
|
||||
|
||||
//
|
||||
// if the query starts with a ^ it indicates the search is looking for items which start with the term
|
||||
// still do the like match, but mandate that all items returned must start with the provided word
|
||||
//
|
||||
$words = explode(" ", $search);
|
||||
$searchsql = "";
|
||||
$intwordcount = 0;
|
||||
if (count($words) > 0)
|
||||
{
|
||||
foreach ($words as $word)
|
||||
{
|
||||
//
|
||||
// see if the first word had a caret, which indicates search must start with term
|
||||
//
|
||||
if ($intwordcount == 0 && (strpos($word, "^") === 0))
|
||||
$searchsql.= sprintf(" and b.name like %s", $this->db->escapeString(substr($word, 1)."%"));
|
||||
else
|
||||
$searchsql.= sprintf(" and b.name like %s", $this->db->escapeString("%".$word."%"));
|
||||
|
||||
$intwordcount++;
|
||||
}
|
||||
}
|
||||
|
||||
$exccatlist = "";
|
||||
if (count($excludedcats) > 0)
|
||||
$exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") ";
|
||||
|
||||
$res = $this->db->mysqliQuery(sprintf("
|
||||
SELECT b.*,
|
||||
g.name AS group_name,
|
||||
r.guid,
|
||||
(SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum'
|
||||
FROM binaries b
|
||||
INNER JOIN groups g ON g.ID = b.groupID
|
||||
LEFT OUTER JOIN releases r ON r.ID = b.releaseID
|
||||
WHERE 1=1 %s %s order by DATE DESC LIMIT %d ",
|
||||
$searchsql, $exccatlist, $limit));
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all binaries for a release.
|
||||
*/
|
||||
public function getForReleaseId($id)
|
||||
{
|
||||
return $this->db->mysqliQuery(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a binary row.
|
||||
*/
|
||||
public function getById($id)
|
||||
{
|
||||
|
||||
return $this->db->mysqliQueryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of blacklists from database.
|
||||
*/
|
||||
public function getBlacklist($activeonly=true)
|
||||
{
|
||||
|
||||
$where = "";
|
||||
if ($activeonly)
|
||||
$where = " where binaryblacklist.status = 1 ";
|
||||
|
||||
return $this->db->mysqliQuery("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex,
|
||||
groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist
|
||||
left outer JOIN groups ON groups.name = binaryblacklist.groupname
|
||||
".$where."
|
||||
ORDER BY coalesce(groupname,'zzz')");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a blacklist row from database.
|
||||
*/
|
||||
public function getBlacklistByID($id)
|
||||
{
|
||||
return $this->db->mysqliQueryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a blacklist row from database.
|
||||
*/
|
||||
public function deleteBlacklist($id)
|
||||
{
|
||||
|
||||
return $this->db->mysqliQuery(sprintf("delete from binaryblacklist where ID = %d", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a blacklist row.
|
||||
*/
|
||||
public function updateBlacklist($regex)
|
||||
{
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $this->db->escapeString($groupname));
|
||||
}
|
||||
|
||||
$this->db->mysqliQuery(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new blacklist row.
|
||||
*/
|
||||
public function addBlacklist($regex)
|
||||
{
|
||||
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $this->db->escapeString($groupname));
|
||||
}
|
||||
|
||||
return $this->db->mysqliQueryInsert(sprintf("insert into binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ",
|
||||
$groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new binary row and its associated parts.
|
||||
*/
|
||||
public function delete($id)
|
||||
{
|
||||
|
||||
$this->db->mysqliQuery(sprintf("delete from parts where binaryID = %d", $id));
|
||||
$this->db->mysqliQuery(sprintf("delete from binaries where ID = %d", $id));
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
<?php
|
||||
require_once(dirname(__FILE__)."/../../../config.php");
|
||||
?>
|
||||
@@ -1,18 +0,0 @@
|
||||
<?php
|
||||
require_once("/var/www/newznab/www/config.php");
|
||||
require_once("/var/www/newznab/www/lib/framework/db.php");
|
||||
|
||||
$db = new DB();
|
||||
|
||||
echo "This script will convert only your parts and binaries table to InnoDB with compressed row format..\n";
|
||||
echo "This may take a while depending on how large your tables are. Do not stop this script. It will finish.\n\n";
|
||||
|
||||
echo "Converting the binaries table to InnoDB with compressed row format.\n";
|
||||
$db->query("ALTER TABLE binaries ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;");
|
||||
echo "Finished converting the binaries table to InnoDB with compressed row format..\n\n";
|
||||
|
||||
echo "Converting the parts table to InnoDB with compressed row format.\n";
|
||||
$db->query("ALTER TABLE parts ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;");
|
||||
echo "Finished converting the parts table to InnoDB with compressed row format..\n\n";
|
||||
|
||||
?>
|
||||
@@ -1,312 +0,0 @@
|
||||
<?php
|
||||
require_once("innodb/binaries.php");
|
||||
require_once("framework/db.php");
|
||||
require_once(WWW_DIR."/lib/Net_NNTP/NNTP/Client.php");
|
||||
|
||||
/**
|
||||
* This class extends the standard PEAR NNTP class with some extra features.
|
||||
*/
|
||||
class Nntp extends Net_NNTP_Client
|
||||
{
|
||||
/**
|
||||
* Start an NNTP connection.
|
||||
*/
|
||||
function doConnect()
|
||||
{
|
||||
$enc = false;
|
||||
if (defined("NNTP_SSLENABLED") && NNTP_SSLENABLED == true)
|
||||
$enc = 'ssl';
|
||||
|
||||
$ret = $this->connect(NNTP_SERVER, $enc, NNTP_PORT);
|
||||
if(PEAR::isError($ret))
|
||||
{
|
||||
echo "Cannot connect to server ".NNTP_SERVER.(!$enc?" (nonssl) ":"(ssl) ").": ".$ret->getMessage();
|
||||
die();
|
||||
}
|
||||
if(!defined(NNTP_USERNAME) && NNTP_USERNAME!="" )
|
||||
{
|
||||
$ret2 = $this->authenticate(NNTP_USERNAME, NNTP_PASSWORD);
|
||||
if(PEAR::isError($ret2))
|
||||
{
|
||||
echo "Cannot authenticate to server ".NNTP_SERVER.(!$enc?" (nonssl) ":" (ssl) ")." - ".NNTP_USERNAME." (".$ret2->getMessage().")";
|
||||
die();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* End an NNTP connection.
|
||||
*/
|
||||
function doQuit()
|
||||
{
|
||||
$this->quit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve an NNTP message and decode it.
|
||||
*/
|
||||
function getMessage($groupname, $partMsgId)
|
||||
{
|
||||
$summary = $this->selectGroup($groupname);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$body = $this->getBody('<'.$partMsgId.'>', true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
//echo 'NntpPrc : Error fetching part number '.$partMsgId.' in '.$groupname.' (Server response: '. $body->getMessage().')\n';
|
||||
return false;
|
||||
}
|
||||
|
||||
$message = $this->decodeYenc($body);
|
||||
if (!$message)
|
||||
{
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
echo $message . "\n";
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a series of NNTP messages and decode them.
|
||||
*/
|
||||
function getMessages($groupname, $msgIds)
|
||||
{
|
||||
$summary = $this->selectGroup($groupname);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach($msgIds as $msgId)
|
||||
{
|
||||
$messageID = '<'.$msgId.'>';
|
||||
$body = $this->getBody($messageID, true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
//echo 'NntpPrc : Error fetching '.$messageID.' in '.$groupname.' (Server response: '. $body->getMessage().')';
|
||||
return false;
|
||||
}
|
||||
|
||||
$dec = $this->decodeYenc($body);
|
||||
if (!$dec)
|
||||
{
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
$message .= $dec;
|
||||
}
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve all NNTP messages associated with a binaries.ID
|
||||
*/
|
||||
function getBinary($binaryId, $isNfo=false)
|
||||
{
|
||||
$db = new DB();
|
||||
$bin = new Binaries();
|
||||
|
||||
$binary = $bin->getById($binaryId);
|
||||
if (!$binary)
|
||||
return false;
|
||||
|
||||
$summary = $this->selectGroup($binary['groupname']);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId));
|
||||
|
||||
//
|
||||
// Dont attempt to download nfos which are larger than one part.
|
||||
//
|
||||
if (sizeof($resparts) > 1 && $isNfo === true)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach($resparts as $part)
|
||||
{
|
||||
$messageID = '<'.$part['messageID'].'>';
|
||||
$body = $this->getBody($messageID, true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
//echo 'NntpPrc : Error fetching part number '.$part['messageID'].' in '.$binary['groupname'].' (Server response: '. $body->getMessage().')';
|
||||
return false;
|
||||
}
|
||||
|
||||
$dec = $this->decodeYenc($body);
|
||||
if (!$dec)
|
||||
{
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
$message .= $dec;
|
||||
}
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get XZVER for a range of NNTP messages.
|
||||
*/
|
||||
function getXOverview($range, $_names = true, $_forceNames = true)
|
||||
{
|
||||
// Fetch overview from server
|
||||
$overview = $this->cmdXZver($range);
|
||||
if (PEAR::isError($overview)) {
|
||||
return $overview;
|
||||
}
|
||||
|
||||
// Use field names from overview format as keys?
|
||||
if ($_names)
|
||||
{
|
||||
// Already cached?
|
||||
if (is_null($this->_overviewFormatCache)) {
|
||||
// Fetch overview format
|
||||
$format = $this->getOverviewFormat($_forceNames, true);
|
||||
if (PEAR::isError($format)){
|
||||
return $format;
|
||||
}
|
||||
|
||||
// Prepend 'Number' field
|
||||
$format = array_merge(array('Number' => false), $format);
|
||||
|
||||
// Cache format
|
||||
$this->_overviewFormatCache = $format;
|
||||
}
|
||||
else
|
||||
{
|
||||
$format = $this->_overviewFormatCache;
|
||||
}
|
||||
|
||||
// Loop through all articles
|
||||
foreach ($overview as $key => $article)
|
||||
{
|
||||
if (sizeof($format) == sizeof($article))
|
||||
{
|
||||
//Replace overview using $format as keys, $article as values
|
||||
$overview[$key] = array_combine(array_keys($format), $article);
|
||||
|
||||
// If article prefixed by field name, remove it
|
||||
foreach($format as $fkey=>$fval)
|
||||
{
|
||||
if ($fval === true)
|
||||
{
|
||||
$overview[$key][$fkey] = trim(str_replace($fkey.':', '', $overview[$key][$fkey]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch (true)
|
||||
{
|
||||
// Expect one article
|
||||
case is_null($range);
|
||||
case is_int($range);
|
||||
case is_string($range) && ctype_digit($range):
|
||||
case is_string($range) && substr($range, 0, 1) == '<' && substr($range, -1, 1) == '>':
|
||||
if (count($overview) == 0) {
|
||||
return false;
|
||||
} else {
|
||||
return reset($overview);
|
||||
}
|
||||
break;
|
||||
|
||||
// Expect multiple articles
|
||||
default:
|
||||
return $overview;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send XZVER command over NNTP connection.
|
||||
*/
|
||||
function cmdXZver($range = NULL)
|
||||
{
|
||||
if (is_null($range))
|
||||
$command = 'XZVER';
|
||||
else
|
||||
$command = 'XZVER ' . $range;
|
||||
|
||||
$response = $this->_sendCommand($command);
|
||||
|
||||
switch ($response) {
|
||||
case 224: // 224, RFC2980: 'Overview information follows'
|
||||
$data = $this->_getTextResponse();
|
||||
|
||||
//de-yenc
|
||||
$dec = $this->decodeYenc(implode("\r\n", $data));
|
||||
if (!$dec)
|
||||
{
|
||||
$this->throwError("yenc decode failure");
|
||||
}
|
||||
|
||||
//inflate deflated string
|
||||
$data = explode("\r\n", gzinflate($dec));
|
||||
|
||||
foreach ($data as $key => $value)
|
||||
$data[$key] = explode("\t", ltrim($value));
|
||||
|
||||
return $data;
|
||||
break;
|
||||
case 412: // 412, RFC2980: 'No news group current selected'
|
||||
$this->throwError("No news group current selected ({$this->_currentStatusResponse()})", $response);
|
||||
break;
|
||||
case 420: // 420, RFC2980: 'No article(s) selected'
|
||||
$this->throwError("No article(s) selected ({$this->_currentStatusResponse()})", $response);
|
||||
break;
|
||||
case 502: // 502 RFC2980: 'no permission'
|
||||
$this->throwError("No permission ({$this->_currentStatusResponse()})", $response);
|
||||
break;
|
||||
case 500: // 500 RFC2980: 'unknown command'
|
||||
$this->throwError("XZver not supported ({$this->_currentStatusResponse()})", $response);
|
||||
break;
|
||||
default:
|
||||
return $this->_handleUnexpectedResponse($response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a yenc encoded string.
|
||||
*/
|
||||
function decodeYenc($yencodedvar)
|
||||
{
|
||||
$input = array();
|
||||
preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $yencodedvar, $input);
|
||||
if (isset($input[1]))
|
||||
{
|
||||
$ret = "";
|
||||
$input = trim(preg_replace("/\r\n/im", "", preg_replace("/(^=yend.*)/im", "", preg_replace("/(^=ypart.*\\r\\n)/im", "", preg_replace("/(^=ybegin.*\\r\\n)/im", "", $input[1], 1), 1), 1)));
|
||||
|
||||
for( $chr = 0; $chr < strlen($input) ; $chr++)
|
||||
$ret .= ($input[$chr] != "=" ? chr(ord($input[$chr]) - 42) : chr((ord($input[++$chr]) - 64) - 42));
|
||||
|
||||
return $ret;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
<?php
|
||||
declare(ticks = 1);
|
||||
|
||||
/**
|
||||
* Class that wraps PHP POSIX and PCNTL functions to easily implement
|
||||
* process forking and pseudo-threading
|
||||
*
|
||||
* @author Don Bauer <lordgnu@me.com>
|
||||
* @link https://github.com/lordgnu/PowerSpawn
|
||||
* @version 1.0
|
||||
*/
|
||||
class PowerSpawn
|
||||
{
|
||||
private $myChildren;
|
||||
private $parentPID;
|
||||
private $shutdownCallback = null;
|
||||
private $killCallback = null;
|
||||
|
||||
public $maxChildren = 10; // Max number of children allowed to Spawn
|
||||
public $timeLimit = 0; // Time limit in seconds (0 to disable)
|
||||
public $sleepCount = 1; // Number of seconds to sleep on Tick()
|
||||
|
||||
public $childData; // Variable for storage of data to be passed to the next spawned child
|
||||
public $complete;
|
||||
|
||||
public function __construct() {
|
||||
if (function_exists('pcntl_fork') && function_exists('posix_getpid')) {
|
||||
// Everything is good
|
||||
$this->parentPID = $this->myPID();
|
||||
$this->myChildren = array();
|
||||
$this->complete = false;
|
||||
|
||||
// Install the signal handler
|
||||
pcntl_signal(SIGCHLD, array($this, 'sigHandler'));
|
||||
} else {
|
||||
die("You must have POSIX and PCNTL functions to use PowerSpawn\n");
|
||||
}
|
||||
}
|
||||
|
||||
public function __destruct() {
|
||||
|
||||
}
|
||||
|
||||
public function sigHandler($signo) {
|
||||
switch ($signo) {
|
||||
case SIGCHLD:
|
||||
$this->checkChildren();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public function checkChildren() {
|
||||
foreach ($this->myChildren as $i => $child) {
|
||||
// Check for time running and if still running
|
||||
if ($this->pidDead($child['pid']) != 0) {
|
||||
// Child is dead
|
||||
unset($this->myChildren[$i]);
|
||||
} elseif ($this->timeLimit > 0) {
|
||||
// Check the time limit
|
||||
if (time() - $child['time'] >= $this->timeLimit) {
|
||||
// Child had exceeded time limit
|
||||
$this->killChild($child['pid']);
|
||||
unset($this->myChildren[$i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function myPID() {
|
||||
return posix_getpid();
|
||||
}
|
||||
|
||||
public function myParent() {
|
||||
return posix_getppid();
|
||||
}
|
||||
|
||||
public function spawnChild() {
|
||||
$time = time();
|
||||
$pid = pcntl_fork();
|
||||
if ($pid) $this->myChildren[] = array('time'=>$time,'pid'=>$pid);
|
||||
}
|
||||
|
||||
public function killChild($pid = 0) {
|
||||
if ($pid > 0) {
|
||||
posix_kill($pid, SIGTERM);
|
||||
if ($this->killCallback !== null) call_user_func($this->killCallback);
|
||||
}
|
||||
}
|
||||
|
||||
public function parentCheck() {
|
||||
if ($this->myPID() == $this->parentPID) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function pidDead($pid = 0) {
|
||||
if ($pid > 0) {
|
||||
return pcntl_waitpid($pid, $status, WUNTRACED OR WNOHANG);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public function setCallback($callback = null) {
|
||||
$this->shutdownCallback = $callback;
|
||||
}
|
||||
|
||||
public function setKillCallback($callback = null) {
|
||||
$this->killCallback = $callback;
|
||||
}
|
||||
|
||||
public function childCount() {
|
||||
return count($this->myChildren);
|
||||
}
|
||||
|
||||
public function runParentCode() {
|
||||
if (!$this->complete) {
|
||||
return $this->parentCheck();
|
||||
} else {
|
||||
if ($this->shutdownCallback !== null)
|
||||
call_user_func($this->shutdownCallback);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function runChildCode() {
|
||||
return !$this->parentCheck();
|
||||
}
|
||||
|
||||
public function spawnReady() {
|
||||
if (count($this->myChildren) < $this->maxChildren) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function shutdown() {
|
||||
while($this->childCount()) {
|
||||
$this->checkChildren();
|
||||
$this->tick();
|
||||
}
|
||||
$this->complete = true;
|
||||
}
|
||||
|
||||
public function tick() {
|
||||
sleep($this->sleepCount);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
https://github.com/lordgnu/PowerSpawn for usage examples
|
||||
*/
|
||||
@@ -1,146 +0,0 @@
|
||||
<?php
|
||||
require_once("framework/db.php");
|
||||
|
||||
class Sites
|
||||
{
|
||||
const REGISTER_STATUS_OPEN = 0;
|
||||
const REGISTER_STATUS_INVITE = 1;
|
||||
const REGISTER_STATUS_CLOSED = 2;
|
||||
|
||||
const ERR_BADUNRARPATH = -1;
|
||||
const ERR_BADFFMPEGPATH = -2;
|
||||
const ERR_BADMEDIAINFOPATH = -3;
|
||||
const ERR_BADNZBPATH = -4;
|
||||
const ERR_DEEPNOUNRAR = -5;
|
||||
const ERR_BADTMPUNRARPATH = -6;
|
||||
const ERR_BADLAMEPATH = -7;
|
||||
const ERR_SABCOMPLETEPATH = -8;
|
||||
|
||||
public function version()
|
||||
{
|
||||
return "0.2.3p";
|
||||
}
|
||||
|
||||
public function update($form)
|
||||
{
|
||||
$db = new DB();
|
||||
$site = $this->row2Object($form);
|
||||
|
||||
if (substr($site->nzbpath, strlen($site->nzbpath) - 1) != '/')
|
||||
$site->nzbpath = $site->nzbpath."/";
|
||||
|
||||
//
|
||||
// Validate site settings
|
||||
//
|
||||
if ($site->mediainfopath != "" && !is_file($site->mediainfopath))
|
||||
return Sites::ERR_BADMEDIAINFOPATH;
|
||||
|
||||
if ($site->ffmpegpath != "" && !is_file($site->ffmpegpath))
|
||||
return Sites::ERR_BADFFMPEGPATH;
|
||||
|
||||
if ($site->unrarpath != "" && !is_file($site->unrarpath))
|
||||
return Sites::ERR_BADUNRARPATH;
|
||||
|
||||
if ($site->nzbpath != "" && !file_exists($site->nzbpath))
|
||||
return Sites::ERR_BADNZBPATH;
|
||||
|
||||
if ($site->checkpasswordedrar == 2 && !is_file($site->unrarpath))
|
||||
return Sites::ERR_DEEPNOUNRAR;
|
||||
|
||||
if ($site->tmpunrarpath != "" && !file_exists($site->tmpunrarpath))
|
||||
return Sites::ERR_BADTMPUNRARPATH;
|
||||
|
||||
if ($site->lamepath != "" && !file_exists($site->lamepath))
|
||||
return Sites::ERR_BADLAMEPATH;
|
||||
|
||||
if ($site->sabcompletedir != "" && !file_exists($site->sabcompletedir))
|
||||
return Sites::ERR_SABCOMPLETEPATH;
|
||||
|
||||
$sql = $sqlKeys = array();
|
||||
foreach($form as $settingK=>$settingV)
|
||||
{
|
||||
$sql[] = sprintf("WHEN %s THEN %s", $db->escapeString($settingK), $db->escapeString(trim($settingV)));
|
||||
$sqlKeys[] = $db->escapeString($settingK);
|
||||
}
|
||||
|
||||
$db->query(sprintf("UPDATE site SET value = CASE setting %s END WHERE setting IN (%s)", implode(' ', $sql), implode(', ', $sqlKeys)));
|
||||
|
||||
return $site;
|
||||
}
|
||||
|
||||
public function get()
|
||||
{
|
||||
$db = new DB();
|
||||
$rows = $db->query("select * from site");
|
||||
|
||||
if ($rows === false)
|
||||
return false;
|
||||
|
||||
return $this->rows2Object($rows);
|
||||
}
|
||||
|
||||
public function rows2Object($rows)
|
||||
{
|
||||
$obj = new stdClass;
|
||||
foreach($rows as $row)
|
||||
$obj->{$row['setting']} = $row['value'];
|
||||
|
||||
$obj->{'version'} = $this->version();
|
||||
return $obj;
|
||||
}
|
||||
|
||||
public function row2Object($row)
|
||||
{
|
||||
$obj = new stdClass;
|
||||
$rowKeys = array_keys($row);
|
||||
foreach($rowKeys as $key)
|
||||
$obj->{$key} = $row[$key];
|
||||
|
||||
return $obj;
|
||||
}
|
||||
|
||||
public function getUnappliedPatches($site)
|
||||
{
|
||||
preg_match("/\d+/", $site->dbversion, $matches);
|
||||
$currentrev = $matches[0];
|
||||
|
||||
$patchpath = WWW_DIR."../db/patch/0.2.3/";
|
||||
$patchfiles = glob($patchpath."*.sql");
|
||||
$missingpatch = array();
|
||||
foreach($patchfiles as $file)
|
||||
{
|
||||
$filecontents = file_get_contents($file);
|
||||
if (preg_match("/Rev\: (\d+)/", $filecontents, $matches))
|
||||
{
|
||||
$patchrev = $matches[1];
|
||||
if ($patchrev > $currentrev)
|
||||
$missingpatch[] = $file;
|
||||
}
|
||||
}
|
||||
return $missingpatch;
|
||||
}
|
||||
|
||||
public function updateItem($setting, $value)
|
||||
{
|
||||
$db = new DB();
|
||||
$sql = sprintf("update site set value = %s where setting = %s", $db->escapeString($value), $db->escapeString($setting));
|
||||
return $db->query($sql);
|
||||
}
|
||||
|
||||
public function updateLatestRegexRevision($rev)
|
||||
{
|
||||
return $this->updateItem("latestregexrevision", $rev);
|
||||
}
|
||||
|
||||
public function getLicense($html=false)
|
||||
{
|
||||
$n = "\r\n";
|
||||
if ($html)
|
||||
$n = "<br/>";
|
||||
|
||||
return $n."newznab ".$this->version()." Copyright (C) ".date("Y")." newznab.com".$n."
|
||||
|
||||
This program is distributed with a commercial licence. See LICENCE.txt for
|
||||
further details.".$n;
|
||||
}
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
<?php
|
||||
|
||||
require_once("config.php");
|
||||
require_once("lib/framework/db.php");
|
||||
|
||||
$db = new DB();
|
||||
|
||||
$retval = "";
|
||||
|
||||
if ($argc == 1) {
|
||||
echo "no arguments specified - php nzb-import.php /path/to/nzb bool_use_filenames\n";
|
||||
return;
|
||||
}
|
||||
|
||||
$filestoprocess = Array();
|
||||
$browserpostednames = Array();
|
||||
|
||||
|
||||
$strTerminator = "\n";
|
||||
$path = $argv[1];
|
||||
$usenzbname = (isset($argv[2]) && $argv[2] == 'true') ? true : false;
|
||||
|
||||
if (substr($path, strlen($path) - 1) != '/')
|
||||
$path = $path . "/";
|
||||
|
||||
$groups = $db->query("SELECT ID, name FROM groups");
|
||||
foreach ($groups as $group)
|
||||
$siteGroups[$group["name"]] = $group["ID"];
|
||||
|
||||
if (!isset($groups) || count($groups) == 0) {
|
||||
echo "no groups available in the database, add first.\n";
|
||||
} else {
|
||||
$nzbCount = 0;
|
||||
|
||||
|
||||
|
||||
if (count($filestoprocess) == 0)
|
||||
$filestoprocess = glob($path . "*.nzb");
|
||||
$start = date('Y-m-d H:i:s');
|
||||
|
||||
foreach ($filestoprocess as $nzbFile) {
|
||||
$importfailed = false;
|
||||
$transaction = "";
|
||||
$nzb = file_get_contents($nzbFile);
|
||||
|
||||
$xml = @simplexml_load_string($nzb);
|
||||
if (!$xml || strtolower($xml->getName()) != 'nzb') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$i = 0;
|
||||
foreach ($xml->file as $file) {
|
||||
//file info
|
||||
$groupID = -1;
|
||||
$name = (string)$file->attributes()->subject;
|
||||
$fromname = (string)$file->attributes()->poster;
|
||||
$unixdate = (string)$file->attributes()->date;
|
||||
$date = date("Y-m-d H:i:s", (string)$file->attributes()->date);
|
||||
|
||||
//groups
|
||||
$groupArr = array();
|
||||
foreach ($file->groups->group as $group) {
|
||||
$group = (string)$group;
|
||||
if (array_key_exists($group, $siteGroups)) {
|
||||
$groupID = $siteGroups[$group];
|
||||
}
|
||||
$groupArr[] = $group;
|
||||
}
|
||||
|
||||
if ($groupID != -1) {
|
||||
$xref = implode(': ', $groupArr) . ':';
|
||||
|
||||
$totalParts = sizeof($file->segments->segment);
|
||||
|
||||
//insert binary
|
||||
$binaryHash = md5($name . $fromname . $groupID);
|
||||
|
||||
|
||||
if ($usenzbname) { //Use the binary names
|
||||
$usename = str_replace('.nzb', '', basename($nzbFile));
|
||||
|
||||
$binarySql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded, importname, relname, relpart, reltotalpart, procstat, categoryID, regexID, reqID) values (%s, %s, %s, %s, %s, %s, %s, NOW(), %s, replace(%s, '_', ' '), %d, %d, %d, %s, %d, %s);",
|
||||
$db->escapeString($name), $db->escapeString($fromname), $db->escapeString($date),
|
||||
$db->escapeString($xref), $db->escapeString($totalParts), $db->escapeString($groupID), $db->escapeString($binaryHash), $db->escapeString($nzbFile),
|
||||
$db->escapeString($usename), 1, 1, 5, "null", "null", "null");
|
||||
|
||||
$transaction = $transaction . $binarySql;
|
||||
|
||||
} else { //Don't use the binary names
|
||||
$binarySql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded, importname) values (%s, %s, %s, %s, %s, %s, %s, NOW(), %s);",
|
||||
$db->escapeString($name), $db->escapeString($fromname), $db->escapeString($date),
|
||||
$db->escapeString($xref), $db->escapeString($totalParts), $db->escapeString($groupID), $db->escapeString($binaryHash), $db->escapeString($nzbFile));
|
||||
|
||||
$transaction = $transaction . $binarySql;
|
||||
}
|
||||
|
||||
$sessionVarSQL = "SET @binID = last_insert_id();";
|
||||
$transaction = $transaction . $sessionVarSQL;
|
||||
|
||||
//segments (i.e. parts)
|
||||
if (count($file->segments->segment) > 0) {
|
||||
$partsSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) values ";
|
||||
foreach ($file->segments->segment as $segment) {
|
||||
$messageId = (string)$segment;
|
||||
$partnumber = $segment->attributes()->number;
|
||||
$size = $segment->attributes()->bytes;
|
||||
|
||||
$partsSql .= sprintf("(%s, %s, 0, %s, %s, NOW()),",
|
||||
"@binID", $db->escapeString($messageId), $db->escapeString($partnumber),
|
||||
$db->escapeString($size));
|
||||
}
|
||||
$partsSql = substr($partsSql, 0, -1) . ";";
|
||||
$transaction = $transaction . $partsSql;
|
||||
}
|
||||
|
||||
} else {
|
||||
$importfailed = true;
|
||||
|
||||
echo ("no group found for " . $name . " (one of " . implode(', ', $groupArr) . " are missing)" . $strTerminator);
|
||||
flush();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$importfailed) {
|
||||
$db->multiQueryTransaction($transaction);
|
||||
|
||||
$nzbCount++;
|
||||
@unlink($nzbFile);
|
||||
|
||||
echo ("imported " . $nzbFile . $strTerminator);
|
||||
flush();
|
||||
}
|
||||
if ($nzbCount == 100) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
$seconds = strtotime(date('Y-m-d H:i:s')) - strtotime($start);
|
||||
$retval .= 'Processed ' . $nzbCount . ' nzbs in ' . $seconds . ' second(s)';
|
||||
|
||||
echo $retval;
|
||||
die();
|
||||
@@ -1,24 +0,0 @@
|
||||
<?php
|
||||
|
||||
require("config.php");
|
||||
require_once("lib/groups.php");
|
||||
require_once("lib/innodb/binaries.php");
|
||||
|
||||
if (isset($argv[1]))
|
||||
{
|
||||
$group = $argv[1];
|
||||
echo "Updating group {$group}\n";
|
||||
|
||||
$g = new Groups;
|
||||
$group = $g->getByName($group);
|
||||
|
||||
$bin = new Binaries;
|
||||
$bin->updateGroup(null, $group);
|
||||
}
|
||||
else
|
||||
{
|
||||
$binaries = new Binaries;
|
||||
$binaries->updateAllGroups();
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,77 +0,0 @@
|
||||
<?php
|
||||
require("config.php");
|
||||
require_once(dirname(__FILE__)."/lib/framework/db.php");
|
||||
require_once(dirname(__FILE__)."/lib/groups.php");
|
||||
require_once(dirname(__FILE__)."/lib/innodb/binaries.php");
|
||||
require_once(dirname(__FILE__)."/lib/powerspawn.php");
|
||||
|
||||
$groups = new Groups;
|
||||
$groupList = $groups->getActive();
|
||||
unset($groups);
|
||||
|
||||
$ps = new PowerSpawn;
|
||||
$ps->setCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 10;
|
||||
$ps->timelimit = 0; // Disable child timeout
|
||||
|
||||
echo "Starting threaded binary update process\n";
|
||||
|
||||
while ($ps->runParentCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
if (count($groupList))
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->spawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->childData = array_pop($groupList);
|
||||
echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n";
|
||||
$ps->spawnChild();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
$ps->tick();
|
||||
#echo "- \n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "No more groups to process - Initiating shutdown\n";
|
||||
$ps->shutdown();
|
||||
echo "Shutdown complete\n";
|
||||
}
|
||||
}
|
||||
|
||||
unset($groupList);
|
||||
|
||||
if ($ps->runChildCode())
|
||||
{
|
||||
$group = $ps->childData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->myPID());
|
||||
|
||||
echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n";
|
||||
|
||||
$param = $group['name'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'update_binaries.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded update process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,15 +0,0 @@
|
||||
<?php
|
||||
|
||||
$newzpath = getenv('NEWZPATH');
|
||||
require_once("$newzpath/www/config.php");
|
||||
require_once(WWW_DIR."/lib/backfill.php");
|
||||
|
||||
if (isset($argv[1]))
|
||||
$groupName = $argv[1];
|
||||
else
|
||||
$groupName = '';
|
||||
|
||||
$backfill = new Backfill();
|
||||
$backfill->backfillAllGroups($groupName);
|
||||
|
||||
?>
|
||||
@@ -1,77 +0,0 @@
|
||||
<?php
|
||||
$newzpath = getenv('NEWZPATH');
|
||||
require_once("$newzpath/www/config.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once("powerspawn.php");
|
||||
|
||||
$groups = new Groups;
|
||||
$groupList = $groups->getActive();
|
||||
unset($groups);
|
||||
|
||||
$ps = new PowerSpawn;
|
||||
$ps->setCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 10;
|
||||
$ps->timeLimit = 0; // Disable child timeout
|
||||
|
||||
echo "Starting threaded backfill process\n";
|
||||
|
||||
while ($ps->runParentCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
if (count($groupList))
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->spawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->childData = array_pop($groupList);
|
||||
echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n";
|
||||
$ps->spawnChild();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
$ps->tick();
|
||||
#echo ". \n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "No more groups to process - Initiating shutdown\n";
|
||||
$ps->shutdown();
|
||||
echo "Shutdown complete\n";
|
||||
}
|
||||
}
|
||||
|
||||
unset($groupList);
|
||||
|
||||
if ($ps->runChildCode())
|
||||
{
|
||||
$group = $ps->childData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->myPID());
|
||||
|
||||
echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n";
|
||||
|
||||
$param = $group['name'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'backfill.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded backfill process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,174 +0,0 @@
|
||||
<?php
|
||||
declare(ticks = 1);
|
||||
|
||||
/**
|
||||
* Class that wraps PHP POSIX and PCNTL functions to easily implement
|
||||
* process forking and pseudo-threading
|
||||
*
|
||||
* @author Don Bauer <lordgnu@me.com>
|
||||
* @link https://github.com/lordgnu/PowerSpawn
|
||||
* @version 1.0
|
||||
*/
|
||||
class PowerSpawn
|
||||
{
|
||||
private $myChildren;
|
||||
private $parentPID;
|
||||
private $shutdownCallback = null;
|
||||
private $killCallback = null;
|
||||
|
||||
public $maxChildren = 10; // Max number of children allowed to Spawn
|
||||
public $timeLimit = 0; // Time limit in seconds (0 to disable)
|
||||
public $sleepCount = 1; // Number of seconds to sleep on Tick()
|
||||
|
||||
public $childData; // Variable for storage of data to be passed to the next spawned child
|
||||
public $complete;
|
||||
|
||||
public function __construct() {
|
||||
if (function_exists('pcntl_fork') && function_exists('posix_getpid')) {
|
||||
// Everything is good
|
||||
$this->parentPID = $this->myPID();
|
||||
$this->myChildren = array();
|
||||
$this->complete = false;
|
||||
|
||||
// Install the signal handler
|
||||
pcntl_signal(SIGCHLD, array($this, 'sigHandler'));
|
||||
} else {
|
||||
die("You must have POSIX and PCNTL functions to use PowerSpawn\n");
|
||||
}
|
||||
}
|
||||
|
||||
public function __destruct() {
|
||||
|
||||
}
|
||||
|
||||
public function sigHandler($signo) {
|
||||
switch ($signo) {
|
||||
case SIGCHLD:
|
||||
$this->checkChildren();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public function checkChildren() {
|
||||
foreach ($this->myChildren as $i => $child) {
|
||||
// Check for time running and if still running
|
||||
if ($this->pidDead($child['pid']) != 0) {
|
||||
// Child is dead
|
||||
unset($this->myChildren[$i]);
|
||||
} elseif ($this->timeLimit > 0) {
|
||||
// Check the time limit
|
||||
if (time() - $child['time'] >= $this->timeLimit) {
|
||||
// Child had exceeded time limit
|
||||
$this->killChild($child['pid']);
|
||||
unset($this->myChildren[$i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function myPID() {
|
||||
return posix_getpid();
|
||||
}
|
||||
|
||||
public function myParent() {
|
||||
return posix_getppid();
|
||||
}
|
||||
|
||||
public function spawnChild() {
|
||||
$time = time();
|
||||
$pid = pcntl_fork();
|
||||
if ($pid) $this->myChildren[] = array('time'=>$time,'pid'=>$pid);
|
||||
}
|
||||
|
||||
public function killChild($pid = 0) {
|
||||
if ($pid > 0) {
|
||||
posix_kill($pid, SIGTERM);
|
||||
if ($this->killCallback !== null) call_user_func($this->killCallback);
|
||||
}
|
||||
}
|
||||
|
||||
public function parentCheck() {
|
||||
if ($this->myPID() == $this->parentPID) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function pidDead($pid = 0) {
|
||||
if ($pid > 0) {
|
||||
return pcntl_waitpid($pid, $status, WUNTRACED OR WNOHANG);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public function setCallback($callback = null) {
|
||||
$this->shutdownCallback = $callback;
|
||||
}
|
||||
|
||||
public function setKillCallback($callback = null) {
|
||||
$this->killCallback = $callback;
|
||||
}
|
||||
|
||||
public function childCount() {
|
||||
return count($this->myChildren);
|
||||
}
|
||||
|
||||
public function runParentCode() {
|
||||
if (!$this->complete) {
|
||||
return $this->parentCheck();
|
||||
} else {
|
||||
if ($this->shutdownCallback !== null)
|
||||
call_user_func($this->shutdownCallback);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function runChildCode() {
|
||||
return !$this->parentCheck();
|
||||
}
|
||||
|
||||
public function spawnReady() {
|
||||
if (count($this->myChildren) < $this->maxChildren) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function shutdown() {
|
||||
while($this->childCount()) {
|
||||
$this->checkChildren();
|
||||
$this->tick();
|
||||
}
|
||||
$this->complete = true;
|
||||
}
|
||||
|
||||
public function tick() {
|
||||
sleep($this->sleepCount);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
https://github.com/lordgnu/PowerSpawn for usage examples
|
||||
*/
|
||||
@@ -1,25 +0,0 @@
|
||||
<?php
|
||||
|
||||
$newzpath = getenv('NEWZPATH');
|
||||
require_once("$newzpath/www/config.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
|
||||
if (isset($argv[1]))
|
||||
{
|
||||
$group = $argv[1];
|
||||
echo "Updating group {$group}\n";
|
||||
|
||||
$g = new Groups;
|
||||
$group = $g->getByName($group);
|
||||
|
||||
$bin = new Binaries;
|
||||
$bin->updateGroup(null, $group);
|
||||
}
|
||||
else
|
||||
{
|
||||
$binaries = new Binaries;
|
||||
$binaries->updateAllGroups();
|
||||
}
|
||||
|
||||
?>
|
||||
@@ -1,77 +0,0 @@
|
||||
<?php
|
||||
$newzpath = getenv('NEWZPATH');
|
||||
require("$newzpath/www/config.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once("powerspawn.php");
|
||||
|
||||
$groups = new Groups;
|
||||
$groupList = $groups->getActive();
|
||||
unset($groups);
|
||||
|
||||
$ps = new PowerSpawn;
|
||||
$ps->setCallback('psUpdateComplete');
|
||||
$ps->maxChildren = 10;
|
||||
$ps->timelimit = 0; // Disable child timeout
|
||||
|
||||
echo "Starting threaded binary update process\n";
|
||||
|
||||
while ($ps->runParentCode())
|
||||
{
|
||||
// Start the parent loop
|
||||
if (count($groupList))
|
||||
{
|
||||
// We still have groups to process
|
||||
if ($ps->spawnReady())
|
||||
{
|
||||
// Spawn another thread
|
||||
$ps->childData = array_pop($groupList);
|
||||
echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n";
|
||||
$ps->spawnChild();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There are no more slots available to run
|
||||
$ps->tick();
|
||||
#echo "- \n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// No more groups to process
|
||||
echo "No more groups to process - Initiating shutdown\n";
|
||||
$ps->shutdown();
|
||||
echo "Shutdown complete\n";
|
||||
}
|
||||
}
|
||||
|
||||
unset($groupList);
|
||||
|
||||
if ($ps->runChildCode())
|
||||
{
|
||||
$group = $ps->childData;
|
||||
|
||||
$thread = sprintf("%05d",$ps->myPID());
|
||||
|
||||
echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n";
|
||||
|
||||
$param = $group['name'];
|
||||
|
||||
$dir = dirname(__FILE__);
|
||||
$file = 'update_binaries.php';
|
||||
|
||||
$output = shell_exec("php {$dir}/{$file} {$param}");
|
||||
|
||||
echo "[Thread-{$thread}] Completed update for group {$group['name']}\n";
|
||||
}
|
||||
|
||||
// Exit to call back to parent - Let know that child has completed
|
||||
exit(0);
|
||||
|
||||
// Create callback function
|
||||
function psUpdateComplete()
|
||||
{
|
||||
echo "[Thread-MASTER] Threaded update process complete\n";
|
||||
}
|
||||
|
||||
?>
|
||||
+4
-8
@@ -18,20 +18,19 @@ if [ "$THREADS" == "true" -a "$INNODB" == "true" ]; then
|
||||
|
||||
#make active groups current
|
||||
if [[ $BINARIES == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
cd $NEWZNAB_PATH
|
||||
[ -f update_binaries_threaded.php ] && $PHP update_binaries_threaded.php
|
||||
fi
|
||||
|
||||
#get backfill for all active groups
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
cd $NEWZNAB_PATH
|
||||
[ -f backfill_threaded.php ] && $PHP backfill_threaded.php
|
||||
fi
|
||||
|
||||
wait
|
||||
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
#increment backfill days
|
||||
$MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}"
|
||||
fi
|
||||
@@ -65,7 +64,6 @@ elif [ "$THREADS" != "true" -a "$INNODB" == "true" ]; then
|
||||
wait
|
||||
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
#increment backfill days
|
||||
$MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}"
|
||||
fi
|
||||
@@ -87,20 +85,19 @@ elif [ "$THREADS" == "true" -a "$INNODB" != "true" ]; then
|
||||
|
||||
#make active groups current
|
||||
if [[ $BINARIES == "true" ]] ; then
|
||||
cd $MYISAM_PATH
|
||||
cd $NEWZNAB_PATH
|
||||
[ -f update_binaries_threaded.php ] && $PHP update_binaries_threaded.php
|
||||
fi
|
||||
|
||||
#get backfill for all active groups
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $MYISAM_PATH
|
||||
cd $NEWZNAB_PATH
|
||||
[ -f backfill_threaded.php ] && $PHP backfill_threaded.php
|
||||
fi
|
||||
|
||||
wait
|
||||
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
#increment backfill days
|
||||
$MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}"
|
||||
fi
|
||||
@@ -135,7 +132,6 @@ elif [ "$THREADS" != "true" -a "$INNODB" != "true" ]; then
|
||||
wait
|
||||
|
||||
if [[ $BACKFILL == "true" ]] ; then
|
||||
cd $INNODB_PATH
|
||||
#increment backfill days
|
||||
$MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}"
|
||||
fi
|
||||
|
||||
+10
-9
@@ -7,6 +7,7 @@ export NEWZPATH="/var/www/newznab"
|
||||
export NEWZNAB_PATH=$NEWZPATH"/misc/update_scripts"
|
||||
export TESTING_PATH=$NEWZPATH"/misc/testing"
|
||||
export ADMIN_PATH=$NEWZPATH"/www/admin"
|
||||
export INNODB_PATH=$TESTING_PATH"/innodb"
|
||||
export USERNAME="what is your name" # this is the user name that will run these scripts
|
||||
export NEWZNAB_IMPORT_SLEEP_TIME="1" # in seconds - this includes import_nzb backfill and current fill, 0 may cause errors
|
||||
export NEWZNAB_POST_SLEEP_TIME="1" # in seconds - this is for post processing - sleep between loops, 0 may cause errors
|
||||
@@ -14,16 +15,16 @@ export MAXDAYS="210" #max days for backfill
|
||||
export NZBS="/path/to/nzbs" #The path to the nzb dump you downloaded from torrents
|
||||
|
||||
#Choose to run the threaded or non-threaded newznab scripts true/false
|
||||
export THREADS="true"
|
||||
export THREADS="false"
|
||||
|
||||
#Choose your database, comment the one true/false
|
||||
export INNODB="true"
|
||||
export INNODB="false"
|
||||
|
||||
#Choose to run update_cleanup.php true/false
|
||||
export CLEANUP="true"
|
||||
export CLEANUP="false"
|
||||
|
||||
#Choose to run update_binaries true/false
|
||||
export BINARIES="true"
|
||||
export BINARIES="false"
|
||||
|
||||
#Choose to run backfill script true/false
|
||||
export BACKFILL="true"
|
||||
@@ -32,11 +33,11 @@ export BACKFILL="true"
|
||||
export IMPORT="true"
|
||||
|
||||
#Select some monitoring script, if they are not installed, it will not affect the running of the scripts
|
||||
export USE_HTOP="true"
|
||||
export USE_NMON="true"
|
||||
export USE_BWMNG="true"
|
||||
export USE_IOTOP="true"
|
||||
export USE_MYTOP="true"
|
||||
export USE_HTOP="false"
|
||||
export USE_NMON="false"
|
||||
export USE_BWMNG="false"
|
||||
export USE_IOTOP="false"
|
||||
export USE_MYTOP="false"
|
||||
|
||||
#By using this script you understand that the programmer is not responsible for any loss of data, users, or sanity.
|
||||
#You also agree that you were smart enough to make a backup of your database and files. Do you agree? yes/no
|
||||
|
||||
@@ -18,10 +18,6 @@ if [[ $AGREED == "no" ]]; then
|
||||
exit
|
||||
fi
|
||||
|
||||
export INNODB_PATH=$DIR"/bin/innodb"
|
||||
export MYISAM_PATH=$DIR"/bin/myisam"
|
||||
export START_PATH=$DIR
|
||||
|
||||
#delete some files
|
||||
[ -f bin/lib/postprocess4.php ];rm bin/lib/postprocess4.php
|
||||
[ -f bin/processAlternate4.php ];rm bin/processAlternate4.php
|
||||
|
||||
Reference in New Issue
Block a user