diff --git a/README.md b/README.md index 58b68fcf0..282e86f48 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SETUP - * These scripts were written and tested on Ubuntu 12.10 where bash is located at /bin/bash. You may need to create a symlink or edit these script accordingly. + * These scripts were written and tested on Ubuntu 12.10 where bash is located at /bin/bash. You may need to create a symlink or edit these scripts accordingly. * Please backup your database first. Something like this should do it. @@ -11,16 +11,16 @@ * If you decide to convert your database, I recommend using [kevin123's github](https://github.com/kevinlekiller/Newznab-Barracuda.git). I recommend only converting the binaries a parts table, using compressed tables. But, there are many choices. I suggest you read his README and follow his recommendations. Or, simply: - `cd /var/www/newznab/misc/testing/` + `cd /var/www/newznab/misc/testing` - `git clone https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git innodb` + `git clone https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git kev-innodb` - `cd innodb/lib/innodb` + `cd innodb/lib/kev-innodb` `php convertToInnoDB.php` - * If, you have already converted your database, and didn't to the steps above. You will need to clone [kevin123's github](https://github.com/kevinlekiller/Newznab-InnoDB-Dropin.git) and get the scripts. + * Now, you will need to clone [andrewit's github](https://github.com/itandrew/Newznab-InnoDB-Dropin.git) and get his scripts. `cd /var/www/newznab/misc/testing/` @@ -33,10 +33,6 @@ `git clone https://github.com/jonnyboy/newznab-tmux.git tmux` - -or- - - `git clone https://github.com/kevinlekiller/newznab-tmux tmux` - `cd tmux` `nano edit_these.sh` @@ -56,7 +52,7 @@ * If something looks stalled, it probably isn't. If all 13 panes are still there, it is most likely, as it should be. - * update_cleanup needs to be uncommented to actually do something, and update_parsing is good for fixing a few releases everytime it runs, not a silver bullet though + * **misc/testing/update_cleanup.php** needs to be edited to actually do something, and update_parsing is good for fixing a few releases everytime it runs, not a silver bullet though * If you are running this on an OVH or kimsufi server, you may need to run sudo ./start.sh because they built grsecurity into the kernel. @@ -64,6 +60,8 @@ * Join in the converstion at irc://moonlight.se.eu.synirc.net/newznab-tmux. + * The development branch of this git is still under heavy development. + * Thanks go to all who offered their assistance and improvement to these scripts.
diff --git a/bin/innodb/backfill.php b/bin/innodb/backfill.php deleted file mode 100755 index 3d1b0c72f..000000000 --- a/bin/innodb/backfill.php +++ /dev/null @@ -1,14 +0,0 @@ -backfillAllGroups($groupName); - -?> diff --git a/bin/innodb/backfill_threaded.php b/bin/innodb/backfill_threaded.php deleted file mode 100755 index 820b69e01..000000000 --- a/bin/innodb/backfill_threaded.php +++ /dev/null @@ -1,76 +0,0 @@ -getActive(); -unset($groups); - -$ps = new PowerSpawn; -$ps->setCallback('psUpdateComplete'); -$ps->maxChildren = 10; -$ps->timeLimit = 0; // Disable child timeout - -echo "Starting threaded backfill process\n"; - -while ($ps->runParentCode()) -{ - // Start the parent loop - if (count($groupList)) - { - // We still have groups to process - if ($ps->spawnReady()) - { - // Spawn another thread - $ps->childData = array_pop($groupList); - echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; - $ps->spawnChild(); - } - else - { - // There are no more slots available to run - $ps->tick(); - #echo ". \n"; - } - } - else - { - // No more groups to process - echo "No more groups to process - Initiating shutdown\n"; - $ps->shutdown(); - echo "Shutdown complete\n"; - } -} - -unset($groupList); - -if ($ps->runChildCode()) -{ - $group = $ps->childData; - - $thread = sprintf("%05d",$ps->myPID()); - - echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n"; - - $param = $group['name']; - - $dir = dirname(__FILE__); - $file = 'backfill.php'; - - $output = shell_exec("php {$dir}/{$file} {$param}"); - - echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; -} - -// Exit to call back to parent - Let know that child has completed -exit(0); - -// Create callback function -function psUpdateComplete() -{ - echo "[Thread-MASTER] Threaded backfill process complete\n"; -} - -?> diff --git a/bin/innodb/config.php b/bin/innodb/config.php deleted file mode 100755 index 7ec2852f4..000000000 --- a/bin/innodb/config.php +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/bin/innodb/lib/backfill.php b/bin/innodb/lib/backfill.php deleted file mode 100755 index 6e34c2d7c..000000000 --- a/bin/innodb/lib/backfill.php +++ /dev/null @@ -1,300 +0,0 @@ -db = new DB(); - } - else - { - $this->db = $db; - } - $this->n = "\n"; - } - - /** - * Update all active groups categories and descriptions. - */ - function backfillAllGroups($groupName='', $backfillDate=null) - { - $n = $this->n; - $groups = new Groups; - $res = false; - if ($groupName != '') - { - $grp = $groups->getByName($groupName); - if ($grp) - $res = array($grp); - } - else - { - $res = $groups->getActive(); - } - - if ($res) - { - $nntp = new Nntp(); - $nntp->doConnect(); - - foreach($res as $groupArr) - { - $this->backfillGroup($nntp, $groupArr, $backfillDate); - } - - $nntp->doQuit(); - } - else - { - echo "No groups specified. Ensure groups are added to newznab's database for updating.$n"; - } - } - - /** - * Update a group back to a specified date. - */ - function backfillGroup($nntp, $groupArr, $backfillDate=null) - { - $this->db->disableAutoCommit(); //Turn off auto committing - - $binaries = new Binaries($this->db); - - $n = $this->n; - $this->startGroup = microtime(true); - - echo 'Processing '.$groupArr['name'].$n; - - $data = $nntp->selectGroup($groupArr['name']); - if(PEAR::isError($data)) - { - echo "Could not select group (bad name?): {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return; - } - if ($backfillDate) - $targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date - else - $targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target - if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0) - { - echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return; - } - - echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~"; - echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400)); - echo " days.".$n."Local first = ".$groupArr['first_record']." ("; - echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400)); - echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n"; - - if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish - { - echo "Nothing to do, we already have the target post.$n $n"; - - $this->db->commit(); //Not an error so commit and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - //get first and last part numbers from newsgroup - if($targetpost < $data['first']) - { - echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n"; - echo "Skipping Group $n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - //calculate total number of parts - $total = $groupArr['first_record'] - $targetpost; - $done = false; - //set first and last, moving the window by maxxMssgs - $last = $groupArr['first_record'] - 1; - $first = $last - $binaries->messagebuffer + 1; //set initial "chunk" - if($targetpost > $first) //just in case this is the last chunk we needed - $first = $targetpost; - while($done === false) - { - $binaries->startLoop = microtime(true); - echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n; - flush(); - $success = $binaries->scan($nntp, $groupArr, $first, $last, 'backfill'); - if (!$success) - { - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID'])); - - $this->db->commit(); //At this point we are ready to commit a set up updates to the db. - - if($first==$targetpost) - $done = true; - else - { //Keep going: set new last, new first, check for last chunk. - $last = $first - 1; - $first = $last - $binaries->messagebuffer + 1; - if($targetpost > $first) - $first = $targetpost; - } - } - $first_record_postdate = $this->postdate($nntp,$first,false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate - - $this->db->commit(); //Commit and re-enable auto committing we are done with this group. - $this->db->enableAutoCommit(); - - $timeGroup = number_format(microtime(true) - $this->startGroup, 2); - echo "Group processed in $timeGroup seconds $n"; - } - - /** - * Returns single timestamp from a local article number. - */ - function postdate($nntp,$post,$debug=true) - { - $n = $this->n; - $attempts=0; - do - { - $msgs = $nntp->getOverview($post."-".$post,true,false); - if(PEAR::isError($msgs)) - { - echo "Error {$msgs->code}: {$msgs->message}$n"; - echo "Returning from postdate$n"; - return ""; - } - - if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date'])) - { - $success=false; - } else { - $date = $msgs[0]['Date']; - $success=true; - } - if($debug && $attempts > 0) echo "retried $attempts time(s)".$n; - $attempts++; - } while($attempts <= 3 && $success == false); - - if (!$success) { return ""; } - - if($debug) echo "DEBUG: postdate for post: $post came back $date ("; - $date = strtotime($date); - if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n; - return $date; - } - - /** - * Calculates the post number for a given number of days back in a group. - */ - function daytopost($nntp, $group, $days, $debug=true) - { - $n = $this->n; - $pddebug = false; //DEBUG every postdate call?!?! - if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n; - - $data = $nntp->selectGroup($group); - if(PEAR::isError($data)) - { - echo "Error {$data->code}: {$data->message}$n"; - echo "Returning from daytopost$n"; - return ""; - } - $goaldate = date('U')-(86400*$days); //goaltimestamp - $totalnumberofarticles = $data['last'] - $data['first']; - $upperbound = $data['last']; - $lowerbound = $data['first']; - - if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n"; - if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); } - - $firstDate = $this->postdate($nntp, $data['first'], $pddebug); - $lastDate = $this->postdate($nntp, $data['last'], $pddebug); - if ($goaldate < $firstDate) - { - echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n"; - echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n"; - return $data['first']; - } - elseif ($goaldate > $lastDate) - { - echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n"; - echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n"; - return ""; - } - if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n"; - - $interval = floor(($upperbound - $lowerbound) * 0.5); - $dateofnextone = ""; - $templowered = ""; - - if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n"; - - $dateofnextone = $lastDate; - - while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up - { - $nskip = 1; - while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate) - { - $upperbound = $upperbound - $interval - ($nskip - 1); - if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n"; - $nskip = $nskip * 2; - } - if(!$templowered) - { - $interval = ceil(($interval/2)); - if($debug) echo "Set interval to $interval articles. $n"; - } - $dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug); - - $skip = 1; - while(!$dateofnextone) - { - $upperbound = $upperbound - $skip; - $skip = $skip * 2; - if($debug) echo "Getting next article date... $upperbound\n"; - $dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug); - } - } - echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n"; - return $upperbound; - } - - /** - * Calculate the number of days from a timestamp. - */ - private function daysOld($timestamp) - { - return round((time()-$timestamp)/86400, 1); - } - - /** - * Calculate the number of days from a date. - */ - private function dateToDays($backfillDate) - { - return floor(-($backfillDate - time())/(60*60*24)); - } -} diff --git a/bin/innodb/lib/binaries.php b/bin/innodb/lib/binaries.php deleted file mode 100755 index a7b8b2236..000000000 --- a/bin/innodb/lib/binaries.php +++ /dev/null @@ -1,742 +0,0 @@ -n = "\n"; - - $s = new Sites(); - $site = $s->get(); - $this->compressedHeaders = ($site->compressedheaders == "1") ? true : false; - $this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000; - $this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false; - $this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000; - $this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3; - - $this->blackList = array(); //cache of our black/white list - $this->message = array(); - - if($db == null) - { - $this->db = new DB(); - } - else - { - $this->db = $db; - } - - } - - /* - * Allows you to set the db that the current object should use - */ - public function setDB($db) - { - $this->db = $db; - } - - /** - * Process headers and store in database for all active groups. - */ - function updateAllGroups() - { - $n = $this->n; - $groups = new Groups; - $res = $groups->getActive(); - - $s = new Sites(); - echo $s->getLicense(); - - if ($res) - { - shuffle($res); - $alltime = microtime(true); - echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n; - - $nntp = new Nntp(); - $nntp->doConnect(); - - $pos = 0; - foreach($res as $groupArr) - { - $pos++; - echo 'Group '.$pos.' of '.sizeof($res).$n; - $this->message = array(); - $this->updateGroup($nntp, $groupArr); - } - - $nntp->doQuit(); - echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n; - } - else - { - echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n"; - } - } - - /** - * Process headers and store in database for a group. - */ - function updateGroup($nntp=null, $groupArr) - { - $this->db->disableAutoCommit(); - - $blnDoDisconnect = false; - if ($nntp == null) - { - $nntp = new Nntp(); - $nntp->doConnect(); - $this->message = array(); - $blnDoDisconnect = true; - } - - $backfill = new Backfill(); - $n = $this->n; - $this->startGroup = microtime(true); - - echo 'Processing '.$groupArr['name'].$n; - - // Connect to server - $data = $nntp->selectGroup($groupArr['name']); - if (PEAR::isError($data)) - { - echo "Could not select group (bad name?): {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - - //Attempt to repair any missing parts before grabbing new ones - $this->partRepair($nntp, $groupArr); - - //Get first and last part numbers from newsgroup - $last = $grouplast = $data['last']; - - // For new newsgroups - determine here how far you want to go back. - if ($groupArr['last_record'] == 0) - { - if ($this->NewGroupScanByDays) - { - $first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true); - if ($first == '') - { - echo "Skipping group: {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - } - else - { - if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan)) - $first = $data['first']; - else - $first = $data['last'] - $this->NewGroupMsgsToScan; - } - $first_record_postdate = $backfill->postdate($nntp, $first, false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID'])); - } - else - { - if ($data['last'] < $groupArr['last_record']) - { - echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - $first = $groupArr['last_record'] + 1; - } - - // Generate postdates for first and last records, for those that upgraded - if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID'])); - - // Deactivate empty groups - if (($data['last'] - $data['first']) <= 5) - $this->db->mysqliQuery(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString('0'), $groupArr['ID'])); - - // Calculate total number of parts - $total = $grouplast - $first + 1; - - // If total is bigger than 0 it means we have new parts in the newsgroup - if($total > 0) - { - echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n; - echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n; - if ($groupArr['last_record'] == 0) - echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n; - - $done = false; - - // Get all the parts (in portions of $this->messagebuffer to not use too much memory) - while ($done === false) - { - $this->startLoop = microtime(true); - - if ($total > $this->messagebuffer) - { - if ($first + $this->messagebuffer > $grouplast) - $last = $grouplast; - else - $last = $first + $this->messagebuffer; - } - - echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n; - flush(); - - //get headers from newsgroup - $lastId = $this->scan($nntp, $groupArr, $first, $last); - if ($lastId === false) - { - //scan failed - skip group - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - $this->db->mysqliQuery(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($lastId), $groupArr['ID'])); - - $this->db->commit(); //At this point we are ready to commit a whole group to the db. - - if ($last == $grouplast) - $done = true; - else - { - $last = $lastId; - $first = $last + 1; - } - } - - $last_record_postdate = $backfill->postdate($nntp,$last,false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate - $timeGroup = number_format(microtime(true) - $this->startGroup, 2); - echo "Group processed in $timeGroup seconds $n $n"; - } - else - { - echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n; - } - - if ($blnDoDisconnect) - { - $nntp->doQuit(); - } - - /* - * Got through the updating of this group successfully - * so commit and re-enable the auto committing. - */ - $this->db->commit(); //For anything not yet committed. - $this->db->enableAutoCommit(); - - } - - /** - * Download a range of usenet messages. Store binaries with subjects matching a - * specific pattern in the database. - */ - function scan($nntp, $groupArr, $first, $last, $type='update') - { - $n = $this->n; - $this->startHeaders = microtime(true); - - if ($this->compressedHeaders) - $msgs = $nntp->getXOverview($first."-".$last, true, false); - else - $msgs = $nntp->getOverview($first."-".$last, true, false); - - if (PEAR::isError($msgs) && $msgs->code == 400) - { - echo "NNTP connection timed out. Reconnecting...$n"; - $nntp->doConnect(); - $nntp->selectGroup($groupArr['name']); - if ($this->compressedHeaders) - $msgs = $nntp->getXOverview($first."-".$last, true, false); - else - $msgs = $nntp->getOverview($first."-".$last, true, false); - } - - $rangerequested = range($first, $last); - $msgsreceived = array(); - $msgsblacklisted = array(); - $msgsignored = array(); - $msgsinserted = array(); - $msgsnotinserted = array(); - - $timeHeaders = number_format(microtime(true) - $this->startHeaders, 2); - - if(PEAR::isError($msgs)) - { - echo "Error {$msgs->code}: {$msgs->message}$n"; - echo "Skipping group$n"; - return false; - } - - $this->startUpdate = microtime(true); - if (is_array($msgs)) - { - //loop headers, figure out parts - foreach($msgs AS $msg) - { - if (!isset($msg['Number'])) - continue; - - $msgsreceived[] = $msg['Number']; - $msgPart = $msgTotalParts = 0; - - $pattern = '|\((\d+)[\/](\d+)\)|i'; - preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); - $matchcnt = sizeof($matches[0]); - for ($i=0; $i<$matchcnt; $i++) - { - $msgPart = $matches[1][$i]; - $msgTotalParts = $matches[2][$i]; - } - - if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue - { - $msgsignored[] = $msg['Number']; - continue; - } - - //Filter binaries based on black/white list - if ($this->isBlackListed($msg, $groupArr['name'])) - { - $msgsblacklisted[] = $msg['Number']; - continue; - } - - if((int)$msgPart > 0 && (int)$msgTotalParts > 0) - { - $subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject']))); - - if(!isset($this->message[$subject])) - { - $this->message[$subject] = $msg; - $this->message[$subject]['MaxParts'] = (int) $msgTotalParts; - $this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']); - } - if((int)$msgPart > 0) - { - $this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']); - } - } - } - unset($msg); - unset($msgs); - $count = 0; - $updatecount = 0; - $partcount = 0; - $maxnum = $last; - - $rangenotreceived = array_diff($rangerequested, $msgsreceived); - - - if ($type != 'partrepair') - echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n"; - - if ($type == 'update' && sizeof($msgsreceived) == 0) - { - echo "Error: Server did not return any articles.$n"; - echo "Skipping group$n"; - return false; - } - - if (sizeof($rangenotreceived) > 0) { - switch($type) - { - case 'backfill': - //don't add missing articles - break; - case 'partrepair': - case 'update': - default: - $this->addMissingParts($rangenotreceived, $groupArr['ID']); - break; - } - echo "Server did not return ".count($rangenotreceived)." article(s).$n"; - } - - if(isset($this->message) && count($this->message)) - { - $maxnum = $first; - //insert binaries and parts into database. when binary already exists; only insert new parts - foreach($this->message AS $subject => $data) - { - if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') - { - $binaryHash = md5($subject.$data['From'].$groupArr['ID']); - $res = $this->db->mysqliQueryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $this->db->escapeString($binaryHash))); - if(!$res) - { - $sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $this->db->escapeString($subject), $this->db->escapeString($data['From']), $this->db->escapeString($data['Date']), $this->db->escapeString($data['Xref']), $this->db->escapeString($data['MaxParts']), $groupArr['ID'], $this->db->escapeString($binaryHash)); - $binaryID = $this->db->mysqliQueryInsert($sql); - $count++; - if ($count%500==0) echo "$count bin adds..."; - } - else - { - $binaryID = $res["ID"]; - $updatecount++; - if ($updatecount%500==0) echo "$updatecount bin updates..."; - } - - foreach($data['Parts'] AS $partdata) - { - $maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum; - $partcount++; - $pidata = $this->db->mysqliQueryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $this->db->escapeString($partdata['Message-ID']), $this->db->escapeString($partdata['number']), $this->db->escapeString(round($partdata['part'])), $this->db->escapeString($partdata['size'])), false); - if (!$pidata) { - $msgsnotinserted[] = $partdata['number']; - } else { - $msgsinserted[] = $partdata['number']; - } - } - } - } - //TODO: determine whether to add to missing articles if insert failed - if (sizeof($msgsnotinserted) > 0) - { - echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n; - $this->addMissingParts($msgsnotinserted, $groupArr['ID']); - } - if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output - } - $timeUpdate = number_format(microtime(true) - $this->startUpdate, 2); - $timeLoop = number_format(microtime(true)-$this->startLoop, 2); - - if ($type != 'partrepair') - { - echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.'; - echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n"; - } - unset($this->message); - unset($data); - return $maxnum; - } - else - { - echo "Error: Can't get parts from server (msgs not array) $n"; - echo "Skipping group$n"; - return false; - } - } - - /** - * Go through all rows in partrepair table and see if theyve arrived on usenet yet. - */ - private function partRepair($nntp, $groupArr) - { - $n = $this->n; - - //get all parts in partrepair table - - $missingParts = $this->db->mysqliQuery(sprintf("SELECT * FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 30000", $groupArr['ID'])); - $partsRepaired = $partsFailed = 0; - - if (sizeof($missingParts) > 0) - { - echo 'Attempting to repair '.sizeof($missingParts).' parts...'; - - //loop through each part to group into ranges - $ranges = array(); - $lastnum = $lastpart = 0; - foreach($missingParts as $part) - { - if (($lastnum+1) == $part['numberID']) { - $ranges[$lastpart] = $part['numberID']; - } else { - $lastpart = $part['numberID']; - $ranges[$lastpart] = $part['numberID']; - } - $lastnum = $part['numberID']; - } - - //download missing parts in ranges - foreach($ranges as $partfrom=>$partto) - { - $this->startLoop = microtime(true); - - echo "."; - - //get article from newsgroup - $this->scan($nntp, $groupArr, $partfrom, $partto, 'partrepair'); - - //check if the articles were added - $articles = implode(',', range($partfrom, $partto)); - $sql = sprintf("SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", $groupArr['ID'], $articles); - - $result = $this->db->mysqliQueryDirect($sql); - while ($r = mysqli_fetch_assoc($result)) - { - if (isset($r['number']) && $r['number'] == $r['numberID']) - { - $partsRepaired++; - - //article was added, delete from partrepair - $this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE ID=%d", $r['ID'])); - } - else - { - $partsFailed++; - - //article was not added, increment attempts - $this->db->mysqliQuery(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d", $r['ID'])); - } - } - } - - echo $n.$partsRepaired.' parts repaired.'.$n; - } - - //remove articles that we cant fetch after 5 attempts - $this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $groupArr['ID'])); - - } - - /** - * Insert a missing part to the database. - */ - private function addMissingParts($numbers, $groupID) - { - - $added = false; - $insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES "; - foreach($numbers as $number) - { - if ($number > 0) - { - $added = true; - $insertStr .= sprintf("(%u, %d), ", $number, $groupID); - } - } - if ($added) - { - $insertStr = substr($insertStr, 0, -2); - $insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1"; - return $this->db->mysqliQueryInsert($insertStr, false); - } - - return -1; - } - - /** - * Return internally cached list of binary blacklist patterns. - */ - public function retrieveBlackList() - { - if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; } - $blackList = $this->getBlacklist(true); - $this->blackList = $blackList; - return $blackList; - } - - /** - * Test if a message subject is blacklisted. - */ - public function isBlackListed($msg, $groupName) - { - $blackList = $this->retrieveBlackList(); - $field = array(); - if (isset($msg["Subject"])) - $field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"]; - - if (isset($msg["From"])) - $field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"]; - - if (isset($msg["Message-ID"])) - $field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"]; - - foreach ($blackList as $blist) - { - if (preg_match('/^'.$blist['groupname'].'$/i', $groupName)) - { - //blacklist - if ($blist['optype'] == 1) - { - if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) { - return true; - } - } - else if ($blist['optype'] == 2) - { - if (!preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) { - return true; - } - } - } - } - - return false; - } - - /** - * Rawsearch. Perform a simple like match on binary subjects matching a pattern. - */ - public function search($search, $limit=1000, $excludedcats=array()) - { - - - // - // if the query starts with a ^ it indicates the search is looking for items which start with the term - // still do the like match, but mandate that all items returned must start with the provided word - // - $words = explode(" ", $search); - $searchsql = ""; - $intwordcount = 0; - if (count($words) > 0) - { - foreach ($words as $word) - { - // - // see if the first word had a caret, which indicates search must start with term - // - if ($intwordcount == 0 && (strpos($word, "^") === 0)) - $searchsql.= sprintf(" and b.name like %s", $this->db->escapeString(substr($word, 1)."%")); - else - $searchsql.= sprintf(" and b.name like %s", $this->db->escapeString("%".$word."%")); - - $intwordcount++; - } - } - - $exccatlist = ""; - if (count($excludedcats) > 0) - $exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") "; - - $res = $this->db->mysqliQuery(sprintf(" - SELECT b.*, - g.name AS group_name, - r.guid, - (SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum' - FROM binaries b - INNER JOIN groups g ON g.ID = b.groupID - LEFT OUTER JOIN releases r ON r.ID = b.releaseID - WHERE 1=1 %s %s order by DATE DESC LIMIT %d ", - $searchsql, $exccatlist, $limit)); - - return $res; - } - - /** - * Get all binaries for a release. - */ - public function getForReleaseId($id) - { - return $this->db->mysqliQuery(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id)); - } - - /** - * Get a binary row. - */ - public function getById($id) - { - - return $this->db->mysqliQueryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id)); - } - - /** - * Get list of blacklists from database. - */ - public function getBlacklist($activeonly=true) - { - - $where = ""; - if ($activeonly) - $where = " where binaryblacklist.status = 1 "; - - return $this->db->mysqliQuery("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex, - groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist - left outer JOIN groups ON groups.name = binaryblacklist.groupname - ".$where." - ORDER BY coalesce(groupname,'zzz')"); - } - - /** - * Get a blacklist row from database. - */ - public function getBlacklistByID($id) - { - return $this->db->mysqliQueryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id)); - } - - /** - * Delete a blacklist row from database. - */ - public function deleteBlacklist($id) - { - - return $this->db->mysqliQuery(sprintf("delete from binaryblacklist where ID = %d", $id)); - } - - /** - * Update a blacklist row. - */ - public function updateBlacklist($regex) - { - - $groupname = $regex["groupname"]; - if ($groupname == "") - $groupname = "null"; - else - { - $groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname); - $groupname = sprintf("%s", $this->db->escapeString($groupname)); - } - - $this->db->mysqliQuery(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"])); - } - - /** - * Add a new blacklist row. - */ - public function addBlacklist($regex) - { - - - $groupname = $regex["groupname"]; - if ($groupname == "") - $groupname = "null"; - else - { - $groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname); - $groupname = sprintf("%s", $this->db->escapeString($groupname)); - } - - return $this->db->mysqliQueryInsert(sprintf("insert into binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ", - $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"])); - } - - /** - * Add a new binary row and its associated parts. - */ - public function delete($id) - { - - $this->db->mysqliQuery(sprintf("delete from parts where binaryID = %d", $id)); - $this->db->mysqliQuery(sprintf("delete from binaries where ID = %d", $id)); - } -} diff --git a/bin/innodb/lib/framework/db.php b/bin/innodb/lib/framework/db.php deleted file mode 100755 index e85e2bcc0..000000000 --- a/bin/innodb/lib/framework/db.php +++ /dev/null @@ -1,175 +0,0 @@ -mysqli = new mysqli(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME); - if (mysqli_connect_errno()) { - printf("Connect failed: %s\n", mysqli_connect_error()); - exit(); - } - } - - public function escapeString($str) - { - return "'".mysql_real_escape_string($str)."'"; - } - - public function makeLookupTable($rows, $keycol) - { - $arr = array(); - foreach($rows as $row) - $arr[$row[$keycol]] = $row; - return $arr; - } - - public function queryInsert($query, $returnlastid=true) - { - $result = mysql_query($query); - return ($returnlastid) ? mysql_insert_id() : $result; - } - - public function queryOneRow($query) - { - $rows = $this->query($query); - - if (!$rows) - return false; - - if ($rows) - return $rows[0]; - else - return $rows; - } - - public function query($query) - { - $result = mysql_query($query); - - if ($result === false || $result === true) - return array(); - - $rows = array(); - - while ($row = mysql_fetch_assoc($result)) - $rows[] = $row; - - mysql_free_result($result); - return $rows; - } - - public function queryDirect($query) - { - $result = mysql_query($query); - return $result; - } - - - public function optimise() - { - $ret = array(); - $alltables = $this->query("SHOW TABLES"); - - foreach ($alltables as $tablename) - { - $ret[] = $tablename['Tables_in_'.DB_NAME]; - $this->queryDirect("REPAIR TABLE `".$tablename['Tables_in_'.DB_NAME]."`"); - $this->queryDirect("OPTIMIZE TABLE `".$tablename['Tables_in_'.DB_NAME]."`"); - } - - return $ret; - } - - /* Mysqli Functions */ - public function multiQueryTransaction($query) - { - $this->mysqli->autocommit(FALSE); - $this->mysqli->multi_query($query); - - while($this->mysqli->next_result()) { - $result = $this->mysqli->use_result(); - if($result instanceof mysqli_result) - $result->free(); - } - - $this->mysqli->commit(); - } - - public function disableAutoCommit() - { - $this->mysqli->autocommit(FALSE); - } - - public function mysqliQuery($sql) - { - $result = $this->mysqli->query($sql); - if ($result === false || $result === true) - return array(); - - $rows = array(); - - while ($row = mysqli_fetch_assoc($result)) - $rows[] = $row; - - mysqli_free_result($result); - return $rows; - } - - public function mysqliQueryOneRow($query) - { - $rows = $this->mysqliQuery($query); - - if (!$rows) - return false; - - if ($rows) - return $rows[0]; - else - return $rows; - } - - public function mysqliQueryInsert($query, $returnlastid=true) - { - $result = $this->mysqli->query($query); - return ($returnlastid) ? $this->mysqli->insert_id : $result; - } - - public function commit() - { - $this->mysqli->commit(); - } - - public function rollback() - { - $this->mysqli->rollback(); - } - - public function mysqliQueryDirect($query) - { - return $this->mysqli->query($query); - } - - - public function enableAutoCommit() - { - $this->mysqli->autocommit(TRUE); - } -} -?> diff --git a/bin/innodb/lib/groups.php b/bin/innodb/lib/groups.php deleted file mode 100755 index 571f5b5bb..000000000 --- a/bin/innodb/lib/groups.php +++ /dev/null @@ -1,27 +0,0 @@ -query("SELECT * FROM groups WHERE active = 1 ORDER BY name"); - } - - /** - * Get a group row by name. - */ - public function getByName($grp) - { - $db = new DB(); - return $db->queryOneRow(sprintf("select * from groups where name = '%s' ", $grp)); - } - -} diff --git a/bin/innodb/lib/innodb/backfill.php b/bin/innodb/lib/innodb/backfill.php deleted file mode 100755 index b91a5bc0e..000000000 --- a/bin/innodb/lib/innodb/backfill.php +++ /dev/null @@ -1,300 +0,0 @@ -db = new DB(); - } - else - { - $this->db = $db; - } - $this->n = "\n"; - } - - /** - * Update all active groups categories and descriptions. - */ - function backfillAllGroups($groupName='', $backfillDate=null) - { - $n = $this->n; - $groups = new Groups; - $res = false; - if ($groupName != '') - { - $grp = $groups->getByName($groupName); - if ($grp) - $res = array($grp); - } - else - { - $res = $groups->getActive(); - } - - if ($res) - { - $nntp = new Nntp(); - $nntp->doConnect(); - - foreach($res as $groupArr) - { - $this->backfillGroup($nntp, $groupArr, $backfillDate); - } - - $nntp->doQuit(); - } - else - { - echo "No groups specified. Ensure groups are added to newznab's database for updating.$n"; - } - } - - /** - * Update a group back to a specified date. - */ - function backfillGroup($nntp, $groupArr, $backfillDate=null) - { - $this->db->disableAutoCommit(); //Turn off auto committing - - $binaries = new Binaries($this->db); - - $n = $this->n; - $this->startGroup = microtime(true); - - echo 'Processing '.$groupArr['name'].$n; - - $data = $nntp->selectGroup($groupArr['name']); - if(PEAR::isError($data)) - { - echo "Could not select group (bad name?): {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return; - } - if ($backfillDate) - $targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date - else - $targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target - if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0) - { - echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return; - } - - echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~"; - echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400)); - echo " days.".$n."Local first = ".$groupArr['first_record']." ("; - echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400)); - echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n"; - - if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish - { - echo "Nothing to do, we already have the target post.$n $n"; - - $this->db->commit(); //Not an error so commit and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - //get first and last part numbers from newsgroup - if($targetpost < $data['first']) - { - echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n"; - echo "Skipping Group $n"; - - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - //calculate total number of parts - $total = $groupArr['first_record'] - $targetpost; - $done = false; - //set first and last, moving the window by maxxMssgs - $last = $groupArr['first_record'] - 1; - $first = $last - $binaries->messagebuffer + 1; //set initial "chunk" - if($targetpost > $first) //just in case this is the last chunk we needed - $first = $targetpost; - while($done === false) - { - $binaries->startLoop = microtime(true); - echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n; - flush(); - $success = $binaries->scan($nntp, $groupArr, $first, $last, 'backfill'); - if (!$success) - { - $this->db->rollback(); //Rollback and re-enable autocommitting - $this->db->enableAutoCommit(); - return ""; - } - - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID'])); - - $this->db->commit(); //At this point we are ready to commit a set up updates to the db. - - if($first==$targetpost) - $done = true; - else - { //Keep going: set new last, new first, check for last chunk. - $last = $first - 1; - $first = $last - $binaries->messagebuffer + 1; - if($targetpost > $first) - $first = $targetpost; - } - } - $first_record_postdate = $this->postdate($nntp,$first,false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate - - $this->db->commit(); //Commit and re-enable auto committing we are done with this group. - $this->db->enableAutoCommit(); - - $timeGroup = number_format(microtime(true) - $this->startGroup, 2); - echo "Group processed in $timeGroup seconds $n"; - } - - /** - * Returns single timestamp from a local article number. - */ - function postdate($nntp,$post,$debug=true) - { - $n = $this->n; - $attempts=0; - do - { - $msgs = $nntp->getOverview($post."-".$post,true,false); - if(PEAR::isError($msgs)) - { - echo "Error {$msgs->code}: {$msgs->message}$n"; - echo "Returning from postdate$n"; - return ""; - } - - if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date'])) - { - $success=false; - } else { - $date = $msgs[0]['Date']; - $success=true; - } - if($debug && $attempts > 0) echo "retried $attempts time(s)".$n; - $attempts++; - } while($attempts <= 3 && $success == false); - - if (!$success) { return ""; } - - if($debug) echo "DEBUG: postdate for post: $post came back $date ("; - $date = strtotime($date); - if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n; - return $date; - } - - /** - * Calculates the post number for a given number of days back in a group. - */ - function daytopost($nntp, $group, $days, $debug=true) - { - $n = $this->n; - $pddebug = false; //DEBUG every postdate call?!?! - if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n; - - $data = $nntp->selectGroup($group); - if(PEAR::isError($data)) - { - echo "Error {$data->code}: {$data->message}$n"; - echo "Returning from daytopost$n"; - return ""; - } - $goaldate = date('U')-(86400*$days); //goaltimestamp - $totalnumberofarticles = $data['last'] - $data['first']; - $upperbound = $data['last']; - $lowerbound = $data['first']; - - if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n"; - if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); } - - $firstDate = $this->postdate($nntp, $data['first'], $pddebug); - $lastDate = $this->postdate($nntp, $data['last'], $pddebug); - if ($goaldate < $firstDate) - { - echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n"; - echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n"; - return $data['first']; - } - elseif ($goaldate > $lastDate) - { - echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n"; - echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n"; - return ""; - } - if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n"; - - $interval = floor(($upperbound - $lowerbound) * 0.5); - $dateofnextone = ""; - $templowered = ""; - - if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n"; - - $dateofnextone = $lastDate; - - while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up - { - $nskip = 1; - while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate) - { - $upperbound = $upperbound - $interval - ($nskip - 1); - if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n"; - $nskip = $nskip * 2; - } - if(!$templowered) - { - $interval = ceil(($interval/2)); - if($debug) echo "Set interval to $interval articles. $n"; - } - $dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug); - - $skip = 1; - while(!$dateofnextone) - { - $upperbound = $upperbound - $skip; - $skip = $skip * 2; - if($debug) echo "Getting next article date... $upperbound\n"; - $dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug); - } - } - echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n"; - return $upperbound; - } - - /** - * Calculate the number of days from a timestamp. - */ - private function daysOld($timestamp) - { - return round((time()-$timestamp)/86400, 1); - } - - /** - * Calculate the number of days from a date. - */ - private function dateToDays($backfillDate) - { - return floor(-($backfillDate - time())/(60*60*24)); - } -} diff --git a/bin/innodb/lib/innodb/binaries.php b/bin/innodb/lib/innodb/binaries.php deleted file mode 100755 index 0179b2d02..000000000 --- a/bin/innodb/lib/innodb/binaries.php +++ /dev/null @@ -1,741 +0,0 @@ -n = "\n"; - - $s = new Sites(); - $site = $s->get(); - $this->compressedHeaders = ($site->compressedheaders == "1") ? true : false; - $this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000; - $this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false; - $this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000; - $this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3; - - $this->blackList = array(); //cache of our black/white list - $this->message = array(); - - if($db == null) - { - $this->db = new DB(); - } - else - { - $this->db = $db; - } - - } - - /* - * Allows you to set the db that the current object should use - */ - public function setDB($db) - { - $this->db = $db; - } - - /** - * Process headers and store in database for all active groups. - */ - function updateAllGroups() - { - $n = $this->n; - $groups = new Groups; - $res = $groups->getActive(); - - $s = new Sites(); - echo $s->getLicense(); - - if ($res) - { - shuffle($res); - $alltime = microtime(true); - echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n; - - $nntp = new Nntp(); - $nntp->doConnect(); - - $pos = 0; - foreach($res as $groupArr) - { - $pos++; - echo 'Group '.$pos.' of '.sizeof($res).$n; - $this->message = array(); - $this->updateGroup($nntp, $groupArr); - } - - $nntp->doQuit(); - echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n; - } - else - { - echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n"; - } - } - - /** - * Process headers and store in database for a group. - */ - function updateGroup($nntp=null, $groupArr) - { - $this->db->disableAutoCommit(); - - $blnDoDisconnect = false; - if ($nntp == null) - { - $nntp = new Nntp(); - $nntp->doConnect(); - $this->message = array(); - $blnDoDisconnect = true; - } - - $backfill = new Backfill(); - $n = $this->n; - $this->startGroup = microtime(true); - - echo 'Processing '.$groupArr['name'].$n; - - // Connect to server - $data = $nntp->selectGroup($groupArr['name']); - if (PEAR::isError($data)) - { - echo "Could not select group (bad name?): {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - - //Attempt to repair any missing parts before grabbing new ones - $this->partRepair($nntp, $groupArr); - - //Get first and last part numbers from newsgroup - $last = $grouplast = $data['last']; - - // For new newsgroups - determine here how far you want to go back. - if ($groupArr['last_record'] == 0) - { - if ($this->NewGroupScanByDays) - { - $first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true); - if ($first == '') - { - echo "Skipping group: {$groupArr['name']}$n"; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - } - else - { - if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan)) - $first = $data['first']; - else - $first = $data['last'] - $this->NewGroupMsgsToScan; - } - $first_record_postdate = $backfill->postdate($nntp, $first, false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $this->db->escapeString($first), $groupArr['ID'])); - } - else - { - if ($data['last'] < $groupArr['last_record']) - { - echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n; - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - $first = $groupArr['last_record'] + 1; - } - - // Generate postdates for first and last records, for those that upgraded - if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) - $this->db->mysqliQuery(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID'])); - - // Deactivate empty groups - if (($data['last'] - $data['first']) <= 5) - $this->db->mysqliQuery(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString('0'), $groupArr['ID'])); - - // Calculate total number of parts - $total = $grouplast - $first + 1; - - // If total is bigger than 0 it means we have new parts in the newsgroup - if($total > 0) - { - echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n; - echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n; - if ($groupArr['last_record'] == 0) - echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n; - - $done = false; - - // Get all the parts (in portions of $this->messagebuffer to not use too much memory) - while ($done === false) - { - $this->startLoop = microtime(true); - - if ($total > $this->messagebuffer) - { - if ($first + $this->messagebuffer > $grouplast) - $last = $grouplast; - else - $last = $first + $this->messagebuffer; - } - - echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n; - flush(); - - //get headers from newsgroup - $lastId = $this->scan($nntp, $groupArr, $first, $last); - if ($lastId === false) - { - //scan failed - skip group - - $this->db->rollback(); //Rollback and re-enable auto committing - $this->db->enableAutoCommit(); - return; - } - $this->db->mysqliQuery(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $this->db->escapeString($lastId), $groupArr['ID'])); - - $this->db->commit(); //At this point we are ready to commit a whole group to the db. - - if ($last == $grouplast) - $done = true; - else - { - $last = $lastId; - $first = $last + 1; - } - } - - $last_record_postdate = $backfill->postdate($nntp,$last,false); - $this->db->mysqliQuery(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate - $timeGroup = number_format(microtime(true) - $this->startGroup, 2); - echo "Group processed in $timeGroup seconds $n $n"; - } - else - { - echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n; - } - - if ($blnDoDisconnect) - { - $nntp->doQuit(); - } - - /* - * Got through the updating of this group successfully - * so commit and re-enable the auto committing. - */ - $this->db->commit(); //For anything not yet committed. - $this->db->enableAutoCommit(); - - } - - /** - * Download a range of usenet messages. Store binaries with subjects matching a - * specific pattern in the database. - */ - function scan($nntp, $groupArr, $first, $last, $type='update') - { - $n = $this->n; - $this->startHeaders = microtime(true); - - if ($this->compressedHeaders) - $msgs = $nntp->getXOverview($first."-".$last, true, false); - else - $msgs = $nntp->getOverview($first."-".$last, true, false); - - if (PEAR::isError($msgs) && $msgs->code == 400) - { - echo "NNTP connection timed out. Reconnecting...$n"; - $nntp->doConnect(); - $nntp->selectGroup($groupArr['name']); - if ($this->compressedHeaders) - $msgs = $nntp->getXOverview($first."-".$last, true, false); - else - $msgs = $nntp->getOverview($first."-".$last, true, false); - } - - $rangerequested = range($first, $last); - $msgsreceived = array(); - $msgsblacklisted = array(); - $msgsignored = array(); - $msgsinserted = array(); - $msgsnotinserted = array(); - - $timeHeaders = number_format(microtime(true) - $this->startHeaders, 2); - - if(PEAR::isError($msgs)) - { - echo "Error {$msgs->code}: {$msgs->message}$n"; - echo "Skipping group$n"; - return false; - } - - $this->startUpdate = microtime(true); - if (is_array($msgs)) - { - //loop headers, figure out parts - foreach($msgs AS $msg) - { - if (!isset($msg['Number'])) - continue; - - $msgsreceived[] = $msg['Number']; - $msgPart = $msgTotalParts = 0; - - $pattern = '|\((\d+)[\/](\d+)\)|i'; - preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); - $matchcnt = sizeof($matches[0]); - for ($i=0; $i<$matchcnt; $i++) - { - $msgPart = $matches[1][$i]; - $msgTotalParts = $matches[2][$i]; - } - - if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue - { - $msgsignored[] = $msg['Number']; - continue; - } - - //Filter binaries based on black/white list - if ($this->isBlackListed($msg, $groupArr['name'])) - { - $msgsblacklisted[] = $msg['Number']; - continue; - } - - if((int)$msgPart > 0 && (int)$msgTotalParts > 0) - { - $subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject']))); - - if(!isset($this->message[$subject])) - { - $this->message[$subject] = $msg; - $this->message[$subject]['MaxParts'] = (int) $msgTotalParts; - $this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']); - } - if((int)$msgPart > 0) - { - $this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']); - } - } - } - unset($msg); - unset($msgs); - $count = 0; - $updatecount = 0; - $partcount = 0; - $maxnum = $last; - - $rangenotreceived = array_diff($rangerequested, $msgsreceived); - - - if ($type != 'partrepair') - echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n"; - - if ($type == 'update' && sizeof($msgsreceived) == 0) - { - echo "Error: Server did not return any articles.$n"; - echo "Skipping group$n"; - return false; - } - - if (sizeof($rangenotreceived) > 0) { - switch($type) - { - case 'backfill': - //don't add missing articles - break; - case 'partrepair': - case 'update': - default: - $this->addMissingParts($rangenotreceived, $groupArr['ID']); - break; - } - echo "Server did not return ".count($rangenotreceived)." article(s).$n"; - } - - if(isset($this->message) && count($this->message)) - { - $maxnum = $first; - //insert binaries and parts into database. when binary already exists; only insert new parts - foreach($this->message AS $subject => $data) - { - if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') - { - $binaryHash = md5($subject.$data['From'].$groupArr['ID']); - $res = $this->db->mysqliQueryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $this->db->escapeString($binaryHash))); - if(!$res) - { - $sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $this->db->escapeString($subject), $this->db->escapeString($data['From']), $this->db->escapeString($data['Date']), $this->db->escapeString($data['Xref']), $this->db->escapeString($data['MaxParts']), $groupArr['ID'], $this->db->escapeString($binaryHash)); - $binaryID = $this->db->mysqliQueryInsert($sql); - $count++; - if ($count%500==0) echo "$count bin adds..."; - } - else - { - $binaryID = $res["ID"]; - $updatecount++; - if ($updatecount%500==0) echo "$updatecount bin updates..."; - } - - foreach($data['Parts'] AS $partdata) - { - $maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum; - $partcount++; - $pidata = $this->db->mysqliQueryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $this->db->escapeString($partdata['Message-ID']), $this->db->escapeString($partdata['number']), $this->db->escapeString(round($partdata['part'])), $this->db->escapeString($partdata['size'])), false); - if (!$pidata) { - $msgsnotinserted[] = $partdata['number']; - } else { - $msgsinserted[] = $partdata['number']; - } - } - } - } - //TODO: determine whether to add to missing articles if insert failed - if (sizeof($msgsnotinserted) > 0) - { - echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n; - $this->addMissingParts($msgsnotinserted, $groupArr['ID']); - } - if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output - } - $timeUpdate = number_format(microtime(true) - $this->startUpdate, 2); - $timeLoop = number_format(microtime(true)-$this->startLoop, 2); - - if ($type != 'partrepair') - { - echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.'; - echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n"; - } - unset($this->message); - unset($data); - return $maxnum; - } - else - { - echo "Error: Can't get parts from server (msgs not array) $n"; - echo "Skipping group$n"; - return false; - } - } - - /** - * Go through all rows in partrepair table and see if theyve arrived on usenet yet. - */ - private function partRepair($nntp, $groupArr) - { - $n = $this->n; - - //get all parts in partrepair table - - $missingParts = $this->db->mysqliQuery(sprintf("SELECT * FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC LIMIT 30000", $groupArr['ID'])); - $partsRepaired = $partsFailed = 0; - - if (sizeof($missingParts) > 0) - { - echo 'Attempting to repair '.sizeof($missingParts).' parts...'; - - //loop through each part to group into ranges - $ranges = array(); - $lastnum = $lastpart = 0; - foreach($missingParts as $part) - { - if (($lastnum+1) == $part['numberID']) { - $ranges[$lastpart] = $part['numberID']; - } else { - $lastpart = $part['numberID']; - $ranges[$lastpart] = $part['numberID']; - } - $lastnum = $part['numberID']; - } - - //download missing parts in ranges - foreach($ranges as $partfrom=>$partto) - { - $this->startLoop = microtime(true); - - echo "."; - - //get article from newsgroup - $this->scan($nntp, $groupArr, $partfrom, $partto, 'partrepair'); - - //check if the articles were added - $articles = implode(',', range($partfrom, $partto)); - $sql = sprintf("SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC", $groupArr['ID'], $articles); - - $result = $this->db->mysqliQueryDirect($sql); - while ($r = mysqli_fetch_assoc($result)) - { - if (isset($r['number']) && $r['number'] == $r['numberID']) - { - $partsRepaired++; - - //article was added, delete from partrepair - $this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE ID=%d", $r['ID'])); - } - else - { - $partsFailed++; - - //article was not added, increment attempts - $this->db->mysqliQuery(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d", $r['ID'])); - } - } - } - - echo $n.$partsRepaired.' parts repaired.'.$n; - } - - //remove articles that we cant fetch after 5 attempts - $this->db->mysqliQuery(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $groupArr['ID'])); - - } - - /** - * Insert a missing part to the database. - */ - private function addMissingParts($numbers, $groupID) - { - - $added = false; - $insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES "; - foreach($numbers as $number) - { - if ($number > 0) - { - $added = true; - $insertStr .= sprintf("(%u, %d), ", $number, $groupID); - } - } - if ($added) - { - $insertStr = substr($insertStr, 0, -2); - $insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1"; - return $this->db->mysqliQueryInsert($insertStr, false); - } - - return -1; - } - - /** - * Return internally cached list of binary blacklist patterns. - */ - public function retrieveBlackList() - { - if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; } - $blackList = $this->getBlacklist(true); - $this->blackList = $blackList; - return $blackList; - } - - /** - * Test if a message subject is blacklisted. - */ - public function isBlackListed($msg, $groupName) - { - $blackList = $this->retrieveBlackList(); - $field = array(); - if (isset($msg["Subject"])) - $field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"]; - - if (isset($msg["From"])) - $field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"]; - - if (isset($msg["Message-ID"])) - $field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"]; - - foreach ($blackList as $blist) - { - if (preg_match('/^'.$blist['groupname'].'$/i', $groupName)) - { - //blacklist - if ($blist['optype'] == 1) - { - if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) { - return true; - } - } - else if ($blist['optype'] == 2) - { - if (!preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) { - return true; - } - } - } - } - - return false; - } - - /** - * Rawsearch. Perform a simple like match on binary subjects matching a pattern. - */ - public function search($search, $limit=1000, $excludedcats=array()) - { - - - // - // if the query starts with a ^ it indicates the search is looking for items which start with the term - // still do the like match, but mandate that all items returned must start with the provided word - // - $words = explode(" ", $search); - $searchsql = ""; - $intwordcount = 0; - if (count($words) > 0) - { - foreach ($words as $word) - { - // - // see if the first word had a caret, which indicates search must start with term - // - if ($intwordcount == 0 && (strpos($word, "^") === 0)) - $searchsql.= sprintf(" and b.name like %s", $this->db->escapeString(substr($word, 1)."%")); - else - $searchsql.= sprintf(" and b.name like %s", $this->db->escapeString("%".$word."%")); - - $intwordcount++; - } - } - - $exccatlist = ""; - if (count($excludedcats) > 0) - $exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") "; - - $res = $this->db->mysqliQuery(sprintf(" - SELECT b.*, - g.name AS group_name, - r.guid, - (SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum' - FROM binaries b - INNER JOIN groups g ON g.ID = b.groupID - LEFT OUTER JOIN releases r ON r.ID = b.releaseID - WHERE 1=1 %s %s order by DATE DESC LIMIT %d ", - $searchsql, $exccatlist, $limit)); - - return $res; - } - - /** - * Get all binaries for a release. - */ - public function getForReleaseId($id) - { - return $this->db->mysqliQuery(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id)); - } - - /** - * Get a binary row. - */ - public function getById($id) - { - - return $this->db->mysqliQueryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id)); - } - - /** - * Get list of blacklists from database. - */ - public function getBlacklist($activeonly=true) - { - - $where = ""; - if ($activeonly) - $where = " where binaryblacklist.status = 1 "; - - return $this->db->mysqliQuery("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex, - groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist - left outer JOIN groups ON groups.name = binaryblacklist.groupname - ".$where." - ORDER BY coalesce(groupname,'zzz')"); - } - - /** - * Get a blacklist row from database. - */ - public function getBlacklistByID($id) - { - return $this->db->mysqliQueryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id)); - } - - /** - * Delete a blacklist row from database. - */ - public function deleteBlacklist($id) - { - - return $this->db->mysqliQuery(sprintf("delete from binaryblacklist where ID = %d", $id)); - } - - /** - * Update a blacklist row. - */ - public function updateBlacklist($regex) - { - - $groupname = $regex["groupname"]; - if ($groupname == "") - $groupname = "null"; - else - { - $groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname); - $groupname = sprintf("%s", $this->db->escapeString($groupname)); - } - - $this->db->mysqliQuery(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"])); - } - - /** - * Add a new blacklist row. - */ - public function addBlacklist($regex) - { - - - $groupname = $regex["groupname"]; - if ($groupname == "") - $groupname = "null"; - else - { - $groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname); - $groupname = sprintf("%s", $this->db->escapeString($groupname)); - } - - return $this->db->mysqliQueryInsert(sprintf("insert into binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ", - $groupname, $this->db->escapeString($regex["regex"]), $regex["status"], $this->db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"])); - } - - /** - * Add a new binary row and its associated parts. - */ - public function delete($id) - { - - $this->db->mysqliQuery(sprintf("delete from parts where binaryID = %d", $id)); - $this->db->mysqliQuery(sprintf("delete from binaries where ID = %d", $id)); - } -} diff --git a/bin/innodb/lib/innodb/config.php b/bin/innodb/lib/innodb/config.php deleted file mode 100755 index 8202b16b3..000000000 --- a/bin/innodb/lib/innodb/config.php +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/bin/innodb/lib/innodb/convertToInnoDB.php b/bin/innodb/lib/innodb/convertToInnoDB.php deleted file mode 100755 index aaee5c13d..000000000 --- a/bin/innodb/lib/innodb/convertToInnoDB.php +++ /dev/null @@ -1,18 +0,0 @@ -query("ALTER TABLE binaries ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;"); -echo "Finished converting the binaries table to InnoDB with compressed row format..\n\n"; - -echo "Converting the parts table to InnoDB with compressed row format.\n"; -$db->query("ALTER TABLE parts ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;"); -echo "Finished converting the parts table to InnoDB with compressed row format..\n\n"; - -?> diff --git a/bin/innodb/lib/nntp.php b/bin/innodb/lib/nntp.php deleted file mode 100755 index 5d068cdb7..000000000 --- a/bin/innodb/lib/nntp.php +++ /dev/null @@ -1,312 +0,0 @@ -connect(NNTP_SERVER, $enc, NNTP_PORT); - if(PEAR::isError($ret)) - { - echo "Cannot connect to server ".NNTP_SERVER.(!$enc?" (nonssl) ":"(ssl) ").": ".$ret->getMessage(); - die(); - } - if(!defined(NNTP_USERNAME) && NNTP_USERNAME!="" ) - { - $ret2 = $this->authenticate(NNTP_USERNAME, NNTP_PASSWORD); - if(PEAR::isError($ret2)) - { - echo "Cannot authenticate to server ".NNTP_SERVER.(!$enc?" (nonssl) ":" (ssl) ")." - ".NNTP_USERNAME." (".$ret2->getMessage().")"; - die(); - } - } - } - - /** - * End an NNTP connection. - */ - function doQuit() - { - $this->quit(); - } - - /** - * Retrieve an NNTP message and decode it. - */ - function getMessage($groupname, $partMsgId) - { - $summary = $this->selectGroup($groupname); - $message = $dec = ''; - - if (PEAR::isError($summary)) - { - echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n"; - return false; - } - - $body = $this->getBody('<'.$partMsgId.'>', true); - if (PEAR::isError($body)) - { - //echo 'NntpPrc : Error fetching part number '.$partMsgId.' in '.$groupname.' (Server response: '. $body->getMessage().')\n'; - return false; - } - - $message = $this->decodeYenc($body); - if (!$message) - { - // - // Yenc decode failed - // - return false; - } - - echo $message . "\n"; - return $message; - } - - /** - * Retrieve a series of NNTP messages and decode them. - */ - function getMessages($groupname, $msgIds) - { - $summary = $this->selectGroup($groupname); - $message = $dec = ''; - - if (PEAR::isError($summary)) - { - echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n"; - return false; - } - - foreach($msgIds as $msgId) - { - $messageID = '<'.$msgId.'>'; - $body = $this->getBody($messageID, true); - if (PEAR::isError($body)) - { - //echo 'NntpPrc : Error fetching '.$messageID.' in '.$groupname.' (Server response: '. $body->getMessage().')'; - return false; - } - - $dec = $this->decodeYenc($body); - if (!$dec) - { - // - // Yenc decode failed - // - return false; - } - - $message .= $dec; - } - return $message; - } - - /** - * Retrieve all NNTP messages associated with a binaries.ID - */ - function getBinary($binaryId, $isNfo=false) - { - $db = new DB(); - $bin = new Binaries(); - - $binary = $bin->getById($binaryId); - if (!$binary) - return false; - - $summary = $this->selectGroup($binary['groupname']); - $message = $dec = ''; - - if (PEAR::isError($summary)) - { - echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n"; - return false; - } - - $resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId)); - - // - // Dont attempt to download nfos which are larger than one part. - // - if (sizeof($resparts) > 1 && $isNfo === true) - { - return false; - } - - foreach($resparts as $part) - { - $messageID = '<'.$part['messageID'].'>'; - $body = $this->getBody($messageID, true); - if (PEAR::isError($body)) - { - //echo 'NntpPrc : Error fetching part number '.$part['messageID'].' in '.$binary['groupname'].' (Server response: '. $body->getMessage().')'; - return false; - } - - $dec = $this->decodeYenc($body); - if (!$dec) - { - // - // Yenc decode failed - // - return false; - } - - $message .= $dec; - } - return $message; - } - - /** - * Get XZVER for a range of NNTP messages. - */ - function getXOverview($range, $_names = true, $_forceNames = true) - { - // Fetch overview from server - $overview = $this->cmdXZver($range); - if (PEAR::isError($overview)) { - return $overview; - } - - // Use field names from overview format as keys? - if ($_names) - { - // Already cached? - if (is_null($this->_overviewFormatCache)) { - // Fetch overview format - $format = $this->getOverviewFormat($_forceNames, true); - if (PEAR::isError($format)){ - return $format; - } - - // Prepend 'Number' field - $format = array_merge(array('Number' => false), $format); - - // Cache format - $this->_overviewFormatCache = $format; - } - else - { - $format = $this->_overviewFormatCache; - } - - // Loop through all articles - foreach ($overview as $key => $article) - { - if (sizeof($format) == sizeof($article)) - { - //Replace overview using $format as keys, $article as values - $overview[$key] = array_combine(array_keys($format), $article); - - // If article prefixed by field name, remove it - foreach($format as $fkey=>$fval) - { - if ($fval === true) - { - $overview[$key][$fkey] = trim(str_replace($fkey.':', '', $overview[$key][$fkey])); - } - } - } - } - } - - switch (true) - { - // Expect one article - case is_null($range); - case is_int($range); - case is_string($range) && ctype_digit($range): - case is_string($range) && substr($range, 0, 1) == '<' && substr($range, -1, 1) == '>': - if (count($overview) == 0) { - return false; - } else { - return reset($overview); - } - break; - - // Expect multiple articles - default: - return $overview; - } - } - - /** - * Send XZVER command over NNTP connection. - */ - function cmdXZver($range = NULL) - { - if (is_null($range)) - $command = 'XZVER'; - else - $command = 'XZVER ' . $range; - - $response = $this->_sendCommand($command); - - switch ($response) { - case 224: // 224, RFC2980: 'Overview information follows' - $data = $this->_getTextResponse(); - - //de-yenc - $dec = $this->decodeYenc(implode("\r\n", $data)); - if (!$dec) - { - $this->throwError("yenc decode failure"); - } - - //inflate deflated string - $data = explode("\r\n", gzinflate($dec)); - - foreach ($data as $key => $value) - $data[$key] = explode("\t", ltrim($value)); - - return $data; - break; - case 412: // 412, RFC2980: 'No news group current selected' - $this->throwError("No news group current selected ({$this->_currentStatusResponse()})", $response); - break; - case 420: // 420, RFC2980: 'No article(s) selected' - $this->throwError("No article(s) selected ({$this->_currentStatusResponse()})", $response); - break; - case 502: // 502 RFC2980: 'no permission' - $this->throwError("No permission ({$this->_currentStatusResponse()})", $response); - break; - case 500: // 500 RFC2980: 'unknown command' - $this->throwError("XZver not supported ({$this->_currentStatusResponse()})", $response); - break; - default: - return $this->_handleUnexpectedResponse($response); - } - } - - /** - * Decode a yenc encoded string. - */ - function decodeYenc($yencodedvar) - { - $input = array(); - preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $yencodedvar, $input); - if (isset($input[1])) - { - $ret = ""; - $input = trim(preg_replace("/\r\n/im", "", preg_replace("/(^=yend.*)/im", "", preg_replace("/(^=ypart.*\\r\\n)/im", "", preg_replace("/(^=ybegin.*\\r\\n)/im", "", $input[1], 1), 1), 1))); - - for( $chr = 0; $chr < strlen($input) ; $chr++) - $ret .= ($input[$chr] != "=" ? chr(ord($input[$chr]) - 42) : chr((ord($input[++$chr]) - 64) - 42)); - - return $ret; - } - return false; - } -} diff --git a/bin/innodb/lib/powerspawn.php b/bin/innodb/lib/powerspawn.php deleted file mode 100644 index bdc5860c5..000000000 --- a/bin/innodb/lib/powerspawn.php +++ /dev/null @@ -1,174 +0,0 @@ - - * @link https://github.com/lordgnu/PowerSpawn - * @version 1.0 - */ -class PowerSpawn -{ - private $myChildren; - private $parentPID; - private $shutdownCallback = null; - private $killCallback = null; - - public $maxChildren = 10; // Max number of children allowed to Spawn - public $timeLimit = 0; // Time limit in seconds (0 to disable) - public $sleepCount = 1; // Number of seconds to sleep on Tick() - - public $childData; // Variable for storage of data to be passed to the next spawned child - public $complete; - - public function __construct() { - if (function_exists('pcntl_fork') && function_exists('posix_getpid')) { - // Everything is good - $this->parentPID = $this->myPID(); - $this->myChildren = array(); - $this->complete = false; - - // Install the signal handler - pcntl_signal(SIGCHLD, array($this, 'sigHandler')); - } else { - die("You must have POSIX and PCNTL functions to use PowerSpawn\n"); - } - } - - public function __destruct() { - - } - - public function sigHandler($signo) { - switch ($signo) { - case SIGCHLD: - $this->checkChildren(); - break; - } - } - - public function checkChildren() { - foreach ($this->myChildren as $i => $child) { - // Check for time running and if still running - if ($this->pidDead($child['pid']) != 0) { - // Child is dead - unset($this->myChildren[$i]); - } elseif ($this->timeLimit > 0) { - // Check the time limit - if (time() - $child['time'] >= $this->timeLimit) { - // Child had exceeded time limit - $this->killChild($child['pid']); - unset($this->myChildren[$i]); - } - } - } - } - - public function myPID() { - return posix_getpid(); - } - - public function myParent() { - return posix_getppid(); - } - - public function spawnChild() { - $time = time(); - $pid = pcntl_fork(); - if ($pid) $this->myChildren[] = array('time'=>$time,'pid'=>$pid); - } - - public function killChild($pid = 0) { - if ($pid > 0) { - posix_kill($pid, SIGTERM); - if ($this->killCallback !== null) call_user_func($this->killCallback); - } - } - - public function parentCheck() { - if ($this->myPID() == $this->parentPID) { - return true; - } else { - return false; - } - } - - public function pidDead($pid = 0) { - if ($pid > 0) { - return pcntl_waitpid($pid, $status, WUNTRACED OR WNOHANG); - } else { - return 0; - } - } - - public function setCallback($callback = null) { - $this->shutdownCallback = $callback; - } - - public function setKillCallback($callback = null) { - $this->killCallback = $callback; - } - - public function childCount() { - return count($this->myChildren); - } - - public function runParentCode() { - if (!$this->complete) { - return $this->parentCheck(); - } else { - if ($this->shutdownCallback !== null) - call_user_func($this->shutdownCallback); - return false; - } - } - - public function runChildCode() { - return !$this->parentCheck(); - } - - public function spawnReady() { - if (count($this->myChildren) < $this->maxChildren) { - return true; - } else { - return false; - } - } - - public function shutdown() { - while($this->childCount()) { - $this->checkChildren(); - $this->tick(); - } - $this->complete = true; - } - - public function tick() { - sleep($this->sleepCount); - } -} - - -/* -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -https://github.com/lordgnu/PowerSpawn for usage examples -*/ diff --git a/bin/innodb/lib/site.php b/bin/innodb/lib/site.php deleted file mode 100755 index a9d6fd6e1..000000000 --- a/bin/innodb/lib/site.php +++ /dev/null @@ -1,146 +0,0 @@ -row2Object($form); - - if (substr($site->nzbpath, strlen($site->nzbpath) - 1) != '/') - $site->nzbpath = $site->nzbpath."/"; - - // - // Validate site settings - // - if ($site->mediainfopath != "" && !is_file($site->mediainfopath)) - return Sites::ERR_BADMEDIAINFOPATH; - - if ($site->ffmpegpath != "" && !is_file($site->ffmpegpath)) - return Sites::ERR_BADFFMPEGPATH; - - if ($site->unrarpath != "" && !is_file($site->unrarpath)) - return Sites::ERR_BADUNRARPATH; - - if ($site->nzbpath != "" && !file_exists($site->nzbpath)) - return Sites::ERR_BADNZBPATH; - - if ($site->checkpasswordedrar == 2 && !is_file($site->unrarpath)) - return Sites::ERR_DEEPNOUNRAR; - - if ($site->tmpunrarpath != "" && !file_exists($site->tmpunrarpath)) - return Sites::ERR_BADTMPUNRARPATH; - - if ($site->lamepath != "" && !file_exists($site->lamepath)) - return Sites::ERR_BADLAMEPATH; - - if ($site->sabcompletedir != "" && !file_exists($site->sabcompletedir)) - return Sites::ERR_SABCOMPLETEPATH; - - $sql = $sqlKeys = array(); - foreach($form as $settingK=>$settingV) - { - $sql[] = sprintf("WHEN %s THEN %s", $db->escapeString($settingK), $db->escapeString(trim($settingV))); - $sqlKeys[] = $db->escapeString($settingK); - } - - $db->query(sprintf("UPDATE site SET value = CASE setting %s END WHERE setting IN (%s)", implode(' ', $sql), implode(', ', $sqlKeys))); - - return $site; - } - - public function get() - { - $db = new DB(); - $rows = $db->query("select * from site"); - - if ($rows === false) - return false; - - return $this->rows2Object($rows); - } - - public function rows2Object($rows) - { - $obj = new stdClass; - foreach($rows as $row) - $obj->{$row['setting']} = $row['value']; - - $obj->{'version'} = $this->version(); - return $obj; - } - - public function row2Object($row) - { - $obj = new stdClass; - $rowKeys = array_keys($row); - foreach($rowKeys as $key) - $obj->{$key} = $row[$key]; - - return $obj; - } - - public function getUnappliedPatches($site) - { - preg_match("/\d+/", $site->dbversion, $matches); - $currentrev = $matches[0]; - - $patchpath = WWW_DIR."../db/patch/0.2.3/"; - $patchfiles = glob($patchpath."*.sql"); - $missingpatch = array(); - foreach($patchfiles as $file) - { - $filecontents = file_get_contents($file); - if (preg_match("/Rev\: (\d+)/", $filecontents, $matches)) - { - $patchrev = $matches[1]; - if ($patchrev > $currentrev) - $missingpatch[] = $file; - } - } - return $missingpatch; - } - - public function updateItem($setting, $value) - { - $db = new DB(); - $sql = sprintf("update site set value = %s where setting = %s", $db->escapeString($value), $db->escapeString($setting)); - return $db->query($sql); - } - - public function updateLatestRegexRevision($rev) - { - return $this->updateItem("latestregexrevision", $rev); - } - - public function getLicense($html=false) - { - $n = "\r\n"; - if ($html) - $n = "
"; - - return $n."newznab ".$this->version()." Copyright (C) ".date("Y")." newznab.com".$n." - -This program is distributed with a commercial licence. See LICENCE.txt for -further details.".$n; - } -} diff --git a/bin/innodb/nzb-import.php b/bin/innodb/nzb-import.php deleted file mode 100755 index fc78e7148..000000000 --- a/bin/innodb/nzb-import.php +++ /dev/null @@ -1,143 +0,0 @@ -query("SELECT ID, name FROM groups"); -foreach ($groups as $group) - $siteGroups[$group["name"]] = $group["ID"]; - -if (!isset($groups) || count($groups) == 0) { - echo "no groups available in the database, add first.\n"; -} else { - $nzbCount = 0; - - - - if (count($filestoprocess) == 0) - $filestoprocess = glob($path . "*.nzb"); - $start = date('Y-m-d H:i:s'); - - foreach ($filestoprocess as $nzbFile) { - $importfailed = false; - $transaction = ""; - $nzb = file_get_contents($nzbFile); - - $xml = @simplexml_load_string($nzb); - if (!$xml || strtolower($xml->getName()) != 'nzb') { - continue; - } - - $i = 0; - foreach ($xml->file as $file) { - //file info - $groupID = -1; - $name = (string)$file->attributes()->subject; - $fromname = (string)$file->attributes()->poster; - $unixdate = (string)$file->attributes()->date; - $date = date("Y-m-d H:i:s", (string)$file->attributes()->date); - - //groups - $groupArr = array(); - foreach ($file->groups->group as $group) { - $group = (string)$group; - if (array_key_exists($group, $siteGroups)) { - $groupID = $siteGroups[$group]; - } - $groupArr[] = $group; - } - - if ($groupID != -1) { - $xref = implode(': ', $groupArr) . ':'; - - $totalParts = sizeof($file->segments->segment); - - //insert binary - $binaryHash = md5($name . $fromname . $groupID); - - - if ($usenzbname) { //Use the binary names - $usename = str_replace('.nzb', '', basename($nzbFile)); - - $binarySql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded, importname, relname, relpart, reltotalpart, procstat, categoryID, regexID, reqID) values (%s, %s, %s, %s, %s, %s, %s, NOW(), %s, replace(%s, '_', ' '), %d, %d, %d, %s, %d, %s);", - $db->escapeString($name), $db->escapeString($fromname), $db->escapeString($date), - $db->escapeString($xref), $db->escapeString($totalParts), $db->escapeString($groupID), $db->escapeString($binaryHash), $db->escapeString($nzbFile), - $db->escapeString($usename), 1, 1, 5, "null", "null", "null"); - - $transaction = $transaction . $binarySql; - - } else { //Don't use the binary names - $binarySql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded, importname) values (%s, %s, %s, %s, %s, %s, %s, NOW(), %s);", - $db->escapeString($name), $db->escapeString($fromname), $db->escapeString($date), - $db->escapeString($xref), $db->escapeString($totalParts), $db->escapeString($groupID), $db->escapeString($binaryHash), $db->escapeString($nzbFile)); - - $transaction = $transaction . $binarySql; - } - - $sessionVarSQL = "SET @binID = last_insert_id();"; - $transaction = $transaction . $sessionVarSQL; - - //segments (i.e. parts) - if (count($file->segments->segment) > 0) { - $partsSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) values "; - foreach ($file->segments->segment as $segment) { - $messageId = (string)$segment; - $partnumber = $segment->attributes()->number; - $size = $segment->attributes()->bytes; - - $partsSql .= sprintf("(%s, %s, 0, %s, %s, NOW()),", - "@binID", $db->escapeString($messageId), $db->escapeString($partnumber), - $db->escapeString($size)); - } - $partsSql = substr($partsSql, 0, -1) . ";"; - $transaction = $transaction . $partsSql; - } - - } else { - $importfailed = true; - - echo ("no group found for " . $name . " (one of " . implode(', ', $groupArr) . " are missing)" . $strTerminator); - flush(); - break; - } - } - - if (!$importfailed) { - $db->multiQueryTransaction($transaction); - - $nzbCount++; - @unlink($nzbFile); - - echo ("imported " . $nzbFile . $strTerminator); - flush(); - } - if ($nzbCount == 100) { - break; - } - } -} -$seconds = strtotime(date('Y-m-d H:i:s')) - strtotime($start); -$retval .= 'Processed ' . $nzbCount . ' nzbs in ' . $seconds . ' second(s)'; - -echo $retval; -die(); diff --git a/bin/innodb/update_binaries.php b/bin/innodb/update_binaries.php deleted file mode 100755 index db5a59596..000000000 --- a/bin/innodb/update_binaries.php +++ /dev/null @@ -1,24 +0,0 @@ -getByName($group); - - $bin = new Binaries; - $bin->updateGroup(null, $group); -} -else -{ - $binaries = new Binaries; - $binaries->updateAllGroups(); -} - -?> diff --git a/bin/innodb/update_binaries_threaded.php b/bin/innodb/update_binaries_threaded.php deleted file mode 100755 index 965d47d33..000000000 --- a/bin/innodb/update_binaries_threaded.php +++ /dev/null @@ -1,77 +0,0 @@ -getActive(); -unset($groups); - -$ps = new PowerSpawn; -$ps->setCallback('psUpdateComplete'); -$ps->maxChildren = 10; -$ps->timelimit = 0; // Disable child timeout - -echo "Starting threaded binary update process\n"; - -while ($ps->runParentCode()) -{ - // Start the parent loop - if (count($groupList)) - { - // We still have groups to process - if ($ps->spawnReady()) - { - // Spawn another thread - $ps->childData = array_pop($groupList); - echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; - $ps->spawnChild(); - } - else - { - // There are no more slots available to run - $ps->tick(); - #echo "- \n"; - } - } - else - { - // No more groups to process - echo "No more groups to process - Initiating shutdown\n"; - $ps->shutdown(); - echo "Shutdown complete\n"; - } -} - -unset($groupList); - -if ($ps->runChildCode()) -{ - $group = $ps->childData; - - $thread = sprintf("%05d",$ps->myPID()); - - echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n"; - - $param = $group['name']; - - $dir = dirname(__FILE__); - $file = 'update_binaries.php'; - - $output = shell_exec("php {$dir}/{$file} {$param}"); - - echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; -} - -// Exit to call back to parent - Let know that child has completed -exit(0); - -// Create callback function -function psUpdateComplete() -{ - echo "[Thread-MASTER] Threaded update process complete\n"; -} - -?> diff --git a/bin/myisam/backfill.php b/bin/myisam/backfill.php deleted file mode 100644 index 43690e0d7..000000000 --- a/bin/myisam/backfill.php +++ /dev/null @@ -1,15 +0,0 @@ -backfillAllGroups($groupName); - -?> diff --git a/bin/myisam/backfill_threaded.php b/bin/myisam/backfill_threaded.php deleted file mode 100755 index 06c7ec0e5..000000000 --- a/bin/myisam/backfill_threaded.php +++ /dev/null @@ -1,77 +0,0 @@ -getActive(); -unset($groups); - -$ps = new PowerSpawn; -$ps->setCallback('psUpdateComplete'); -$ps->maxChildren = 10; -$ps->timeLimit = 0; // Disable child timeout - -echo "Starting threaded backfill process\n"; - -while ($ps->runParentCode()) -{ - // Start the parent loop - if (count($groupList)) - { - // We still have groups to process - if ($ps->spawnReady()) - { - // Spawn another thread - $ps->childData = array_pop($groupList); - echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; - $ps->spawnChild(); - } - else - { - // There are no more slots available to run - $ps->tick(); - #echo ". \n"; - } - } - else - { - // No more groups to process - echo "No more groups to process - Initiating shutdown\n"; - $ps->shutdown(); - echo "Shutdown complete\n"; - } -} - -unset($groupList); - -if ($ps->runChildCode()) -{ - $group = $ps->childData; - - $thread = sprintf("%05d",$ps->myPID()); - - echo "[Thread-{$thread}] Begining backfill processing for group {$group['name']}\n"; - - $param = $group['name']; - - $dir = dirname(__FILE__); - $file = 'backfill.php'; - - $output = shell_exec("php {$dir}/{$file} {$param}"); - - echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; -} - -// Exit to call back to parent - Let know that child has completed -exit(0); - -// Create callback function -function psUpdateComplete() -{ - echo "[Thread-MASTER] Threaded backfill process complete\n"; -} - -?> diff --git a/bin/myisam/powerspawn.php b/bin/myisam/powerspawn.php deleted file mode 100644 index bdc5860c5..000000000 --- a/bin/myisam/powerspawn.php +++ /dev/null @@ -1,174 +0,0 @@ - - * @link https://github.com/lordgnu/PowerSpawn - * @version 1.0 - */ -class PowerSpawn -{ - private $myChildren; - private $parentPID; - private $shutdownCallback = null; - private $killCallback = null; - - public $maxChildren = 10; // Max number of children allowed to Spawn - public $timeLimit = 0; // Time limit in seconds (0 to disable) - public $sleepCount = 1; // Number of seconds to sleep on Tick() - - public $childData; // Variable for storage of data to be passed to the next spawned child - public $complete; - - public function __construct() { - if (function_exists('pcntl_fork') && function_exists('posix_getpid')) { - // Everything is good - $this->parentPID = $this->myPID(); - $this->myChildren = array(); - $this->complete = false; - - // Install the signal handler - pcntl_signal(SIGCHLD, array($this, 'sigHandler')); - } else { - die("You must have POSIX and PCNTL functions to use PowerSpawn\n"); - } - } - - public function __destruct() { - - } - - public function sigHandler($signo) { - switch ($signo) { - case SIGCHLD: - $this->checkChildren(); - break; - } - } - - public function checkChildren() { - foreach ($this->myChildren as $i => $child) { - // Check for time running and if still running - if ($this->pidDead($child['pid']) != 0) { - // Child is dead - unset($this->myChildren[$i]); - } elseif ($this->timeLimit > 0) { - // Check the time limit - if (time() - $child['time'] >= $this->timeLimit) { - // Child had exceeded time limit - $this->killChild($child['pid']); - unset($this->myChildren[$i]); - } - } - } - } - - public function myPID() { - return posix_getpid(); - } - - public function myParent() { - return posix_getppid(); - } - - public function spawnChild() { - $time = time(); - $pid = pcntl_fork(); - if ($pid) $this->myChildren[] = array('time'=>$time,'pid'=>$pid); - } - - public function killChild($pid = 0) { - if ($pid > 0) { - posix_kill($pid, SIGTERM); - if ($this->killCallback !== null) call_user_func($this->killCallback); - } - } - - public function parentCheck() { - if ($this->myPID() == $this->parentPID) { - return true; - } else { - return false; - } - } - - public function pidDead($pid = 0) { - if ($pid > 0) { - return pcntl_waitpid($pid, $status, WUNTRACED OR WNOHANG); - } else { - return 0; - } - } - - public function setCallback($callback = null) { - $this->shutdownCallback = $callback; - } - - public function setKillCallback($callback = null) { - $this->killCallback = $callback; - } - - public function childCount() { - return count($this->myChildren); - } - - public function runParentCode() { - if (!$this->complete) { - return $this->parentCheck(); - } else { - if ($this->shutdownCallback !== null) - call_user_func($this->shutdownCallback); - return false; - } - } - - public function runChildCode() { - return !$this->parentCheck(); - } - - public function spawnReady() { - if (count($this->myChildren) < $this->maxChildren) { - return true; - } else { - return false; - } - } - - public function shutdown() { - while($this->childCount()) { - $this->checkChildren(); - $this->tick(); - } - $this->complete = true; - } - - public function tick() { - sleep($this->sleepCount); - } -} - - -/* -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -https://github.com/lordgnu/PowerSpawn for usage examples -*/ diff --git a/bin/myisam/update_binaries.php b/bin/myisam/update_binaries.php deleted file mode 100644 index bedcb7fc4..000000000 --- a/bin/myisam/update_binaries.php +++ /dev/null @@ -1,25 +0,0 @@ -getByName($group); - - $bin = new Binaries; - $bin->updateGroup(null, $group); -} -else -{ - $binaries = new Binaries; - $binaries->updateAllGroups(); -} - -?> diff --git a/bin/myisam/update_binaries_threaded.php b/bin/myisam/update_binaries_threaded.php deleted file mode 100755 index 6e472fe2b..000000000 --- a/bin/myisam/update_binaries_threaded.php +++ /dev/null @@ -1,77 +0,0 @@ -getActive(); -unset($groups); - -$ps = new PowerSpawn; -$ps->setCallback('psUpdateComplete'); -$ps->maxChildren = 10; -$ps->timelimit = 0; // Disable child timeout - -echo "Starting threaded binary update process\n"; - -while ($ps->runParentCode()) -{ - // Start the parent loop - if (count($groupList)) - { - // We still have groups to process - if ($ps->spawnReady()) - { - // Spawn another thread - $ps->childData = array_pop($groupList); - echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) ." group(s) to update after this\n"; - $ps->spawnChild(); - } - else - { - // There are no more slots available to run - $ps->tick(); - #echo "- \n"; - } - } - else - { - // No more groups to process - echo "No more groups to process - Initiating shutdown\n"; - $ps->shutdown(); - echo "Shutdown complete\n"; - } -} - -unset($groupList); - -if ($ps->runChildCode()) -{ - $group = $ps->childData; - - $thread = sprintf("%05d",$ps->myPID()); - - echo "[Thread-{$thread}] Begining processing for group {$group['name']}\n"; - - $param = $group['name']; - - $dir = dirname(__FILE__); - $file = 'update_binaries.php'; - - $output = shell_exec("php {$dir}/{$file} {$param}"); - - echo "[Thread-{$thread}] Completed update for group {$group['name']}\n"; -} - -// Exit to call back to parent - Let know that child has completed -exit(0); - -// Create callback function -function psUpdateComplete() -{ - echo "[Thread-MASTER] Threaded update process complete\n"; -} - -?> diff --git a/bin/workhorse.sh b/bin/workhorse.sh index cd86525cd..29ad4ad7f 100755 --- a/bin/workhorse.sh +++ b/bin/workhorse.sh @@ -18,20 +18,19 @@ if [ "$THREADS" == "true" -a "$INNODB" == "true" ]; then #make active groups current if [[ $BINARIES == "true" ]] ; then - cd $INNODB_PATH + cd $NEWZNAB_PATH [ -f update_binaries_threaded.php ] && $PHP update_binaries_threaded.php fi #get backfill for all active groups if [[ $BACKFILL == "true" ]] ; then - cd $INNODB_PATH + cd $NEWZNAB_PATH [ -f backfill_threaded.php ] && $PHP backfill_threaded.php fi wait if [[ $BACKFILL == "true" ]] ; then - cd $INNODB_PATH #increment backfill days $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" fi @@ -65,7 +64,6 @@ elif [ "$THREADS" != "true" -a "$INNODB" == "true" ]; then wait if [[ $BACKFILL == "true" ]] ; then - cd $INNODB_PATH #increment backfill days $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" fi @@ -87,20 +85,19 @@ elif [ "$THREADS" == "true" -a "$INNODB" != "true" ]; then #make active groups current if [[ $BINARIES == "true" ]] ; then - cd $MYISAM_PATH + cd $NEWZNAB_PATH [ -f update_binaries_threaded.php ] && $PHP update_binaries_threaded.php fi #get backfill for all active groups if [[ $BACKFILL == "true" ]] ; then - cd $MYISAM_PATH + cd $NEWZNAB_PATH [ -f backfill_threaded.php ] && $PHP backfill_threaded.php fi wait if [[ $BACKFILL == "true" ]] ; then - cd $INNODB_PATH #increment backfill days $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" fi @@ -135,7 +132,6 @@ elif [ "$THREADS" != "true" -a "$INNODB" != "true" ]; then wait if [[ $BACKFILL == "true" ]] ; then - cd $INNODB_PATH #increment backfill days $MYSQL -u$DB_USER -h $DB_HOST --password=$DB_PASSWORD $DB_NAME -e "${MYSQL_CMD}" fi diff --git a/edit_these.sh b/edit_these.sh index 36eb700f0..bbb3bd583 100755 --- a/edit_these.sh +++ b/edit_these.sh @@ -7,6 +7,7 @@ export NEWZPATH="/var/www/newznab" export NEWZNAB_PATH=$NEWZPATH"/misc/update_scripts" export TESTING_PATH=$NEWZPATH"/misc/testing" export ADMIN_PATH=$NEWZPATH"/www/admin" +export INNODB_PATH=$TESTING_PATH"/innodb" export USERNAME="what is your name" # this is the user name that will run these scripts export NEWZNAB_IMPORT_SLEEP_TIME="1" # in seconds - this includes import_nzb backfill and current fill, 0 may cause errors export NEWZNAB_POST_SLEEP_TIME="1" # in seconds - this is for post processing - sleep between loops, 0 may cause errors @@ -14,16 +15,16 @@ export MAXDAYS="210" #max days for backfill export NZBS="/path/to/nzbs" #The path to the nzb dump you downloaded from torrents #Choose to run the threaded or non-threaded newznab scripts true/false -export THREADS="true" +export THREADS="false" #Choose your database, comment the one true/false -export INNODB="true" +export INNODB="false" #Choose to run update_cleanup.php true/false -export CLEANUP="true" +export CLEANUP="false" #Choose to run update_binaries true/false -export BINARIES="true" +export BINARIES="false" #Choose to run backfill script true/false export BACKFILL="true" @@ -32,11 +33,11 @@ export BACKFILL="true" export IMPORT="true" #Select some monitoring script, if they are not installed, it will not affect the running of the scripts -export USE_HTOP="true" -export USE_NMON="true" -export USE_BWMNG="true" -export USE_IOTOP="true" -export USE_MYTOP="true" +export USE_HTOP="false" +export USE_NMON="false" +export USE_BWMNG="false" +export USE_IOTOP="false" +export USE_MYTOP="false" #By using this script you understand that the programmer is not responsible for any loss of data, users, or sanity. #You also agree that you were smart enough to make a backup of your database and files. Do you agree? yes/no diff --git a/start.sh b/start.sh index 3bb8a14ec..1f243b3ed 100755 --- a/start.sh +++ b/start.sh @@ -18,10 +18,6 @@ if [[ $AGREED == "no" ]]; then exit fi -export INNODB_PATH=$DIR"/bin/innodb" -export MYISAM_PATH=$DIR"/bin/myisam" -export START_PATH=$DIR - #delete some files [ -f bin/lib/postprocess4.php ];rm bin/lib/postprocess4.php [ -f bin/processAlternate4.php ];rm bin/processAlternate4.php