starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

really updated kevin123's compression patch

Browse Source
This commit is contained in:
jonnyboy
2013-02-11 22:27:04 +01:00
parent c05a1c31b9
commit 871af75ab6
8 changed files with 6088 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kevin123/Net_NNTP/LICENSE
Executable
+48
View File
@@ -0,0 +1,48 @@
+-----------------------------------------------------------------------+
| |
| W3C® SOFTWARE NOTICE AND LICENSE |
| http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 |
| |
| This work (and included software, documentation such as READMEs, |
| or other related items) is being provided by the copyright holders |
| under the following license. By obtaining, using and/or copying |
| this work, you (the licensee) agree that you have read, understood, |
| and will comply with the following terms and conditions. |
| |
| Permission to copy, modify, and distribute this software and its |
| documentation, with or without modification, for any purpose and |
| without fee or royalty is hereby granted, provided that you include |
| the following on ALL copies of the software and documentation or |
| portions thereof, including modifications: |
| |
| 1. The full text of this NOTICE in a location viewable to users |
| of the redistributed or derivative work. |
| |
| 2. Any pre-existing intellectual property disclaimers, notices, |
| or terms and conditions. If none exist, the W3C Software Short |
| Notice should be included (hypertext is preferred, text is |
| permitted) within the body of any redistributed or derivative |
| code. |
| |
| 3. Notice of any changes or modifications to the files, including |
| the date changes were made. (We recommend you provide URIs to |
| the location from which the code is derived.) |
| |
| THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT |
| HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, |
| INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR |
| FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE |
| OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, |
| COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. |
| |
| COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, |
| SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE |
| SOFTWARE OR DOCUMENTATION. |
| |
| The name and trademarks of copyright holders may NOT be used in |
| advertising or publicity pertaining to the software without |
| specific, written prior permission. Title to copyright in this |
| software and any associated documentation will at all times |
| remain with copyright holders. |
| |
+-----------------------------------------------------------------------+
kevin123/Net_NNTP/NNTP/Client.php
Executable
+1549
View File
File diff suppressed because it is too large Load Diff
kevin123/Net_NNTP/NNTP/Protocol/Client.php
Executable
+2196
View File
File diff suppressed because it is too large Load Diff
kevin123/Net_NNTP/NNTP/Protocol/Responsecode.php
Executable
+509
View File
@@ -0,0 +1,509 @@
<?php
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 foldmethod=marker: */
/**
*
*
* PHP versions 4 and 5
*
* <pre>
* +-----------------------------------------------------------------------+
* | |
* | W3C® SOFTWARE NOTICE AND LICENSE |
* | http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 |
* | |
* | This work (and included software, documentation such as READMEs, |
* | or other related items) is being provided by the copyright holders |
* | under the following license. By obtaining, using and/or copying |
* | this work, you (the licensee) agree that you have read, understood, |
* | and will comply with the following terms and conditions. |
* | |
* | Permission to copy, modify, and distribute this software and its |
* | documentation, with or without modification, for any purpose and |
* | without fee or royalty is hereby granted, provided that you include |
* | the following on ALL copies of the software and documentation or |
* | portions thereof, including modifications: |
* | |
* | 1. The full text of this NOTICE in a location viewable to users |
* | of the redistributed or derivative work. |
* | |
* | 2. Any pre-existing intellectual property disclaimers, notices, |
* | or terms and conditions. If none exist, the W3C Software Short |
* | Notice should be included (hypertext is preferred, text is |
* | permitted) within the body of any redistributed or derivative |
* | code. |
* | |
* | 3. Notice of any changes or modifications to the files, including |
* | the date changes were made. (We recommend you provide URIs to |
* | the location from which the code is derived.) |
* | |
* | THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT |
* | HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, |
* | INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR |
* | FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE |
* | OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, |
* | COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. |
* | |
* | COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, |
* | SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE |
* | SOFTWARE OR DOCUMENTATION. |
* | |
* | The name and trademarks of copyright holders may NOT be used in |
* | advertising or publicity pertaining to the software without |
* | specific, written prior permission. Title to copyright in this |
* | software and any associated documentation will at all times |
* | remain with copyright holders. |
* | |
* +-----------------------------------------------------------------------+
* </pre>
*
* @category Net
* @package Net_NNTP
* @author Heino H. Gehlsen <heino@gehlsen.dk>
* @copyright 2002-2011 Heino H. Gehlsen <heino@gehlsen.dk>. All Rights Reserved.
* @license http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 W3C® SOFTWARE NOTICE AND LICENSE
* @version SVN: $Id: Responsecode.php 306619 2010-12-24 12:16:07Z heino $
* @link http://pear.php.net/package/Net_NNTP
* @see
* @since File available since release 1.3.0
*/
// {{{ Constants: Connection
/**
* 'Server ready - posting allowed' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_READY_POSTING_ALLOWED', 200);
/**
* 'Server ready - no posting allowed' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_READY_POSTING_PROHIBITED', 201);
/**
* 'Closing connection - goodbye!' (RFC977)
*
* @access public
* @since ?
*/
//define('NET_NNTP_PROTOCOL_RESPONSECODE_DISCONNECTING_REQUESTED', 205); ///// goodbye
/**
* 'Service discontinued' (RFC977)
*
* @access public
* @since ?
*/
//define('NET_NNTP_PROTOCOL_RESPONSECODE_DISCONNECTING_FORCED', 400); ///// unavailable / discontinued
/**
* 'Slave status noted' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_SLAVE_RECOGNIZED', 202);
// }}}
// {{{ Constants: Common errors
/**
* 'Command not recognized' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_UNKNOWN_COMMAND', 500);
/**
* 'Command syntax error' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_SYNTAX_ERROR', 501);
/**
* 'Access restriction or permission denied' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NOT_PERMITTED', 502);
/**
* 'Program fault - command not performed' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NOT_SUPPORTED', 503);
// }}}
// {{{ Constants: Group selection
/**
* 'Group selected' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_GROUP_SELECTED', 211);
/**
* 'No such news group' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_GROUP', 411);
// }}}
// {{{ Constants: Article retrieval
/**
* 'Article retrieved - head and body follow' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_ARTICLE_FOLLOWS', 220);
/**
* 'Article retrieved - head follows' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_HEAD_FOLLOWS', 221);
/**
* 'Article retrieved - body follows' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_BODY_FOLLOWS', 222);
/**
* 'Article retrieved - request text separately' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_ARTICLE_SELECTED', 223);
/**
* 'No newsgroup has been selected' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_GROUP_SELECTED', 412);
/**
* 'No current article has been selected' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_ARTICLE_SELECTED', 420);
/**
* 'No next article in this group' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_NEXT_ARTICLE', 421);
/**
* 'No previous article in this group' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_PREVIOUS_ARTICLE', 422);
/**
* 'No such article number in this group' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_ARTICLE_NUMBER', 423);
/**
* 'No such article found' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_ARTICLE_ID', 430);
// }}}
// {{{ Constants: Transferring
/**
* 'Send article to be transferred' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_SEND', 335);
/**
* 'Article transferred ok' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_SUCCESS', 235);
/**
* 'Article not wanted - do not send it' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_UNWANTED', 435);
/**
* 'Transfer failed - try again later' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_FAILURE', 436);
/**
* 'Article rejected - do not try again' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_REJECTED', 437);
// }}}
// {{{ Constants: Posting
/**
* 'Send article to be posted' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_SEND', 340);
/**
* 'Article posted ok' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_SUCCESS', 240);
/**
* 'Posting not allowed' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_PROHIBITED', 440);
/**
* 'Posting failed' (RFC977)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_FAILURE', 441);
// }}}
// {{{ Constants: Authorization
/**
* 'Authorization required for this command' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_REQUIRED', 450);
/**
* 'Continue with authorization sequence' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_CONTINUE', 350);
/**
* 'Authorization accepted' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_ACCEPTED', 250);
/**
* 'Authorization rejected' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_REJECTED', 452);
// }}}
// {{{ Constants: Authentication
/**
* 'Authentication required' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_REQUIRED', 480);
/**
* 'More authentication information required' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_CONTINUE', 381);
/**
* 'Authentication accepted' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_ACCEPTED', 281);
/**
* 'Authentication rejected' (RFC2980)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_REJECTED', 482);
// }}}
// {{{ Constants: Misc
/**
* 'Help text follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_HELP_FOLLOWS', 100);
/**
* 'Capabilities list follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_CAPABILITIES_FOLLOW', 101);
/**
* 'Server date and time' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_SERVER_DATE', 111);
/**
* 'Information follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_GROUPS_FOLLOW', 215);
/**
* 'Overview information follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_OVERVIEW_FOLLOWS', 224);
/**
* 'Headers follow' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_HEADERS_FOLLOW', 225);
/**
* 'List of new articles follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NEW_ARTICLES_FOLLOW', 230);
/**
* 'List of new newsgroups follows' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_NEW_GROUPS_FOLLOW', 231);
/**
* 'The server is in the wrong mode; the indicated capability should be used to change the mode' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_WRONG_MODE', 401);
/**
* 'Internal fault or problem preventing action being taken' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_INTERNAL_FAULT', 403);
/**
* 'Command unavailable until suitable privacy has been arranged' (Draft)
*
* (the client must negotiate appropriate privacy protection on the connection.
* This will involve the use of a privacy extension such as [NNTP-TLS].)
*
* @access public
* @since ?
*/
//define('NET_NNTP_PROTOCOL_RESPONSECODE_ENCRYPTION_REQUIRED', 483);
/**
* 'Error in base64-encoding [RFC3548] of an argument' (Draft)
*
* @access public
*/
define('NET_NNTP_PROTOCOL_RESPONSECODE_BASE64_ENCODING_ERROR', 504);
// }}}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* c-hanging-comment-ender-p: nil
* End:
*/
?>
kevin123/backfill.php
Executable
+300
View File
@@ -0,0 +1,300 @@
<?php
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/groups.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/binaries.php");
/**
* Retrieves messages from usenet based on provided backfill-to date.
*/
class Backfill
{
/**
* Default constructor.
*/
function Backfill()
{
$this->n = "\n";
}
/**
* Update all active groups categories and descriptions.
*/
function backfillAllGroups($groupName='', $groupPost='', $backfillDate=null, $backfillPost=null)
{
$n = $this->n;
$groups = new Groups;
$res = false;
if ($groupName != '')
{
$grp = $groups->getByName($groupName);
if ($grp)
$res = array($grp);
}
else
{
$res = $groups->getActive();
}
if ($res)
{
$nntp = new Nntp();
if ($nntp->doNXFConnect()) {
$nntpc = new Nntp();
$nntpc->doConnect();
foreach($res as $groupArr)
{
$this->backfillGroup($nntp, $nntpc, $groupArr, $backfillDate, $backfillPost=$groupPost);
}
$nntp->doQuit();
$nntpc->doQuit();
} else {
echo "Failed to get NNTP connection.$n";
}
}
else
{
echo "No groups specified. Ensure groups are added to newznab's database for updating.$n";
}
}
/**
* Update a group back to a specified date.
*/
function backfillGroup($nntp, $nntpc, $groupArr, $backfillDate=null, $backfillPost=null)
{
$db = new DB();
$db->disableAutoCommit();
$binaries = new Binaries();
$n = $this->n;
$this->startGroup = microtime(true);
echo 'Processing '.$groupArr['name'].$n;
$data = $nntp->selectGroup($groupArr['name']);
if(PEAR::isError($data))
{
echo "Could not select group (bad name?): {$groupArr['name']}$n";
$db->rollback();
return;
}
$datat = $nntpc->selectGroup($groupArr['name']);
if(PEAR::isError($datat))
{
echo "Could not select group (bad name?): {$groupArr['name']}$n";
$db->rollback();
return;
}
if ($backfillPost) {
$targetpost = round($groupArr['first_record'] - $backfillPost);
} else {
if ($backfillDate)
$targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date
else
$targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target
}
if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0)
{
echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n";
$db->rollback();
return;
}
echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~";
echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400));
echo " days.".$n."Local first = ".$groupArr['first_record']." (";
echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400));
echo " days). Going to backfill $backfillPost posts, which is post $targetpost.$n";
if ($backfillPost)
{
echo " days). Going to backfill $backfillPost posts, which is post $targetpost.$n";
}
else { echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n"; }
if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish
{
echo "Nothing to do, we already have the target post.$n $n";
$db->commit(); //Not an error so commit and re-enable autocommitting
return "";
}
//get first and last part numbers from newsgroup
if($targetpost < $data['first'])
{
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n";
echo "Skipping Group $n";
$db->rollback();
return "";
}
//calculate total number of parts
$total = $groupArr['first_record'] - $targetpost;
$done = false;
//set first and last, moving the window by maxxMssgs
$last = $groupArr['first_record'] - 1;
$first = $last - $binaries->messagebuffer + 1; //set initial "chunk"
if($targetpost > $first) //just in case this is the last chunk we needed
$first = $targetpost;
while($done === false)
{
$binaries->startLoop = microtime(true);
echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n;
flush();
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
if (!$success)
{
$db->rollback();
return "";
}
$db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
$db->commit(false);
if($first==$targetpost)
$done = true;
else
{ //Keep going: set new last, new first, check for last chunk.
$last = $first - 1;
$first = $last - $binaries->messagebuffer + 1;
if($targetpost > $first)
$first = $targetpost;
}
}
$first_record_postdate = $this->postdate($nntp,$first,false);
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate
$db->commit();
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in $timeGroup seconds $n";
}
/**
* Returns single timestamp from a local article number.
*/
function postdate($nntp,$post,$debug=true)
{
$n = $this->n;
$attempts=0;
do
{
$msgs = $nntp->getOverview($post."-".$post,true,false);
if(PEAR::isError($msgs))
{
echo "Error {$msgs->code}: {$msgs->message}$n";
echo "Returning from postdate$n";
return "";
}
if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date']))
{
$success=false;
} else {
$date = $msgs[0]['Date'];
$success=true;
}
if($debug && $attempts > 0) echo "retried $attempts time(s)".$n;
$attempts++;
} while($attempts <= 3 && $success == false);
if (!$success) { return ""; }
if($debug) echo "DEBUG: postdate for post: $post came back $date (";
$date = strtotime($date);
if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n;
return $date;
}
/**
* Calculates the post number for a given number of days back in a group.
*/
function daytopost($nntp, $group, $days, $debug=true)
{
$n = $this->n;
$pddebug = false; //DEBUG every postdate call?!?!
if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n;
$data = $nntp->selectGroup($group);
if(PEAR::isError($data))
{
echo "Error {$data->code}: {$data->message}$n";
echo "Returning from daytopost$n";
return "";
}
$goaldate = date('U')-(86400*$days); //goaltimestamp
$totalnumberofarticles = $data['last'] - $data['first'];
$upperbound = $data['last'];
$lowerbound = $data['first'];
if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n";
if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); }
$firstDate = $this->postdate($nntp, $data['first'], $pddebug);
$lastDate = $this->postdate($nntp, $data['last'], $pddebug);
if ($goaldate < $firstDate)
{
echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n";
echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n";
return $data['first'];
}
elseif ($goaldate > $lastDate)
{
echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n";
echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n";
return "";
}
if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n";
$interval = floor(($upperbound - $lowerbound) * 0.5);
$dateofnextone = "";
$templowered = "";
if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n";
$dateofnextone = $lastDate;
while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up
{
$nskip = 1;
while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate)
{
$upperbound = $upperbound - $interval - ($nskip - 1);
if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n";
$nskip = $nskip * 2;
}
if(!$templowered)
{
$interval = ceil(($interval/2));
if($debug) echo "Set interval to $interval articles. $n";
}
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
$skip = 1;
while(!$dateofnextone)
{
$upperbound = $upperbound - $skip;
$skip = $skip * 2;
if($debug) echo "Getting next article date... $upperbound\n";
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
}
}
echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n";
return $upperbound;
}
/**
* Calculate the number of days from a timestamp.
*/
private function daysOld($timestamp)
{
return round((time()-$timestamp)/86400, 1);
}
/**
* Calculate the number of days from a date.
*/
private function dateToDays($backfillDate)
{
return floor(-($backfillDate - time())/(60*60*24));
}
}
kevin123/binaries.php
Executable
+856
View File
@@ -0,0 +1,856 @@
<?php
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/groups.php");
require_once(WWW_DIR."/lib/backfill.php");
require_once(WWW_DIR."/lib/Net_NNTP/NNTP/Client.php");
/**
* This class manages the downloading of binaries and parts from usenet, and the
* managing of data in the binaries and parts tables.
*/
class Binaries
{
const OPT_BLACKLIST = 1;
const OPT_WHITELIST = 2;
const BLACKLIST_FIELD_SUBJECT = 1;
const BLACKLIST_FIELD_FROM = 2;
const BLACKLIST_FIELD_MESSAGEID = 3;
/**
* Default constructor
*/
function Binaries()
{
$this->n = "\n";
$s = new Sites();
$site = $s->get();
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
$this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000;
$this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false;
$this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000;
$this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3;
$this->blackList = array(); //cache of our black/white list
$this->message = array();
}
/**
* Process headers and store in database for all active groups.
*/
function updateAllGroups()
{
$n = $this->n;
$groups = new Groups;
$res = $groups->getActive();
$s = new Sites();
echo $s->getLicense();
if ($res)
{
shuffle($res);
$alltime = microtime(true);
echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n;
$nntp = new Nntp();
if ($nntp->doConnect()) {
$pos = 0;
foreach($res as $groupArr)
{
$pos++;
echo 'Group '.$pos.' of '.sizeof($res).$n;
$this->message = array();
$this->updateGroup($nntp, $groupArr);
}
$nntp->doQuit();
echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n;
} else {
echo "Failed to get NNTP connection.$n";
}
}
else
{
echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n";
}
}
/**
* Process headers and store in database for a group.
*/
function updateGroup($nntp=null, $groupArr)
{
$blnDoDisconnect = false;
if ($nntp == null)
{
$nntp = new Nntp();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection.$n";
return;
}
$this->message = array();
$blnDoDisconnect = true;
}
$db = new DB();
$backfill = new Backfill();
$db->disableAutoCommit();
$n = $this->n;
$this->startGroup = microtime(true);
$this->startLoop = microtime(true);
echo 'Processing '.$groupArr['name'].$n;
// Connect to server
$data = $nntp->selectGroup($groupArr['name']);
if (PEAR::isError($data))
{
echo "Could not select group (bad name?): {$groupArr['name']}$n";
$db->rollback();
return;
}
//Attempt to repair any missing parts before grabbing new ones
$this->partRepair($nntp, $groupArr);
//Get first and last part numbers from newsgroup
$last = $grouplast = $data['last'];
// For new newsgroups - determine here how far you want to go back.
if ($groupArr['last_record'] == 0)
{
if ($this->NewGroupScanByDays)
{
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
if ($first == '')
{
echo "Skipping group: {$groupArr['name']}$n";
$db->rollback();
return;
}
}
else
{
if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan))
$first = $data['first'];
else
$first = $data['last'] - $this->NewGroupMsgsToScan;
}
$first_record_postdate = $backfill->postdate($nntp, $first, false);
$db->query(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
}
else
{
if ($data['last'] < $groupArr['last_record'])
{
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n;
$db->rollback();
return;
}
$first = $groupArr['last_record'] + 1;
}
// Generate postdates for first and last records, for those that upgraded
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0"))
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID']));
// Deactivate empty groups
if (($data['last'] - $data['first']) <= 5)
$db->query(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID']));
// Calculate total number of parts
$total = $grouplast - $first + 1;
// If total is bigger than 0 it means we have new parts in the newsgroup
if($total > 0)
{
echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n;
echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n;
if ($groupArr['last_record'] == 0)
echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n;
$done = false;
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
while ($done === false)
{
$this->startLoop = microtime(true);
if ($total > $this->messagebuffer)
{
if ($first + $this->messagebuffer > $grouplast)
$last = $grouplast;
else
$last = $first + $this->messagebuffer;
}
echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n;
flush();
//get headers from newsgroup
$lastId = $this->scan($nntp, $groupArr, $first, $last);
if ($lastId === false)
{
//scan failed - skip group
$db->rollback();
return;
}
$db->query(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($lastId), $groupArr['ID']));
$db->commit(false);
if ($last == $grouplast)
$done = true;
else
{
$last = $lastId;
$first = $last + 1;
}
}
$last_record_postdate = $backfill->postdate($nntp,$last,false);
$db->query(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in $timeGroup seconds $n $n";
}
else
{
echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n;
}
if ($blnDoDisconnect)
{
$nntp->doQuit();
}
/*
* Got through the updating of this group successfully
* so commit and re-enable the auto committing.
*/
$db->commit(true);
}
/**
* Download a range of usenet messages. Store binaries with subjects matching a
* specific pattern in the database.
*/
function scan($nntp, $groupArr, $first, $last, $type='update')
{
$db = new Db();
$n = $this->n;
$this->startHeaders = microtime(true);
if ($this->compressedHeaders)
{
$nntpc = new Nntp();
$nntpc->doNXFConnect();
$response = $nntpc->_sendCommand('XFEATURE COMPRESS GZIP');
if (PEAR::isError($response) || $response != 290)
{
$response2 = $nntpc->_sendCommand('XZVER');
if (PEAR::isError($response2) || $response2 != 412)
{
$msgs = $nntp->getOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
else
{
$msgs = $nntp->getXOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
}
else
{
$msgs = $nntp->getOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
}
else
$msgs = $nntp->getOverview($first."-".$last, true, false);
if (PEAR::isError($msgs) && ($msgs->code == 400 || $msgs->code == 503))
{
echo "NNTP connection timed out. Reconnecting...$n";
if (!$nntp->doConnect()) {
// TODO: What now?
echo "Failed to get NNTP connection.$n";
return;
}
$nntp->selectGroup($groupArr['name']);
if ($this->compressedHeaders)
{
$nntpc = new Nntp();
$nntpc->doNXFConnect();
$response = $nntpc->_sendCommand('XFEATURE COMPRESS GZIP');
if (PEAR::isError($response) || $response != 290)
{
$response2 = $nntpc->_sendCommand('XZVER');
if (PEAR::isError($response2) || $response2 != 412)
{
$msgs = $nntp->getOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
else
{
$msgs = $nntp->getXOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
}
else
{
$msgs = $nntp->getOverview($first."-".$last, true, false);
$nntpc->doQuit();
}
}
else
$msgs = $nntp->getOverview($first."-".$last, true, false);
}
$rangerequested = range($first, $last);
$msgsreceived = array();
$msgsblacklisted = array();
$msgsignored = array();
$msgsinserted = array();
$msgsnotinserted = array();
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
if(PEAR::isError($msgs))
{
echo "Error {$msgs->code}: {$msgs->message}$n";
echo "Skipping group$n";
return false;
}
$this->startUpdate = microtime(true);
if (is_array($msgs))
{
//loop headers, figure out parts
foreach($msgs AS $msg)
{
if (!isset($msg['Number']))
continue;
$msgsreceived[] = $msg['Number'];
$msgPart = $msgTotalParts = 0;
$pattern = '|\((\d+)[\/](\d+)\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i=0; $i<$matchcnt; $i++)
{
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue
{
$msgsignored[] = $msg['Number'];
continue;
}
//Filter binaries based on black/white list
if ($this->isBlackListed($msg, $groupArr['name']))
{
$msgsblacklisted[] = $msg['Number'];
continue;
}
if((int)$msgPart > 0 && (int)$msgTotalParts > 0)
{
$subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject'])));
if(!isset($this->message[$subject]))
{
$this->message[$subject] = $msg;
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
}
if((int)$msgPart > 0)
{
$this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']);
}
}
}
unset($msg);
unset($msgs);
$count = 0;
$updatecount = 0;
$partcount = 0;
$maxnum = $last;
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
if ($type != 'partrepair')
echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n";
if ($type == 'update' && sizeof($msgsreceived) == 0)
{
echo "Error: Server did not return any articles.$n";
echo "Skipping group$n";
return false;
}
if (sizeof($rangenotreceived) > 0) {
switch($type)
{
case 'backfill':
//don't add missing articles
break;
case 'partrepair':
case 'update':
default:
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
break;
}
echo "Server did not return ".count($rangenotreceived)." article(s).$n";
}
if(isset($this->message) && count($this->message))
{
$maxnum = $first;
//insert binaries and parts into database. when binary already exists; only insert new parts
foreach($this->message AS $subject => $data)
{
if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '')
{
$binaryHash = md5($subject.$data['From'].$groupArr['ID']);
$res = $db->queryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $db->escapeString($binaryHash)));
if(!$res)
{
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $db->escapeString($subject), $db->escapeString($data['From']), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash));
$binaryID = $db->queryInsert($sql);
$count++;
if ($count%500==0) echo "$count bin adds...";
}
else
{
$binaryID = $res["ID"];
$updatecount++;
if ($updatecount%500==0) echo "$updatecount bin updates...";
}
foreach($data['Parts'] AS $partdata)
{
$maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum;
$query = sprintf
(
"SELECT number FROM `parts` WHERE number = '%s' LIMIT 1",
$partdata['number']
);
$result = $db->query($query);
if (!count($result))
{
$partcount++;
$pidata = $db->queryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size'])), false);
if (!$pidata) {
$msgsnotinserted[] = $partdata['number'];
} else {
$msgsinserted[] = $partdata['number'];
}
}
}
}
}
//TODO: determine whether to add to missing articles if insert failed
if (sizeof($msgsnotinserted) > 0)
{
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n;
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
}
if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output
}
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
$timeLoop = number_format(microtime(true)-$this->startLoop, 2);
if ($type != 'partrepair')
{
echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.';
echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n";
}
unset($this->message);
unset($data);
return $maxnum;
}
else
{
echo "Error: Can't get parts from server (msgs not array) $n";
echo "Skipping group$n";
return false;
}
}
/**
* Go through all rows in partrepair table and see if theyve arrived on usenet yet.
*
* @param Nntp $nntp
* @param array $group
* @return bool
*/
private function partRepair($nntp, $group)
{
$db = new DB;
$parts = array();
$chunks = array();
$result = array();
$query = sprintf
(
"SELECT numberID FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC",
$group['ID']
);
$result = $db->query($query);
if (!count($result))
return false;
foreach ($result as $item)
$parts[] = $item['numberID'];
if (count($parts))
{
$matched = array();
printf("Repair: supposed to repair %s parts.%s", count($parts), $this->n);
foreach ($parts as $key => $item)
{
if (in_array(substr($item, 0, -2), $matched))
continue;
# when moving to php >= 5.3
#preg_filter(sprintf("~%s~", substr($item, 0, -3)), '$0', $parts);
$result = preg_grep(sprintf("~%s~", substr($item, 0, -2)), $parts);
if (count($result))
{
$matched[] = substr($item, 0, -2);
array_push($chunks, $result);
foreach ($result as $key => $val)
unset($parts[$key]);
}
}
$chunks = $this->getSuperUniqueArray($chunks);
if (!count($chunks))
{
printf("Repair: unable to extract parts for repair! Please report this is a bug, together with a dump of your partrepair table.\n", $this->n);
return false;
}
$repaired = 0;
foreach ($chunks as $chunk)
{
$start = current($chunk);
$end = end($chunk);
# TODO: if less than 3 chunks do 3 single calls to scan()
if (count($chunk) < 3) { }
$range = ($end - $start);
printf("Repair: + %s-%s (%s missing, %s articles overhead)%s", $start, $end, count($chunk), $range, $this->n);
$this->scan($nntp, $group, $start, $end, 'partrepair');
$query = sprintf
(
"SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC",
$group['ID'], implode(',', $chunk)
);
$result = $db->query($query);
foreach ($result as $item)
{
# TODO: rewrite.. stupid
if ($item['number'] == $item['numberID'])
{
#printf("Repair: %s repaired.%s", $item['ID'], $this->n);
$db->query(sprintf("DELETE FROM partrepair WHERE ID=%d LIMIT 1", $item['ID']));
$repaired++;
continue;
}
else
{
#printf("Repair: %s has not arrived yet or deleted.%s", $item['numberID'], $this->n);
$db->query(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d LIMIT 1", $item['ID']));
}
}
}
$db->query(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $group['ID']));
printf("Repair: repaired %s.%s", $repaired, $this->n);
printf("Repair: cleaned %s parts.%s", $db->getAffectedRows(), $this->n);
return true;
}
return false;
}
/**
* Insert a missing part to the database.
*/
private function addMissingParts($numbers, $groupID)
{
$db = new DB;
$added = false;
$insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES ";
foreach($numbers as $number)
{
if ($number > 0)
{
$added = true;
$insertStr .= sprintf("(%u, %d), ", $number, $groupID);
}
}
if ($added)
{
$insertStr = substr($insertStr, 0, -2);
$insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1";
return $db->queryInsert($insertStr, false);
}
return -1;
}
/**
* Return internally cached list of binary blacklist patterns.
*/
public function retrieveBlackList()
{
if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; }
$blackList = $this->getBlacklist(true);
$this->blackList = $blackList;
return $blackList;
}
/**
* Test if a message subject is blacklisted.
*/
public function isBlackListed($msg, $groupName)
{
$blackList = $this->retrieveBlackList();
$field = array();
if (isset($msg["Subject"]))
$field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"];
if (isset($msg["From"]))
$field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"];
if (isset($msg["Message-ID"]))
$field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"];
// if a white list is detected we now are required to
// only accept the entry if it matches at least 1 whitelist
// while a blacklist will over-ride all
$whitelist = array();
$matches_whitelist = false;
foreach ($blackList as $blist)
{
if (preg_match('/^'.$blist['groupname'].'$/i', $groupName))
{
//blacklist
if ($blist['optype'] == Binaries::OPT_BLACKLIST)
{
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
return true;
}
}
else if ($blist['optype'] == Binaries::OPT_WHITELIST)
{
$whitelist[] = $blist['regex'];
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
// Flag that we matched the white list
$matches_whitelist = true;
}
}
}
}
# We parsed entire matching list entries at this point.. now we need
# to handle the whitelist (if it was enabled)
if(count($whitelist) > 0 && !$matches_whitelist)
{
# We failed to match white list
return true;
}
return false;
}
/**
* Rawsearch. Perform a simple like match on binary subjects matching a pattern.
*/
public function search($search, $limit=1000, $excludedcats=array())
{
$db = new DB();
//
// if the query starts with a ^ it indicates the search is looking for items which start with the term
// still do the like match, but mandate that all items returned must start with the provided word
//
$words = explode(" ", $search);
$searchsql = "";
$intwordcount = 0;
if (count($words) > 0)
{
foreach ($words as $word)
{
//
// see if the first word had a caret, which indicates search must start with term
//
if ($intwordcount == 0 && (strpos($word, "^") === 0))
$searchsql.= sprintf(" and b.name like %s", $db->escapeString(substr($word, 1)."%"));
else
$searchsql.= sprintf(" and b.name like %s", $db->escapeString("%".$word."%"));
$intwordcount++;
}
}
$exccatlist = "";
if (count($excludedcats) > 0)
$exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") ";
$res = $db->query(sprintf("
SELECT b.*,
g.name AS group_name,
r.guid,
(SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum'
FROM binaries b
INNER JOIN groups g ON g.ID = b.groupID
LEFT OUTER JOIN releases r ON r.ID = b.releaseID
WHERE 1=1 %s %s order by DATE DESC LIMIT %d ",
$searchsql, $exccatlist, $limit));
return $res;
}
/**
* Get all binaries for a release.
*/
public function getForReleaseId($id)
{
$db = new DB();
return $db->query(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id));
}
/**
* Get a binary row.
*/
public function getById($id)
{
$db = new DB();
return $db->queryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id));
}
/**
* Get list of blacklists from database.
*/
public function getBlacklist($activeonly=true)
{
$db = new DB();
$where = "";
if ($activeonly)
$where = " where binaryblacklist.status = 1 ";
return $db->query("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex,
groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist
left outer JOIN groups ON groups.name = binaryblacklist.groupname
".$where."
ORDER BY coalesce(groupname,'zzz')");
}
/**
* Get a blacklist row from database.
*/
public function getBlacklistByID($id)
{
$db = new DB();
return $db->queryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id));
}
/**
* Delete a blacklist row from database.
*/
public function deleteBlacklist($id)
{
$db = new DB();
return $db->query(sprintf("delete from binaryblacklist where ID = %d", $id));
}
/**
* Update a blacklist row.
*/
public function updateBlacklist($regex)
{
$db = new DB();
$groupname = $regex["groupname"];
if ($groupname == "")
$groupname = "null";
else
{
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
$groupname = sprintf("%s", $db->escapeString($groupname));
}
$db->query(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $db->escapeString($regex["regex"]), $regex["status"], $db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"]));
}
/**
* Add a new blacklist row.
*/
public function addBlacklist($regex)
{
$db = new DB();
$groupname = $regex["groupname"];
if ($groupname == "")
$groupname = "null";
else
{
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
$groupname = sprintf("%s", $db->escapeString($groupname));
}
return $db->queryInsert(sprintf("insert into binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ",
$groupname, $db->escapeString($regex["regex"]), $regex["status"], $db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"]));
}
/**
* Add a new binary row and its associated parts.
*/
public function delete($id)
{
$db = new DB();
$db->query(sprintf("delete from parts where binaryID = %d", $id));
$db->query(sprintf("delete from binaries where ID = %d", $id));
}
# http://php.net/manual/en/function.array-unique.php#97285
public function getSuperUniqueArray($array)
{
$result = array_map("unserialize", array_unique(array_map("serialize", $array)));
foreach ($result as $key => $value)
{
if (is_array($value))
$result[$key] = $this->getSuperUniqueArray($value);
}
return $result;
}
}
kevin123/nntp.php
Executable
+629
View File
@@ -0,0 +1,629 @@
<?php
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/Net_NNTP/NNTP/Client.php");
/**
* This class extends the standard PEAR NNTP class with some extra features.
*/
class Nntp extends Net_NNTP_Client
{
public $XFCompression = false;
/**
* Start an NNTP connection. With Xfeature compression.
*/
function doConnect($attempt=1)
{
if ($this->_isConnected()) {
return true;
}
// Attempt to connect up to 5 times before giving up.
$maxAttempts = 5;
$connected = true;
$s = new Sites();
$site = $s->get();
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
$enc = false;
if (defined("NNTP_SSLENABLED") && NNTP_SSLENABLED == true)
$enc = 'ssl';
$ret = $this->connect(NNTP_SERVER, $enc, NNTP_PORT);
if(PEAR::isError($ret))
{
$err = "Cannot connect to server ".NNTP_SERVER.(!$enc?" (nonssl) ":"(ssl) ").": ". $ret->getMessage();
echo $err;
$connected = false;
}
if(!defined(NNTP_USERNAME) && NNTP_USERNAME!="" )
{
$ret2 = $this->authenticate(NNTP_USERNAME, NNTP_PASSWORD);
if(PEAR::isError($ret2))
{
$err = "Cannot authenticate to server ".NNTP_SERVER.(!$enc?" (nonssl) ":" (ssl) ")." - ". NNTP_USERNAME." (".$ret2->getMessage().")";
echo $err;
$connected = false;
}
}
if ($this->compressedHeaders)
{
$response = $this->_sendCommand('XFEATURE COMPRESS GZIP');
if (PEAR::isError($response) || $response != 290)
{
//echo "NNTP: XFeature not supported.\n";
}
else
{
$this->enableXFCompression();
}
}
if ($attempt < $maxAttempts && !$connected) {
sleep(5);
$connected = $this->doConnect($attempt+1);
}
if (!$connected && $attempt == 1) {
echo "\nTried to connect ".$maxAttempts." times, but couldn't. Check your settings and connection.\n";
}
return $connected;
}
/**
* Start an NNTP connection. Without Xfeature compression.
*/
function doNXFConnect($attempt=1)
{
if ($this->_isConnected()) {
return true;
}
// Attempt to connect up to 5 times before giving up.
$maxAttempts = 5;
$connected = true;
$enc = false;
if (defined("NNTP_SSLENABLED") && NNTP_SSLENABLED == true)
$enc = 'ssl';
$ret = $this->connect(NNTP_SERVER, $enc, NNTP_PORT);
if(PEAR::isError($ret))
{
$err = "Cannot connect to server ".NNTP_SERVER.(!$enc?" (nonssl) ":"(ssl) ").": ". $ret->getMessage();
echo $err;
$connected = false;
}
if(!defined(NNTP_USERNAME) && NNTP_USERNAME!="" )
{
$ret2 = $this->authenticate(NNTP_USERNAME, NNTP_PASSWORD);
if(PEAR::isError($ret2))
{
$err = "Cannot authenticate to server ".NNTP_SERVER.(!$enc?" (nonssl) ":" (ssl) ")." - ". NNTP_USERNAME." (".$ret2->getMessage().")";
echo $err;
$connected = false;
}
}
if ($attempt < $maxAttempts && !$connected) {
sleep(5);
$connected = $this->doConnect($attempt+1);
}
if (!$connected && $attempt == 1) {
echo "\nTried to connect ".$maxAttempts." times, but couldn't. Check your settings and connection.\n";
}
return $connected;
}
/**
* End an NNTP connection.
*/
function doQuit()
{
$this->quit();
}
/**
* Retrieve an NNTP message and decode it.
*/
function getMessage($groupname, $partMsgId)
{
$summary = $this->selectGroup($groupname);
$message = $dec = '';
if (PEAR::isError($summary))
{
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
return false;
}
$body = $this->getBody('<'.$partMsgId.'>', true);
if (PEAR::isError($body))
{
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $partMsgId, $groupname, $body->getMessage());
return false;
}
$message = $this->decodeYenc($body);
if (!$message)
{
//
// Yenc decode failed
//
return false;
}
return $message;
}
/**
* Retrieve a series of NNTP messages and decode them.
*/
function getMessages($groupname, $msgIds)
{
$summary = $this->selectGroup($groupname);
$message = $dec = '';
if (PEAR::isError($summary))
{
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
return false;
}
foreach($msgIds as $msgId)
{
$messageID = '<'.$msgId.'>';
$body = $this->getBody($messageID, true);
if (PEAR::isError($body))
{
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $messageID, $groupname, $body->getMessage());
return false;
}
$dec = $this->decodeYenc($body);
if (!$dec)
{
//
// Yenc decode failed
//
return false;
}
$message .= $dec;
}
return $message;
}
/**
* Retrieve all NNTP messages associated with a binaries.ID
*/
function getBinary($binaryId, $isNfo=false)
{
$db = new DB();
$bin = new Binaries();
$binary = $bin->getById($binaryId);
if (!$binary)
{
printf("NntpPrc: Unable to locate binary: %s\n", $binaryId);
return false;
}
$summary = $this->selectGroup($binary['groupname']);
$message = $dec = '';
if (PEAR::isError($summary))
{
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
return false;
}
$resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId));
// Dont attempt to download nfos which are larger than one part.
if (sizeof($resparts) > 1 && $isNfo === true)
{
echo "NntpPrc : Error Nfo is too large... skipping.\n";
return false;
}
foreach($resparts as $part)
{
$messageID = '<'.$part['messageID'].'>';
$body = $this->getBody($messageID, true);
if (PEAR::isError($body))
{
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $part['messageID'], $binary['groupname'], $body->getMessage());
return false;
}
$dec = $this->decodeYenc($body);
if (!$dec)
{
printf("NntpPrc: Unable to decode body of binary: %s\n", $binaryId);
// Yenc decode failed
return false;
}
$message .= $dec;
}
return $message;
}
/**
* Decode a yenc encoded string.
*/
function decodeYenc($yencodedvar)
{
$input = array();
preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $yencodedvar, $input);
if (isset($input[1]))
{
$ret = "";
$input = trim(preg_replace("/\r\n/im", "", preg_replace("/(^=yend.*)/im", "", preg_replace("/(^=ypart.*\\r\\n)/im", "", preg_replace("/(^=ybegin.*\\r\\n)/im", "", $input[1], 1), 1), 1)));
for( $chr = 0; $chr < strlen($input) ; $chr++)
$ret .= ($input[$chr] != "=" ? chr(ord($input[$chr]) - 42) : chr((ord($input[++$chr]) - 64) - 42));
return $ret;
}
return false;
}
/**
* Get XZVER for a range of NNTP messages.
*/
function getXOverview($range, $_names = true, $_forceNames = true)
{
$overview = $this->cmdXOver($range);
if (PEAR::isError($overview)) {
return $overview; }
// Use field names from overview format as keys?
if ($_names)
{
// Already cached?
if (is_null($this->_overviewFormatCache)) {
// Fetch overview format
$format = $this->getOverviewFormat($_forceNames, true);
if (PEAR::isError($format)){
return $format;
}
// Prepend 'Number' field
$format = array_merge(array('Number' => false), $format);
// Cache format
$this->_overviewFormatCache = $format;
}
else
{
$format = $this->_overviewFormatCache;
}
// Loop through all articles
foreach ($overview as $key => $article)
{
if (sizeof($format) == sizeof($article))
{
//Replace overview using $format as keys, $article as values
$overview[$key] = array_combine(array_keys($format), $article);
// If article prefixed by field name, remove it
foreach($format as $fkey=>$fval)
{
if ($fval === true)
{
$overview[$key][$fkey] = trim(str_replace($fkey.':', '', $overview[$key][$fkey]));
}
}
}
}
}
switch (true)
{
// Expect one article
case is_null($range);
case is_int($range);
case is_string($range) && ctype_digit($range):
case is_string($range) && substr($range, 0, 1) == '<' && substr($range, -1, 1) == '>':
if (count($overview) == 0) {
return false;
} else {
return reset($overview);
}
break;
// Expect multiple articles
default:
return $overview;
}
}
// }}}
// {{{ cmdXZver()
/*
* Based on code from http://wonko.com/software/yenc/, but
* simplified because XZVER and the likes don't implement
* yenc properly
*/
private function yencDecode($string, $destination = "") {
$encoded = array();
$header = array();
$decoded = '';
# Extract the yEnc string itself
preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $string, $encoded);
$encoded = $encoded[1];
# Extract the filesize and filename from the yEnc header
preg_match("/^=ybegin.*size=([^ $]+).*name=([^\\r\\n]+)/im", $encoded, $header);
$filesize = $header[1];
$filename = $header[2];
# Remove the header and footer from the string before parsing it.
$encoded = preg_replace("/(^=ybegin.*\\r\\n)/im", "", $encoded, 1);
$encoded = preg_replace("/(^=yend.*)/im", "", $encoded, 1);
# Remove linebreaks and whitespace from the string
$encoded = trim(str_replace("\r\n", "", $encoded));
// Decode
$strLength = strlen($encoded);
for($i = 0; $i < $strLength; $i++) {
$c = $encoded[$i];
if ($c == '=') {
$i++;
$decoded .= chr((ord($encoded[$i]) - 64) - 42);
} else {
$decoded .= chr(ord($c) - 42);
}
}
// Make sure the decoded filesize is the same as the size specified in the header.
if (strlen($decoded) != $filesize) {
throw new Exception("Filesize in yEnc header en filesize found do not match up");
}
return $decoded;
}
/**
* Fetch message header from message number $first until $last
* The format of the returned array is:
* $messages[message_id][header_name]
* @param optional string $range articles to fetch
* @return mixed (array) nested array of message and there headers on success or (object) pear_error on failure
* @access protected
*/
function cmdXZver($range = null)
{
if (is_null($range))
$command = 'XZVER';
else
$command = 'XZVER ' . $range;
$response = $this->_sendCommand($command);
switch ($response) {
case 224: // RFC2980: 'Overview information follows'
$data = $this->_getCompressedResponse();
foreach ($data as $key => $value)
$data[$key] = explode("\t", trim($value));
return $data;
break;
case 412: // RFC2980: 'No news group current selected'
return $this->throwError('No news group current selected', $response, $this->_currentStatusResponse());
break;
case 420: // RFC2980: 'No article(s) selected'
return $this->throwError('No article(s) selected', $response, $this->_currentStatusResponse());
break;
case 502: // RFC2980: 'no permission'
return $this->throwError('No permission', $response, $this->_currentStatusResponse());
break;
case 500: // RFC2980: 'unknown command'
$this->throwError("XZver not supported ({$this->_currentStatusResponse()})", $response);
break;
default:
return $this->_handleUnexpectedResponse($response);
}
}
/**
* Retrieve blob
* Get data and assume we do not hit any blindspots
* @return mixed (array) text response on success or (object) pear_error on failure
* @access private
*/
function _getCompressedResponse()
{
$data = array();
// We can have two kinds of compressed support:
// - yEnc encoding
// - Just a gzip drop
// We try to autodetect which one this uses
$line = @fread($this->_socket, 1024);
if (substr($line, 0, 7) == '=ybegin') {
$data = $this->_getTextResponse();
$data = $line . "\r\n" . implode("", $data);
$data = $this->yencDecode($data);
$data = explode("\r\n", gzinflate($data));
return $data;
}
// We cannot use blocked I/O on this one
$streamMetadata = stream_get_meta_data($this->_socket);
stream_set_blocking($this->_socket, false);
// Continue until connection is lost or we don't receive any data anymore
$tries = 0;
$uncompressed = '';
while (!feof($this->_socket)) {
# Retrieve and append up to 32k characters from the server
$received = @fread($this->_socket, 32768);
if (strlen($received) == 0) {
$tries++;
# Try decompression
$uncompressed = @gzuncompress($line);
if (($uncompressed !== false) || ($tries > 500)) {
break;
}
if ($tries % 50 == 0) {
}
}
# an error occured
if ($received === false) {
@fclose($this->_socket);
$this->_socket = false;
}
$line .= $received;
}
# and set the stream to its original blocked(?) value
stream_set_blocking($this->_socket, $streamMetadata['blocked']);
$data = explode("\r\n", $uncompressed);
$dataCount = count($data);
# Gzipped compress includes the "." and linefeed in the compressed stream, skip those.
if ($dataCount >= 2) {
if (($data[($dataCount - 2)] == ".") && (empty($data[($dataCount - 1)]))) {
array_pop($data);
array_pop($data);
}
$data = array_filter($data);
}
return $data;
}
/**
* Enable XFeature compression support for the current connection.
*/
function enableXFCompression()
{
$response = $this->_sendCommand('XFEATURE COMPRESS GZIP');
if (PEAR::isError($response) || $response != 290) {
echo "Xfeature compression not supported!\n";
return false;
}
$this->XFCompression = true;
echo "XFeature compression enabled\n";
return true;
}
/**
* Override to intercept any Xfeature compressed responses.
*/
function _getTextResponse()
{
if ($this->XFCompression && isset($this->_currentStatusResponse[1])
&& stripos($this->_currentStatusResponse[1], 'COMPRESS=GZIP') !== false)
{
return $this->_getXFCompressedTextResponse();
}
return parent::_getTextResponse();
}
function _getXFCompressedTextResponse()
{
$tries = 0;
$bytesreceived = 0;
$totalbytesreceived = 0;
$completed = false;
$data = null;
//build binary array that represents zero results basically a compressed empty string terminated with .(period) char(13) char(10)
$emptyreturnend = chr(0x03).chr(0x00).chr(0x00).chr(0x00).chr(0x00).chr(0x01).chr(0x2e).chr(0x0d).chr(0x0a);
$emptyreturn = chr(0x78).chr(0x9C).$emptyreturnend;
$emptyreturn2 = chr(0x78).chr(0x01).$emptyreturnend;
$emptyreturn3 = chr(0x78).chr(0x5e).$emptyreturnend;
$emptyreturn4 = chr(0x78).chr(0xda).$emptyreturnend;
while (!feof($this->_socket))
{
$completed = false;
//get data from the stream
$buffer = fgets($this->_socket);
//get byte count and update total bytes
$bytesreceived = strlen($buffer);
//if we got no bytes at all try one more time to pull data.
if ($bytesreceived == 0)
{
$buffer = fgets($this->_socket);
}
//get any socket error codes
$errorcode = socket_last_error();
//if the buffer is zero its zero...
if ($bytesreceived === 0)
return $this->throwError('No data returned.', 1000);
//did we have any socket errors?
if ($errorcode === 0)
{
//append buffer to final data object
$data .= $buffer;
$totalbytesreceived = $totalbytesreceived+$bytesreceived;
//output byte count in real time once we have 1MB of data
if ($totalbytesreceived > 10240)
if ($totalbytesreceived%128 == 0)
{
echo "bytes recived: ";
echo $totalbytesreceived;
echo "\r";
}
//check to see if we have the magic terminator on the byte stream
$b1 = null;
if ($bytesreceived > 2)
if (ord($buffer[$bytesreceived-3]) == 0x2e && ord($buffer[$bytesreceived-2]) == 0x0d && ord($buffer[$bytesreceived-1]) == 0x0a)//substr($buffer,-3) == ".\r\n"
{
//check to see if the returned binary string is 11 bytes long generally and indcator
//of an compressed empty string probably don't need this check
if ($totalbytesreceived==11)
{
//compare the data to the empty string if the data is a compressed empty string
//throw an error else return the data
if (($data === $emptyreturn)||($data === $emptyreturn2)||($data === $emptyreturn3)||($data === $emptyreturn4))
{
echo "empty gzip stream\n";
return $this->throwError('No data returned.', 1000);
}
}
else
{
echo "\n";
$completed = true;
}
}
}
else
{
echo "failed to read from socket\n";
return $this->throwError('Failed to read line from socket.', 1000);
}
if ($completed)
{
//check to see if the header is valid for a gzip stream
if(ord($data[0]) == 0x78 && in_array(ord($data[1]),array(0x01,0x5e,0x9c,0xda)))
{
$decomp = @gzuncompress(mb_substr ( $data , 0 ,-3, '8bit' ));
}
else
{
echo "Invalid header on gzip stream.\n";
return $this->throwError('Invalid gzip stream.', 1000);
}
if ($decomp != false)
{
$decomp = explode("\r\n", trim($decomp));
return $decomp;
}
else
{
$tries++;
echo "Decompression Failed Retry Number: $tries \n";
}
}
}
//throw an error if we get out of the loop
if (!feof($this->_socket))
{
return "Error: unexpected fgets() fail\n";
}
return $this->throwError('Decompression Failed, connection closed.', 1000);
}
}
scripts/update_svn.sh
+1 -1
View File
@@ -19,7 +19,7 @@ cd $NEWZPATH"/misc/update_scripts"
php5 update_database_version.php
#purge smarty cache
rm /www/lib/smarty/templates_c/*
rm -v /www/lib/smarty/templates_c/*
echo " "