mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 00:01:21 +00:00
removed kevin123's compression mod, added nice option
This commit is contained in:
jonnyboy
+8
-6
@@ -2,7 +2,7 @@
|
||||
|
||||
require(dirname(__FILE__)."/config.php");
|
||||
require(WWW_DIR.'/lib/postprocess.php');
|
||||
$version="0.1r718";
|
||||
$version="0.1r719";
|
||||
|
||||
$db = new DB();
|
||||
|
||||
@@ -38,7 +38,7 @@ $_DB_PASSWORD = escapeshellarg(getenv('DB_PASSWORD'));
|
||||
$_DB_PASS = getenv('DB_PASSWORD');
|
||||
$_mysql = getenv('MYSQL');
|
||||
$_nice = getenv('NICE');
|
||||
$_php = $_nice." -n19 ".getenv('PHP');
|
||||
$_php = $_nice." -n${array['NICENESS']} ".getenv('PHP');
|
||||
$_tmux = getenv('TMUXCMD');
|
||||
$_sed = getenv('SED');
|
||||
$_tee = getenv('TEE');
|
||||
@@ -1178,10 +1178,12 @@ while( $i > 0 )
|
||||
}
|
||||
|
||||
//set command for running backfill
|
||||
if (( $array['KEVIN_SAFER'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
|
||||
$log = writelog($panes0[3]);
|
||||
$_backfill_cmd = "cd $_bin && $_php safer_backfill_parts.php 2>&1 $log";
|
||||
} elseif ( $array['BACKFILL_THREADS'] == "true" ) {
|
||||
//if (( $array['KEVIN_SAFER'] == "true" ) && ( $optimize_safe_to_run != "true" )) {
|
||||
//if ( $optimize_safe_to_run != "true" ) {
|
||||
//$log = writelog($panes0[3]);
|
||||
//$_backfill_cmd = "cd $_bin && $_php safer_backfill_parts.php 2>&1 $log";
|
||||
//} else
|
||||
if ( $array['BACKFILL_THREADS'] == "true" ) {
|
||||
$log = writelog($panes0[3]);
|
||||
$_backfill_cmd = "cd $_bin && $_php backfill_threaded.php 2>&1 $log && $mysql_command_1 2>&1 $log";
|
||||
} else {
|
||||
|
||||
@@ -12,6 +12,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
#if this is set to false, the script will run 1 loop and terminate
|
||||
export RUNNING="true"
|
||||
|
||||
#these scripts set the 'nice'ness of each script, default is 19, the lowest, the highest is -20
|
||||
#anything between -1 and -20 require root/sudo to run
|
||||
export NICENESS="19"
|
||||
|
||||
############################################################
|
||||
|
||||
#Set paths
|
||||
@@ -150,22 +154,6 @@ export MAXDAYS="210"
|
||||
|
||||
############################################################
|
||||
|
||||
#use kevin123's safer_backfill_parts.php instead of normal backfill or backfill threaded
|
||||
#this is the script I use, it does 1 group at a time from z to a (wanted to start with tv groups first) 100k parts,
|
||||
#then the script stops (once per loop), if your first_record_postdate on the group is 2012-06-24
|
||||
#it will be skipped (target reached). When that group is done, it will do another ( again from z to a).
|
||||
#this does not use increment, it works by the date set below
|
||||
#you also need to enable kevin's compression mod, those files are needed and you still need to enable BACKFILL
|
||||
export KEVIN_SAFER="false"
|
||||
|
||||
#set the date to go back to, must be in the format of YYYY-MM-DD, like 2012-06-24
|
||||
export KEVIN_DATE="2012-06-24"
|
||||
|
||||
#set the number of articles/headers to download at one time
|
||||
export KEVIN_PARTS="100000"
|
||||
|
||||
############################################################
|
||||
|
||||
#Set the path to the nzb dump you downloaded from torrents, this is the path to bulk files folder of nzbs
|
||||
#this does not recurse through subfolders, unless you set NZB_THREADS to true
|
||||
export NZBS="/path/to/nzbs"
|
||||
@@ -369,13 +357,6 @@ export POWERLINE="false"
|
||||
|
||||
############################################################
|
||||
|
||||
#use kevin123's compression mod true/false
|
||||
#this will copy kevin123's compression mod to the correct location when you update_svn.sh or fix_files.sh
|
||||
#this ensures that kevin123's files are present after updating svn
|
||||
export KEVINS_COMP="false"
|
||||
|
||||
############################################################
|
||||
|
||||
#set your LANG to which ever you like, only effects these scripts
|
||||
export LANG="en_US.UTF-8"
|
||||
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
+-----------------------------------------------------------------------+
|
||||
| |
|
||||
| W3C® SOFTWARE NOTICE AND LICENSE |
|
||||
| http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 |
|
||||
| |
|
||||
| This work (and included software, documentation such as READMEs, |
|
||||
| or other related items) is being provided by the copyright holders |
|
||||
| under the following license. By obtaining, using and/or copying |
|
||||
| this work, you (the licensee) agree that you have read, understood, |
|
||||
| and will comply with the following terms and conditions. |
|
||||
| |
|
||||
| Permission to copy, modify, and distribute this software and its |
|
||||
| documentation, with or without modification, for any purpose and |
|
||||
| without fee or royalty is hereby granted, provided that you include |
|
||||
| the following on ALL copies of the software and documentation or |
|
||||
| portions thereof, including modifications: |
|
||||
| |
|
||||
| 1. The full text of this NOTICE in a location viewable to users |
|
||||
| of the redistributed or derivative work. |
|
||||
| |
|
||||
| 2. Any pre-existing intellectual property disclaimers, notices, |
|
||||
| or terms and conditions. If none exist, the W3C Software Short |
|
||||
| Notice should be included (hypertext is preferred, text is |
|
||||
| permitted) within the body of any redistributed or derivative |
|
||||
| code. |
|
||||
| |
|
||||
| 3. Notice of any changes or modifications to the files, including |
|
||||
| the date changes were made. (We recommend you provide URIs to |
|
||||
| the location from which the code is derived.) |
|
||||
| |
|
||||
| THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT |
|
||||
| HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, |
|
||||
| INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR |
|
||||
| FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE |
|
||||
| OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, |
|
||||
| COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. |
|
||||
| |
|
||||
| COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, |
|
||||
| SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE |
|
||||
| SOFTWARE OR DOCUMENTATION. |
|
||||
| |
|
||||
| The name and trademarks of copyright holders may NOT be used in |
|
||||
| advertising or publicity pertaining to the software without |
|
||||
| specific, written prior permission. Title to copyright in this |
|
||||
| software and any associated documentation will at all times |
|
||||
| remain with copyright holders. |
|
||||
| |
|
||||
+-----------------------------------------------------------------------+
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,509 +0,0 @@
|
||||
<?php
|
||||
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 foldmethod=marker: */
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* PHP versions 4 and 5
|
||||
*
|
||||
* <pre>
|
||||
* +-----------------------------------------------------------------------+
|
||||
* | |
|
||||
* | W3C® SOFTWARE NOTICE AND LICENSE |
|
||||
* | http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 |
|
||||
* | |
|
||||
* | This work (and included software, documentation such as READMEs, |
|
||||
* | or other related items) is being provided by the copyright holders |
|
||||
* | under the following license. By obtaining, using and/or copying |
|
||||
* | this work, you (the licensee) agree that you have read, understood, |
|
||||
* | and will comply with the following terms and conditions. |
|
||||
* | |
|
||||
* | Permission to copy, modify, and distribute this software and its |
|
||||
* | documentation, with or without modification, for any purpose and |
|
||||
* | without fee or royalty is hereby granted, provided that you include |
|
||||
* | the following on ALL copies of the software and documentation or |
|
||||
* | portions thereof, including modifications: |
|
||||
* | |
|
||||
* | 1. The full text of this NOTICE in a location viewable to users |
|
||||
* | of the redistributed or derivative work. |
|
||||
* | |
|
||||
* | 2. Any pre-existing intellectual property disclaimers, notices, |
|
||||
* | or terms and conditions. If none exist, the W3C Software Short |
|
||||
* | Notice should be included (hypertext is preferred, text is |
|
||||
* | permitted) within the body of any redistributed or derivative |
|
||||
* | code. |
|
||||
* | |
|
||||
* | 3. Notice of any changes or modifications to the files, including |
|
||||
* | the date changes were made. (We recommend you provide URIs to |
|
||||
* | the location from which the code is derived.) |
|
||||
* | |
|
||||
* | THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT |
|
||||
* | HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, |
|
||||
* | INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR |
|
||||
* | FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE |
|
||||
* | OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, |
|
||||
* | COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. |
|
||||
* | |
|
||||
* | COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, |
|
||||
* | SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE |
|
||||
* | SOFTWARE OR DOCUMENTATION. |
|
||||
* | |
|
||||
* | The name and trademarks of copyright holders may NOT be used in |
|
||||
* | advertising or publicity pertaining to the software without |
|
||||
* | specific, written prior permission. Title to copyright in this |
|
||||
* | software and any associated documentation will at all times |
|
||||
* | remain with copyright holders. |
|
||||
* | |
|
||||
* +-----------------------------------------------------------------------+
|
||||
* </pre>
|
||||
*
|
||||
* @category Net
|
||||
* @package Net_NNTP
|
||||
* @author Heino H. Gehlsen <heino@gehlsen.dk>
|
||||
* @copyright 2002-2011 Heino H. Gehlsen <heino@gehlsen.dk>. All Rights Reserved.
|
||||
* @license http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 W3C® SOFTWARE NOTICE AND LICENSE
|
||||
* @version SVN: $Id: Responsecode.php 306619 2010-12-24 12:16:07Z heino $
|
||||
* @link http://pear.php.net/package/Net_NNTP
|
||||
* @see
|
||||
* @since File available since release 1.3.0
|
||||
*/
|
||||
|
||||
|
||||
|
||||
// {{{ Constants: Connection
|
||||
|
||||
/**
|
||||
* 'Server ready - posting allowed' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_READY_POSTING_ALLOWED', 200);
|
||||
|
||||
/**
|
||||
* 'Server ready - no posting allowed' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_READY_POSTING_PROHIBITED', 201);
|
||||
|
||||
|
||||
/**
|
||||
* 'Closing connection - goodbye!' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
* @since ?
|
||||
*/
|
||||
//define('NET_NNTP_PROTOCOL_RESPONSECODE_DISCONNECTING_REQUESTED', 205); ///// goodbye
|
||||
|
||||
/**
|
||||
* 'Service discontinued' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
* @since ?
|
||||
*/
|
||||
//define('NET_NNTP_PROTOCOL_RESPONSECODE_DISCONNECTING_FORCED', 400); ///// unavailable / discontinued
|
||||
|
||||
|
||||
/**
|
||||
* 'Slave status noted' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_SLAVE_RECOGNIZED', 202);
|
||||
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Common errors
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Command not recognized' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_UNKNOWN_COMMAND', 500);
|
||||
|
||||
/**
|
||||
* 'Command syntax error' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_SYNTAX_ERROR', 501);
|
||||
|
||||
/**
|
||||
* 'Access restriction or permission denied' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NOT_PERMITTED', 502);
|
||||
|
||||
/**
|
||||
* 'Program fault - command not performed' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NOT_SUPPORTED', 503);
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Group selection
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Group selected' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_GROUP_SELECTED', 211);
|
||||
|
||||
/**
|
||||
* 'No such news group' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_GROUP', 411);
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Article retrieval
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Article retrieved - head and body follow' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_ARTICLE_FOLLOWS', 220);
|
||||
|
||||
/**
|
||||
* 'Article retrieved - head follows' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_HEAD_FOLLOWS', 221);
|
||||
|
||||
/**
|
||||
* 'Article retrieved - body follows' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_BODY_FOLLOWS', 222);
|
||||
|
||||
/**
|
||||
* 'Article retrieved - request text separately' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_ARTICLE_SELECTED', 223);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'No newsgroup has been selected' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_GROUP_SELECTED', 412);
|
||||
|
||||
|
||||
/**
|
||||
* 'No current article has been selected' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_ARTICLE_SELECTED', 420);
|
||||
|
||||
/**
|
||||
* 'No next article in this group' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_NEXT_ARTICLE', 421);
|
||||
|
||||
/**
|
||||
* 'No previous article in this group' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_PREVIOUS_ARTICLE', 422);
|
||||
|
||||
|
||||
/**
|
||||
* 'No such article number in this group' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_ARTICLE_NUMBER', 423);
|
||||
|
||||
/**
|
||||
* 'No such article found' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NO_SUCH_ARTICLE_ID', 430);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Transferring
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Send article to be transferred' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_SEND', 335);
|
||||
|
||||
/**
|
||||
* 'Article transferred ok' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_SUCCESS', 235);
|
||||
|
||||
/**
|
||||
* 'Article not wanted - do not send it' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_UNWANTED', 435);
|
||||
|
||||
/**
|
||||
* 'Transfer failed - try again later' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_FAILURE', 436);
|
||||
|
||||
/**
|
||||
* 'Article rejected - do not try again' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_TRANSFER_REJECTED', 437);
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Posting
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Send article to be posted' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_SEND', 340);
|
||||
|
||||
/**
|
||||
* 'Article posted ok' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_SUCCESS', 240);
|
||||
|
||||
/**
|
||||
* 'Posting not allowed' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_PROHIBITED', 440);
|
||||
|
||||
/**
|
||||
* 'Posting failed' (RFC977)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_POSTING_FAILURE', 441);
|
||||
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Authorization
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Authorization required for this command' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_REQUIRED', 450);
|
||||
|
||||
/**
|
||||
* 'Continue with authorization sequence' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_CONTINUE', 350);
|
||||
|
||||
/**
|
||||
* 'Authorization accepted' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_ACCEPTED', 250);
|
||||
|
||||
/**
|
||||
* 'Authorization rejected' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHORIZATION_REJECTED', 452);
|
||||
|
||||
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Authentication
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Authentication required' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_REQUIRED', 480);
|
||||
|
||||
/**
|
||||
* 'More authentication information required' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_CONTINUE', 381);
|
||||
|
||||
/**
|
||||
* 'Authentication accepted' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_ACCEPTED', 281);
|
||||
|
||||
/**
|
||||
* 'Authentication rejected' (RFC2980)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_AUTHENTICATION_REJECTED', 482);
|
||||
|
||||
|
||||
// }}}
|
||||
// {{{ Constants: Misc
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 'Help text follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_HELP_FOLLOWS', 100);
|
||||
|
||||
/**
|
||||
* 'Capabilities list follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_CAPABILITIES_FOLLOW', 101);
|
||||
|
||||
/**
|
||||
* 'Server date and time' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_SERVER_DATE', 111);
|
||||
|
||||
/**
|
||||
* 'Information follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_GROUPS_FOLLOW', 215);
|
||||
|
||||
/**
|
||||
* 'Overview information follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_OVERVIEW_FOLLOWS', 224);
|
||||
|
||||
/**
|
||||
* 'Headers follow' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_HEADERS_FOLLOW', 225);
|
||||
|
||||
/**
|
||||
* 'List of new articles follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NEW_ARTICLES_FOLLOW', 230);
|
||||
|
||||
/**
|
||||
* 'List of new newsgroups follows' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_NEW_GROUPS_FOLLOW', 231);
|
||||
|
||||
/**
|
||||
* 'The server is in the wrong mode; the indicated capability should be used to change the mode' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_WRONG_MODE', 401);
|
||||
|
||||
/**
|
||||
* 'Internal fault or problem preventing action being taken' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_INTERNAL_FAULT', 403);
|
||||
|
||||
/**
|
||||
* 'Command unavailable until suitable privacy has been arranged' (Draft)
|
||||
*
|
||||
* (the client must negotiate appropriate privacy protection on the connection.
|
||||
* This will involve the use of a privacy extension such as [NNTP-TLS].)
|
||||
*
|
||||
* @access public
|
||||
* @since ?
|
||||
*/
|
||||
//define('NET_NNTP_PROTOCOL_RESPONSECODE_ENCRYPTION_REQUIRED', 483);
|
||||
|
||||
/**
|
||||
* 'Error in base64-encoding [RFC3548] of an argument' (Draft)
|
||||
*
|
||||
* @access public
|
||||
*/
|
||||
define('NET_NNTP_PROTOCOL_RESPONSECODE_BASE64_ENCODING_ERROR', 504);
|
||||
|
||||
// }}}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* c-hanging-comment-ender-p: nil
|
||||
* End:
|
||||
*/
|
||||
|
||||
?>
|
||||
@@ -1,437 +0,0 @@
|
||||
<?php
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/nntp.php");
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
|
||||
/**
|
||||
* Retrieves messages from usenet based on provided backfill-to date.
|
||||
*/
|
||||
class Backfill
|
||||
{
|
||||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
function Backfill()
|
||||
{
|
||||
$this->n = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all active groups categories and descriptions.
|
||||
*/
|
||||
function backfillAllGroups($groupName='', $backfillDate=null)
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = false;
|
||||
if ($groupName != '')
|
||||
{
|
||||
$grp = $groups->getByName($groupName);
|
||||
if ($grp)
|
||||
$res = array($grp);
|
||||
}
|
||||
else
|
||||
{
|
||||
$res = $groups->getActive();
|
||||
}
|
||||
|
||||
if ($res)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
if ($nntp->doConnect(1, true)) {
|
||||
$nntpc = new Nntp();
|
||||
$nntpc->doConnect();
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$this->backfillGroup($nntp, $nntpc, $groupArr, $backfillDate);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
$nntpc->doQuit();
|
||||
} else {
|
||||
echo "Failed to get NNTP connection.$n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database for updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a group back to a specified date.
|
||||
*/
|
||||
function backfillGroup($nntp, $nntpc, $groupArr, $backfillDate=null)
|
||||
{
|
||||
$db = new DB();
|
||||
$db->disableAutoCommit();
|
||||
|
||||
$binaries = new Binaries();
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
$datac = $nntpc->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($datac))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
if ($backfillDate)
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$this->dateToDays($backfillDate),TRUE); // get targetpost based on date
|
||||
else
|
||||
$targetpost = $this->daytopost($nntp,$groupArr['name'],$groupArr['backfill_target'],TRUE); //get targetpost based on days target
|
||||
if($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0)
|
||||
{
|
||||
echo "Group ".$groupArr['name']." has invalid numbers. Have you run update on it? Have you set the backfill days amount?$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
|
||||
echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~";
|
||||
echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400));
|
||||
echo " days.".$n."Local first = ".$groupArr['first_record']." (";
|
||||
echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400));
|
||||
echo " days). Backfill target of ".$groupArr['backfill_target']."days is post $targetpost.$n";
|
||||
|
||||
if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish
|
||||
{
|
||||
echo "Nothing to do, we already have the target post.$n $n";
|
||||
$db->commit(); //Not an error so commit and re-enable autocommitting
|
||||
return "";
|
||||
}
|
||||
//get first and last part numbers from newsgroup
|
||||
if($targetpost < $data['first'])
|
||||
{
|
||||
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n";
|
||||
echo "Skipping Group $n";
|
||||
$db->rollback();
|
||||
return "";
|
||||
}
|
||||
//calculate total number of parts
|
||||
$total = $groupArr['first_record'] - $targetpost;
|
||||
$done = false;
|
||||
//set first and last, moving the window by maxxMssgs
|
||||
$last = $groupArr['first_record'] - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1; //set initial "chunk"
|
||||
if($targetpost > $first) //just in case this is the last chunk we needed
|
||||
$first = $targetpost;
|
||||
while($done === false)
|
||||
{
|
||||
$binaries->startLoop = microtime(true);
|
||||
echo "Getting ".($last-$first+1)." parts (".number_format($first-$targetpost)." in queue)".$n;
|
||||
flush();
|
||||
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
|
||||
if (!$success)
|
||||
{
|
||||
$db->rollback();
|
||||
return "";
|
||||
}
|
||||
|
||||
$db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
|
||||
$db->commit(false);
|
||||
|
||||
if($first==$targetpost)
|
||||
$done = true;
|
||||
else
|
||||
{ //Keep going: set new last, new first, check for last chunk.
|
||||
$last = $first - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1;
|
||||
if($targetpost > $first)
|
||||
$first = $targetpost;
|
||||
}
|
||||
}
|
||||
$first_record_postdate = $this->postdate($nntp,$first,false);
|
||||
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate
|
||||
|
||||
$db->commit();
|
||||
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns single timestamp from a local article number.
|
||||
*/
|
||||
function postdate($nntp,$post,$debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$attempts=0;
|
||||
do
|
||||
{
|
||||
$msgs = $nntp->getOverview($post."-".$post,true,false);
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Returning from postdate$n";
|
||||
return "";
|
||||
}
|
||||
|
||||
if(!isset($msgs[0]['Date']) || $msgs[0]['Date']=="" || is_null($msgs[0]['Date']))
|
||||
{
|
||||
$success=false;
|
||||
} else {
|
||||
$date = $msgs[0]['Date'];
|
||||
$success=true;
|
||||
}
|
||||
if($debug && $attempts > 0) echo "retried $attempts time(s)".$n;
|
||||
$attempts++;
|
||||
} while($attempts <= 3 && $success == false);
|
||||
|
||||
if (!$success) { return ""; }
|
||||
|
||||
if($debug) echo "DEBUG: postdate for post: $post came back $date (";
|
||||
$date = strtotime($date);
|
||||
if($debug) echo "$date seconds unixtime or ".$this->daysOld($date)." days)".$n;
|
||||
return $date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the post number for a given number of days back in a group.
|
||||
*/
|
||||
function daytopost($nntp, $group, $days, $debug=true)
|
||||
{
|
||||
$n = $this->n;
|
||||
$pddebug = false; //DEBUG every postdate call?!?!
|
||||
if ($debug) echo "INFO: daytopost finding post for $group $days days back.".$n;
|
||||
|
||||
$data = $nntp->selectGroup($group);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Error {$data->code}: {$data->message}$n";
|
||||
echo "Returning from daytopost$n";
|
||||
return "";
|
||||
}
|
||||
$goaldate = date('U')-(86400*$days); //goaltimestamp
|
||||
$totalnumberofarticles = $data['last'] - $data['first'];
|
||||
$upperbound = $data['last'];
|
||||
$lowerbound = $data['first'];
|
||||
|
||||
if ($debug) echo "Total Articles: $totalnumberofarticles $n Upper: $upperbound $n Lower: $lowerbound $n Goal: ".date("r", $goaldate)." ($goaldate) $n";
|
||||
if ($data['last']==PHP_INT_MAX) { echo "ERROR: Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.$n"; die(); }
|
||||
|
||||
$firstDate = $this->postdate($nntp, $data['first'], $pddebug);
|
||||
$lastDate = $this->postdate($nntp, $data['last'], $pddebug);
|
||||
if ($goaldate < $firstDate)
|
||||
{
|
||||
echo "WARNING: Backfill target of $days day(s) is older than the first article stored on your news server.$n";
|
||||
echo "Starting from the first available article (".date("r", $firstDate)." or ".$this->daysOld($firstDate)." days).$n";
|
||||
return $data['first'];
|
||||
}
|
||||
elseif ($goaldate > $lastDate)
|
||||
{
|
||||
echo "ERROR: Backfill target of $days day(s) is newer than the last article stored on your news server.$n";
|
||||
echo "To backfill this group you need to set Backfill Days to at least ".ceil($this->daysOld($lastDate)+1)." days (".date("r", $lastDate-86400).").$n";
|
||||
return "";
|
||||
}
|
||||
if ($debug) echo "DEBUG: Searching for postdate $n Goaldate: $goaldate (".date("r", $goaldate).") $n Firstdate: $firstDate (".((is_int($firstDate))?date("r", $firstDate):'n/a').") $n Lastdate: $lastDate (".date("r", $lastDate).") $n";
|
||||
|
||||
$interval = floor(($upperbound - $lowerbound) * 0.5);
|
||||
$dateofnextone = "";
|
||||
$templowered = "";
|
||||
|
||||
if ($debug) echo "Start: ".$data['first']." $n End: ".$data['last']." $n Interval: $interval $n";
|
||||
|
||||
$dateofnextone = $lastDate;
|
||||
|
||||
while($this->daysOld($dateofnextone) < $days) //match on days not timestamp to speed things up
|
||||
{
|
||||
$nskip = 1;
|
||||
while(($tmpDate = $this->postdate($nntp,($upperbound-$interval),$pddebug))>$goaldate)
|
||||
{
|
||||
$upperbound = $upperbound - $interval - ($nskip - 1);
|
||||
if($debug) echo "New upperbound ($upperbound) is ".$this->daysOld($tmpDate)." days old. $n";
|
||||
$nskip = $nskip * 2;
|
||||
}
|
||||
if(!$templowered)
|
||||
{
|
||||
$interval = ceil(($interval/2));
|
||||
if($debug) echo "Set interval to $interval articles. $n";
|
||||
}
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
|
||||
$skip = 1;
|
||||
while(!$dateofnextone)
|
||||
{
|
||||
$upperbound = $upperbound - $skip;
|
||||
$skip = $skip * 2;
|
||||
if($debug) echo "Getting next article date... $upperbound\n";
|
||||
$dateofnextone = $this->postdate($nntp,($upperbound-1),$pddebug);
|
||||
}
|
||||
}
|
||||
echo "Determined to be article $upperbound which is ".$this->daysOld($dateofnextone)." days old (".date("r", $dateofnextone).") $n";
|
||||
return $upperbound;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a timestamp.
|
||||
*/
|
||||
private function daysOld($timestamp)
|
||||
{
|
||||
return round((time()-$timestamp)/86400, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of days from a date.
|
||||
*/
|
||||
private function dateToDays($backfillDate)
|
||||
{
|
||||
return floor(-($backfillDate - time())/(60*60*24));
|
||||
}
|
||||
|
||||
/**
|
||||
* Update groups by post vs date categories and descriptions.
|
||||
*/
|
||||
function backfillPostAllGroups($groupName='', $backfillPost='')
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = false;
|
||||
if ($groupName != '')
|
||||
{
|
||||
$grp = $groups->getByName($groupName);
|
||||
if ($grp)
|
||||
$res = array($grp);
|
||||
}
|
||||
else
|
||||
{
|
||||
$res = $groups->getActive();
|
||||
}
|
||||
|
||||
if ($res)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
if ($nntp->doConnect(1, true)) {
|
||||
$nntpc = new Nntp();
|
||||
$nntpc->doConnect();
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$this->backfillPostGroup($nntp, $nntpc, $groupArr, $backfillPost);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
$nntpc->doQuit();
|
||||
} else {
|
||||
echo "Failed to get NNTP connection.$n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure you have spelled the group name right and the group is enabled.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a group back to a specified post.
|
||||
*/
|
||||
function backfillPostGroup($nntp, $nntpc, $groupArr, $backfillPost)
|
||||
{
|
||||
$db = new DB();
|
||||
$db->disableAutoCommit();
|
||||
|
||||
$binaries = new Binaries();
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
$datac = $nntpc->selectGroup($groupArr['name']);
|
||||
if(PEAR::isError($datac))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
if ($backfillPost)
|
||||
$targetpost = round($groupArr['first_record'] - $backfillPost);
|
||||
else
|
||||
{
|
||||
echo "You must set a target post. ex: php ".$groupArr['name']." 20000$n";
|
||||
die;
|
||||
}
|
||||
if($groupArr['first_record'] == 0)
|
||||
{
|
||||
echo "Group ".$groupArr['name']." has invalid numbers. Have you run update_binaires on it?$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
|
||||
echo "Group ".$data["group"].": server has ".$data['first']." - ".$data['last'].", or ~";
|
||||
echo((int) (($this->postdate($nntp,$data['last'],FALSE) - $this->postdate($nntp,$data['first'],FALSE))/86400));
|
||||
echo " days.".$n."Local first = ".$groupArr['first_record']." (";
|
||||
echo((int) ((date('U') - $this->postdate($nntp,$groupArr['first_record'],FALSE))/86400));
|
||||
echo " days). Going to backfill $backfillPost posts, which is post $targetpost.$n";
|
||||
|
||||
if($targetpost >= $groupArr['first_record']) //if our estimate comes back with stuff we already have, finish
|
||||
{
|
||||
echo "Nothing to do, we already have the target post.$n $n";
|
||||
$db->commit(); //Not an error so commit and re-enable autocommitting
|
||||
return "";
|
||||
}
|
||||
//get first and last part numbers from newsgroup
|
||||
if($targetpost < $data['first'])
|
||||
{
|
||||
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.$n";
|
||||
echo "Skipping Group $n";
|
||||
$db->rollback();
|
||||
return "";
|
||||
}
|
||||
//calculate total number of parts
|
||||
$total = $groupArr['first_record'] - $targetpost;
|
||||
$done = false;
|
||||
//set first and last, moving the window by maxxMssgs
|
||||
$last = $groupArr['first_record'] - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1; //set initial "chunk"
|
||||
if($targetpost > $first) //just in case this is the last chunk we needed
|
||||
$first = $targetpost;
|
||||
while($done === false)
|
||||
{
|
||||
$binaries->startLoop = microtime(true);
|
||||
echo "Getting ".($last-$first+1)." parts (".($first-$targetpost)." in queue)".$n;
|
||||
flush();
|
||||
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
|
||||
if (!$success)
|
||||
{
|
||||
$db->rollback();
|
||||
return "";
|
||||
}
|
||||
|
||||
$db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
|
||||
$db->commit(false);
|
||||
|
||||
if($first==$targetpost)
|
||||
$done = true;
|
||||
else
|
||||
{ //Keep going: set new last, new first, check for last chunk.
|
||||
$last = $first - 1;
|
||||
$first = $last - $binaries->messagebuffer + 1;
|
||||
if($targetpost > $first)
|
||||
$first = $targetpost;
|
||||
}
|
||||
}
|
||||
$first_record_postdate = $this->postdate($nntp,$first,false);
|
||||
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$first_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate
|
||||
|
||||
$db->commit();
|
||||
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n";
|
||||
}
|
||||
}
|
||||
@@ -1,858 +0,0 @@
|
||||
<?php
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
require_once(WWW_DIR."/lib/nntp.php");
|
||||
require_once(WWW_DIR."/lib/groups.php");
|
||||
require_once(WWW_DIR."/lib/backfill.php");
|
||||
|
||||
/**
|
||||
* This class manages the downloading of binaries and parts from usenet, and the
|
||||
* managing of data in the binaries and parts tables.
|
||||
*/
|
||||
class Binaries
|
||||
{
|
||||
const OPT_BLACKLIST = 1;
|
||||
const OPT_WHITELIST = 2;
|
||||
|
||||
const BLACKLIST_FIELD_SUBJECT = 1;
|
||||
const BLACKLIST_FIELD_FROM = 2;
|
||||
const BLACKLIST_FIELD_MESSAGEID = 3;
|
||||
|
||||
/**
|
||||
* Default constructor
|
||||
*/
|
||||
function Binaries()
|
||||
{
|
||||
$this->n = "\n";
|
||||
|
||||
$s = new Sites();
|
||||
$site = $s->get();
|
||||
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
|
||||
$this->messagebuffer = (!empty($site->maxmssgs)) ? $site->maxmssgs : 20000;
|
||||
$this->NewGroupScanByDays = ($site->newgroupscanmethod == "1") ? true : false;
|
||||
$this->NewGroupMsgsToScan = (!empty($site->newgroupmsgstoscan)) ? $site->newgroupmsgstoscan : 50000;
|
||||
$this->NewGroupDaysToScan = (!empty($site->newgroupdaystoscan)) ? $site->newgroupdaystoscan : 3;
|
||||
|
||||
$this->blackList = array(); //cache of our black/white list
|
||||
$this->message = array();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for all active groups.
|
||||
*/
|
||||
function updateAllGroups()
|
||||
{
|
||||
$n = $this->n;
|
||||
$groups = new Groups;
|
||||
$res = $groups->getActive();
|
||||
|
||||
$s = new Sites();
|
||||
echo $s->getLicense();
|
||||
|
||||
if ($res)
|
||||
{
|
||||
shuffle($res);
|
||||
$alltime = microtime(true);
|
||||
echo 'Updating: '.sizeof($res).' groups - Using compression? '.(($this->compressedHeaders)?'Yes':'No').$n;
|
||||
|
||||
$nntp = new Nntp();
|
||||
if ($nntp->doConnect()) {
|
||||
|
||||
$pos = 0;
|
||||
foreach($res as $groupArr)
|
||||
{
|
||||
$pos++;
|
||||
echo 'Group '.$pos.' of '.sizeof($res).$n;
|
||||
$this->message = array();
|
||||
$this->updateGroup($nntp, $groupArr);
|
||||
}
|
||||
|
||||
$nntp->doQuit();
|
||||
echo 'Updating completed in '.number_format(microtime(true) - $alltime, 2).' seconds'.$n;
|
||||
} else {
|
||||
echo "Failed to get NNTP connection.$n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.$n";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process headers and store in database for a group.
|
||||
*/
|
||||
function updateGroup($nntp=null, $groupArr)
|
||||
{
|
||||
$blnDoDisconnect = false;
|
||||
if ($nntp == null)
|
||||
{
|
||||
$nntp = new Nntp();
|
||||
if (!$nntp->doConnect()) {
|
||||
echo "Failed to get NNTP connection.$n";
|
||||
return;
|
||||
}
|
||||
$this->message = array();
|
||||
$blnDoDisconnect = true;
|
||||
}
|
||||
|
||||
$db = new DB();
|
||||
$backfill = new Backfill();
|
||||
|
||||
$db->disableAutoCommit();
|
||||
|
||||
$n = $this->n;
|
||||
$this->startGroup = microtime(true);
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
echo 'Processing '.$groupArr['name'].$n;
|
||||
|
||||
// Connect to server
|
||||
$data = $nntp->selectGroup($groupArr['name']);
|
||||
if (PEAR::isError($data))
|
||||
{
|
||||
echo "Could not select group (bad name?): {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
|
||||
//Attempt to repair any missing parts before grabbing new ones
|
||||
$this->partRepair($nntp, $groupArr);
|
||||
|
||||
//Get first and last part numbers from newsgroup
|
||||
$last = $grouplast = $data['last'];
|
||||
|
||||
// For new newsgroups - determine here how far you want to go back.
|
||||
if ($groupArr['last_record'] == 0)
|
||||
{
|
||||
if ($this->NewGroupScanByDays)
|
||||
{
|
||||
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
|
||||
if ($first == '')
|
||||
{
|
||||
echo "Skipping group: {$groupArr['name']}$n";
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['first'] > ($data['last'] - $this->NewGroupMsgsToScan))
|
||||
$first = $data['first'];
|
||||
else
|
||||
$first = $data['last'] - $this->NewGroupMsgsToScan;
|
||||
}
|
||||
$first_record_postdate = $backfill->postdate($nntp, $first, false);
|
||||
$db->query(sprintf("UPDATE groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(".$first_record_postdate.") WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($data['last'] < $groupArr['last_record'])
|
||||
{
|
||||
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}".$n;
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
$first = $groupArr['last_record'] + 1;
|
||||
}
|
||||
|
||||
// Generate postdates for first and last records, for those that upgraded
|
||||
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0"))
|
||||
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['first_record'],false)."), last_record_postdate = FROM_UNIXTIME(".$backfill->postdate($nntp,$groupArr['last_record'],false).") WHERE ID = %d", $groupArr['ID']));
|
||||
|
||||
// Deactivate empty groups
|
||||
if (($data['last'] - $data['first']) <= 5)
|
||||
$db->query(sprintf("UPDATE groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID']));
|
||||
|
||||
// Calculate total number of parts
|
||||
$total = $grouplast - $first + 1;
|
||||
|
||||
// If total is bigger than 0 it means we have new parts in the newsgroup
|
||||
if($total > 0)
|
||||
{
|
||||
echo "Group ".$data["group"]." has ".number_format($total)." new parts.".$n;
|
||||
echo "First: ".$data['first']." Last: ".$data['last']." Local last: ".$groupArr['last_record'].$n;
|
||||
if ($groupArr['last_record'] == 0)
|
||||
echo "New group starting with ".(($this->NewGroupScanByDays) ? $this->NewGroupDaysToScan." days" : $this->NewGroupMsgsToScan." messages")." worth.".$n;
|
||||
|
||||
$done = false;
|
||||
|
||||
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
|
||||
while ($done === false)
|
||||
{
|
||||
$this->startLoop = microtime(true);
|
||||
|
||||
if ($total > $this->messagebuffer)
|
||||
{
|
||||
if ($first + $this->messagebuffer > $grouplast)
|
||||
$last = $grouplast;
|
||||
else
|
||||
$last = $first + $this->messagebuffer;
|
||||
}
|
||||
|
||||
echo "Getting ".number_format($last-$first+1)." parts (".$first." to ".$last.") - ".number_format($grouplast - $last)." in queue".$n;
|
||||
flush();
|
||||
|
||||
//get headers from newsgroup
|
||||
$lastId = $this->scan($nntp, $groupArr, $first, $last);
|
||||
|
||||
if ($lastId === false)
|
||||
{
|
||||
//scan failed - skip group
|
||||
$db->rollback();
|
||||
return;
|
||||
}
|
||||
$db->query(sprintf("UPDATE groups SET last_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($lastId), $groupArr['ID']));
|
||||
$db->commit(false);
|
||||
|
||||
if ($last == $grouplast)
|
||||
$done = true;
|
||||
else
|
||||
{
|
||||
$last = $lastId;
|
||||
$first = $last + 1;
|
||||
}
|
||||
}
|
||||
|
||||
$last_record_postdate = $backfill->postdate($nntp,$last,false);
|
||||
$db->query(sprintf("UPDATE groups SET last_record_postdate = FROM_UNIXTIME(".$last_record_postdate."), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's last postdate
|
||||
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
|
||||
echo "Group processed in $timeGroup seconds $n $n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "No new records for ".$data["group"]." (first $first last $last total $total) grouplast ".$groupArr['last_record'].$n.$n;
|
||||
|
||||
}
|
||||
|
||||
if ($blnDoDisconnect)
|
||||
{
|
||||
$nntp->doQuit();
|
||||
}
|
||||
/*
|
||||
* Got through the updating of this group successfully
|
||||
* so commit and re-enable the auto committing.
|
||||
*/
|
||||
$db->commit(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a range of usenet messages. Store binaries with subjects matching a
|
||||
* specific pattern in the database.
|
||||
*/
|
||||
function scan($nntp, $groupArr, $first, $last, $type='update')
|
||||
{
|
||||
$db = new Db();
|
||||
$n = $this->n;
|
||||
$this->startHeaders = microtime(true);
|
||||
|
||||
if ($this->compressedHeaders)
|
||||
{
|
||||
$nntpn = new Nntp();
|
||||
$nntpn->doConnect(1,true);
|
||||
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
|
||||
if (PEAR::isError($response) || $response != 290)
|
||||
{
|
||||
$response2 = $nntpn->_sendCommand('group alt.binaries.multimedia');
|
||||
$response3 = $nntpn->_sendCommand('xzver');
|
||||
if (PEAR::isError($response3) || $response3 != 224)
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
|
||||
if (PEAR::isError($msgs) && ($msgs->code == 400 || $msgs->code == 503))
|
||||
{
|
||||
echo "NNTP connection timed out. Reconnecting...$n";
|
||||
if (!$nntp->doConnect()) {
|
||||
// TODO: What now?
|
||||
echo "Failed to get NNTP connection.$n";
|
||||
return;
|
||||
}
|
||||
$nntp->selectGroup($groupArr['name']);
|
||||
if ($this->compressedHeaders)
|
||||
{
|
||||
$nntpn = new Nntp();
|
||||
$nntpn->doConnect(1,true);
|
||||
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
|
||||
if (PEAR::isError($response) || $response != 290)
|
||||
{
|
||||
$response2 = $nntpn->_sendCommand('group alt.binaries.multimedia');
|
||||
$response3 = $nntpn->_sendCommand('xzver');
|
||||
if (PEAR::isError($response3) || $response3 != 224)
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getXOverview($first."-".$last, true, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$nntpn->doQuit();
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
$msgs = $nntp->getOverview($first."-".$last, true, false);
|
||||
}
|
||||
|
||||
$rangerequested = range($first, $last);
|
||||
$msgsreceived = array();
|
||||
$msgsblacklisted = array();
|
||||
$msgsignored = array();
|
||||
$msgsinserted = array();
|
||||
$msgsnotinserted = array();
|
||||
|
||||
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
|
||||
|
||||
if(PEAR::isError($msgs))
|
||||
{
|
||||
echo "Error {$msgs->code}: {$msgs->message}$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->startUpdate = microtime(true);
|
||||
if (is_array($msgs))
|
||||
{
|
||||
//loop headers, figure out parts
|
||||
foreach($msgs AS $msg)
|
||||
{
|
||||
if (!isset($msg['Number']))
|
||||
continue;
|
||||
|
||||
$msgsreceived[] = $msg['Number'];
|
||||
$msgPart = $msgTotalParts = 0;
|
||||
|
||||
$pattern = '|\((\d+)[\/](\d+)\)|i';
|
||||
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
|
||||
$matchcnt = sizeof($matches[0]);
|
||||
for ($i=0; $i<$matchcnt; $i++)
|
||||
{
|
||||
$msgPart = $matches[1][$i];
|
||||
$msgTotalParts = $matches[2][$i];
|
||||
}
|
||||
|
||||
if (!isset($msg['Subject']) || $matchcnt == 0) // not a binary post most likely.. continue
|
||||
{
|
||||
$msgsignored[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
//Filter binaries based on black/white list
|
||||
if ($this->isBlackListed($msg, $groupArr['name']))
|
||||
{
|
||||
$msgsblacklisted[] = $msg['Number'];
|
||||
continue;
|
||||
}
|
||||
|
||||
if((int)$msgPart > 0 && (int)$msgTotalParts > 0)
|
||||
{
|
||||
$subject = utf8_encode(trim(preg_replace('|\('.$msgPart.'[\/]'.$msgTotalParts.'\)|i', '', $msg['Subject'])));
|
||||
|
||||
if(!isset($this->message[$subject]))
|
||||
{
|
||||
$this->message[$subject] = $msg;
|
||||
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
|
||||
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
|
||||
}
|
||||
if((int)$msgPart > 0)
|
||||
{
|
||||
$this->message[$subject]['Parts'][(int)$msgPart] = array('Message-ID' => substr($msg['Message-ID'],1,-1), 'number' => $msg['Number'], 'part' => (int)$msgPart, 'size' => $msg['Bytes']);
|
||||
}
|
||||
}
|
||||
}
|
||||
unset($msg);
|
||||
unset($msgs);
|
||||
$count = 0;
|
||||
$updatecount = 0;
|
||||
$partcount = 0;
|
||||
$maxnum = $last;
|
||||
|
||||
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
|
||||
|
||||
|
||||
if ($type != 'partrepair')
|
||||
echo "Received ".sizeof($msgsreceived)." articles of ".($last-$first+1)." requested, ".sizeof($msgsblacklisted)." blacklisted, ".sizeof($msgsignored)." not binaries $n";
|
||||
|
||||
if ($type == 'update' && sizeof($msgsreceived) == 0)
|
||||
{
|
||||
echo "Error: Server did not return any articles.$n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (sizeof($rangenotreceived) > 0) {
|
||||
switch($type)
|
||||
{
|
||||
case 'backfill':
|
||||
//don't add missing articles
|
||||
break;
|
||||
case 'partrepair':
|
||||
case 'update':
|
||||
default:
|
||||
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
|
||||
break;
|
||||
}
|
||||
echo "Server did not return ".count($rangenotreceived)." article(s).$n";
|
||||
}
|
||||
|
||||
if(isset($this->message) && count($this->message))
|
||||
{
|
||||
$maxnum = $first;
|
||||
//insert binaries and parts into database. when binary already exists; only insert new parts
|
||||
foreach($this->message AS $subject => $data)
|
||||
{
|
||||
if(isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '')
|
||||
{
|
||||
$binaryHash = md5($subject.$data['From'].$groupArr['ID']);
|
||||
$res = $db->queryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $db->escapeString($binaryHash)));
|
||||
if(!$res)
|
||||
{
|
||||
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $db->escapeString($subject), $db->escapeString($data['From']), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash));
|
||||
$binaryID = $db->queryInsert($sql);
|
||||
$count++;
|
||||
if ($count%500==0) echo "$count bin adds...";
|
||||
}
|
||||
else
|
||||
{
|
||||
$binaryID = $res["ID"];
|
||||
$updatecount++;
|
||||
if ($updatecount%500==0) echo "$updatecount bin updates...";
|
||||
}
|
||||
|
||||
foreach($data['Parts'] AS $partdata)
|
||||
{
|
||||
$maxnum = ($partdata['number'] > $maxnum) ? $partdata['number'] : $maxnum;
|
||||
|
||||
$query = sprintf
|
||||
(
|
||||
"SELECT number FROM `parts` WHERE number = '%s' LIMIT 1",
|
||||
$partdata['number']
|
||||
);
|
||||
|
||||
$result = $db->query($query);
|
||||
if (!count($result))
|
||||
{
|
||||
$partcount++;
|
||||
$pidata = $db->queryInsert(sprintf("INSERT INTO parts (binaryID, messageID, number, partnumber, size, dateadded) VALUES (%d, %s, %s, %s, %s, now())", $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size'])), false);
|
||||
if (!$pidata) {
|
||||
$msgsnotinserted[] = $partdata['number'];
|
||||
} else {
|
||||
$msgsinserted[] = $partdata['number'];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//TODO: determine whether to add to missing articles if insert failed
|
||||
if (sizeof($msgsnotinserted) > 0)
|
||||
{
|
||||
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert'.$n;
|
||||
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
|
||||
}
|
||||
if (($count >= 500) || ($updatecount >= 500)) { echo $n; } //line break for bin adds output
|
||||
}
|
||||
|
||||
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
|
||||
$timeLoop = number_format(microtime(true)-$this->startLoop, 2);
|
||||
|
||||
if ($type != 'partrepair')
|
||||
{
|
||||
echo number_format($count).' new, '.number_format($updatecount).' updated, '.number_format($partcount).' parts.';
|
||||
echo " $timeHeaders headers, $timeUpdate update, $timeLoop range.$n";
|
||||
}
|
||||
unset($this->message);
|
||||
unset($data);
|
||||
return $maxnum;
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "Error: Can't get parts from server (msgs not array) $n";
|
||||
echo "Skipping group$n";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Go through all rows in partrepair table and see if theyve arrived on usenet yet.
|
||||
*
|
||||
* @param Nntp $nntp
|
||||
* @param array $group
|
||||
* @return bool
|
||||
*/
|
||||
private function partRepair($nntp, $group)
|
||||
{
|
||||
$db = new DB;
|
||||
|
||||
$parts = array();
|
||||
$chunks = array();
|
||||
$result = array();
|
||||
|
||||
$query = sprintf
|
||||
(
|
||||
"SELECT numberID FROM partrepair WHERE groupID = %d AND attempts < 5 ORDER BY numberID ASC",
|
||||
$group['ID']
|
||||
);
|
||||
|
||||
$result = $db->query($query);
|
||||
if (!count($result))
|
||||
return false;
|
||||
|
||||
foreach ($result as $item)
|
||||
$parts[] = $item['numberID'];
|
||||
|
||||
if (count($parts))
|
||||
{
|
||||
$matched = array();
|
||||
printf("Repair: supposed to repair %s parts.%s", count($parts), $this->n);
|
||||
foreach ($parts as $key => $item)
|
||||
{
|
||||
if (in_array(substr($item, 0, -2), $matched))
|
||||
continue;
|
||||
|
||||
# when moving to php >= 5.3
|
||||
#preg_filter(sprintf("~%s~", substr($item, 0, -3)), '$0', $parts);
|
||||
|
||||
$result = preg_grep(sprintf("~%s~", substr($item, 0, -2)), $parts);
|
||||
if (count($result))
|
||||
{
|
||||
|
||||
$matched[] = substr($item, 0, -2);
|
||||
array_push($chunks, $result);
|
||||
|
||||
foreach ($result as $key => $val)
|
||||
unset($parts[$key]);
|
||||
}
|
||||
}
|
||||
|
||||
$chunks = $this->getSuperUniqueArray($chunks);
|
||||
|
||||
if (!count($chunks))
|
||||
{
|
||||
printf("Repair: unable to extract parts for repair! Please report this is a bug, together with a dump of your partrepair table.\n", $this->n);
|
||||
return false;
|
||||
}
|
||||
|
||||
$repaired = 0;
|
||||
foreach ($chunks as $chunk)
|
||||
{
|
||||
$start = current($chunk);
|
||||
$end = end($chunk);
|
||||
|
||||
# TODO: if less than 3 chunks do 3 single calls to scan()
|
||||
if (count($chunk) < 3) { }
|
||||
|
||||
$range = ($end - $start);
|
||||
|
||||
printf("Repair: + %s-%s (%s missing, %s articles overhead)%s", $start, $end, count($chunk), $range, $this->n);
|
||||
$this->scan($nntp, $group, $start, $end, 'partrepair');
|
||||
|
||||
$query = sprintf
|
||||
(
|
||||
"SELECT pr.ID, pr.numberID, p.number from partrepair pr LEFT JOIN parts p ON p.number = pr.numberID WHERE pr.groupID=%d AND pr.numberID IN (%s) ORDER BY pr.numberID ASC",
|
||||
$group['ID'], implode(',', $chunk)
|
||||
);
|
||||
|
||||
$result = $db->query($query);
|
||||
foreach ($result as $item)
|
||||
{
|
||||
# TODO: rewrite.. stupid
|
||||
if ($item['number'] == $item['numberID'])
|
||||
{
|
||||
#printf("Repair: %s repaired.%s", $item['ID'], $this->n);
|
||||
$db->query(sprintf("DELETE FROM partrepair WHERE ID=%d LIMIT 1", $item['ID']));
|
||||
$repaired++;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
#printf("Repair: %s has not arrived yet or deleted.%s", $item['numberID'], $this->n);
|
||||
$db->query(sprintf("UPDATE partrepair SET attempts=attempts+1 WHERE ID=%d LIMIT 1", $item['ID']));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$db->query(sprintf("DELETE FROM partrepair WHERE attempts >= 5 AND groupID = %d", $group['ID']));
|
||||
printf("Repair: repaired %s.%s", $repaired, $this->n);
|
||||
printf("Repair: cleaned %s parts.%s", $db->getAffectedRows(), $this->n);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a missing part to the database.
|
||||
*/
|
||||
private function addMissingParts($numbers, $groupID)
|
||||
{
|
||||
$db = new DB;
|
||||
$added = false;
|
||||
$insertStr = "INSERT INTO partrepair (numberID, groupID) VALUES ";
|
||||
foreach($numbers as $number)
|
||||
{
|
||||
if ($number > 0)
|
||||
{
|
||||
$added = true;
|
||||
$insertStr .= sprintf("(%u, %d), ", $number, $groupID);
|
||||
}
|
||||
}
|
||||
if ($added)
|
||||
{
|
||||
$insertStr = substr($insertStr, 0, -2);
|
||||
$insertStr .= " ON DUPLICATE KEY UPDATE attempts=attempts+1";
|
||||
return $db->queryInsert($insertStr, false);
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return internally cached list of binary blacklist patterns.
|
||||
*/
|
||||
public function retrieveBlackList()
|
||||
{
|
||||
if (is_array($this->blackList) && !empty($this->blackList)) { return $this->blackList; }
|
||||
$blackList = $this->getBlacklist(true);
|
||||
$this->blackList = $blackList;
|
||||
return $blackList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test if a message subject is blacklisted.
|
||||
*/
|
||||
public function isBlackListed($msg, $groupName)
|
||||
{
|
||||
$blackList = $this->retrieveBlackList();
|
||||
$field = array();
|
||||
if (isset($msg["Subject"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_SUBJECT] = $msg["Subject"];
|
||||
|
||||
if (isset($msg["From"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_FROM] = $msg["From"];
|
||||
|
||||
if (isset($msg["Message-ID"]))
|
||||
$field[Binaries::BLACKLIST_FIELD_MESSAGEID] = $msg["Message-ID"];
|
||||
|
||||
// if a white list is detected we now are required to
|
||||
// only accept the entry if it matches at least 1 whitelist
|
||||
// while a blacklist will over-ride all
|
||||
$whitelist = array();
|
||||
$matches_whitelist = false;
|
||||
|
||||
foreach ($blackList as $blist)
|
||||
{
|
||||
if (preg_match('/^'.$blist['groupname'].'$/i', $groupName))
|
||||
{
|
||||
//blacklist
|
||||
if ($blist['optype'] == Binaries::OPT_BLACKLIST)
|
||||
{
|
||||
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else if ($blist['optype'] == Binaries::OPT_WHITELIST)
|
||||
{
|
||||
$whitelist[] = $blist['regex'];
|
||||
if (preg_match('/'.$blist['regex'].'/i', $field[$blist['msgcol']])) {
|
||||
// Flag that we matched the white list
|
||||
$matches_whitelist = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# We parsed entire matching list entries at this point.. now we need
|
||||
# to handle the whitelist (if it was enabled)
|
||||
if(count($whitelist) > 0 && !$matches_whitelist)
|
||||
{
|
||||
# We failed to match white list
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rawsearch. Perform a simple like match on binary subjects matching a pattern.
|
||||
*/
|
||||
public function search($search, $limit=1000, $excludedcats=array())
|
||||
{
|
||||
$db = new DB();
|
||||
|
||||
//
|
||||
// if the query starts with a ^ it indicates the search is looking for items which start with the term
|
||||
// still do the like match, but mandate that all items returned must start with the provided word
|
||||
//
|
||||
$words = explode(" ", $search);
|
||||
$searchsql = "";
|
||||
$intwordcount = 0;
|
||||
if (count($words) > 0)
|
||||
{
|
||||
foreach ($words as $word)
|
||||
{
|
||||
//
|
||||
// see if the first word had a caret, which indicates search must start with term
|
||||
//
|
||||
if ($intwordcount == 0 && (strpos($word, "^") === 0))
|
||||
$searchsql.= sprintf(" and b.name like %s", $db->escapeString(substr($word, 1)."%"));
|
||||
else
|
||||
$searchsql.= sprintf(" and b.name like %s", $db->escapeString("%".$word."%"));
|
||||
|
||||
$intwordcount++;
|
||||
}
|
||||
}
|
||||
|
||||
$exccatlist = "";
|
||||
if (count($excludedcats) > 0)
|
||||
$exccatlist = " and b.categoryID not in (".implode(",", $excludedcats).") ";
|
||||
|
||||
$res = $db->query(sprintf("
|
||||
SELECT b.*,
|
||||
g.name AS group_name,
|
||||
r.guid,
|
||||
(SELECT COUNT(ID) FROM parts p where p.binaryID = b.ID) as 'binnum'
|
||||
FROM binaries b
|
||||
INNER JOIN groups g ON g.ID = b.groupID
|
||||
LEFT OUTER JOIN releases r ON r.ID = b.releaseID
|
||||
WHERE 1=1 %s %s order by DATE DESC LIMIT %d ",
|
||||
$searchsql, $exccatlist, $limit));
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all binaries for a release.
|
||||
*/
|
||||
public function getForReleaseId($id)
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->query(sprintf("select binaries.* from binaries where releaseID = %d order by relpart", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
* Get a binary row.
|
||||
*/
|
||||
public function getById($id)
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->queryOneRow(sprintf("select binaries.*, groups.name as groupname from binaries left outer join groups on binaries.groupID = groups.ID where binaries.ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of blacklists from database.
|
||||
*/
|
||||
public function getBlacklist($activeonly=true)
|
||||
{
|
||||
$db = new DB();
|
||||
|
||||
$where = "";
|
||||
if ($activeonly)
|
||||
$where = " where binaryblacklist.status = 1 ";
|
||||
|
||||
return $db->query("SELECT binaryblacklist.ID, binaryblacklist.optype, binaryblacklist.status, binaryblacklist.description, binaryblacklist.groupname AS groupname, binaryblacklist.regex,
|
||||
groups.ID AS groupID, binaryblacklist.msgcol FROM binaryblacklist
|
||||
left outer JOIN groups ON groups.name = binaryblacklist.groupname
|
||||
".$where."
|
||||
ORDER BY coalesce(groupname,'zzz')");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a blacklist row from database.
|
||||
*/
|
||||
public function getBlacklistByID($id)
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->queryOneRow(sprintf("select * from binaryblacklist where ID = %d ", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a blacklist row from database.
|
||||
*/
|
||||
public function deleteBlacklist($id)
|
||||
{
|
||||
$db = new DB();
|
||||
return $db->query(sprintf("delete from binaryblacklist where ID = %d", $id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a blacklist row.
|
||||
*/
|
||||
public function updateBlacklist($regex)
|
||||
{
|
||||
$db = new DB();
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $db->escapeString($groupname));
|
||||
}
|
||||
|
||||
$db->query(sprintf("update binaryblacklist set groupname=%s, regex=%s, status=%d, description=%s, optype=%d, msgcol=%d where ID = %d ", $groupname, $db->escapeString($regex["regex"]), $regex["status"], $db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"], $regex["id"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new blacklist row.
|
||||
*/
|
||||
public function addBlacklist($regex)
|
||||
{
|
||||
$db = new DB();
|
||||
|
||||
$groupname = $regex["groupname"];
|
||||
if ($groupname == "")
|
||||
$groupname = "null";
|
||||
else
|
||||
{
|
||||
$groupname = preg_replace("/a\.b\./i", "alt.binaries.", $groupname);
|
||||
$groupname = sprintf("%s", $db->escapeString($groupname));
|
||||
}
|
||||
|
||||
return $db->queryInsert(sprintf("INSERT INTO binaryblacklist (groupname, regex, status, description, optype, msgcol) values (%s, %s, %d, %s, %d, %d) ",
|
||||
$groupname, $db->escapeString($regex["regex"]), $regex["status"], $db->escapeString($regex["description"]), $regex["optype"], $regex["msgcol"]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new binary row and its associated parts.
|
||||
*/
|
||||
public function delete($id)
|
||||
{
|
||||
$db = new DB();
|
||||
$db->query(sprintf("delete from parts where binaryID = %d", $id));
|
||||
$db->query(sprintf("delete from binaries where ID = %d", $id));
|
||||
}
|
||||
|
||||
# http://php.net/manual/en/function.array-unique.php#97285
|
||||
public function getSuperUniqueArray($array)
|
||||
{
|
||||
$result = array_map("unserialize", array_unique(array_map("serialize", $array)));
|
||||
foreach ($result as $key => $value)
|
||||
{
|
||||
if (is_array($value))
|
||||
$result[$key] = $this->getSuperUniqueArray($value);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
@@ -1,601 +0,0 @@
|
||||
<?php
|
||||
require_once(WWW_DIR."/lib/binaries.php");
|
||||
require_once(WWW_DIR."/lib/framework/db.php");
|
||||
require_once(WWW_DIR."/lib/Net_NNTP/NNTP/Client.php");
|
||||
|
||||
/**
|
||||
* This class extends the standard PEAR NNTP class with some extra features.
|
||||
*/
|
||||
class Nntp extends Net_NNTP_Client
|
||||
{
|
||||
/**
|
||||
* Start an NNTP connection.
|
||||
*/
|
||||
public $XFCompression = false;
|
||||
function doConnect($attempt=1, $overridecompression=false)
|
||||
{
|
||||
if ($this->_isConnected()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Attempt to connect up to 5 times before giving up.
|
||||
$maxAttempts = 5;
|
||||
|
||||
$connected = true;
|
||||
|
||||
$s = new Sites();
|
||||
$site = $s->get();
|
||||
$this->compressedHeaders = ($site->compressedheaders == "1") ? true : false;
|
||||
$enc = false;
|
||||
if (defined("NNTP_SSLENABLED") && NNTP_SSLENABLED == true)
|
||||
$enc = 'ssl';
|
||||
|
||||
$ret = $this->connect(NNTP_SERVER, $enc, NNTP_PORT);
|
||||
if(PEAR::isError($ret))
|
||||
{
|
||||
$err = "Cannot connect to server ".NNTP_SERVER.(!$enc?" (nonssl) ":"(ssl) ").": ". $ret->getMessage();
|
||||
echo $err;
|
||||
$connected = false;
|
||||
}
|
||||
if(!defined(NNTP_USERNAME) && NNTP_USERNAME!="" )
|
||||
{
|
||||
$ret2 = $this->authenticate(NNTP_USERNAME, NNTP_PASSWORD);
|
||||
if(PEAR::isError($ret2))
|
||||
{
|
||||
$err = "Cannot authenticate to server ".NNTP_SERVER.(!$enc?" (nonssl) ":" (ssl) ")." - ". NNTP_USERNAME." (".$ret2->getMessage().")";
|
||||
echo $err;
|
||||
$connected = false;
|
||||
}
|
||||
}
|
||||
if ($this->compressedHeaders && !$overridecompression)
|
||||
{
|
||||
$response = $this->_sendCommand('XFEATURE COMPRESS GZIP');
|
||||
if (PEAR::isError($response) || $response != 290)
|
||||
{
|
||||
//echo "NNTP: XFeature not supported.\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
$this->enableXFCompression();
|
||||
}
|
||||
}
|
||||
if ($attempt < $maxAttempts && !$connected) {
|
||||
sleep(5);
|
||||
$connected = $this->doConnect($attempt+1);
|
||||
}
|
||||
|
||||
if (!$connected && $attempt == 1) {
|
||||
echo "\nTried to connect ".$maxAttempts." times, but couldn't. Check your settings and connection.\n";
|
||||
}
|
||||
|
||||
return $connected;
|
||||
}
|
||||
/**
|
||||
* End an NNTP connection.
|
||||
*/
|
||||
function doQuit()
|
||||
{
|
||||
$this->quit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve an NNTP message and decode it.
|
||||
*/
|
||||
function getMessage($groupname, $partMsgId)
|
||||
{
|
||||
$summary = $this->selectGroup($groupname);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$body = $this->getBody('<'.$partMsgId.'>', true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $partMsgId, $groupname, $body->getMessage());
|
||||
return false;
|
||||
}
|
||||
|
||||
$message = $this->decodeYenc($body);
|
||||
if (!$message)
|
||||
{
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a series of NNTP messages and decode them.
|
||||
*/
|
||||
function getMessages($groupname, $msgIds)
|
||||
{
|
||||
$summary = $this->selectGroup($groupname);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach($msgIds as $msgId)
|
||||
{
|
||||
$messageID = '<'.$msgId.'>';
|
||||
$body = $this->getBody($messageID, true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $messageID, $groupname, $body->getMessage());
|
||||
return false;
|
||||
}
|
||||
|
||||
$dec = $this->decodeYenc($body);
|
||||
if (!$dec)
|
||||
{
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
$message .= $dec;
|
||||
}
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve all NNTP messages associated with a binaries.ID
|
||||
*/
|
||||
function getBinary($binaryId, $isNfo=false)
|
||||
{
|
||||
$db = new DB();
|
||||
$bin = new Binaries();
|
||||
|
||||
$binary = $bin->getById($binaryId);
|
||||
if (!$binary)
|
||||
{
|
||||
printf("NntpPrc: Unable to locate binary: %s\n", $binaryId);
|
||||
return false;
|
||||
}
|
||||
|
||||
$summary = $this->selectGroup($binary['groupname']);
|
||||
$message = $dec = '';
|
||||
|
||||
if (PEAR::isError($summary))
|
||||
{
|
||||
echo "NntpPrc : ".substr($summary->getMessage(), 0, 30)."\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId));
|
||||
|
||||
//
|
||||
// Dont attempt to download nfos which are larger than one part.
|
||||
//
|
||||
if (sizeof($resparts) > 1 && $isNfo === true)
|
||||
{
|
||||
echo "NntpPrc : Error Nfo is too large... skipping.\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach($resparts as $part)
|
||||
{
|
||||
$messageID = '<'.$part['messageID'].'>';
|
||||
$body = $this->getBody($messageID, true);
|
||||
if (PEAR::isError($body))
|
||||
{
|
||||
printf("NntpPrc : Error fetching part number %s in %s (Server response: %s)\n", $part['messageID'], $binary['groupname'], $body->getMessage());
|
||||
return false;
|
||||
}
|
||||
|
||||
$dec = $this->decodeYenc($body);
|
||||
if (!$dec)
|
||||
{
|
||||
printf("NntpPrc: Unable to decode body of binary: %s\n", $binaryId);
|
||||
|
||||
//
|
||||
// Yenc decode failed
|
||||
//
|
||||
return false;
|
||||
}
|
||||
|
||||
$message .= $dec;
|
||||
}
|
||||
return $message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get XZVER for a range of NNTP messages.
|
||||
*/
|
||||
function getXOverview($range, $_names = true, $_forceNames = true)
|
||||
{
|
||||
// Fetch overview from server
|
||||
$overview = $this->cmdXZver($range);
|
||||
if (PEAR::isError($overview)) {
|
||||
return $overview;
|
||||
}
|
||||
|
||||
// Use field names from overview format as keys?
|
||||
if ($_names)
|
||||
{
|
||||
// Already cached?
|
||||
if (is_null($this->_overviewFormatCache)) {
|
||||
// Fetch overview format
|
||||
$format = $this->getOverviewFormat($_forceNames, true);
|
||||
if (PEAR::isError($format)){
|
||||
return $format;
|
||||
}
|
||||
|
||||
// Prepend 'Number' field
|
||||
$format = array_merge(array('Number' => false), $format);
|
||||
|
||||
// Cache format
|
||||
$this->_overviewFormatCache = $format;
|
||||
}
|
||||
else
|
||||
{
|
||||
$format = $this->_overviewFormatCache;
|
||||
}
|
||||
|
||||
// Loop through all articles
|
||||
foreach ($overview as $key => $article)
|
||||
{
|
||||
if (sizeof($format) == sizeof($article))
|
||||
{
|
||||
//Replace overview using $format as keys, $article as values
|
||||
$overview[$key] = array_combine(array_keys($format), $article);
|
||||
|
||||
// If article prefixed by field name, remove it
|
||||
foreach($format as $fkey=>$fval)
|
||||
{
|
||||
if ($fval === true)
|
||||
{
|
||||
$overview[$key][$fkey] = trim(str_replace($fkey.':', '', $overview[$key][$fkey]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch (true)
|
||||
{
|
||||
// Expect one article
|
||||
case is_null($range);
|
||||
case is_int($range);
|
||||
case is_string($range) && ctype_digit($range):
|
||||
case is_string($range) && substr($range, 0, 1) == '<' && substr($range, -1, 1) == '>':
|
||||
if (count($overview) == 0) {
|
||||
return false;
|
||||
} else {
|
||||
return reset($overview);
|
||||
}
|
||||
break;
|
||||
|
||||
// Expect multiple articles
|
||||
default:
|
||||
return $overview;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a yenc encoded string.
|
||||
*/
|
||||
function decodeYenc($yencodedvar)
|
||||
{
|
||||
$input = array();
|
||||
preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $yencodedvar, $input);
|
||||
if (isset($input[1]))
|
||||
{
|
||||
$ret = "";
|
||||
$input = trim(preg_replace("/\r\n/im", "", preg_replace("/(^=yend.*)/im", "", preg_replace("/(^=ypart.*\\r\\n)/im", "", preg_replace("/(^=ybegin.*\\r\\n)/im", "", $input[1], 1), 1), 1)));
|
||||
|
||||
for( $chr = 0; $chr < strlen($input) ; $chr++)
|
||||
$ret .= ($input[$chr] != "=" ? chr(ord($input[$chr]) - 42) : chr((ord($input[++$chr]) - 64) - 42));
|
||||
|
||||
return $ret;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// }}}
|
||||
// {{{ cmdXZver()
|
||||
/*
|
||||
* Based on code from http://wonko.com/software/yenc/, but
|
||||
* simplified because XZVER and the likes don't implement
|
||||
* yenc properly
|
||||
*/
|
||||
private function yencDecode($string, $destination = "") {
|
||||
$encoded = array();
|
||||
$header = array();
|
||||
$decoded = '';
|
||||
|
||||
# Extract the yEnc string itself
|
||||
preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $string, $encoded);
|
||||
$encoded = $encoded[1];
|
||||
|
||||
# Extract the filesize and filename from the yEnc header
|
||||
preg_match("/^=ybegin.*size=([^ $]+).*name=([^\\r\\n]+)/im", $encoded, $header);
|
||||
$filesize = $header[1];
|
||||
$filename = $header[2];
|
||||
|
||||
# Remove the header and footer from the string before parsing it.
|
||||
$encoded = preg_replace("/(^=ybegin.*\\r\\n)/im", "", $encoded, 1);
|
||||
$encoded = preg_replace("/(^=yend.*)/im", "", $encoded, 1);
|
||||
|
||||
# Remove linebreaks and whitespace from the string
|
||||
$encoded = trim(str_replace("\r\n", "", $encoded));
|
||||
|
||||
// Decode
|
||||
$strLength = strlen($encoded);
|
||||
for($i = 0; $i < $strLength; $i++) {
|
||||
$c = $encoded[$i];
|
||||
|
||||
if ($c == '=') {
|
||||
$i++;
|
||||
$decoded .= chr((ord($encoded[$i]) - 64) - 42);
|
||||
} else {
|
||||
$decoded .= chr(ord($c) - 42);
|
||||
}
|
||||
}
|
||||
// Make sure the decoded filesize is the same as the size specified in the header.
|
||||
if (strlen($decoded) != $filesize) {
|
||||
throw new Exception("Filesize in yEnc header en filesize found do not match up");
|
||||
}
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch message header from message number $first until $last
|
||||
* The format of the returned array is:
|
||||
* $messages[message_id][header_name]
|
||||
* @param optional string $range articles to fetch
|
||||
* @return mixed (array) nested array of message and there headers on success or (object) pear_error on failure
|
||||
* @access protected
|
||||
*/
|
||||
function cmdXZver($range = null)
|
||||
{
|
||||
if (is_null($range))
|
||||
$command = 'XZVER';
|
||||
else
|
||||
$command = 'XZVER ' . $range;
|
||||
$response = $this->_sendCommand($command);
|
||||
|
||||
switch ($response) {
|
||||
case 224: // RFC2980: 'Overview information follows'
|
||||
$data = $this->_getCompressedResponse();
|
||||
foreach ($data as $key => $value)
|
||||
$data[$key] = explode("\t", trim($value));
|
||||
|
||||
return $data;
|
||||
break;
|
||||
case 412: // RFC2980: 'No news group current selected'
|
||||
return $this->throwError('No news group current selected', $response, $this->_currentStatusResponse());
|
||||
break;
|
||||
case 420: // RFC2980: 'No article(s) selected'
|
||||
return $this->throwError('No article(s) selected', $response, $this->_currentStatusResponse());
|
||||
break;
|
||||
case 502: // RFC2980: 'no permission'
|
||||
return $this->throwError('No permission', $response, $this->_currentStatusResponse());
|
||||
break;
|
||||
case 500: // RFC2980: 'unknown command'
|
||||
$this->throwError("XZver not supported ({$this->_currentStatusResponse()})", $response);
|
||||
break;
|
||||
default:
|
||||
return $this->_handleUnexpectedResponse($response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve blob
|
||||
* Get data and assume we do not hit any blindspots
|
||||
* @return mixed (array) text response on success or (object) pear_error on failure
|
||||
* @access private
|
||||
*/
|
||||
function _getCompressedResponse()
|
||||
{
|
||||
$data = array();
|
||||
|
||||
// We can have two kinds of compressed support:
|
||||
// - yEnc encoding
|
||||
// - Just a gzip drop
|
||||
// We try to autodetect which one this uses
|
||||
|
||||
$line = @fread($this->_socket, 1024);
|
||||
if (substr($line, 0, 7) == '=ybegin') {
|
||||
$data = $this->_getTextResponse();
|
||||
$data = $line . "\r\n" . implode("", $data);
|
||||
$data = $this->yencDecode($data);
|
||||
$data = explode("\r\n", gzinflate($data));
|
||||
return $data;
|
||||
}
|
||||
// We cannot use blocked I/O on this one
|
||||
$streamMetadata = stream_get_meta_data($this->_socket);
|
||||
stream_set_blocking($this->_socket, false);
|
||||
|
||||
// Continue until connection is lost or we don't receive any data anymore
|
||||
$tries = 0;
|
||||
$uncompressed = '';
|
||||
while (!feof($this->_socket)) {
|
||||
|
||||
# Retrieve and append up to 32k characters from the server
|
||||
$received = @fread($this->_socket, 32768);
|
||||
if (strlen($received) == 0) {
|
||||
$tries++;
|
||||
# Try decompression
|
||||
$uncompressed = @gzuncompress($line);
|
||||
if (($uncompressed !== false) || ($tries > 500)) {
|
||||
break;
|
||||
}
|
||||
if ($tries % 50 == 0) {
|
||||
}
|
||||
}
|
||||
# an error occured
|
||||
if ($received === false) {
|
||||
@fclose($this->_socket);
|
||||
$this->_socket = false;
|
||||
}
|
||||
$line .= $received;
|
||||
}
|
||||
# and set the stream to its original blocked(?) value
|
||||
stream_set_blocking($this->_socket, $streamMetadata['blocked']);
|
||||
$data = explode("\r\n", $uncompressed);
|
||||
$dataCount = count($data);
|
||||
|
||||
# Gzipped compress includes the "." and linefeed in the compressed stream, skip those.
|
||||
if ($dataCount >= 2) {
|
||||
if (($data[($dataCount - 2)] == ".") && (empty($data[($dataCount - 1)]))) {
|
||||
array_pop($data);
|
||||
array_pop($data);
|
||||
}
|
||||
$data = array_filter($data);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable XFeature compression support for the current connection.
|
||||
*/
|
||||
function enableXFCompression()
|
||||
{
|
||||
$response = $this->_sendCommand('XFEATURE COMPRESS GZIP');
|
||||
|
||||
if (PEAR::isError($response) || $response != 290) {
|
||||
echo "Xfeature compression not supported!\n";
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->XFCompression = true;
|
||||
echo "XFeature compression enabled\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override to intercept any Xfeature compressed responses.
|
||||
*/
|
||||
function _getTextResponse()
|
||||
{
|
||||
if ($this->XFCompression && isset($this->_currentStatusResponse[1])
|
||||
&& stripos($this->_currentStatusResponse[1], 'COMPRESS=GZIP') !== false)
|
||||
{
|
||||
return $this->_getXFCompressedTextResponse();
|
||||
}
|
||||
return parent::_getTextResponse();
|
||||
}
|
||||
|
||||
function _getXFCompressedTextResponse()
|
||||
{
|
||||
$tries = 0;
|
||||
$bytesreceived = 0;
|
||||
$totalbytesreceived = 0;
|
||||
$completed = false;
|
||||
$data = null;
|
||||
//build binary array that represents zero results basically a compressed empty string terminated with .(period) char(13) char(10)
|
||||
$emptyreturnend = chr(0x03).chr(0x00).chr(0x00).chr(0x00).chr(0x00).chr(0x01).chr(0x2e).chr(0x0d).chr(0x0a);
|
||||
$emptyreturn = chr(0x78).chr(0x9C).$emptyreturnend;
|
||||
$emptyreturn2 = chr(0x78).chr(0x01).$emptyreturnend;
|
||||
$emptyreturn3 = chr(0x78).chr(0x5e).$emptyreturnend;
|
||||
$emptyreturn4 = chr(0x78).chr(0xda).$emptyreturnend;
|
||||
|
||||
while (!feof($this->_socket))
|
||||
{
|
||||
$completed = false;
|
||||
//get data from the stream
|
||||
$buffer = fgets($this->_socket);
|
||||
//get byte count and update total bytes
|
||||
$bytesreceived = strlen($buffer);
|
||||
//if we got no bytes at all try one more time to pull data.
|
||||
if ($bytesreceived == 0)
|
||||
{
|
||||
$buffer = fgets($this->_socket);
|
||||
}
|
||||
//get any socket error codes
|
||||
$errorcode = socket_last_error();
|
||||
|
||||
//if the buffer is zero its zero...
|
||||
if ($bytesreceived === 0)
|
||||
return $this->throwError('No data returned.', 1000);
|
||||
//did we have any socket errors?
|
||||
if ($errorcode === 0)
|
||||
{
|
||||
//append buffer to final data object
|
||||
$data .= $buffer;
|
||||
$totalbytesreceived = $totalbytesreceived+$bytesreceived;
|
||||
|
||||
//output byte count in real time once we have 1MB of data
|
||||
if ($totalbytesreceived > 10240)
|
||||
if ($totalbytesreceived%128 == 0)
|
||||
{
|
||||
echo "bytes received: ";
|
||||
echo $totalbytesreceived;
|
||||
echo "\r";
|
||||
}
|
||||
|
||||
//check to see if we have the magic terminator on the byte stream
|
||||
$b1 = null;
|
||||
if ($bytesreceived > 2)
|
||||
if (ord($buffer[$bytesreceived-3]) == 0x2e && ord($buffer[$bytesreceived-2]) == 0x0d && ord($buffer[$bytesreceived-1]) == 0x0a)//substr($buffer,-3) == ".\r\n"
|
||||
{
|
||||
//check to see if the returned binary string is 11 bytes long generally and indcator
|
||||
//of an compressed empty string probably don't need this check
|
||||
if ($totalbytesreceived==11)
|
||||
{
|
||||
//compare the data to the empty string if the data is a compressed empty string
|
||||
//throw an error else return the data
|
||||
if (($data === $emptyreturn)||($data === $emptyreturn2)||($data === $emptyreturn3)||($data === $emptyreturn4))
|
||||
{
|
||||
echo "empty gzip stream\n";
|
||||
return $this->throwError('No data returned.', 1000);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "\n";
|
||||
$completed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "failed to read from socket\n";
|
||||
return $this->throwError('Failed to read line from socket.', 1000);
|
||||
}
|
||||
|
||||
if ($completed)
|
||||
{
|
||||
//check to see if the header is valid for a gzip stream
|
||||
if(ord($data[0]) == 0x78 && in_array(ord($data[1]),array(0x01,0x5e,0x9c,0xda)))
|
||||
{
|
||||
$decomp = @gzuncompress(mb_substr ( $data , 0 ,-3, '8bit' ));
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "Invalid header on gzip stream.\n";
|
||||
return $this->throwError('Invalid gzip stream.', 1000);
|
||||
}
|
||||
|
||||
if ($decomp != false)
|
||||
{
|
||||
$decomp = explode("\r\n", trim($decomp));
|
||||
return $decomp;
|
||||
}
|
||||
else
|
||||
{
|
||||
$tries++;
|
||||
echo "Decompression Failed Retry Number: $tries \n";
|
||||
}
|
||||
}
|
||||
}
|
||||
//throw an error if we get out of the loop
|
||||
if (!feof($this->_socket))
|
||||
{
|
||||
return "Error: unexpected fgets() fail\n";
|
||||
}
|
||||
return $this->throwError('Decompression Failed, connection closed.', 1000);
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ rm -fv *
|
||||
|
||||
#edit cleanup scripts
|
||||
if [[ $CLEANUP_EDIT == "true" ]]; then
|
||||
echo "editing cleanup scripts"
|
||||
sed -i -e 's/^$echo =.*$/$echo = false;/' $TESTING_PATH/update_parsing.php
|
||||
sed -i -e 's/^$limited =.*$/$limited = false;/' $TESTING_PATH/update_parsing.php
|
||||
sed -i -e 's/^$echo =.*$/$echo = false;/' $TESTING_PATH/update_cleanup.php
|
||||
@@ -31,6 +32,7 @@ fi
|
||||
|
||||
#edit powerprocess.php
|
||||
if [[ $FIX_POSIX == "true" ]]; then
|
||||
echo "editing powerprocess"
|
||||
sed -i -e 's/case SIGSTKFLT:/\/\/case SIGSTKFLT:/' $NEWZPATH/www/lib/powerprocess.php
|
||||
sed -i -e 's/return 'SIGSTKFLT';/\/\/return 'SIGSTKFLT';/' $NEWZPATH/www/lib/powerprocess.php
|
||||
sed -i -e 's/case SIGCLD:/\/\/case SIGCLD:/' $NEWZPATH/www/lib/powerprocess.php
|
||||
@@ -43,6 +45,7 @@ fi
|
||||
|
||||
#attempt to get english only from IMDB
|
||||
if [[ $EN_IMDB == "true" ]]; then
|
||||
echo "edit movie language"
|
||||
sed -i -e 's/akas.imdb/www.imdb/g' $NEWZPATH/www/lib/movie.php
|
||||
sed -i -e 's/akas.imdb/www.imdb/g' $TESTING_PATH/update_parsing.php
|
||||
sed -i -e 's/curl_setopt($ch, CURLOPT_URL, $url);/curl_setopt($ch, CURLOPT_URL, $url);\
|
||||
|
||||
@@ -38,7 +38,7 @@ if $TMUXCMD -q has-session -t $TMUX_SESSION; then
|
||||
else
|
||||
printf "The above is just a TMUX notice, it is saying TMUX, that you do not have a TMUX session currently running. It is not an error. It is TMUX"
|
||||
printf "\033]0; $TMUX_SESSION\007\003\n"
|
||||
$TMUXCMD -f $TMUX_CONF new-session -d -s $TMUX_SESSION -n Monitor 'printf "\033]2;Monitor\033\\" && cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n 19 $PHP monitor.php'
|
||||
$TMUXCMD -f $TMUX_CONF new-session -d -s $TMUX_SESSION -n Monitor 'printf "\033]2;Monitor\033\\" && cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n$NICENESS $PHP monitor.php'
|
||||
|
||||
if [ ! -f $NEWZPATH/www/lib/postprocess.php.orig ]; then
|
||||
cp $NEWZPATH/www/lib/postprocess.php $NEWZPATH/www/lib/postprocess.php.orig
|
||||
@@ -229,9 +229,6 @@ else
|
||||
# $SED -i -e "s/order by tvrage.releasetitle asc/order by tvrage.releasetitle DESC/" bin/lib/tvrage.php
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#start tmux
|
||||
#printf "\033]0; $TMUX_SESSION\007\003\n"
|
||||
#$TMUXCMD -f $TMUX_CONF attach-session - $TMUX_SESSION || new-session -d -s $TMUX_SESSION -n $TMUX_SESSION 'cd bin && echo "Monitor Started" && echo "It might take a minute for everything to spinup......" && $NICE -n 19 $PHP monitor.php'
|
||||
|
||||
Reference in New Issue
Block a user