starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-31 18:28:54 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
9cce4dd02265ec1074deaa4e6aa2dadb1a501f7e
newznab-tmux/bin/spotnab.php
T
jonnyboy 6e3d364f8c added spotnab
2013-02-10 19:58:16 +01:00

2290 lines
67 KiB
PHP
Raw Blame History

<?php
// Author l2g
// Date: Jan 27th, 2013
// Version: 0.92.1
//
// Description:
// This script is an implementation of an spotnet and newznab to which
// multiple servers can share comments and information amongst each other
//
// The implementation is based loosely on how spotweb works. If you don't know
// what spotweb is, feel free to Google it.
//
// With an spotnab, you have 2 options:
// 1. Fetch comments (and potential other things as we dream them up) from
// other newznab sources and apply them to your own comment section.
//
// Fetched content is scanned and decrypted using a password that the
// newznab server who posted it chooses to share with you. If you can't
// decrypt the post then you move along. Since all newznab servers choose
// to share and or not to share. Spotnab will only populate your database
// with comments from the sources who choose to share they're comments
// with you.
//
// A few public keys will be configured out of the box so that you can
// retrieve your comments from existing active sites should you choose
// to enable them.
//
// 2. Post comments; send all the comments made by your local users of your
// site to usenet encrypted by your own secure private key. Only those you
// share your public key will be allowed to decrypt it for their own
// server.
//
// Releases are uniquely bundled and shared across NewzNab servers through the
// use of the releases table. To uniquely identify a release across remote
// servers we need to extract the Message-ID from the first segment of the first
// file of the matched release (as it was posted to usenet). This ID is unique
// to usenet, so it'll be unique to us as well. We can achive this information
// from parsing the NZB file generated by the NewzNab server.
//
// This value can be certain to not conflict and uniquely identify a release
// across all NewzNab Servers.
//
// Content is posted and retrieved to Usnet in JSON encrypted using the key
// belonging to the NewzNab Server.
//
// Structure is as follows:
// {
// server: {
// code: <string>,
// title: <string>,
// },
// postdate_utc: 'YYYY-MM-DD hh:mm:ss.ms',
// comments: [
// {
// GID: <char[32]>,
// CID: <char[32]>,
// comment: <string>,
// is_visible: <bool>,
// username: <string>,
// postdate_utc: 'YYYY-MM-DD hh:mm:ss',
// }
// ]
// }
//
// server: Meta information on the server that posted the spot.
//
// server
// -/ code: This is taken from the users local database `site` table. It is
// the `code` the user configured their newznab server as.
//
// -/ title: This is also taken from the users local database `site` table. It
// is the the `title` the user configured their newznab server as.
//
// postdate_utc: This is the current time measured in the number of seconds since
// the Unix Epoch (January 1 1970 00:00:00 GMT). This represents
// the local UTC time of when this message was posted. It doesn't
// reflect the age of the content to follow.
//
// comments: This is an array of comment objects
//
// comments
// -/ GID: This is the Global Identifier for the release the comment
// is in relation to.
//
// -/ CID: This is a unique global identifier used to identify the comment
// itself. This allows the server to publish the comment again
// if there is an update to it's data (such as toggling it to a
// hidden type). This should be used to prevent a duplicate
// entry in the comments table when dealing with remote content.
//
// -/ comment: This is the actual comment itself associated with the release
//
// -/ is_visible: Whether or not the comment should be marked visible or not.
//
// -/ username: The username of the person who made the comment.
//
// -/ postdate_utc: The timestamp (in UTC) that the comment was created at.
//
/* The following database modifications are required for this to work:
DROP TABLE IF EXISTS `spotnab_sources`;
CREATE TABLE `spotnab_sources` (
`ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`userName` VARCHAR(64) NOT NULL DEFAULT 'NNTP',
`userEmail` VARCHAR(128) NOT NULL DEFAULT 'SPOT@NNTP.COM',
`usenetGroup` VARCHAR(64) NOT NULL DEFAULT 'alt.binaries.test2',
`publicKey` VARCHAR(512) NOT NULL DEFAULT '',
`active` TINYINT(1) NOT NULL DEFAULT '1',
`description` VARCHAR(255) NULL DEFAULT '',
`lastUpdate` DATETIME DEFAULT NULL,
PRIMARY KEY (`ID`)
) ENGINE=MYISAM DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
-- We must update releases table to track Global Identifier
ALTER TABLE `releases` ADD `GID` VARCHAR( 32 ) NULL AFTER `ID` ;
ALTER TABLE `releases` ADD INDEX `ix_releases_GID` ( `GID` );
-- We need a way of adding a signature to our posts so others can pull them if
-- they choose. The newznab code choosen is also very important (exists already
-- in the sites table) This is stamped with each push to the news server so
-- others can see where the post came from.
DELETE FROM `site` WHERE `setting` IN
('spotnabsitekey', 'spotnabsitepubkey', 'spotnabsiteprvkey', 'spotnabuser',
'spotnabemail', 'spotnabgroup', 'spotnabpost');
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabsitepubkey', '', NOW( ));
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabsiteprvkey', '', NOW( ));
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabuser', 'NNTP', NOW( ));
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabemail', 'SPOT@NNTP.COM', NOW( ));
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabgroup', 'alt.binaries.test2', NOW( ));
-- We need a reference point we can use to track when we last did our updates to
-- the remote server... Set this value to '1' if you want to post your comments
-- to usenet for others to share from.
INSERT INTO `site` (`ID` , `setting` , `value` , `updateddate` )
VALUES ( NULL , 'spotnabpost', '1', NOW( ));
-------------------------------------------------------------------------------
-- PHASE 2
-------------------------------------------------------------------------------
-- Merging `spotnab_comments` into `releasecomments`:
-- First prepare comments table:
-------------------------------------------------------------------------------
ALTER TABLE `releasecomment` ADD `sourceID` BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER `ID` ;
ALTER TABLE `releasecomment` ADD `GID` VARCHAR( 32 ) DEFAULT NULL AFTER `sourceID` ;
ALTER TABLE `releasecomment` ADD `CID` VARCHAR( 32 ) DEFAULT NULL AFTER `GID` ;
ALTER TABLE `releasecomment` ADD `username` VARCHAR( 50 ) DEFAULT NULL AFTER `userID` ;
ALTER TABLE `releasecomment` ADD `isVisible` TINYINT(1) DEFAULT 1 AFTER `text` ;
ALTER TABLE `releasecomment` ADD `isSynced` TINYINT(1) DEFAULT 0 AFTER `isVisible` ;
ALTER TABLE `releasecomment` ADD INDEX `ix_releasecomment_CID` ( `CID` );
ALTER TABLE `releasecomment` ADD INDEX `ix_releasecomment_GID` ( `GID` );
-- run this if the above sql statements are okay
DROP TABLE IF EXISTS `spotnab_comments`;
-- Things to consider with the table merge... releasecomments continues to work as normal
-- however new stuff fetched from usenet populate the sourceID, GID, and CID
-- Comments are now based on the GID (and no longer the releaseID); because of this we need
-- to gracefully phase out the releaseID column; This is done during the -g switch of this
-- tool which handles
INSTRUCTIONS
============
Both posting and fetching will 'NOT' work unless you've generated
Global Release ID values for all of your current releases... this is
done by running the command:
#> php spotnab.php -g
POSTING
1. To run all of the SQL statements identifed above
2. You 'must' have all Global Release ID's defined for each
release or the comments associated with it will not be
included in the post.
3. Generate yourself your own set of SSL keys:
#> php spotnab.php -k
4. At this point your ready to post... If you run the following SQL
statement, you'll mark all your existing comments:
-- Ensure spotnab posting is enabled:
UPDATE `site` set `value` = '1' WHERE `setting` = 'spotnabpost';
-- Posting actually uses this date as a reference to when
-- it last posted. So set it to a date so far back in time
-- that all your existing comments become declared as unposted.
UPDATE `site` set `updateddate` = '1980-01-01 00:00:00'
WHERE `setting` = 'spotnabpost' AND `value` = '1';
FETCHING
1. You need to acquire and enable sources. A source is as simple
as someone providing you their public key they generated using
the --keygen (-k) switch.
2. Fetch releases:
#> php spotnab.php -f
3. Consider fetching your own posts back just to test things out...:
SET @key := (SELECT value FROM site WHERE setting = 'spotnabsitepubkey');
INSERT INTO `spotnab_sources` (`ID` , `userName`, `userEmail` ,
`usenetGroup`, `publicKey` , `active` ,
`description` , `lastUpdate` )
VALUES ( NULL , 'NNTP', 'SPOT@NNTP.COM', 'alt.binaries.test2', @key,
'1', '1', 'My Key Fetch Test', NULL);
Then you can run the fetch release command again to poll for
stuff you posted already (if you did).
#> php spotnab.php -f
SHARING
1. Use the -k switch on the tool to retrieve your public key...
share it... let people use the comments posted on your site
get them to do the same to you. Eliminating crappy content
couldn't be more easy.
FUTURE CONSIDERATIONS
- Share NZB's and release content accumulated from a group so
people can pick and choose to fetch someone's already hard
earned processing power. Why have 1000 people process
the same group when 1 person could just share their results.
*/
require_once("config.php");
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/nzb.php");
require_once(WWW_DIR."/lib/site.php");
require_once(WWW_DIR."/lib/releases.php");
// Subject Lines will always be a hash key that helps scanning identify whether
// or not the contents are valid or not. the hash key contains is built
// by applying the password key against the md5 some of the message content
// itself. Unmatched content is ignored.
$shortopts = "";
$shortopts .= "G";
$shortopts .= "g";
$shortopts .= "t";
$shortopts .= "p";
$shortopts .= "f";
$shortopts .= "k";
$shortopts .= "K";
$longopts = array(
"spotnab-post",
"spotnab-fetch",
"populate-gid",
"populate-fix-gid",
"keygen",
"force-keygen",
"test",
);
// Set Memory Restrictions To avoid running script if memory value is too low
// Minimum allowable memory set to 512MB
define("MIN_MEMORY", 536870912);
$options = getopt($shortopts, $longopts);
function display_help(){
echo "\n";
echo "SpotNab v0.92.1, Author: l2g\n";
echo "Syntax: spotnab.php <action>\n";
echo "\n";
echo "Actions:\n";
echo " -g, --populate-gid This could be considered phase one of "
."this project.\n";
echo " requiring that your releases database"
." table is up to date\n";
echo " with all GID (Global Identifiers) so"
." it can correctly\n";
echo " communicate with other servers that "
."share the same content\n";
echo " -G, --populate-fix-gid Same as -g except broken nzb files are "
."also broken releases.\n";
echo " Specifying this switch will remove "
."these dead releses.\n";
echo " -f, --spotnab-fetch Get latest spotnab content from "
."usenet using the information\n";
echo " from the servers configured.\n";
echo " -p, --spotnab-post Post latest updates from local system "
."to usenet.\n";
echo "\n";
echo " -t, --test Produces a whole lot of garbage, but "
."is used for testing\n";
echo " the internals of the class...\n";
echo " -k, --keygen Generate a new SSL Public/Private Key "
."pair only if one isn't\n";
echo " already generated.\n";
echo " -K, --force-keygen Generate a new SSL Public/Private Key "
."pair\n";
echo "\n";
$max_mem = get_max_mem();
if ($max_mem < MIN_MEMORY){
echo "Warning: your memory is very low. Consider running this script as:\n";
echo " #> php -d memory_limit=2G spotnab.php <action>\n";
echo "\n";
}
}
// Silent Error Handler (used to shut up noisy XML exceptions)
// We don't care if the nzb is corrupt with so many additional lines of
// info... that is someone elses problem and defeats the purpose
// and readability of this script... we output a more relaxed
// error in these events
// we use the silent error handler for remote connection failures as well
function snHandleError($errno, $errstr, $errfile, $errline, array $errcontext){
if (0 === error_reporting())
return false;
if(!defined('E_STRICT'))define('E_STRICT', 2048);
switch($errno){
case E_WARNING:
case E_NOTICE:
case E_STRICT:
return;
default:
break;
};
throw new ErrorException($errstr, 0, $errno, $errfile, $errline);
}
function get_max_mem(){
// returns the current memory allocated for this process
$val = trim(ini_get('memory_limit'));
$last = strtolower($val[strlen($val)-1]);
switch($last) {
// The 'G' modifier is available since PHP 5.1.0
case 'g':
$val *= 1024;
case 'm':
$val *= 1024;
case 'k':
$val *= 1024;
}
return $val;
}
// Useful for debugging hex
function hex_dump($data, $newline="\n")
{
static $from = '';
static $to = '';
static $width = 16; # number of bytes per line
static $pad = '.'; # padding for non-visible characters
if ($from===''){
for ($i=0; $i<=0xFF; $i++)
{
$from .= chr($i);
$to .= ($i >= 0x20 && $i <= 0x7E) ? chr($i) : $pad;
}
}
$hex = str_split(bin2hex($data), $width*2);
$chars = str_split(strtr($data, $from, $to), $width);
$offset = 0;
$data_out = '';
foreach ($hex as $i => $line){
$data_out .= sprintf('%6X',$offset).' : '.
implode(' ', str_split($line,2)) .
' [' . $chars[$i] . ']' . $newline;
$offset += $width;
}
return $data_out;
}
// Create a NNTP Exception type so we can identify it from others
class SpotNabException extends Exception { }
class SpotNab {
// Segment Identifier domain is used to help build segments
// prior to them being posted.
const SEGID_DOMAIN = 'sample.com';
// Subject line can look like this:
// 01c9478809c80ccb07246d19852ed33b0a5f5d8d-20130125030511
// 517a6210bd588e964654f75b807d65d55d420f5e-20130125002943
// 33012186754c9c75050848d825ad6c2ae1af2e4d-20130125002849
// To speed up fetch process, we can parse this to determine wether
// or not to continue or not.
const FETCH_SUBJECT_REGEX = '/^(?P<checksum>[0-9a-z]{40})-(?P<utcref>[0-9]{14})$/i';
// How many consecutive misses in a row do we allow while trying to retrieve
// historic messages do we allow before assuming that we've exceeded the
// the retention area. In which case we just return the last date we
// matched before the miss count started
const FETCH_MAX_MISSES = 15;
// Maximum number of messages to process at one time
const FETCH_USENET_BATCH = 25000;
// Maximum number of messages to look back if one source
// is lookin like it hasn't posted anything in a very long
// time... we stop counting back headers when we reached
// this figure
const FETCH_MAXIMUM_HEADERS = 25000;
// Maximum age (in seconds) we look back for a source
const FETCH_MAXIMUM_AGE = 172800;
// Verify Fetch Range; This is the number of records to look back after
// a post to be sure that the post was successful. The number may appear
// kind of high, but consider a large active group like a.b.boneless and
// this number makes sense. Plus it doesn't take that long to rescan
// a few headers.
const VERIFY_FETCH_HEADER_COUNT = 300;
// The php function openssl_public_encrypt() seems to fail whenever it's
// passed more then this many characters into its buffer, therefore
// we need to encrypt in batches if the content is longer or we
// fail hard.
const SSL_MAX_BUF_LEN = 245;
// If SSL_MAX_BUF_LEN is to large, we need to use delimiters
// to help separate the batches as they are processed
const SSL_BUF_DELIMITER = ':::';
private $_nntp;
// Meta Information is fetched from DB
private $_post_site;
private $_post_title;
private $_post_user;
private $_post_email;
private $_post_key;
private $_post_group;
/* SSL Public & Private Keys */
private $_ssl_pubkey;
private $_ssl_prvkey;
/* Encryption Variables */
private $_cypher_adjust;
private $_cypher_modulus;
/* Maximum of 3 deep encryption bases */
private $_cypher_scramble;
// **********************************************************************
function __construct($post_user=Null, $post_email=Null, $post_key=Null,
$post_group=Null) {
/*
Constructor
*/
$this->_nntp = new NNTP();
$this->_post_user = $post_user;
$this->_post_email = $post_email;
$this->_post_key = $post_key;
$this->_post_group = $post_group;
$db = new DB();
// Fetch Meta information
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'code'");
$this->_post_site = ($res !== false)?$res['value']:Null;
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'title'");
$this->_post_title = ($res !== false)?$res['value']:Null;
if ($this->_post_user === Null){
// Fetch the SpotNab EmailID
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabuser'");
$this->_post_user = ($res !== false)?$res['value']:Null;
}
if ($this->_post_email === Null){
// Fetch the SpotNab EmailID
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabemail'");
$this->_post_email = ($res !== false)?$res['value']:Null;
}
if ($this->_post_key === Null){
// Fetch the SpotNab Decrypt/Encrypt Key
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabsitekey'");
$this->_post_key = ($res !== false)?$res['value']:Null;
}
if ($this->_post_group === Null){
// Fetch the SpotNab Usenet Group to post to
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabgroup'");
$this->_post_group = ($res !== false)?$res['value']:Null;
}
// Fetch our keys
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabsitepubkey'");
$this->_ssl_pubkey = ($res !== false)?trim($res['value']):false;
if($this->_ssl_pubkey)
$this->_ssl_pubkey = $this->decompstr($this->_ssl_pubkey);
$res = $db->queryOneRow("SELECT value FROM site WHERE "
."setting = 'spotnabsiteprvkey'");
$this->_ssl_prvkey = ($res !== false)?trim($res['value']):false;
if($this->_ssl_prvkey)
$this->_ssl_prvkey = $this->decompstr($this->_ssl_prvkey);
}
// ***********************************************************************
public function fetch($reftime=Null, $comments=true, $retries=3){
/*
* This function queries all enabled sources and fetches any content
* they are sharing.
*/
$db = new DB();
$last = $first = Null;
// Return Value; Initialize it to Okay
// we'll change it to false if we have to.
$fetch_okay = true;
if($reftime === Null){
// Fetch local time (but look back 1 day)
$reftime = $this->utc2local((time()-(60*60*24)));
}
// First we find all active sources and build a hash table we can
// use to simplify fetching.
$sql = "SELECT * FROM spotnab_sources WHERE active = 1 ".
"ORDER BY usenetGroup,lastUpdate DESC";
$res = $db->query($sql);
$group_hash = array();
if(!count($res)){
printf("%s WARNING - There are no source(s) defined to fetch".
" from.\n", date("Y-m-d H:i:s"));
return true;
}
foreach($res as $source){
$ghash = trim($source['usenetGroup']);
if(!array_key_exists($ghash, $group_hash)){
// Because our results are sorted by group, if we enter
// here then we're processing a brand new group...
$group_hash[$ghash] = array();
}
// Reference time is in UTC on Usenet but local in our database
// this isn't intentionally confusing, this is done so all our local
// times reflect those across the world, and it also makes joins to
// the table much easier since they join doesn't have to accomodate
// for the utc time itself.
// Therefore, we need to take the lastUpdate time and convert it to
// UTC for processing.
$ref = $source['lastUpdate'];
if(!$ref){
// We've never fetched from the group before, so we'll use
// the reftime passed to the function
$ref = $reftime;
}
$group_hash[$ghash][] = array(
'ID' => $source['ID'],
'key' => $this->decompstr(trim($source['publicKey'])),
'user' => trim($source['userName']),
'email' => trim($source['userEmail']),
// Store local reference time
'ref' => strtotime($ref),
);
}
// We want to resort the internal arrays by they're ref time
// so that the oldest (longest without an update) is processed
// first
// We set a cap on how many days in the past we look
$_max_age = time() - SpotNab::FETCH_MAXIMUM_AGE;
foreach(array_keys($group_hash) as $key){
$_ref = array();
foreach($group_hash[$key] as $id => $source){
# Source Time (within reason)
$_ref[$id] =
($source['ref'] < $_max_age)?$_max_age:$source['ref'];
}
// Sort results (oldest in time first)
array_multisort($_ref, SORT_ASC, $group_hash[$key]);
}
// Now we fetch headers
set_error_handler('snHandleError');
// Connect to server
try{
$this->_nntp->doConnect();
}
catch(Exception $e){
printf("%s ERROR - Failed to connect to Usenet\n",
date("Y-m-d H:i:s"));
return false;
}
foreach($group_hash as $group => $hash){
printf("%s INFO - Fetching $group for %d source(s).\n",
date("Y-m-d H:i:s"), count($hash));
$summary = $this->_nntp->selectGroup($group);
while(PEAR::isError($summary))
{
// Reset Connection (This happens first time
// around since _nntpReset() will perform
// the initial connect if it hasn't been already
$summary = $this->_nntpReset($group);
// Track retry attempts
$retries--;
if($retries <= 0){
// Retry Atempts exausted
printf("%s ERROR - Exausted connect/fetch retries "
."attempts; aborting.\n",
date("Y-m-d H:i:s"));
return false;
}
}
// We can safely use the first $hash entry since we've
// already sorted it in ascending order above, so this
// is the time furthest back
$first = $this->_first_article_by_date($group, $hash[0]['ref']);
if($first === false){
// Fail
printf("%s WARNING - Could not determine starting article. "
."skipping group.\n", date("Y-m-d H:i:s"));
continue;
}
// Group Processing Initialization
$processed = 0;
$batch = $last = intval($summary['last']);
$total = $last-$first;
// Track how many records were inserted, updated
$inserted = 0;
$updated = 0;
printf("%s INFO - Fetching records(s) ", date("Y-m-d H:i:s"));
// Select Group
while($fetch_okay && $processed < $total)
{
try
{
// Prepare Initial Batch
if ($total > SpotNab::FETCH_USENET_BATCH)
$batch = $first + SpotNab::FETCH_USENET_BATCH;
// Batch Processing
while ($processed < $total)
{
$headers = $this->_get_headers($group,
"$first-$batch", $retries);
if($headers === false){
// Retry Atempts exausted
$fetch_okay = false;
break;
}
// Process the header batch
$saved = $this->process_headers($headers, $hash);
if($saved !== false)
{
$inserted += $saved[0];
$updated = $saved[1];
}
$processed += ($batch-$first);
// Increment starting index
$first += ($batch-$first);
if ($last-$first >= SpotNab::FETCH_USENET_BATCH){
// Fetch next batch
$batch = $first + SpotNab::FETCH_USENET_BATCH;
}else{
$batch = $last;
}
//echo "$first-$batch, processed=$processed\n";
//print_r($headers);
}
}catch(Exception $e){
// Reset Connection
$fetch_okay = $this->_nntpReset($group);
// Track retry attempts
$retries--;
if($retries <= 0){
// Retry Atempts exausted
$fetch_okay = false;
break;
}
continue;
}
}
echo " Done\n";
printf("%s INFO - Results: %d new and %d updated comment(s).\n",
date("Y-m-d H:i:s"), $inserted, $updated);
}
// Restore handler
restore_error_handler();
// Ensure We're not connected
try{$this->_nntp->doQuit();}
catch(Exception $e)
{/* do nothing */}
return $fetch_okay;
}
// ***********************************************************************
private function _first_article_by_date($group, $refdate, $retries=3){
// fetches the first article starting at the time specified
// by ref time.
//
// ref time is expected to be a local time in format:
// YYYY-MM-DD hh:mm:ss or as integer
//
// This function returns the first message id to scan from
// based on the time specified. If no articles are found
// or something bad happens, false is returned.
$interval = 1;
// if we start charting into an area where retention period
// isn't present, we're dealing with a blank/dead record that
// is lost.... to many blanks and we have to abort.
$misses = 0;
if(is_string($refdate)){
// Convert to Integer (Local Time)
$refdate = strtotime($refdate);
}
printf("%s INFO - Scanning %s for start article",
date("Y-m-d H:i:s"), $group);
while(($retries > 0) && ($interval > 0)){
$summary = $this->_nntp->selectGroup($group);
if(PEAR::isError($summary))
{
// Reset Connection
$this->_nntpReset();
// Track retry attempts
$retries--;
if($retries <= 0){
// Retry Atempts exausted
break;
}
continue;
}
$last = intval($summary['last']);
$first = intval($summary['first']);
$curdate = $lastdate = Null;
$curid = $lastid = false;
$interval = 0;
while($retries > 0){
// Adjust Interval
if(($last - $first) > 3)
$interval = floor(($last - $first)/2);
else
$interval = 1;
if($misses >= SpotNab::FETCH_MAX_MISSES){
// Misses reached
$last = intval($summary['last']);
if (($last-$lastid) > SpotNab::FETCH_MAXIMUM_HEADERS){
// We exceeded our maximum header limit
// adjust accordingly
$lastid = $last - SpotNab::FETCH_MAXIMUM_HEADERS;
}
// Return pointer
echo " ".($last-$lastid)." record(s) back.\n";
return $lastid;
}
// Swap
$lastdate = $curdate;
$lastid = $curid;
$msgs = $this->_get_headers(
$group, ($last-$interval), $retries);
if($msgs === false){
// fail (retry limit exceeded)
echo "Failed.\n";
return false;
}
// Reset Miss Count
$misses = 0;
// Save Tracker
$curdate = strtotime($msgs[0]['Date']);
$curid = intval($msgs[0]['Number']);
//echo "\nCUR:".date("Y-m-d H:i:s", $curdate)
// ." REF:".date("Y-m-d H:i:s", $refdate)
// ." LAST:".date("Y-m-d H:i:s", $lastdate)
// ." $interval\n";
if($interval == 1){
// We're soo close now...
if($refdate > $curdate && $refdate > $lastdate){
$last = intval($summary['last']);
if (($last-$curid) > SpotNab::FETCH_MAXIMUM_HEADERS){
// We exceeded our maximum header limit
// adjust accordingly
$curid = $last - SpotNab::FETCH_MAXIMUM_HEADERS;
}
// Found content
echo " ".($last-$curid)." record(s) back.\n";
return $curid;
}else if($refdate > $curdate && $refdate > $lastdate){
// Close... Shuffle forward a bit
$first+=2;
}else{
// Slide window and try again
$last-=2;
}
$interval=2;
continue;
}
// Make some noise
if($interval%2)echo ".";
// Adjust Boundaries
if($curdate > $refdate){
// We need to look further forward
$last = $curid+1;
}else if ($curdate <= $refdate){
// We need To look further back
$first = $curid-1;
}
}
}
echo "n/a m:$misses,i:$interval\n";
return false;
}
// ***********************************************************************
public function process_headers($headers, $group_hash, $save=true){
/*
* We iterate over the provided headers (generated by
* $this->_get_headers() to a structure that is at the very
* minimum looking like this:
*
* array (
* [0] => array (
* 'Number': <int>
* 'Subject': <string>
* 'From': <string>
* 'Date': <string>
* 'Message-ID': <string>
* 'Bytes': <int>
* 'Lines': <int>
* 'Epoch': <int>
* ),
* ...
* )
* From the structure above, we process our group hash and retrieve
* all the binary data we need on valid content.
*
* A group_hash() record looks like this:
* array(
* array(
* 'ID': <int>,
* 'key': <string>,
* 'user': <string>,
* 'email': <string>,
* 'ref': <int>,
* ),
* array(
* 'ID': <int>,
* 'key': <string>,
* 'user': <string>,
* 'email': <string>,
* 'ref': <int>,
* ),
* )
*/
if(!count($group_hash)){
// Nothing to process
return array();
}
//
// Prepare some general SQL Commands for saving later if all goes well
//
$db = new DB();
// Comments
// old: $sql_new_cmt = "INSERT INTO spotnab_comments (".
// old: "ID, sourceID, username, releaseHash, commentHash, isVisible, ".
// old: "comment, dateCreated, isSynced) VALUES (".
// old: "NULL, %d, %s, %s, %s, %d, %s, %s, NOW())";
// old: $sql_upd_cmt = "UPDATE spotnab_comments SET ".
// old: "isVisible = %d, comment = %s, isSynced = 1".
// old: "WHERE sourceID = %d AND releaseHash = %s AND commentHash = %s";
// old: $sql_fnd_cmt = "SELECT count(ID) as cnt FROM spotnab_comments ".
// old: "WHERE sourceID = %d AND releaseHash = %s AND commentHash = %s";
$sql_new_cmt = "INSERT INTO releasecomment (".
"ID, sourceID, username, userID, GID, CID, isVisible, ".
"`text`, createddate, isSynced) VALUES (".
"NULL, %d, %s, 0, %s, %s, %d, %s, %s, 1)";
$sql_upd_cmt = "UPDATE releasecomment SET ".
"isVisible = %d, `text` = %s".
"WHERE sourceID = %d AND GID = %s AND CID = %s";
$sql_fnd_cmt = "SELECT count(ID) as cnt FROM releasecomment ".
"WHERE sourceID = %d AND GID = %s AND CID = %s";
// Sync Times
$sql_sync = "UPDATE spotnab_sources SET lastUpdate = %s ".
"WHERE ID = %d";
$matches = Null;
$processed = 0;
$updates = 0;
$inserts = 0;
foreach ($headers as $header){
// Preform some general scanning the header to determine
// if it could possibly be a valid post.
// Make sure the Message-ID is posted using the fixed SEGID_DOMAIN
// <blabl.12423@SEGID_DOMAIN>
if(!preg_match('/^<[^@]+@(?P<segid>.*)>$/',
$header['Message-ID'], $matches))
continue;
if($matches['segid'] != SpotNab::SEGID_DOMAIN)
continue;
// Now we check the subject line; it provides the first part of
// the key to determining if we should handle the message or not
if(!preg_match(SpotNab::FETCH_SUBJECT_REGEX,
$header['Subject'], $matches)){
continue;
}
// We have a match; So populate potential variables
$checksum = $matches['checksum'];
$refdate = $matches['utcref'];
$refdate_epoch = @strtotime($matches['utcref']. " UTC");
if($refdate_epoch === false || $refdate_epoch < 0){
// Bad time specified
continue;
}
// PreKey is used to attempt to run the decode algorithm
// a head of time.. if we can decrypt this we can probably
// assume the body will decode too (and won't be a waste of
// time to download it)
foreach($group_hash as $hash){
// Track how many records we handled
$processed++;
// First check the ref date... if it's newer then what we've
// already processed, then we'll just keep on chugging along.
if($refdate_epoch < $hash['ref']){
continue;
}
// Scan header information for supported matches
if(!preg_match('/^(?P<user>[^<]+)<(?P<email>[^>]+)>$/',
$header['From'], $matches))
continue;
// Match against our sources posts
if(trim($matches['user']) != $hash['user'])
continue;
if(trim($matches['email']) != $hash['email'])
continue;
// If we reach here, we've found a header we can process
// The next step is to download the header's body
// We'll do some final verifications on it such as detect
// if the checksum is okay, and verify that the timestamp
// within the body matches that of the header... then we
// can start processing the guts of the body.
if($save){
// Download Body
$body = $this->_get_body($header['Group'],
$header['Message-ID']);
if($body === false){
continue;
}
//echo "DEBUG Close Match:\n";
//print_r($header);
// Decode Body
$body = $this->decodePost($body, $hash['key']);
if($body === false)
continue; // Decode failed
// Verify Body
if(!is_array($body))
continue; // not any array
if(!(bool)count(array_filter(array_keys($body), 'is_string')))
continue; // not an associative array
if((!array_key_exists('server', $body)) ||
(!array_key_exists('postdate_utc', $body)))
continue; // base structure missing
// Compare postdate_utc and ensure it matches header
// timestamp
if(preg_replace('/[^0-9]/', '',
$body['postdate_utc']) != $refdate)
continue;
// Comment Handling
if(array_key_exists('comments',$body) &&
is_array($body['comments'])){
foreach($body['comments'] as $comment){
// Verify Comment is parseable
if(!is_array($comment))
continue; // not an array
if(!count(array_filter(array_keys($comment))))
continue; // not an associative array
// Store isVisible flag
$is_visible = 1;
if(array_key_exists('is_visible', $comment))
$is_visible = (intval($comment['is_visible'])>0)?1:0;
// Check that comment doesn't already exist
$res = @$db->queryOneRow(sprintf($sql_fnd_cmt,
$hash['ID'],
$db->escapeString($comment['GID']),
$db->escapeString($comment['CID'])));
// Store Results in DB
if($res && intval($res['cnt'])>0){
// Make some noise
echo '.';
$res = $db->query(sprintf($sql_upd_cmt,
$is_visible,
$db->escapeString($comment['comment']),
$hash['ID'],
$db->escapeString($comment['GID']),
$db->escapeString($comment['CID'])
));
$updates += ($db->getAffectedRows()>0)?1:0;
}else{
// Make some noise
echo '+';
// Perform Insert
$res = @$db->query(sprintf($sql_new_cmt,
$hash['ID'],
$db->escapeString($comment['username']),
$db->escapeString($comment['GID']),
$db->escapeString($comment['CID']),
$is_visible,
$db->escapeString($comment['comment']),
// Convert createddate to Local
$db->escapeString($this->utc2local(
$comment['postdate_utc']))
));
$inserts += 1;
}
}
}
// Update spotnab_sources table, set lastUpdate to the
// timestamp parsed from the header.
$db->query(sprintf($sql_sync,
$db->escapeString(
$this->utc2local($body['postdate_utc'])),
$hash['ID']
)
);
}else{
// Debug non/save mode; mark update
$updates += 1;
}
// always break if we made it this far... no mater how many
// other groups are being processed, we've already matched
// for this article, so we don't need to process it for
// other sources.
break;
}
}
return array($inserts, $updates);
}
// ***********************************************************************
private function _get_body($group, $id, $retries=3){
/*
* Fetch the body of a given Message-ID taken from the headers
* The function then returns the raw content
*/
$matches = Null;
if(preg_match("/^\s*<(.*)>\s*$/", $id, $matches))
// Ensure we're always dealing with a properly
// formatted ID
$id = $matches[1];
// The returned result will be stored in $raw
$raw = Null;
do
{
$raw = $this->_nntp->getBody("<".$id.">", true);
if(PEAR::isError($raw))
{
// Reset Connection (This happens first time
// around since _nntpReset() will perform
// the initial connect if it hasn't been already
$this->_nntpReset($group);
// Track retry attempts
$retries--;
if($retries <= 0){
// Retry Atempts exausted
printf("%s ERROR - Failed Segment Fetch: %s/%s ...\n",
date("Y-m-d H:i:s"), $group, $id);
return false;
}
continue;
}
// Retrieved Data
return $raw;
}while($retries > 0);
// Fail
return false;
}
// ***********************************************************************
private function _get_headers($group, $range, $retries=3, $sort=true){
/*
*
* There is to much involved with fetching article headers
* that bloat and make a lot of code repetative...
* This function returns the headers of the specified range
* in an array() of associative array() always to make life
* easy... alternativly, if this function fails then false
* is returned.
*
* We also convert all time scanned into its Epoch value
* with the returned results for easier parsing; this
* is done to order results as well.
*/
// epoch array is used for sorting fetched results
$epoch = array();
// Header parsing for associative array returned
$min_headers = array('Number', 'Subject', 'From', 'Date',
'Message-ID', 'Bytes', 'Lines');
do
{
$msgs = $this->_nntp->getOverview($range, true, false);
if(PEAR::isError($msgs))
{
$this->_nntpReset($group);
// Track retry attempts
$retries--;
if($retries <= 0){
// Retry Atempts exausted
break;
}
continue;
}
// If we get here, then we fetched the header block okay
// Clean up bad results but don't mark fetch as a failure
// just report what it found.. (nothing). We do this because
// PEAR::isError() never threw, so the response has to be valid
// even though it's inconsistent
if(!$msgs)return array();
if(!is_array($msgs))return array();
// For whatever reason, we sometimes get an array of
// associative array returned, and all other times we just
// get an associative array. Convert the associative array
// if we get one to an array of associative array just to
// simplify the response and make it esier to work with
if((bool)count(array_filter(array_keys($msgs), 'is_string'))){
// convert to an array of assocative array
$msgs = array($msgs);
}
for($i=0;$i<count($msgs);$i++){
$skip=false;
foreach($min_headers as $key){
if(!array_key_exists($key, $msgs[$i])){
unset($msgs[$i]);
$i--;
$skip=true;
break;
}
}
if($skip)continue;
// Update Record With Epoch Value (# of sec from Jan, 1980)
$epoch[$i] = $msgs[$i]['Epoch'] = strtotime($msgs[$i]['Date']);
// It's easier to track the group information if it's
// stored with the header segment
$epoch[$i] = $msgs[$i]['Group'] = $group;
}
if($sort && count($msgs)>1)
// Content is already sorted by articles, but if the
// sort flag is specified, then content is re-sorted by the
// messages stored epoch time
array_multisort($epoch, SORT_ASC, $msgs);
return $msgs;
}while($retries >0);
return false;
}
// ***********************************************************************
public function post($reftime=Null, $comments=true, $retries=3){
/*
* This function posts to usenet if there are any new updates
* to report that are flagged for transmit.
*
* The specified $reftime is presumed to be local *not utc*
*/
$reftime_local = $reftime;
$article = Null;
if($reftime_local === Null){
// Fetch local time
$reftime_local = $this->utc2local();
}
// Header
$message = array(
'server' => array(
'code' => $this->_post_site,
'title' => $this->_post_title,
),
'postdate_utc' => $this->local2utc($reftime_local),
'comments' => array()
);
// Store Comments
$message['comments'] = $this->unPostedComments($reftime_local);
$db = new DB();
$sql = sprintf("UPDATE releasecomment "
."SET isSynced = 1 WHERE "
."releaseID != 0 "
."AND userID != 0 "
."AND sourceID = 0 "
."AND isSynced = 0");
//$sql = "UPDATE site SET updateddate = NOW( ) ".
// "WHERE setting = 'spotnabpost' ";
printf("%s INFO - Detected %d new comment(s) ...\n",
date("Y-m-d H:i:s"), count($message['comments']));
// Later this block of code will look like this:
// if($comments || $releases || ... )
if(!is_array($message['comments'])){
// nothing to post; update database to safe future
// fetching time
$res = @$db->query($sql);
return true;
}
if(!($this->_ssl_prvkey && $this->_ssl_pubkey))
{
printf("%s ERROR - Posting not possible without generated "
."SSL keys ...", date("Y-m-d H:i:s"));
return false;
}
printf("%s INFO - Encoding article ...", date("Y-m-d H:i:s"));
// Encode Message so it can be posted
$article = $this->encodePost($message, $reftime_local);
if($article === false){
echo "Failed.\n";
return false;
}
echo "Done.\n";
// Post message
printf("%s INFO - Posting article ...", date("Y-m-d H:i:s"));
if ($this->_postArticle($article, $retries))
{
// Post is good; update database
$res = @$db->query($sql);
echo "Done.\n";
return true;
}
echo "Failed.\n";
// If code reached here, then we failed to post content
return false;
}
// ***********************************************************************
private function _postArticle ($article, $retries=3)
{
// Extract message id
if(!preg_match('/Message-ID: <(?P<id>[^>]+)>/', $article[0], $matches)){
// we couldn't extract the message id
return false;
}
$msg_id = $matches['id'];
set_error_handler('snHandleError');
// Connect to server
$this->_nntp->doConnect();
while($retries > 0)
{
try
{
$summary = $this->_nntp->selectGroup($this->_post_group);
if(PEAR::isError($summary)){
$summary = $this->_nntpReset($this->_post_group);
$retries--;
continue;
}
// Check if server will receive an article
$_err = $this->_nntp->cmdPost();
if (PEAR::isError($_err)) {
$summary = $this->_nntpReset($this->_post_group);
$retries--;
continue;
}
// Actually send the article
$_err = $this->_nntp->cmdPost2($article);
}catch(Exception $e){
// Ensure We're not connected
try{$this->_nntp->doQuit();}
catch(Exception $e)
{/* do nothing */}
// Post failed
$retries--;
// try again
continue;
}
// Now we verify the post worked okay
$summary = $this->_nntp->selectGroup($this->_post_group);
if(PEAR::isError($summary)) {
$summary = $this->_nntpReset($this->_post_group);
$retries--;
continue;
}
$last = intval($summary['last']);
$batch = $last - SpotNab::VERIFY_FETCH_HEADER_COUNT;
$headers = $this->_get_headers($this->_post_group,
"$batch-$last", $retries);
// Ensure We're not connected
try{$this->_nntp->doQuit();}
catch(Exception $e)
{/* do nothing */}
if($headers === false){
// We failed
return false;
}
// Okay we have headers to scan (we'll work back from the end to
// the front to speed the process up)
for($i=count($headers)-1; $i >0; $i--){
if(!array_key_exists('Message-ID', $headers[$i]))
continue;
if ($headers[$i]['Message-ID'] == "<$msg_id>")
// Great! we found the post we just posted
return true;
}
return false;
// Restore handler
restore_error_handler();
// We're done
return true;
}
// Restore handler
restore_error_handler();
return false;
}
// ***********************************************************************
private function _nntpReset ($group=Null)
{
// Reset Connection
try{$this->_nntp->doQuit();}
catch(Exception $e)
{/* do nothing */}
// Attempt to reconnect
try{$this->_nntp->doConnect();}
catch(Exception $e){return false;}
if($group !== Null)
{
// Reselect group if specified
$summary = $this->_nntp->selectGroup($this->_post_group);
if(PEAR::isError($summary))
return false;
return $summary;
}
return true;
}
// ***********************************************************************
public function nzbToGID($nzbfile){
/*
Scans an nzb file specified and returns the global identifer (GID)
Null is returned if there is a problem.
*/
// Scans an individual NZB file for nfo information
$filename = basename($nzbfile);
// Extract NZB Content (gzip supported)
ob_start();
@readgzfile($nzbfile);
$nzbdata = ob_get_contents();
ob_end_clean();
// Calculate how many bytes we have read into memory
// and compare them against how much free memory we have left...
// if there is a discrepency, we'll need to write the content back
// to disk uncompressed and read it back in a buffered version
$bytes = strlen($nzbdata)."\n";
$free = memory_get_usage(true);
if(($free - $bytes) < (1024*1024*10))
{
// Work with cache files instead of memory to avoid overflowing
$cache = tmpfile();
fwrite($cache, $nzbdata);
fseek($cache, 0);
// Free Memory
unset($nzbdata);
// Read header information
$nzbdata = fread($cache, 1024);
// Gets rid of all namespace definitions
$nzbdata = preg_replace('/xmlns[^=]*="[^"]*"/i', "", $nzbdata, 1);
// Gets rid of all namespace references
$nzbdata = preg_replace("/[a-zA-Z]+:([a-zA-Z]+[=>])/", "$1", $nzbdata);
// Read in the rest of the file now
while (!feof($cache))
$nzbdata .= fread($cache, 8192);
fclose($cache);
}
else
{
// Gets rid of all namespace definitions
$nzbdata = preg_replace('/xmlns[^=]*="[^"]*"/i', "", $nzbdata, 1);
// Gets rid of all namespace references
$nzbdata = preg_replace("/[a-zA-Z]+:([a-zA-Z]+[=>])/", "$1", $nzbdata);
}
// Set error Handler
set_error_handler('snHandleError');
try{
$xml = new SimpleXMLElement($nzbdata);
}catch(Exception $e){
// unparseable
return Null;
}
// NZB Information is no longer needed now
// free it's memory up
unset($nzbdata);
// Restore handler as any future errors really are... code errors :)
restore_error_handler();
// Fetch First Segment of the first set of segments of first file
$segment = $xml->xpath("//file[1]/segments[1]/segment[1]");
if(is_array($segment) && !count($segment))
{
// This happens if a nzb file exists with no segment entries
// This is a critical bug some-where else in the code, but we can
// never post GID's using this information
return Null;
}
$segment = "".(is_array($segment)?$segment[0]:$segment);
if(empty($segment))
return Null;
// Return Global Identifer
return md5($segment);
}
// ***********************************************************************
public function getRandomStr($len) {
// Valid Characters
static $vc = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890';
$unique = '';
for($i = 0; $i < $len; $i++)
$unique .= $vc[mt_rand(0, strlen($vc) - 1)];
return $unique;
}
// ***********************************************************************
public function decodePost($message, $key=Null, $decrypt=true) {
// Decode Yenc
$message = $this->_nntp->decodeYenc($message);
// Decompress Messsage
$message = @gzuncompress($message);
if ($key === Null)
$key = $this->_post_key;
// Decrypt Message
if($decrypt){
$message = $this->decrypt($message, $key);
if($message === false){
// fail
return false;
}
}
// Convert from base64
$message = base64_decode($message);
if($message === false){
// Fail
return false;
}
$message = json_decode($message, true);
if($message === false){
// Fail
return false;
}
return $message;
}
// ***********************************************************************
public function encodePost($message, $reftime=Null, $debug=False) {
/*
Assembles and encodes a message ready to be posted onto
a usenet server.
false is returned if the function fails,
If a reftime is specified, it is presumed that it will be in
an integer format and it will be localtime
If the debug is set to true, then a third part of the
article is returned containing header information that would
look as though _get_header() returned it
*/
// Assumed to be in Y-m-d H:i:s format or int
// convert local time into UTC
$reftime = $this->local2utc($reftime, "YmdHis");
$msgid = sprintf('<%s.%d@%s>',
$this->getRandomStr(32),
time(),
SpotNab::SEGID_DOMAIN
);
if(!is_string($message)){
// If message is not already in string format, then
// it's in it's assembled mixed array format... we
// need to convert it to json before proceeding
$message = json_encode($message, JSON_HEX_TAG|JSON_HEX_APOS|
JSON_HEX_QUOT|JSON_HEX_AMP|JSON_UNESCAPED_UNICODE);
if($message === false){
// Fail
return false;
}
}
// nntp posting expects an array as follows:
// array(
// [0] => 'Message Header'
// [1] => 'Message Body'
// );
// Convert to base64
$message = base64_encode($message);
// Encrypt Message
$message = $this->encrypt($message);
if($message === false){
// fail
return false;
}
// Compress Messsage
$message = @gzcompress($message, 9);
// Yenc Binary Content
$message = $this->encodeYenc($message, md5($message));
//
// Prepare Header
//
// Checksum ID
$checksum = sha1($message);
// Prepare Subject
$subject = sprintf("%s-%s",
// checksum against message transmitted
$checksum,
// Save UTC Time
$reftime
);
$site = new Sites();
$header = "Subject: " . $subject . "\r\n";
$header .= "Newsgroups: " . $this->_post_group . "\r\n";
$header .= "Message-ID: $msgid\r\n";
$header .= "X-Newsreader: NewzNab v" . $site->version() ."\r\n";
$header .= "X-No-Archive: yes\r\n";
$user = trim($this->_post_user);
$email = trim($this->_post_email);
$header .= "From: ".$user. " <" . $email . ">\r\n";
// Binary Content
$header .= 'Content-Type: text/plain; charset=ISO-8859-1' . "\r\n";
$header .= 'Content-Transfer-Encoding: 8bit' . "\r\n";
// Assemble Article in structure NNTP expects
$article = array($header, $message);
if($debug){
// Append some debug data to the article
$article[] = array(
'Number' => 1234,
'Subject' => $subject,
'From' => sprintf('%s <%s>', $this->_post_user,
$this->_post_email),
'Date' => date(DATE_RFC822, strtotime(
$this->utc2local($reftime))),
'Message-ID' => $msgid,
'Bytes' => strlen($message),
'Lines' => '1',
'Epoch' => strtotime($this->utc2local($reftime)),
'Group' => 'alt.binaries.test2'
);
}
return $article;
}
// ***********************************************************************
public function unPostedComments() {
/*
* This function returns a list of comments that have not been
* otherwise posted to usenet.
*
* $from and $to will configure themselves if set to NULL
* but otherwise it's expected format is string "Y-m-d H:i:s"
*/
$db = new DB();
// Make sure we ca npost
$sql = "SELECT count(ID) as cnt FROM site "
."WHERE setting = 'spotnabpost' AND value = '1'";
$res = $db->queryOneRow($sql);
if(!$res)
// Nothing, return empty array list
return Null;
if(!intval($res['cnt']))
// Disabled
return Null;
// Now we fetch for any new posts since reference point
$sql = sprintf("SELECT r.GID, rc.ID, rc.text, u.username, "
."rc.isVisible, rc.createddate, rc.host "
."FROM releasecomment rc "
."LEFT JOIN releases r ON r.ID = rc.releaseID AND rc.releaseID != 0 "
."LEFT JOIN users u ON rc.userID = u.ID AND rc.userID != 0 "
."WHERE r.GID IS NOT NULL "
."AND sourceID = 0 AND isSynced = 0");
$res = $db->query($sql);
if(!$res)
return Null;
// Now we prepare a comments array to return with
$comments = array();
foreach($res as $comment){
// If we don't have a GID then we can't make the post;
// the user hasn't set up there database to store the GID's
// correctly
if(empty($comment['GID']))
continue;
// Hash a unique Comment ID to associate with this message
$CID = md5($comment['ID'].$comment['username'].$comment['createddate'].$comment['host']);
$comments[] = array(
// Release Global ID
'GID' => $comment['GID'],
// Comment ID
'CID' => $CID,
// Store comment
'comment' => $comment['text'],
// Store comment
'username' => $comment['username'],
// Store visibility flag
'is_visible' => $comment['isVisible'],
// Convert createddate to UTC
'postdate_utc' => $this->local2utc($comment['createddate'])
);
}
// Return Results if they are present
return (count($comments)>0)?$comments:Null;
}
// ***********************************************************************
public function utc2local($utc=Null, $format="Y-m-d H:i:s") {
/*
* Takes a utc time as input and outputs local
* If no argument is specified then current local
* time is returned.
*/
if(is_string($utc)){
return date($format, strtotime($utc. " UTC"));
}else if(is_int($utc)){
return date($format, strtotime(date($format, $utc)." UTC"));
}
return date($format);
}
// ***********************************************************************
public function local2utc($local=Null, $format="Y-m-d H:i:s") {
/*
* Takes a local time as input and outputs UTC
* If no argument is specified then current UTC
* time is returned.
*/
if(is_string($local)){
return gmdate($format, strtotime($local));
}else if(is_int($local)){
return gmdate($format, $local);
}
return gmdate($format);
}
// ***********************************************************************
public function encodeYenc($message, $filename, $linelen = 128,
$crc32 = true)
{
/* This code was found http://everything2.com/title/yEnc+PHP+Class
* It could probably get placed into the NNTP() code with it's
* partner decodeYenc()
*/
// yEnc 1.3 draft doesn't allow line lengths of more than 254 bytes.
if ($linelen > 254)
$linelen = 254;
if ($linelen < 1)
return false;
$encoded = "";
// Encode each character of the message one at a time.
for( $i = 0; $i < strlen($message); $i++)
{
$value = (ord($message{$i}) + 42) % 256;
// Escape NULL, TAB, LF, CR, space, . and = characters.
if ($value == 0 || $value == 9 || $value == 10 ||
$value == 13 || $value == 32 || $value == 46 ||
$value == 61)
$encoded .= "=".chr(($value + 64) % 256);
else
$encoded .= chr($value);
}
// Wrap the lines to $linelen characters
// TODO: Make sure we don't split escaped characters in half, as per the yEnc spec.
$encoded = trim(chunk_split($encoded, $linelen));
// Tack a yEnc header onto the encoded message.
$encoded = "=ybegin line=$linelen size=".strlen($message)
." name=".trim($filename)."\r\n".$encoded;
$encoded .= "\r\n=yend size=".strlen($message);
// Add a CRC32 checksum if desired.
if ($crc32 === true)
$encoded .= " crc32=".strtolower(sprintf("%04X", crc32($message)));
return $encoded."\r\n";
}
// ***********************************************************************
public function keygen($passphrase=Null, $bits=2048,
$type=OPENSSL_KEYTYPE_RSA)
{
//Generate Key
$res = openssl_pkey_new(
array(
'private_key_bits' => $bits,
'private_key_type' => $type
)
);
// Get Private Key
openssl_pkey_export($res, $prvkey, $passphrase);
// Get Public Key
$details = openssl_pkey_get_details($res);
if($details === false){
return false;
}
$pubkey = $details['key'];
return array(
'pubkey' => $this->compstr($pubkey),
'prvkey' => $this->compstr($prvkey),
);
}
// ***********************************************************************
public function encrypt ($source, $prvkey=Null, $passphrase=Null){
// Encryption performed using private key
if($prvkey === Null)
// Default Key if none is specified
$prvkey = $this->_ssl_prvkey;
if(!$prvkey)
// Still no key...
return false;
// Load Public Key into array
$crypttext='';
$pkey = openssl_get_privatekey($prvkey, $passphrase);
if($pkey === false)
// bad key
return false;
$batch = $len = strlen($source);
$ptr = 0;
$encrypted = '';
while($len > 0){
// Prepare batch size
$batch = (($len - SpotNab::SSL_MAX_BUF_LEN) > 0)?SpotNab::SSL_MAX_BUF_LEN:$len;
$res = openssl_private_encrypt(substr($source, $ptr, $batch), $crypttext, $pkey);
if($res === false){
// Encryption failed
openssl_free_key($pkey);
return false;
}
$encrypted .= $crypttext . SpotNab::SSL_BUF_DELIMITER;
$len -= $batch;
$ptr += $batch;
}
openssl_free_key($pkey);
return $encrypted;
}
// ***********************************************************************
public function decrypt ($source, $pubkey=Null){
// Decryption performed using public key
if($pubkey === Null)
// Default Key if none is specified
$pubkey = $this->_ssl_pubkey;
if(!$pubkey)
// Still no key...
return false;
$pkey = openssl_get_publickey($pubkey);
if($pkey === false){
// bad key
//echo openssl_error_string();
return false;
}
$cryptlist = explode(SpotNab::SSL_BUF_DELIMITER, $source);
$decrypted = '';
foreach($cryptlist as $crypt){
if(!strlen($crypt))break;
$res = openssl_public_decrypt($crypt, $out, $pkey, OPENSSL_PKCS1_PADDING);
if($res === false){
// Decryption failed
//echo "DEBUG: ".openssl_error_string()."\n";
openssl_free_key($pkey);
return false;
}
$decrypted .= $out;
}
openssl_free_key($pkey);
return $decrypted;
}
// ***********************************************************************
public function compstr ($str){
/*
* Compress a string
*/
$str = @gzcompress($str);
return base64_encode($str);
}
// ***********************************************************************
public function decompstr ($str){
/*
* De-compress a string
*/
$str = base64_decode($str);
return @gzuncompress($str);
}
}
if(!$options){ display_help(); exit(1);}
if(!count($options)){ display_help(); exit(1);}
$max_mem = get_max_mem();
if ($max_mem < MIN_MEMORY){
echo "Warning: your memory is very low. Consider running this script as:\n";
echo " #> php -d memory_limit=2G spotnab.php <action>\n";
}
if(array_key_exists("t", $options) ||
array_key_exists("test", $options)){
$sn = new SpotNab();
printf("%s INFO - Testing SSL Key Generator ...",
date("Y-m-d H:i:s"));
$keys = $sn->keygen();
if(is_array($keys) &&
array_key_exists("pubkey", $keys) &&
array_key_exists("prvkey", $keys))
{
$prvkey = $sn->decompstr($keys['prvkey']);
$pubkey = $sn->decompstr($keys['pubkey']);
$refc = $sn->getRandomStr(80);
$refd = $sn->decrypt($sn->encrypt($refc, $prvkey), $pubkey);
echo ($refc == $refd)?"Successful!\n":"Failed!\n";
}else{
echo "Failed!\n";
}
printf("%s INFO - Testing SSL encryption/decryption ...",
date("Y-m-d H:i:s"));
$preMsg = $sn->getRandomStr(800);
$postMsg = $sn->decrypt($sn->encrypt($preMsg));
if($postMsg === false){
echo "Failed!\n";
}else if(!strcmp($preMsg, $postMsg)){
echo "Successful!\n";
}else{
echo "Failed!\n";
}
printf("%s INFO - Testing message encode/decode ...",
date("Y-m-d H:i:s"));
$before = array(
'server' => array(
'code' => 'l2g',
'title' => 'l2g newznab'
),
'postdate_utc' => $sn->local2utc(),
'comments' => array(
array(
'GID' => 'ABCDEFHIJKLMNOPQRSTUVWXYZ0123456',
'CID' => 'ABCDEFHIJKLMNOPQRSTUVWXYZ0123456',
'comment' => 'I just saw l2g blow his nose in his shirt; thats disgusting.',
'username' => 'bb',
'is_visible' => 1,
'postdate_utc' => $sn->local2utc(time()-86400)
),
array(
'GID' => 'ABCDEFHIJKLMNOPQRSTUVWXYZ0123456',
'CID' => 'ABCDEFHIJKLMNOPQRSTUVWXYZ0123456',
'username' => 'sy',
'is_visible' => 1,
'comment' => 'That is really disgusting... l2g is such a dick.',
'postdate_utc' => $sn->local2utc(time()-86000)
)
)
);
$article = $sn->encodePost($before, Null, true);
if($article !== false){
$after = $sn->decodePost($article[1]);
if($before === $after){
echo "Successful!\n";
}else{
echo "Failed!\n";
}
}else{
echo "Failed!\n";
}
printf("%s INFO - Testing UTC/Local conversions [1/4]...",
date("Y-m-d H:i:s"));
$refa = $sn->utc2local();
$refb = $sn->utc2local($sn->local2utc($refa));
echo ($refa == $refb)?"Successful!\n":"Failed!\n";
printf("%s INFO - Testing UTC/Local conversions [2/4]...",
date("Y-m-d H:i:s"));
$refa = $sn->local2utc();
$refb = $sn->local2utc($sn->utc2local($refa));
echo ($refa == $refb)?"Successful!\n":"Failed!\n";
printf("%s INFO - Testing UTC/Local conversions [3/4]...",
date("Y-m-d H:i:s"));
$refa = $sn->utc2local(time());
$refb = $sn->utc2local($sn->local2utc($refa));
echo ($refa == $refb)?"Successful!\n":"Failed!\n";
printf("%s INFO - Testing UTC/Local conversions [4/4]...",
date("Y-m-d H:i:s"));
$refa = $sn->local2utc(time());
$refb = $sn->local2utc($sn->utc2local($refa));
echo ($refa == $refb)?"Successful!\n":"Failed!\n";
printf("%s INFO - Testing fake usenet parse ...",
date("Y-m-d H:i:s"));
// Fake group hash table
$hash = array(array(
'ID' => 0,
'key' => $sn->decompstr($keys['pubkey']),
'user' => 'NNTP',
'email' => 'SPOT@NNTP.COM',
// We want to find new content, so to make our header
// new, we need to take our ref time and back down
// one second so it can be processed..
'ref' => $article[2]['Epoch']-1
));
// Fake headers (use debug information from encodePost)
$headers = array($article[2]);
$matched = $sn->process_headers($headers, $hash, false);
if($matched !== false){
$inserted = $matched[0];
$updated = $matched[1];
echo ($matched > 0)?"Successful!\n":"Failed!\n";
}else{
echo "Failed\n";
}
}
$delete_broken_releases = false;
if(array_key_exists("G", $options) ||
array_key_exists("populate-fix-gid", $options)){
// Toggle Broken Release Fix
$delete_broken_releases = true;
// Force populate-gid cleanup
$options['populate-gid'] = true;
}
if(array_key_exists("g", $options) ||
array_key_exists("populate-gid", $options)){
$db = new DB();
$nzb = new NZB();
$sn = new SpotNab();
$processed = 0;
printf("%s INFO - Populating GID in releases table.\n", date("Y-m-d H:i:s"));
$fsql = 'SELECT ID, name, guid FROM releases WHERE '
.'GID IS NULL ORDER BY adddate DESC LIMIT %d,%d';
$usql = "UPDATE releases SET GID = '%s' WHERE ID = %d";
$offset = 0;
while(1){
$res = $db->query(sprintf($fsql, $offset, 5000));
if(!($res && count($res) > 0))
break;
foreach ($res as $r){
$nzbfile = $nzb->getNZBPath($r["guid"]);
if($nzbfile === Null){
printf("\n%s WARNING - Could not extract NZB file "
."information for: %d/%s ...\n", date("Y-m-d H:i:s"),
intval($r["ID"]), $r["name"]);
$offset++;
continue;
}
if(!is_file($nzbfile)){
if($delete_broken_releases){
$release = new Releases();
$release->delete($r['ID']);
printf("\n%s WARNING - Fixed broken release: %s ...\n",
date("Y-m-d H:i:s"), $r['name']);
// Free the variable in an attempt to recover memory
unset($release);
}else{
printf("\n%s WARNING - Missing NZB File: %d/%s ...\n",
date("Y-m-d H:i:s"), intval($r["ID"]), $r["name"]);
$offset++;
}
continue;
}
// Increment Process Count
$processed += 1;
$gid = $sn->nzbToGID($nzbfile);
// Free the variable in an attempt to recover memory
unset($nzbfile);
if(!$gid){
if($delete_broken_releases){
$release = new Releases();
$release->delete($r['ID']);
printf("\n%s WARNING - Fixed broken release: %s ...\n",
date("Y-m-d H:i:s"), $r['name']);
// Free the variable in an attempt to recover memory
unset($release);
}else{
printf("\n%s WARNING - Failed to retrieve GID for: %s ...\n",
date("Y-m-d H:i:s"), $r['name']);
$offset++;
}
continue;
}
// Update DB With Global Identifer
$ures = $db->query(sprintf($usql, $gid, $r['ID']));
if($db->getAffectedRows() < 0){
printf("\n%s ERROR - Failed to update: %s ...\n",
date("Y-m-d H:i:s"), $r['name']);
}
// make noise...
echo '.';
}
}
if($processed)echo "\n";
printf("%s INFO - Processed: %d record(s)\n",
date("Y-m-d H:i:s"), $processed);
printf("%s INFO - Updating GID in releasecomments table ...", date("Y-m-d H:i:s"));
# Batch update for comment table
$usql = 'UPDATE releasecomment, releases '
.'SET releasecomment.GID = releases.GID '
.'WHERE releases.ID = releasecomment.releaseID '
.'AND releasecomment.GID IS NULL '
.'AND releases.GID IS NOT NULL ';
// Reset Processed Count
$processed = 0;
$res = $db->query(sprintf($usql));
$affected = $db->getAffectedRows();
if($affected >= 0)
$processed += $affected;
echo " Done\n";
printf("%s INFO - Processed: %d comments(s)\n",
date("Y-m-d H:i:s"), $processed);
}
$force_keygen_save = false;
if(array_key_exists("K", $options) ||
array_key_exists("force-keygen", $options)){
// Enable flag to force keygen to save always
$force_keygen_save = true;
// Ensure keygen is ran
$options['keygen'] = true;
}
if(array_key_exists("k", $options) ||
array_key_exists("keygen", $options)){
$do_keygen = true;
$db = new DB();
$sn = new SpotNab();
$sql_prv = "SELECT value from site ".
"WHERE setting = 'spotnabsiteprvkey'";
$sql_pub = "SELECT value from site ".
"WHERE setting = 'spotnabsitepubkey'";
if($force_keygen_save === false){
$do_keygen = false;
// Check to see if keys are already generated first
// before continuing.
$res_prv = $db->queryOneRow($sql_prv);
$res_pub = $db->queryOneRow($sql_pub);
if($res_pub && $res_prv){
$prvkey = $sn->decompstr($res_prv['value']);
$pubkey = $sn->decompstr($res_pub['value']);
$str_in = $sn->getRandomStr(80);
$str_out = $sn->decrypt($sn->encrypt($str_in, $prvkey), $pubkey);
if($str_in != $str_out){
$do_keygen = true;
}
}
}
if($do_keygen)
{
printf("%s INFO - Generating new SSL Keys\n",
date("Y-m-d H:i:s"));
$keys = $sn->keygen();
if(is_array($keys)){
// Save Keys
$sql = sprintf("UPDATE site SET value = %s ".
"WHERE setting = 'spotnabsitepubkey'",
$db->escapeString($keys['pubkey']));
$db->query($sql);
printf("%s INFO - Saved new public key.\n",
date("Y-m-d H:i:s"));
//echo $keys['pubkey']."\n";
$sql = sprintf("UPDATE site SET value = %s ".
"WHERE setting = 'spotnabsiteprvkey'",
$db->escapeString($keys['prvkey']));
$db->query($sql);
printf("%s INFO - Saved new private key.\n",
date("Y-m-d H:i:s"));
}
}
$res_pub = $db->queryOneRow($sql_pub);
if($res_pub)
printf("SPOTNAB PUBLIC KEY (Begin copy from next line):\n%s\n", $res_pub['value']);
}
if(array_key_exists("p", $options) ||
array_key_exists("spotnab-post", $options)){
$sn = new SpotNab();
$sn->post();
}
if(array_key_exists("f", $options) ||
array_key_exists("spotnab-fetch", $options)){
$sn = new SpotNab();
$sn->fetch();
}
Reference in New Issue View Git Blame Copy Permalink