This commit is contained in:
Darko
2013-08-16 11:52:11 +02:00
parent 3396daeff4
commit e8c3f60183
11 changed files with 9 additions and 4826 deletions
+4 -4
View File
@@ -49,10 +49,10 @@ if [[ $KEVIN_SAFER == "true" ]] || [[ $PARSING_MOD == "true" ]]; then
cp -frv * $NEWZPATH/www/lib/
fi
#copy needed files for hash_decrypt and fixReleaseNames scripts
if [[ $HASH == "true" ]] || [[ $FIXRELEASES == "true" ]]; then
cd $DIR"/test/files to copy/www/lib"
cp -frv * $NEWZPATH/www/lib/
fi
#if [[ $HASH == "true" ]] || [[ $FIXRELEASES == "true" ]]; then
# cd $DIR"/test/files to copy/www/lib"
# cp -frv * $NEWZPATH/www/lib/
#fi
#set user/group to www
echo "Fixing permisions, this can take some time if you have a large set of releases"
File diff suppressed because it is too large Load Diff
-242
View File
@@ -1,242 +0,0 @@
<?php
require_once(WWW_DIR . "/lib/framework/cache.php");
class DB
{
//
// the element relstatus of table releases is used to hold the status of the release
// The variable is a bitwise AND of status
// List of processed constants - used in releases table. Constants need to be powers of 2: 1, 2, 4, 8, 16 etc...
const NFO_PROCESSED_NAMEFIXER = 1; // We have processed the release against its .nfo file in the namefixer
const PREHASH_PROCESSED_NAMEFIXER = 2; // We have processed the release against a predb name
private static $initialized = false;
private static $mysqli = null;
private static $usingInnoDB = null;
private static $batchSize = 1000;
function DB()
{
if (DB::$initialized === false) {
if(defined('DB_PORT')){
DB::$mysqli = new mysqli(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME, DB_PORT);
}else{
DB::$mysqli = new mysqli(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME);
}
if (mysqli_connect_errno()) {
printf("Fatal error: %s", mysqli_connect_error());
exit();
}
DB::$mysqli->select_db(DB_NAME)
or die("Fatal error: could not select database! Check your config.");
DB::$mysqli->set_charset("utf8");
DB::$usingInnoDB = defined('DB_INNODB') ? DB_INNODB : false;
DB::$initialized = true;
}
}
public function getBatchSize()
{
return DB::$batchSize;
}
public function escapeString($str)
{
return "'" . DB::$mysqli->real_escape_string($str) . "'";
}
public function makeLookupTable($rows, $keycol)
{
$arr = array();
foreach ($rows as $row)
$arr[$row[$keycol]] = $row;
return $arr;
}
public function queryInsert($query, $returnlastid = true)
{
if($query=="")
return false;
$result = DB::$mysqli->query($query);
return ($returnlastid) ? DB::$mysqli->insert_id : $result;
}
public function queryOneRow($query, $useCache = false, $cacheTTL = '')
{
if($query=="")
return false;
$rows = $this->query($query, $useCache, $cacheTTL);
return ($rows ? $rows[0] : false);
}
public function query($query, $useCache = false, $cacheTTL = '')
{
if($query=="")
return false;
if ($useCache) {
$cache = new Cache();
if ($cache->enabled && $cache->exists($query)) {
$ret = $cache->fetch($query);
if ($ret !== false)
return $ret;
}
}
$result = DB::$mysqli->query($query);
if ($result === false || $result === true)
return array();
$rows = array();
while ($row = $this->getAssocArray($result))
$rows[] = $row;
$this->freeResult($result);
if ($useCache)
if ($cache->enabled)
$cache->store($query, $rows, $cacheTTL);
return $rows;
}
public function queryDirect($query, $unbuffered = false)
{
if($query=="")
return false;
if($unbuffered)
{
$ret = DB::$mysqli->query($query, MYSQLI_USE_RESULT);
}
else
{
$ret = DB::$mysqli->query($query);
}
return $ret;
}
public function freeResult($result)
{
$result->free_result();
}
//*addedd from nZEDb for testing
public function fetchArray($result)
{
return (is_null($result) ? null : $result->fetch_array());
}
//* end of insert for testing
public function getNumRows($result)
{
return $result->num_rows;
}
public function disableAutoCommit()
{
if (DB::$usingInnoDB == false)
return;
DB::$mysqli->autocommit(false);
}
public function disableForeignKeyChecks()
{
if (DB::$usingInnoDB == false)
return;
$this->query("SET foreign_key_checks=0;");
}
public function enableForeignKeyChecks()
{
if (DB::$usingInnoDB == false)
return;
$this->query("SET foreign_key_checks=1;");
}
public function commit($enableAutoCommit = true)
{
if (DB::$usingInnoDB == false)
return;
DB::$mysqli->commit();
if ($enableAutoCommit == true)
$this->enableAutoCommit();
}
public function rollback($enableAutoCommit = true)
{
if (DB::$usingInnoDB == false)
return;
DB::$mysqli->rollback();
if ($enableAutoCommit == true)
$this->enableAutoCommit();
}
public function enableAutoCommit()
{
if (DB::$usingInnoDB == false)
return;
DB::$mysqli->autocommit(true);
}
public function usingInnoDB()
{
return DB::$usingInnoDB;
}
public function getAssocArray($result)
{
return $result->fetch_assoc();
}
public function getRow($result)
{
return $result->fetch_row();
}
public function optimise($force = false)
{
$ret = array();
if ($force)
$alltables = $this->query("show table status");
else
$alltables = $this->query("show table status where Data_free != 0");
foreach ($alltables as $tablename)
{
$ret[] = $tablename['Name'];
if (strtolower($tablename['Engine']) == "myisam")
$this->queryDirect("REPAIR TABLE `" . $tablename['Name'] . "`");
$this->queryDirect("OPTIMIZE TABLE `" . $tablename['Name'] . "`");
$this->queryDirect("ANALYZE TABLE `" . $tablename['Name'] . "`");
}
return $ret;
}
public function getAffectedRows()
{
return DB::$mysqli->affected_rows;
}
public function getLastError()
{
return DB::$mysqli->error;
}
}
-291
View File
@@ -1,291 +0,0 @@
<?php
require_once(WWW_DIR. "/lib/framework/db.php");
require_once(WWW_DIR. "lib/category.php");
require_once(WWW_DIR."/lib/site.php");
require_once(WWW_DIR."/lib/releases.php");
/**
* This class handles data access for groups.
*/
class Groups
{
/**
* Get all group rows.
*/
public function getAll($orderby=null)
{
$order = ($orderby == null) ? 'name_desc' : $orderby;
$orderArr = explode("_", $order);
switch($orderArr[0]) {
case 'name':
$orderfield = 'groups.name';
break;
case 'description':
$orderfield = 'groups.description';
break;
case 'releases':
$orderfield = 'num_releases';
break;
case 'updated':
$orderfield = 'groups.last_updated';
break;
default:
$orderfield = 'groups.name';
break;
}
$ordersort = (isset($orderArr[1]) && preg_match('/^asc|desc$/i', $orderArr[1])) ? $orderArr[1] : 'desc';
$orderby = $orderfield." ".$ordersort;
$db = new DB();
return $db->query(sprintf("SELECT groups.*, COALESCE(rel.num, 0) AS num_releases
FROM groups
LEFT OUTER JOIN
( SELECT groupID, COUNT(ID) AS num FROM releases group by groupID ) rel ON rel.groupID = groups.ID
ORDER BY %s",$orderby));
}
/**
* Get all group rows for use in a select list.
*/
public function getGroupsForSelect()
{
$db = new DB();
$categories = $db->query("SELECT * FROM groups WHERE active = 1 ORDER BY name");
$temp_array = array();
$temp_array[-1] = "--Please Select--";
foreach($categories as $category)
$temp_array[$category["name"]] = $category["name"];
return $temp_array;
}
/**
* Get a group row by its ID.
*/
public function getByID($id)
{
$db = new DB();
return $db->queryOneRow(sprintf("select * from groups where ID = %d ", $id));
}
/**
* Get all active group rows.
*/
public function getActive()
{
$db = new DB();
return $db->query("SELECT * FROM groups WHERE active = 1 ORDER BY name");
}
/**
* Get a group row by name.
*/
public function getByName($grp)
{
$db = new DB();
return $db->queryOneRow(sprintf("select * from groups where name = '%s' ", $grp));
}
public function getByNameByID($id)
{
$db = new DB();
$res = $db->queryOneRow(sprintf("select name from groups where ID = %d ", $id));
return $res["name"];
}
/**
* Get count of all groups, filter by name.
*/
public function getCount($groupname="", $activeonly=false)
{
$db = new DB();
$grpsql = '';
if ($groupname != "")
$grpsql .= sprintf("and groups.name like %s ", $db->escapeString("%".$groupname."%"));
if ($activeonly == true)
$grpsql .= "and active=1 ";
$res = $db->queryOneRow(sprintf("select count(ID) as num from groups where 1=1 %s", $grpsql));
return $res["num"];
}
/**
* Get groups rows for browse list by limit.
*/
public function getRange($start, $num, $groupname="", $activeonly=false)
{
$db = new DB();
if ($start === false)
$limit = "";
else
$limit = " LIMIT ".$start.",".$num;
$grpsql = '';
if ($groupname != "")
$grpsql .= sprintf("and groups.name like %s ", $db->escapeString("%".$groupname."%"));
if ($activeonly == true)
$grpsql .= "and active=1 ";
$sql = sprintf("SELECT groups.*, COALESCE(rel.num, 0) AS num_releases
FROM groups
LEFT OUTER JOIN
(
SELECT groupID, COUNT(ID) AS num FROM releases group by groupID
) rel ON rel.groupID = groups.ID WHERE 1=1 %s ORDER BY groups.name ".$limit, $grpsql);
return $db->query($sql);
}
/**
* Add a new group row.
*/
public function add($group)
{
$db = new DB();
if ($group["minfilestoformrelease"] == "" || $group["minfilestoformrelease"] == "0")
$minfiles = 'null';
else
$minfiles = $group["minfilestoformrelease"] + 0;
if ($group["minsizetoformrelease"] == "" || $group["minsizetoformrelease"] == "0")
$minsizetoformrelease = 'null';
else
$minsizetoformrelease = $db->escapeString($group["minsizetoformrelease"]);
if ($group["backfill_target"] == "" || $group["backfill_target"] == "0")
$backfill_target = '0';
else
$backfill_target = $group["backfill_target"] + 0;
$first = (isset($group["first_record"]) ? $group["first_record"] : "0");
$last = (isset($group["last_record"]) ? $group["last_record"] : "0");
$sql = sprintf("insert into groups (name, description, first_record, last_record, last_updated, active, minfilestoformrelease, minsizetoformrelease, backfill_target) values (%s, %s, %s, %s, null, %d, %s, %s, %d) ",$db->escapeString($group["name"]), $db->escapeString($group["description"]), $db->escapeString($first), $db->escapeString($last), $group["active"], $minfiles, $minsizetoformrelease, $backfill_target);
return $db->queryInsert($sql);
}
/**
* Delete a group.
*/
public function delete($id)
{
$db = new DB();
return $db->query(sprintf("delete from groups where ID = %d", $id));
}
/**
* Reset all stats about a group, like its first_record.
*/
public function reset($id)
{
$db = new DB();
return $db->query(sprintf("update groups set backfill_target=0, first_record=0, first_record_postdate=null, last_record=0, last_record_postdate=null, last_updated=null where ID = %d", $id));
}
/**
* Reset all stats about a group and delete all releases and binaries associated with that group.
*/
public function purge($id)
{
require_once(WWW_DIR."/lib/binaries.php");
$db = new DB();
$releases = new Releases();
$binaries = new Binaries();
$this->reset($id);
$rels = $db->query(sprintf("select ID from releases where groupID = %d", $id));
foreach ($rels as $rel)
$releases->delete($rel["ID"]);
$bins = $db->query(sprintf("select ID from binaries where groupID = %d", $id));
foreach ($bins as $bin)
$binaries->delete($bin["ID"]);
}
/**
* Update a group row.
*/
public function update($group)
{
$db = new DB();
if ($group["minfilestoformrelease"] == "" || $group["minfilestoformrelease"] == "0")
$minfiles = 'null';
else
$minfiles = $group["minfilestoformrelease"] + 0;
if ($group["minsizetoformrelease"] == "" || $group["minsizetoformrelease"] == "0")
$minsizetoformrelease = 'null';
else
$minsizetoformrelease = $db->escapeString($group["minsizetoformrelease"]);
return $db->query(sprintf("update groups set name=%s, description = %s, backfill_target = %s , active=%d, minfilestoformrelease=%s, minsizetoformrelease=%s where ID = %d ",$db->escapeString($group["name"]), $db->escapeString($group["description"]), $db->escapeString($group["backfill_target"]),$group["active"] , $minfiles, $minsizetoformrelease, $group["id"] ));
}
/**
* Update the list of newsgroups from nntp provider matching a regex and return an array of messages.
*/
function addBulk($groupList, $active = 1)
{
require_once(WWW_DIR."/lib/binaries.php");
require_once(WWW_DIR."/lib/nntp.php");
$ret = array();
if ($groupList == "")
{
$ret[] = "No group list provided.";
}
else
{
$db = new DB();
$nntp = new Nntp;
if (!$nntp->doConnect()) {
$ret[] = "Failed to get NNTP connection";
return $ret;
}
$groups = $nntp->getGroups();
$nntp->doQuit();
$regfilter = "/(" . str_replace (array ('.','*'), array ('\.','.*?'), $groupList) . ")$/";
foreach($groups AS $group)
{
if (preg_match ($regfilter, $group['group']) > 0)
{
$res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $db->escapeString($group['group'])));
if($res)
{
$db->query(sprintf("UPDATE groups SET active = %d where ID = %d", $active, $res["ID"]));
$ret[] = array ('group' => $group['group'], 'msg' => 'Updated');
}
else
{
$desc = "";
$db->queryInsert(sprintf("INSERT INTO groups (name, description, active) VALUES (%s, %s, %d)", $db->escapeString($group['group']), $db->escapeString($desc), $active));
$ret[] = array ('group' => $group['group'], 'msg' => 'Created');
}
}
}
}
return $ret;
}
/**
* Update a group to be active/inactive.
*/
public function updateGroupStatus($id, $status = 0)
{
$db = new DB();
$db->query(sprintf("UPDATE groups SET active = %d WHERE id = %d", $status, $id));
$status = ($status == 0) ? 'deactivated' : 'activated';
return "Group $id has been $status.";
}
}
-712
View File
@@ -1,712 +0,0 @@
<?php
require_once(WWW_DIR."/lib/framework/db.php");
require_once(WWW_DIR."/lib/movie.php");
require_once(WWW_DIR."/lib/tvrage.php");
require_once(WWW_DIR."/lib/nntp.php");
require_once(WWW_DIR."/lib/nzb.php");
require_once(WWW_DIR."/lib/nzbinfo.php");
require_once(WWW_DIR."/lib/rarinfo/par2info.php");
// Silent Error Handler (used to shut up noisy XML exceptions)
// We don't care if the nzb is corrupt with so many additional lines of
// info... that is someone elses problem and defeats the purpose
// and readability of this script... we output a more relaxed
// error in these events
// we use the silent error handler for remote connection failures as well
function nfoHandleError($errno, $errstr, $errfile, $errline, array $errcontext){
if (0 === error_reporting())
return false;
if(!defined('E_STRICT'))define('E_STRICT', 2048);
switch($errno){
case E_WARNING:
case E_NOTICE:
case E_STRICT:
return;
default:
break;
};
throw new ErrorException($errstr, 0, $errno, $errfile, $errline);
}
/**
* This class looks up nfo files and handles storage/retrieval of them from the database.
*/
class Nfo
{
/**
* @access public
* @var how many releases to process at once (hence 1 release
* could have more then 10 possible nfo matches
*/
const NNTP_BATCH_COUNT = 10;
/**
* @access public
* @var Number of retries to usenet on a failed batch fetching before
* giving up a batch and moving on
*/
const USENET_RETRY_COUNT = 5;
/**
* @access public
* @var Defines the maximum size a single segment can be before we can rule
* after a binary has matched a releaseregex
*/
const NFO_MAX_FILESIZE = 50000;
/**
* @access public
* @var Database flag for no NFO found
*/
const FLAG_NFO_MISSING = -1;
/**
* @access public
* @var Database flag NFO pending scan
*/
const FLAG_NFO_PENDING = 0;
/**
* Default constructor.
*/
function Nfo($verbose=false, $use_obfuscated=true, $use_fuzzy=false)
{
$this->use_fuzzy=$use_fuzzy;
$this->use_obfuscated=$use_obfuscated;
$this->verbose=$verbose;
}
private function nfo_scan(&$nzbInfo){
//
// Phase 1, iterate over nzb file for a relative
// match on a possible nfo file
// - 1 segment
// - within byte size
//
// Array of all possible matches to return
$nfo_idx = array();
// Search for all entries that have a single segment
if (empty($nzbInfo->segmentfiles))
// Nothing to Return
return array();
// Fetch Meta Information
if (isset($nzbInfo->metadata['name']))
$name = $nzbInfo->metadata['name'];
else
$name = ""; //???
$unordered_list=array();
foreach($nzbInfo->segmentfiles as $segment){
if ($segment['filesize'] > Nfo::NFO_MAX_FILESIZE)
continue;
$unordered_list[] = array(
"name" => $name,
"subject" => $segment['subject'],
"bytes" => $segment['filesize'],
"segment" => $segment['segments'],
"groups" => $segment['groups']
);
}
//
// Filter built list above based on subject line info
// hence... eliminate par2, nzb ... etc files
//
// processing is done in 2 steps, the first step finds the
// most likely .nfo files, while the second keeps a backup
// of potential others in the unlikelyhood the content
// parsed here is bad
//
foreach($unordered_list as $idx => $n){
if (preg_match("/\.(nfo)([^a-z0-9]+|$)/i", $n["subject"])){
if($this->verbose) echo "[nfo] ";
$nfo_idx[]=$n;
}
}
// Handle Releases with Obfuscation
// Releases titled: f4ca0f95896da1d41254bf49791a86a2
if($this->use_obfuscated)
foreach($unordered_list as $idx => $n){
if (preg_match("/\.(sfv)([^a-z0-9]+|$)/i", $n["subject"]))
continue;
if (preg_match("/\.(nzb)([^a-z0-9]+|$)/i", $n["subject"]))
continue;
if (preg_match("/\.[0-9]+([^a-z0-9\.-]+|$)/i", $n["subject"])){
if($this->verbose) echo "[obfs] ";
$nfo_idx[]=$n;
}
}
// Fuzzy Parsing sees if it can identify other possible nfo's however
// they are appended to the end of the list obvious nfo's are always
// processed first
if($this->use_fuzzy)
foreach($unordered_list as $idx => $n){
if (preg_match("/\.(txt|diz)([^a-z0-9]+|$)/i", $n["subject"])){
if($this->verbose) echo "[fuzz] ";
$nfo_idx[]=$n;
}
}
// Return array of matched content in order (aprox)
// from very possible to... possible...
return $nfo_idx;
}
private function is_binary(&$raw){
// Returns true if data passed in is binary, otherwise
// returns false,
$has_binary = (
0 or substr_count($raw, "^\r\n")/512 > 0.3
or substr_count($raw, "^ -~")/512 > 0.3
or substr_count($raw, "\x00") > 0
);
if($has_binary)
{
// Before we rule it completely out, see if we can detect it
// as utf-16
$result = iconv($in_charset = 'UTF-16LE' , $out_charset = 'UTF-8', $raw);
if (false !== $result)
{
// not binary, we decoded it
// we're dealing with a utf-16 type file...
// store it as utf-8
$raw = $result;
return false;
}
}
// Return the binary flag
return ($has_binary)?true:false;
}
private function is_par2(&$raw){
// Returns true if data passed in is binary, otherwise
// returns false,
return (substr($raw, 4) == "PAR2");
}
private function is_sfv(&$raw){
// scan a content and return true if it is detected to be
// an sfv file, otherwise return false
// First we identify acceptable sfv lines, anything that
// does not match against the below causes this function to
// exit gracefully and report that were not dealing with
// an sfv file
$sfv_regex = array(
// the sfv information itself
'/^\s*([^; \t]+)\s+([^; \t]+)[ \t]*(;|$)/',
// sfv comments
'/^\s*;/',
// empty lines that contain nothing
'/^$/',
);
// itreate over each line of file, if all regex's match
// on every line then we are dealing with an sfv file
foreach(preg_split("/((\r?\n)|(\r\n?))/", $raw) as $line){
$matches=false;
foreach($sfv_regex as $regex)
if(preg_match($regex, $line)){
$matches=true;
break;
}
if(!$matches)
return false;
}
return true;
}
private function store_blob($nfometa, $blobhash, $removed){
// This takes a array of blobs with their index id being
// the release id; In the event we fetch the data and deem
// it no good, we need to add it to the skipped array which
// must be passed into the function.
$db = new DB();
$db->disableAutoCommit();
foreach($blobhash as $uid => $blob){
$query = sprintf(
"REPLACE INTO releasenfo (ID, releaseID, binaryID, nfo) ".
"VALUES (NULL, %d, 0, compress(%s));",
$uid, $db->escapeString($blob));
$id = $db->queryInsert($query);
if(!$id){
if($this->verbose) echo "!";
}else{
$query = sprintf("UPDATE releases SET releasenfoID = %d WHERE ID = %d LIMIT 1",
$id, $uid);
$res = $db->query($query);
if($this->verbose) echo "s";
}
}
$db->commit(false);
// Now we update the database with entries that have no nfo files
// associated with the release
foreach($removed as $uid){
$res = $this->setNfoMissing($uid);
if($db->getAffectedRows() <= 0){
if($this->verbose) echo "!";
}else{
if($this->verbose) echo "s";
}
}
$db->commit(); // re-enables auto-commit
}
private function parse_blobs(&$nfometa, &$nfoblob){
// Parses an array of array of blobs and determines the most
// ideal nfo from them.
//
// $nfoblob is expected as follows
//
// $nfoblob = array(
// [<releaseID>] = array(
// [0] = <potential nfo file>,
// [1] = <potential nfo file>,
// ...
// ),
// [<releaseID>] = array(
// [0] = <potential nfo file>,
// ),
// ...
// )
//
// Meanwhile, $nfometa is expected as follows:
// $nfometa = array(
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// "alt.binaries.mygroupc",
// ...
// )
// [segment] = array(<segment id>),
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// ...
// )
// ),
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// ...
// )
// [segment] = array(<segment id>),
// ),
// ...
// The function strips indexes that appear invalid
// and stores the most ideal match per release
$parsed_blob = array();
$parsed_meta = array();
foreach($nfometa as $uid => $info){
$ideal = Null;
$tossed = 0;
$total = count($info);
foreach($info as $idx => $entry){
// No magic yet here... to come soon!
// for now we save first 'valid' entry
// Some simple checks right off the top... if there is
// no blob or the data failed to fetch, we can rule
// this entry out right away
if(!array_key_exists($uid, $nfoblob)){
if($this->verbose) echo '-';
continue;
}
if(!array_key_exists($idx, $nfoblob[$uid])){
if($this->verbose) echo '-';
continue;
}
if($nfoblob[(string)$uid][$idx] === Null){
if($this->verbose) echo '-';
continue;
}
// Eliminate detected xml (usually nzb) files
if(preg_match('/xmlns[^=]*="[^"]*"/i', $nfoblob[$uid][$idx]) ||
preg_match("/(\<\?xml[\d\D]*\?\>)/i", $nfoblob[$uid][$idx])){
if($this->verbose) echo '-';
continue;
}
// We do not want to pick up sfv files
if($this->is_sfv($nfoblob[$uid][$idx])){
if($this->verbose) echo '-';
continue;
}
// We do not want to pick up par2 files
if($this->is_par2($nfoblob[$uid][$idx])){
if($this->verbose) echo '-';
continue;
}
// Ideally if code reaches this far
// we can assume we've matched and we
// san skip further parsing
$ideal = $idx;
break;
}
if($ideal !== Null){
// An ideal match was found
$parsed_blob[(string)$uid] = $nfoblob[$uid][$ideal];
$parsed_meta[(string)$uid] = $nfometa[$uid][$ideal];
if($this->verbose) echo '+';
}else{
// No valid data
unset($parsed_blob[(string)$uid]);
unset($parsed_meta[(string)$uid]);
}
}
// perform swap with new parsed data by elminating the array containing
// the possible matches with the absolute match itself...
// no longer is anyone dealing with an array of array after calling
// this function
$nfoblob = $parsed_blob;
$nfometa = $parsed_meta;
return count($nfoblob);
}
private function _nfo_grab($nfometa, &$blobhash){
// nfometa should be an array() of segments from nzb file
// it will then populate the blobhash which uses the segments
// as hash entries for the blob data.
// nfometa is an array of arrays simiar to the following
// structure:
//
// The list is structured in such a way that the most ideal
// matches are at the front, while less likely ones at the
// back of the array
//
// $nfometa = array(
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// "alt.binaries.mygroupc",
// ...
// )
// [segment] = array(<segment id>),
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// ...
// )
// ),
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// ...
// )
// [segment] = array(<segment id>),
// ),
// ...
// )
$nntp = new Nntp();
// Connect to server (we throw an exception if we fail) which
// is caught upstairs with the nfo_grab() function
// no error handling is needed here
$nntp->doConnect(1, true);
foreach($nfometa as $uid => $matches){
$blobhash[$uid] = array();
foreach($matches as $idx => $match){
$fetched = false;
foreach($match["groups"] as $group){
// Don not try other groups if we already got it
if($fetched)break;
// Select the group and then attempt to fetch the article
$blob = $nntp->getMessages($group, $match["segment"], false);
if ($blob === false){
if($this->verbose) echo '*';
continue;
}
// Mark that we fetched it to prevent fetching more
// of the same thing
$fetched = true;
if($this->verbose) echo '.';
// Update blob with decrypted version and store
if ($this->is_binary($blob)){
// Binary data is not acceptable, we only
// work with text from here on out.
continue;
}
// Read-able ascii at this point... store it
$blobhash[$uid][$idx] = $blob;
}
if(!$fetched)
// handle empty/failed segments
$blobhash[$uid][$idx] = Null;
}
}
$nntp->doQuit();
}
private function nfo_grab($nfometa, &$blobhash){
// It is possible for connection to drop while attempting
// to fetch nfo content, to accomodate for the exceptions
// thrown during this time we wrap the real nfo_grab()
// in a try catch block with a silent exception catcher
$retries = nfo::USENET_RETRY_COUNT;
set_error_handler('nfoHandleError');
$_blobhash = array();
while($retries >0){
try{
$this->_nfo_grab($nfometa, $_blobhash);
break;
}catch (Exception $e){
// Connection lost
if($this->verbose) echo sprintf("\n%s Connection lost to usenet (%d retries left).\n",
'NfoProc', $retries);
// Decrement retry count
$retries--;
// Reset blobhash
$_blobhash = array();
continue;
}
}
// Restore handler as any future errors really are... code errors :)
restore_error_handler();
if($retries>0){
foreach ($_blobhash as $k => $v)
$blobhash[(string)$k]=$v;
return true;
}
return false;
}
private function scan_releases(&$processed, &$total, $limit=Null){
// Scan all nzb files whos releases match against data
// that has no nfo files associated with it.
//
// nzb files are further parsed for nfo segments that can
// be extracted and applied to the release
$nzb = new NZB();
$db = new DB();
// How many releases to handle at a time
$batch=Nfo::NNTP_BATCH_COUNT;
// Build NFO List
$nfometa = array();
// Missing NFO Query (oldest first so they don't expire on us)
$mnfo = "SELECT ID,guid, name FROM releases r ".
"WHERE r.releasenfoID = ".Nfo::FLAG_NFO_PENDING.
" ORDER BY postdate DESC";
if ($limit !==Null and $limit > 0)
$mnfo .= " LIMIT $limit";
$res = $db->query($mnfo);
if($res){
foreach($res as $r){
$nzbfile = $nzb->getNZBPath($r["guid"]);
if(!is_file($nzbfile)){
if($this->verbose) echo sprintf("%s Missing NZB File: %d/%s ...\n",
'NfoProc', intval($r["ID"]), $r["name"]);
$this->setNfoMissing($r["ID"]);
continue;
}
$nzbInfo = new NzbInfo();
if (!$nzbInfo->loadFromFile($nzbfile))
{
if($this->verbose) echo sprintf("%s Unable to parse NZB File: %d/%s ...\n",
'NfoProc', intval($r["ID"]), $r["name"]);
$this->setNfoMissing($r["ID"]);
continue;
}
$total+=1;
$filename = basename($nzbfile);
if($this->verbose) echo sprintf("NfoProc : Scanning %s - ", $r["name"]);
$matches = $this->nfo_scan($nzbInfo);
unset($nzbInfo);
if(is_array($matches)){
if(!count($matches)){
if($this->verbose) echo "nfo missing.\n";
$this->setNfoMissing($r["ID"]);
continue;
}
}else{
if($this->verbose) echo "corrupt nzb.\n";
$this->setNfoMissing($r["ID"]);
continue;
}
if($this->verbose) echo count($matches)." possible nfo(s).\n";
$processed++;
// Hash Matches by Release ID
$nfometa[(string)$r["ID"]] = $matches;
if(!($processed%$batch))
{
$nfoblob = array();
if($this->verbose) echo "NfoProc : Retrieval ...";
if($this->nfo_grab($nfometa, $nfoblob)){
$before = array_keys($nfoblob);
$this->parse_blobs($nfometa, $nfoblob);
$after = array_keys($nfoblob);
$removed = array_diff($before, $after);
$this->store_blob($nfometa, $nfoblob, $removed);
}
if($this->verbose) echo "\n";
// Reset nfo list array
$nfometa = array();
}
}
if(($processed%$batch)){
$nfoblob = array();
if($this->verbose) echo "NfoProc : Retrieval ...";
if($this->nfo_grab($nfometa, $nfoblob)){
$before = array_keys($nfoblob);
$this->parse_blobs($nfometa, $nfoblob);
$after = array_keys($nfoblob);
$removed = array_diff($before, $after);
$this->store_blob($nfometa, $nfoblob, $removed);
}
if($this->verbose) echo "\n";
}
}
return true;
}
//Add release nfo, imported from nZEDb
public function addReleaseNfo($relid)
{
$db = new DB();
return $db->queryInsert(sprintf("INSERT IGNORE INTO releasenfo (releaseID) VALUE (%d)", $relid));
}
// end of imported part
//Add isNFO, imported from nZEDb
// Confirm that the .nfo file is not something else.
public function isNFO($possibleNFO)
{
$ok = false;
if ($possibleNFO !== false)
{
if (!preg_match('/(<?xml|;\s*Generated\sby.+SF\w|^\s*PAR|\.[a-z0-9]{2,7}\s[a-z0-9]{8}|^\s*RAR|\A.{0,10}(JFIF|matroska|ftyp|ID3))/i', $possibleNFO))
{
if (strlen($possibleNFO) < 45 * 1024)
{
// exif_imagetype needs a minimum size or else it doesn't work.
if (strlen($possibleNFO) > 15)
{
// Check if it's a picture - EXIF.
if (@exif_imagetype($possibleNFO) == false)
{
// Check if it's a picture - JFIF.
if ($this->check_JFIF($possibleNFO) == false)
{
// Check if it's a par2.
$par2info = new Par2Info();
$par2info->setData($possibleNFO);
if ($par2info->error)
{
$ok = true;
}
}
}
}
}
}
}
return $ok;
}
// Check if the possible NFO is a JFIF.
function check_JFIF($filename)
{
$fp = @fopen($filename, 'r');
if ($fp)
{
// JFIF often (but not always) starts at offset 6.
if (fseek($fp, 6) == 0)
{
// JFIF header is 16 bytes.
if (($bytes = fread($fp, 16)) !== false)
{
// Make sure it is JFIF header.
if (substr($bytes, 0, 4) == "JFIF")
return true;
else
return false;
}
}
}
}
//end of import
/**
* Delete a releasenfo row.
*/
public function deleteReleaseNfo($relid)
{
$db = new DB();
return $db->query(sprintf("delete from releasenfo where releaseID = %d", $relid));
}
/**
* Mark a release as missing so it isn't ever parsed again
*/
private function setNfoMissing($relid)
{
$db = new DB();
$q = sprintf("UPDATE releases SET releasenfoID = %d ".
"WHERE ID = %d", Nfo::FLAG_NFO_MISSING, $relid);
return $db->query($q);
}
/**
* Returns the nfo from the database (blob)
*/
public function getNfo($relid, &$nfoout)
{
$db = new DB();
// Has NFO Query
$mnfo = "SELECT uncompress(rn.nfo) as nfo FROM releases r ".
"INNER JOIN releasenfo rn ON rn.releaseID = r.ID AND rn.ID = r.releasenfoID ".
"WHERE rn.nfo IS NOT NULL AND r.ID = %d LIMIT 1";
$res = $db->queryOneRow(sprintf($mnfo, $relid));
if($res && isset($res['nfo']))
{
$nfoout=$res['nfo'];
return true;
}
return false;
}
/**
* Process Nfo's
*/
public function processNfoFiles($batch=50)
{
$processed = 0;
$total = 0;
$this->scan_releases($processed, $total, $batch);
if($this->verbose) echo sprintf("NfoProc : Complete %d NFOs detected from %d scanned NZB files.\n", $processed, $total);
return $total;
}
}
File diff suppressed because it is too large Load Diff
+1
View File
@@ -13,6 +13,7 @@
require_once(dirname(__FILE__)."/../bin/config.php");
require_once("namefixer.php");
require_once("prehash.php");
require_once("functions.php");
$n = "\n";
+1
View File
@@ -2,6 +2,7 @@
require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."lib/groups.php");
require_once("prehash.php");
require_once("functions.php");
+1
View File
@@ -4,6 +4,7 @@ require_once(WWW_DIR. "lib/framework/db.php");
require_once(WWW_DIR. "lib/category.php");
require_once(WWW_DIR. "lib/groups.php");
require_once("namecleaner.php");
require_once("functions.php");
//This script is adapted from nZEDb
/* Values of relnamestatus:
+1
View File
@@ -5,6 +5,7 @@ require_once(WWW_DIR."lib/category.php");
require_once(WWW_DIR."lib/groups.php");
require_once(WWW_DIR."lib/nfo.php");
require_once(WWW_DIR."lib/site.php");
require_once("functions.php");
/*
* Class for inserting names/categories/md5 etc from predb sources into the DB, also for matching names on files / subjects.
+1
View File
@@ -8,6 +8,7 @@ require_once(dirname(__FILE__)."/../bin/config.php");
require_once(WWW_DIR."lib/framework/db.php");
require_once(WWW_DIR."lib/releases.php");
require_once(WWW_DIR."lib/site.php");
require_once("functions.php");
if (!isset($argv[1]) && !isset($argv[2]))
{