Move files around.

This commit is contained in:
Darko
2015-05-28 12:36:00 +02:00
parent a5ea73deec
commit ae0434ac11
1403 changed files with 13 additions and 18 deletions
@@ -0,0 +1,6 @@
<?php
require_once dirname(dirname(dirname(dirname(dirname(__DIR__))))) . DIRECTORY_SEPARATOR . 'www' . DIRECTORY_SEPARATOR . 'config.php';
if (is_file(dirname(__DIR__) . DIRECTORY_SEPARATOR . 'settings.php')) {
require_once dirname(__DIR__) . DIRECTORY_SEPARATOR . 'settings.php';
}
@@ -0,0 +1,338 @@
<?php
if (!isset($argv[1])) {
exit("This script is not intended to be run manually." . PHP_EOL);
}
require_once dirname(__FILE__) . '/../../../../../www/config.php';
use newznab\db\DB;
use newznab\processing\PProcess;
use newznab\processing\post\ProcessAdditional;
// Are we coming from python or php ? $options[0] => (string): python|php
// The type of process we want to do: $options[1] => (string): releases
$options = explode(' ', $argv[1]);
switch ($options[1]) {
// Runs backFill interval or all.
// $options[2] => (string)group name, Name of group to work on.
// $options[3] => (int) backfill type from tmux settings. 1 = Backfill interval , 2 = Bakfill all
case 'backfill':
if (in_array((int)$options[3], [1, 2])) {
$pdo = new DB();
$value = $pdo->queryOneRow("SELECT value FROM tmux WHERE setting = 'backfill_qty'");
if ($value !== false) {
$nntp = nntp($pdo);
(new \Backfill())->backfillAllGroups($options[2], ($options[3] == 1 ? '' : $value['value']));
}
}
break;
/* BackFill up to x number of articles for all groups.
*
* $options[2] => (string) Group name.
* $options[3] => (int) Quantity of articles to download.
*/
case 'backfill_all_quantity':
$pdo = new DB();
$nntp = nntp($pdo);
(new \Backfill())->backfillAllGroups($options[2], $options[3]);
break;
// BackFill a single group, 10000 parts.
// $options[2] => (string)group name, Name of group to work on.
case 'backfill_all_quick':
$pdo = new DB();
$nntp = nntp($pdo);
(new \Backfill())->backfillAllGroups($options[2], 10000, 'normal');
break;
/* Get a range of article headers for a group.
*
* $options[2] => (string) backfill/binaries
* $options[3] => (string) Group name.
* $options[4] => (int) First article number in range.
* $options[5] => (int) Last article number in range.
* $options[6] => (int) Number of threads.
*/
case 'get_range':
$pdo = new DB();
$nntp = nntp($pdo);
$groups = new \Groups();
$s = new Sites();
$site = $s->get();
$groupMySQL = $groups->getByName($options[3]);
if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) {
if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) {
return;
}
}
$binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]);
$return = $binaries->scan($groupMySQL, $options[4], $options[5], ($site->safepartrepair == 1 ? 'update' : 'backfill'));
if (empty($return)) {
exit();
}
$columns = [];
switch ($options[2]) {
case 'binaries':
if ($return['lastArticleNumber'] <= $groupMySQL['last_record']){
exit();
}
$columns[1] = sprintf(
'last_record_postdate = %s',
$pdo->from_unixtime(
(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate']))
)
);
$columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']);
$query = sprintf(
'UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s',
$columns[1],
$columns[2],
$groupMySQL['id'],
$return['lastArticleNumber']
);
break;
case 'backfill':
if ($return['firstArticleNumber'] >= $groupMySQL['first_record']){
exit();
}
$columns[1] = sprintf(
'first_record_postdate = %s',
$pdo->from_unixtime(
(is_numeric($return['firstArticleDate']) ? $return['firstArticleDate'] : strtotime($return['firstArticleDate']))
)
);
$columns[2] = sprintf('first_record = %s', $return['firstArticleNumber']);
$query = sprintf(
'UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND first_record > %s',
$columns[1],
$columns[2],
$groupMySQL['id'],
$return['firstArticleNumber']
);
break;
default:
exit();
}
$pdo->queryExec($query);
break;
/* Do part repair for a group.
*
* $options[2] => (string) Group name.
*/
case 'part_repair':
$pdo = new DB();
$groups = new \Groups(['Settings' => $pdo]);
$groupMySQL = $groups->getByName($options[2]);
$nntp = nntp($pdo);
// Select group, here, only once
$data = $nntp->selectGroup($groupMySQL['name']);
if ($nntp->isError($data)) {
if ($nntp->dataError($nntp, $groupMySQL['name']) === false) {
exit();
}
}
(new \Binaries())->partRepair($groupMySQL);
break;
// Process releases.
// $options[2] => (string)groupCount, number of groups terminated by _ | (int)groupid, group to work on
case 'releases':
$pdo = new DB();
$s = new Sites();
$site = $s->get();
$releases = new \Releases(['Settings' => $pdo]);
//Runs function that are per group
if (is_numeric($options[2])) {
processReleases($site, $releases, $options[2]);
} else {
// Run functions that run on releases table after all others completed.
$groupCount = rtrim($options[2], '_');
if (!is_numeric($groupCount)) {
$groupCount = 1;
}
$releases->deletedReleasesByGroup();
$releases->deleteReleases();
$releases->processRequestIDs('', (5000 * $groupCount), true);
$releases->processRequestIDs('', (1000 * $groupCount), false);
$releases->categorizeReleases(2);
}
break;
// Process all local requestid for a single group.
// $options[2] => (int)groupid, group to work on
case 'requestid':
if (is_numeric($options[2])) {
(new \RequestIDLocal(['Echo' => true]))->lookupRequestIDs(['GroupID' => $options[2], 'limit' => 5000]);
}
break;
/* Update a single group's article headers.
*
* $options[2] => (string) Group name.
*/
case 'update_group_headers':
$pdo = new DB();
$nntp = nntp($pdo);
$groups = new \Groups();
$groupMySQL = $groups->getByName($options[2]);
(new \Binaries(['NNTP' => $nntp, 'Groups' => $groups, 'Settings' => $pdo]))->updateGroup($groupMySQL);
break;
// Do a single group (update_binaries/backFill/update_releases/postprocess).
// $options[2] => (int)groupid, group to work on
case 'update_per_group':
if (is_numeric($options[2])) {
$pdo = new DB();
// Get the group info from MySQL.
$groupMySQL = $pdo->queryOneRow(sprintf('SELECT * FROM groups WHERE id = %d', $options[2]));
if ($groupMySQL === false) {
exit('ERROR: Group not found with id ' . $options[2] . PHP_EOL);
}
// Connect to NNTP.
$nntp = nntp($pdo);
$backFill = new \Backfill();
// Update the group for new binaries.
(new \Binaries())->updateGroup($groupMySQL);
// BackFill the group with 20k articles.
$backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal');
// Create releases.
processReleases(new \Releases(['Settings' => $pdo]), $options[2]);
// Post process the releases.
(new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start($options[2]);
(new \Info(['Echo' => true, 'Settings' => $pdo]))->processNfoFiles($nntp, $options[2]);
}
break;
// Post process additional and NFO.
// $options[2] => (char)Letter or number a-f 0-9, first character of release guid.
case 'pp_additional':
case 'pp_nfo':
if (charCheck($options[2])) {
$pdo = new DB();
// Create the connection here and pass, this is for post processing, so check for alternate.
$nntp = nntp($pdo, true);
if ($options[1] === 'pp_nfo') {
(new \Info(['Echo' => true, 'Settings' => $pdo]))->processNfoFiles($nntp, '', $options[2]);
} else {
(new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start('', $options[2]);
}
}
break;
/* Post process movies.
*
* $options[2] (char) Single character, first letter of release guid.
* $options[3] (int) Process all releases or renamed releases only.
*/
case 'pp_movie':
if (charCheck($options[2])) {
$pdo = new DB();
(new PProcess(['Settings' => $pdo]))->processMovies('', $options[2], (isset($options[3]) ? $options[3] : ''));
}
break;
/* Post process TV.
*
* $options[2] (char) Single character, first letter of release guid.
* $options[3] (int) Process all releases or renamed releases only.
*/
case 'pp_tv':
if (charCheck($options[2])) {
$pdo = new DB();
(new PProcess(['Settings' => $pdo]))->processTv('', $options[2], (isset($options[3]) ? $options[3] : ''));
}
break;
}
/**
* Create / process releases for a groupid.
*
* @param \Releases $releases
* @param int $groupID
*/
function processReleases($site, $releases, $groupID)
{
$s = new Sites();
$site = $s->get();
$releaseCreationLimit = ($site->maxnzbsprocessed != '' ? (int)$site->maxnzbsprocessed : 1000);
$releases->applyRegex($groupID);
$releases->processIncompleteBinaries($groupID);
$releases->createReleases($groupID);
do {
$releasesCount = $releases->createReleases($groupID);
// This loops as long as the number of releases or nzbs added was >= the limit (meaning there are more waiting to be created)
} while (($releasesCount['added'] + $releasesCount['dupes']) >= $releaseCreationLimit);
$releases->deleteBinaries($groupID);
}
/**
* Check if the character contains a-f or 0-9.
*
* @param string $char
*
* @return bool
*/
function charCheck($char)
{
if (in_array($char, ['a','b','c','d','e','f','0','1','2','3','4','5','6','7','8','9'])) {
return true;
}
return false;
}
/**
* Check if the group should be processed.
*
* @param \newznab\db\DB $pdo
* @param int $groupID
*/
function collectionCheck(&$pdo, $groupID)
{
if ($pdo->queryOneRow(sprintf('SELECT id FROM collections_%d LIMIT 1', $groupID)) === false) {
exit();
}
}
/**
* Connect to usenet, return NNTP object.
*
* @param \DB $pdo
* @param bool $alternate Use alternate NNTP provider.
*
* @return NNTP
*/
function &nntp(&$pdo, $alternate = false)
{
$nntp = new \NNTP(['Settings' => $pdo]);
$s = new Sites();
$site = $s->get();
if (($alternate && $site->alternate_nntp == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
exit("ERROR: Unable to connect to usenet." . PHP_EOL);
}
return $nntp;
}
@@ -0,0 +1 @@
/settings.php
@@ -0,0 +1,19 @@
####These multi-processing scripts require a POSIX compliant operating system and the PHP pcntl extension.
####binaries.php
This will download new headers for all active groups using your binaries threads site setting.
You can pass a argument, a number to limit the max amount of new headers to download.
####releases.php
This is identical to the python releases_threaded.py
This will create new releases/delete unwanted releases, process requestid's, categorize releases by group
using your release threads site setting.
####update_per_group.php:
This is identical to the python update_threaded.py
This will download new headers for all active groups, backfill 20k headers from all backfill enabled groups,
create new releases/delete unwanted releases, process requestid's, categorize releases, process additional and NFO
by group using your release threads site setting.
@@ -0,0 +1,8 @@
<?php
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\Forking;
// Check if argument 1 is numeric, which is to limit article count.
(new Forking())->processWorkType(
'backfill', (isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? array(0 => $argv[1]) : array(0 => false))
);
@@ -0,0 +1,10 @@
<?php
if (!isset($argv[1]) || !is_numeric($argv[1])) {
exit(
'Argument 1 => (Number) Set to 0 to ignore, else fetches up to x new headers for every active group.' . PHP_EOL
);
}
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\Forking;
(new Forking())->processWorkType('binaries', array(0 => $argv[1]));
@@ -0,0 +1,20 @@
<?php
if (!isset($argv[1]) || !in_array($argv[1], ['nfo', 'filename', 'md5', 'par2', 'miscsorter', 'predbft'])) {
exit(
'First argument (mandatory):' . PHP_EOL .
'nfo => Attempt to fix release name using the nfo.' . PHP_EOL .
'filename => Attempt to fix release name using the filenames.' . PHP_EOL .
'md5 => Attempt to fix release name using the MD5.' . PHP_EOL .
'par2 => Attempt to fix release name using the par2.' . PHP_EOL .
'miscsorter => Attempt to fix release name using magic.' . PHP_EOL .
'predbft => Attempt to fix release name using Predb full text matching.' . PHP_EOL . PHP_EOL
);
}
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\Forking;
(new Forking())->processWorkType('fixRelNames_' . $argv[1], [0 => $argv[1]]);
@@ -0,0 +1,34 @@
<?php
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\ForkingImportNZB;
if (!isset($argv[1]) || !is_dir($argv[1])) {
exit(
'First argument (mandatory):' . PHP_EOL .
'Path to a folder, containing folders with .nzb or .nzb.gz files inside them.' . PHP_EOL .
'If you supply a path containing only files, the files will be ignored.' . PHP_EOL .
'The sub-folders will be searched recursively for NZB files.' . PHP_EOL . PHP_EOL .
'Second argument (optional):' . PHP_EOL .
'Number of processes, how many processes to run max at a time. (default is 1)' . PHP_EOL . PHP_EOL .
'Third argument (optional):' . PHP_EOL .
'true|false => Delete the NZB files after they are imported (recommended), if you stop and restart you will have to go over the imported files again.' . PHP_EOL . PHP_EOL .
'Fourth argument (optional)' . PHP_EOL .
'true|false => Delete the NZB if importing it fails (not recommended).' . PHP_EOL .
'Fifth argument (optional):' . PHP_EOL .
'true|false => Use the NZB file name as the release name (not recommended), the names in the NZB are better.' . PHP_EOL . PHP_EOL .
'Sixth argument (optional):' . PHP_EOL .
'How many NZB files to import per process, if this is not set, it will do 50,000 per process.' . PHP_EOL . PHP_EOL .
'Note that successfully imported NZB files WILL be deleted.' . PHP_EOL
);
}
(new ForkingImportNZB())->start(
$argv[1],
(isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 1),
(isset($argv[3]) && $argv[3] === 'true' ? 'true' : 'false'),
(isset($argv[4]) && $argv[4] === 'true' ? 'true' : 'false'),
(isset($argv[5]) && $argv[5] === 'true' ? 'true' : 'false'),
(isset($argv[6]) && is_numeric($argv[6]) && $argv[6] > 0 ? $argv[6] : 50000)
);
@@ -0,0 +1,21 @@
<?php
if (!isset($argv[1]) || !in_array($argv[1], ['ama', 'add', 'mov', 'nfo', 'sha', 'tv'])) {
exit(
'First argument (mandatory):' . PHP_EOL .
'ama => Do amazon processing, this does not use multi-processing, because of amazon API restrictions.' . PHP_EOL .
'add => Do additional (rar|zip) processing.' . PHP_EOL .
'mov => Do movie processing.' . PHP_EOL .
'nfo => Do NFO processing.' . PHP_EOL .
'sha => Do sharing processing, this does not use multi-processing.' . PHP_EOL .
'tv => Do TV processing.' . PHP_EOL . PHP_EOL .
'Second argument (optional):' . PHP_EOL .
'true|false => Only post-process renamed releases. This is for the mov|tv options.' . PHP_EOL
);
}
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\Forking;
(new Forking())->processWorkType('postProcess_' . $argv[1], (isset($argv[2]) && $argv[2] === 'true' ? [0 => true] : []));
@@ -0,0 +1,7 @@
<?php
declare(ticks=1);
require('.do_not_run/require.php');
use newznab\libraries\Forking;
(new Forking())->processWorkType('releases');
@@ -0,0 +1,7 @@
<?php
declare(ticks=1);
require('.do_not_run/require.php');
use \newznab\libraries\Forking;
(new Forking())->processWorkType('request_id');
@@ -0,0 +1,15 @@
<?php
if (!isset($argv[1]) || !in_array($argv[1], ['backfill', 'binaries'])) {
exit(
'First argument (mandatory):' . PHP_EOL .
'binaries => Do Safe Binaries update.' . PHP_EOL .
'backfill => Do Safe Backfill update.' . PHP_EOL
);
}
declare(ticks=1);
require('.do_not_run/require.php');
use \newznab\libraries\Forking;
(new Forking())->processWorkType('safe_' . $argv[1]);
@@ -0,0 +1,49 @@
<?php
/**
* Maximum time in seconds a child can stay alive.
* Keeps long processes, like groups with millions of parts from preventing other groups to update.
*
* @default 1800
*/
define('NN_MULTIPROCESSING_MAX_CHILD_TIME', 1800);
/**
* How much work can 1 child do.
* Increasing this currently reduces performance with no benefits, here for testing or future use.
*
* @default 1
*/
define('NN_MULTIPROCESSING_MAX_CHILD_WORK', 1);
/**
* This setting can be used to override your site settings, to cap the maximum amount of child processes.
* Set to 0 to ignore.
*
* @default 0
*/
define('NN_MULTIPROCESSING_MAX_CHILDREN_OVERRIDE', 0);
/**
* Which type of messages to display on the screen.
*
* -1 All messages.
* 2 Critical messages.
* 4 Warnings.
* 6 Info messages.
* 7 Debug messages.
*
* @default 6
*/
define('NN_MULTIPROCESSING_LOG_TYPE', 6);
/**
* How to display the text output (echo's) from the child processes.
*
* 0 - Do not display any anything.
* 1 - Display the text as the child outputs it in real time. (This will mix all the child process text output.)
* 2 - Display the text when the child is done. (This will display the text serially.)
*
* @default 1
*/
define('NN_MULTIPROCESSING_CHILD_OUTPUT_TYPE', 1);
@@ -0,0 +1,8 @@
<?php
declare(ticks=1);
require('.do_not_run/require.php');
use \newznab\libraries\Forking;
// This is the same as the python update_threaded.php
(new Forking())->processWorkType('update_per_group');