starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new regexes, update predb_import_daily_batch script, add new PreDb class to manage predb imports.

Browse Source
This commit is contained in:
DariusIII
2015-11-09 12:13:14 +01:00
parent 18528b66c4
commit 5a873a5082
13 changed files with 889 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/data/predb_import_daily_batch.php
Regular → Executable
+136 -166
View File
@@ -1,115 +1,135 @@
<?php
/**
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program (see LICENSE.txt in the base directory. If
* not, see:
*
* @link <http://www.gnu.org/licenses/>.
* @author niel
* @copyright 2015 nZEDb
*/
/* TODO better tune the queries for performance, including using prepared statements and
pre-fetching groupid and other data for faster inclusion in the main query.
/* TODO better tune the queries for performance, including pre-fetching group_id and other data for
faster inclusion in the main query.
*/
require_once realpath(dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR . 'indexer.php');
use newznab\db\Settings;
use newznab\db\PreDb;
use newznab\utility\Utility;
require_once realpath(dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR . 'indexer.php');
unset($config);
if (!Utility::isWin()) {
$fullPath = DS;
$paths = preg_split('#/#', NN_RES);
foreach ($paths as $path) {
if ($path !== '') {
$fullPath .= $path . DS;
if (!is_readable($fullPath) || !is_executable($fullPath)) {
exit('The (' . $fullPath . ') folder must be readable and executable by all.' .
PHP_EOL);
}
}
if (NN_DEBUG) {
echo "Checking resource path\n";
}
unset($fullPath, $paths, $path);
$canExeRead = Utility::canExecuteRead(NN_RES);
if (is_string($canExeRead)) {
exit($canExeRead);
}
unset($canExeRead);
}
if (NN_DEBUG) {
echo "Checking directory is writable\n";
}
if (!is_writable(NN_RES)) {
exit('The (' . NN_RES . ') folder must be writable.' . PHP_EOL);
}
$progress = rw_progress(settings_array());
if (!isset($argv[1]) || !is_numeric($argv[1]) && $argv[1] != 'progress' || !isset($argv[2]) ||
!in_array($argv[2], ['local', 'remote']) || !isset($argv[3]) ||
!in_array($argv[3], ['true', 'false'])
) {
exit('This script quickly imports the daily PreDB dumps.' . PHP_EOL .
'Argument 1: Enter the unix time of the patch to start at.' . PHP_EOL .
'You can find the unix time in the file name of the patch, it\'s the long number.' .
PHP_EOL .
'You can put in 0 to import all the daily PreDB dumps.' . PHP_EOL .
'You can put in progress to track progress of the imports and only import newer ones.' .
PHP_EOL .
'Argument 2: If your MySQL server is local, type local else type remote.' . PHP_EOL .
'Argument 3: Show output of dump_predb.php or not, true | false' . PHP_EOL
'Argument 1: Enter the unix time of the patch to start at.' . PHP_EOL .
'You can find the unix time in the file name of the patch, it\'s the long number.' .
PHP_EOL .
'You can put in 0 to import all the daily PreDB dumps.' . PHP_EOL .
'You can put in progress to track progress of the imports and only import newer ones.' .
PHP_EOL .
'Argument 2: If your MySQL server is local, type local else type remote.' . PHP_EOL .
'Argument 3: Show output of queries or not, true | false' . PHP_EOL
);
}
$fileName = '_predb_dump.csv.gz';
$innerUrl = 'fb2pffwwriruyco';
$baseUrl = 'https://www.dropbox.com/sh/' . $innerUrl;
$folderUrl['url'] = $baseUrl . '/AACy9Egno_v2kcziVHuvWbbxa';
$result = Utility::getUrl($folderUrl);
if (!$result) {
exit('Error connecting to dropbox.com, try again later?' . PHP_EOL);
if (NN_DEBUG) {
echo "Parameter check completed\n";
}
$result = preg_match_all('#<a [\w"=-]+ href="https://www.dropbox.com/sh/' . $innerUrl . '/(\S+/\d+' . $fileName .
'\?dl=0)"#',
$result,
$all_matches
);
if ($result) {
exec('clear');
$all_matches = array_unique($all_matches[1]);
$total = count($all_matches);
$pdo = new Settings();
$url = 'https://api.github.com/repos/nZEDb/nZEDbPre_Dumps/contents/dumps/';
$filePattern = '(?P<filename>(?P<stamp>\d+)_predb_dump\.csv\.gz)';
if ($argv[1] != 'progress') {
$progress['last'] = !is_numeric($argv[1]) ? time() : $argv[1];
}
if (NN_DEBUG) {
echo "Fetching predb_dump list from GitHub\n";
}
$pdo->queryExec('DROP TABLE IF EXISTS tmp_pre');
$pdo->queryExec('CREATE TABLE tmp_pre LIKE prehash');
$result = Utility::getUrl(
[
'url' => $url,
'requestheaders' => [
'Content-Type: application/json',
'User-Agent: nZEDb'
]
]);
// Drop id as it is not needed and incurs overhead creating each id.
$pdo->queryExec('ALTER TABLE tmp_pre DROP COLUMN id');
if ($result === false) {
exit('Error connecting to GitHub, try again later?' . PHP_EOL);
}
// Add a column for the group's name which is included instead of the groupid, which may be
// different between individual databases
$pdo->queryExec('ALTER TABLE tmp_pre ADD COLUMN groupname VARCHAR (255)');
if (NN_DEBUG) {
echo "Extracting filenames from list.\n";
}
// Drop indexes on tmp_pre
$pdo->queryExec('ALTER TABLE tmp_pre DROP INDEX `ix_prehash_nfo`, DROP INDEX `ix_prehash_predate`, DROP INDEX `ix_prehash_source`, DROP INDEX `ix_prehash_title`, DROP INDEX `ix_prehash_requestid`');
$data = json_decode($result, true);
if (is_null($data)) {
exit("Error: $result");
}
foreach ($all_matches as $matches) {
if (preg_match('#^(.+)/(\d+)_#', $matches, $match)) {
$total = count($data);
$predb = new PreDb();
$progress = $predb->progress(settings_array());
foreach ($data as $file) {
if (preg_match("#^https://raw\.githubusercontent\.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$#",
$file['download_url'])) {
if (preg_match("#^$filePattern$#", $file['name'], $match)) {
$timematch = $progress['last'];
// Skip patches the user does not want.
if ($match[2] < $timematch) {
echo 'Skipping dump ' . $match[2] . ', as your minimum unix time argument is ' .
$timematch . PHP_EOL;
if ($match[1] < $timematch) {
echo 'Skipping dump ' . $match[2] .
', as your minimum unix time argument is ' .
$timematch . PHP_EOL;
--$total;
continue;
}
// Download the dump.
$file['url'] = $baseUrl . '/' . $match[1] . '/' . $match[2] . $fileName . '?dl=1';
$dump = Utility::getUrl($file);
$dump = Utility::getUrl(['url' => $file['download_url']]);
echo "Downloading: {$file['download_url']}\n";
if (!$dump) {
echo 'Error downloading dump ' . $match[2] . ' you can try manually importing it.' .
PHP_EOL;
echo "Error downloading dump {$match[2]} you can try manually importing it." .
PHP_EOL;
continue;
} else {
if (NN_DEBUG) {
echo "Dump {$match[2]} downloaded\n";
}
}
// Make sure we didn't get a HTML page.
if (strlen($dump) < 5000 && strpos($dump, '<!DOCTYPE html>') !== false) {
echo 'The dump file ' . $match[2] . ' might be missing from dropbox.' . PHP_EOL;
// Make sure we didn't get an HTML page.
if (strpos($dump, '<!DOCTYPE html>') !== false) {
echo "The dump file {$match[2]} might be missing from GitHub." . PHP_EOL;
continue;
}
@@ -117,35 +137,68 @@ if ($result) {
$dump = gzdecode($dump);
if (!$dump) {
echo 'Error decompressing dump ' . $match[2] . '.' . PHP_EOL;
echo "Error decompressing dump {$match[2]}." . PHP_EOL;
continue;
}
// Store the dump.
$dumpFile = NN_RES . $match[2] . '_predb_dump.csv';
$fetched = file_put_contents($dumpFile, $dump);
$fetched = file_put_contents($dumpFile, $dump);
if (!$fetched) {
echo 'Error storing dump file ' . $match[2] . ' in (' . NN_RES . ').' . PHP_EOL;
echo "Error storing dump file {$match[2]} in (" . NN_RES . ').' .
PHP_EOL;
continue;
}
// Make sure it's readable by all.
chmod($dumpFile, 0777);
$local = strtolower($argv[2]) == 'local' ? true : false;
$local = strtolower($argv[2]) == 'local' ? true : false;
$verbose = $argv[3] == true ? true : false;
importDump($dumpFile, $local, $verbose);
if ($verbose) {
echo $predb->log->info("Clearing import table");
}
// Truncate to clear any old data
$predb->executeTruncate();
// Import file into predb_imports
$predb->executeLoadData(
[
'fields' => '\\t\\t',
'lines' => '\\r\\n',
'local' => $local,
'path' => $dumpFile,
]);
// Remove any titles where length <=8
if ($verbose === true) {
echo $predb->log->info("Deleting any records where title <=8 from Temporary Table");
}
$predb->executeDeleteShort();
// Add any groups that do not currently exist
$predb->executeAddGroups();
// Fill the group_id
$predb->executeUpdateGroupID();
echo $predb->log->info("Inserting records from temporary table into predb table");
$predb->executeInsert();
// Delete the dump.
// unlink($dumpFile);
unlink($dumpFile);
$progress = rw_progress(settings_array($match[2] + 1, $progress), false);
echo 'Successfully imported PreDB dump ' . $match[2] . ' ' . (--$total) .
' dumps remaining to import.' . PHP_EOL;
$progress = $predb->progress(settings_array($match[2] + 1, $progress),
['read' => false]);
echo "Successfully imported PreDB dump {$match[2]}, " . (--$total) .
' dumps remaining.' . PHP_EOL;
} else {
echo "Ignoring: {$file['download_url']}\n";
}
} else if (NN_DEBUG) {
echo "^https://raw.githubusercontent.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$\n {$file['download_url']}\n";
}
// Drop tmp_pre table
$pdo->queryExec('DROP TABLE IF EXISTS tmp_pre');
}
function settings_array($last = null, $settings = null)
@@ -161,87 +214,4 @@ function settings_array($last = null, $settings = null)
return $settings;
}
function rw_progress($settings, $read = true)
{
if (!$read || !is_file(__DIR__ . DS . 'prehash_progress.txt')) {
file_put_contents(__DIR__ . DS . 'prehash_progress.txt', base64_encode(serialize($settings)));
} else {
$settings = unserialize(base64_decode(file_get_contents(__DIR__ . DS .
'prehash_progress.txt'
)
)
);
}
return $settings;
}
// This function duplicates how dump_predb works but does not drop the hashes/triggers as recreating
// potentially millions for the small addition that dailies add, isn't worth it.
function importDump($path, $local, $verbose = true, $table = 'prehash')
{
global $pdo;
// Create temp table to allow updating
if ($verbose) {
echo $pdo->log->info("Creating temporary table");
}
// Truncate to clear any old data
$pdo->queryDirect("TRUNCATE TABLE tmp_pre");
// Import file into tmp_pre
$sqlLoad = sprintf(
"LOAD DATA %s INFILE '%s' IGNORE INTO TABLE tmp_pre FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);",
($local === false ? 'LOCAL' : ''),
$path
);
if (NN_DEBUG) {
echo $pdo->log->header($sqlLoad);
}
$pdo->queryDirect($sqlLoad);
// Remove any titles where length <=8
if ($verbose) {
echo $pdo->log->info("Deleting any records where title <=8 from Temporary Table");
}
$pdo->queryDirect("DELETE FROM tmp_pre WHERE LENGTH(title) <= 8");
// Add any groups that do not currently exist
$sqlAddGroups = <<<SQL_ADD_GROUPS
INSERT IGNORE INTO groups (`name`, description)
SELECT groupname, 'Added by predb import script'
FROM tmp_pre AS t LEFT JOIN groups AS g ON t.`groupname` = g.`name`
WHERE t.`groupname` IS NOT NULL AND g.`name` IS NULL
GROUP BY groupname;
SQL_ADD_GROUPS;
$pdo->queryDirect($sqlAddGroups);
// Fill the groupid
$pdo->queryDirect("UPDATE tmp_pre AS t SET groupid = (SELECT id FROM groups WHERE name = t.groupname) WHERE groupname IS NOT NULL");
// Insert and update table
$sqlInsert = <<<SQL_INSERT
INSERT INTO $table (title, nfo, size, files, filename, nuked, nukereason, category, predate, SOURCE, requestid, groupid)
SELECT t.title, t.nfo, t.size, t.files, t.filename, t.nuked, t.nukereason, t.category, t.predate, t.source, t.requestid, t.groupid
FROM tmp_pre AS t
ON DUPLICATE KEY UPDATE prehash.nfo = IF(prehash.nfo IS NULL, t.nfo, prehash.nfo),
prehash.size = IF(prehash.size IS NULL, t.size, prehash.size),
prehash.files = IF(prehash.files IS NULL, t.files, prehash.files),
prehash.filename = IF(prehash.filename = '', t.filename, prehash.filename),
prehash.nuked = IF(t.nuked > 0, t.nuked, prehash.nuked),
prehash.nukereason = IF(t.nuked > 0, t.nukereason, prehash.nukereason),
prehash.category = IF(prehash.category IS NULL, t.category, prehash.category),
prehash.requestid = IF(prehash.requestid = 0, t.requestid, prehash.requestid),
prehash.groupid = IF(prehash.groupid = 0, t.groupid, prehash.groupid);
SQL_INSERT;
echo $pdo->log->info("Inserting records from temporary table into $table");
if (NN_DEBUG) {
echo $pdo->log->primary($sqlInsert);
}
if ($pdo->queryDirect($sqlInsert) === false) {
echo "FAILED\n";
}
}
?>