mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 17:28:55 +00:00
DariusIII
254 lines
8.2 KiB
PHP
254 lines
8.2 KiB
PHP
<?php
|
|
|
|
/* TODO better tune the queries for performance, including using prepared statements and
|
|
pre-fetching groupid and other data for faster inclusion in the main query.
|
|
*/
|
|
|
|
use newznab\db\Settings;
|
|
use newznab\utility\Utility;
|
|
|
|
$config = dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR . 'www' . DIRECTORY_SEPARATOR .
|
|
'config.php';
|
|
|
|
|
|
if (!is_file($config)) {
|
|
exit('Place this script in the testing folder of tmux.' . PHP_EOL);
|
|
}
|
|
require_once dirname(__FILE__) . '/../../www/config.php';
|
|
unset($config);
|
|
|
|
if (!Utility::isWin()) {
|
|
$fullPath = DS;
|
|
$paths = preg_split('#/#', NN_RES);
|
|
foreach ($paths as $path) {
|
|
if ($path !== '') {
|
|
$fullPath .= $path . DS;
|
|
if (!is_readable($fullPath) || !is_executable($fullPath)) {
|
|
exit('The (' . $fullPath . ') folder must be readable and executable by all.' .
|
|
PHP_EOL);
|
|
}
|
|
}
|
|
}
|
|
unset($fullPath, $paths, $path);
|
|
}
|
|
|
|
if (!is_writable(NN_RES)) {
|
|
exit('The (' . NN_RES . ') folder must be writable.' . PHP_EOL);
|
|
}
|
|
|
|
$progress = rw_progress(settings_array());
|
|
|
|
if (!isset($argv[1]) || !is_numeric($argv[1]) && $argv[1] != 'progress' || !isset($argv[2]) ||
|
|
!in_array($argv[2], ['local', 'remote']) || !isset($argv[3]) ||
|
|
!in_array($argv[3], ['true', 'false'])
|
|
) {
|
|
exit('This script quickly imports the daily PreDB dumps.' . PHP_EOL .
|
|
'Argument 1: Enter the unix time of the patch to start at.' . PHP_EOL .
|
|
'You can find the unix time in the file name of the patch, it\'s the long number.' .
|
|
PHP_EOL .
|
|
'You can put in 0 to import all the daily PreDB dumps.' . PHP_EOL .
|
|
'You can put in progress to track progress of the imports and only import newer ones.' .
|
|
PHP_EOL .
|
|
'Argument 2: If your MySQL server is local, type local else type remote.' . PHP_EOL .
|
|
'Argument 3: Show output of dump_predb.php or not, true | false' . PHP_EOL
|
|
);
|
|
}
|
|
$fileName = '_predb_dump.csv.gz';
|
|
$innerUrl = 'fb2pffwwriruyco';
|
|
$baseUrl = 'https://www.dropbox.com/sh/' . $innerUrl;
|
|
$folderUrl['url'] = $baseUrl . '/AACy9Egno_v2kcziVHuvWbbxa';
|
|
|
|
$result = Utility::getUrl($folderUrl);
|
|
|
|
if (!$result) {
|
|
exit('Error connecting to dropbox.com, try again later?' . PHP_EOL);
|
|
}
|
|
|
|
$result = preg_match_all('#<a [\w"=-]+ href="https://www.dropbox.com/sh/' . $innerUrl . '/(\S+/\d+' . $fileName .
|
|
'\?dl=0)"#',
|
|
$result,
|
|
$all_matches
|
|
);
|
|
if ($result) {
|
|
exec('clear');
|
|
$all_matches = array_unique($all_matches[1]);
|
|
$total = count($all_matches);
|
|
$pdo = new Settings();
|
|
|
|
if ($argv[1] != 'progress') {
|
|
$progress['last'] = !is_numeric($argv[1]) ? time() : $argv[1];
|
|
}
|
|
|
|
$pdo->queryExec('DROP TABLE IF EXISTS tmp_pre');
|
|
$pdo->queryExec('CREATE TABLE tmp_pre LIKE prehash');
|
|
|
|
// Drop id as it is not needed and incurs overhead creating each id.
|
|
$pdo->queryExec('ALTER TABLE tmp_pre DROP COLUMN id');
|
|
|
|
// Add a column for the group's name which is included instead of the groupid, which may be
|
|
// different between individual databases
|
|
$pdo->queryExec('ALTER TABLE tmp_pre ADD COLUMN groupname VARCHAR (255)');
|
|
|
|
// Drop indexes on tmp_pre
|
|
$pdo->queryExec('ALTER TABLE tmp_pre DROP INDEX `ix_prehash_nfo`, DROP INDEX `ix_prehash_predate`, DROP INDEX `ix_prehash_source`, DROP INDEX `ix_prehash_title`, DROP INDEX `ix_prehash_requestid`');
|
|
|
|
foreach ($all_matches as $matches) {
|
|
if (preg_match('#^(.+)/(\d+)_#', $matches, $match)) {
|
|
$timematch = $progress['last'];
|
|
|
|
// Skip patches the user does not want.
|
|
if ($match[2] < $timematch) {
|
|
echo 'Skipping dump ' . $match[2] . ', as your minimum unix time argument is ' .
|
|
$timematch . PHP_EOL;
|
|
--$total;
|
|
continue;
|
|
}
|
|
|
|
// Download the dump.
|
|
$file['url'] = $baseUrl . '/' . $match[1] . '/' . $match[2] . $fileName . '?dl=1';
|
|
$dump = Utility::getUrl($file);
|
|
|
|
if (!$dump) {
|
|
echo 'Error downloading dump ' . $match[2] . ' you can try manually importing it.' .
|
|
PHP_EOL;
|
|
continue;
|
|
}
|
|
|
|
// Make sure we didn't get a HTML page.
|
|
if (strlen($dump) < 5000 && strpos($dump, '<!DOCTYPE html>') !== false) {
|
|
echo 'The dump file ' . $match[2] . ' might be missing from dropbox.' . PHP_EOL;
|
|
continue;
|
|
}
|
|
|
|
// Decompress.
|
|
$dump = gzdecode($dump);
|
|
|
|
if (!$dump) {
|
|
echo 'Error decompressing dump ' . $match[2] . '.' . PHP_EOL;
|
|
continue;
|
|
}
|
|
|
|
// Store the dump.
|
|
$dumpFile = NN_RES . $match[2] . '_predb_dump.csv';
|
|
$fetched = file_put_contents($dumpFile, $dump);
|
|
if (!$fetched) {
|
|
echo 'Error storing dump file ' . $match[2] . ' in (' . NN_RES . ').' . PHP_EOL;
|
|
continue;
|
|
}
|
|
|
|
// Make sure it's readable by all.
|
|
chmod($dumpFile, 0777);
|
|
$local = strtolower($argv[2]) == 'local' ? true : false;
|
|
$verbose = $argv[3] == true ? true : false;
|
|
importDump($dumpFile, $local, $verbose);
|
|
|
|
// Delete the dump.
|
|
// unlink($dumpFile);
|
|
|
|
$progress = rw_progress(settings_array($match[2] + 1, $progress), false);
|
|
echo 'Successfully imported PreDB dump ' . $match[2] . ' ' . (--$total) .
|
|
' dumps remaining to import.' . PHP_EOL;
|
|
}
|
|
}
|
|
|
|
// Drop tmp_pre table
|
|
$pdo->queryExec('DROP TABLE IF EXISTS tmp_pre');
|
|
}
|
|
|
|
function settings_array($last = null, $settings = null)
|
|
{
|
|
if (is_null($settings)) {
|
|
$settings['last'] = 0;
|
|
}
|
|
|
|
if (!is_null($last)) {
|
|
$settings['last'] = $last;
|
|
}
|
|
|
|
return $settings;
|
|
}
|
|
|
|
function rw_progress($settings, $read = true)
|
|
{
|
|
if (!$read || !is_file(__DIR__ . DS . 'prehash_progress.txt')) {
|
|
file_put_contents(__DIR__ . DS . 'prehash_progress.txt', base64_encode(serialize($settings)));
|
|
} else {
|
|
$settings = unserialize(base64_decode(file_get_contents(__DIR__ . DS .
|
|
'prehash_progress.txt'
|
|
)
|
|
)
|
|
);
|
|
}
|
|
|
|
return $settings;
|
|
}
|
|
|
|
// This function duplicates how dump_predb works but does not drop the hashes/triggers as recreating
|
|
// potentially millions for the small addition that dailies add, isn't worth it.
|
|
function importDump($path, $local, $verbose = true, $table = 'prehash')
|
|
{
|
|
global $pdo;
|
|
|
|
// Create temp table to allow updating
|
|
if ($verbose) {
|
|
echo $pdo->log->info("Creating temporary table");
|
|
}
|
|
|
|
// Truncate to clear any old data
|
|
$pdo->queryDirect("TRUNCATE TABLE tmp_pre");
|
|
|
|
// Import file into tmp_pre
|
|
$sqlLoad = sprintf(
|
|
"LOAD DATA %s INFILE '%s' IGNORE INTO TABLE tmp_pre FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);",
|
|
($local === false ? 'LOCAL' : ''),
|
|
$path
|
|
);
|
|
if (NN_DEBUG) {
|
|
echo $pdo->log->header($sqlLoad);
|
|
}
|
|
$pdo->queryDirect($sqlLoad);
|
|
|
|
// Remove any titles where length <=8
|
|
if ($verbose) {
|
|
echo $pdo->log->info("Deleting any records where title <=8 from Temporary Table");
|
|
}
|
|
$pdo->queryDirect("DELETE FROM tmp_pre WHERE LENGTH(title) <= 8");
|
|
|
|
// Add any groups that do not currently exist
|
|
$sqlAddGroups = <<<SQL_ADD_GROUPS
|
|
INSERT IGNORE INTO groups (`name`, description)
|
|
SELECT groupname, 'Added by predb import script'
|
|
FROM tmp_pre AS t LEFT JOIN groups AS g ON t.`groupname` = g.`name`
|
|
WHERE t.`groupname` IS NOT NULL AND g.`name` IS NULL
|
|
GROUP BY groupname;
|
|
SQL_ADD_GROUPS;
|
|
|
|
$pdo->queryDirect($sqlAddGroups);
|
|
|
|
// Fill the groupid
|
|
$pdo->queryDirect("UPDATE tmp_pre AS t SET groupid = (SELECT id FROM groups WHERE name = t.groupname) WHERE groupname IS NOT NULL");
|
|
|
|
// Insert and update table
|
|
$sqlInsert = <<<SQL_INSERT
|
|
INSERT INTO $table (title, nfo, size, files, filename, nuked, nukereason, category, predate, SOURCE, requestid, groupid)
|
|
SELECT t.title, t.nfo, t.size, t.files, t.filename, t.nuked, t.nukereason, t.category, t.predate, t.source, t.requestid, t.groupid
|
|
FROM tmp_pre AS t
|
|
ON DUPLICATE KEY UPDATE prehash.nfo = IF(prehash.nfo IS NULL, t.nfo, prehash.nfo),
|
|
prehash.size = IF(prehash.size IS NULL, t.size, prehash.size),
|
|
prehash.files = IF(prehash.files IS NULL, t.files, prehash.files),
|
|
prehash.filename = IF(prehash.filename = '', t.filename, prehash.filename),
|
|
prehash.nuked = IF(t.nuked > 0, t.nuked, prehash.nuked),
|
|
prehash.nukereason = IF(t.nuked > 0, t.nukereason, prehash.nukereason),
|
|
prehash.category = IF(prehash.category IS NULL, t.category, prehash.category),
|
|
prehash.requestid = IF(prehash.requestid = 0, t.requestid, prehash.requestid),
|
|
prehash.groupid = IF(prehash.groupid = 0, t.groupid, prehash.groupid);
|
|
SQL_INSERT;
|
|
|
|
echo $pdo->log->info("Inserting records from temporary table into $table");
|
|
if (NN_DEBUG) {
|
|
echo $pdo->log->primary($sqlInsert);
|
|
}
|
|
if ($pdo->queryDirect($sqlInsert) === false) {
|
|
echo "FAILED\n";
|
|
}
|
|
} |