mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Update predb_daily_import_batch.php
This commit is contained in:
DariusIII
@@ -46,7 +46,7 @@ if (!is_writable(NN_RES)) {
|
||||
exit('The (' . NN_RES . ') folder must be writable.' . PHP_EOL);
|
||||
}
|
||||
|
||||
if (!isset($argv[1]) || !is_numeric($argv[1]) && $argv[1] != 'progress' || !isset($argv[2]) ||
|
||||
if (!isset($argv[1]) || (!is_numeric($argv[1]) && $argv[1] !== 'progress') || !isset($argv[2]) ||
|
||||
!in_array($argv[2], ['local', 'remote']) || !isset($argv[3]) ||
|
||||
!in_array($argv[3], ['true', 'false'])
|
||||
) {
|
||||
@@ -67,143 +67,194 @@ if (NN_DEBUG) {
|
||||
|
||||
$url = 'https://api.github.com/repos/nZEDb/nZEDbPre_Dumps/contents/dumps/';
|
||||
$filePattern = '(?P<filename>(?P<stamp>\d+)_predb_dump\.csv\.gz)';
|
||||
$client = new Client();
|
||||
|
||||
if (NN_DEBUG) {
|
||||
echo "Fetching predb_dump list from GitHub\n";
|
||||
echo "Fetching predb_dump directory list from GitHub\n";
|
||||
}
|
||||
|
||||
$result = $client->request('GET', $url,
|
||||
[
|
||||
'requestheaders' => [
|
||||
'Content-Type: application/json',
|
||||
'User-Agent: NNTmux'
|
||||
]
|
||||
]);
|
||||
$result = getDirListing($url);
|
||||
|
||||
if ($result === false) {
|
||||
exit('Error connecting to GitHub, try again later?' . PHP_EOL);
|
||||
}
|
||||
$dirs = json_decode($result, true);
|
||||
|
||||
if (NN_DEBUG) {
|
||||
echo "Extracting filenames from list.\n";
|
||||
}
|
||||
|
||||
$data = json_decode($result->getBody()->getContents(), true);
|
||||
if (is_null($data)) {
|
||||
if (is_null($dirs) ||
|
||||
(isset($dirs['message']) && substr($dirs['message'], 0, 27) == 'API rate limit exceeded for')) {
|
||||
exit("Error: $result");
|
||||
}
|
||||
|
||||
$total = count($data);
|
||||
|
||||
if (NN_DEBUG) {
|
||||
echo "Fetching predb_dump lists from GitHub\n";
|
||||
}
|
||||
|
||||
foreach ($dirs as $dir) {
|
||||
if ($dir['name'] == '0README.txt') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$result = getDirListing($url . $dir['name'] . '/');
|
||||
|
||||
if (NN_DEBUG) {
|
||||
echo "Extracting filenames from list.\n";
|
||||
}
|
||||
|
||||
$temp = json_decode($result, true);
|
||||
if (is_null($temp)) {
|
||||
exit("Error: $result");
|
||||
}
|
||||
|
||||
$data[$dir['name']] = $temp;
|
||||
}
|
||||
|
||||
$total = 0;
|
||||
foreach ($data as $dir => $files) {
|
||||
$total += count($files);
|
||||
}
|
||||
$total --;
|
||||
|
||||
$predb = new PreDb();
|
||||
|
||||
$progress = $predb->progress(settings_array());
|
||||
|
||||
foreach ($data as $file) {
|
||||
if (preg_match("#^https://raw\.githubusercontent\.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$#",
|
||||
$file['download_url'])) {
|
||||
if (preg_match("#^$filePattern$#", $file['name'], $match)) {
|
||||
$timematch = $progress['last'];
|
||||
foreach ($data as $dir => $files) {
|
||||
foreach ($files as $file) {
|
||||
//var_dump($file);
|
||||
if (preg_match("#^https://raw\.githubusercontent\.com/nZEDb/nZEDbPre_Dumps/master/dumps/$dir/$filePattern$#",
|
||||
$file['download_url'])) {
|
||||
if (preg_match("#^$filePattern$#", $file['name'], $match)) {
|
||||
$timematch = $progress['last'];
|
||||
|
||||
// Skip patches the user does not want.
|
||||
if ($match[1] < $timematch) {
|
||||
echo 'Skipping dump ' . $match[2] .
|
||||
', as your minimum unix time argument is ' .
|
||||
$timematch . PHP_EOL;
|
||||
--$total;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Download the dump.
|
||||
$dump = $client->get($file['download_url'])->getBody();
|
||||
echo "Downloading: {$file['download_url']}\n";
|
||||
|
||||
if (!$dump) {
|
||||
echo "Error downloading dump {$match[2]} you can try manually importing it." .
|
||||
PHP_EOL;
|
||||
continue;
|
||||
} else {
|
||||
if (NN_DEBUG) {
|
||||
echo "Dump {$match[2]} downloaded\n";
|
||||
// Skip patches the user does not want.
|
||||
if ($match[1] < $timematch) {
|
||||
echo 'Skipping dump ' . $match[2] .
|
||||
', as your minimum unix time argument is ' .
|
||||
$timematch . PHP_EOL;
|
||||
--$total;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Download the dump.
|
||||
$dump = Utility::getUrl(['url' => $file['download_url']]);
|
||||
echo "Downloading: {$file['download_url']}\n";
|
||||
|
||||
if (!$dump) {
|
||||
echo "Error downloading dump {$match[2]} you can try manually importing it." .
|
||||
PHP_EOL;
|
||||
continue;
|
||||
} else {
|
||||
if (NN_DEBUG) {
|
||||
echo "Dump {$match[2]} downloaded\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure we didn't get an HTML page.
|
||||
if (strpos($dump, '<!DOCTYPE html>') !== false) {
|
||||
echo "The dump file {$match[2]} might be missing from GitHub." . PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Decompress.
|
||||
$dump = gzdecode($dump);
|
||||
|
||||
if (!$dump) {
|
||||
echo "Error decompressing dump {$match[2]}." . PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Store the dump.
|
||||
$dumpFile = NN_RES . $match[2] . '_predb_dump.csv';
|
||||
$fetched = file_put_contents($dumpFile, $dump);
|
||||
if (!$fetched) {
|
||||
echo "Error storing dump file {$match[2]} in (" . NN_RES . ').' .
|
||||
PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Make sure it's readable by all.
|
||||
chmod($dumpFile, 0777);
|
||||
$local = strtolower($argv[2]) === 'local';
|
||||
$verbose = $argv[3] === true;
|
||||
|
||||
if ($verbose) {
|
||||
echo $predb->log->info('Clearing import table');
|
||||
}
|
||||
|
||||
// Truncate to clear any old data
|
||||
$predb->executeTruncate();
|
||||
|
||||
// Import file into predb_imports
|
||||
$predb->executeLoadData(
|
||||
[
|
||||
'fields' => '\\t\\t',
|
||||
'lines' => '\\r\\n',
|
||||
'local' => $local,
|
||||
'path' => $dumpFile,
|
||||
]);
|
||||
|
||||
// Remove any titles where length <=8
|
||||
if ($verbose === true) {
|
||||
echo $predb->log->info('Deleting any records where title <=8 from Temporary Table');
|
||||
}
|
||||
$predb->executeDeleteShort();
|
||||
|
||||
// Add any groups that do not currently exist
|
||||
$predb->executeAddGroups();
|
||||
|
||||
// Fill the groups_id
|
||||
$predb->executeUpdateGroupID();
|
||||
|
||||
echo $predb->log->info('Inserting records from temporary table into predb table');
|
||||
$predb->executeInsert();
|
||||
|
||||
// Delete the dump.
|
||||
unlink($dumpFile);
|
||||
|
||||
$progress = $predb->progress(settings_array($match[2] + 1, $progress),
|
||||
['read' => false]);
|
||||
echo sprintf("Successfully imported PreDB dump %d (%s), %d dumps remaining\n",
|
||||
$match[2],
|
||||
date('Y-m-d', $match[2]),
|
||||
--$total
|
||||
);
|
||||
} else {
|
||||
echo "Ignoring: {$file['download_url']}\n";
|
||||
}
|
||||
// Decompress.
|
||||
$dump = gzdecode($dump);
|
||||
|
||||
if (!$dump) {
|
||||
echo "Error decompressing dump {$match[2]}." . PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Store the dump.
|
||||
$dumpFile = NN_RES . $match[2] . '_predb_dump.csv';
|
||||
$fetched = file_put_contents($dumpFile, $dump);
|
||||
if (!$fetched) {
|
||||
echo "Error storing dump file {$match[2]} in (" . NN_RES . ').' .
|
||||
PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Make sure it's readable by all.
|
||||
chmod($dumpFile, 0777);
|
||||
$local = strtolower($argv[2]) == 'local' ? true : false;
|
||||
$verbose = $argv[3] == true ? true : false;
|
||||
|
||||
if ($verbose) {
|
||||
echo $predb->log->info("Clearing import table");
|
||||
}
|
||||
|
||||
// Truncate to clear any old data
|
||||
$predb->executeTruncate();
|
||||
|
||||
// Import file into predb_imports
|
||||
$predb->executeLoadData(
|
||||
[
|
||||
'fields' => '\\t\\t',
|
||||
'lines' => '\\r\\n',
|
||||
'local' => $local,
|
||||
'path' => $dumpFile,
|
||||
]);
|
||||
|
||||
// Remove any titles where length <=8
|
||||
if ($verbose === true) {
|
||||
echo $predb->log->info("Deleting any records where title <=8 from Temporary Table");
|
||||
}
|
||||
$predb->executeDeleteShort();
|
||||
|
||||
// Add any groups that do not currently exist
|
||||
$predb->executeAddGroups();
|
||||
|
||||
// Fill the groups_id
|
||||
$predb->executeUpdateGroupID();
|
||||
|
||||
echo $predb->log->info("Inserting records from temporary table into predb table");
|
||||
$predb->executeInsert();
|
||||
|
||||
// Delete the dump.
|
||||
unlink($dumpFile);
|
||||
|
||||
$progress = $predb->progress(settings_array($match[2] + 1, $progress),
|
||||
['read' => false]);
|
||||
echo "Successfully imported PreDB dump {$match[2]}, " . (--$total) .
|
||||
' dumps remaining.' . PHP_EOL;
|
||||
} else {
|
||||
echo "Ignoring: {$file['download_url']}\n";
|
||||
if (NN_DEBUG) {
|
||||
echo "^https://raw.githubusercontent.com/nZEDb/nZEDbPre_Dumps/master/dumps/$dir/$filePattern$\n {$file['download_url']}\n";
|
||||
}
|
||||
}
|
||||
} else if (NN_DEBUG) {
|
||||
echo "^https://raw.githubusercontent.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$\n {$file['download_url']}\n";
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
function settings_array($last = null, $settings = null)
|
||||
{
|
||||
if (is_null($settings)) {
|
||||
if ($settings === null) {
|
||||
$settings['last'] = 0;
|
||||
}
|
||||
|
||||
if (!is_null($last)) {
|
||||
if ($last !== null) {
|
||||
$settings['last'] = $last;
|
||||
}
|
||||
|
||||
return $settings;
|
||||
}
|
||||
|
||||
function getDirListing($url)
|
||||
{
|
||||
$result = Utility::getUrl(
|
||||
[
|
||||
'url' => $url,
|
||||
'requestheaders' => [
|
||||
'Content-Type: application/json',
|
||||
'User-Agent: nZEDb'
|
||||
]
|
||||
]);
|
||||
|
||||
if ($result === false) {
|
||||
exit('Error connecting to GitHub, try again later?' . PHP_EOL);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user