From 18ea6d922c0b4a5e63330fd44c3f28c1b2d9378f Mon Sep 17 00:00:00 2001 From: DariusIII Date: Wed, 29 Mar 2017 15:31:05 +0200 Subject: [PATCH] Update predb_daily_import_batch.php --- cli/data/predb_import_daily_batch.php | 271 +++++++++++++++----------- 1 file changed, 161 insertions(+), 110 deletions(-) diff --git a/cli/data/predb_import_daily_batch.php b/cli/data/predb_import_daily_batch.php index b38366a88..0d75ff26b 100755 --- a/cli/data/predb_import_daily_batch.php +++ b/cli/data/predb_import_daily_batch.php @@ -46,7 +46,7 @@ if (!is_writable(NN_RES)) { exit('The (' . NN_RES . ') folder must be writable.' . PHP_EOL); } -if (!isset($argv[1]) || !is_numeric($argv[1]) && $argv[1] != 'progress' || !isset($argv[2]) || +if (!isset($argv[1]) || (!is_numeric($argv[1]) && $argv[1] !== 'progress') || !isset($argv[2]) || !in_array($argv[2], ['local', 'remote']) || !isset($argv[3]) || !in_array($argv[3], ['true', 'false']) ) { @@ -67,143 +67,194 @@ if (NN_DEBUG) { $url = 'https://api.github.com/repos/nZEDb/nZEDbPre_Dumps/contents/dumps/'; $filePattern = '(?P(?P\d+)_predb_dump\.csv\.gz)'; -$client = new Client(); if (NN_DEBUG) { - echo "Fetching predb_dump list from GitHub\n"; + echo "Fetching predb_dump directory list from GitHub\n"; } -$result = $client->request('GET', $url, - [ - 'requestheaders' => [ - 'Content-Type: application/json', - 'User-Agent: NNTmux' - ] - ]); +$result = getDirListing($url); -if ($result === false) { - exit('Error connecting to GitHub, try again later?' . PHP_EOL); -} +$dirs = json_decode($result, true); -if (NN_DEBUG) { - echo "Extracting filenames from list.\n"; -} - -$data = json_decode($result->getBody()->getContents(), true); -if (is_null($data)) { +if (is_null($dirs) || + (isset($dirs['message']) && substr($dirs['message'], 0, 27) == 'API rate limit exceeded for')) { exit("Error: $result"); } -$total = count($data); + +if (NN_DEBUG) { + echo "Fetching predb_dump lists from GitHub\n"; +} + +foreach ($dirs as $dir) { + if ($dir['name'] == '0README.txt') { + continue; + } + + $result = getDirListing($url . $dir['name'] . '/'); + + if (NN_DEBUG) { + echo "Extracting filenames from list.\n"; + } + + $temp = json_decode($result, true); + if (is_null($temp)) { + exit("Error: $result"); + } + + $data[$dir['name']] = $temp; +} + +$total = 0; +foreach ($data as $dir => $files) { + $total += count($files); +} +$total --; + $predb = new PreDb(); $progress = $predb->progress(settings_array()); -foreach ($data as $file) { - if (preg_match("#^https://raw\.githubusercontent\.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$#", - $file['download_url'])) { - if (preg_match("#^$filePattern$#", $file['name'], $match)) { - $timematch = $progress['last']; +foreach ($data as $dir => $files) { + foreach ($files as $file) { + //var_dump($file); + if (preg_match("#^https://raw\.githubusercontent\.com/nZEDb/nZEDbPre_Dumps/master/dumps/$dir/$filePattern$#", + $file['download_url'])) { + if (preg_match("#^$filePattern$#", $file['name'], $match)) { + $timematch = $progress['last']; - // Skip patches the user does not want. - if ($match[1] < $timematch) { - echo 'Skipping dump ' . $match[2] . - ', as your minimum unix time argument is ' . - $timematch . PHP_EOL; - --$total; - continue; - } - - // Download the dump. - $dump = $client->get($file['download_url'])->getBody(); - echo "Downloading: {$file['download_url']}\n"; - - if (!$dump) { - echo "Error downloading dump {$match[2]} you can try manually importing it." . - PHP_EOL; - continue; - } else { - if (NN_DEBUG) { - echo "Dump {$match[2]} downloaded\n"; + // Skip patches the user does not want. + if ($match[1] < $timematch) { + echo 'Skipping dump ' . $match[2] . + ', as your minimum unix time argument is ' . + $timematch . PHP_EOL; + --$total; + continue; } + + // Download the dump. + $dump = Utility::getUrl(['url' => $file['download_url']]); + echo "Downloading: {$file['download_url']}\n"; + + if (!$dump) { + echo "Error downloading dump {$match[2]} you can try manually importing it." . + PHP_EOL; + continue; + } else { + if (NN_DEBUG) { + echo "Dump {$match[2]} downloaded\n"; + } + } + + // Make sure we didn't get an HTML page. + if (strpos($dump, '') !== false) { + echo "The dump file {$match[2]} might be missing from GitHub." . PHP_EOL; + continue; + } + + // Decompress. + $dump = gzdecode($dump); + + if (!$dump) { + echo "Error decompressing dump {$match[2]}." . PHP_EOL; + continue; + } + + // Store the dump. + $dumpFile = NN_RES . $match[2] . '_predb_dump.csv'; + $fetched = file_put_contents($dumpFile, $dump); + if (!$fetched) { + echo "Error storing dump file {$match[2]} in (" . NN_RES . ').' . + PHP_EOL; + continue; + } + + // Make sure it's readable by all. + chmod($dumpFile, 0777); + $local = strtolower($argv[2]) === 'local'; + $verbose = $argv[3] === true; + + if ($verbose) { + echo $predb->log->info('Clearing import table'); + } + + // Truncate to clear any old data + $predb->executeTruncate(); + + // Import file into predb_imports + $predb->executeLoadData( + [ + 'fields' => '\\t\\t', + 'lines' => '\\r\\n', + 'local' => $local, + 'path' => $dumpFile, + ]); + + // Remove any titles where length <=8 + if ($verbose === true) { + echo $predb->log->info('Deleting any records where title <=8 from Temporary Table'); + } + $predb->executeDeleteShort(); + + // Add any groups that do not currently exist + $predb->executeAddGroups(); + + // Fill the groups_id + $predb->executeUpdateGroupID(); + + echo $predb->log->info('Inserting records from temporary table into predb table'); + $predb->executeInsert(); + + // Delete the dump. + unlink($dumpFile); + + $progress = $predb->progress(settings_array($match[2] + 1, $progress), + ['read' => false]); + echo sprintf("Successfully imported PreDB dump %d (%s), %d dumps remaining\n", + $match[2], + date('Y-m-d', $match[2]), + --$total + ); + } else { + echo "Ignoring: {$file['download_url']}\n"; } - // Decompress. - $dump = gzdecode($dump); - - if (!$dump) { - echo "Error decompressing dump {$match[2]}." . PHP_EOL; - continue; - } - - // Store the dump. - $dumpFile = NN_RES . $match[2] . '_predb_dump.csv'; - $fetched = file_put_contents($dumpFile, $dump); - if (!$fetched) { - echo "Error storing dump file {$match[2]} in (" . NN_RES . ').' . - PHP_EOL; - continue; - } - - // Make sure it's readable by all. - chmod($dumpFile, 0777); - $local = strtolower($argv[2]) == 'local' ? true : false; - $verbose = $argv[3] == true ? true : false; - - if ($verbose) { - echo $predb->log->info("Clearing import table"); - } - - // Truncate to clear any old data - $predb->executeTruncate(); - - // Import file into predb_imports - $predb->executeLoadData( - [ - 'fields' => '\\t\\t', - 'lines' => '\\r\\n', - 'local' => $local, - 'path' => $dumpFile, - ]); - - // Remove any titles where length <=8 - if ($verbose === true) { - echo $predb->log->info("Deleting any records where title <=8 from Temporary Table"); - } - $predb->executeDeleteShort(); - - // Add any groups that do not currently exist - $predb->executeAddGroups(); - - // Fill the groups_id - $predb->executeUpdateGroupID(); - - echo $predb->log->info("Inserting records from temporary table into predb table"); - $predb->executeInsert(); - - // Delete the dump. - unlink($dumpFile); - - $progress = $predb->progress(settings_array($match[2] + 1, $progress), - ['read' => false]); - echo "Successfully imported PreDB dump {$match[2]}, " . (--$total) . - ' dumps remaining.' . PHP_EOL; } else { - echo "Ignoring: {$file['download_url']}\n"; + if (NN_DEBUG) { + echo "^https://raw.githubusercontent.com/nZEDb/nZEDbPre_Dumps/master/dumps/$dir/$filePattern$\n {$file['download_url']}\n"; + } } - } else if (NN_DEBUG) { - echo "^https://raw.githubusercontent.com/nZEDb/nZEDbPre_Dumps/master/dumps/$filePattern$\n {$file['download_url']}\n"; } } +//////////////////////////////////////////////////////////////////////////////////////////////////// + function settings_array($last = null, $settings = null) { - if (is_null($settings)) { + if ($settings === null) { $settings['last'] = 0; } - if (!is_null($last)) { + if ($last !== null) { $settings['last'] = $last; } return $settings; } + +function getDirListing($url) +{ + $result = Utility::getUrl( + [ + 'url' => $url, + 'requestheaders' => [ + 'Content-Type: application/json', + 'User-Agent: nZEDb' + ] + ]); + + if ($result === false) { + exit('Error connecting to GitHub, try again later?' . PHP_EOL); + } + + return $result; +}