queryExec('DROP TABLE IF EXISTS tmp_pre'); $pdo->queryExec('CREATE TABLE tmp_pre LIKE prehash'); // Drop id as it is not needed and incurs overhead creating each id. $pdo->queryExec('ALTER TABLE tmp_pre DROP COLUMN id'); // Add a column for the group's name which is included instead of the groupid, which may be // different between individual databases $pdo->queryExec('ALTER TABLE tmp_pre ADD COLUMN groupname VARCHAR (255)'); // Drop indexes on tmp_pre $pdo->queryExec('ALTER TABLE tmp_pre DROP INDEX `ix_prehash_nfo`, DROP INDEX `ix_prehash_predate`, DROP INDEX `ix_prehash_source`, DROP INDEX `ix_prehash_title`, DROP INDEX `ix_prehash_requestid`'); foreach ($all_matches as $matches) { if (preg_match('#^(.+)/(\d+)_#', $matches, $match)) { $timematch = -1 + $progress['last']; // Skip patches the user does not want. if ($match[2] < $timematch) { echo 'Skipping dump ' . $match[2] . ', as your minimum unix time argument is ' . $timematch . PHP_EOL; --$total; continue; } // Download the dump. $file['url'] = $baseUrl . '/' . $match[1] . '/' . $match[2] . $fileName . '?dl=1'; $dump = Utility::getUrl($file); if (!$dump) { echo 'Error downloading dump ' . $match[2] . ' you can try manually importing it.' . PHP_EOL; continue; } // Make sure we didn't get a HTML page. if (strlen($dump) < 5000 && strpos($dump, '') !== false) { echo 'The dump file ' . $match[2] . ' might be missing from dropbox.' . PHP_EOL; continue; } // Decompress. $dump = gzdecode($dump); if (!$dump) { echo 'Error decompressing dump ' . $match[2] . '.' . PHP_EOL; continue; } // Store the dump. $dumpFile = NN_RES . $match[2] . '_predb_dump.csv'; $fetched = file_put_contents($dumpFile, $dump); if (!$fetched) { echo 'Error storing dump file ' . $match[2] . ' in (' . NN_RES . ').' . PHP_EOL; continue; } // Make sure it's readable by all. chmod($dumpFile, 0777); $local = strtolower($argv[2]) == 'local' ? true : false; $verbose = $argv[3] == true ? true : false; importDump($dumpFile, $local, $verbose); // Delete the dump. // unlink($dumpFile); $progress = rw_progress(settings_array($match[2] + 1, $progress), false); echo 'Successfully imported PreDB dump ' . $match[2] . ' ' . (--$total) . ' dumps remaining to import.' . PHP_EOL; } } // Drop tmp_pre table $pdo->queryExec('DROP TABLE IF EXISTS tmp_pre'); } function settings_array($last = null, $settings = null) { if (is_null($settings)) { $settings['last'] = 0; } if (!is_null($last)) { $settings['last'] = $last; } return $settings; } function rw_progress($settings, $read = true) { if (!$read || !is_file(__DIR__ . DS . 'prehash_progress.txt')) { file_put_contents(__DIR__ . DS . 'prehash_progress.txt', base64_encode(serialize($settings))); } else { $settings = unserialize(base64_decode(file_get_contents(__DIR__ . DS . 'prehash_progress.txt' ) ) ); } return $settings; } // This function duplicates how dump_predb works but does not drop the hashes/triggers as recreating // potentially millions for the small addition that dailies add, isn't worth it. function importDump($path, $local, $verbose = true, $table = 'prehash') { global $pdo; // Create temp table to allow updating if ($verbose) { echo $pdo->log->info("Creating temporary table"); } // TRuncate to clear any old data $pdo->queryDirect("TRUNCATE TABLE tmp_pre"); // Import file into tmp_pre $sqlLoad = sprintf( "LOAD DATA %s INFILE '%s' IGNORE INTO TABLE tmp_pre FIELDS TERMINATED BY '\\t\\t' ENCLOSED BY \"'\" LINES TERMINATED BY '\\r\\n' (title, nfo, size, files, filename, nuked, nukereason, category, predate, source, requestid, groupname);", ($local === false ? 'LOCAL' : ''), $path ); if (NN_DEBUG) { echo $pdo->log->header($sqlLoad); } $pdo->queryDirect($sqlLoad); // Remove any titles where length <=8 if ($verbose) { echo $pdo->log->info("Deleting any records where title <=8 from Temporary Table"); } $pdo->queryDirect("DELETE FROM tmp_pre WHERE LENGTH(title) <= 8"); // Add any groups that do not currently exist $sqlAddGroups = <<queryDirect($sqlAddGroups); // Fill the groupid $pdo->queryDirect("UPDATE tmp_pre AS t SET groupid = (SELECT id FROM groups WHERE name = t.groupname) WHERE groupname IS NOT NULL"); // Insert and update table $sqlInsert = << 0, t.nuked, prehash.nuked), prehash.nukereason = IF(t.nuked > 0, t.nukereason, prehash.nukereason), prehash.category = IF(prehash.category IS NULL, t.category, prehash.category), prehash.requestid = IF(prehash.requestid = 0, t.requestid, prehash.requestid), prehash.groupid = IF(prehash.groupid = 0, t.groupid, prehash.groupid); SQL_INSERT; echo $pdo->log->info("Inserting records from temporary table into $table"); if (NN_DEBUG) { echo $pdo->log->primary($sqlInsert); } if ($pdo->queryDirect($sqlInsert) === false) { echo "FAILED\n"; } }