mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Revert Binaries
This commit is contained in:
DariusIII
+328
-275
@@ -333,7 +333,7 @@ class Binaries
|
||||
// We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet)
|
||||
$leaveOver = $this->messageBuffer;
|
||||
|
||||
// If this is not a new group, go from our newest to the servers newest.
|
||||
// If this is not a new group, go from our newest to the servers newest.
|
||||
} else {
|
||||
// Set our oldest wanted to our newest local article.
|
||||
$first = $groupMySQL['last_record'];
|
||||
@@ -377,14 +377,14 @@ class Binaries
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
(
|
||||
(int) $groupMySQL['last_record'] === 0
|
||||
? 'New group '.$groupNNTP['group'].' starting with '.
|
||||
(
|
||||
$this->_newGroupScanByDays
|
||||
? $this->_newGroupDaysToScan.' days'
|
||||
: number_format($this->_newGroupMessagesToScan).' messages'
|
||||
).' worth.'
|
||||
: 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.'
|
||||
(int) $groupMySQL['last_record'] === 0
|
||||
? 'New group '.$groupNNTP['group'].' starting with '.
|
||||
(
|
||||
$this->_newGroupScanByDays
|
||||
? $this->_newGroupDaysToScan.' days'
|
||||
: number_format($this->_newGroupMessagesToScan).' messages'
|
||||
).' worth.'
|
||||
: 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.'
|
||||
).
|
||||
' Leaving '.number_format($leaveOver).
|
||||
" for next pass.\nServer oldest: ".number_format($groupNNTP['first']).
|
||||
@@ -410,8 +410,8 @@ class Binaries
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->header(
|
||||
PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first).
|
||||
' to '.number_format($last).') from '.$groupMySQL['name'].' - ('.
|
||||
number_format($groupLast - $last).' articles in queue).'
|
||||
' to '.number_format($last).') from '.$groupMySQL['name'].' - ('.
|
||||
number_format($groupLast - $last).' articles in queue).'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -479,15 +479,15 @@ class Binaries
|
||||
$endGroup = now()->diffInSeconds($startGroup);
|
||||
$this->colorCli->primary(
|
||||
PHP_EOL.'Group '.$groupMySQL['name'].' processed in '.
|
||||
$endGroup.Str::plural(' second', $endGroup)
|
||||
$endGroup.Str::plural(' second', $endGroup)
|
||||
);
|
||||
}
|
||||
} elseif ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
'No new articles for '.$groupMySQL['name'].' (first '.number_format($first).
|
||||
', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']).
|
||||
', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']).
|
||||
' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record'])
|
||||
', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']).
|
||||
', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']).
|
||||
' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record'])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -689,7 +689,7 @@ class Binaries
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->alternate(
|
||||
'Server did not return '.$notReceivedCount.
|
||||
' articles from '.$this->groupMySQL['name'].'.'
|
||||
' articles from '.$this->groupMySQL['name'].'.'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -711,183 +711,193 @@ class Binaries
|
||||
*/
|
||||
protected function storeHeaders(array $headers = []): void
|
||||
{
|
||||
DB::transaction(function () use ($headers) {
|
||||
$binariesUpdate = $collectionIDs = $articles = [];
|
||||
$binariesUpdate = $collectionIDs = $articles = [];
|
||||
|
||||
$partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES ';
|
||||
DB::beginTransaction();
|
||||
|
||||
// Loop articles, figure out files/parts.
|
||||
foreach ($headers as $this->header) {
|
||||
// Set up the info for inserting into parts/binaries/collections tables.
|
||||
if (! isset($articles[$this->header['matches'][1]])) {
|
||||
// check whether file count should be ignored (XXX packs for now only).
|
||||
$whitelistMatch = false;
|
||||
if ($this->_ignoreFileCount($this->groupMySQL['name'], $this->header['matches'][1])) {
|
||||
$whitelistMatch = true;
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
$partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES ';
|
||||
|
||||
// Loop articles, figure out files/parts.
|
||||
foreach ($headers as $this->header) {
|
||||
// Set up the info for inserting into parts/binaries/collections tables.
|
||||
if (! isset($articles[$this->header['matches'][1]])) {
|
||||
// check whether file count should be ignored (XXX packs for now only).
|
||||
$whitelistMatch = false;
|
||||
if ($this->_ignoreFileCount($this->groupMySQL['name'], $this->header['matches'][1])) {
|
||||
$whitelistMatch = true;
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
}
|
||||
|
||||
// Attempt to find the file count. If it is not found, set it to 0.
|
||||
if (! $whitelistMatch && ! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $this->header['matches'][1], $fileCount)) {
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
}
|
||||
|
||||
$collMatch = $this->_collectionsCleaning->collectionsCleaner(
|
||||
$this->header['matches'][1]
|
||||
);
|
||||
|
||||
// Used to group articles together when forming the release.
|
||||
$this->header['CollectionKey'] = $collMatch['name'].$fileCount[3];
|
||||
|
||||
// If this header's collection key isn't in memory, attempt to insert the collection
|
||||
if (! isset($collectionIDs[$this->header['CollectionKey']])) {
|
||||
/* Date from header should be a string this format:
|
||||
* 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500
|
||||
* Still make sure it's not unix time, convert it to unix time if it is.
|
||||
*/
|
||||
$this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date']));
|
||||
|
||||
// Get the current unixtime from PHP.
|
||||
$now = now()->timestamp;
|
||||
|
||||
$xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref');
|
||||
|
||||
$tempHeaderXrefs = [];
|
||||
foreach (explode(' ', $this->header['Xref']) as $headerXref) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) {
|
||||
$tempHeaderXrefs[] = $hit[0];
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to find the file count. If it is not found, set it to 0.
|
||||
if (! $whitelistMatch && ! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $this->header['matches'][1], $fileCount)) {
|
||||
$fileCount[1] = $fileCount[3] = 0;
|
||||
$tempXrefsData = [];
|
||||
|
||||
if ($xrefsData !== null) {
|
||||
foreach (explode(' ', $xrefsData) as $xrefData) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) {
|
||||
$tempXrefsData[] = $match1[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$collMatch = $this->_collectionsCleaning->collectionsCleaner(
|
||||
$this->header['matches'][1]
|
||||
);
|
||||
|
||||
// Used to group articles together when forming the release.
|
||||
$this->header['CollectionKey'] = $collMatch['name'].$fileCount[3];
|
||||
|
||||
// If this header's collection key isn't in memory, attempt to insert the collection
|
||||
if (! isset($collectionIDs[$this->header['CollectionKey']])) {
|
||||
/* Date from header should be a string this format:
|
||||
* 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500
|
||||
* Still make sure it's not unix time, convert it to unix time if it is.
|
||||
*/
|
||||
$this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date']));
|
||||
|
||||
// Get the current unixtime from PHP.
|
||||
$now = now()->timestamp;
|
||||
|
||||
$xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref');
|
||||
|
||||
$tempHeaderXrefs = [];
|
||||
foreach (explode(' ', $this->header['Xref']) as $headerXref) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) {
|
||||
$tempHeaderXrefs[] = $hit[0];
|
||||
}
|
||||
$finalXrefArray = [];
|
||||
foreach ($tempHeaderXrefs as $tempHeaderXref) {
|
||||
if (! in_array($tempHeaderXref, $tempXrefsData, false)) {
|
||||
$finalXrefArray[] = $tempHeaderXref;
|
||||
}
|
||||
}
|
||||
|
||||
$tempXrefsData = [];
|
||||
$finaXref = implode(' ', $finalXrefArray);
|
||||
|
||||
if ($xrefsData !== null) {
|
||||
foreach (explode(' ', $xrefsData) as $xrefData) {
|
||||
if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) {
|
||||
$tempXrefsData[] = $match1[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
$xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref));
|
||||
|
||||
$finalXrefArray = [];
|
||||
foreach ($tempHeaderXrefs as $tempHeaderXref) {
|
||||
if (! in_array($tempHeaderXref, $tempXrefsData, false)) {
|
||||
$finalXrefArray[] = $tempHeaderXref;
|
||||
}
|
||||
}
|
||||
$date = $this->header['Date'] > $now ? $now : $this->header['Date'];
|
||||
$unixtime = is_numeric($this->header['Date']) ? $date : $now;
|
||||
|
||||
$finaXref = implode(' ', $finalXrefArray);
|
||||
$random = random_bytes(16);
|
||||
|
||||
$xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref));
|
||||
$collectionID = false;
|
||||
|
||||
$date = $this->header['Date'] > $now ? $now : $this->header['Date'];
|
||||
$unixtime = is_numeric($this->header['Date']) ? $date : $now;
|
||||
|
||||
$random = random_bytes(16);
|
||||
|
||||
$collectionID = false;
|
||||
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
INSERT INTO collections (subject, fromname, date, xref, groups_id,
|
||||
totalfiles, collectionhash, collection_regexes_id, dateadded)
|
||||
VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW())
|
||||
ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", escapeString(substr(utf8_encode($this->header['matches'][1]), 0, 255)), escapeString(utf8_encode($this->header['From'])), $unixtime, escapeString(implode(' ', $tempHeaderXrefs)), $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), $collMatch['id'], $xref, sodium_bin2hex($random)));
|
||||
$collectionID = $this->_pdo->lastInsertId();
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
if ($collectionID === false) {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted[] = $this->header['Number'];
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
$collectionIDs[$this->header['CollectionKey']] = $collectionID;
|
||||
} else {
|
||||
$collectionID = $collectionIDs[$this->header['CollectionKey']];
|
||||
}
|
||||
|
||||
// Binary Hash should be unique to the group
|
||||
$hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']);
|
||||
|
||||
$binaryID = false;
|
||||
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize)
|
||||
VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d)
|
||||
ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(utf8_encode($this->header['matches'][1])), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes']));
|
||||
$binaryID = $this->_pdo->lastInsertId();
|
||||
$collectionID = $this->_pdo->lastInsertId();
|
||||
DB::commit();
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
DB::rollBack();
|
||||
}
|
||||
|
||||
if ($binaryID === false) {
|
||||
if ($collectionID === false) {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted[] = $this->header['Number'];
|
||||
}
|
||||
DB::rollBack();
|
||||
DB::beginTransaction();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$binariesUpdate[$binaryID]['Size'] = 0;
|
||||
$binariesUpdate[$binaryID]['Parts'] = 0;
|
||||
|
||||
$articles[$this->header['matches'][1]]['CollectionID'] = $collectionID;
|
||||
$articles[$this->header['matches'][1]]['BinaryID'] = $binaryID;
|
||||
$collectionIDs[$this->header['CollectionKey']] = $collectionID;
|
||||
} else {
|
||||
$binaryID = $articles[$this->header['matches'][1]]['BinaryID'];
|
||||
$binariesUpdate[$binaryID]['Size'] += $this->header['Bytes'];
|
||||
$binariesUpdate[$binaryID]['Parts']++;
|
||||
$collectionID = $collectionIDs[$this->header['CollectionKey']];
|
||||
}
|
||||
|
||||
// In case there are quotes in the message id
|
||||
$this->header['Message-ID'] = addslashes($this->header['Message-ID']);
|
||||
// Binary Hash should be unique to the group
|
||||
$hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']);
|
||||
|
||||
// Strip the < and >, saves space in DB.
|
||||
$this->header['Message-ID'][0] = "'";
|
||||
$binaryID = false;
|
||||
|
||||
$partsQuery .=
|
||||
'('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',".
|
||||
$this->header['matches'][2].','.$this->header['Bytes'].'),';
|
||||
}
|
||||
|
||||
unset($headers); // Reclaim memory.
|
||||
|
||||
// Start of inserting into SQL.
|
||||
$this->startUpdate = now();
|
||||
|
||||
// End of processing headers.
|
||||
$this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning);
|
||||
$binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES ';
|
||||
foreach ($binariesUpdate as $binaryID => $binary) {
|
||||
$binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),';
|
||||
}
|
||||
$binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts';
|
||||
$binariesQuery = rtrim($binariesQuery, ',').$binariesEnd;
|
||||
|
||||
// Check if we got any binaries. If we did, try to insert them.
|
||||
if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) ? true : $this->runQuery($binariesQuery)) {
|
||||
if (\strlen($partsQuery) === \strlen($partsCheck) ? true : $this->runQuery(rtrim($partsQuery, ','))) {
|
||||
} else {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted += $this->headersReceived;
|
||||
try {
|
||||
DB::insert(sprintf("
|
||||
INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize)
|
||||
VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d)
|
||||
ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(utf8_encode($this->header['matches'][1])), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes']));
|
||||
$binaryID = $this->_pdo->lastInsertId();
|
||||
DB::commit();
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
DB::rollBack();
|
||||
}
|
||||
|
||||
if ($binaryID === false) {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted[] = $this->header['Number'];
|
||||
}
|
||||
DB::rollBack();
|
||||
DB::beginTransaction();
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$binariesUpdate[$binaryID]['Size'] = 0;
|
||||
$binariesUpdate[$binaryID]['Parts'] = 0;
|
||||
|
||||
$articles[$this->header['matches'][1]]['CollectionID'] = $collectionID;
|
||||
$articles[$this->header['matches'][1]]['BinaryID'] = $binaryID;
|
||||
} else {
|
||||
$binaryID = $articles[$this->header['matches'][1]]['BinaryID'];
|
||||
$binariesUpdate[$binaryID]['Size'] += $this->header['Bytes'];
|
||||
$binariesUpdate[$binaryID]['Parts']++;
|
||||
}
|
||||
|
||||
// In case there are quotes in the message id
|
||||
$this->header['Message-ID'] = addslashes($this->header['Message-ID']);
|
||||
|
||||
// Strip the < and >, saves space in DB.
|
||||
$this->header['Message-ID'][0] = "'";
|
||||
|
||||
$partsQuery .=
|
||||
'('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',".
|
||||
$this->header['matches'][2].','.$this->header['Bytes'].'),';
|
||||
}
|
||||
|
||||
unset($headers); // Reclaim memory.
|
||||
|
||||
// Start of inserting into SQL.
|
||||
$this->startUpdate = now();
|
||||
|
||||
// End of processing headers.
|
||||
$this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning);
|
||||
$binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES ';
|
||||
foreach ($binariesUpdate as $binaryID => $binary) {
|
||||
$binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),';
|
||||
}
|
||||
$binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts';
|
||||
$binariesQuery = rtrim($binariesQuery, ',').$binariesEnd;
|
||||
|
||||
// Check if we got any binaries. If we did, try to insert them.
|
||||
if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) ? true : $this->runQuery($binariesQuery)) {
|
||||
if (\strlen($partsQuery) === \strlen($partsCheck) ? true : $this->runQuery(rtrim($partsQuery, ','))) {
|
||||
DB::commit();
|
||||
} else {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted += $this->headersReceived;
|
||||
}
|
||||
DB::rollBack();
|
||||
}
|
||||
}, 10);
|
||||
|
||||
} else {
|
||||
if ($this->addToPartRepair) {
|
||||
$this->headersNotInserted += $this->headersReceived;
|
||||
}
|
||||
DB::rollBack();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -937,8 +947,8 @@ class Binaries
|
||||
{
|
||||
$this->colorCli->primary(
|
||||
'Received '.\count($this->headersReceived).
|
||||
' articles of '.number_format($this->last - $this->first + 1).' requested, '.
|
||||
$this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.'
|
||||
' articles of '.number_format($this->last - $this->first + 1).' requested, '.
|
||||
$this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -950,18 +960,40 @@ class Binaries
|
||||
$currentMicroTime = now();
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->alternateOver($this->timeHeaders.'s').
|
||||
$this->colorCli->primaryOver(' to download articles, ').
|
||||
$this->colorCli->alternateOver($this->timeCleaning.'s').
|
||||
$this->colorCli->primaryOver(' to process collections, ').
|
||||
$this->colorCli->alternateOver($this->timeInsert.'s').
|
||||
$this->colorCli->primaryOver(' to insert binaries/parts, ').
|
||||
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s').
|
||||
$this->colorCli->primaryOver(' for part repair, ').
|
||||
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s').
|
||||
$this->colorCli->primary(' total.');
|
||||
$this->colorCli->primaryOver(' to download articles, ').
|
||||
$this->colorCli->alternateOver($this->timeCleaning.'s').
|
||||
$this->colorCli->primaryOver(' to process collections, ').
|
||||
$this->colorCli->alternateOver($this->timeInsert.'s').
|
||||
$this->colorCli->primaryOver(' to insert binaries/parts, ').
|
||||
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s').
|
||||
$this->colorCli->primaryOver(' for part repair, ').
|
||||
$this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s').
|
||||
$this->colorCli->primary(' total.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* If we failed to insert Collections/Binaries/Parts, rollback the transaction and add the parts to part repair.
|
||||
*
|
||||
* @param array $headers Array of headers containing sub-arrays with parts.
|
||||
* @return array Array of article numbers to add to part repair.
|
||||
*
|
||||
* @throws \Exception
|
||||
* @throws \Throwable
|
||||
*/
|
||||
protected function _rollbackAddToPartRepair(array $headers): array
|
||||
{
|
||||
$headersNotInserted = [];
|
||||
foreach ($headers as $header) {
|
||||
foreach ($header as $file) {
|
||||
$headersNotInserted[] = $file['Parts']['number'];
|
||||
}
|
||||
}
|
||||
DB::rollBack();
|
||||
|
||||
return $headersNotInserted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to get missing article headers.
|
||||
*
|
||||
@@ -972,103 +1004,120 @@ class Binaries
|
||||
*/
|
||||
public function partRepair(array $groupArr): void
|
||||
{
|
||||
DB::transaction(function () use ($groupArr) {
|
||||
// Get all parts in partrepair table.
|
||||
$missingParts = MissedPart::select()->where(['groups_id' => $groupArr['id'], 'attempts' => $this->_partRepairMaxTries])->orderBy('numberid', 'asc')->limit($this->_partRepairLimit)->get()->toArray();
|
||||
// Get all parts in partrepair table.
|
||||
$missingParts = [];
|
||||
try {
|
||||
$missingParts = DB::select(sprintf('
|
||||
SELECT * FROM missed_parts
|
||||
WHERE groups_id = %d AND attempts < %d
|
||||
ORDER BY numberid ASC LIMIT %d', $groupArr['id'], $this->_partRepairMaxTries, $this->_partRepairLimit));
|
||||
} catch (\PDOException $e) {
|
||||
if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') {
|
||||
$this->colorCli->notice('Deadlock occurred');
|
||||
DB::rollBack();
|
||||
}
|
||||
}
|
||||
|
||||
$missingCount = \count($missingParts);
|
||||
if ($missingCount > 0) {
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
'Attempting to repair '.
|
||||
number_format($missingCount).
|
||||
' parts.'
|
||||
);
|
||||
}
|
||||
|
||||
// Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4.
|
||||
$ranges = $partList = [];
|
||||
$firstPart = $lastNum = $missingParts[0]->numberid;
|
||||
|
||||
foreach ($missingParts as $part) {
|
||||
if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) {
|
||||
$ranges[] = [
|
||||
'partfrom' => $firstPart,
|
||||
'partto' => $lastNum,
|
||||
'partlist' => $partList,
|
||||
];
|
||||
|
||||
$firstPart = $part->numberid;
|
||||
$partList = [];
|
||||
}
|
||||
$partList[] = $part->numberid;
|
||||
$lastNum = $part->numberid;
|
||||
}
|
||||
|
||||
$ranges[] = [
|
||||
'partfrom' => $firstPart,
|
||||
'partto' => $lastNum,
|
||||
'partlist' => $partList,
|
||||
];
|
||||
|
||||
// Download missing parts in ranges.
|
||||
foreach ($ranges as $range) {
|
||||
$partFrom = $range['partfrom'];
|
||||
$partTo = $range['partto'];
|
||||
$partList = $range['partlist'];
|
||||
|
||||
$missingCount = \count($missingParts);
|
||||
if ($missingCount > 0) {
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
'Attempting to repair '.
|
||||
number_format($missingCount).
|
||||
' parts.'
|
||||
);
|
||||
echo \chr(random_int(45, 46)).PHP_EOL;
|
||||
}
|
||||
|
||||
// Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4.
|
||||
$ranges = $partList = [];
|
||||
$firstPart = $lastNum = $missingParts[0]->numberid;
|
||||
// Get article headers from newsgroup.
|
||||
$this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList);
|
||||
}
|
||||
|
||||
foreach ($missingParts as $part) {
|
||||
if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) {
|
||||
$ranges[] = [
|
||||
'partfrom' => $firstPart,
|
||||
'partto' => $lastNum,
|
||||
'partlist' => $partList,
|
||||
];
|
||||
|
||||
$firstPart = $part->numberid;
|
||||
$partList = [];
|
||||
}
|
||||
$partList[] = $part->numberid;
|
||||
$lastNum = $part->numberid;
|
||||
}
|
||||
|
||||
$ranges[] = [
|
||||
'partfrom' => $firstPart,
|
||||
'partto' => $lastNum,
|
||||
'partlist' => $partList,
|
||||
];
|
||||
|
||||
// Download missing parts in ranges.
|
||||
foreach ($ranges as $range) {
|
||||
$partFrom = $range['partfrom'];
|
||||
$partTo = $range['partto'];
|
||||
$partList = $range['partlist'];
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
echo \chr(random_int(45, 46)).PHP_EOL;
|
||||
}
|
||||
|
||||
// Get article headers from newsgroup.
|
||||
$this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList);
|
||||
}
|
||||
|
||||
// Calculate parts repaired
|
||||
$result = DB::select(
|
||||
sprintf(
|
||||
'
|
||||
// Calculate parts repaired
|
||||
$result = DB::select(
|
||||
sprintf(
|
||||
'
|
||||
SELECT COUNT(id) AS num
|
||||
FROM missed_parts
|
||||
WHERE groups_id = %d
|
||||
AND numberid <= %d',
|
||||
$groupArr['id'],
|
||||
$missingParts[$missingCount - 1]->numberid
|
||||
)
|
||||
);
|
||||
$groupArr['id'],
|
||||
$missingParts[$missingCount - 1]->numberid
|
||||
)
|
||||
);
|
||||
|
||||
$partsRepaired = 0;
|
||||
if ($result > 0) {
|
||||
$partsRepaired = ($missingCount - $result[0]->num);
|
||||
}
|
||||
$partsRepaired = 0;
|
||||
if ($result > 0) {
|
||||
$partsRepaired = ($missingCount - $result[0]->num);
|
||||
}
|
||||
|
||||
// Update attempts on remaining parts for active group
|
||||
if (isset($missingParts[$missingCount - 1]->id)) {
|
||||
DB::update(
|
||||
sprintf(
|
||||
'
|
||||
// Update attempts on remaining parts for active group
|
||||
if (isset($missingParts[$missingCount - 1]->id)) {
|
||||
DB::update(
|
||||
sprintf(
|
||||
'
|
||||
UPDATE missed_parts
|
||||
SET attempts = attempts + 1
|
||||
WHERE groups_id = %d
|
||||
AND numberid <= %d',
|
||||
$groupArr['id'],
|
||||
$missingParts[$missingCount - 1]->numberid
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
PHP_EOL.
|
||||
number_format($partsRepaired).
|
||||
' parts repaired.'
|
||||
);
|
||||
}
|
||||
$groupArr['id'],
|
||||
$missingParts[$missingCount - 1]->numberid
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Remove articles that we cant fetch after x attempts.
|
||||
MissedPart::query()->where('attempts', '>=', $this->_partRepairMaxTries)->where('groups_id', $groupArr['id'])->delete();
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
PHP_EOL.
|
||||
number_format($partsRepaired).
|
||||
' parts repaired.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove articles that we cant fetch after x attempts.
|
||||
DB::transaction(function () use ($groupArr) {
|
||||
DB::delete(
|
||||
sprintf(
|
||||
'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d',
|
||||
$this->_partRepairMaxTries,
|
||||
$groupArr['id']
|
||||
)
|
||||
);
|
||||
}, 10);
|
||||
}
|
||||
|
||||
@@ -1230,7 +1279,7 @@ class Binaries
|
||||
if ($this->_echoCLI) {
|
||||
$this->colorCli->primary(
|
||||
PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime).
|
||||
', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.'
|
||||
', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1240,21 +1289,17 @@ class Binaries
|
||||
/**
|
||||
* Add article numbers from missing headers to DB.
|
||||
*
|
||||
* @param array $numbers The article numbers of the missing headers.
|
||||
* @param int $groupID The ID of this groups.
|
||||
*
|
||||
* @throws \Throwable
|
||||
* @param array $numbers The article numbers of the missing headers.
|
||||
* @param int $groupID The ID of this groups.
|
||||
*/
|
||||
private function addMissingParts(array $numbers, int $groupID): string
|
||||
{
|
||||
DB::transaction(function () use ($numbers, $groupID) {
|
||||
$insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES ';
|
||||
foreach ($numbers as $number) {
|
||||
$insertStr .= '('.$number.','.$groupID.'),';
|
||||
}
|
||||
$insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES ';
|
||||
foreach ($numbers as $number) {
|
||||
$insertStr .= '('.$number.','.$groupID.'),';
|
||||
}
|
||||
|
||||
DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1');
|
||||
}, 10);
|
||||
DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1');
|
||||
|
||||
return $this->_pdo->lastInsertId();
|
||||
}
|
||||
@@ -1269,7 +1314,13 @@ class Binaries
|
||||
*/
|
||||
private function removeRepairedParts(array $numbers, int $groupID): void
|
||||
{
|
||||
MissedPart::whereIn('numberid', $numbers)->where('groups_id', $groupID)->delete();
|
||||
$sql = 'DELETE FROM missed_parts WHERE numberid in (';
|
||||
foreach ($numbers as $number) {
|
||||
$sql .= $number.',';
|
||||
}
|
||||
DB::transaction(static function () use ($groupID, $sql) {
|
||||
DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupID);
|
||||
}, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1438,6 +1489,10 @@ class Binaries
|
||||
*/
|
||||
public function delete(int $collectionID): void
|
||||
{
|
||||
DB::transaction(static function () use ($collectionID) {
|
||||
DB::delete(sprintf('DELETE FROM collections WHERE id = %d', $collectionID));
|
||||
}, 10);
|
||||
|
||||
Collection::query()->where('id', $collectionID)->delete();
|
||||
}
|
||||
|
||||
@@ -1470,26 +1525,24 @@ class Binaries
|
||||
|
||||
protected function runQuery($query): bool
|
||||
{
|
||||
DB::transaction(function () use ($query) {
|
||||
try {
|
||||
return DB::insert($query);
|
||||
} catch (QueryException $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
} catch (\PDOException $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
try {
|
||||
return DB::insert($query);
|
||||
} catch (QueryException $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
});
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
} catch (\PDOException $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
} catch (\Throwable $e) {
|
||||
if (config('app.debug') === true) {
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
$this->colorCli->debug('Query error occurred.');
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user