diff --git a/Blacklight/Binaries.php b/Blacklight/Binaries.php index 7496694ad..a3747a254 100644 --- a/Blacklight/Binaries.php +++ b/Blacklight/Binaries.php @@ -333,7 +333,7 @@ class Binaries // We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet) $leaveOver = $this->messageBuffer; - // If this is not a new group, go from our newest to the servers newest. + // If this is not a new group, go from our newest to the servers newest. } else { // Set our oldest wanted to our newest local article. $first = $groupMySQL['last_record']; @@ -377,14 +377,14 @@ class Binaries if ($this->_echoCLI) { $this->colorCli->primary( ( - (int) $groupMySQL['last_record'] === 0 - ? 'New group '.$groupNNTP['group'].' starting with '. - ( - $this->_newGroupScanByDays - ? $this->_newGroupDaysToScan.' days' - : number_format($this->_newGroupMessagesToScan).' messages' - ).' worth.' - : 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.' + (int) $groupMySQL['last_record'] === 0 + ? 'New group '.$groupNNTP['group'].' starting with '. + ( + $this->_newGroupScanByDays + ? $this->_newGroupDaysToScan.' days' + : number_format($this->_newGroupMessagesToScan).' messages' + ).' worth.' + : 'Group '.$groupNNTP['group'].' has '.number_format($realTotal).' new articles.' ). ' Leaving '.number_format($leaveOver). " for next pass.\nServer oldest: ".number_format($groupNNTP['first']). @@ -410,8 +410,8 @@ class Binaries if ($this->_echoCLI) { $this->colorCli->header( PHP_EOL.'Getting '.number_format($last - $first + 1).' articles ('.number_format($first). - ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. - number_format($groupLast - $last).' articles in queue).' + ' to '.number_format($last).') from '.$groupMySQL['name'].' - ('. + number_format($groupLast - $last).' articles in queue).' ); } @@ -479,15 +479,15 @@ class Binaries $endGroup = now()->diffInSeconds($startGroup); $this->colorCli->primary( PHP_EOL.'Group '.$groupMySQL['name'].' processed in '. - $endGroup.Str::plural(' second', $endGroup) + $endGroup.Str::plural(' second', $endGroup) ); } } elseif ($this->_echoCLI) { $this->colorCli->primary( 'No new articles for '.$groupMySQL['name'].' (first '.number_format($first). - ', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']). - ', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']). - ' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record']) + ', last '.number_format($last).', grouplast '.number_format($groupMySQL['last_record']). + ', total '.number_format($total).")\n".'Server oldest: '.number_format($groupNNTP['first']). + ' Server newest: '.number_format($groupNNTP['last']).' Local newest: '.number_format($groupMySQL['last_record']) ); } } @@ -689,7 +689,7 @@ class Binaries if ($this->_echoCLI) { $this->colorCli->alternate( 'Server did not return '.$notReceivedCount. - ' articles from '.$this->groupMySQL['name'].'.' + ' articles from '.$this->groupMySQL['name'].'.' ); } } @@ -711,183 +711,193 @@ class Binaries */ protected function storeHeaders(array $headers = []): void { - DB::transaction(function () use ($headers) { - $binariesUpdate = $collectionIDs = $articles = []; + $binariesUpdate = $collectionIDs = $articles = []; - $partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '; + DB::beginTransaction(); - // Loop articles, figure out files/parts. - foreach ($headers as $this->header) { - // Set up the info for inserting into parts/binaries/collections tables. - if (! isset($articles[$this->header['matches'][1]])) { - // check whether file count should be ignored (XXX packs for now only). - $whitelistMatch = false; - if ($this->_ignoreFileCount($this->groupMySQL['name'], $this->header['matches'][1])) { - $whitelistMatch = true; - $fileCount[1] = $fileCount[3] = 0; + $partsQuery = $partsCheck = 'INSERT IGNORE INTO parts (binaries_id, number, messageid, partnumber, size) VALUES '; + + // Loop articles, figure out files/parts. + foreach ($headers as $this->header) { + // Set up the info for inserting into parts/binaries/collections tables. + if (! isset($articles[$this->header['matches'][1]])) { + // check whether file count should be ignored (XXX packs for now only). + $whitelistMatch = false; + if ($this->_ignoreFileCount($this->groupMySQL['name'], $this->header['matches'][1])) { + $whitelistMatch = true; + $fileCount[1] = $fileCount[3] = 0; + } + + // Attempt to find the file count. If it is not found, set it to 0. + if (! $whitelistMatch && ! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $this->header['matches'][1], $fileCount)) { + $fileCount[1] = $fileCount[3] = 0; + } + + $collMatch = $this->_collectionsCleaning->collectionsCleaner( + $this->header['matches'][1] + ); + + // Used to group articles together when forming the release. + $this->header['CollectionKey'] = $collMatch['name'].$fileCount[3]; + + // If this header's collection key isn't in memory, attempt to insert the collection + if (! isset($collectionIDs[$this->header['CollectionKey']])) { + /* Date from header should be a string this format: + * 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500 + * Still make sure it's not unix time, convert it to unix time if it is. + */ + $this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date'])); + + // Get the current unixtime from PHP. + $now = now()->timestamp; + + $xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref'); + + $tempHeaderXrefs = []; + foreach (explode(' ', $this->header['Xref']) as $headerXref) { + if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) { + $tempHeaderXrefs[] = $hit[0]; + } } - // Attempt to find the file count. If it is not found, set it to 0. - if (! $whitelistMatch && ! preg_match('/[[(\s](\d{1,5})(\/|[\s_]of[\s_]|-)(\d{1,5})[])\s$:]/i', $this->header['matches'][1], $fileCount)) { - $fileCount[1] = $fileCount[3] = 0; + $tempXrefsData = []; + + if ($xrefsData !== null) { + foreach (explode(' ', $xrefsData) as $xrefData) { + if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) { + $tempXrefsData[] = $match1[0]; + } + } } - $collMatch = $this->_collectionsCleaning->collectionsCleaner( - $this->header['matches'][1] - ); - - // Used to group articles together when forming the release. - $this->header['CollectionKey'] = $collMatch['name'].$fileCount[3]; - - // If this header's collection key isn't in memory, attempt to insert the collection - if (! isset($collectionIDs[$this->header['CollectionKey']])) { - /* Date from header should be a string this format: - * 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500 - * Still make sure it's not unix time, convert it to unix time if it is. - */ - $this->header['Date'] = (is_numeric($this->header['Date']) ? $this->header['Date'] : strtotime($this->header['Date'])); - - // Get the current unixtime from PHP. - $now = now()->timestamp; - - $xrefsData = Collection::whereCollectionhash(sha1($this->header['CollectionKey']))->value('xref'); - - $tempHeaderXrefs = []; - foreach (explode(' ', $this->header['Xref']) as $headerXref) { - if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $headerXref, $hit) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $headerXref, $hit)) { - $tempHeaderXrefs[] = $hit[0]; - } + $finalXrefArray = []; + foreach ($tempHeaderXrefs as $tempHeaderXref) { + if (! in_array($tempHeaderXref, $tempXrefsData, false)) { + $finalXrefArray[] = $tempHeaderXref; } + } - $tempXrefsData = []; + $finaXref = implode(' ', $finalXrefArray); - if ($xrefsData !== null) { - foreach (explode(' ', $xrefsData) as $xrefData) { - if (preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)(\:\d+)/', $xrefData, $match1) || preg_match('/(^[a-zA-Z]{2,3}\.(bin(aries|arios|aer))\.[a-zA-Z0-9]?.+)/', $xrefData, $match1)) { - $tempXrefsData[] = $match1[0]; - } - } - } + $xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref)); - $finalXrefArray = []; - foreach ($tempHeaderXrefs as $tempHeaderXref) { - if (! in_array($tempHeaderXref, $tempXrefsData, false)) { - $finalXrefArray[] = $tempHeaderXref; - } - } + $date = $this->header['Date'] > $now ? $now : $this->header['Date']; + $unixtime = is_numeric($this->header['Date']) ? $date : $now; - $finaXref = implode(' ', $finalXrefArray); + $random = random_bytes(16); - $xref = sprintf('xref = CONCAT(xref, "\\n"%s ),', escapeString($finaXref)); + $collectionID = false; - $date = $this->header['Date'] > $now ? $now : $this->header['Date']; - $unixtime = is_numeric($this->header['Date']) ? $date : $now; - - $random = random_bytes(16); - - $collectionID = false; - - try { - DB::insert(sprintf(" + try { + DB::insert(sprintf(" INSERT INTO collections (subject, fromname, date, xref, groups_id, totalfiles, collectionhash, collection_regexes_id, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', %d, NOW()) ON DUPLICATE KEY UPDATE %s dateadded = NOW(), noise = '%s'", escapeString(substr(utf8_encode($this->header['matches'][1]), 0, 255)), escapeString(utf8_encode($this->header['From'])), $unixtime, escapeString(implode(' ', $tempHeaderXrefs)), $this->groupMySQL['id'], $fileCount[3], sha1($this->header['CollectionKey']), $collMatch['id'], $xref, sodium_bin2hex($random))); - $collectionID = $this->_pdo->lastInsertId(); - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - } - - if ($collectionID === false) { - if ($this->addToPartRepair) { - $this->headersNotInserted[] = $this->header['Number']; - } - - continue; - } - $collectionIDs[$this->header['CollectionKey']] = $collectionID; - } else { - $collectionID = $collectionIDs[$this->header['CollectionKey']]; - } - - // Binary Hash should be unique to the group - $hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']); - - $binaryID = false; - - try { - DB::insert(sprintf(" - INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) - VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) - ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(utf8_encode($this->header['matches'][1])), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes'])); - $binaryID = $this->_pdo->lastInsertId(); + $collectionID = $this->_pdo->lastInsertId(); + DB::commit(); } catch (\Throwable $e) { if (config('app.debug') === true) { Log::error($e->getMessage()); } + DB::rollBack(); } - if ($binaryID === false) { + if ($collectionID === false) { if ($this->addToPartRepair) { $this->headersNotInserted[] = $this->header['Number']; } + DB::rollBack(); + DB::beginTransaction(); continue; } - - $binariesUpdate[$binaryID]['Size'] = 0; - $binariesUpdate[$binaryID]['Parts'] = 0; - - $articles[$this->header['matches'][1]]['CollectionID'] = $collectionID; - $articles[$this->header['matches'][1]]['BinaryID'] = $binaryID; + $collectionIDs[$this->header['CollectionKey']] = $collectionID; } else { - $binaryID = $articles[$this->header['matches'][1]]['BinaryID']; - $binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; - $binariesUpdate[$binaryID]['Parts']++; + $collectionID = $collectionIDs[$this->header['CollectionKey']]; } - // In case there are quotes in the message id - $this->header['Message-ID'] = addslashes($this->header['Message-ID']); + // Binary Hash should be unique to the group + $hash = md5($this->header['matches'][1].$this->header['From'].$this->groupMySQL['id']); - // Strip the < and >, saves space in DB. - $this->header['Message-ID'][0] = "'"; + $binaryID = false; - $partsQuery .= - '('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',". - $this->header['matches'][2].','.$this->header['Bytes'].'),'; - } - - unset($headers); // Reclaim memory. - - // Start of inserting into SQL. - $this->startUpdate = now(); - - // End of processing headers. - $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning); - $binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '; - foreach ($binariesUpdate as $binaryID => $binary) { - $binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),'; - } - $binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts'; - $binariesQuery = rtrim($binariesQuery, ',').$binariesEnd; - - // Check if we got any binaries. If we did, try to insert them. - if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) ? true : $this->runQuery($binariesQuery)) { - if (\strlen($partsQuery) === \strlen($partsCheck) ? true : $this->runQuery(rtrim($partsQuery, ','))) { - } else { - if ($this->addToPartRepair) { - $this->headersNotInserted += $this->headersReceived; + try { + DB::insert(sprintf(" + INSERT INTO binaries (binaryhash, name, collections_id, totalparts, currentparts, filenumber, partsize) + VALUES (UNHEX('%s'), %s, %d, %d, 1, %d, %d) + ON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $hash, escapeString(utf8_encode($this->header['matches'][1])), $collectionID, $this->header['matches'][3], $fileCount[1], $this->header['Bytes'], $this->header['Bytes'])); + $binaryID = $this->_pdo->lastInsertId(); + DB::commit(); + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error($e->getMessage()); } + DB::rollBack(); } + + if ($binaryID === false) { + if ($this->addToPartRepair) { + $this->headersNotInserted[] = $this->header['Number']; + } + DB::rollBack(); + DB::beginTransaction(); + + continue; + } + + $binariesUpdate[$binaryID]['Size'] = 0; + $binariesUpdate[$binaryID]['Parts'] = 0; + + $articles[$this->header['matches'][1]]['CollectionID'] = $collectionID; + $articles[$this->header['matches'][1]]['BinaryID'] = $binaryID; + } else { + $binaryID = $articles[$this->header['matches'][1]]['BinaryID']; + $binariesUpdate[$binaryID]['Size'] += $this->header['Bytes']; + $binariesUpdate[$binaryID]['Parts']++; + } + + // In case there are quotes in the message id + $this->header['Message-ID'] = addslashes($this->header['Message-ID']); + + // Strip the < and >, saves space in DB. + $this->header['Message-ID'][0] = "'"; + + $partsQuery .= + '('.$binaryID.','.$this->header['Number'].','.rtrim($this->header['Message-ID'], '>')."',". + $this->header['matches'][2].','.$this->header['Bytes'].'),'; + } + + unset($headers); // Reclaim memory. + + // Start of inserting into SQL. + $this->startUpdate = now(); + + // End of processing headers. + $this->timeCleaning = $this->startUpdate->diffInSeconds($this->startCleaning); + $binariesQuery = $binariesCheck = 'INSERT INTO binaries (id, partsize, currentparts) VALUES '; + foreach ($binariesUpdate as $binaryID => $binary) { + $binariesQuery .= '('.$binaryID.','.$binary['Size'].','.$binary['Parts'].'),'; + } + $binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts'; + $binariesQuery = rtrim($binariesQuery, ',').$binariesEnd; + + // Check if we got any binaries. If we did, try to insert them. + if (\strlen($binariesCheck.$binariesEnd) === \strlen($binariesQuery) ? true : $this->runQuery($binariesQuery)) { + if (\strlen($partsQuery) === \strlen($partsCheck) ? true : $this->runQuery(rtrim($partsQuery, ','))) { + DB::commit(); } else { if ($this->addToPartRepair) { $this->headersNotInserted += $this->headersReceived; } + DB::rollBack(); } - }, 10); - + } else { + if ($this->addToPartRepair) { + $this->headersNotInserted += $this->headersReceived; + } + DB::rollBack(); + } } /** @@ -937,8 +947,8 @@ class Binaries { $this->colorCli->primary( 'Received '.\count($this->headersReceived). - ' articles of '.number_format($this->last - $this->first + 1).' requested, '. - $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' + ' articles of '.number_format($this->last - $this->first + 1).' requested, '. + $this->headersBlackListed.' blacklisted, '.$this->notYEnc.' not yEnc.' ); } @@ -950,18 +960,40 @@ class Binaries $currentMicroTime = now(); if ($this->_echoCLI) { $this->colorCli->alternateOver($this->timeHeaders.'s'). - $this->colorCli->primaryOver(' to download articles, '). - $this->colorCli->alternateOver($this->timeCleaning.'s'). - $this->colorCli->primaryOver(' to process collections, '). - $this->colorCli->alternateOver($this->timeInsert.'s'). - $this->colorCli->primaryOver(' to insert binaries/parts, '). - $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s'). - $this->colorCli->primaryOver(' for part repair, '). - $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s'). - $this->colorCli->primary(' total.'); + $this->colorCli->primaryOver(' to download articles, '). + $this->colorCli->alternateOver($this->timeCleaning.'s'). + $this->colorCli->primaryOver(' to process collections, '). + $this->colorCli->alternateOver($this->timeInsert.'s'). + $this->colorCli->primaryOver(' to insert binaries/parts, '). + $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startPR).'s'). + $this->colorCli->primaryOver(' for part repair, '). + $this->colorCli->alternateOver($currentMicroTime->diffInSeconds($this->startLoop).'s'). + $this->colorCli->primary(' total.'); } } + /** + * If we failed to insert Collections/Binaries/Parts, rollback the transaction and add the parts to part repair. + * + * @param array $headers Array of headers containing sub-arrays with parts. + * @return array Array of article numbers to add to part repair. + * + * @throws \Exception + * @throws \Throwable + */ + protected function _rollbackAddToPartRepair(array $headers): array + { + $headersNotInserted = []; + foreach ($headers as $header) { + foreach ($header as $file) { + $headersNotInserted[] = $file['Parts']['number']; + } + } + DB::rollBack(); + + return $headersNotInserted; + } + /** * Attempt to get missing article headers. * @@ -972,103 +1004,120 @@ class Binaries */ public function partRepair(array $groupArr): void { - DB::transaction(function () use ($groupArr) { - // Get all parts in partrepair table. - $missingParts = MissedPart::select()->where(['groups_id' => $groupArr['id'], 'attempts' => $this->_partRepairMaxTries])->orderBy('numberid', 'asc')->limit($this->_partRepairLimit)->get()->toArray(); + // Get all parts in partrepair table. + $missingParts = []; + try { + $missingParts = DB::select(sprintf(' + SELECT * FROM missed_parts + WHERE groups_id = %d AND attempts < %d + ORDER BY numberid ASC LIMIT %d', $groupArr['id'], $this->_partRepairMaxTries, $this->_partRepairLimit)); + } catch (\PDOException $e) { + if ($e->getMessage() === 'SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction') { + $this->colorCli->notice('Deadlock occurred'); + DB::rollBack(); + } + } + + $missingCount = \count($missingParts); + if ($missingCount > 0) { + if ($this->_echoCLI) { + $this->colorCli->primary( + 'Attempting to repair '. + number_format($missingCount). + ' parts.' + ); + } + + // Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4. + $ranges = $partList = []; + $firstPart = $lastNum = $missingParts[0]->numberid; + + foreach ($missingParts as $part) { + if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) { + $ranges[] = [ + 'partfrom' => $firstPart, + 'partto' => $lastNum, + 'partlist' => $partList, + ]; + + $firstPart = $part->numberid; + $partList = []; + } + $partList[] = $part->numberid; + $lastNum = $part->numberid; + } + + $ranges[] = [ + 'partfrom' => $firstPart, + 'partto' => $lastNum, + 'partlist' => $partList, + ]; + + // Download missing parts in ranges. + foreach ($ranges as $range) { + $partFrom = $range['partfrom']; + $partTo = $range['partto']; + $partList = $range['partlist']; - $missingCount = \count($missingParts); - if ($missingCount > 0) { if ($this->_echoCLI) { - $this->colorCli->primary( - 'Attempting to repair '. - number_format($missingCount). - ' parts.' - ); + echo \chr(random_int(45, 46)).PHP_EOL; } - // Loop through each part to group into continuous ranges with a maximum range of messagebuffer/4. - $ranges = $partList = []; - $firstPart = $lastNum = $missingParts[0]->numberid; + // Get article headers from newsgroup. + $this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList); + } - foreach ($missingParts as $part) { - if (($part->numberid - $firstPart) > ($this->messageBuffer / 4)) { - $ranges[] = [ - 'partfrom' => $firstPart, - 'partto' => $lastNum, - 'partlist' => $partList, - ]; - - $firstPart = $part->numberid; - $partList = []; - } - $partList[] = $part->numberid; - $lastNum = $part->numberid; - } - - $ranges[] = [ - 'partfrom' => $firstPart, - 'partto' => $lastNum, - 'partlist' => $partList, - ]; - - // Download missing parts in ranges. - foreach ($ranges as $range) { - $partFrom = $range['partfrom']; - $partTo = $range['partto']; - $partList = $range['partlist']; - - if ($this->_echoCLI) { - echo \chr(random_int(45, 46)).PHP_EOL; - } - - // Get article headers from newsgroup. - $this->scan($groupArr, $partFrom, $partTo, 'partrepair', $partList); - } - - // Calculate parts repaired - $result = DB::select( - sprintf( - ' + // Calculate parts repaired + $result = DB::select( + sprintf( + ' SELECT COUNT(id) AS num FROM missed_parts WHERE groups_id = %d AND numberid <= %d', - $groupArr['id'], - $missingParts[$missingCount - 1]->numberid - ) - ); + $groupArr['id'], + $missingParts[$missingCount - 1]->numberid + ) + ); - $partsRepaired = 0; - if ($result > 0) { - $partsRepaired = ($missingCount - $result[0]->num); - } + $partsRepaired = 0; + if ($result > 0) { + $partsRepaired = ($missingCount - $result[0]->num); + } - // Update attempts on remaining parts for active group - if (isset($missingParts[$missingCount - 1]->id)) { - DB::update( - sprintf( - ' + // Update attempts on remaining parts for active group + if (isset($missingParts[$missingCount - 1]->id)) { + DB::update( + sprintf( + ' UPDATE missed_parts SET attempts = attempts + 1 WHERE groups_id = %d AND numberid <= %d', - $groupArr['id'], - $missingParts[$missingCount - 1]->numberid - ) - ); - } - - if ($this->_echoCLI) { - $this->colorCli->primary( - PHP_EOL. - number_format($partsRepaired). - ' parts repaired.' - ); - } + $groupArr['id'], + $missingParts[$missingCount - 1]->numberid + ) + ); } - // Remove articles that we cant fetch after x attempts. - MissedPart::query()->where('attempts', '>=', $this->_partRepairMaxTries)->where('groups_id', $groupArr['id'])->delete(); + if ($this->_echoCLI) { + $this->colorCli->primary( + PHP_EOL. + number_format($partsRepaired). + ' parts repaired.' + ); + } + } + + // Remove articles that we cant fetch after x attempts. + DB::transaction(function () use ($groupArr) { + DB::delete( + sprintf( + 'DELETE FROM missed_parts WHERE attempts >= %d AND groups_id = %d', + $this->_partRepairMaxTries, + $groupArr['id'] + ) + ); }, 10); } @@ -1230,7 +1279,7 @@ class Binaries if ($this->_echoCLI) { $this->colorCli->primary( PHP_EOL.'Found article #'.$wantedArticle.' which has a date of '.date('r', $articleTime). - ', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.' + ', vs wanted date of '.date('r', $goalTime).'. Difference from goal is '.Carbon::createFromTimestamp($goalTime)->diffInDays(Carbon::createFromTimestamp($articleTime)).'days.' ); } @@ -1240,21 +1289,17 @@ class Binaries /** * Add article numbers from missing headers to DB. * - * @param array $numbers The article numbers of the missing headers. - * @param int $groupID The ID of this groups. - * - * @throws \Throwable + * @param array $numbers The article numbers of the missing headers. + * @param int $groupID The ID of this groups. */ private function addMissingParts(array $numbers, int $groupID): string { - DB::transaction(function () use ($numbers, $groupID) { - $insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; - foreach ($numbers as $number) { - $insertStr .= '('.$number.','.$groupID.'),'; - } + $insertStr = 'INSERT INTO missed_parts (numberid, groups_id) VALUES '; + foreach ($numbers as $number) { + $insertStr .= '('.$number.','.$groupID.'),'; + } - DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); - }, 10); + DB::insert(rtrim($insertStr, ',').' ON DUPLICATE KEY UPDATE attempts=attempts+1'); return $this->_pdo->lastInsertId(); } @@ -1269,7 +1314,13 @@ class Binaries */ private function removeRepairedParts(array $numbers, int $groupID): void { - MissedPart::whereIn('numberid', $numbers)->where('groups_id', $groupID)->delete(); + $sql = 'DELETE FROM missed_parts WHERE numberid in ('; + foreach ($numbers as $number) { + $sql .= $number.','; + } + DB::transaction(static function () use ($groupID, $sql) { + DB::delete(rtrim($sql, ',').') AND groups_id = '.$groupID); + }, 10); } /** @@ -1438,6 +1489,10 @@ class Binaries */ public function delete(int $collectionID): void { + DB::transaction(static function () use ($collectionID) { + DB::delete(sprintf('DELETE FROM collections WHERE id = %d', $collectionID)); + }, 10); + Collection::query()->where('id', $collectionID)->delete(); } @@ -1470,26 +1525,24 @@ class Binaries protected function runQuery($query): bool { - DB::transaction(function () use ($query) { - try { - return DB::insert($query); - } catch (QueryException $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); - } catch (\PDOException $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); - } catch (\Throwable $e) { - if (config('app.debug') === true) { - Log::error($e->getMessage()); - } - $this->colorCli->debug('Query error occurred.'); + try { + return DB::insert($query); + } catch (QueryException $e) { + if (config('app.debug') === true) { + Log::error($e->getMessage()); } - }); + $this->colorCli->debug('Query error occurred.'); + } catch (\PDOException $e) { + if (config('app.debug') === true) { + Log::error($e->getMessage()); + } + $this->colorCli->debug('Query error occurred.'); + } catch (\Throwable $e) { + if (config('app.debug') === true) { + Log::error($e->getMessage()); + } + $this->colorCli->debug('Query error occurred.'); + } return false; }